def processNickList(nicks, platforms=None, rutaDescarga="./", avoidProcessing=True, avoidDownload=True, nThreads=12, maltego=False, verbosity=1, logFolder="./logs"): ''' Method that receives as a parameter a series of nicks and verifies whether those nicks have a profile associated in different social networks. List of parameters that the method receives: :param nicks: list of nicks to process. :param platforms: list of <Platform> objects to be processed. :param rutaDescarga: local file where saving the obtained information. :param avoidProcessing: boolean var that defines whether the profiles will NOT be processed. :param avoidDownload: boolean var that defines whether the profiles will NOT be downloaded (stored in this version). :param maltego: parameter to tell usufy.py that he has been invoked by Malego. :param verbosity: the level of verbosity to be used. :param logFolder: the path to the log folder. :return: Returns a dictionary where the key is the nick and the value another dictionary where the keys are the social networks and te value is the corresponding URL. ''' osrframework.utils.logger.setupLogger(loggerName="osrframework.usufy", verbosity=verbosity, logFolder=logFolder) logger = logging.getLogger("osrframework.usufy") if platforms == None: platforms = platform_selection.getAllPlatformNames("usufy") # Defining the output results variable res = [] # Processing the whole list of terms... for nick in nicks: logger.info("Looking for '" + nick + "' in " + str(len(platforms)) + " different platforms:\n" +str( [ str(plat) for plat in platforms ] ) ) # If the process is executed by the current app, we use the Processes. It is faster than pools. if nThreads <= 0 or nThreads > len(platforms): nThreads = len(platforms) # Using threads in a pool if we are not running the program in main args = [] # We need to create all the arguments that will be needed #print platforms for plat in platforms: args.append (( plat, nick, rutaDescarga, avoidProcessing, avoidDownload)) logger.info("Launching " + str(nThreads) + " different threads...") # We define the pool pool = Pool(nThreads) #print json.dumps(args, indent=2) # We call the wrapping function with all the args previously generated #poolResults = pool.apply_async(multi_run_wrapper,(args)) poolResults = pool.map(multi_run_wrapper,args) pool.close() profiles = [] for r in poolResults: # We need to recover the results and check if they are not an empty json if r != "{}": profiles.append(json.loads(r)) res+=profiles #res = profiles return res
def main(args): ''' Main function. This function is created in this way so as to let other applications make use of the full configuration capabilities of the application. ''' # Recovering the logger # Calling the logger when being imported osrframework.utils.logger.setupLogger(loggerName="osrframework.usufy", verbosity=args.verbose, logFolder=args.logfolder) # From now on, the logger can be recovered like this: logger = logging.getLogger("osrframework.usufy") # Printing the results if requested if not args.maltego: print banner.text sayingHello = """usufy.py Copyright (C) F. Brezo and Y. Rubio (i3visio) 2014-2017 This program comes with ABSOLUTELY NO WARRANTY. This is free software, and you are welcome to redistribute it under certain conditions. For additional info, visit <http://www.gnu.org/licenses/gpl-3.0.txt>.""" logger.info(sayingHello) print sayingHello print logger.info("Starting usufy.py...") if args.license: logger.info("Looking for the license...") # showing the license try: with open ("COPYING", "r") as iF: contenido = iF.read().splitlines() for linea in contenido: print linea except Exception: try: # Trying to recover the COPYING file... with open ("/usr/share/osrframework/COPYING", "r") as iF: contenido = iF.read().splitlines() for linea in contenido: print linea except: logger.error("ERROR: there has been an error when opening the COPYING file.\n\tThe file contains the terms of the GPLv3 under which this software is distributed.\n\tIn case of doubts, verify the integrity of the files or contact [email protected].") elif args.fuzz: logger.info("Performing the fuzzing tasks...") res = fuzzUsufy(args.fuzz, args.fuzz_config) logger.info("Recovered platforms:\n" + str(res)) else: logger.debug("Recovering the list of platforms to be processed...") # Recovering the list of platforms to be launched listPlatforms = platform_selection.getPlatformsByName(platformNames=args.platforms, tags=args.tags, mode="usufy", excludePlatformNames=args.exclude) logger.debug("Platforms recovered.") if args.info: # Information actions... if args.info == 'list_platforms': infoPlatforms="Listing the platforms:\n" for p in listPlatforms: infoPlatforms += "\t\t" + (str(p) + ": ").ljust(16, ' ') + str(p.tags)+"\n" logger.info(infoPlatforms) return infoPlatforms elif args.info == 'list_tags': logger.info("Listing the tags:") tags = {} # Going through all the selected platforms to get their tags for p in listPlatforms: for t in p.tags: if t not in tags.keys(): tags[t] = 1 else: tags[t] += 1 infoTags = "List of tags:\n" # Displaying the results in a sorted list for t in tags.keys(): infoTags += "\t\t" + (t + ": ").ljust(16, ' ') + str(tags[t]) + " time(s)\n" logger.info(infoTags) return infoTags else: pass # performing the test elif args.benchmark: logger.warning("The benchmark mode may last some minutes as it will be performing similar queries to the ones performed by the program in production. ") logger.info("Launching the benchmarking tests...") platforms = platform_selection.getAllPlatformNames("usufy") res = benchmark.doBenchmark(platforms) strTimes = "" for e in sorted(res.keys()): strTimes += str(e) + "\t" + str(res[e]) + "\n" logger.info(strTimes) return strTimes # showing the tags of the usufy platforms elif args.show_tags: logger.info("Collecting the list of tags...") tags = platform_selection.getAllPlatformNamesByTag("usufy") logger.info(json.dumps(tags, indent=2)) print "This is the list of platforms grouped by tag." print print json.dumps(tags, indent=2, sort_keys=True) print print "[Tip] Remember that you can always launch the platform using the -t option followed by any of the aforementioned." print return tags # Executing the corresponding process... else: # Showing the execution time... if not args.maltego: startTime= dt.datetime.now() print str(startTime) +"\tStarting search in " + str(len(listPlatforms)) + " platform(s)... Relax!" print print "\tPress <Ctrl + C> to stop..." print # Defining the list of users to monitor nicks = [] logger.debug("Recovering nicknames to be processed...") if args.nicks: for n in args.nicks: # TO-DO # A trick to avoid having the processing of the properties when being queried by Maltego if "properties.i3visio" not in n: nicks.append(n) else: # Reading the nick files try: nicks = args.list.read().splitlines() except: logger.error("ERROR: there has been an error when opening the file that stores the nicks.\tPlease, check the existence of this file.") # Definning the results res = [] if args.output_folder != None: # if Verifying an output folder was selected logger.debug("Preparing the output folder...") if not args.maltego: if not os.path.exists(args.output_folder): logger.warning("The output folder \'" + args.output_folder + "\' does not exist. The system will try to create it.") os.makedirs(args.output_folder) # Launching the process... ###try: res = processNickList(nicks, listPlatforms, args.output_folder, avoidProcessing = args.avoid_processing, avoidDownload = args.avoid_download, nThreads=args.threads, verbosity= args.verbose, logFolder=args.logfolder) ###except Exception as e: ###print "Exception grabbed when processing the nicks: " + str(e) ###print traceback.print_stack() else: try: res = processNickList(nicks, listPlatforms, nThreads=args.threads, verbosity= args.verbose, logFolder=args.logfolder) except Exception as e: print "Exception grabbed when processing the nicks: " + str(e) print traceback.print_stack() logger.info("Listing the results obtained...") # We are going to iterate over the results... strResults = "\t" # Structure returned """ [ { "attributes": [ { "attributes": [], "type": "i3visio.uri", "value": "http://twitter.com/i3visio" }, { "attributes": [], "type": "i3visio.alias", "value": "i3visio" }, { "attributes": [], "type": "i3visio.platform", "value": "Twitter" } ], "type": "i3visio.profile", "value": "Twitter - i3visio" } , ... ] """ for r in res: # The format of the results (attributes) for a given nick is a list as follows: for att in r["attributes"]: # iterating through the attributes platform = "" uri = "" for details in att["attributes"]: if details["type"] == "i3visio.platform": platform = details["value"] if details["type"] == "i3visio.uri": uri = details["value"] try: strResults+= (str(platform) + ":").ljust(16, ' ')+ " "+ str(uri)+"\n\t\t" except: pass logger.info(strResults) # Generating summary files for each ... if args.extension: # Storing the file... logger.info("Creating output files as requested.") if not args.maltego: # Verifying if the outputPath exists if not os.path.exists (args.output_folder): logger.warning("The output folder \'" + args.output_folder + "\' does not exist. The system will try to create it.") os.makedirs(args.output_folder) # Grabbing the results fileHeader = os.path.join(args.output_folder, args.file_header) # Iterating through the given extensions to print its values for ext in args.extension: # Generating output files general.exportUsufy(res, ext, fileHeader) # Generating the Maltego output if args.maltego: general.listToMaltego(res) # Printing the results if requested else: print "A summary of the results obtained are shown in the following table:" #print res print unicode(general.usufyToTextExport(res)) print if args.web_browser: general.openResultsInBrowser(res) print "You can find all the information collected in the following files:" for ext in args.extension: # Showing the output files print "\t-" + fileHeader + "." + ext # Showing the execution time... print endTime= dt.datetime.now() print str(endTime) +"\tFinishing execution..." print print "Total time used:\t" + str(endTime-startTime) print "Average seconds/query:\t" + str((endTime-startTime).total_seconds()/len(listPlatforms)) +" seconds" print # Urging users to place an issue on Github... print print "Did something go wrong? Is a platform reporting false positives? Do you need to integrate a new one?" print "Then, place an issue in the Github project: <https://github.com/i3visio/osrframework/issues>." print "Note that otherwise, we won't know about it!" print return res
def processNickList(nicks, platforms=None, rutaDescarga="./", avoidProcessing=True, avoidDownload=True, nThreads=12, maltego=False, verbosity=1, logFolder="./logs"): ''' Method that receives as a parameter a series of nicks and verifies whether those nicks have a profile associated in different social networks. List of parameters that the method receives: :param nicks: list of nicks to process. :param platforms: list of <Platform> objects to be processed. :param rutaDescarga: local file where saving the obtained information. :param avoidProcessing: boolean var that defines whether the profiles will NOT be processed. :param avoidDownload: boolean var that defines whether the profiles will NOT be downloaded (stored in this version). :param maltego: parameter to tell usufy.py that he has been invoked by Malego. :param verbosity: the level of verbosity to be used. :param logFolder: the path to the log folder. :return: Returns a dictionary where the key is the nick and the value another dictionary where the keys are the social networks and te value is the corresponding URL. ''' osrframework.utils.logger.setupLogger(loggerName="osrframework.usufy", verbosity=verbosity, logFolder=logFolder) logger = logging.getLogger("osrframework.usufy") if platforms == None: platforms = platform_selection.getAllPlatformNames("usufy") # Defining the output results variable res = [] # Processing the whole list of terms... for nick in nicks: logger.info("Looking for '" + nick + "' in " + str(len(platforms)) + " different platforms:\n" +str( [ str(plat) for plat in platforms ] ) ) # If the process is executed by the current app, we use the Processes. It is faster than pools. if nThreads <= 0 or nThreads > len(platforms): nThreads = len(platforms) logger.info("Launching " + str(nThreads) + " different threads...") # Using threads in a pool if we are not running the program in main # Example catched from: https://stackoverflow.com/questions/11312525/catch-ctrlc-sigint-and-exit-multiprocesses-gracefully-in-python original_sigint_handler = signal.signal(signal.SIGINT, signal.SIG_IGN) pool = Pool(nThreads) signal.signal(signal.SIGINT, original_sigint_handler) poolResults = [] try: def log_result(result): # This is called whenever foo_pool(i) returns a result. # result_list is modified only by the main process, not the pool workers. poolResults.append(result) for plat in platforms: # We need to create all the arguments that will be needed parameters = ( plat, nick, rutaDescarga, avoidProcessing, avoidDownload, ) pool.apply_async (pool_function, args= parameters, callback = log_result ) # Waiting for results to be finished while len(poolResults) < len(platforms): #print "Waiiting to finish all!" pass # Closing normal termination pool.close() except KeyboardInterrupt: print "\n[!] Process manually stopped by the user. Terminating workers.\n" pool.terminate() print "[!] The following platforms were not processed:" pending = "" for p in platforms: processed = False for processedPlatform in poolResults: if str(p) == processedPlatform["platform"]: processed = True break if not processed: print "\t- " + str(p) pending += " " + str(p).lower() print print "[!] If you want to relaunch the app with these platforms you can always run the command with: " print "\t usufy.py ... -p " + pending print print "[!] If you prefer to avoid these platforms you can manually evade them for whatever reason with: " print "\t usufy.py ... -x " + pending print pool.join() profiles = [] # Processing the results # ---------------------- for serArray in poolResults: data = serArray["data"] # We need to recover the results and check if they are not an empty json or None if data != None: array = json.loads(data) for r in array: if r != "{}": profiles.append(r) res+=profiles return res
def usufy_main(args): ''' Main function. This function is created in this way so as to let other applications make use of the full configuration capabilities of the application. ''' # Recovering the logger # Calling the logger when being imported osrframework.utils.logger.setupLogger(loggerName="osrframework.usufy", verbosity=args.verbose, logFolder=args.logfolder) # From now on, the logger can be recovered like this: logger = logging.getLogger("osrframework.usufy") logger.info("""usufy.py Copyright (C) F. Brezo and Y. Rubio (i3visio) 2015 This program comes with ABSOLUTELY NO WARRANTY. This is free software, and you are welcome to redistribute it under certain conditions. For details, run: \tpython usufy.py --license""") logger.info("Starting usufy.py...") if args.license: logger.info("Looking for the license...") # showing the license try: with open ("COPYING", "r") as iF: contenido = iF.read().splitlines() for linea in contenido: print linea except Exception: logger.error("ERROR: there has been an error when opening the COPYING file.\n\tThe file contains the terms of the GPLv3 under which this software is distributed.\n\tIn case of doubts, verify the integrity of the files or contact [email protected].") elif args.fuzz: logger.info("Performing the fuzzing tasks...") res = fuzzUsufy(args.fuzz, args.fuzz_config) logger.info("Recovered platforms:\n" + str(res)) else: logger.debug("Recovering the list of platforms to be processed...") # Recovering the list of platforms to be launched listPlatforms = platform_selection.getPlatformsByName(platformNames=args.platforms, tags=args.tags, mode="usufy") logger.debug("Platforms recovered.") if args.info: # Information actions... if args.info == 'list_platforms': infoPlatforms="Listing the platforms:\n" for p in listPlatforms: infoPlatforms += "\t\t" + (str(p) + ": ").ljust(16, ' ') + str(p.tags)+"\n" logger.info(infoPlatforms) return infoPlatforms elif args.info == 'list_tags': logger.info("Listing the tags:") tags = {} # Going through all the selected platforms to get their tags for p in listPlatforms: for t in p.tags: if t not in tags.keys(): tags[t] = 1 else: tags[t] += 1 infoTags = "List of tags:\n" # Displaying the results in a sorted list for t in tags.keys(): infoTags += "\t\t" + (t + ": ").ljust(16, ' ') + str(tags[t]) + " time(s)\n" logger.info(infoTags) return infoTags else: pass # performing the test elif args.benchmark: logger.warning("The benchmark mode may last some minutes as it will be performing similar queries to the ones performed by the program in production. ") logger.info("Launching the benchmarking tests...") platforms = platform_selection.getAllPlatformNames("usufy") res = benchmark.doBenchmark(platforms) strTimes = "" for e in sorted(res.keys()): strTimes += str(e) + "\t" + str(res[e]) + "\n" logger.info(strTimes) return strTimes # Executing the corresponding process... else: # Defining the list of users to monitor nicks = [] logger.debug("Recovering nicknames to be processed...") if args.nicks: for n in args.nicks: # TO-DO # A trick to avoid having the processing of the properties when being queried by Maltego if "properties.i3visio" not in n: nicks.append(n) else: # Reading the nick files try: nicks = args.list.read().splitlines() except: logger.error("ERROR: there has been an error when opening the file that stores the nicks.\tPlease, check the existence of this file.") # Checking if performing any transform was required if args.squatting: logger.debug("Making basic transformations on the provided nicknames...") # Iterating to process _ and or . #nicks = profilesquatting.generatingProfiles(nicks, args.profilesquatting) nicks = profilesquatting.getNewNicks(nicks, logName = "usufy", modes = args.squatting, nonValidChars = args.nonvalid) logger.info("Obtained nicks:\n" + str(nicks)) logger.debug("Profilesquatting nicknames recovered.") if args.info == 'list_users': strNicks = "" for n in nicks: strNicks += n + "\n" logger.info("Generated nicks:\n----------------\n" + strNicks) # Storing the file... logger.info("Creating output files as requested.") # Verifying if the outputPath exists if not os.path.exists (args.output_folder): logger.warning("The output folder \'" + args.output_folder + "\' does not exist. The system will try to create it.") os.makedirs(args.output_folder) strTime = general.getCurrentStrDatetime() logger.info("Writing generated nicks to a text file.") with open (os.path.join(args.output_folder, "nicks_" + strTime +".txt"), "w") as oF: oF.write( strNicks ) # if this option was selected, we will jsut return this and exist return nicks if args.output_folder != None: # if Verifying an output folder was selected logger.debug("Preparing the output folder...") if not os.path.exists(args.output_folder): logger.warning("The output folder \'" + args.output_folder + "\' does not exist. The system will try to create it.") os.makedirs(args.output_folder) # Launching the process... res = processNickList(nicks, listPlatforms, args.output_folder, avoidProcessing = args.avoid_processing, avoidDownload = args.avoid_download, nThreads=args.threads, verbosity= args.verbose, logFolder=args.logfolder) else: res = processNickList(nicks, listPlatforms, nThreads=args.threads, verbosity= args.verbose, logFolder=args.logfolder) logger.info("Listing the results obtained...") # We are going to iterate over the results... strResults = "\t" # Structure returned """ [ { "attributes": [ { "attributes": [], "type": "i3visio.uri", "value": "http://twitter.com/i3visio" }, { "attributes": [], "type": "i3visio.alias", "value": "i3visio" }, { "attributes": [], "type": "i3visio.platform", "value": "Twitter" } ], "type": "i3visio.profile", "value": "Twitter - i3visio" } , ... ] """ for r in res: # The format of the results (attributes) for a given nick is a list as follows: for att in r["attributes"]: # iterating through the attributes platform = "" uri = "" for details in att["attributes"]: if details["type"] == "i3visio.platform": platform = details["value"] if details["type"] == "i3visio.uri": uri = details["value"] try: strResults+= (str(platform) + ":").ljust(16, ' ')+ " "+ str(uri)+"\n\t\t" except: pass logger.info(strResults) # Generating summary files for each ... if args.extension: # Storing the file... logger.info("Creating output files as requested.") if not args.output_folder: args.output_folder = "./" else: # Verifying if the outputPath exists if not os.path.exists (args.output_folder): logger.warning("The output folder \'" + args.output_folder + "\' does not exist. The system will try to create it.") os.makedirs(args.output_folder) # Grabbing the results fileHeader = os.path.join(args.output_folder, args.file_header + general.getCurrentStrDatetime()) # Iterating through the given extensions to print its values for ext in args.extension: # Generating output files general.exportUsufy(res, ext, fileHeader) # Generating the Maltego output if args.maltego: general.listToMaltego(res) # Printing the results if requested if not args.maltego: print "A summary of the results obtained are the following table:" print general.usufyToTextExport(res) print "You can find all the information collected in the following files:" for ext in args.extension: # Generating output files print "\t-" + fileHeader + "." + ext return res
def processNickList(nicks, platforms=None, rutaDescarga="./", avoidProcessing=True, avoidDownload=True, nThreads=12, maltego=False, verbosity=1, logFolder="./logs"): ''' Method that receives as a parameter a series of nicks and verifies whether those nicks have a profile associated in different social networks. List of parameters that the method receives: :param nicks: list of nicks to process. :param platforms: list of <Platform> objects to be processed. :param rutaDescarga: local file where saving the obtained information. :param avoidProcessing: boolean var that defines whether the profiles will NOT be processed. :param avoidDownload: boolean var that defines whether the profiles will NOT be downloaded (stored in this version). :param maltego: parameter to tell usufy.py that he has been invoked by Malego. :param verbosity: the level of verbosity to be used. :param logFolder: the path to the log folder. :return: Returns a dictionary where the key is the nick and the value another dictionary where the keys are the social networks and te value is the corresponding URL. ''' osrframework.utils.logger.setupLogger(loggerName="osrframework.usufy", verbosity=verbosity, logFolder=logFolder) logger = logging.getLogger("osrframework.usufy") if platforms == None: platforms = platform_selection.getAllPlatformNames("usufy") # Defining the output results variable res = [] # Processing the whole list of terms... for nick in nicks: logger.info("Looking for '" + nick + "' in " + str(len(platforms)) + " different platforms:\n" + str([str(plat) for plat in platforms])) # Using threads in a pool if we are not running the program in main args = [] # We need to create all the arguments that will be needed #print platforms for plat in platforms: args.append( (plat, nick, rutaDescarga, avoidProcessing, avoidDownload)) # If the process is executed by the current app, we use the Processes. It is faster than pools. if nThreads <= 0 or nThreads > len(platforms): nThreads = len(platforms) logger.info("Launching " + str(nThreads) + " different threads...") # We define the pool pool = Pool(nThreads) # We call the wrapping function with all the args previously generated #poolResults = pool.apply_async(multi_run_wrapper,(args)) poolResults = pool.map(multi_run_wrapper, args) pool.close() profiles = [] for serArray in poolResults: # We need to recover the results and check if they are not an empty json array = json.loads(serArray) for r in array: if r != "{}": profiles.append(r) res += profiles #res = profiles return res
def main(params=None): """ Main function to launch usufy. The function is created in this way so as to let other applications make use of the full configuration capabilities of the application. The parameters received are used as parsed by this modules `getParser()`. Args: ----- params: A list with the parameters as grabbed by the terminal. It is None when this is called by an entry_point. Returns: -------- dict: A Json representing the matching results. """ # Grabbing the parser parser = getParser() if params != None: args = parser.parse_args(params) else: args = parser.parse_args() # Recovering the logger # Calling the logger when being imported osrframework.utils.logger.setupLogger(loggerName="osrframework.usufy", verbosity=args.verbose, logFolder=args.logfolder) # From now on, the logger can be recovered like this: logger = logging.getLogger("osrframework.usufy") print(general.title(banner.text)) sayingHello = """ Usufy | Copyright (C) F. Brezo and Y. Rubio (i3visio) 2014-2018 This program comes with ABSOLUTELY NO WARRANTY. This is free software, and you are welcome to redistribute it under certain conditions. For additional info, visit """ + general.LICENSE_URL + "\n" logger.info(sayingHello) print(general.title(sayingHello)) logger.info("Starting usufy...") if args.license: general.showLicense() elif args.fuzz: logger.info("Performing the fuzzing tasks...") res = fuzzUsufy(args.fuzz, args.fuzz_config) logger.info("Recovered platforms:\n" + str(res)) else: logger.debug("Recovering the list of platforms to be processed...") # Recovering the list of platforms to be launched listPlatforms = platform_selection.getPlatformsByName( platformNames=args.platforms, tags=args.tags, mode="usufy", excludePlatformNames=args.exclude) logger.debug("Platforms recovered.") if args.info: # Information actions... if args.info == 'list_platforms': infoPlatforms = "Listing the platforms:\n" for p in listPlatforms: infoPlatforms += "\t\t" + (str(p) + ": ").ljust( 16, ' ') + str(p.tags) + "\n" logger.info(infoPlatforms) return infoPlatforms elif args.info == 'list_tags': logger.info("Listing the tags:") tags = {} # Going through all the selected platforms to get their tags for p in listPlatforms: for t in p.tags: if t not in tags.keys(): tags[t] = 1 else: tags[t] += 1 infoTags = "List of tags:\n" # Displaying the results in a sorted list for t in tags.keys(): infoTags += "\t\t" + (t + ": ").ljust(16, ' ') + str( tags[t]) + " time(s)\n" logger.info(infoTags) return infoTags else: pass # performing the test elif args.benchmark: logger.warning( "The benchmark mode may last some minutes as it will be performing similar queries to the ones performed by the program in production. " ) logger.info("Launching the benchmarking tests...") platforms = platform_selection.getAllPlatformNames("usufy") res = benchmark.doBenchmark(platforms) strTimes = "" for e in sorted(res.keys()): strTimes += str(e) + "\t" + str(res[e]) + "\n" logger.info(strTimes) return strTimes # showing the tags of the usufy platforms elif args.show_tags: logger.info("Collecting the list of tags...") tags = platform_selection.getAllPlatformNamesByTag("usufy") logger.info(json.dumps(tags, indent=2)) print( general.info( "This is the list of platforms grouped by tag.\n")) print(json.dumps(tags, indent=2, sort_keys=True)) print( general.info( "[Tip] Remember that you can always launch the platform using the -t option followed by any of the aforementioned.\n" )) return tags # Executing the corresponding process... else: # Showing the execution time... startTime = dt.datetime.now() print( str(startTime) + "\tStarting search in " + general.emphasis(str(len(listPlatforms))) + " platform(s)... Relax!\n") print(general.emphasis("\tPress <Ctrl + C> to stop...\n")) # Defining the list of users to monitor nicks = [] logger.debug("Recovering nicknames to be processed...") if args.nicks: for n in args.nicks: # TO-DO # A trick to avoid having the processing of the properties when being queried by Maltego if "properties.i3visio" not in n: nicks.append(n) else: # Reading the nick files try: nicks = args.list.read().splitlines() except: logger.error( "ERROR: there has been an error when opening the file that stores the nicks.\tPlease, check the existence of this file." ) # Definning the results res = [] if args.output_folder != None: # if Verifying an output folder was selected logger.debug("Preparing the output folder...") if not os.path.exists(args.output_folder): logger.warning( "The output folder \'" + args.output_folder + "\' does not exist. The system will try to create it.") os.makedirs(args.output_folder) # Launching the process... res = processNickList(nicks, listPlatforms, args.output_folder, avoidProcessing=args.avoid_processing, avoidDownload=args.avoid_download, nThreads=args.threads, verbosity=args.verbose, logFolder=args.logfolder) else: try: res = processNickList(nicks, listPlatforms, nThreads=args.threads, verbosity=args.verbose, logFolder=args.logfolder) except Exception as e: print( general.error( "Exception grabbed when processing the nicks: " + str(e))) print(general.error(traceback.print_stack())) logger.info("Listing the results obtained...") # We are going to iterate over the results... strResults = "\t" # Structure returned """ [ { print "attributes": [ { "attributes": [], "type": "i3visio.uri", "value": "http://twitter.com/i3visio" }, { "attributes": [], "type": "i3visio.alias", "value": "i3visio" }, { "attributes": [], "type": "i3visio.platform", "value": "Twitter" } ], "type": "i3visio.profile", "value": "Twitter - i3visio" } , ... ] """ for r in res: # The format of the results (attributes) for a given nick is a list as follows: for att in r["attributes"]: # iterating through the attributes platform = "" uri = "" for details in att["attributes"]: if details["type"] == "i3visio.platform": platform = details["value"] if details["type"] == "i3visio.uri": uri = details["value"] try: strResults += (str(platform) + ":").ljust( 16, ' ') + " " + str(uri) + "\n\t\t" except: pass logger.info(strResults) # Generating summary files for each ... if args.extension: # Storing the file... logger.info("Creating output files as requested.") # Verifying if the outputPath exists if not os.path.exists(args.output_folder): logger.warning( "The output folder \'" + args.output_folder + "\' does not exist. The system will try to create it.") os.makedirs(args.output_folder) # Grabbing the results fileHeader = os.path.join(args.output_folder, args.file_header) # Iterating through the given extensions to print its values for ext in args.extension: # Generating output files general.exportUsufy(res, ext, fileHeader) now = dt.datetime.now() print( str(now) + "\tA summary of the results obtained is shown below:\n") print(general.success(general.usufyToTextExport(res))) if args.web_browser: general.openResultsInBrowser(res) now = dt.datetime.now() print("\n" + str(now) + "\tYou can find all the information here:") for ext in args.extension: # Showing the output files print("\t" + general.emphasis(fileHeader + "." + ext)) # Showing the execution time... endTime = dt.datetime.now() print("\n" + str(endTime) + "\tFinishing execution...\n") print("Total time consumed:\t" + general.emphasis(str(endTime - startTime))) print("Average seconds/query:\t" + general.emphasis( str((endTime - startTime).total_seconds() / len(listPlatforms))) + " seconds\n") # Urging users to place an issue on Github... print(banner.footer) if params: return res
def main(args): ''' Main function. This function is created in this way so as to let other applications make use of the full configuration capabilities of the application. ''' # Recovering the logger # Calling the logger when being imported osrframework.utils.logger.setupLogger(loggerName="osrframework.usufy", verbosity=args.verbose, logFolder=args.logfolder) # From now on, the logger can be recovered like this: logger = logging.getLogger("osrframework.usufy") # Printing the results if requested if not args.maltego: print banner.text sayingHello = """usufy.py Copyright (C) F. Brezo and Y. Rubio (i3visio) 2015 This program comes with ABSOLUTELY NO WARRANTY. This is free software, and you are welcome to redistribute it under certain conditions. For additional info, visit <http://www.gnu.org/licenses/gpl-3.0.txt>.""" logger.info(sayingHello) print sayingHello print logger.info("Starting usufy.py...") if args.license: logger.info("Looking for the license...") # showing the license try: with open ("COPYING", "r") as iF: contenido = iF.read().splitlines() for linea in contenido: print linea except Exception: try: # Trying to recover the COPYING file... with open ("/usr/share/osrframework/COPYING", "r") as iF: contenido = iF.read().splitlines() for linea in contenido: print linea except: logger.error("ERROR: there has been an error when opening the COPYING file.\n\tThe file contains the terms of the GPLv3 under which this software is distributed.\n\tIn case of doubts, verify the integrity of the files or contact [email protected].") elif args.fuzz: logger.info("Performing the fuzzing tasks...") res = fuzzUsufy(args.fuzz, args.fuzz_config) logger.info("Recovered platforms:\n" + str(res)) else: logger.debug("Recovering the list of platforms to be processed...") # Recovering the list of platforms to be launched listPlatforms = platform_selection.getPlatformsByName(platformNames=args.platforms, tags=args.tags, mode="usufy") logger.debug("Platforms recovered.") if args.info: # Information actions... if args.info == 'list_platforms': infoPlatforms="Listing the platforms:\n" for p in listPlatforms: infoPlatforms += "\t\t" + (str(p) + ": ").ljust(16, ' ') + str(p.tags)+"\n" logger.info(infoPlatforms) return infoPlatforms elif args.info == 'list_tags': logger.info("Listing the tags:") tags = {} # Going through all the selected platforms to get their tags for p in listPlatforms: for t in p.tags: if t not in tags.keys(): tags[t] = 1 else: tags[t] += 1 infoTags = "List of tags:\n" # Displaying the results in a sorted list for t in tags.keys(): infoTags += "\t\t" + (t + ": ").ljust(16, ' ') + str(tags[t]) + " time(s)\n" logger.info(infoTags) return infoTags else: pass # performing the test elif args.benchmark: logger.warning("The benchmark mode may last some minutes as it will be performing similar queries to the ones performed by the program in production. ") logger.info("Launching the benchmarking tests...") platforms = platform_selection.getAllPlatformNames("usufy") res = benchmark.doBenchmark(platforms) strTimes = "" for e in sorted(res.keys()): strTimes += str(e) + "\t" + str(res[e]) + "\n" logger.info(strTimes) return strTimes # Executing the corresponding process... else: # Showing the execution time... if not args.maltego: startTime= dt.datetime.now() print str(startTime) +"\tStarting search in " + str(len(listPlatforms)) + " platform(s)... Be patient!\n" # Defining the list of users to monitor nicks = [] logger.debug("Recovering nicknames to be processed...") if args.nicks: for n in args.nicks: # TO-DO # A trick to avoid having the processing of the properties when being queried by Maltego if "properties.i3visio" not in n: nicks.append(n) else: # Reading the nick files try: nicks = args.list.read().splitlines() except: logger.error("ERROR: there has been an error when opening the file that stores the nicks.\tPlease, check the existence of this file.") if args.output_folder != None: # if Verifying an output folder was selected logger.debug("Preparing the output folder...") if not args.maltego: if not os.path.exists(args.output_folder): logger.warning("The output folder \'" + args.output_folder + "\' does not exist. The system will try to create it.") os.makedirs(args.output_folder) # Launching the process... res = processNickList(nicks, listPlatforms, args.output_folder, avoidProcessing = args.avoid_processing, avoidDownload = args.avoid_download, nThreads=args.threads, verbosity= args.verbose, logFolder=args.logfolder) else: res = processNickList(nicks, listPlatforms, nThreads=args.threads, verbosity= args.verbose, logFolder=args.logfolder) logger.info("Listing the results obtained...") # We are going to iterate over the results... strResults = "\t" # Structure returned """ [ { "attributes": [ { "attributes": [], "type": "i3visio.uri", "value": "http://twitter.com/i3visio" }, { "attributes": [], "type": "i3visio.alias", "value": "i3visio" }, { "attributes": [], "type": "i3visio.platform", "value": "Twitter" } ], "type": "i3visio.profile", "value": "Twitter - i3visio" } , ... ] """ for r in res: # The format of the results (attributes) for a given nick is a list as follows: for att in r["attributes"]: # iterating through the attributes platform = "" uri = "" for details in att["attributes"]: if details["type"] == "i3visio.platform": platform = details["value"] if details["type"] == "i3visio.uri": uri = details["value"] try: strResults+= (str(platform) + ":").ljust(16, ' ')+ " "+ str(uri)+"\n\t\t" except: pass logger.info(strResults) # Generating summary files for each ... if args.extension: # Storing the file... logger.info("Creating output files as requested.") if not args.maltego: # Verifying if the outputPath exists if not os.path.exists (args.output_folder): logger.warning("The output folder \'" + args.output_folder + "\' does not exist. The system will try to create it.") os.makedirs(args.output_folder) # Grabbing the results fileHeader = os.path.join(args.output_folder, args.file_header) # Iterating through the given extensions to print its values if not args.maltego: for ext in args.extension: # Generating output files general.exportUsufy(res, ext, fileHeader) # Generating the Maltego output if args.maltego: general.listToMaltego(res) # Printing the results if requested if not args.maltego: print "A summary of the results obtained are shown in the following table:" #print res print unicode(general.usufyToTextExport(res)) print print "You can find all the information collected in the following files:" for ext in args.extension: # Showing the output files print "\t-" + fileHeader + "." + ext # Showing the execution time... if not args.maltego: print endTime= dt.datetime.now() print str(endTime) +"\tFinishing execution..." print print "Total time used:\t" + str(endTime-startTime) print "Average seconds/query:\t" + str((endTime-startTime).total_seconds()/len(listPlatforms)) +" seconds" print # Urging users to place an issue on Github... if not args.maltego: print print "Did something go wrong? Is a platform reporting false positives? Do you need to integrate a new one?" print "Then, place an issue in the Github project: <https://github.com/i3visio/osrframework/issues>." print "Note that otherwise, we won't know about it!" print return res