Ejemplo n.º 1
0
def fuzzUsufy(fDomains = None, fFuzzStruct = None):
    '''
        Method to guess the usufy path against a list of domains or subdomains.

        :param fDomains:    a list to strings containing the domains and (optionally) a nick.
        :param fFuzzStruct:    a list to strings containing the transforms to be performed.

        :return:    Dictionary of {domain: url}.
    '''
    logger = logging.getLogger("osrframework.usufy")

    if fFuzzStruct == None:
        # Loading these structures by default
        fuzzingStructures = [
                    "http://<DOMAIN>/<USERNAME>",
                    "http://<DOMAIN>/~<USERNAME>",
                    "http://<DOMAIN>/?action=profile;user=<USERNAME>",
                    "http://<DOMAIN>/causes/author/<USERNAME>",
                    "http://<DOMAIN>/channel/<USERNAME>",
                    "http://<DOMAIN>/community/profile/<USERNAME>",
                    "http://<DOMAIN>/component/comprofiler/userprofiler/<USERNAME>",
                    "http://<DOMAIN>/details/@<USERNAME>",
                    "http://<DOMAIN>/foros/member.php?username=<USERNAME>",
                    "http://<DOMAIN>/forum/member/<USERNAME>",
                    "http://<DOMAIN>/forum/member.php?username=<USERNAME>",
                    "http://<DOMAIN>/forum/profile.php?mode=viewprofile&u=<USERNAME>",
                    "http://<DOMAIN>/home/<USERNAME>",
                    "http://<DOMAIN>/index.php?action=profile;user=<USERNAME>",
                    "http://<DOMAIN>/member_profile.php?u=<USERNAME>",
                    "http://<DOMAIN>/member.php?username=<USERNAME>",
                    "http://<DOMAIN>/members/?username=<USERNAME>",
                    "http://<DOMAIN>/members/<USERNAME>",
                    "http://<DOMAIN>/members/view/<USERNAME>",
                    "http://<DOMAIN>/mi-espacio/<USERNAME>",
                    "http://<DOMAIN>/u<USERNAME>",
                    "http://<DOMAIN>/u/<USERNAME>",
                    "http://<DOMAIN>/user-<USERNAME>",
                    "http://<DOMAIN>/user/<USERNAME>",
                    "http://<DOMAIN>/user/<USERNAME>.html",
                    "http://<DOMAIN>/users/<USERNAME>",
                    "http://<DOMAIN>/usr/<USERNAME>",
                    "http://<DOMAIN>/usuario/<USERNAME>",
                    "http://<DOMAIN>/usuarios/<USERNAME>",
                    "http://<DOMAIN>/en/users/<USERNAME>",
                    "http://<DOMAIN>/people/<USERNAME>",
                    "http://<DOMAIN>/profil/<USERNAME>",
                    "http://<DOMAIN>/profile/<USERNAME>",
                    "http://<DOMAIN>/profile/page/<USERNAME>",
                    "http://<DOMAIN>/rapidforum/index.php?action=profile;user=<USERNAME>",
                    "http://<DOMAIN>/social/usuarios/<USERNAME>",
                    "http://<USERNAME>.<DOMAIN>",
                    "http://<USERNAME>.<DOMAIN>/user/"
                ]
    else:
        try:
            fuzzingStructures = fFuzzStruct.read().splitlines()
        except:
            logger.error("Usufy could NOT open the following file: " + fFuzzStruct )

    res = {}

    lines = fDomains.read().splitlines()

    # Going through all the lines
    for l in lines:
        domain = l.split()[0]
        print "Performing tests for" + domain + "..."

        # selecting the number of nicks to be tested in this domain
        nick = l.split()[1]

        # Choosing the errors from the input file
        #errors = l.split('\t')[2:]

        # possibleURLs found
        possibleURL = []

        for struct in fuzzingStructures:
            # initiating list
            urlToTry = struct.replace("<DOMAIN>", domain)
            test = urlToTry.replace("<USERNAME>", nick.lower())
            print "Processing "+ test + "..."
            i3Browser = browser.Browser()
            try:
                html = i3Browser.recoverURL(test)
                if nick in html:
                    possibleURL.append(test)
                    print "\tPossible usufy found!!!\n"
            except:
                logger.error("The resource could not be downloaded.")

        #print possibleURL
        res[domain] = possibleURL

    print json.dumps(res, indent = 2)
    return res
Ejemplo n.º 2
0
def main(args):
    '''
        Main function. This function is created in this way so as to let other applications make use of the full configuration capabilities of the application.
    '''
    # Recovering the logger
    # Calling the logger when being imported
    osrframework.utils.logger.setupLogger(loggerName="osrframework.usufy", verbosity=args.verbose, logFolder=args.logfolder)
    # From now on, the logger can be recovered like this:
    logger = logging.getLogger("osrframework.usufy")
    # Printing the results if requested
    if not args.maltego:
        print banner.text

        sayingHello = """usufy.py Copyright (C) F. Brezo and Y. Rubio (i3visio) 2014-2017
This program comes with ABSOLUTELY NO WARRANTY.
This is free software, and you are welcome to redistribute it under certain conditions. For additional info, visit <http://www.gnu.org/licenses/gpl-3.0.txt>."""
        logger.info(sayingHello)
        print sayingHello
        print
        logger.info("Starting usufy.py...")

    if args.license:
        logger.info("Looking for the license...")
        # showing the license
        try:
            with open ("COPYING", "r") as iF:
                contenido = iF.read().splitlines()
                for linea in contenido:
                    print linea
        except Exception:
            try:
                # Trying to recover the COPYING file...
                with open ("/usr/share/osrframework/COPYING", "r") as iF:
                    contenido = iF.read().splitlines()
                    for linea in contenido:
                        print linea
            except:
                logger.error("ERROR: there has been an error when opening the COPYING file.\n\tThe file contains the terms of the GPLv3 under which this software is distributed.\n\tIn case of doubts, verify the integrity of the files or contact [email protected].")
    elif args.fuzz:
        logger.info("Performing the fuzzing tasks...")
        res = fuzzUsufy(args.fuzz, args.fuzz_config)
        logger.info("Recovered platforms:\n" + str(res))
    else:
        logger.debug("Recovering the list of platforms to be processed...")
        # Recovering the list of platforms to be launched
        listPlatforms = platform_selection.getPlatformsByName(platformNames=args.platforms, tags=args.tags, mode="usufy", excludePlatformNames=args.exclude)
        logger.debug("Platforms recovered.")

        if args.info:
            # Information actions...
            if args.info == 'list_platforms':
                infoPlatforms="Listing the platforms:\n"
                for p in listPlatforms:
                    infoPlatforms += "\t\t" + (str(p) + ": ").ljust(16, ' ') + str(p.tags)+"\n"
                logger.info(infoPlatforms)
                return infoPlatforms
            elif args.info == 'list_tags':
                logger.info("Listing the tags:")
                tags = {}
                # Going through all the selected platforms to get their tags
                for p in listPlatforms:
                    for t in p.tags:
                        if t not in tags.keys():
                            tags[t] = 1
                        else:
                            tags[t] += 1
                infoTags = "List of tags:\n"
                # Displaying the results in a sorted list
                for t in tags.keys():
                    infoTags += "\t\t" + (t + ": ").ljust(16, ' ') + str(tags[t]) + "  time(s)\n"
                logger.info(infoTags)
                return infoTags
            else:
                pass

        # performing the test
        elif args.benchmark:
            logger.warning("The benchmark mode may last some minutes as it will be performing similar queries to the ones performed by the program in production. ")
            logger.info("Launching the benchmarking tests...")
            platforms = platform_selection.getAllPlatformNames("usufy")
            res = benchmark.doBenchmark(platforms)
            strTimes = ""
            for e in sorted(res.keys()):
                strTimes += str(e) + "\t" + str(res[e]) + "\n"
            logger.info(strTimes)
            return strTimes

        # showing the tags of the usufy platforms
        elif args.show_tags:
            logger.info("Collecting the list of tags...")
            tags = platform_selection.getAllPlatformNamesByTag("usufy")
            logger.info(json.dumps(tags, indent=2))
            print "This is the list of platforms grouped by tag."
            print
            print json.dumps(tags, indent=2, sort_keys=True)
            print
            print "[Tip] Remember that you can always launch the platform using the -t option followed by any of the aforementioned."
            print
            return tags

        # Executing the corresponding process...
        else:
            # Showing the execution time...
            if not args.maltego:
                startTime= dt.datetime.now()
                print str(startTime) +"\tStarting search in " + str(len(listPlatforms)) + " platform(s)... Relax!"
                print
                print "\tPress <Ctrl + C> to stop..."
                print

            # Defining the list of users to monitor
            nicks = []
            logger.debug("Recovering nicknames to be processed...")
            if args.nicks:
                for n in args.nicks:
                    # TO-DO
                    #     A trick to avoid having the processing of the properties when being queried by Maltego
                    if "properties.i3visio" not in n:
                        nicks.append(n)
            else:
                # Reading the nick files
                try:
                    nicks = args.list.read().splitlines()
                except:
                    logger.error("ERROR: there has been an error when opening the file that stores the nicks.\tPlease, check the existence of this file.")

            # Definning the results
            res = []

            if args.output_folder != None:
                # if Verifying an output folder was selected
                logger.debug("Preparing the output folder...")
                if not args.maltego:
                    if not os.path.exists(args.output_folder):
                        logger.warning("The output folder \'" + args.output_folder + "\' does not exist. The system will try to create it.")
                        os.makedirs(args.output_folder)
                # Launching the process...
                ###try:
                res = processNickList(nicks, listPlatforms, args.output_folder, avoidProcessing = args.avoid_processing, avoidDownload = args.avoid_download, nThreads=args.threads, verbosity= args.verbose, logFolder=args.logfolder)
                ###except Exception as e:
                    ###print "Exception grabbed when processing the nicks: " + str(e)
                    ###print traceback.print_stack()
            else:
                try:
                    res = processNickList(nicks, listPlatforms, nThreads=args.threads, verbosity= args.verbose, logFolder=args.logfolder)
                except Exception as e:
                    print "Exception grabbed when processing the nicks: " + str(e)
                    print traceback.print_stack()

            logger.info("Listing the results obtained...")
            # We are going to iterate over the results...
            strResults = "\t"

            # Structure returned
            """
            [
                {
                  "attributes": [
                    {
                      "attributes": [],
                      "type": "i3visio.uri",
                      "value": "http://twitter.com/i3visio"
                    },
                    {
                      "attributes": [],
                      "type": "i3visio.alias",
                      "value": "i3visio"
                    },
                    {
                      "attributes": [],
                      "type": "i3visio.platform",
                      "value": "Twitter"
                    }
                  ],
                  "type": "i3visio.profile",
                  "value": "Twitter - i3visio"
                }
                ,
                ...
            ]
            """
            for r in res:
                # The format of the results (attributes) for a given nick is a list as follows:

                for att in r["attributes"]:
                    # iterating through the attributes
                    platform = ""
                    uri = ""
                    for details in att["attributes"]:
                        if details["type"] == "i3visio.platform":
                            platform = details["value"]
                        if details["type"] == "i3visio.uri":
                            uri = details["value"]
                    try:
                        strResults+= (str(platform) + ":").ljust(16, ' ')+ " "+ str(uri)+"\n\t\t"
                    except:
                        pass

                logger.info(strResults)

            # Generating summary files for each ...
            if args.extension:
                # Storing the file...
                logger.info("Creating output files as requested.")
                if not args.maltego:
                    # Verifying if the outputPath exists
                    if not os.path.exists (args.output_folder):
                        logger.warning("The output folder \'" + args.output_folder + "\' does not exist. The system will try to create it.")
                        os.makedirs(args.output_folder)

                    # Grabbing the results
                    fileHeader = os.path.join(args.output_folder, args.file_header)

                    # Iterating through the given extensions to print its values
                    for ext in args.extension:
                        # Generating output files
                        general.exportUsufy(res, ext, fileHeader)

            # Generating the Maltego output
            if args.maltego:
                general.listToMaltego(res)
            # Printing the results if requested
            else:
                print "A summary of the results obtained are shown in the following table:"
                #print res
                print unicode(general.usufyToTextExport(res))

                print

                if args.web_browser:
                    general.openResultsInBrowser(res)

                print "You can find all the information collected in the following files:"
                for ext in args.extension:
                    # Showing the output files
                    print "\t-" + fileHeader + "." + ext

                # Showing the execution time...
                print
                endTime= dt.datetime.now()
                print str(endTime) +"\tFinishing execution..."
                print
                print "Total time used:\t" + str(endTime-startTime)
                print "Average seconds/query:\t" + str((endTime-startTime).total_seconds()/len(listPlatforms)) +" seconds"
                print

                # Urging users to place an issue on Github...
                print
                print "Did something go wrong? Is a platform reporting false positives? Do you need to integrate a new one?"
                print "Then, place an issue in the Github project: <https://github.com/i3visio/osrframework/issues>."
                print "Note that otherwise, we won't know about it!"
                print

            return res
Ejemplo n.º 3
0
def fuzzUsufy(fDomains = None, fFuzzStruct = None):
    ''' 
        Method to guess the usufy path against a list of domains or subdomains.
        
        :param fDomains:    a list to strings containing the domains and (optionally) a nick.
        :param fFuzzStruct:    a list to strings containing the transforms to be performed.
        
        :return:    Dictionary of {domain: url}.
    '''
    logger = logging.getLogger("osrframework.usufy")
    
    if fFuzzStruct == None:
        # Loading these structures by default
        fuzzingStructures = [
                    "http://<USERNAME>.<DOMAIN>",
                    "http://<USERNAME>.<DOMAIN>/user/",
                    "http://<DOMAIN>/<USERNAME>",
                    "http://<DOMAIN>/user/<USERNAME>",
                    "http://<DOMAIN>/users/<USERNAME>",
                    "http://<DOMAIN>/en/users/<USERNAME>",
                    "http://<DOMAIN>/profil/<USERNAME>",
                    "http://<DOMAIN>/profile/<USERNAME>",
                    "http://<DOMAIN>/members/<USERNAME>",
                    "http://<DOMAIN>/channel/<USERNAME>",
                    "http://<DOMAIN>/u/<USERNAME>",
                    "http://<DOMAIN>/home/<USERNAME>",
                    "http://<DOMAIN>/people/<USERNAME>",
                    "http://<DOMAIN>/usr/<USERNAME>",
                    "http://<DOMAIN>/~<USERNAME>",
                    "http://<DOMAIN>/user-<USERNAME>",
                    "http://<DOMAIN>/causes/author/<USERNAME>",
                    "http://<DOMAIN>/profile/page/<USERNAME>",
                    "http://<DOMAIN>/component/comprofiler/userprofiler/<USERNAME>",
                    "http://<DOMAIN>/social/usuarios/<USERNAME>",
                    "http://<DOMAIN>/mi-espacio/<USERNAME>",
                    "http://<DOMAIN>/forum/profile.php?mode=viewprofile&u=<USERNAME>",
                    "http://<DOMAIN>/index.php?action=profile;user=<USERNAME>",
                    "http://<DOMAIN>/member.php?username=<USERNAME>",
                    "http://<DOMAIN>/members/?username=<USERNAME>",
                    "http://<DOMAIN>/forum/member.php?username=<USERNAME>",
                    "http://<DOMAIN>/member.php?username=<USERNAME>",
                    "http://<DOMAIN>/rapidforum/index.php?action=profile;user=<USERNAME>",
                ]
    else:
        try:
            fuzzingStructures = fFuzzStruct.read().splitlines()
        except:
            logger.error("Usufy could NOT open the following file: " + fFuzzStruct )
    
    res = {}
    
    lines = fDomains.read().splitlines()
    
    # Going through all the lines
    for l in lines:
        domain = l.split('\t')[0]
        print "Performing tests for" + domain + "..."
        
        # selecting the number of nicks to be tested in this domain
        nick = l.split('\t')[1]
        
        # Choosing the errors from the input file
        #errors = l.split('\t')[2:]
        
        # possibleURLs found
        possibleURL = []
        
        for struct in fuzzingStructures:
            # initiating list
            urlToTry = struct.replace("<DOMAIN>", domain)
            test = urlToTry.replace("<USERNAME>", nick.lower())
            print "Processing "+ test + "..."
            i3Browser = browser.Browser()
            try:
                html = i3Browser.recoverURL(test)
                if nick in html:
                    possibleURL.append(test)      
                    print "Usufy found!!!\n"
                    break
            except:
                logger.error("The resource could not be downloaded.")
               
        #print possibleURL                
        res[domain] = possibleURL

    print json.dumps(res, indent = 2)
    return res
Ejemplo n.º 4
0
def usufy_main(args):
    ''' 
        Main function. This function is created in this way so as to let other applications make use of the full configuration capabilities of the application.    
    '''
    # Recovering the logger
    # Calling the logger when being imported
    osrframework.utils.logger.setupLogger(loggerName="osrframework.usufy", verbosity=args.verbose, logFolder=args.logfolder)    
    # From now on, the logger can be recovered like this:
    logger = logging.getLogger("osrframework.usufy")

    logger.info("""usufy.py Copyright (C) F. Brezo and Y. Rubio (i3visio) 2015
This program comes with ABSOLUTELY NO WARRANTY.
This is free software, and you are welcome to redistribute it under certain conditions.
For details, run:
\tpython usufy.py --license""")
    
    logger.info("Starting usufy.py...")

    if args.license:
        logger.info("Looking for the license...")
        # showing the license
        try:
            with open ("COPYING", "r") as iF:
                contenido = iF.read().splitlines()
                for linea in contenido:    
                    print linea
        except Exception:
            logger.error("ERROR: there has been an error when opening the COPYING file.\n\tThe file contains the terms of the GPLv3 under which this software is distributed.\n\tIn case of doubts, verify the integrity of the files or contact [email protected].")
    elif args.fuzz:
        logger.info("Performing the fuzzing tasks...")
        res = fuzzUsufy(args.fuzz, args.fuzz_config)
        logger.info("Recovered platforms:\n" + str(res))
    else:
        logger.debug("Recovering the list of platforms to be processed...")
        # Recovering the list of platforms to be launched
        listPlatforms = platform_selection.getPlatformsByName(platformNames=args.platforms, tags=args.tags, mode="usufy")
        logger.debug("Platforms recovered.")

        if args.info:
            # Information actions...
            if args.info == 'list_platforms':
                infoPlatforms="Listing the platforms:\n"
                for p in listPlatforms:
                    infoPlatforms += "\t\t" + (str(p) + ": ").ljust(16, ' ') + str(p.tags)+"\n"
                logger.info(infoPlatforms)
                return infoPlatforms
            elif args.info == 'list_tags':
                logger.info("Listing the tags:")
                tags = {}
                # Going through all the selected platforms to get their tags
                for p in listPlatforms:
                    for t in p.tags:
                        if t not in tags.keys():
                            tags[t] = 1
                        else:
                            tags[t] += 1
                infoTags = "List of tags:\n"
                # Displaying the results in a sorted list
                for t in tags.keys():
                    infoTags += "\t\t" + (t + ": ").ljust(16, ' ') + str(tags[t]) + "  time(s)\n"
                logger.info(infoTags)
                return infoTags
            else:
                pass
                
        # performing the test
        elif args.benchmark:
            logger.warning("The benchmark mode may last some minutes as it will be performing similar queries to the ones performed by the program in production. ")
            logger.info("Launching the benchmarking tests...")
            platforms = platform_selection.getAllPlatformNames("usufy")
            res = benchmark.doBenchmark(platforms)
            strTimes = ""
            for e in sorted(res.keys()):
                strTimes += str(e) + "\t" + str(res[e]) + "\n"
            logger.info(strTimes)
            return strTimes
        # Executing the corresponding process...
        else:
            # Defining the list of users to monitor
            nicks = []
            logger.debug("Recovering nicknames to be processed...")
            if args.nicks:
                for n in args.nicks:
                    # TO-DO
                    #     A trick to avoid having the processing of the properties when being queried by Maltego
                    if "properties.i3visio" not in n:
                        nicks.append(n)
            else:
                # Reading the nick files
                try:
                    nicks = args.list.read().splitlines()
                except:
                    logger.error("ERROR: there has been an error when opening the file that stores the nicks.\tPlease, check the existence of this file.")

            # Checking if performing any transform was required
            if args.squatting:
                logger.debug("Making basic transformations on the provided nicknames...")
                # Iterating to process _ and or .
                #nicks = profilesquatting.generatingProfiles(nicks, args.profilesquatting)
                nicks = profilesquatting.getNewNicks(nicks, logName = "usufy", modes = args.squatting, nonValidChars = args.nonvalid)
                logger.info("Obtained nicks:\n" + str(nicks))
                
                logger.debug("Profilesquatting nicknames recovered.")
                if args.info == 'list_users':            
                    strNicks = ""
                    for n in nicks:
                        strNicks += n + "\n"
                    logger.info("Generated nicks:\n----------------\n" + strNicks)
                    # Storing the file...
                    logger.info("Creating output files as requested.")
                    # Verifying if the outputPath exists
                    if not os.path.exists (args.output_folder):
                        logger.warning("The output folder \'" + args.output_folder + "\' does not exist. The system will try to create it.")
                        os.makedirs(args.output_folder)
                            
                    strTime = general.getCurrentStrDatetime()
                    logger.info("Writing generated nicks to a text file.")
                    with open (os.path.join(args.output_folder, "nicks_" + strTime +".txt"), "w") as oF:
                        oF.write( strNicks )
                    # if this option was selected, we will jsut return this and exist
                    return nicks            

            if args.output_folder != None:    
                # if Verifying an output folder was selected
                logger.debug("Preparing the output folder...")
                if not os.path.exists(args.output_folder):
                    logger.warning("The output folder \'" + args.output_folder + "\' does not exist. The system will try to create it.")
                    os.makedirs(args.output_folder)
                # Launching the process...
                res = processNickList(nicks, listPlatforms, args.output_folder, avoidProcessing = args.avoid_processing, avoidDownload = args.avoid_download, nThreads=args.threads, verbosity= args.verbose, logFolder=args.logfolder)
            else:
                res = processNickList(nicks, listPlatforms, nThreads=args.threads, verbosity= args.verbose, logFolder=args.logfolder)
                
            
            logger.info("Listing the results obtained...")
            # We are going to iterate over the results...
            strResults = "\t"   
            
            # Structure returned 
            """
            [
                {
                  "attributes": [
                    {
                      "attributes": [], 
                      "type": "i3visio.uri", 
                      "value": "http://twitter.com/i3visio"
                    }, 
                    {
                      "attributes": [], 
                      "type": "i3visio.alias", 
                      "value": "i3visio"
                    }, 
                    {
                      "attributes": [], 
                      "type": "i3visio.platform", 
                      "value": "Twitter"
                    }
                  ], 
                  "type": "i3visio.profile", 
                  "value": "Twitter - i3visio"
                }
                ,
                ...
            ]                    
            """    
            for r in res:               
                # The format of the results (attributes) for a given nick is a list as follows:
                for att in r["attributes"]:
                    # iterating through the attributes
                    platform = ""
                    uri = ""
                    for details in att["attributes"]:
                        if details["type"] == "i3visio.platform":
                            platform = details["value"]
                        if details["type"] == "i3visio.uri":
                            uri = details["value"]                                
                    try:
                        strResults+= (str(platform) + ":").ljust(16, ' ')+ " "+ str(uri)+"\n\t\t"
                    except:
                        pass

                logger.info(strResults)

                # Generating summary files for each ...
                if args.extension:
                    # Storing the file...
                    logger.info("Creating output files as requested.")
                    if not args.output_folder:
                        args.output_folder = "./"
                    else:
                        # Verifying if the outputPath exists
                        if not os.path.exists (args.output_folder):
                            logger.warning("The output folder \'" + args.output_folder + "\' does not exist. The system will try to create it.")
                            os.makedirs(args.output_folder)
                            
                    # Grabbing the results 
                    fileHeader = os.path.join(args.output_folder, args.file_header + general.getCurrentStrDatetime())

                    # Iterating through the given extensions to print its values
                    for ext in args.extension:
                        # Generating output files
                        general.exportUsufy(res, ext, fileHeader)
                        
                # Generating the Maltego output    
                if args.maltego:
                    general.listToMaltego(res)

                # Printing the results if requested
                if not args.maltego:
                    print "A summary of the results obtained are the following table:"
                    print general.usufyToTextExport(res)
                    print "You can find all the information collected in the following files:"                    
                    for ext in args.extension:
                        # Generating output files
                        print "\t-" + fileHeader + "." + ext
                return res
Ejemplo n.º 5
0
def main(params=None):
    """
    Main function to launch usufy.

    The function is created in this way so as to let other applications make
    use of the full configuration capabilities of the application. The
    parameters received are used as parsed by this modules `getParser()`.

    Args:
    -----
        params: A list with the parameters as grabbed by the terminal. It is
            None when this is called by an entry_point.

    Returns:
    --------
        dict: A Json representing the matching results.
    """
    # Grabbing the parser
    parser = getParser()

    if params != None:
        args = parser.parse_args(params)
    else:
        args = parser.parse_args()

    # Recovering the logger
    # Calling the logger when being imported
    osrframework.utils.logger.setupLogger(loggerName="osrframework.usufy",
                                          verbosity=args.verbose,
                                          logFolder=args.logfolder)
    # From now on, the logger can be recovered like this:
    logger = logging.getLogger("osrframework.usufy")

    print(general.title(banner.text))

    sayingHello = """
Usufy | Copyright (C) F. Brezo and Y. Rubio (i3visio) 2014-2018

This program comes with ABSOLUTELY NO WARRANTY. This is free software, and you
are welcome to redistribute it under certain conditions. For additional info,
visit """ + general.LICENSE_URL + "\n"
    logger.info(sayingHello)
    print(general.title(sayingHello))
    logger.info("Starting usufy...")

    if args.license:
        general.showLicense()
    elif args.fuzz:
        logger.info("Performing the fuzzing tasks...")
        res = fuzzUsufy(args.fuzz, args.fuzz_config)
        logger.info("Recovered platforms:\n" + str(res))
    else:
        logger.debug("Recovering the list of platforms to be processed...")
        # Recovering the list of platforms to be launched
        listPlatforms = platform_selection.getPlatformsByName(
            platformNames=args.platforms,
            tags=args.tags,
            mode="usufy",
            excludePlatformNames=args.exclude)
        logger.debug("Platforms recovered.")

        if args.info:
            # Information actions...
            if args.info == 'list_platforms':
                infoPlatforms = "Listing the platforms:\n"
                for p in listPlatforms:
                    infoPlatforms += "\t\t" + (str(p) + ": ").ljust(
                        16, ' ') + str(p.tags) + "\n"
                logger.info(infoPlatforms)
                return infoPlatforms
            elif args.info == 'list_tags':
                logger.info("Listing the tags:")
                tags = {}
                # Going through all the selected platforms to get their tags
                for p in listPlatforms:
                    for t in p.tags:
                        if t not in tags.keys():
                            tags[t] = 1
                        else:
                            tags[t] += 1
                infoTags = "List of tags:\n"
                # Displaying the results in a sorted list
                for t in tags.keys():
                    infoTags += "\t\t" + (t + ": ").ljust(16, ' ') + str(
                        tags[t]) + "  time(s)\n"
                logger.info(infoTags)
                return infoTags
            else:
                pass

        # performing the test
        elif args.benchmark:
            logger.warning(
                "The benchmark mode may last some minutes as it will be performing similar queries to the ones performed by the program in production. "
            )
            logger.info("Launching the benchmarking tests...")
            platforms = platform_selection.getAllPlatformNames("usufy")
            res = benchmark.doBenchmark(platforms)
            strTimes = ""
            for e in sorted(res.keys()):
                strTimes += str(e) + "\t" + str(res[e]) + "\n"
            logger.info(strTimes)
            return strTimes

        # showing the tags of the usufy platforms
        elif args.show_tags:
            logger.info("Collecting the list of tags...")
            tags = platform_selection.getAllPlatformNamesByTag("usufy")
            logger.info(json.dumps(tags, indent=2))
            print(
                general.info(
                    "This is the list of platforms grouped by tag.\n"))
            print(json.dumps(tags, indent=2, sort_keys=True))
            print(
                general.info(
                    "[Tip] Remember that you can always launch the platform using the -t option followed by any of the aforementioned.\n"
                ))
            return tags

        # Executing the corresponding process...
        else:
            # Showing the execution time...
            startTime = dt.datetime.now()
            print(
                str(startTime) + "\tStarting search in " +
                general.emphasis(str(len(listPlatforms))) +
                " platform(s)... Relax!\n")
            print(general.emphasis("\tPress <Ctrl + C> to stop...\n"))

            # Defining the list of users to monitor
            nicks = []
            logger.debug("Recovering nicknames to be processed...")
            if args.nicks:
                for n in args.nicks:
                    # TO-DO
                    #     A trick to avoid having the processing of the properties when being queried by Maltego
                    if "properties.i3visio" not in n:
                        nicks.append(n)
            else:
                # Reading the nick files
                try:
                    nicks = args.list.read().splitlines()
                except:
                    logger.error(
                        "ERROR: there has been an error when opening the file that stores the nicks.\tPlease, check the existence of this file."
                    )

            # Definning the results
            res = []

            if args.output_folder != None:
                # if Verifying an output folder was selected
                logger.debug("Preparing the output folder...")
                if not os.path.exists(args.output_folder):
                    logger.warning(
                        "The output folder \'" + args.output_folder +
                        "\' does not exist. The system will try to create it.")
                    os.makedirs(args.output_folder)
                # Launching the process...
                res = processNickList(nicks,
                                      listPlatforms,
                                      args.output_folder,
                                      avoidProcessing=args.avoid_processing,
                                      avoidDownload=args.avoid_download,
                                      nThreads=args.threads,
                                      verbosity=args.verbose,
                                      logFolder=args.logfolder)

            else:
                try:
                    res = processNickList(nicks,
                                          listPlatforms,
                                          nThreads=args.threads,
                                          verbosity=args.verbose,
                                          logFolder=args.logfolder)
                except Exception as e:
                    print(
                        general.error(
                            "Exception grabbed when processing the nicks: " +
                            str(e)))
                    print(general.error(traceback.print_stack()))

            logger.info("Listing the results obtained...")
            # We are going to iterate over the results...
            strResults = "\t"

            # Structure returned
            """
            [
                {        print

                  "attributes": [
                    {
                      "attributes": [],
                      "type": "i3visio.uri",
                      "value": "http://twitter.com/i3visio"
                    },
                    {
                      "attributes": [],
                      "type": "i3visio.alias",
                      "value": "i3visio"
                    },
                    {
                      "attributes": [],
                      "type": "i3visio.platform",
                      "value": "Twitter"
                    }
                  ],
                  "type": "i3visio.profile",
                  "value": "Twitter - i3visio"
                }
                ,
                ...
            ]
            """
            for r in res:
                # The format of the results (attributes) for a given nick is a list as follows:

                for att in r["attributes"]:
                    # iterating through the attributes
                    platform = ""
                    uri = ""
                    for details in att["attributes"]:
                        if details["type"] == "i3visio.platform":
                            platform = details["value"]
                        if details["type"] == "i3visio.uri":
                            uri = details["value"]
                    try:
                        strResults += (str(platform) + ":").ljust(
                            16, ' ') + " " + str(uri) + "\n\t\t"
                    except:
                        pass

                logger.info(strResults)

            # Generating summary files for each ...
            if args.extension:
                # Storing the file...
                logger.info("Creating output files as requested.")
                # Verifying if the outputPath exists
                if not os.path.exists(args.output_folder):
                    logger.warning(
                        "The output folder \'" + args.output_folder +
                        "\' does not exist. The system will try to create it.")
                    os.makedirs(args.output_folder)

                # Grabbing the results
                fileHeader = os.path.join(args.output_folder, args.file_header)

                # Iterating through the given extensions to print its values
                for ext in args.extension:
                    # Generating output files
                    general.exportUsufy(res, ext, fileHeader)

            now = dt.datetime.now()
            print(
                str(now) +
                "\tA summary of the results obtained is shown below:\n")
            print(general.success(general.usufyToTextExport(res)))

            if args.web_browser:
                general.openResultsInBrowser(res)

            now = dt.datetime.now()
            print("\n" + str(now) + "\tYou can find all the information here:")
            for ext in args.extension:
                # Showing the output files
                print("\t" + general.emphasis(fileHeader + "." + ext))

            # Showing the execution time...
            endTime = dt.datetime.now()
            print("\n" + str(endTime) + "\tFinishing execution...\n")
            print("Total time consumed:\t" +
                  general.emphasis(str(endTime - startTime)))
            print("Average seconds/query:\t" + general.emphasis(
                str((endTime - startTime).total_seconds() /
                    len(listPlatforms))) + " seconds\n")

            # Urging users to place an issue on Github...
            print(banner.footer)

    if params:
        return res
Ejemplo n.º 6
0
def main(args):
    '''
        Main function. This function is created in this way so as to let other applications make use of the full configuration capabilities of the application.
    '''
    # Recovering the logger
    # Calling the logger when being imported
    osrframework.utils.logger.setupLogger(loggerName="osrframework.usufy", verbosity=args.verbose, logFolder=args.logfolder)
    # From now on, the logger can be recovered like this:
    logger = logging.getLogger("osrframework.usufy")
    # Printing the results if requested
    if not args.maltego:
        print banner.text

        sayingHello = """usufy.py Copyright (C) F. Brezo and Y. Rubio (i3visio) 2015
This program comes with ABSOLUTELY NO WARRANTY.
This is free software, and you are welcome to redistribute it under certain conditions. For additional info, visit <http://www.gnu.org/licenses/gpl-3.0.txt>."""
        logger.info(sayingHello)
        print sayingHello
        print
        logger.info("Starting usufy.py...")

    if args.license:
        logger.info("Looking for the license...")
        # showing the license
        try:
            with open ("COPYING", "r") as iF:
                contenido = iF.read().splitlines()
                for linea in contenido:
                    print linea
        except Exception:
            try:
                # Trying to recover the COPYING file...
                with open ("/usr/share/osrframework/COPYING", "r") as iF:
                    contenido = iF.read().splitlines()
                    for linea in contenido:
                        print linea
            except:
                logger.error("ERROR: there has been an error when opening the COPYING file.\n\tThe file contains the terms of the GPLv3 under which this software is distributed.\n\tIn case of doubts, verify the integrity of the files or contact [email protected].")
    elif args.fuzz:
        logger.info("Performing the fuzzing tasks...")
        res = fuzzUsufy(args.fuzz, args.fuzz_config)
        logger.info("Recovered platforms:\n" + str(res))
    else:
        logger.debug("Recovering the list of platforms to be processed...")
        # Recovering the list of platforms to be launched
        listPlatforms = platform_selection.getPlatformsByName(platformNames=args.platforms, tags=args.tags, mode="usufy")
        logger.debug("Platforms recovered.")

        if args.info:
            # Information actions...
            if args.info == 'list_platforms':
                infoPlatforms="Listing the platforms:\n"
                for p in listPlatforms:
                    infoPlatforms += "\t\t" + (str(p) + ": ").ljust(16, ' ') + str(p.tags)+"\n"
                logger.info(infoPlatforms)
                return infoPlatforms
            elif args.info == 'list_tags':
                logger.info("Listing the tags:")
                tags = {}
                # Going through all the selected platforms to get their tags
                for p in listPlatforms:
                    for t in p.tags:
                        if t not in tags.keys():
                            tags[t] = 1
                        else:
                            tags[t] += 1
                infoTags = "List of tags:\n"
                # Displaying the results in a sorted list
                for t in tags.keys():
                    infoTags += "\t\t" + (t + ": ").ljust(16, ' ') + str(tags[t]) + "  time(s)\n"
                logger.info(infoTags)
                return infoTags
            else:
                pass

        # performing the test
        elif args.benchmark:
            logger.warning("The benchmark mode may last some minutes as it will be performing similar queries to the ones performed by the program in production. ")
            logger.info("Launching the benchmarking tests...")
            platforms = platform_selection.getAllPlatformNames("usufy")
            res = benchmark.doBenchmark(platforms)
            strTimes = ""
            for e in sorted(res.keys()):
                strTimes += str(e) + "\t" + str(res[e]) + "\n"
            logger.info(strTimes)
            return strTimes
        # Executing the corresponding process...
        else:
            # Showing the execution time...
            if not args.maltego:
                startTime= dt.datetime.now()
                print str(startTime) +"\tStarting search in " + str(len(listPlatforms)) + " platform(s)... Be patient!\n"

            # Defining the list of users to monitor
            nicks = []
            logger.debug("Recovering nicknames to be processed...")
            if args.nicks:
                for n in args.nicks:
                    # TO-DO
                    #     A trick to avoid having the processing of the properties when being queried by Maltego
                    if "properties.i3visio" not in n:
                        nicks.append(n)
            else:
                # Reading the nick files
                try:
                    nicks = args.list.read().splitlines()
                except:
                    logger.error("ERROR: there has been an error when opening the file that stores the nicks.\tPlease, check the existence of this file.")

            if args.output_folder != None:
                # if Verifying an output folder was selected
                logger.debug("Preparing the output folder...")
                if not args.maltego:
                    if not os.path.exists(args.output_folder):
                        logger.warning("The output folder \'" + args.output_folder + "\' does not exist. The system will try to create it.")
                        os.makedirs(args.output_folder)
                # Launching the process...
                res = processNickList(nicks, listPlatforms, args.output_folder, avoidProcessing = args.avoid_processing, avoidDownload = args.avoid_download, nThreads=args.threads, verbosity= args.verbose, logFolder=args.logfolder)
            else:
                res = processNickList(nicks, listPlatforms, nThreads=args.threads, verbosity= args.verbose, logFolder=args.logfolder)


            logger.info("Listing the results obtained...")
            # We are going to iterate over the results...
            strResults = "\t"

            # Structure returned
            """
            [
                {
                  "attributes": [
                    {
                      "attributes": [],
                      "type": "i3visio.uri",
                      "value": "http://twitter.com/i3visio"
                    },
                    {
                      "attributes": [],
                      "type": "i3visio.alias",
                      "value": "i3visio"
                    },
                    {
                      "attributes": [],
                      "type": "i3visio.platform",
                      "value": "Twitter"
                    }
                  ],
                  "type": "i3visio.profile",
                  "value": "Twitter - i3visio"
                }
                ,
                ...
            ]
            """
            for r in res:
                # The format of the results (attributes) for a given nick is a list as follows:

                for att in r["attributes"]:
                    # iterating through the attributes
                    platform = ""
                    uri = ""
                    for details in att["attributes"]:
                        if details["type"] == "i3visio.platform":
                            platform = details["value"]
                        if details["type"] == "i3visio.uri":
                            uri = details["value"]
                    try:
                        strResults+= (str(platform) + ":").ljust(16, ' ')+ " "+ str(uri)+"\n\t\t"
                    except:
                        pass

                logger.info(strResults)

            # Generating summary files for each ...
            if args.extension:
                # Storing the file...
                logger.info("Creating output files as requested.")
                if not args.maltego:
                    # Verifying if the outputPath exists
                    if not os.path.exists (args.output_folder):
                        logger.warning("The output folder \'" + args.output_folder + "\' does not exist. The system will try to create it.")
                        os.makedirs(args.output_folder)

                # Grabbing the results
                fileHeader = os.path.join(args.output_folder, args.file_header)

                # Iterating through the given extensions to print its values
                if not args.maltego:
                    for ext in args.extension:
                        # Generating output files
                        general.exportUsufy(res, ext, fileHeader)

            # Generating the Maltego output
            if args.maltego:
                general.listToMaltego(res)

            # Printing the results if requested
            if not args.maltego:
                print "A summary of the results obtained are shown in the following table:"
                #print res
                print unicode(general.usufyToTextExport(res))

                print

                print "You can find all the information collected in the following files:"
                for ext in args.extension:
                    # Showing the output files
                    print "\t-" + fileHeader + "." + ext

            # Showing the execution time...
            if not args.maltego:
                print
                endTime= dt.datetime.now()
                print str(endTime) +"\tFinishing execution..."
                print
                print "Total time used:\t" + str(endTime-startTime)
                print "Average seconds/query:\t" + str((endTime-startTime).total_seconds()/len(listPlatforms)) +" seconds"
                print

            # Urging users to place an issue on Github...
            if not args.maltego:
                print
                print "Did something go wrong? Is a platform reporting false positives? Do you need to integrate a new one?"
                print "Then, place an issue in the Github project: <https://github.com/i3visio/osrframework/issues>."
                print "Note that otherwise, we won't know about it!"
                print

            return res