def getParser():
import osrframework.utils.configuration as configuration
DEFAULT_VALUES = configuration.returnListOfConfigurationValues("searchfy")
parser = argparse.ArgumentParser(description='searchfy.py - Piece of software that performs a query on the platforms in OSRFramework.', prog='searchfy.py', epilog='Check the README.md file for further details on the usage of this program or follow us on Twitter in <http://twitter.com/i3visio>.', add_help=False)
parser._optionals.title = "Input options (one required)"
# Defining the mutually exclusive group for the main options
groupMain = parser.add_mutually_exclusive_group(required=True)
# Adding the main options
groupMain.add_argument('--license', required=False, action='store_true', default=False, help='shows the GPLv3+ license and exists.')
groupMain.add_argument('-q', '--queries', metavar='<searches>', nargs='+', action='store', help = 'the list of queries to be performed).')
listAll = platform_selection.getAllPlatformNames("searchfy")
# Configuring the processing options
groupProcessing = parser.add_argument_group('Processing arguments', 'Configuring the way in which searchfy will process the identified profiles.')
#groupProcessing.add_argument('-L', '--logfolder', metavar='<path_to_log_folder', required=False, default = './logs', action='store', help='path to the log folder. If none was provided, ./logs is assumed.')
# Getting a sample header for the output files
groupProcessing.add_argument('-e', '--extension', metavar='<sum_ext>', nargs='+', choices=['csv', 'gml', 'json', 'mtz', 'ods', 'png', 'txt', 'xls', 'xlsx' ], required=False, default = DEFAULT_VALUES["extension"], action='store', help='output extension for the summary files. Default: xls.')
groupProcessing.add_argument('-F', '--file_header', metavar='<alternative_header_file>', required=False, default = DEFAULT_VALUES["file_header"], action='store', help='Header for the output filenames to be generated. If None was provided the following will be used: profiles.<extension>' )
groupProcessing.add_argument('-m', '--maltego', required=False, action='store_true', help='Parameter specified to let usufy.py know that he has been launched by a Maltego Transform.')
groupProcessing.add_argument('-o', '--output_folder', metavar='<path_to_output_folder>', required=False, default = DEFAULT_VALUES["output_folder"], action='store', help='output folder for the generated documents. While if the paths does not exist, usufy.py will try to create; if this argument is not provided, usufy will NOT write any down any data. Check permissions if something goes wrong.')
groupProcessing.add_argument('-p', '--platforms', metavar='<platform>', choices=listAll, nargs='+', required=False, default = DEFAULT_VALUES["platforms"] ,action='store', help='select the platforms where you want to perform the search amongst the following: ' + str(listAll) + '. More than one option can be selected.')
groupProcessing.add_argument('--process', required=False, default =False ,action='store_true', help='whether to process the info in the profiles recovered. NOTE: this would be much slower.')
groupProcessing.add_argument('-w', '--web_browser', required=False, action='store_true', help='opening the URIs returned in the default web browser.')
# About options
groupAbout = parser.add_argument_group('About arguments', 'Showing additional information about this program.')
groupAbout.add_argument('-h', '--help', action='help', help='shows this help and exists.')
#groupAbout.add_argument('-v', '--verbose', metavar='<verbosity>', choices=[0, 1, 2], required=False, action='store', default=1, help='select the verbosity level: 0 - none; 1 - normal (default); 2 - debug.', type=int)
groupAbout.add_argument('--version', action='version', version='%(prog)s ' +" " +__version__, help='shows the version of the program and exists.')
return parser
def research(program=None):
"""Prepare research UI for the main tools in the framework.
"""
platOptions = platform_selection.getAllPlatformNames(program)
if not program:
return render_template('research.html',
mt_research='class=current',
mr_main='class=current')
else:
params = request.args.get('query_text')
if params != None:
# We perform an additional check to see if we are receiving an array
if params[0] == "[":
# TODO: make it more flexible. This is a workaround to process whois results
params = params.replace("u'", "")
params = params.replace("'", "")
params = params.replace(",", "")
params = params.replace("[", "")
params = params.replace("]", "")
return render_template('research-' + program + '.html',
mt_research='class=current',
query_text=params,
plat_options=platOptions)
else:
return render_template('research-' + program + '.html',
mt_research='class=current',
plat_options=platOptions)
def getParser():
parser = argparse.ArgumentParser(description='searchfy.py - Piece of software that performs a query on the platforms in OSRFramework.', prog='searchfy.py', epilog='Check the README.md file for further details on the usage of this program or follow us on Twitter in <http://twitter.com/i3visio>.', add_help=False)
parser._optionals.title = "Input options (one required)"
# Defining the mutually exclusive group for the main options
groupMain = parser.add_mutually_exclusive_group(required=True)
# Adding the main options
groupMain.add_argument('--license', required=False, action='store_true', default=False, help='shows the GPLv3+ license and exists.')
groupMain.add_argument('-q', '--queries', metavar='<searches>', nargs='+', action='store', help = 'the list of queries to be performed).')
listAll = platform_selection.getAllPlatformNames("searchfy")
# Configuring the processing options
groupProcessing = parser.add_argument_group('Processing arguments', 'Configuring the way in which searchfy will process the identified profiles.')
#groupProcessing.add_argument('-L', '--logfolder', metavar='<path_to_log_folder', required=False, default = './logs', action='store', help='path to the log folder. If none was provided, ./logs is assumed.')
# Getting a sample header for the output files
groupProcessing.add_argument('-e', '--extension', metavar='<sum_ext>', nargs='+', choices=['csv', 'gml', 'json', 'mtz', 'ods', 'png', 'txt', 'xls', 'xlsx' ], required=False, default = ['xls'], action='store', help='output extension for the summary files. Default: xls.')
groupProcessing.add_argument('-F', '--file_header', metavar='<alternative_header_file>', required=False, default = "profiles", action='store', help='Header for the output filenames to be generated. If None was provided the following will be used: profiles.<extension>' )
groupProcessing.add_argument('-m', '--maltego', required=False, action='store_true', help='Parameter specified to let usufy.py know that he has been launched by a Maltego Transform.')
groupProcessing.add_argument('-o', '--output_folder', metavar='<path_to_output_folder>', required=False, default = './results', action='store', help='output folder for the generated documents. While if the paths does not exist, usufy.py will try to create; if this argument is not provided, usufy will NOT write any down any data. Check permissions if something goes wrong.')
groupProcessing.add_argument('-p', '--platforms', metavar='<platform>', choices=listAll, nargs='+', required=False, default =['all'] ,action='store', help='select the platforms where you want to perform the search amongst the following: ' + str(listAll) + '. More than one option can be selected.')
groupProcessing.add_argument('--process', required=False, default =False ,action='store_true', help='whether to process the info in the profiles recovered. NOTE: this would be much slower.')
# About options
groupAbout = parser.add_argument_group('About arguments', 'Showing additional information about this program.')
groupAbout.add_argument('-h', '--help', action='help', help='shows this help and exists.')
#groupAbout.add_argument('-v', '--verbose', metavar='<verbosity>', choices=[0, 1, 2], required=False, action='store', default=1, help='select the verbosity level: 0 - none; 1 - normal (default); 2 - debug.', type=int)
groupAbout.add_argument('--version', action='version', version='%(prog)s ' +" " +__version__, help='shows the version of the program and exists.')
return parser
def getParser():
DEFAULT_VALUES = configuration.returnListOfConfigurationValues("searchfy")
# Capturing errors just in case the option is not found in the configuration
try:
excludeList = [DEFAULT_VALUES["exclude_platforms"]]
except:
excludeList = []
parser = argparse.ArgumentParser(description='searchfy.py - Piece of software that performs a query on the platforms in OSRFramework.', prog='searchfy.py', epilog='Check the README.md file for further details on the usage of this program or follow us on Twitter in <http://twitter.com/i3visio>.', add_help=False)
parser._optionals.title = "Input options (one required)"
# Adding the main options
groupMain = parser.add_mutually_exclusive_group(required=True)
groupMain.add_argument('--license', required=False, action='store_true', default=False, help='shows the GPLv3+ license and exists.')
groupMain.add_argument('-q', '--queries', metavar='<searches>', nargs='+', action='store', help = 'the list of queries to be performed).')
listAll = platform_selection.getAllPlatformNames("searchfy")
# Configuring the processing options
groupProcessing = parser.add_argument_group('Processing arguments', 'Configuring the way in which searchfy will process the identified profiles.')
groupProcessing.add_argument('-e', '--extension', metavar='<sum_ext>', nargs='+', choices=['csv', 'gml', 'json', 'ods', 'png', 'txt', 'xls', 'xlsx' ], required=False, default=DEFAULT_VALUES["extension"], action='store', help='output extension for the summary files. Default: xls.')
groupProcessing.add_argument('-F', '--file_header', metavar='<alternative_header_file>', required=False, default=DEFAULT_VALUES["file_header"], action='store', help='Header for the output filenames to be generated. If None was provided the following will be used: profiles.<extension>' )
groupProcessing.add_argument('-o', '--output_folder', metavar='<path_to_output_folder>', required=False, default=DEFAULT_VALUES["output_folder"], action='store', help='output folder for the generated documents. While if the paths does not exist, usufy.py will try to create; if this argument is not provided, usufy will NOT write any down any data. Check permissions if something goes wrong.')
groupProcessing.add_argument('-p', '--platforms', metavar='<platform>', choices=listAll, nargs='+', required=False, default=DEFAULT_VALUES["platforms"] ,action='store', help='select the platforms where you want to perform the search amongst the following: ' + str(listAll) + '. More than one option can be selected.')
groupProcessing.add_argument('--process', required=False, default =False ,action='store_true', help='whether to process the info in the profiles recovered. NOTE: this would be much slower.')
groupProcessing.add_argument('-w', '--web_browser', required=False, action='store_true', help='opening the URIs returned in the default web browser.')
groupProcessing.add_argument('-x', '--exclude', metavar='<platform>', choices=listAll, nargs='+', required=False, default=excludeList, action='store', help='select the platforms that you want to exclude from the processing.')
# About options
groupAbout = parser.add_argument_group('About arguments', 'Showing additional information about this program.')
groupAbout.add_argument('-h', '--help', action='help', help='shows this help and exists.')
groupAbout.add_argument('--version', action='version', version='[%(prog)s] OSRFramework ' + osrframework.__version__, help='shows the version of the program and exists.')
return parser
def processNickList(nicks, platforms=None, rutaDescarga="./", avoidProcessing=True, avoidDownload=True, nThreads=12, maltego=False, verbosity=1, logFolder="./logs"):
'''
Method that receives as a parameter a series of nicks and verifies whether those nicks have a profile associated in different social networks.
List of parameters that the method receives:
:param nicks: list of nicks to process.
:param platforms: list of <Platform> objects to be processed.
:param rutaDescarga: local file where saving the obtained information.
:param avoidProcessing: boolean var that defines whether the profiles will NOT be processed.
:param avoidDownload: boolean var that defines whether the profiles will NOT be downloaded (stored in this version).
:param maltego: parameter to tell usufy.py that he has been invoked by Malego.
:param verbosity: the level of verbosity to be used.
:param logFolder: the path to the log folder.
:return:
Returns a dictionary where the key is the nick and the value another dictionary where the keys are the social networks and te value is the corresponding URL.
'''
osrframework.utils.logger.setupLogger(loggerName="osrframework.usufy", verbosity=verbosity, logFolder=logFolder)
logger = logging.getLogger("osrframework.usufy")
if platforms == None:
platforms = platform_selection.getAllPlatformNames("usufy")
# Defining the output results variable
res = []
# Processing the whole list of terms...
for nick in nicks:
logger.info("Looking for '" + nick + "' in " + str(len(platforms)) + " different platforms:\n" +str( [ str(plat) for plat in platforms ] ) )
# If the process is executed by the current app, we use the Processes. It is faster than pools.
if nThreads <= 0 or nThreads > len(platforms):
nThreads = len(platforms)
# Using threads in a pool if we are not running the program in main
args = []
# We need to create all the arguments that will be needed
#print platforms
for plat in platforms:
args.append (( plat, nick, rutaDescarga, avoidProcessing, avoidDownload))
logger.info("Launching " + str(nThreads) + " different threads...")
# We define the pool
pool = Pool(nThreads)
#print json.dumps(args, indent=2)
# We call the wrapping function with all the args previously generated
#poolResults = pool.apply_async(multi_run_wrapper,(args))
poolResults = pool.map(multi_run_wrapper,args)
pool.close()
profiles = []
for r in poolResults:
# We need to recover the results and check if they are not an empty json
if r != "{}":
profiles.append(json.loads(r))
res+=profiles
#res = profiles
return res
def getParser():
import osrframework.utils.configuration as configuration
DEFAULT_VALUES = configuration.returnListOfConfigurationValues("usufy")
# Capturing errors just in case the option is not found in the configuration
try:
excludeList = [DEFAULT_VALUES["exclude_platforms"]]
except:
excludeList = []
# Recovering all the possible options
platOptions = platform_selection.getAllPlatformNames("usufy")
parser = argparse.ArgumentParser(description='usufy.py - Piece of software that checks the existence of a profile for a given user in up to ' + str(len(platOptions)-1)+ ' different platforms.', prog='usufy.py', epilog='Check the README.md file for further details on the usage of this program or follow us on Twitter in <http://twitter.com/i3visio>.', add_help=False)
parser._optionals.title = "Input options (one required)"
# Defining the mutually exclusive group for the main options
groupMainOptions = parser.add_mutually_exclusive_group(required=True)
# Adding the main options
groupMainOptions.add_argument('--info', metavar='<action>', choices=['list_platforms', 'list_tags'], action='store', help='select the action to be performed amongst the following: list_platforms (list the details of the selected platforms), list_tags (list the tags of the selected platforms). Afterwards, it exists.')
groupMainOptions.add_argument('--license', required=False, action='store_true', default=False, help='shows the GPLv3+ license and exists.')
groupMainOptions.add_argument('-b', '--benchmark', action='store_true', default=False, help='perform the benchmarking tasks.')
groupMainOptions.add_argument('-f', '--fuzz', metavar='<path_to_fuzzing_list>', action='store', type=argparse.FileType('r'), help='this option will try to find usufy-like URLs. The list of fuzzing platforms in the file should be (one per line): <BASE_DOMAIN>\t<VALID_NICK>')
groupMainOptions.add_argument('-l', '--list', metavar='<path_to_nick_list>', action='store', type=argparse.FileType('r'), help='path to the file where the list of nicks to verify is stored (one per line).')
groupMainOptions.add_argument('-n', '--nicks', metavar='<nick>', nargs='+', action='store', help = 'the list of nicks to process (at least one is required).')
groupMainOptions.add_argument('--show_tags', action='store_true', default=False, help='it will show the platforms grouped by tags.')
# Selecting the platforms where performing the search
groupPlatforms = parser.add_argument_group('Platform selection arguments', 'Criteria for selecting the platforms where performing the search.')
groupPlatforms.add_argument('-p', '--platforms', metavar='<platform>', choices=platOptions, nargs='+', required=False, default=DEFAULT_VALUES["platforms"], action='store', help='select the platforms where you want to perform the search amongst the following: ' + str(platOptions) + '. More than one option can be selected.')
groupPlatforms.add_argument('-t', '--tags', metavar='<tag>', default = [], nargs='+', required=False, action='store', help='select the list of tags that fit the platforms in which you want to perform the search. More than one option can be selected.')
groupPlatforms.add_argument('-x', '--exclude', metavar='<platform>', choices=platOptions, nargs='+', required=False, default=excludeList, action='store', help='select the platforms that you want to exclude from the processing.')
# Configuring the processing options
groupProcessing = parser.add_argument_group('Processing arguments', 'Configuring the way in which usufy will process the identified profiles.')
groupProcessing.add_argument('--avoid_download', required=False, action='store_true', default=False, help='argument to force usufy NOT to store the downloadable version of the profiles.')
groupProcessing.add_argument('--avoid_processing', required=False, action='store_true', default=False, help='argument to force usufy NOT to perform any processing task with the valid profiles.')
groupProcessing.add_argument('--fuzz_config', metavar='<path_to_fuzz_list>', action='store', type=argparse.FileType('r'), help='path to the fuzzing config details. Wildcards such as the domains or the nicknames should come as: <DOMAIN>, <USERNAME>.')
groupProcessing.add_argument('--nonvalid', metavar='<not_valid_characters>', required=False, default = '\\|<>=', action='store', help="string containing the characters considered as not valid for nicknames." )
groupProcessing.add_argument('-e', '--extension', metavar='<sum_ext>', nargs='+', choices=['csv', 'gml', 'json', 'mtz', 'ods', 'png', 'txt', 'xls', 'xlsx' ], required=False, default=DEFAULT_VALUES["extension"], action='store', help='output extension for the summary files. Default: xls.')
groupProcessing.add_argument('-L', '--logfolder', metavar='<path_to_log_folder', required=False, default = './logs', action='store', help='path to the log folder. If none was provided, ./logs is assumed.')
groupProcessing.add_argument('-m', '--maltego', required=False, action='store_true', help='parameter specified to let usufy.py know that he has been launched by a Maltego Transform.')
groupProcessing.add_argument('-o', '--output_folder', metavar='<path_to_output_folder>', required=False, default=DEFAULT_VALUES["output_folder"], action='store', help='output folder for the generated documents. While if the paths does not exist, usufy.py will try to create; if this argument is not provided, usufy will NOT write any down any data. Check permissions if something goes wrong.')
groupProcessing.add_argument('-w', '--web_browser', required=False, action='store_true', help='opening the uris returned in the default web browser.')
# Getting a sample header for the output files
groupProcessing.add_argument('-F', '--file_header', metavar='<alternative_header_file>', required=False, default=DEFAULT_VALUES["file_header"], action='store', help='Header for the output filenames to be generated. If None was provided the following will be used: profiles.<extension>.' )
groupProcessing.add_argument('-T', '--threads', metavar='<num_threads>', required=False, action='store', default=int(DEFAULT_VALUES["threads"]), type=int, help='write down the number of threads to be used (default 32). If 0, the maximum number possible will be used, which may make the system feel unstable.')
# About options
groupAbout = parser.add_argument_group('About arguments', 'Showing additional information about this program.')
groupAbout.add_argument('-h', '--help', action='help', help='shows this help and exists.')
groupAbout.add_argument('-v', '--verbose', metavar='<verbosity>', choices=[0, 1, 2], required=False, action='store', default=1, help='select the verbosity level: 0 - none; 1 - normal (default); 2 - debug.', type=int)
groupAbout.add_argument('--version', action='version', version='%(prog)s ' +__version__, help='shows the version of the program and exists.')
return parser
def getParser():
# Recovering all the possible options
platOptions = platform_selection.getAllPlatformNames("usufy")
parser = argparse.ArgumentParser(description='usufy.py - Piece of software that checks the existence of a profile for a given user in up to ' + str(len(platOptions)-1)+ ' different platforms.', prog='usufy.py', epilog='Check the README.md file for further details on the usage of this program or follow us on Twitter in <http://twitter.com/i3visio>.', add_help=False)
parser._optionals.title = "Input options (one required)"
# Defining the mutually exclusive group for the main options
groupMainOptions = parser.add_mutually_exclusive_group(required=True)
# Adding the main options
groupMainOptions.add_argument('--info', metavar='<action>', choices=['list_platforms', 'list_tags'], action='store', help='select the action to be performed amongst the following: list_platforms (list the details of the selected platforms), list_tags (list the tags of the selected platforms). Afterwards, it exists.')
groupMainOptions.add_argument('--license', required=False, action='store_true', default=False, help='shows the GPLv3+ license and exists.')
groupMainOptions.add_argument('-b', '--benchmark', action='store_true', default=False, help='perform the benchmarking tasks.')
groupMainOptions.add_argument('-f', '--fuzz', metavar='<path_to_fuzzing_list>', action='store', type=argparse.FileType('r'), help='this option will try to find usufy-like URLs. The list of fuzzing platforms in the file should be (one per line): <BASE_DOMAIN>\t<VALID_NICK>')
groupMainOptions.add_argument('-l', '--list', metavar='<path_to_nick_list>', action='store', type=argparse.FileType('r'), help='path to the file where the list of nicks to verify is stored (one per line).')
groupMainOptions.add_argument('-n', '--nicks', metavar='<nick>', nargs='+', action='store', help = 'the list of nicks to process (at least one is required).')
# Selecting the platforms where performing the search
groupPlatforms = parser.add_argument_group('Platform selection arguments', 'Criteria for selecting the platforms where performing the search.')
groupPlatforms.add_argument('-p', '--platforms', metavar='<platform>', choices=platOptions, nargs='+', required=False, default =['all'] ,action='store', help='select the platforms where you want to perform the search amongst the following: ' + str(platOptions) + '. More than one option can be selected.')
groupPlatforms.add_argument('-t', '--tags', metavar='<tag>', default = [], nargs='+', required=False, action='store', help='select the list of tags that fit the platforms in which you want to perform the search. More than one option can be selected.')
# Configuring the processing options
groupProcessing = parser.add_argument_group('Processing arguments', 'Configuring the way in which usufy will process the identified profiles.')
groupProcessing.add_argument('--avoid_download', required=False, action='store_true', default=False, help='argument to force usufy NOT to store the downloadable version of the profiles.')
groupProcessing.add_argument('--avoid_processing', required=False, action='store_true', default=False, help='argument to force usufy NOT to perform any processing task with the valid profiles.')
groupProcessing.add_argument('--fuzz_config', metavar='<path_to_fuzz_list>', action='store', type=argparse.FileType('r'), help='path to the fuzzing config details. Wildcards such as the domains or the nicknames should come as: <DOMAIN>, <USERNAME>.')
groupProcessing.add_argument('--nonvalid', metavar='<not_valid_characters>', required=False, default = '\\|<>=', action='store', help="string containing the characters considered as not valid for nicknames." )
groupProcessing.add_argument('-c', '--credentials', metavar='<path_to_credentials_file', required=False, default = './creds.txt', action='store', help='path to the credentials file. If none was provided, ./creds.txt is assumed.')
groupProcessing.add_argument('-e', '--extension', metavar='<sum_ext>', nargs='+', choices=['csv', 'gml', 'json', 'mtz', 'ods', 'png', 'txt', 'xls', 'xlsx' ], required=False, default = ['xls'], action='store', help='output extension for the summary files. Default: xls.')
groupProcessing.add_argument('-L', '--logfolder', metavar='<path_to_log_folder', required=False, default = './logs', action='store', help='path to the log folder. If none was provided, ./logs is assumed.')
groupProcessing.add_argument('-m', '--maltego', required=False, action='store_true', help='Parameter specified to let usufy.py know that he has been launched by a Maltego Transform.')
groupProcessing.add_argument('-o', '--output_folder', metavar='<path_to_output_folder>', required=False, default = './results', action='store', help='output folder for the generated documents. While if the paths does not exist, usufy.py will try to create; if this argument is not provided, usufy will NOT write any down any data. Check permissions if something goes wrong.')
# Getting a sample header for the output files
groupProcessing.add_argument('-F', '--file_header', metavar='<alternative_header_file>', required=False, default = "profiles", action='store', help='Header for the output filenames to be generated. If None was provided the following will be used: profiles.<extension>.' )
# [TO-DO]: to be revisited
groupProcessing.add_argument('-s', '--squatting', metavar='<level>', nargs='+', choices=['basic', 'l33t', 'local', 'years', 'words', 'all'], required=False, default=[], action='store', help="select the level of profilesquatting to be looked for (in order of execution): words (for adding sensitive words such as 'real', 'home', 'news', etc.); l33t (l33t m0d3); years (ending in numbers); local (looking for localized endings: '_es', '_en', '_fr', etc.); basic (changing '-', '.' and ' '); and all.")
groupProcessing.add_argument('-T', '--threads', metavar='<num_threads>', required=False, action='store', default=32, type=int, help='write down the number of threads to be used (default 32). If 0, the maximum number possible will be used, which may make the system feel unstable.')
# About options
groupAbout = parser.add_argument_group('About arguments', 'Showing additional information about this program.')
groupAbout.add_argument('-h', '--help', action='help', help='shows this help and exists.')
groupAbout.add_argument('-v', '--verbose', metavar='<verbosity>', choices=[0, 1, 2], required=False, action='store', default=1, help='select the verbosity level: 0 - none; 1 - normal (default); 2 - debug.', type=int)
groupAbout.add_argument('--version', action='version', version='%(prog)s ' +__version__, help='shows the version of the program and exists.')
return parser
def processNickList(nicks,
platforms=None,
rutaDescarga="./",
avoidProcessing=True,
avoidDownload=True,
nThreads=12,
maltego=False,
verbosity=1,
logFolder="./logs"):
'''
Method that receives as a parameter a series of nicks and verifies whether those nicks have a profile associated in different social networks.
List of parameters that the method receives:
:param nicks: list of nicks to process.
:param platforms: list of <Platform> objects to be processed.
:param rutaDescarga: local file where saving the obtained information.
:param avoidProcessing: boolean var that defines whether the profiles will NOT be processed.
:param avoidDownload: boolean var that defines whether the profiles will NOT be downloaded (stored in this version).
:param maltego: parameter to tell usufy.py that he has been invoked by Malego.
:param verbosity: the level of verbosity to be used.
:param logFolder: the path to the log folder.
:return:
Returns a dictionary where the key is the nick and the value another dictionary where the keys are the social networks and te value is the corresponding URL.
'''
osrframework.utils.logger.setupLogger(loggerName="osrframework.usufy",
verbosity=verbosity,
logFolder=logFolder)
logger = logging.getLogger("osrframework.usufy")
if platforms == None:
platforms = platform_selection.getAllPlatformNames("usufy")
# Defining the output results variable
res = []
# Processing the whole list of terms...
for nick in nicks:
logger.info("Looking for '" + nick + "' in " + str(len(platforms)) +
" different platforms:\n" +
str([str(plat) for plat in platforms]))
# Using threads in a pool if we are not running the program in main
args = []
# We need to create all the arguments that will be needed
#print platforms
for plat in platforms:
args.append(
(plat, nick, rutaDescarga, avoidProcessing, avoidDownload))
# If the process is executed by the current app, we use the Processes. It is faster than pools.
if nThreads <= 0 or nThreads > len(platforms):
nThreads = len(platforms)
logger.info("Launching " + str(nThreads) + " different threads...")
# We define the pool
pool = Pool(nThreads)
# We call the wrapping function with all the args previously generated
#poolResults = pool.apply_async(multi_run_wrapper,(args))
poolResults = pool.map(multi_run_wrapper, args)
pool.close()
profiles = []
for serArray in poolResults:
# We need to recover the results and check if they are not an empty json
array = json.loads(serArray)
for r in array:
if r != "{}":
profiles.append(r)
res += profiles
#res = profiles
return res
def usufy_main(args):
'''
Main function. This function is created in this way so as to let other applications make use of the full configuration capabilities of the application.
'''
# Recovering the logger
# Calling the logger when being imported
osrframework.utils.logger.setupLogger(loggerName="osrframework.usufy", verbosity=args.verbose, logFolder=args.logfolder)
# From now on, the logger can be recovered like this:
logger = logging.getLogger("osrframework.usufy")
logger.info("""usufy.py Copyright (C) F. Brezo and Y. Rubio (i3visio) 2015
This program comes with ABSOLUTELY NO WARRANTY.
This is free software, and you are welcome to redistribute it under certain conditions.
For details, run:
\tpython usufy.py --license""")
logger.info("Starting usufy.py...")
if args.license:
logger.info("Looking for the license...")
# showing the license
try:
with open ("COPYING", "r") as iF:
contenido = iF.read().splitlines()
for linea in contenido:
print linea
except Exception:
logger.error("ERROR: there has been an error when opening the COPYING file.\n\tThe file contains the terms of the GPLv3 under which this software is distributed.\n\tIn case of doubts, verify the integrity of the files or contact [email protected].")
elif args.fuzz:
logger.info("Performing the fuzzing tasks...")
res = fuzzUsufy(args.fuzz, args.fuzz_config)
logger.info("Recovered platforms:\n" + str(res))
else:
logger.debug("Recovering the list of platforms to be processed...")
# Recovering the list of platforms to be launched
listPlatforms = platform_selection.getPlatformsByName(platformNames=args.platforms, tags=args.tags, mode="usufy")
logger.debug("Platforms recovered.")
if args.info:
# Information actions...
if args.info == 'list_platforms':
infoPlatforms="Listing the platforms:\n"
for p in listPlatforms:
infoPlatforms += "\t\t" + (str(p) + ": ").ljust(16, ' ') + str(p.tags)+"\n"
logger.info(infoPlatforms)
return infoPlatforms
elif args.info == 'list_tags':
logger.info("Listing the tags:")
tags = {}
# Going through all the selected platforms to get their tags
for p in listPlatforms:
for t in p.tags:
if t not in tags.keys():
tags[t] = 1
else:
tags[t] += 1
infoTags = "List of tags:\n"
# Displaying the results in a sorted list
for t in tags.keys():
infoTags += "\t\t" + (t + ": ").ljust(16, ' ') + str(tags[t]) + " time(s)\n"
logger.info(infoTags)
return infoTags
else:
pass
# performing the test
elif args.benchmark:
logger.warning("The benchmark mode may last some minutes as it will be performing similar queries to the ones performed by the program in production. ")
logger.info("Launching the benchmarking tests...")
platforms = platform_selection.getAllPlatformNames("usufy")
res = benchmark.doBenchmark(platforms)
strTimes = ""
for e in sorted(res.keys()):
strTimes += str(e) + "\t" + str(res[e]) + "\n"
logger.info(strTimes)
return strTimes
# Executing the corresponding process...
else:
# Defining the list of users to monitor
nicks = []
logger.debug("Recovering nicknames to be processed...")
if args.nicks:
for n in args.nicks:
# TO-DO
# A trick to avoid having the processing of the properties when being queried by Maltego
if "properties.i3visio" not in n:
nicks.append(n)
else:
# Reading the nick files
try:
nicks = args.list.read().splitlines()
except:
logger.error("ERROR: there has been an error when opening the file that stores the nicks.\tPlease, check the existence of this file.")
# Checking if performing any transform was required
if args.squatting:
logger.debug("Making basic transformations on the provided nicknames...")
# Iterating to process _ and or .
#nicks = profilesquatting.generatingProfiles(nicks, args.profilesquatting)
nicks = profilesquatting.getNewNicks(nicks, logName = "usufy", modes = args.squatting, nonValidChars = args.nonvalid)
logger.info("Obtained nicks:\n" + str(nicks))
logger.debug("Profilesquatting nicknames recovered.")
if args.info == 'list_users':
strNicks = ""
for n in nicks:
strNicks += n + "\n"
logger.info("Generated nicks:\n----------------\n" + strNicks)
# Storing the file...
logger.info("Creating output files as requested.")
# Verifying if the outputPath exists
if not os.path.exists (args.output_folder):
logger.warning("The output folder \'" + args.output_folder + "\' does not exist. The system will try to create it.")
os.makedirs(args.output_folder)
strTime = general.getCurrentStrDatetime()
logger.info("Writing generated nicks to a text file.")
with open (os.path.join(args.output_folder, "nicks_" + strTime +".txt"), "w") as oF:
oF.write( strNicks )
# if this option was selected, we will jsut return this and exist
return nicks
if args.output_folder != None:
# if Verifying an output folder was selected
logger.debug("Preparing the output folder...")
if not os.path.exists(args.output_folder):
logger.warning("The output folder \'" + args.output_folder + "\' does not exist. The system will try to create it.")
os.makedirs(args.output_folder)
# Launching the process...
res = processNickList(nicks, listPlatforms, args.output_folder, avoidProcessing = args.avoid_processing, avoidDownload = args.avoid_download, nThreads=args.threads, verbosity= args.verbose, logFolder=args.logfolder)
else:
res = processNickList(nicks, listPlatforms, nThreads=args.threads, verbosity= args.verbose, logFolder=args.logfolder)
logger.info("Listing the results obtained...")
# We are going to iterate over the results...
strResults = "\t"
# Structure returned
"""
[
{
"attributes": [
{
"attributes": [],
"type": "i3visio.uri",
"value": "http://twitter.com/i3visio"
},
{
"attributes": [],
"type": "i3visio.alias",
"value": "i3visio"
},
{
"attributes": [],
"type": "i3visio.platform",
"value": "Twitter"
}
],
"type": "i3visio.profile",
"value": "Twitter - i3visio"
}
,
...
]
"""
for r in res:
# The format of the results (attributes) for a given nick is a list as follows:
for att in r["attributes"]:
# iterating through the attributes
platform = ""
uri = ""
for details in att["attributes"]:
if details["type"] == "i3visio.platform":
platform = details["value"]
if details["type"] == "i3visio.uri":
uri = details["value"]
try:
strResults+= (str(platform) + ":").ljust(16, ' ')+ " "+ str(uri)+"\n\t\t"
except:
pass
logger.info(strResults)
# Generating summary files for each ...
if args.extension:
# Storing the file...
logger.info("Creating output files as requested.")
if not args.output_folder:
args.output_folder = "./"
else:
# Verifying if the outputPath exists
if not os.path.exists (args.output_folder):
logger.warning("The output folder \'" + args.output_folder + "\' does not exist. The system will try to create it.")
os.makedirs(args.output_folder)
# Grabbing the results
fileHeader = os.path.join(args.output_folder, args.file_header + general.getCurrentStrDatetime())
# Iterating through the given extensions to print its values
for ext in args.extension:
# Generating output files
general.exportUsufy(res, ext, fileHeader)
# Generating the Maltego output
if args.maltego:
general.listToMaltego(res)
# Printing the results if requested
if not args.maltego:
print "A summary of the results obtained are the following table:"
print general.usufyToTextExport(res)
print "You can find all the information collected in the following files:"
for ext in args.extension:
# Generating output files
print "\t-" + fileHeader + "." + ext
return res
def getParser():
# Recovering all the possible options
platOptions = platform_selection.getAllPlatformNames("usufy")
parser = argparse.ArgumentParser(
description="usufy.py - Piece of software that checks the existence of a profile for a given user in up to "
+ str(len(platOptions) - 1)
+ " different platforms.",
prog="usufy.py",
epilog="Check the README.md file for further details on the usage of this program or follow us on Twitter in <http://twitter.com/i3visio>.",
add_help=False,
)
parser._optionals.title = "Input options (one required)"
# Defining the mutually exclusive group for the main options
groupMainOptions = parser.add_mutually_exclusive_group(required=True)
# Adding the main options
groupMainOptions.add_argument(
"--info",
metavar="<action>",
choices=["list_platforms", "list_tags"],
action="store",
help="select the action to be performed amongst the following: list_platforms (list the details of the selected platforms), list_tags (list the tags of the selected platforms). Afterwards, it exists.",
)
groupMainOptions.add_argument(
"--license", required=False, action="store_true", default=False, help="shows the GPLv3+ license and exists."
)
groupMainOptions.add_argument(
"-b", "--benchmark", action="store_true", default=False, help="perform the benchmarking tasks."
)
groupMainOptions.add_argument(
"-f",
"--fuzz",
metavar="<path_to_fuzzing_list>",
action="store",
type=argparse.FileType("r"),
help="this option will try to find usufy-like URLs. The list of fuzzing platforms in the file should be (one per line): <BASE_DOMAIN>\t<VALID_NICK>",
)
groupMainOptions.add_argument(
"-l",
"--list",
metavar="<path_to_nick_list>",
action="store",
type=argparse.FileType("r"),
help="path to the file where the list of nicks to verify is stored (one per line).",
)
groupMainOptions.add_argument(
"-n",
"--nicks",
metavar="<nick>",
nargs="+",
action="store",
help="the list of nicks to process (at least one is required).",
)
# Selecting the platforms where performing the search
groupPlatforms = parser.add_argument_group(
"Platform selection arguments", "Criteria for selecting the platforms where performing the search."
)
groupPlatforms.add_argument(
"-p",
"--platforms",
metavar="<platform>",
choices=platOptions,
nargs="+",
required=False,
default=["all"],
action="store",
help="select the platforms where you want to perform the search amongst the following: "
+ str(platOptions)
+ ". More than one option can be selected.",
)
groupPlatforms.add_argument(
"-t",
"--tags",
metavar="<tag>",
default=[],
nargs="+",
required=False,
action="store",
help="select the list of tags that fit the platforms in which you want to perform the search. More than one option can be selected.",
)
# Configuring the processing options
groupProcessing = parser.add_argument_group(
"Processing arguments", "Configuring the way in which usufy will process the identified profiles."
)
groupProcessing.add_argument(
"--avoid_download",
required=False,
action="store_true",
default=False,
help="argument to force usufy NOT to store the downloadable version of the profiles.",
)
groupProcessing.add_argument(
"--avoid_processing",
required=False,
action="store_true",
default=False,
help="argument to force usufy NOT to perform any processing task with the valid profiles.",
)
groupProcessing.add_argument(
"--fuzz_config",
metavar="<path_to_fuzz_list>",
action="store",
type=argparse.FileType("r"),
help="path to the fuzzing config details. Wildcards such as the domains or the nicknames should come as: <DOMAIN>, <USERNAME>.",
)
groupProcessing.add_argument(
"--nonvalid",
metavar="<not_valid_characters>",
required=False,
default="\\|<>=",
action="store",
help="string containing the characters considered as not valid for nicknames.",
)
groupProcessing.add_argument(
"-c",
"--credentials",
metavar="<path_to_credentials_file",
required=False,
default="./creds.txt",
action="store",
help="path to the credentials file. If none was provided, ./creds.txt is assumed.",
)
groupProcessing.add_argument(
"-e",
"--extension",
metavar="<sum_ext>",
nargs="+",
choices=["csv", "gml", "json", "mtz", "ods", "png", "txt", "xls", "xlsx"],
required=False,
default=["csv"],
action="store",
help="output extension for the summary files. Default: xls.",
)
groupProcessing.add_argument(
"-L",
"--logfolder",
metavar="<path_to_log_folder",
required=False,
default="./logs",
action="store",
help="path to the log folder. If none was provided, ./logs is assumed.",
)
groupProcessing.add_argument(
"-m",
"--maltego",
required=False,
action="store_true",
help="Parameter specified to let usufy.py know that he has been launched by a Maltego Transform.",
)
groupProcessing.add_argument(
"-o",
"--output_folder",
metavar="<path_to_output_folder>",
required=False,
default="./results",
action="store",
help="output folder for the generated documents. While if the paths does not exist, usufy.py will try to create; if this argument is not provided, usufy will NOT write any down any data. Check permissions if something goes wrong.",
)
# Getting a sample header for the output files
groupProcessing.add_argument(
"-F",
"--file_header",
metavar="<alternative_header_file>",
required=False,
default="profiles",
action="store",
help="Header for the output filenames to be generated. If None was provided the following will be used: profiles.<extension>.",
)
# [TO-DO]: to be revisited
groupProcessing.add_argument(
"-s",
"--squatting",
metavar="<level>",
nargs="+",
choices=["basic", "l33t", "local", "years", "words", "all"],
required=False,
default=[],
action="store",
help="select the level of profilesquatting to be looked for (in order of execution): words (for adding sensitive words such as 'real', 'home', 'news', etc.); l33t (l33t m0d3); years (ending in numbers); local (looking for localized endings: '_es', '_en', '_fr', etc.); basic (changing '-', '.' and ' '); and all.",
)
groupProcessing.add_argument(
"-T",
"--threads",
metavar="<num_threads>",
required=False,
action="store",
default=32,
type=int,
help="write down the number of threads to be used (default 32). If 0, the maximum number possible will be used, which may make the system feel unstable.",
)
# About options
groupAbout = parser.add_argument_group("About arguments", "Showing additional information about this program.")
groupAbout.add_argument("-h", "--help", action="help", help="shows this help and exists.")
groupAbout.add_argument(
"-v",
"--verbose",
metavar="<verbosity>",
choices=[0, 1, 2],
required=False,
action="store",
default=1,
help="select the verbosity level: 0 - none; 1 - normal (default); 2 - debug.",
type=int,
)
groupAbout.add_argument(
"--version",
action="version",
version="%(prog)s " + __version__,
help="shows the version of the program and exists.",
)
return parser
general.listToMaltego(res)
# Printing the results if requested
if not args.maltego:
print "A summary of the results obtained are the following table:"
print general.usufyToTextExport(res)
print "You can find all the information collected in the following files:"
for ext in args.extension:
# Generating output files
print "\t-" + fileHeader + "." + ext
return res
if __name__ == "__main__":
# Recovering all the possible options
platOptions = platform_selection.getAllPlatformNames("usufy")
parser = argparse.ArgumentParser(description='usufy.py - Piece of software that checks the existence of a profile for a given user in up to ' + str(len(platOptions)-1)+ ' different platforms.', prog='usufy.py', epilog='Check the README.md file for further details on the usage of this program.', add_help=False)
parser._optionals.title = "Input options (one required)"
# Defining the mutually exclusive group for the main options
groupMainOptions = parser.add_mutually_exclusive_group(required=True)
# Adding the main options
groupMainOptions.add_argument('--info', metavar='<action>', choices=['list_platforms', 'list_tags'], action='store', help='select the action to be performed amongst the following: list_platforms (list the details of the selected platforms), list_tags (list the tags of the selected platforms). Afterwards, it exists.')
groupMainOptions.add_argument('--license', required=False, action='store_true', default=False, help='shows the GPLv3+ license and exists.')
groupMainOptions.add_argument('-b', '--benchmark', action='store_true', default=False, help='perform the benchmarking tasks.')
groupMainOptions.add_argument('-f', '--fuzz', metavar='<path_to_fuzzing_list>', action='store', type=argparse.FileType('r'), help='this option will try to find usufy-like URLs. The list of fuzzing platforms in the file should be (one per line): <BASE_DOMAIN>\t<VALID_NICK>')
groupMainOptions.add_argument('-l', '--list', metavar='<path_to_nick_list>', action='store', type=argparse.FileType('r'), help='path to the file where the list of nicks to verify is stored (one per line).')
groupMainOptions.add_argument('-n', '--nicks', metavar='<nick>', nargs='+', action='store', help = 'the list of nicks to process (at least one is required).')
# Selecting the platforms where performing the search
def run(program):
"""Loading OSRFramework output...
"""
platOptions = platform_selection.getAllPlatformNames(program)
# Loading the stored global data
global loaded_data
global DATA_FOLDER
output_folder = DATA_FOLDER
answer = []
form = request.form
if "terminal-form" in form.keys():
strParams = request.form['tex_command']
# Splitting the query
params = shlex.split(strParams)
# Manually adding the data folder if NOT provided
if "-o " not in strParams:
params += ["-o", output_folder]
elif "windowed-form" in form.keys():
# Manually building params
params = []
# Iterating through all the attributes. We will use its name to identify them
for key in form.keys():
if key == "tex_query":
# Adding the queries from the first text file
if program != "searchfy":
params += ["-n"]
else:
params += ["-q"]
# Splitting the query
splittedQuery = shlex.split(request.form[key])
# Adding the parameters
params += splittedQuery
elif key == "select_platforms":
# Adding the parameter depending on the platform
if program == "domainfy":
params += ["-t"]
elif program == "mailfy":
params += ["-d"]
else:
params += ["-p"]
# This is a MultiDict. We have to ad an iteration
for pName in form.getlist(key):
params += [pName]
elif "export_" in key:
params += ["-e"]
params += [key.split("_")[1]]
elif key == "open_url":
params += ["-w"]
elif key == "tex_filename":
params += ["-F"]
params += [request.form[key]]
params += ["-o", DATA_FOLDER]
# Selecting the appropriate program
if program == "domainfy":
args = domainfy.getParser().parse_args(params)
elif program == "entify":
args = entify.getParser().parse_args(params)
elif program == "mailfy":
args = mailfy.getParser().parse_args(params)
elif program == "phonefy":
args = phonefy.getParser().parse_args(params)
elif program == "searchfy":
args = searchfy.getParser().parse_args(params)
elif program == "usufy":
args = usufy.getParser().parse_args(params)
# Return output. text/html is required for most browsers to show the text
try:
answer = runQuery(program=program, args=args)
except:
abort(400)
# Reading CSV
try:
with open(os.path.join(args.output_folder,
args.file_header + ".csv")) as iF:
everything = iF.read().splitlines()
loaded_data["csv"] = ""
for i, line in enumerate(everything):
loaded_data["csv"] += line
# Checking if it is the last line. This is done to avoid extra lines.
if i + 1 != len(everything):
loaded_data["csv"] += "\n"
except:
pass
return render_template('research-' + program + '.html',
mt_research='class=current',
plat_options=platOptions,
text_results=general.usufyToTextExport(answer),
command=buildCommandFromParams(program, params))
def main(args):
'''
Main function. This function is created in this way so as to let other applications make use of the full configuration capabilities of the application.
'''
# Recovering the logger
# Calling the logger when being imported
osrframework.utils.logger.setupLogger(loggerName="osrframework.usufy", verbosity=args.verbose, logFolder=args.logfolder)
# From now on, the logger can be recovered like this:
logger = logging.getLogger("osrframework.usufy")
# Printing the results if requested
if not args.maltego:
print banner.text
sayingHello = """usufy.py Copyright (C) F. Brezo and Y. Rubio (i3visio) 2015
This program comes with ABSOLUTELY NO WARRANTY.
This is free software, and you are welcome to redistribute it under certain conditions. For additional info, visit <http://www.gnu.org/licenses/gpl-3.0.txt>."""
logger.info(sayingHello)
print sayingHello
print
logger.info("Starting usufy.py...")
if args.license:
logger.info("Looking for the license...")
# showing the license
try:
with open ("COPYING", "r") as iF:
contenido = iF.read().splitlines()
for linea in contenido:
print linea
except Exception:
try:
# Trying to recover the COPYING file...
with open ("/usr/share/osrframework/COPYING", "r") as iF:
contenido = iF.read().splitlines()
for linea in contenido:
print linea
except:
logger.error("ERROR: there has been an error when opening the COPYING file.\n\tThe file contains the terms of the GPLv3 under which this software is distributed.\n\tIn case of doubts, verify the integrity of the files or contact [email protected].")
elif args.fuzz:
logger.info("Performing the fuzzing tasks...")
res = fuzzUsufy(args.fuzz, args.fuzz_config)
logger.info("Recovered platforms:\n" + str(res))
else:
logger.debug("Recovering the list of platforms to be processed...")
# Recovering the list of platforms to be launched
listPlatforms = platform_selection.getPlatformsByName(platformNames=args.platforms, tags=args.tags, mode="usufy")
logger.debug("Platforms recovered.")
if args.info:
# Information actions...
if args.info == 'list_platforms':
infoPlatforms="Listing the platforms:\n"
for p in listPlatforms:
infoPlatforms += "\t\t" + (str(p) + ": ").ljust(16, ' ') + str(p.tags)+"\n"
logger.info(infoPlatforms)
return infoPlatforms
elif args.info == 'list_tags':
logger.info("Listing the tags:")
tags = {}
# Going through all the selected platforms to get their tags
for p in listPlatforms:
for t in p.tags:
if t not in tags.keys():
tags[t] = 1
else:
tags[t] += 1
infoTags = "List of tags:\n"
# Displaying the results in a sorted list
for t in tags.keys():
infoTags += "\t\t" + (t + ": ").ljust(16, ' ') + str(tags[t]) + " time(s)\n"
logger.info(infoTags)
return infoTags
else:
pass
# performing the test
elif args.benchmark:
logger.warning("The benchmark mode may last some minutes as it will be performing similar queries to the ones performed by the program in production. ")
logger.info("Launching the benchmarking tests...")
platforms = platform_selection.getAllPlatformNames("usufy")
res = benchmark.doBenchmark(platforms)
strTimes = ""
for e in sorted(res.keys()):
strTimes += str(e) + "\t" + str(res[e]) + "\n"
logger.info(strTimes)
return strTimes
# Executing the corresponding process...
else:
# Showing the execution time...
if not args.maltego:
startTime= dt.datetime.now()
print str(startTime) +"\tStarting search in " + str(len(listPlatforms)) + " platform(s)... Be patient!\n"
# Defining the list of users to monitor
nicks = []
logger.debug("Recovering nicknames to be processed...")
if args.nicks:
for n in args.nicks:
# TO-DO
# A trick to avoid having the processing of the properties when being queried by Maltego
if "properties.i3visio" not in n:
nicks.append(n)
else:
# Reading the nick files
try:
nicks = args.list.read().splitlines()
except:
logger.error("ERROR: there has been an error when opening the file that stores the nicks.\tPlease, check the existence of this file.")
if args.output_folder != None:
# if Verifying an output folder was selected
logger.debug("Preparing the output folder...")
if not args.maltego:
if not os.path.exists(args.output_folder):
logger.warning("The output folder \'" + args.output_folder + "\' does not exist. The system will try to create it.")
os.makedirs(args.output_folder)
# Launching the process...
res = processNickList(nicks, listPlatforms, args.output_folder, avoidProcessing = args.avoid_processing, avoidDownload = args.avoid_download, nThreads=args.threads, verbosity= args.verbose, logFolder=args.logfolder)
else:
res = processNickList(nicks, listPlatforms, nThreads=args.threads, verbosity= args.verbose, logFolder=args.logfolder)
logger.info("Listing the results obtained...")
# We are going to iterate over the results...
strResults = "\t"
# Structure returned
"""
[
{
"attributes": [
{
"attributes": [],
"type": "i3visio.uri",
"value": "http://twitter.com/i3visio"
},
{
"attributes": [],
"type": "i3visio.alias",
"value": "i3visio"
},
{
"attributes": [],
"type": "i3visio.platform",
"value": "Twitter"
}
],
"type": "i3visio.profile",
"value": "Twitter - i3visio"
}
,
...
]
"""
for r in res:
# The format of the results (attributes) for a given nick is a list as follows:
for att in r["attributes"]:
# iterating through the attributes
platform = ""
uri = ""
for details in att["attributes"]:
if details["type"] == "i3visio.platform":
platform = details["value"]
if details["type"] == "i3visio.uri":
uri = details["value"]
try:
strResults+= (str(platform) + ":").ljust(16, ' ')+ " "+ str(uri)+"\n\t\t"
except:
pass
logger.info(strResults)
# Generating summary files for each ...
if args.extension:
# Storing the file...
logger.info("Creating output files as requested.")
if not args.maltego:
# Verifying if the outputPath exists
if not os.path.exists (args.output_folder):
logger.warning("The output folder \'" + args.output_folder + "\' does not exist. The system will try to create it.")
os.makedirs(args.output_folder)
# Grabbing the results
fileHeader = os.path.join(args.output_folder, args.file_header)
# Iterating through the given extensions to print its values
if not args.maltego:
for ext in args.extension:
# Generating output files
general.exportUsufy(res, ext, fileHeader)
# Generating the Maltego output
if args.maltego:
general.listToMaltego(res)
# Printing the results if requested
if not args.maltego:
print "A summary of the results obtained are shown in the following table:"
#print res
print unicode(general.usufyToTextExport(res))
print
print "You can find all the information collected in the following files:"
for ext in args.extension:
# Showing the output files
print "\t-" + fileHeader + "." + ext
# Showing the execution time...
if not args.maltego:
print
endTime= dt.datetime.now()
print str(endTime) +"\tFinishing execution..."
print
print "Total time used:\t" + str(endTime-startTime)
print "Average seconds/query:\t" + str((endTime-startTime).total_seconds()/len(listPlatforms)) +" seconds"
print
# Urging users to place an issue on Github...
if not args.maltego:
print
print "Did something go wrong? Is a platform reporting false positives? Do you need to integrate a new one?"
print "Then, place an issue in the Github project: <https://github.com/i3visio/osrframework/issues>."
print "Note that otherwise, we won't know about it!"
print
return res
def getParser():
parser = argparse.ArgumentParser(
description=
'phonefy.py - Piece of software that checks the existence of a given series of phones in a bunch of phone number lists associated to malicious activities.',
prog='phonefy.py',
epilog=
'Check the README.md file for further details on the usage of this program or follow us on Twitter in <http://twitter.com/i3visio>.',
add_help=False)
parser._optionals.title = "Input options (one required)"
# Defining the mutually exclusive group for the main options
groupMainOptions = parser.add_mutually_exclusive_group(required=True)
# Adding the main options
groupMainOptions.add_argument('--license',
required=False,
action='store_true',
default=False,
help='shows the GPLv3+ license and exists.')
groupMainOptions.add_argument(
'-n',
'--numbers',
metavar='<phones>',
nargs='+',
action='store',
help='the list of phones to process (at least one is required).')
listAll = platform_selection.getAllPlatformNames("phonefy")
# Configuring the processing options
groupProcessing = parser.add_argument_group(
'Processing arguments',
'Configuring the way in which usufy will process the identified profiles.'
)
#groupProcessing.add_argument('-L', '--logfolder', metavar='<path_to_log_folder', required=False, default = './logs', action='store', help='path to the log folder. If none was provided, ./logs is assumed.')
groupProcessing.add_argument(
'-e',
'--extension',
metavar='<sum_ext>',
nargs='+',
choices=[
'csv', 'gml', 'json', 'mtz', 'ods', 'png', 'txt', 'xls', 'xlsx'
],
required=False,
default=['xls'],
action='store',
help='output extension for the summary files. Default: xls.')
groupProcessing.add_argument(
'-o',
'--output_folder',
metavar='<path_to_output_folder>',
required=False,
default='./results',
action='store',
help=
'output folder for the generated documents. While if the paths does not exist, usufy.py will try to create; if this argument is not provided, usufy will NOT write any down any data. Check permissions if something goes wrong.'
)
groupProcessing.add_argument(
'-p',
'--platforms',
metavar='<platform>',
choices=listAll,
nargs='+',
required=False,
default=['all'],
action='store',
help=
'select the platforms where you want to perform the search amongst the following: '
+ str(listAll) + '. More than one option can be selected.')
# Getting a sample header for the output files
groupProcessing.add_argument(
'-F',
'--file_header',
metavar='<alternative_header_file>',
required=False,
default="profiles",
action='store',
help=
'Header for the output filenames to be generated. If None was provided the following will be used: profiles.<extension>.'
)
groupProcessing.add_argument(
'--quiet',
required=False,
action='store_true',
default=False,
help='tells the program not to show anything.')
groupProcessing.add_argument(
'-w',
'--web_browser',
required=False,
action='store_true',
help='opening the URIs returned in the default web browser.')
# About options
groupAbout = parser.add_argument_group(
'About arguments',
'Showing additional information about this program.')
groupAbout.add_argument('-h',
'--help',
action='help',
help='shows this help and exists.')
#groupAbout.add_argument('-v', '--verbose', metavar='<verbosity>', choices=[0, 1, 2], required=False, action='store', default=1, help='select the verbosity level: 0 - none; 1 - normal (default); 2 - debug.', type=int)
groupAbout.add_argument(
'--version',
action='version',
version='%(prog)s ' + " " + __version__,
help='shows the version of the program and exists.')
return parser
# Writing the results onto a file
if args.output_file != None:
with open(output_file, "w") as oF:
oF.write(json.dumps(results, indent=2) )
if __name__ == "__main__":
parser = argparse.ArgumentParser(description='phonefy.py - Piece of software that checks the existence of a given series of phones in a bunch of phone number lists associated to malicious activities.', prog='phonefy.py', epilog='Check the README.md file for further details on the usage of this program.', add_help=False)
parser._optionals.title = "Input options (one required)"
# Defining the mutually exclusive group for the main options
general = parser.add_mutually_exclusive_group(required=True)
# Adding the main options
general.add_argument('--license', required=False, action='store_true', default=False, help='shows the GPLv3+ license and exists.')
general.add_argument('-n', '--numbers', metavar='<phones>', nargs='+', action='store', help = 'the list of phones to process (at least one is required).')
listAll = platform_selection.getAllPlatformNames("phonefy")
# Configuring the processing options
groupProcessing = parser.add_argument_group('Processing arguments', 'Configuring the way in which usufy will process the identified profiles.')
#groupProcessing.add_argument('-L', '--logfolder', metavar='<path_to_log_folder', required=False, default = './logs', action='store', help='path to the log folder. If none was provided, ./logs is assumed.')
groupProcessing.add_argument('-o', '--output_file', metavar='<path_to_output_file>', action='store', help='path to the output file where the results will be stored in json format.', required=False)
groupProcessing.add_argument('-p', '--platforms', metavar='<platform>', choices=listAll, nargs='+', required=False, default =['all'] ,action='store', help='select the platforms where you want to perform the search amongst the following: ' + str(listAll) + '. More than one option can be selected.')
groupProcessing.add_argument('--quiet', required=False, action='store_true', default=False, help='tells the program not to show anything.')
# About options
groupAbout = parser.add_argument_group('About arguments', 'Showing additional information about this program.')
groupAbout.add_argument('-h', '--help', action='help', help='shows this help and exists.')
#groupAbout.add_argument('-v', '--verbose', metavar='<verbosity>', choices=[0, 1, 2], required=False, action='store', default=1, help='select the verbosity level: 0 - none; 1 - normal (default); 2 - debug.', type=int)
groupAbout.add_argument('--version', action='version', version='%(prog)s ' +__version__, help='shows the version of the program and exists.')
args = parser.parse_args()
def main(args):
'''
Main function. This function is created in this way so as to let other applications make use of the full configuration capabilities of the application.
'''
# Recovering the logger
# Calling the logger when being imported
osrframework.utils.logger.setupLogger(loggerName="osrframework.usufy", verbosity=args.verbose, logFolder=args.logfolder)
# From now on, the logger can be recovered like this:
logger = logging.getLogger("osrframework.usufy")
# Printing the results if requested
if not args.maltego:
print banner.text
sayingHello = """usufy.py Copyright (C) F. Brezo and Y. Rubio (i3visio) 2014-2017
This program comes with ABSOLUTELY NO WARRANTY.
This is free software, and you are welcome to redistribute it under certain conditions. For additional info, visit <http://www.gnu.org/licenses/gpl-3.0.txt>."""
logger.info(sayingHello)
print sayingHello
print
logger.info("Starting usufy.py...")
if args.license:
logger.info("Looking for the license...")
# showing the license
try:
with open ("COPYING", "r") as iF:
contenido = iF.read().splitlines()
for linea in contenido:
print linea
except Exception:
try:
# Trying to recover the COPYING file...
with open ("/usr/share/osrframework/COPYING", "r") as iF:
contenido = iF.read().splitlines()
for linea in contenido:
print linea
except:
logger.error("ERROR: there has been an error when opening the COPYING file.\n\tThe file contains the terms of the GPLv3 under which this software is distributed.\n\tIn case of doubts, verify the integrity of the files or contact [email protected].")
elif args.fuzz:
logger.info("Performing the fuzzing tasks...")
res = fuzzUsufy(args.fuzz, args.fuzz_config)
logger.info("Recovered platforms:\n" + str(res))
else:
logger.debug("Recovering the list of platforms to be processed...")
# Recovering the list of platforms to be launched
listPlatforms = platform_selection.getPlatformsByName(platformNames=args.platforms, tags=args.tags, mode="usufy", excludePlatformNames=args.exclude)
logger.debug("Platforms recovered.")
if args.info:
# Information actions...
if args.info == 'list_platforms':
infoPlatforms="Listing the platforms:\n"
for p in listPlatforms:
infoPlatforms += "\t\t" + (str(p) + ": ").ljust(16, ' ') + str(p.tags)+"\n"
logger.info(infoPlatforms)
return infoPlatforms
elif args.info == 'list_tags':
logger.info("Listing the tags:")
tags = {}
# Going through all the selected platforms to get their tags
for p in listPlatforms:
for t in p.tags:
if t not in tags.keys():
tags[t] = 1
else:
tags[t] += 1
infoTags = "List of tags:\n"
# Displaying the results in a sorted list
for t in tags.keys():
infoTags += "\t\t" + (t + ": ").ljust(16, ' ') + str(tags[t]) + " time(s)\n"
logger.info(infoTags)
return infoTags
else:
pass
# performing the test
elif args.benchmark:
logger.warning("The benchmark mode may last some minutes as it will be performing similar queries to the ones performed by the program in production. ")
logger.info("Launching the benchmarking tests...")
platforms = platform_selection.getAllPlatformNames("usufy")
res = benchmark.doBenchmark(platforms)
strTimes = ""
for e in sorted(res.keys()):
strTimes += str(e) + "\t" + str(res[e]) + "\n"
logger.info(strTimes)
return strTimes
# showing the tags of the usufy platforms
elif args.show_tags:
logger.info("Collecting the list of tags...")
tags = platform_selection.getAllPlatformNamesByTag("usufy")
logger.info(json.dumps(tags, indent=2))
print "This is the list of platforms grouped by tag."
print
print json.dumps(tags, indent=2, sort_keys=True)
print
print "[Tip] Remember that you can always launch the platform using the -t option followed by any of the aforementioned."
print
return tags
# Executing the corresponding process...
else:
# Showing the execution time...
if not args.maltego:
startTime= dt.datetime.now()
print str(startTime) +"\tStarting search in " + str(len(listPlatforms)) + " platform(s)... Relax!"
print
print "\tPress <Ctrl + C> to stop..."
print
# Defining the list of users to monitor
nicks = []
logger.debug("Recovering nicknames to be processed...")
if args.nicks:
for n in args.nicks:
# TO-DO
# A trick to avoid having the processing of the properties when being queried by Maltego
if "properties.i3visio" not in n:
nicks.append(n)
else:
# Reading the nick files
try:
nicks = args.list.read().splitlines()
except:
logger.error("ERROR: there has been an error when opening the file that stores the nicks.\tPlease, check the existence of this file.")
# Definning the results
res = []
if args.output_folder != None:
# if Verifying an output folder was selected
logger.debug("Preparing the output folder...")
if not args.maltego:
if not os.path.exists(args.output_folder):
logger.warning("The output folder \'" + args.output_folder + "\' does not exist. The system will try to create it.")
os.makedirs(args.output_folder)
# Launching the process...
###try:
res = processNickList(nicks, listPlatforms, args.output_folder, avoidProcessing = args.avoid_processing, avoidDownload = args.avoid_download, nThreads=args.threads, verbosity= args.verbose, logFolder=args.logfolder)
###except Exception as e:
###print "Exception grabbed when processing the nicks: " + str(e)
###print traceback.print_stack()
else:
try:
res = processNickList(nicks, listPlatforms, nThreads=args.threads, verbosity= args.verbose, logFolder=args.logfolder)
except Exception as e:
print "Exception grabbed when processing the nicks: " + str(e)
print traceback.print_stack()
logger.info("Listing the results obtained...")
# We are going to iterate over the results...
strResults = "\t"
# Structure returned
"""
[
{
"attributes": [
{
"attributes": [],
"type": "i3visio.uri",
"value": "http://twitter.com/i3visio"
},
{
"attributes": [],
"type": "i3visio.alias",
"value": "i3visio"
},
{
"attributes": [],
"type": "i3visio.platform",
"value": "Twitter"
}
],
"type": "i3visio.profile",
"value": "Twitter - i3visio"
}
,
...
]
"""
for r in res:
# The format of the results (attributes) for a given nick is a list as follows:
for att in r["attributes"]:
# iterating through the attributes
platform = ""
uri = ""
for details in att["attributes"]:
if details["type"] == "i3visio.platform":
platform = details["value"]
if details["type"] == "i3visio.uri":
uri = details["value"]
try:
strResults+= (str(platform) + ":").ljust(16, ' ')+ " "+ str(uri)+"\n\t\t"
except:
pass
logger.info(strResults)
# Generating summary files for each ...
if args.extension:
# Storing the file...
logger.info("Creating output files as requested.")
if not args.maltego:
# Verifying if the outputPath exists
if not os.path.exists (args.output_folder):
logger.warning("The output folder \'" + args.output_folder + "\' does not exist. The system will try to create it.")
os.makedirs(args.output_folder)
# Grabbing the results
fileHeader = os.path.join(args.output_folder, args.file_header)
# Iterating through the given extensions to print its values
for ext in args.extension:
# Generating output files
general.exportUsufy(res, ext, fileHeader)
# Generating the Maltego output
if args.maltego:
general.listToMaltego(res)
# Printing the results if requested
else:
print "A summary of the results obtained are shown in the following table:"
#print res
print unicode(general.usufyToTextExport(res))
print
if args.web_browser:
general.openResultsInBrowser(res)
print "You can find all the information collected in the following files:"
for ext in args.extension:
# Showing the output files
print "\t-" + fileHeader + "." + ext
# Showing the execution time...
print
endTime= dt.datetime.now()
print str(endTime) +"\tFinishing execution..."
print
print "Total time used:\t" + str(endTime-startTime)
print "Average seconds/query:\t" + str((endTime-startTime).total_seconds()/len(listPlatforms)) +" seconds"
print
# Urging users to place an issue on Github...
print
print "Did something go wrong? Is a platform reporting false positives? Do you need to integrate a new one?"
print "Then, place an issue in the Github project: <https://github.com/i3visio/osrframework/issues>."
print "Note that otherwise, we won't know about it!"
print
return res
class OSRFConsoleSearchfy(OSRFConsoleUtil):
"""Class that controls an interactive searchfy program"""
# Setting up the name of the module
UNAME = "searchfy.py"
intro = ""
# Defining the prompt
prompt = 'osrf (' + UNAME.split('.')[0] + ') > '
# Defining the character to create hyphens
ruler = '-'
# Defining the configuration for this module
CONFIG = {}
CONFIG["QUERY"] = {
"DESCRIPTION": "Query to be verified. Escape \" and \'.",
"CURRENT_VALUE": None,
"DEFAULT_VALUE": None,
"REQUIRED": True,
"OPTIONS": []
}
CONFIG["PLATFORMS"] = {
"DESCRIPTION": "Platforms to be checked.",
"CURRENT_VALUE": "all",
"REQUIRED": False,
"DEFAULT_VALUE": "all",
"OPTIONS": platform_selection.getAllPlatformNames("searchfy"),
}
CONFIG["OUTPUT"] = {
"DESCRIPTION":
"The path to the output folder where the files will be created.",
"CURRENT_VALUE": "./",
"DEFAULT_VALUE": "./",
"REQUIRED": False,
"OPTIONS": []
}
CONFIG["EXTENSION"] = {
"DESCRIPTION": "The default extension of the files to be written.",
"CURRENT_VALUE": "csv",
"DEFAULT_VALUE": "csv",
"REQUIRED": False,
"OPTIONS": ['csv', 'xls', 'xlsx', 'json', 'gml']
}
def _getParams(self):
""" Function that creates the array with the params that will work with this function"""
# Creating the parameters as if they were created using the command line
params = [
"-q", self.CONFIG["QUERY"]["CURRENT_VALUE"], "-p",
self.CONFIG["PLATFORMS"]["CURRENT_VALUE"], "-o",
self.CONFIG["OUTPUT"]["CURRENT_VALUE"], "-e",
self.CONFIG["EXTENSION"]["CURRENT_VALUE"]
]
return params
def do_run(self, line):
"""
Running the current application. This method should be redefined for each util.
"""
print
# Checking if all the required parameters have been set
if self._checkIfRequiredAreSet():
print "Collecting the options set by the user..."
# Getting the parser...
parser = searchfy.getParser()
# Generating the parameters
params = self._getParams()
args = parser.parse_args(params)
print "Launching the util with the following parameters: " + str(
params)
print
try:
searchfy.main(args)
except Exception as e:
print "[!] ERROR. Something happenned when launching the utility. Type 'show options' to check the parameters. "
print "Traceback: " + str(e)
else:
print "[!] ERROR. There are required parameters which have not been set."
self.do_show("options")
print
class OSRFConsoleSearchfy(OSRFConsoleUtil):
"""
Class that controls an interactive searchfy program
"""
# Setting up the name of the module
UNAME = "searchfy"
intro = """
Loading """ + UNAME + """..."""
description = """
A tool to look for people using different search engines found in several
platforms. This tool will wrap the search capabilities provided by the
platforms."""
# Defining the prompt
prompt = general.emphasis('\nosrf (' + UNAME + ') > ')
# Defining the character to create hyphens
ruler = '-'
# Defining the configuration for this module
CONFIG = {}
DEFAULT_VALUES = configuration.returnListOfConfigurationValues("searchfy")
# Defining the configuration for this module
CONFIG = {}
CONFIG["QUERY"] = {
"DESCRIPTION": "Query to be verified. Escape \" and \'.",
"CURRENT_VALUE": None,
"DEFAULT_VALUE": None,
"REQUIRED": True,
"OPTIONS": []
}
CONFIG["PLATFORMS"] = {
"DESCRIPTION": "Platforms to be checked.",
"CURRENT_VALUE": DEFAULT_VALUES["platforms"],
"DEFAULT_VALUE": DEFAULT_VALUES["platforms"],
"REQUIRED": False,
"OPTIONS": platform_selection.getAllPlatformNames("searchfy"),
}
CONFIG["OUTPUT"] = {
"DESCRIPTION":
"The path to the output folder where the files will be created.",
"CURRENT_VALUE": DEFAULT_VALUES["output_folder"],
"DEFAULT_VALUE": DEFAULT_VALUES["output_folder"],
"REQUIRED": False,
"OPTIONS": []
}
CONFIG["EXTENSION"] = {
"DESCRIPTION": "The default extension of the files to be written.",
"CURRENT_VALUE": DEFAULT_VALUES["extension"],
"DEFAULT_VALUE": DEFAULT_VALUES["extension"],
"REQUIRED": False,
"OPTIONS": ['csv', 'xls', 'xlsx', 'json', 'gml']
}
def _getParams(self):
"""
Function that creates the array with the params of this function
Returns:
--------
list: A list of the params that can be used
"""
# Creating the parameters as if they were created using the command line
params = ["-q"] + self.CONFIG["QUERY"]["CURRENT_VALUE"] + [
"-p"
] + self.CONFIG["PLATFORMS"]["CURRENT_VALUE"] + [
"-o", self.CONFIG["OUTPUT"]["CURRENT_VALUE"], "-e"
] + self.CONFIG["EXTENSION"]["CURRENT_VALUE"]
return params
def do_run(self, line):
"""
Command that send the order to the framework to launch this util
Args:
-----
line: The string of the line typed.
"""
# Checking if all the required parameters have been set
if self._checkIfRequiredAreSet():
print(general.info("\nCollecting the options set by the user..."))
# Getting the parser...
parser = searchfy.getParser()
# Generating the parameters
params = self._getParams()
args = parser.parse_args(params)
print(
general.info("\nLaunching " + self.UNAME +
" with the following parameters:") +
general.emphasis("\t$ " + self.UNAME + " " +
utils.listToString(params) + "\n"))
try:
searchfy.main(args)
except Exception as e:
print(
general.error(
"\n[!!] ERROR. Something happened when launching the utility. Type 'show options' to check the parameters.\n"
))
print(general.error("Traceback: " + str(e)))
else:
print(
general.error(
"\n[!!] ERROR. There are required parameters which have not been set."
))
self.do_show("options")
print(general.success("Execution ended successfully."))
def getParser():
DEFAULT_VALUES = configuration.returnListOfConfigurationValues("mailfy")
# Capturing errors just in case the option is not found in the configuration
try:
excludeList = [DEFAULT_VALUES["exclude_domains"]]
except:
excludeList = []
# Recovering all the possible options
platOptions = platform_selection.getAllPlatformNames("mailfy")
parser = argparse.ArgumentParser(
description='mailfy - Checking the existence of a given mail.',
prog='mailfy',
epilog=
'Check the README.md file for further details on the usage of this program or follow us on Twitter in <http://twitter.com/i3visio>.',
add_help=False,
conflict_handler='resolve')
parser._optionals.title = "Input options (one required)"
# Adding the main options
groupMainOptions = parser.add_mutually_exclusive_group(required=True)
groupMainOptions.add_argument('--license',
required=False,
action='store_true',
default=False,
help='shows the GPLv3+ license and exists.')
groupMainOptions.add_argument('-m',
'--emails',
metavar='<emails>',
nargs='+',
action='store',
help='the list of emails to be checked.')
groupMainOptions.add_argument('-M',
'--emails_file',
metavar='<emails_file>',
action='store',
help='the file with the list of emails.')
groupMainOptions.add_argument(
'-n',
'--nicks',
metavar='<nicks>',
nargs='+',
action='store',
help='the list of nicks to be checked in the domains selected.')
groupMainOptions.add_argument(
'-N',
'--nicks_file',
metavar='<nicks_file>',
action='store',
help=
'the file with the list of nicks to be checked in the domains selected.'
)
groupMainOptions.add_argument(
'--create_emails',
metavar='<nicks_file>',
action='store',
help=
'the file with the list of nicks to be created in the domains selected.'
)
# Configuring the processing options
groupProcessing = parser.add_argument_group(
'Processing arguments',
'Configuring the way in which mailfy will process the identified profiles.'
)
groupProcessing.add_argument(
'-e',
'--extension',
metavar='<sum_ext>',
nargs='+',
choices=['csv', 'gml', 'json', 'ods', 'png', 'txt', 'xls', 'xlsx'],
required=False,
default=DEFAULT_VALUES["extension"],
action='store',
help='output extension for the summary files. Default: xls.')
groupProcessing.add_argument(
'-d',
'--domains',
metavar='<candidate_domains>',
nargs='+',
choices=['all'] + EMAIL_DOMAINS,
action='store',
help='list of domains where the nick will be looked for.',
required=False,
default=DEFAULT_VALUES["domains"])
groupProcessing.add_argument(
'-o',
'--output_folder',
metavar='<path_to_output_folder>',
required=False,
default=DEFAULT_VALUES["output_folder"],
action='store',
help=
'output folder for the generated documents. While if the paths does not exist, usufy.py will try to create; if this argument is not provided, usufy will NOT write any down any data. Check permissions if something goes wrong.'
)
groupProcessing.add_argument(
'-p',
'--platforms',
metavar='<platform>',
choices=platOptions,
nargs='+',
required=False,
default=["all"],
action='store',
help=
'select the platforms where you want to perform the search amongst the following: {}. More than one option can be selected.'
.format(str(platOptions)))
groupProcessing.add_argument(
'-x',
'--exclude',
metavar='<domain>',
choices=EMAIL_DOMAINS,
nargs='+',
required=False,
default=excludeList,
action='store',
help="select the domains to be excluded from the search.")
groupProcessing.add_argument(
'-F',
'--file_header',
metavar='<alternative_header_file>',
required=False,
default=DEFAULT_VALUES["file_header"],
action='store',
help=
'Header for the output filenames to be generated. If None was provided the following will be used: profiles.<extension>.'
)
groupProcessing.add_argument(
'-T',
'--threads',
metavar='<num_threads>',
required=False,
action='store',
default=int(DEFAULT_VALUES["threads"]),
type=int,
help=
'write down the number of threads to be used (default 16). If 0, the maximum number possible will be used, which may make the system feel unstable.'
)
groupProcessing.add_argument(
'--quiet',
required=False,
action='store_true',
default=False,
help='tells the program not to show anything.')
# About options
groupAbout = parser.add_argument_group(
'About arguments',
'Showing additional information about this program.')
groupAbout.add_argument('-h',
'--help',
action='help',
help='shows this help and exists.')
groupAbout.add_argument(
'--version',
action='version',
version='[%(prog)s] OSRFramework ' + osrframework.__version__,
help='shows the version of the program and exists.')
return parser
def main(params=None):
"""
Main function to launch usufy.
The function is created in this way so as to let other applications make
use of the full configuration capabilities of the application. The
parameters received are used as parsed by this modules `getParser()`.
Args:
-----
params: A list with the parameters as grabbed by the terminal. It is
None when this is called by an entry_point.
Returns:
--------
dict: A Json representing the matching results.
"""
# Grabbing the parser
parser = getParser()
if params != None:
args = parser.parse_args(params)
else:
args = parser.parse_args()
# Recovering the logger
# Calling the logger when being imported
osrframework.utils.logger.setupLogger(loggerName="osrframework.usufy",
verbosity=args.verbose,
logFolder=args.logfolder)
# From now on, the logger can be recovered like this:
logger = logging.getLogger("osrframework.usufy")
print(general.title(banner.text))
sayingHello = """
Usufy | Copyright (C) F. Brezo and Y. Rubio (i3visio) 2014-2018
This program comes with ABSOLUTELY NO WARRANTY. This is free software, and you
are welcome to redistribute it under certain conditions. For additional info,
visit """ + general.LICENSE_URL + "\n"
logger.info(sayingHello)
print(general.title(sayingHello))
logger.info("Starting usufy...")
if args.license:
general.showLicense()
elif args.fuzz:
logger.info("Performing the fuzzing tasks...")
res = fuzzUsufy(args.fuzz, args.fuzz_config)
logger.info("Recovered platforms:\n" + str(res))
else:
logger.debug("Recovering the list of platforms to be processed...")
# Recovering the list of platforms to be launched
listPlatforms = platform_selection.getPlatformsByName(
platformNames=args.platforms,
tags=args.tags,
mode="usufy",
excludePlatformNames=args.exclude)
logger.debug("Platforms recovered.")
if args.info:
# Information actions...
if args.info == 'list_platforms':
infoPlatforms = "Listing the platforms:\n"
for p in listPlatforms:
infoPlatforms += "\t\t" + (str(p) + ": ").ljust(
16, ' ') + str(p.tags) + "\n"
logger.info(infoPlatforms)
return infoPlatforms
elif args.info == 'list_tags':
logger.info("Listing the tags:")
tags = {}
# Going through all the selected platforms to get their tags
for p in listPlatforms:
for t in p.tags:
if t not in tags.keys():
tags[t] = 1
else:
tags[t] += 1
infoTags = "List of tags:\n"
# Displaying the results in a sorted list
for t in tags.keys():
infoTags += "\t\t" + (t + ": ").ljust(16, ' ') + str(
tags[t]) + " time(s)\n"
logger.info(infoTags)
return infoTags
else:
pass
# performing the test
elif args.benchmark:
logger.warning(
"The benchmark mode may last some minutes as it will be performing similar queries to the ones performed by the program in production. "
)
logger.info("Launching the benchmarking tests...")
platforms = platform_selection.getAllPlatformNames("usufy")
res = benchmark.doBenchmark(platforms)
strTimes = ""
for e in sorted(res.keys()):
strTimes += str(e) + "\t" + str(res[e]) + "\n"
logger.info(strTimes)
return strTimes
# showing the tags of the usufy platforms
elif args.show_tags:
logger.info("Collecting the list of tags...")
tags = platform_selection.getAllPlatformNamesByTag("usufy")
logger.info(json.dumps(tags, indent=2))
print(
general.info(
"This is the list of platforms grouped by tag.\n"))
print(json.dumps(tags, indent=2, sort_keys=True))
print(
general.info(
"[Tip] Remember that you can always launch the platform using the -t option followed by any of the aforementioned.\n"
))
return tags
# Executing the corresponding process...
else:
# Showing the execution time...
startTime = dt.datetime.now()
print(
str(startTime) + "\tStarting search in " +
general.emphasis(str(len(listPlatforms))) +
" platform(s)... Relax!\n")
print(general.emphasis("\tPress <Ctrl + C> to stop...\n"))
# Defining the list of users to monitor
nicks = []
logger.debug("Recovering nicknames to be processed...")
if args.nicks:
for n in args.nicks:
# TO-DO
# A trick to avoid having the processing of the properties when being queried by Maltego
if "properties.i3visio" not in n:
nicks.append(n)
else:
# Reading the nick files
try:
nicks = args.list.read().splitlines()
except:
logger.error(
"ERROR: there has been an error when opening the file that stores the nicks.\tPlease, check the existence of this file."
)
# Definning the results
res = []
if args.output_folder != None:
# if Verifying an output folder was selected
logger.debug("Preparing the output folder...")
if not os.path.exists(args.output_folder):
logger.warning(
"The output folder \'" + args.output_folder +
"\' does not exist. The system will try to create it.")
os.makedirs(args.output_folder)
# Launching the process...
res = processNickList(nicks,
listPlatforms,
args.output_folder,
avoidProcessing=args.avoid_processing,
avoidDownload=args.avoid_download,
nThreads=args.threads,
verbosity=args.verbose,
logFolder=args.logfolder)
else:
try:
res = processNickList(nicks,
listPlatforms,
nThreads=args.threads,
verbosity=args.verbose,
logFolder=args.logfolder)
except Exception as e:
print(
general.error(
"Exception grabbed when processing the nicks: " +
str(e)))
print(general.error(traceback.print_stack()))
logger.info("Listing the results obtained...")
# We are going to iterate over the results...
strResults = "\t"
# Structure returned
"""
[
{ print
"attributes": [
{
"attributes": [],
"type": "i3visio.uri",
"value": "http://twitter.com/i3visio"
},
{
"attributes": [],
"type": "i3visio.alias",
"value": "i3visio"
},
{
"attributes": [],
"type": "i3visio.platform",
"value": "Twitter"
}
],
"type": "i3visio.profile",
"value": "Twitter - i3visio"
}
,
...
]
"""
for r in res:
# The format of the results (attributes) for a given nick is a list as follows:
for att in r["attributes"]:
# iterating through the attributes
platform = ""
uri = ""
for details in att["attributes"]:
if details["type"] == "i3visio.platform":
platform = details["value"]
if details["type"] == "i3visio.uri":
uri = details["value"]
try:
strResults += (str(platform) + ":").ljust(
16, ' ') + " " + str(uri) + "\n\t\t"
except:
pass
logger.info(strResults)
# Generating summary files for each ...
if args.extension:
# Storing the file...
logger.info("Creating output files as requested.")
# Verifying if the outputPath exists
if not os.path.exists(args.output_folder):
logger.warning(
"The output folder \'" + args.output_folder +
"\' does not exist. The system will try to create it.")
os.makedirs(args.output_folder)
# Grabbing the results
fileHeader = os.path.join(args.output_folder, args.file_header)
# Iterating through the given extensions to print its values
for ext in args.extension:
# Generating output files
general.exportUsufy(res, ext, fileHeader)
now = dt.datetime.now()
print(
str(now) +
"\tA summary of the results obtained is shown below:\n")
print(general.success(general.usufyToTextExport(res)))
if args.web_browser:
general.openResultsInBrowser(res)
now = dt.datetime.now()
print("\n" + str(now) + "\tYou can find all the information here:")
for ext in args.extension:
# Showing the output files
print("\t" + general.emphasis(fileHeader + "." + ext))
# Showing the execution time...
endTime = dt.datetime.now()
print("\n" + str(endTime) + "\tFinishing execution...\n")
print("Total time consumed:\t" +
general.emphasis(str(endTime - startTime)))
print("Average seconds/query:\t" + general.emphasis(
str((endTime - startTime).total_seconds() /
len(listPlatforms))) + " seconds\n")
# Urging users to place an issue on Github...
print(banner.footer)
if params:
return res
def processNickList(nicks, platforms=None, rutaDescarga="./", avoidProcessing=True, avoidDownload=True, nThreads=12, maltego=False, verbosity=1, logFolder="./logs"):
'''
Method that receives as a parameter a series of nicks and verifies whether those nicks have a profile associated in different social networks.
List of parameters that the method receives:
:param nicks: list of nicks to process.
:param platforms: list of <Platform> objects to be processed.
:param rutaDescarga: local file where saving the obtained information.
:param avoidProcessing: boolean var that defines whether the profiles will NOT be processed.
:param avoidDownload: boolean var that defines whether the profiles will NOT be downloaded (stored in this version).
:param maltego: parameter to tell usufy.py that he has been invoked by Malego.
:param verbosity: the level of verbosity to be used.
:param logFolder: the path to the log folder.
:return:
Returns a dictionary where the key is the nick and the value another dictionary where the keys are the social networks and te value is the corresponding URL.
'''
osrframework.utils.logger.setupLogger(loggerName="osrframework.usufy", verbosity=verbosity, logFolder=logFolder)
logger = logging.getLogger("osrframework.usufy")
if platforms == None:
platforms = platform_selection.getAllPlatformNames("usufy")
# Defining the output results variable
res = []
# Processing the whole list of terms...
for nick in nicks:
logger.info("Looking for '" + nick + "' in " + str(len(platforms)) + " different platforms:\n" +str( [ str(plat) for plat in platforms ] ) )
# If the process is executed by the current app, we use the Processes. It is faster than pools.
if nThreads <= 0 or nThreads > len(platforms):
nThreads = len(platforms)
logger.info("Launching " + str(nThreads) + " different threads...")
# Using threads in a pool if we are not running the program in main
# Example catched from: https://stackoverflow.com/questions/11312525/catch-ctrlc-sigint-and-exit-multiprocesses-gracefully-in-python
original_sigint_handler = signal.signal(signal.SIGINT, signal.SIG_IGN)
pool = Pool(nThreads)
signal.signal(signal.SIGINT, original_sigint_handler)
poolResults = []
try:
def log_result(result):
# This is called whenever foo_pool(i) returns a result.
# result_list is modified only by the main process, not the pool workers.
poolResults.append(result)
for plat in platforms:
# We need to create all the arguments that will be needed
parameters = ( plat, nick, rutaDescarga, avoidProcessing, avoidDownload, )
pool.apply_async (pool_function, args= parameters, callback = log_result )
# Waiting for results to be finished
while len(poolResults) < len(platforms):
#print "Waiiting to finish all!"
pass
# Closing normal termination
pool.close()
except KeyboardInterrupt:
print "\n[!] Process manually stopped by the user. Terminating workers.\n"
pool.terminate()
print "[!] The following platforms were not processed:"
pending = ""
for p in platforms:
processed = False
for processedPlatform in poolResults:
if str(p) == processedPlatform["platform"]:
processed = True
break
if not processed:
print "\t- " + str(p)
pending += " " + str(p).lower()
print
print "[!] If you want to relaunch the app with these platforms you can always run the command with: "
print "\t usufy.py ... -p " + pending
print
print "[!] If you prefer to avoid these platforms you can manually evade them for whatever reason with: "
print "\t usufy.py ... -x " + pending
print
pool.join()
profiles = []
# Processing the results
# ----------------------
for serArray in poolResults:
data = serArray["data"]
# We need to recover the results and check if they are not an empty json or None
if data != None:
array = json.loads(data)
for r in array:
if r != "{}":
profiles.append(r)
res+=profiles
return res
class OSRFConsoleUsufy(OSRFConsoleUtil):
"""Class that controls an interactive usufy program"""
# Setting up the name of the module
UNAME = "usufy.py"
intro = ""
# Defining the prompt
prompt = general.emphasis('osrf (' + UNAME.split('.')[0] + ') > ')
# Defining the character to create hyphens
ruler = '-'
DEFAULT_VALUES = configuration.returnListOfConfigurationValues("usufy")
# Defining the configuration for this module
CONFIG = {}
CONFIG["NICK"] = {
"DESCRIPTION" : "Alias to be verified.",
"CURRENT_VALUE" : None,
"DEFAULT_VALUE" : None,
"REQUIRED" : True,
"OPTIONS" : []
}
CONFIG["PLATFORMS"] = {
"DESCRIPTION" : "Platforms to be checked.",
"CURRENT_VALUE" : DEFAULT_VALUES["platforms"],
"DEFAULT_VALUE" : DEFAULT_VALUES["platforms"],
"REQUIRED" : False,
"OPTIONS" : platform_selection.getAllPlatformNames("usufy"),
}
CONFIG["THREADS"] = {
"DESCRIPTION" : "Number of threads to use.",
"CURRENT_VALUE" : DEFAULT_VALUES["threads"],
"DEFAULT_VALUE" : DEFAULT_VALUES["threads"],
"REQUIRED" : False,
"OPTIONS" : []
}
CONFIG["OUTPUT"] = {
"DESCRIPTION" : "The path to the output folder where the files will be created.",
"CURRENT_VALUE" : DEFAULT_VALUES["output_folder"],
"DEFAULT_VALUE" : DEFAULT_VALUES["output_folder"],
"REQUIRED" : False,
"OPTIONS" : []
}
CONFIG["EXTENSION"] = {
"DESCRIPTION" : "The default extension of the files to be written.",
"CURRENT_VALUE" : DEFAULT_VALUES["extension"],
"DEFAULT_VALUE" : DEFAULT_VALUES["extension"],
"REQUIRED" : False,
"OPTIONS" : ['csv', 'xls', 'xlsx', 'json', 'gml']
}
def _getParams(self):
"""Function that creates the array with the params that will work with this function."""
# Creating the parameters as if they were created using the command line
params = [
"-n" ] + self.CONFIG["NICK"]["CURRENT_VALUE"].split() + [
"-p" ] + self.CONFIG["PLATFORMS"]["CURRENT_VALUE"].split() + [
"-T", str(self.CONFIG["THREADS"]["CURRENT_VALUE"]),
"-o", self.CONFIG["OUTPUT"]["CURRENT_VALUE"],
"-e" ] + self.CONFIG["EXTENSION"]["CURRENT_VALUE"].split()
return params
def do_run(self, line):
"""Running the current application. This method should be redefined for each util."""
print
# Checking if all the required parameters have been set
if self._checkIfRequiredAreSet():
print(general.info("Collecting the options set by the user..."))
# Getting the parser...
parser = usufy.getParser()
# Generating the parameters
params = self._getParams()
args = parser.parse_args(params)
print(general.info("Launching " + self.UNAME + " with the following parameters: ") + general.emphashis(str(params)))
try:
usufy.main(args)
except Exception as e:
print(gemeral.error("[!] ERROR. Something happenned when launching the utility. Type 'show options' to check the parameters. "))
print(general.error("Traceback: " + str(e)))
else:
print(general.error("[!] ERROR. There are required parameters which have not been set."))
self.do_show("options")
print(general.success("Execution ended successfully."))
class OSRFConsoleMain(cmd.Cmd):
"""
OSRFramework console application to control the different framework utils.
Type 'help' to find the commands.
"""
DISCLAIMER = '''\tOSRFConsole ''' + __version__ + ''' - Copyright (C) F. Brezo and Y. Rubio (i3visio) 2016-2017
This program comes with ABSOLUTELY NO WARRANTY.
This is free software, and you are welcome to redistribute it under certain conditions. For additional info, visit to <http://www.gnu.org/licenses/gpl-3.0.txt>.'''
intro = banner.text + "\n" + DISCLAIMER
info = "\n General information"
info += "\n ==================="
info += "\n OSRFramework stands for Open Sources Research Framework. It includes a set of tools that help the analyst in the task of user profiling making use of different OSINT tools. To get additional information about the available commands type 'help'."
info += "\n"
info += "\n Modules available:"
info += "\n ------------------"
info += "\n\t- usufy --> the Jewel of the Chrown. A tool that verifies if a username exists in " + str(len(platform_selection.getAllPlatformNames("usufy"))) + " platforms."
info += "\n\t- mailfy --> a tool to check if a username has been registered in up to " + str(len(mailfy.EMAIL_DOMAINS )) + " email providers."
info += "\n\t- searchfy --> a tool to look for profiles using full names and other info in " + str(len(platform_selection.getAllPlatformNames("searchfy"))) + " platforms."
info += "\n\t- domainfy --> a tool to check the existence of a given domain in up to " + str(domainfy.getNumberTLD()) + " different TLD."
info += "\n\t- phonefy --> a tool that checks if a phone number has been linked to spam practices in " + str(len(platform_selection.getAllPlatformNames("phonefy"))) + " platforms."
info += "\n\t- entify --> a util to look for regular expressions using " + str(len(regexp_selection.getAllRegexpNames())) + " patterns."
info += "\n"
# Appending the self.info data to the headers...
intro += info
# Defining the prompt
prompt = general.emphasis('osrf > ')
ruler = '='
def do_info(self, line):
"""
Command that shows again the general information about the application.
"""
configInfo = "\n Additional configuration files:"
configInfo += "\n -------------------------------"
configInfo += "\n You will be able to find more configuration options in the following files in your system. The relevant paths are the ones that follows:"
# Get the configuration folders in each system
paths = configuration.getConfigPath()
configInfo += "\n\t- '" + os.path.join(paths["appPath"], "accounts.cfg") + "' -> Configuration details about the login credentials already configured in the framework."
configInfo += "\n\t- '" + os.path.join(paths["appPath"], "api_keys.cfg") + "' -> Configuration details about the API credentials already configured."
configInfo += "\n\t- '" + os.path.join(paths["appPath"], "browser.cfg") + "' -> Connection configuration about how the browsers will be connected."
configInfo += "\n\t- '" + os.path.join(paths["appPath"], "general.cfg") + "' -> General configuration of the different utils containing the default options."
configInfo += "\n\t- '" + paths["appPathDefaults"] + "/' -> Directory containing default files as a backup."
configInfo += "\n\t- '" + paths["appPathPlugins"] + "/' -> Directory containing the details of the user defined plugins."
configInfo += "\n\t- '" + paths["appPathPatterns"] + "/' -> Directory containing the user-defined patterns for entify.py."
configInfo += "\n\t- '" + paths["appPathWrappers"] + "/' -> Directory containing the user-defined wrappers for usufy, searchfy and phonefy platforms."
configInfo += "\n"
print(general.title(self.info) + general.info(configInfo))
def do_use(self, line):
"""
This command will define which of the framework's utilities will be loaded.
The available options are the following:
- domainfy
- entify
- mailfy
- phonefy
- searchfy
- usufy
For example, type 'use usufy' to load the usufy util. You can always use
the <TAB> to be helped using the autocomplete options.
"""
if line not in UTILS:
print(general.warning("[!] Util is not correct. Try 'help use' to check the available options."))
return False
elif line == "domainfy":
OSRFConsoleDomainfy().cmdloop()
elif line == "entify":
OSRFConsoleEntify().cmdloop()
elif line == "mailfy":
OSRFConsoleMailfy().cmdloop()
elif line == "phonefy":
OSRFConsolePhonefy().cmdloop()
elif line == "searchfy":
OSRFConsoleSearchfy().cmdloop()
elif line == "usufy":
OSRFConsoleUsufy().cmdloop()
else:
print(general.warning("[!] Not implemented yet. Try 'help use' to check the available options."))
def complete_use(self, text, line, begidx, endidx):
if not text:
completions = UTILS
else:
completions = [ f
for f in UTILS
if f.startswith(text.lower())
]
return completions
def do_exit(self, line):
"""
This command will exit osrfconsole normally.
"""
print(info("Exiting..."))
sys.exit()
# Writing the results onto a file
if args.output_file != None:
with open(output_file, "w") as oF:
oF.write(json.dumps(results, indent=2) )
if __name__ == "__main__":
parser = argparse.ArgumentParser(description='searchfy.py - Piece of software that performs a query on the platforms in OSRFramework.', prog='searchfy.py', epilog='Check the README.md file for further details on the usage of this program.', add_help=False)
parser._optionals.title = "Input options (one required)"
# Defining the mutually exclusive group for the main options
general = parser.add_mutually_exclusive_group(required=True)
# Adding the main options
general.add_argument('--license', required=False, action='store_true', default=False, help='shows the GPLv3+ license and exists.')
general.add_argument('-q', '--queries', metavar='<searches>', nargs='+', action='store', help = 'the list of queries to be performed).')
listAll = platform_selection.getAllPlatformNames("searchfy")
# Configuring the processing options
groupProcessing = parser.add_argument_group('Processing arguments', 'Configuring the way in which searchfy will process the identified profiles.')
#groupProcessing.add_argument('-L', '--logfolder', metavar='<path_to_log_folder', required=False, default = './logs', action='store', help='path to the log folder. If none was provided, ./logs is assumed.')
groupProcessing.add_argument('-o', '--output_file', metavar='<path_to_output_file>', action='store', help='path to the output file where the results will be stored in json format.', required=False)
groupProcessing.add_argument('-p', '--platforms', metavar='<platform>', choices=listAll, nargs='+', required=False, default =['all'] ,action='store', help='select the platforms where you want to perform the search amongst the following: ' + str(listAll) + '. More than one option can be selected.')
groupProcessing.add_argument('--quiet', required=False, action='store_true', default=False, help='tells the program not to show anything.')
# About options
groupAbout = parser.add_argument_group('About arguments', 'Showing additional information about this program.')
groupAbout.add_argument('-h', '--help', action='help', help='shows this help and exists.')
#groupAbout.add_argument('-v', '--verbose', metavar='<verbosity>', choices=[0, 1, 2], required=False, action='store', default=1, help='select the verbosity level: 0 - none; 1 - normal (default); 2 - debug.', type=int)
groupAbout.add_argument('--version', action='version', version='%(prog)s ' +__version__, help='shows the version of the program and exists.')
args = parser.parse_args()
