def getParser():
parser = argparse.ArgumentParser(description='entify.py - entify.py is a program designed to extract using regular expressions all the entities from the files on a given folder. This software also provides an interface to look for these entities in any given text.', prog='entify.py', epilog="Check the README.md file for further details on the usage of this program or follow us on Twitter in <http://twitter.com/i3visio>.", add_help=False)
parser._optionals.title = "Input options (one required)"
# Adding the main options
# Defining the mutually exclusive group for the main options
groupMainOptions = parser.add_mutually_exclusive_group(required=True)
listAll = regexp_selection.getAllRegexpNames()
groupMainOptions.add_argument('-r', '--regexp', metavar='<name>', choices=listAll, action='store', nargs='+', help='select the regular expressions to be looked for amongst the following: ' + str(listAll))
groupMainOptions.add_argument('-R', '--new_regexp', metavar='<regular_expression>', action='store', help='add a new regular expression, for example, for testing purposes.')
# Adding the main options
# Defining the mutually exclusive group for the main options
groupInput = parser.add_mutually_exclusive_group(required=True)
groupInput.add_argument('-i', '--input_folder', metavar='<path_to_input_folder>', default=None, action='store', help='path to the folder to analyse.')
groupInput.add_argument('-w', '--web', metavar='<url>', action='store', default=None, help='URI to be recovered and analysed.')
# adding the option
groupProcessing = parser.add_argument_group('Processing arguments', 'Configuring the processing parameters.')
groupProcessing.add_argument('-e', '--extension', metavar='<sum_ext>', nargs='+', choices=['csv', 'gml', 'json', 'mtz', 'ods', 'png', 'txt', 'xls', 'xlsx' ], required=False, default = ['csv'], action='store', help='output extension for the summary files. Default: xls.')
groupProcessing.add_argument('-o', '--output_folder', metavar='<path_to_output_folder>', required=False, default = './results', action='store', help='output folder for the generated documents. While if the paths does not exist, usufy.py will try to create; if this argument is not provided, usufy will NOT write any down any data. Check permissions if something goes wrong.')
groupProcessing.add_argument('-v', '--verbose', metavar='<verbosity>', choices=[0, 1, 2], required=False, action='store', default=1, help='select the verbosity level: 0 - none; 1 - normal (default); 2 - debug.', type=int)
# Getting a sample header for the output files
groupProcessing.add_argument('-F', '--file_header', metavar='<alternative_header_file>', required=False, default = "profiles", action='store', help='Header for the output filenames to be generated. If None was provided the following will be used: profiles.<extension>.' )
groupProcessing.add_argument('-q', '--quiet', required=False, action='store_true', default=False, help='Asking the program not to show any output.')
groupProcessing.add_argument('-L', '--logfolder', metavar='<path_to_log_folder', required=False, default = './logs', action='store', help='path to the log folder. If none was provided, ./logs is assumed.')
groupProcessing.add_argument('--recursive', action='store_true', default=False, required=False, help='Variable to tell the system to perform a recursive search on the folder tree.')
groupAbout = parser.add_argument_group('About arguments', 'Showing additional information about this program.')
groupAbout.add_argument('-h', '--help', action='help', help='shows this help and exists.')
groupAbout.add_argument('--version', action='version', version='%(prog)s '+" " +__version__, help='shows the version of the program and exists.')
return parser
def getParser():
parser = argparse.ArgumentParser(description='entify.py - entify.py is a program designed to extract using regular expressions all the entities from the files on a given folder. This software also provides an interface to look for these entities in any given text.', prog='entify.py', epilog="Check the README.md file for further details on the usage of this program or follow us on Twitter in <http://twitter.com/i3visio>.", add_help=False)
parser._optionals.title = "Input options (one required)"
# Adding the main options
# Defining the mutually exclusive group for the main options
groupMainOptions = parser.add_mutually_exclusive_group(required=True)
listAll = regexp_selection.getAllRegexpNames()
groupMainOptions.add_argument('-r', '--regexp', metavar='<name>', choices=listAll, action='store', nargs='+', help='select the regular expressions to be looked for amongst the following: ' + str(listAll))
groupMainOptions.add_argument('-R', '--new_regexp', metavar='<regular_expression>', action='store', help='add a new regular expression, for example, for testing purposes.')
# Adding the main options
# Defining the mutually exclusive group for the main options
groupInput = parser.add_mutually_exclusive_group(required=True)
groupInput.add_argument('-i', '--input_folder', metavar='<path_to_input_folder>', default=None, action='store', help='path to the folder to analyse.')
groupInput.add_argument('-w', '--web', metavar='<url>', action='store', default=None, help='URI to be recovered and analysed.')
# adding the option
groupProcessing = parser.add_argument_group('Processing arguments', 'Configuring the processing parameters.')
groupProcessing.add_argument('-e', '--extension', metavar='<sum_ext>', nargs='+', choices=['csv', 'gml', 'json', 'mtz', 'ods', 'png', 'txt', 'xls', 'xlsx' ], required=False, default = ['xls'], action='store', help='output extension for the summary files. Default: xls.')
groupProcessing.add_argument('-o', '--output_folder', metavar='<path_to_output_folder>', required=False, default = './results', action='store', help='output folder for the generated documents. While if the paths does not exist, usufy.py will try to create; if this argument is not provided, usufy will NOT write any down any data. Check permissions if something goes wrong.')
groupProcessing.add_argument('-v', '--verbose', metavar='<verbosity>', choices=[0, 1, 2], required=False, action='store', default=1, help='select the verbosity level: 0 - none; 1 - normal (default); 2 - debug.', type=int)
# Getting a sample header for the output files
groupProcessing.add_argument('-F', '--file_header', metavar='<alternative_header_file>', required=False, default = "profiles", action='store', help='Header for the output filenames to be generated. If None was provided the following will be used: profiles.<extension>.' )
groupProcessing.add_argument('-q', '--quiet', required=False, action='store_true', default=False, help='Asking the program not to show any output.')
groupProcessing.add_argument('-L', '--logfolder', metavar='<path_to_log_folder', required=False, default = './logs', action='store', help='path to the log folder. If none was provided, ./logs is assumed.')
groupProcessing.add_argument('--recursive', action='store_true', default=False, required=False, help='Variable to tell the system to perform a recursive search on the folder tree.')
groupAbout = parser.add_argument_group('About arguments', 'Showing additional information about this program.')
groupAbout.add_argument('-h', '--help', action='help', help='shows this help and exists.')
groupAbout.add_argument('--version', action='version', version='%(prog)s '+" " +__version__, help='shows the version of the program and exists.')
return parser
class OSRFConsoleMain(cmd.Cmd):
"""
OSRFramework console application to control the different framework utils.
Type 'help' to find the commands.
"""
DISCLAIMER = '''\tOSRFConsole ''' + __version__ + ''' - Copyright (C) F. Brezo and Y. Rubio (i3visio) 2016-2017
This program comes with ABSOLUTELY NO WARRANTY.
This is free software, and you are welcome to redistribute it under certain conditions. For additional info, visit to <http://www.gnu.org/licenses/gpl-3.0.txt>.'''
intro = banner.text + "\n" + DISCLAIMER
info = "\n General information"
info += "\n ==================="
info += "\n OSRFramework stands for Open Sources Research Framework. It includes a set of tools that help the analyst in the task of user profiling making use of different OSINT tools. To get additional information about the available commands type 'help'."
info += "\n"
info += "\n Modules available:"
info += "\n ------------------"
info += "\n\t- usufy --> the Jewel of the Chrown. A tool that verifies if a username exists in " + str(len(platform_selection.getAllPlatformNames("usufy"))) + " platforms."
info += "\n\t- mailfy --> a tool to check if a username has been registered in up to " + str(len(mailfy.EMAIL_DOMAINS )) + " email providers."
info += "\n\t- searchfy --> a tool to look for profiles using full names and other info in " + str(len(platform_selection.getAllPlatformNames("searchfy"))) + " platforms."
info += "\n\t- domainfy --> a tool to check the existence of a given domain in up to " + str(domainfy.getNumberTLD()) + " different TLD."
info += "\n\t- phonefy --> a tool that checks if a phone number has been linked to spam practices in " + str(len(platform_selection.getAllPlatformNames("phonefy"))) + " platforms."
info += "\n\t- entify --> a util to look for regular expressions using " + str(len(regexp_selection.getAllRegexpNames())) + " patterns."
info += "\n"
# Appending the self.info data to the headers...
intro += info
# Defining the prompt
prompt = general.emphasis('osrf > ')
ruler = '='
def do_info(self, line):
"""
Command that shows again the general information about the application.
"""
configInfo = "\n Additional configuration files:"
configInfo += "\n -------------------------------"
configInfo += "\n You will be able to find more configuration options in the following files in your system. The relevant paths are the ones that follows:"
# Get the configuration folders in each system
paths = configuration.getConfigPath()
configInfo += "\n\t- '" + os.path.join(paths["appPath"], "accounts.cfg") + "' -> Configuration details about the login credentials already configured in the framework."
configInfo += "\n\t- '" + os.path.join(paths["appPath"], "api_keys.cfg") + "' -> Configuration details about the API credentials already configured."
configInfo += "\n\t- '" + os.path.join(paths["appPath"], "browser.cfg") + "' -> Connection configuration about how the browsers will be connected."
configInfo += "\n\t- '" + os.path.join(paths["appPath"], "general.cfg") + "' -> General configuration of the different utils containing the default options."
configInfo += "\n\t- '" + paths["appPathDefaults"] + "/' -> Directory containing default files as a backup."
configInfo += "\n\t- '" + paths["appPathPlugins"] + "/' -> Directory containing the details of the user defined plugins."
configInfo += "\n\t- '" + paths["appPathPatterns"] + "/' -> Directory containing the user-defined patterns for entify.py."
configInfo += "\n\t- '" + paths["appPathWrappers"] + "/' -> Directory containing the user-defined wrappers for usufy, searchfy and phonefy platforms."
configInfo += "\n"
print(general.title(self.info) + general.info(configInfo))
def do_use(self, line):
"""
This command will define which of the framework's utilities will be loaded.
The available options are the following:
- domainfy
- entify
- mailfy
- phonefy
- searchfy
- usufy
For example, type 'use usufy' to load the usufy util. You can always use
the <TAB> to be helped using the autocomplete options.
"""
if line not in UTILS:
print(general.warning("[!] Util is not correct. Try 'help use' to check the available options."))
return False
elif line == "domainfy":
OSRFConsoleDomainfy().cmdloop()
elif line == "entify":
OSRFConsoleEntify().cmdloop()
elif line == "mailfy":
OSRFConsoleMailfy().cmdloop()
elif line == "phonefy":
OSRFConsolePhonefy().cmdloop()
elif line == "searchfy":
OSRFConsoleSearchfy().cmdloop()
elif line == "usufy":
OSRFConsoleUsufy().cmdloop()
else:
print(general.warning("[!] Not implemented yet. Try 'help use' to check the available options."))
def complete_use(self, text, line, begidx, endidx):
if not text:
completions = UTILS
else:
completions = [ f
for f in UTILS
if f.startswith(text.lower())
]
return completions
def do_exit(self, line):
"""
This command will exit osrfconsole normally.
"""
print(info("Exiting..."))
sys.exit()
class OSRFConsoleEntify(OSRFConsoleUtil):
"""Class that controls an interactive entify program."""
# Setting up the name of the module
UNAME = "entify.py"
intro = ""
# Defining the prompt
prompt = 'osrf (' + UNAME.split('.')[0] + ') > '
# Defining the character to create hyphens
ruler = '-'
DEFAULT_VALUES = configuration.returnListOfConfigurationValues("entify")
# Defining the configuration for this module
CONFIG = {}
CONFIG["URL"] = {
"DESCRIPTION" : "The URL to be checked.",
"CURRENT_VALUE" : None,
"DEFAULT_VALUE" : None,
"REQUIRED" : True,
"OPTIONS" : []
}
CONFIG["REGEXP"] = {
"DESCRIPTION" : "The regular expressions to be checked.",
"CURRENT_VALUE" : "all",
"DEFAULT_VALUE" : "all",
"REQUIRED" : False,
"OPTIONS" : regexp_selection.getAllRegexpNames(),
}
CONFIG["OUTPUT"] = {
"DESCRIPTION" : "The path to the output folder where the files will be created.",
"CURRENT_VALUE" : DEFAULT_VALUES["output_folder"],
"DEFAULT_VALUE" : DEFAULT_VALUES["output_folder"],
"REQUIRED" : False,
"OPTIONS" : []
}
CONFIG["EXTENSION"] = {
"DESCRIPTION" : "The default extension of the files to be written.",
"CURRENT_VALUE" : DEFAULT_VALUES["extension"],
"DEFAULT_VALUE" : DEFAULT_VALUES["extension"],
"REQUIRED" : False,
"OPTIONS" : ['csv', 'xls', 'xlsx', 'json', 'gml']
}
def _getParams(self):
""" Function that creates the array with the params that will work with this function"""
# Creating the parameters as if they were created using the command line
params = [
"-u", self.CONFIG["URL"]["CURRENT_VALUE"],
"-r" ] + self.CONFIG["REGEXP"]["CURRENT_VALUE"].split() + [
"-o", self.CONFIG["OUTPUT"]["CURRENT_VALUE"],
"-e" ] + self.CONFIG["EXTENSION"]["CURRENT_VALUE"].split()
return params
def do_run(self, line):
"""Running the current application. This method should be redefined for each util."""
print
# Checking if all the required parameters have been set
if self._checkIfRequiredAreSet():
print(general.info("Collecting the options set by the user..."))
# Getting the parser...
parser = entify.getParser()
# Generating the parameters
params = self._getParams()
args = parser.parse_args(params)
print(general.info("Launching " + self.UNAME + " with the following parameters: ") + general.emphashis(str(params)))
try:
usufy.main(args)
except Exception as e:
print(gemeral.error("[!] ERROR. Something happened when launching the utility. Type 'show options' to check the parameters. "))
print(general.error("Traceback: " + str(e)))
else:
print(general.error("[!] ERROR. There are required parameters which have not been set."))
self.do_show("options")
print(general.success("Execution ended successfully."))
oF.write(resultsToCSV(results))"""
if "json" in args.extension:
with open(os.path.join(args.output_folder, "results.json"), "w") as oF:
oF.write(general.dictToJson(results))
return results
if __name__ == "__main__":
parser = argparse.ArgumentParser(description='entify.py - entify.py is a program designed to extract using regular expressions all the entities from the files on a given folder. This software also provides an interface to look for these entities in any given text.', prog='entify.py', epilog="", add_help=False)
parser._optionals.title = "Input options (one required)"
# Adding the main options
# Defining the mutually exclusive group for the main options
groupMainOptions = parser.add_mutually_exclusive_group(required=True)
listAll = regexp_selection.getAllRegexpNames()
groupMainOptions.add_argument('-r', '--regexp', metavar='<name>', choices=listAll, action='store', nargs='+', help='select the regular expressions to be looked for amongst the following: ' + str(listAll))
groupMainOptions.add_argument('-R', '--new_regexp', metavar='<regular_expression>', action='store', help='add a new regular expression, for example, for testing purposes.')
# Adding the main options
# Defining the mutually exclusive group for the main options
groupInput = parser.add_mutually_exclusive_group(required=True)
groupInput.add_argument('-i', '--input_folder', metavar='<path_to_input_folder>', default=None, action='store', help='path to the folder to analyse.')
groupInput.add_argument('-w', '--web', metavar='<url>', action='store', default=None, help='URI to be recovered and analysed.')
# adding the option
groupProcessing = parser.add_argument_group('Processing arguments', 'Configuring the processing parameters.')
groupProcessing.add_argument('-e', '--extension', metavar='<sum_ext>', nargs='+', choices=['json'], required=False, default = ['json'], action='store', help='output extension for the summary files (if not provided, json is assumed).')
groupProcessing.add_argument('-o', '--output_folder', metavar='<path_to_output_folder>', action='store', help='path to the output folder where the results will be stored.', required=False)
groupProcessing.add_argument('-v', '--verbose', metavar='<verbosity>', choices=[0, 1, 2], required=False, action='store', default=1, help='select the verbosity level: 0 - none; 1 - normal (default); 2 - debug.', type=int)
groupProcessing.add_argument('-q', '--quiet', required=False, action='store_true', default=False, help='Asking the program not to show any output.')
class OSRFConsoleEntify(OSRFConsoleUtil):
"""
Class that controls an interactive entify program.
"""
# Setting up the name of the module
UNAME = "entify"
intro = """
Loading """ + UNAME + """..."""
description = """
A tool to look for known regular expressions in remote or local resources."""
# Defining the prompt
prompt = general.emphasis('\nosrf (' + UNAME + ') > ')
# Defining the character to create hyphens
ruler = '-'
# Defining the configuration for this module
CONFIG = {}
DEFAULT_VALUES = configuration.returnListOfConfigurationValues("entify")
# Defining the configuration for this module
CONFIG = {}
CONFIG["URL"] = {
"DESCRIPTION" : "The URL to be checked.",
"CURRENT_VALUE" : None,
"DEFAULT_VALUE" : None,
"REQUIRED" : True,
"OPTIONS" : []
}
CONFIG["REGEXP"] = {
"DESCRIPTION" : "The regular expressions to be checked.",
"CURRENT_VALUE" : "all",
"DEFAULT_VALUE" : "all",
"REQUIRED" : False,
"OPTIONS" : regexp_selection.getAllRegexpNames(),
}
CONFIG["OUTPUT"] = {
"DESCRIPTION" : "The path to the output folder where the files will be created.",
"CURRENT_VALUE" : DEFAULT_VALUES["output_folder"],
"DEFAULT_VALUE" : DEFAULT_VALUES["output_folder"],
"REQUIRED" : False,
"OPTIONS" : []
}
CONFIG["EXTENSION"] = {
"DESCRIPTION" : "The default extension of the files to be written.",
"CURRENT_VALUE" : DEFAULT_VALUES["extension"],
"DEFAULT_VALUE" : DEFAULT_VALUES["extension"],
"REQUIRED" : False,
"OPTIONS" : ['csv', 'xls', 'xlsx', 'json', 'gml']
}
def _getParams(self):
"""
Function that creates the array with the params of this function
Returns:
--------
list: A list of the params that can be used
"""
# Creating the parameters as if they were created using the command line
params = [
"-u", self.CONFIG["URL"]["CURRENT_VALUE"],
"-r" ] + self.CONFIG["REGEXP"]["CURRENT_VALUE"] + [
"-o", self.CONFIG["OUTPUT"]["CURRENT_VALUE"],
"-e" ] + self.CONFIG["EXTENSION"]["CURRENT_VALUE"]
return params
def do_run(self, line):
"""
Command that send the order to the framework to launch this util
Args:
-----
line: The string of the line typed.
"""
# Checking if all the required parameters have been set
if self._checkIfRequiredAreSet():
print(general.info("\nCollecting the options set by the user...\n"))
# Getting the parser...
parser = entify.getParser()
# Generating the parameters
params = self._getParams()
args = parser.parse_args(params)
print(general.info("\nLaunching " + self.UNAME + " with the following parameters:") + general.emphasis("\t$ " + self.UNAME + " " + utils.listToString(params) + "\n"))
try:
entify.main(args)
except Exception as e:
print(general.error("\n[!!] ERROR. Something happened when launching the utility. Type 'show options' to check the parameters.\n"))
print(general.error("Traceback: " + str(e)))
else:
print(general.error("\n[!!] ERROR. There are required parameters which have not been set."))
self.do_show("options")
print(general.success("Execution ended successfully."))