def getParser(): parser = argparse.ArgumentParser(description='entify.py - entify.py is a program designed to extract using regular expressions all the entities from the files on a given folder. This software also provides an interface to look for these entities in any given text.', prog='entify.py', epilog="Check the README.md file for further details on the usage of this program or follow us on Twitter in <http://twitter.com/i3visio>.", add_help=False) parser._optionals.title = "Input options (one required)" # Adding the main options # Defining the mutually exclusive group for the main options groupMainOptions = parser.add_mutually_exclusive_group(required=True) listAll = regexp_selection.getAllRegexpNames() groupMainOptions.add_argument('-r', '--regexp', metavar='<name>', choices=listAll, action='store', nargs='+', help='select the regular expressions to be looked for amongst the following: ' + str(listAll)) groupMainOptions.add_argument('-R', '--new_regexp', metavar='<regular_expression>', action='store', help='add a new regular expression, for example, for testing purposes.') # Adding the main options # Defining the mutually exclusive group for the main options groupInput = parser.add_mutually_exclusive_group(required=True) groupInput.add_argument('-i', '--input_folder', metavar='<path_to_input_folder>', default=None, action='store', help='path to the folder to analyse.') groupInput.add_argument('-w', '--web', metavar='<url>', action='store', default=None, help='URI to be recovered and analysed.') # adding the option groupProcessing = parser.add_argument_group('Processing arguments', 'Configuring the processing parameters.') groupProcessing.add_argument('-e', '--extension', metavar='<sum_ext>', nargs='+', choices=['csv', 'gml', 'json', 'mtz', 'ods', 'png', 'txt', 'xls', 'xlsx' ], required=False, default = ['csv'], action='store', help='output extension for the summary files. Default: xls.') groupProcessing.add_argument('-o', '--output_folder', metavar='<path_to_output_folder>', required=False, default = './results', action='store', help='output folder for the generated documents. While if the paths does not exist, usufy.py will try to create; if this argument is not provided, usufy will NOT write any down any data. Check permissions if something goes wrong.') groupProcessing.add_argument('-v', '--verbose', metavar='<verbosity>', choices=[0, 1, 2], required=False, action='store', default=1, help='select the verbosity level: 0 - none; 1 - normal (default); 2 - debug.', type=int) # Getting a sample header for the output files groupProcessing.add_argument('-F', '--file_header', metavar='<alternative_header_file>', required=False, default = "profiles", action='store', help='Header for the output filenames to be generated. If None was provided the following will be used: profiles.<extension>.' ) groupProcessing.add_argument('-q', '--quiet', required=False, action='store_true', default=False, help='Asking the program not to show any output.') groupProcessing.add_argument('-L', '--logfolder', metavar='<path_to_log_folder', required=False, default = './logs', action='store', help='path to the log folder. If none was provided, ./logs is assumed.') groupProcessing.add_argument('--recursive', action='store_true', default=False, required=False, help='Variable to tell the system to perform a recursive search on the folder tree.') groupAbout = parser.add_argument_group('About arguments', 'Showing additional information about this program.') groupAbout.add_argument('-h', '--help', action='help', help='shows this help and exists.') groupAbout.add_argument('--version', action='version', version='%(prog)s '+" " +__version__, help='shows the version of the program and exists.') return parser
def getParser(): parser = argparse.ArgumentParser(description='entify.py - entify.py is a program designed to extract using regular expressions all the entities from the files on a given folder. This software also provides an interface to look for these entities in any given text.', prog='entify.py', epilog="Check the README.md file for further details on the usage of this program or follow us on Twitter in <http://twitter.com/i3visio>.", add_help=False) parser._optionals.title = "Input options (one required)" # Adding the main options # Defining the mutually exclusive group for the main options groupMainOptions = parser.add_mutually_exclusive_group(required=True) listAll = regexp_selection.getAllRegexpNames() groupMainOptions.add_argument('-r', '--regexp', metavar='<name>', choices=listAll, action='store', nargs='+', help='select the regular expressions to be looked for amongst the following: ' + str(listAll)) groupMainOptions.add_argument('-R', '--new_regexp', metavar='<regular_expression>', action='store', help='add a new regular expression, for example, for testing purposes.') # Adding the main options # Defining the mutually exclusive group for the main options groupInput = parser.add_mutually_exclusive_group(required=True) groupInput.add_argument('-i', '--input_folder', metavar='<path_to_input_folder>', default=None, action='store', help='path to the folder to analyse.') groupInput.add_argument('-w', '--web', metavar='<url>', action='store', default=None, help='URI to be recovered and analysed.') # adding the option groupProcessing = parser.add_argument_group('Processing arguments', 'Configuring the processing parameters.') groupProcessing.add_argument('-e', '--extension', metavar='<sum_ext>', nargs='+', choices=['csv', 'gml', 'json', 'mtz', 'ods', 'png', 'txt', 'xls', 'xlsx' ], required=False, default = ['xls'], action='store', help='output extension for the summary files. Default: xls.') groupProcessing.add_argument('-o', '--output_folder', metavar='<path_to_output_folder>', required=False, default = './results', action='store', help='output folder for the generated documents. While if the paths does not exist, usufy.py will try to create; if this argument is not provided, usufy will NOT write any down any data. Check permissions if something goes wrong.') groupProcessing.add_argument('-v', '--verbose', metavar='<verbosity>', choices=[0, 1, 2], required=False, action='store', default=1, help='select the verbosity level: 0 - none; 1 - normal (default); 2 - debug.', type=int) # Getting a sample header for the output files groupProcessing.add_argument('-F', '--file_header', metavar='<alternative_header_file>', required=False, default = "profiles", action='store', help='Header for the output filenames to be generated. If None was provided the following will be used: profiles.<extension>.' ) groupProcessing.add_argument('-q', '--quiet', required=False, action='store_true', default=False, help='Asking the program not to show any output.') groupProcessing.add_argument('-L', '--logfolder', metavar='<path_to_log_folder', required=False, default = './logs', action='store', help='path to the log folder. If none was provided, ./logs is assumed.') groupProcessing.add_argument('--recursive', action='store_true', default=False, required=False, help='Variable to tell the system to perform a recursive search on the folder tree.') groupAbout = parser.add_argument_group('About arguments', 'Showing additional information about this program.') groupAbout.add_argument('-h', '--help', action='help', help='shows this help and exists.') groupAbout.add_argument('--version', action='version', version='%(prog)s '+" " +__version__, help='shows the version of the program and exists.') return parser
class OSRFConsoleMain(cmd.Cmd): """ OSRFramework console application to control the different framework utils. Type 'help' to find the commands. """ DISCLAIMER = '''\tOSRFConsole ''' + __version__ + ''' - Copyright (C) F. Brezo and Y. Rubio (i3visio) 2016-2017 This program comes with ABSOLUTELY NO WARRANTY. This is free software, and you are welcome to redistribute it under certain conditions. For additional info, visit to <http://www.gnu.org/licenses/gpl-3.0.txt>.''' intro = banner.text + "\n" + DISCLAIMER info = "\n General information" info += "\n ===================" info += "\n OSRFramework stands for Open Sources Research Framework. It includes a set of tools that help the analyst in the task of user profiling making use of different OSINT tools. To get additional information about the available commands type 'help'." info += "\n" info += "\n Modules available:" info += "\n ------------------" info += "\n\t- usufy --> the Jewel of the Chrown. A tool that verifies if a username exists in " + str(len(platform_selection.getAllPlatformNames("usufy"))) + " platforms." info += "\n\t- mailfy --> a tool to check if a username has been registered in up to " + str(len(mailfy.EMAIL_DOMAINS )) + " email providers." info += "\n\t- searchfy --> a tool to look for profiles using full names and other info in " + str(len(platform_selection.getAllPlatformNames("searchfy"))) + " platforms." info += "\n\t- domainfy --> a tool to check the existence of a given domain in up to " + str(domainfy.getNumberTLD()) + " different TLD." info += "\n\t- phonefy --> a tool that checks if a phone number has been linked to spam practices in " + str(len(platform_selection.getAllPlatformNames("phonefy"))) + " platforms." info += "\n\t- entify --> a util to look for regular expressions using " + str(len(regexp_selection.getAllRegexpNames())) + " patterns." info += "\n" # Appending the self.info data to the headers... intro += info # Defining the prompt prompt = general.emphasis('osrf > ') ruler = '=' def do_info(self, line): """ Command that shows again the general information about the application. """ configInfo = "\n Additional configuration files:" configInfo += "\n -------------------------------" configInfo += "\n You will be able to find more configuration options in the following files in your system. The relevant paths are the ones that follows:" # Get the configuration folders in each system paths = configuration.getConfigPath() configInfo += "\n\t- '" + os.path.join(paths["appPath"], "accounts.cfg") + "' -> Configuration details about the login credentials already configured in the framework." configInfo += "\n\t- '" + os.path.join(paths["appPath"], "api_keys.cfg") + "' -> Configuration details about the API credentials already configured." configInfo += "\n\t- '" + os.path.join(paths["appPath"], "browser.cfg") + "' -> Connection configuration about how the browsers will be connected." configInfo += "\n\t- '" + os.path.join(paths["appPath"], "general.cfg") + "' -> General configuration of the different utils containing the default options." configInfo += "\n\t- '" + paths["appPathDefaults"] + "/' -> Directory containing default files as a backup." configInfo += "\n\t- '" + paths["appPathPlugins"] + "/' -> Directory containing the details of the user defined plugins." configInfo += "\n\t- '" + paths["appPathPatterns"] + "/' -> Directory containing the user-defined patterns for entify.py." configInfo += "\n\t- '" + paths["appPathWrappers"] + "/' -> Directory containing the user-defined wrappers for usufy, searchfy and phonefy platforms." configInfo += "\n" print(general.title(self.info) + general.info(configInfo)) def do_use(self, line): """ This command will define which of the framework's utilities will be loaded. The available options are the following: - domainfy - entify - mailfy - phonefy - searchfy - usufy For example, type 'use usufy' to load the usufy util. You can always use the <TAB> to be helped using the autocomplete options. """ if line not in UTILS: print(general.warning("[!] Util is not correct. Try 'help use' to check the available options.")) return False elif line == "domainfy": OSRFConsoleDomainfy().cmdloop() elif line == "entify": OSRFConsoleEntify().cmdloop() elif line == "mailfy": OSRFConsoleMailfy().cmdloop() elif line == "phonefy": OSRFConsolePhonefy().cmdloop() elif line == "searchfy": OSRFConsoleSearchfy().cmdloop() elif line == "usufy": OSRFConsoleUsufy().cmdloop() else: print(general.warning("[!] Not implemented yet. Try 'help use' to check the available options.")) def complete_use(self, text, line, begidx, endidx): if not text: completions = UTILS else: completions = [ f for f in UTILS if f.startswith(text.lower()) ] return completions def do_exit(self, line): """ This command will exit osrfconsole normally. """ print(info("Exiting...")) sys.exit()
class OSRFConsoleEntify(OSRFConsoleUtil): """Class that controls an interactive entify program.""" # Setting up the name of the module UNAME = "entify.py" intro = "" # Defining the prompt prompt = 'osrf (' + UNAME.split('.')[0] + ') > ' # Defining the character to create hyphens ruler = '-' DEFAULT_VALUES = configuration.returnListOfConfigurationValues("entify") # Defining the configuration for this module CONFIG = {} CONFIG["URL"] = { "DESCRIPTION" : "The URL to be checked.", "CURRENT_VALUE" : None, "DEFAULT_VALUE" : None, "REQUIRED" : True, "OPTIONS" : [] } CONFIG["REGEXP"] = { "DESCRIPTION" : "The regular expressions to be checked.", "CURRENT_VALUE" : "all", "DEFAULT_VALUE" : "all", "REQUIRED" : False, "OPTIONS" : regexp_selection.getAllRegexpNames(), } CONFIG["OUTPUT"] = { "DESCRIPTION" : "The path to the output folder where the files will be created.", "CURRENT_VALUE" : DEFAULT_VALUES["output_folder"], "DEFAULT_VALUE" : DEFAULT_VALUES["output_folder"], "REQUIRED" : False, "OPTIONS" : [] } CONFIG["EXTENSION"] = { "DESCRIPTION" : "The default extension of the files to be written.", "CURRENT_VALUE" : DEFAULT_VALUES["extension"], "DEFAULT_VALUE" : DEFAULT_VALUES["extension"], "REQUIRED" : False, "OPTIONS" : ['csv', 'xls', 'xlsx', 'json', 'gml'] } def _getParams(self): """ Function that creates the array with the params that will work with this function""" # Creating the parameters as if they were created using the command line params = [ "-u", self.CONFIG["URL"]["CURRENT_VALUE"], "-r" ] + self.CONFIG["REGEXP"]["CURRENT_VALUE"].split() + [ "-o", self.CONFIG["OUTPUT"]["CURRENT_VALUE"], "-e" ] + self.CONFIG["EXTENSION"]["CURRENT_VALUE"].split() return params def do_run(self, line): """Running the current application. This method should be redefined for each util.""" print # Checking if all the required parameters have been set if self._checkIfRequiredAreSet(): print(general.info("Collecting the options set by the user...")) # Getting the parser... parser = entify.getParser() # Generating the parameters params = self._getParams() args = parser.parse_args(params) print(general.info("Launching " + self.UNAME + " with the following parameters: ") + general.emphashis(str(params))) try: usufy.main(args) except Exception as e: print(gemeral.error("[!] ERROR. Something happened when launching the utility. Type 'show options' to check the parameters. ")) print(general.error("Traceback: " + str(e))) else: print(general.error("[!] ERROR. There are required parameters which have not been set.")) self.do_show("options") print(general.success("Execution ended successfully."))
oF.write(resultsToCSV(results))""" if "json" in args.extension: with open(os.path.join(args.output_folder, "results.json"), "w") as oF: oF.write(general.dictToJson(results)) return results if __name__ == "__main__": parser = argparse.ArgumentParser(description='entify.py - entify.py is a program designed to extract using regular expressions all the entities from the files on a given folder. This software also provides an interface to look for these entities in any given text.', prog='entify.py', epilog="", add_help=False) parser._optionals.title = "Input options (one required)" # Adding the main options # Defining the mutually exclusive group for the main options groupMainOptions = parser.add_mutually_exclusive_group(required=True) listAll = regexp_selection.getAllRegexpNames() groupMainOptions.add_argument('-r', '--regexp', metavar='<name>', choices=listAll, action='store', nargs='+', help='select the regular expressions to be looked for amongst the following: ' + str(listAll)) groupMainOptions.add_argument('-R', '--new_regexp', metavar='<regular_expression>', action='store', help='add a new regular expression, for example, for testing purposes.') # Adding the main options # Defining the mutually exclusive group for the main options groupInput = parser.add_mutually_exclusive_group(required=True) groupInput.add_argument('-i', '--input_folder', metavar='<path_to_input_folder>', default=None, action='store', help='path to the folder to analyse.') groupInput.add_argument('-w', '--web', metavar='<url>', action='store', default=None, help='URI to be recovered and analysed.') # adding the option groupProcessing = parser.add_argument_group('Processing arguments', 'Configuring the processing parameters.') groupProcessing.add_argument('-e', '--extension', metavar='<sum_ext>', nargs='+', choices=['json'], required=False, default = ['json'], action='store', help='output extension for the summary files (if not provided, json is assumed).') groupProcessing.add_argument('-o', '--output_folder', metavar='<path_to_output_folder>', action='store', help='path to the output folder where the results will be stored.', required=False) groupProcessing.add_argument('-v', '--verbose', metavar='<verbosity>', choices=[0, 1, 2], required=False, action='store', default=1, help='select the verbosity level: 0 - none; 1 - normal (default); 2 - debug.', type=int) groupProcessing.add_argument('-q', '--quiet', required=False, action='store_true', default=False, help='Asking the program not to show any output.')
class OSRFConsoleEntify(OSRFConsoleUtil): """ Class that controls an interactive entify program. """ # Setting up the name of the module UNAME = "entify" intro = """ Loading """ + UNAME + """...""" description = """ A tool to look for known regular expressions in remote or local resources.""" # Defining the prompt prompt = general.emphasis('\nosrf (' + UNAME + ') > ') # Defining the character to create hyphens ruler = '-' # Defining the configuration for this module CONFIG = {} DEFAULT_VALUES = configuration.returnListOfConfigurationValues("entify") # Defining the configuration for this module CONFIG = {} CONFIG["URL"] = { "DESCRIPTION" : "The URL to be checked.", "CURRENT_VALUE" : None, "DEFAULT_VALUE" : None, "REQUIRED" : True, "OPTIONS" : [] } CONFIG["REGEXP"] = { "DESCRIPTION" : "The regular expressions to be checked.", "CURRENT_VALUE" : "all", "DEFAULT_VALUE" : "all", "REQUIRED" : False, "OPTIONS" : regexp_selection.getAllRegexpNames(), } CONFIG["OUTPUT"] = { "DESCRIPTION" : "The path to the output folder where the files will be created.", "CURRENT_VALUE" : DEFAULT_VALUES["output_folder"], "DEFAULT_VALUE" : DEFAULT_VALUES["output_folder"], "REQUIRED" : False, "OPTIONS" : [] } CONFIG["EXTENSION"] = { "DESCRIPTION" : "The default extension of the files to be written.", "CURRENT_VALUE" : DEFAULT_VALUES["extension"], "DEFAULT_VALUE" : DEFAULT_VALUES["extension"], "REQUIRED" : False, "OPTIONS" : ['csv', 'xls', 'xlsx', 'json', 'gml'] } def _getParams(self): """ Function that creates the array with the params of this function Returns: -------- list: A list of the params that can be used """ # Creating the parameters as if they were created using the command line params = [ "-u", self.CONFIG["URL"]["CURRENT_VALUE"], "-r" ] + self.CONFIG["REGEXP"]["CURRENT_VALUE"] + [ "-o", self.CONFIG["OUTPUT"]["CURRENT_VALUE"], "-e" ] + self.CONFIG["EXTENSION"]["CURRENT_VALUE"] return params def do_run(self, line): """ Command that send the order to the framework to launch this util Args: ----- line: The string of the line typed. """ # Checking if all the required parameters have been set if self._checkIfRequiredAreSet(): print(general.info("\nCollecting the options set by the user...\n")) # Getting the parser... parser = entify.getParser() # Generating the parameters params = self._getParams() args = parser.parse_args(params) print(general.info("\nLaunching " + self.UNAME + " with the following parameters:") + general.emphasis("\t$ " + self.UNAME + " " + utils.listToString(params) + "\n")) try: entify.main(args) except Exception as e: print(general.error("\n[!!] ERROR. Something happened when launching the utility. Type 'show options' to check the parameters.\n")) print(general.error("Traceback: " + str(e))) else: print(general.error("\n[!!] ERROR. There are required parameters which have not been set.")) self.do_show("options") print(general.success("Execution ended successfully."))