def returnListOfCreds(): ''' :return: A list of tuples containing in the first the name of the platform, as read from the accounts.cfg file in the application folder. E. g.: listCreds.append(("<platform>", "<username>", "<password>")) ''' listCreds = [] # If a accounts.cfg has not been found, creating it by copying from default configPath = configuration.getConfigPath("accounts.cfg") # Checking if the configuration file exists if not os.path.exists(configPath): try: # Copy the data from the default folder defaultConfigPath = configuration.getConfigPath(os.path.join("default", "accounts.cfg")) with open(configPath, "w") as oF: with open(defaultConfigPath) as iF: cont = iF.read() oF.write(cont) except Exception, e: print "WARNING. No configuration file could be found and the default file was not found either, so NO credentials have been loaded." print str(e) print return listCreds
def returnListOfCreds(): ''' :return: A list of tuples containing in the first the name of the platform, as read from the accounts.cfg file in the application folder. E. g.: listCreds.append(("<platform>", "<username>", "<password>")) ''' listCreds = [] # If a accounts.cfg has not been found, creating it by copying from default configPath = os.path.join(configuration.getConfigPath()["appPath"], "accounts.cfg") # Checking if the configuration file exists if not os.path.exists(configPath): # Copy the data from the default folder defaultConfigPath = os.path.join( configuration.getConfigPath()["appPathDefaults"], "accounts.cfg") try: with open(defaultConfigPath) as iF: cont = iF.read() with open(configPath, "w") as oF: oF.write(cont) except Exception, e: raise errors.ConfigurationFileNotFoundError( configPath, defaultConfigPath) return listCreds
def returnListOfAPIKeys(): ''' :return: A dictionary containing the API Keys stored in a dictionary depending on the information required by each platform ''' dictAPIKeys = {} # If a api_keys.cfg has not been found, creating it by copying from default configPath = os.path.join(configuration.getConfigPath()["appPath"], "api_keys.cfg") # Checking if the configuration file exists if not os.path.exists(configPath): try: # Copy the data from the default folder defaultConfigPath = os.path.join(configuration.getConfigPath()["appPathDefaults"], "api_keys.cfg") with open(configPath, "w") as oF: with open(defaultConfigPath) as iF: cont = iF.read() oF.write(cont) except Exception, e: print "WARNING. No configuration file could be found and the default file was not found either, so NO API keys have been loaded." print str(e) print return dictAPIKeys
def __init__(self): """ Recovering an instance of a new Browser. """ # Browser self.br = mechanize.Browser() # Cookie Jar self.cj = cookielib.LWPCookieJar() self.br.set_cookiejar(self.cj) # Browser options self.br.set_handle_equiv(True) self.br.set_handle_gzip(False) self.br.set_handle_redirect(True) self.br.set_handle_referer(False) self.br.set_handle_robots(False) self.br.set_handled_schemes(['http', 'https']) # Follows refresh 0 but not hangs on refresh > 0 self.br.set_handle_refresh(mechanize._http.HTTPRefreshProcessor(), max_time=1) # Defining User Agents self.userAgents = [] # Handling proxies self.proxies = {} self.timeout = 2 # Want debugging messages? #self.br.set_debug_http(True) #self.br.set_debug_redirects(True) #self.br.set_debug_responses(True) # Trying to read the configuration # -------------------------------- # If a current.cfg has not been found, creating it by copying from default configPath = configuration.getConfigPath("browser.cfg") configPath = os.path.join(configuration.getConfigPath()["appPath"], "browser.cfg") # Checking if the configuration file exists if not os.path.exists(configPath): try: # Copy the data from the default folder defaultConfigPath = os.path.join(configuration.getConfigPath()["appPathDefaults"], "browser.cfg") with open(defaultConfigPath) as iF: cont = iF.read() with open(configPath, "w") as oF: oF.write(cont) except Exception, e: print("WARNING. No configuration file could be found and the default file was not found either, so configuration will be set as default.") print(str(e)) print() # Storing configuration as default self.userAgents = ['Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/55.0.2883.87 Chrome/55.0.2883.87 Safari/537.36'] self.proxies = {} return None
def returnListOfCreds(): ''' :return: A list of tuples containing in the first the name of the platform, as read from the accounts.cfg file in the application folder. E. g.: listCreds.append(("<platform>", "<username>", "<password>")) ''' listCreds = [] # If a accounts.cfg has not been found, creating it by copying from default configPath = configuration.getConfigPath("accounts.cfg") # Checking if the configuration file exists if not os.path.exists(configPath): try: # Copy the data from the default folder defaultConfigPath = configuration.getConfigPath( os.path.join("default", "accounts.cfg")) with open(configPath, "w") as oF: with open(defaultConfigPath) as iF: cont = iF.read() oF.write(cont) except Exception, e: print "WARNING. No configuration file could be found and the default file was not found either, so NO credentials have been loaded." print str(e) print return listCreds
def __init__(self): """ Recovering an instance of a new Browser. """ # Browser self.br = mechanize.Browser() # Cookie Jar self.cj = cookielib.LWPCookieJar() self.br.set_cookiejar(self.cj) # Browser options self.br.set_handle_equiv(True) self.br.set_handle_gzip(False) self.br.set_handle_redirect(True) self.br.set_handle_referer(False) self.br.set_handle_robots(False) # Follows refresh 0 but not hangs on refresh > 0 self.br.set_handle_refresh(mechanize._http.HTTPRefreshProcessor(), max_time=1) # Want debugging messages? #self.br.set_debug_http(True) #self.br.set_debug_redirects(True) #self.br.set_debug_responses(True) # Trying to read the configuration # -------------------------------- # If a current.cfg has not been found, creating it by copying from default configPath = configuration.getConfigPath("browser.cfg") # Checking if the configuration file exists if not os.path.exists(configPath): try: # Copy the data from the default folder defaultConfigPath = configuration.getConfigPath( os.path.join("default", "browser.cfg")) with open(configPath, "w") as oF: with open(defaultConfigPath) as iF: cont = iF.read() oF.write(cont) except Exception, e: print "WARNING. No configuration file could be found and the default file was not found either, so configuration will be set as default." print str(e) print # Storing configuration as default self.userAgents = [ 'Mozilla/5.0 (Windows NT 5.1; rv:31.0) Gecko/20100101 Firefox/31.0' ] self.proxies = [] return None
def __init__(self): """ Recovering an instance of a new Browser. """ # Browser self.br = mechanize.Browser() # Cookie Jar self.cj = cookielib.LWPCookieJar() self.br.set_cookiejar(self.cj) # Browser options self.br.set_handle_equiv(True) self.br.set_handle_gzip(False) self.br.set_handle_redirect(True) self.br.set_handle_referer(False) self.br.set_handle_robots(False) # Follows refresh 0 but not hangs on refresh > 0 self.br.set_handle_refresh(mechanize._http.HTTPRefreshProcessor(), max_time=1) # Want debugging messages? #self.br.set_debug_http(True) #self.br.set_debug_redirects(True) #self.br.set_debug_responses(True) # Trying to read the configuration # -------------------------------- # If a current.cfg has not been found, creating it by copying from default configPath = configuration.getConfigPath("browser.cfg") configPath = os.path.join(configuration.getConfigPath()["appPath"], "browser.cfg") # Checking if the configuration file exists if not os.path.exists(configPath): try: # Copy the data from the default folder defaultConfigPath = os.path.join(configuration.getConfigPath()["appPathDefaults"], "browser.cfg") with open(configPath, "w") as oF: with open(defaultConfigPath) as iF: cont = iF.read() oF.write(cont) except Exception, e: print "WARNING. No configuration file could be found and the default file was not found either, so configuration will be set as default." print str(e) print # Storing configuration as default self.userAgents = ['Mozilla/5.0 (Windows NT 5.1; rv:31.0) Gecko/20100101 Firefox/31.0'] self.proxies = [] return None
def do_info(self, line): """ Command that shows again the general information about the application. """ configInfo = "\n Additional configuration files:" configInfo += "\n -------------------------------" configInfo += "\n You will be able to find more configuration options in the following files in your system. The relevant paths are the ones that follows:" # Get the configuration folders in each system paths = configuration.getConfigPath() configInfo += "\n\t- '" + os.path.join( paths["appPath"], "accounts.cfg" ) + "' -> Configuration details about the login credentials already configured in the framework." configInfo += "\n\t- '" + os.path.join( paths["appPath"], "api_keys.cfg" ) + "' -> Configuration details about the API credentials already configured." configInfo += "\n\t- '" + os.path.join( paths["appPath"], "browser.cfg" ) + "' -> Connection configuration about how the browsers will be connected." configInfo += "\n\t- '" + os.path.join( paths["appPath"], "general.cfg" ) + "' -> General configuration of the different utils containing the default options." configInfo += "\n\t- '" + paths[ "appPathDefaults"] + "/' -> Directory containing default files as a backup." configInfo += "\n\t- '" + paths[ "appPathPlugins"] + "/' -> Directory containing the details of the user defined plugins." configInfo += "\n\t- '" + paths[ "appPathPatterns"] + "/' -> Directory containing the user-defined patterns for entify.py." configInfo += "\n\t- '" + paths[ "appPathWrappers"] + "/' -> Directory containing the user-defined wrappers for usufy, searchfy and phonefy platforms." configInfo += "\n" print self.info + configInfo
def do_info(self, line): """ Command that shows again the general information about the application Args: ----- line: the string of the line typed. """ configInfo = """ """ + general.emphasis("""Additional configuration files: -------------------------------""") + """ You will be able to find more configuration options in the following files in your system. The relevant paths are the ones that follow:""" # Get the configuration folders in each system paths = configuration.getConfigPath() configInfo += """ - Configuration details about the login credentials in OSRFramework: """ + general.info(os.path.join(paths["appPath"], "accounts.cfg")) + """ - Configuration details about the API credentials already configured: """ + general.info(os.path.join(paths["appPath"], "api_keys.cfg")) + """ - Connection configuration about how the browsers will be connected: """ + general.info(os.path.join(paths["appPath"], "browser.cfg")) + """ - General default configuration of the the utils: """ + general.info(os.path.join(paths["appPath"], "general.cfg")) + """ - Directory containing default files as a backup: """ + general.info(paths["appPathDefaults"]) + """ - Directory containing the user-defined patterns for entify.py: """ + general.info(paths["appPathPatterns"]) + """ - Directory containing the user-defined wrappers for usufy platforms: """ + general.info(paths["appPathWrappers"]) print(self.info + configInfo)
def __init__(self, platform, *args, **kwargs): msg = """ [*] Warning:\t{}. Details: No valid credentials provided for '{}'. Update the configuration file at: '{}'. """.format( self.__class__.__name__, platform, os.path.join(configuration.getConfigPath()["appPath"], "accounts.cfg"), general.emphasis("-x " + platform)) OSRFrameworkException.__init__(self, general.warning(msg)) self.generic = "The credentials for some platforms where NOT provided."
def getConfiguration(configuration_file="server.cfg", package="osrframework_server", section="osrframework_server"): """ Method that recovers the configuration information osrframework_server. TODO: Grab the default file from the package data instead of storing it in the main folder. Args: ----- configuration_file: The configuration file to be read. package: The package name where the default files are stored. section: The secction's name that will be read. Returns: -------- A dictionary containing the default configuration. """ VALUES = {} # If a api_keys.cfg has not been found, creating it by copying from default configPath = os.path.join(configuration.getConfigPath()["appPath"], configuration_file) # Checking if the configuration file exists if not os.path.exists(configPath): print( warning( "[*] No configuration files found for OSRFramework Server.")) print( warning( "[*] Copying default configuration files to OSRFramework configuration folder at '" + configPath + "'.")) # Copy the data from the default folder defaultConfigPath = pkg_resources.resource_filename( package, os.path.join('config', configuration_file)) copyfile(defaultConfigPath, configPath) # Reading the configuration file config = ConfigParser.ConfigParser() config.read(configPath) # Iterating through all the sections, which contain the platforms for s in config.sections(): incomplete = False if s.lower() == section.lower(): # Iterating through parameters for (param, value) in config.items(s): if value != '': VALUES[param] = value return VALUES
def returnListOfAPIKeys(): ''' :return: A dictionary containing the API Keys stored in a dictionary depending on the information required by each platform ''' dictAPIKeys = {} # If a api_keys.cfg has not been found, creating it by copying from default configPath = configuration.getConfigPath("api_keys.cfg") # Checking if the configuration file exists if not os.path.exists(configPath): try: # Copy the data from the default folder defaultConfigPath = configuration.getConfigPath(os.path.join("default", "api_keys.cfg")) with open(configPath, "w") as oF: with open(defaultConfigPath) as iF: cont = iF.read() oF.write(cont) except Exception, e: print "WARNING. No configuration file could be found and the default file was not found either, so NO API keys have been loaded." print str(e) print return listCreds
def getAllPlatformObjects(mode = None): ''' Method that recovers ALL the list of <Platform> classes to be processed.... :param mode: The mode of the search. The following can be chosen: ["phonefy", "usufy", "searchfy"]. :return: Returns a list [] of <Platform> objects. ''' listAll = [] ############################################################################ ############################################################################ # -------------------------------------------------------------------------- # Dinamically collecting all the "official" modules # -------------------------------------------------------------------------- # A list that will contain all of the module names all_modules = [] # Grabbing all the module names for _, name, _ in pkgutil.iter_modules(osrframework.wrappers.__path__): all_modules.append("osrframework.wrappers." + name) # Iterating through all the module names to grab them for moduleName in all_modules: # Importing the module my_module = importlib.import_module(moduleName) # Getting all the classNames. classNames = [m[0] for m in inspect.getmembers(my_module, inspect.isclass) if m[1].__module__ == moduleName] # Dinamically grabbing the first class of the module. IT SHOULD BE ALONE! MyClass = getattr(my_module, classNames[0]) # Instantiating the object newInstance = MyClass() # Adding to the list! listAll.append(newInstance) # -------------------------------------------------------------------------- # Loading user-defined wrappers under [OSRFrameworkHOME]/plugins/wrappers/ # -------------------------------------------------------------------------- # Creating the application paths paths = configuration.getConfigPath() newPath = os.path.abspath(paths["appPathWrappers"]) # Inserting in the System Path if not newPath in sys.path: sys.path.append(newPath) userImportedModules = {} for module in os.listdir(newPath): if module[-3:] == '.py': current = module.replace('.py', '') userImportedModules[current] = __import__(current) del newPath userClasses = [] # Iterating through all the files for userModule in userImportedModules.keys(): my_module = userImportedModules[userModule] # Getting all the classNames. classNames = [m[0] for m in inspect.getmembers(my_module, inspect.isclass) if m[1].__module__ == userModule] # Dinamically grabbing the first class of the module. IT SHOULD BE ALONE! MyClass = getattr(my_module, classNames[0]) # Instantiating the object newInstance = MyClass() # Adding to the list! userClasses.append(newInstance) # -------------------------------------------------------------------------- # Overwriting original modules with the user plugins # -------------------------------------------------------------------------- listToAdd = [] for userClass in userClasses: overwritten = False for i, officialClass in enumerate(listAll): # Checking if the name is the same if str(userClass) == str(officialClass): # Replacing the official module if a user module exists for it listAll[i] = userClass # We stop iterating this loop overwritten = True break if not overwritten: # Appending the new class listToAdd.append(userClass) # Merging listAll and listToAdd listAll = listAll + listToAdd ############################################################################ ############################################################################ creds = credentials.getCredentials() for p in listAll: # Verify if there are credentials to be loaded if p.platformName.lower() in creds.keys(): p.setCredentials(creds[p.platformName.lower()]) if mode == None: return listAll else: # We are returning only those platforms which are required by the mode. selected = [] for p in listAll: if p.isValidMode[mode]: selected.append(p) return selected
def getAllRegexp(): ''' Method that recovers ALL the list of <RegexpObject> classes to be processed.... :return: Returns a list [] of <RegexpObject> classes. ''' listAll = [] ############################################################################ ############################################################################ # -------------------------------------------------------------------------- # Dinamically collecting all the "official" modules # -------------------------------------------------------------------------- # A list that will contain all of the module names all_modules = [] # Grabbing all the module names for _, name, _ in pkgutil.iter_modules(osrframework.patterns.__path__): all_modules.append("osrframework.patterns." + name) # Iterating through all the module names to grab them for moduleName in all_modules: # Importing the module my_module = importlib.import_module(moduleName) # Getting all the classNames. classNames = [ m[0] for m in inspect.getmembers(my_module, inspect.isclass) if m[1].__module__ == moduleName ] # Dinamically grabbing the first class of the module. IT SHOULD BE ALONE! MyClass = getattr(my_module, classNames[0]) # Instantiating the object newInstance = MyClass() # Adding to the list! listAll.append(newInstance) # -------------------------------------------------------------------------- # Loading user-defined wrappers under [OSRFrameworkHOME]/plugins/patterns/ # -------------------------------------------------------------------------- # Creating the application paths paths = configuration.getConfigPath() newPath = os.path.abspath(paths["appPathPatterns"]) # Inserting in the System Path if not newPath in sys.path: sys.path.append(newPath) userImportedModules = {} for module in os.listdir(newPath): if module[-3:] == '.py': current = module.replace('.py', '') userImportedModules[current] = __import__(current) del newPath userClasses = [] # Iterating through all the files for userModule in userImportedModules.keys(): my_module = userImportedModules[userModule] # Getting all the classNames. classNames = [ m[0] for m in inspect.getmembers(my_module, inspect.isclass) if m[1].__module__ == userModule ] # Dinamically grabbing the first class of the module. IT SHOULD BE ALONE! MyClass = getattr(my_module, classNames[0]) # Instantiating the object newInstance = MyClass() # Adding to the list! userClasses.append(newInstance) # -------------------------------------------------------------------------- # Overwriting original modules with the user plugins # -------------------------------------------------------------------------- listToAdd = [] for userClass in userClasses: for i, officialClass in enumerate(listAll): # Checking if the name is the same if str(userClass) == str(officialClass): # Replacing the official module if a user module exists for it listAll[i] = userClass else: if userClass not in listToAdd: # Appending the new class listToAdd.append(userClass) # Merging listAll and listToAdd listAll = listAll + listToAdd ############################################################################ ############################################################################ return listAll
groupAbout = parser.add_argument_group( 'About arguments', 'Showing additional information about this program.') groupAbout.add_argument('-h', '--help', action='help', help='shows this help and exists.') groupAbout.add_argument( '--version', action='version', version=VERSION, help='shows the version of the program and exists.') args = parser.parse_args() # Creating the application path applicationPath = configuration.getConfigPath() applicationPathDefaults = os.path.join(applicationPath, "default") applicationPathTransforms = os.path.join(applicationPath, "transforms") # Copying the default configuration files. if not os.path.exists(applicationPathDefaults): os.makedirs(applicationPathDefaults) if not os.path.exists(applicationPathTransforms): os.makedirs(applicationPathTransforms) configureMaltego(transformsConfigFolder=applicationPathTransforms, base=args.base, debug=args.debug, backupPath=applicationPathDefaults)
from pypandoc import convert read_md = lambda f: convert(f, 'rst') except ImportError: print("[!] pypandoc module not found, could not convert Markdown to RST") read_md = lambda f: open(f, 'r').read() except Exception: read_md = lambda f: open(f, 'r').read() # Reading the .md file try: long_description = read_md(os.path.join(HERE,"README.md")) except: long_description = "" # Creating the application paths paths = configuration.getConfigPath() print("[*] Launching the installation of the osrframework module...") # Launching the setup setup( name="osrframework", version=NEW_VERSION, description="OSRFramework - A set of GPLv3+ OSINT tools developed by i3visio analysts for online research.", author="Felix Brezo and Yaiza Rubio", author_email="*****@*****.**", url="http://github.com/i3visio/osrframework", license="COPYING", keywords = "python osint harvesting profiling username socialmedia forums", entry_points={ 'console_scripts': [ 'alias_generator = osrframework.alias_generator:main',
read_md = lambda f: convert(f, 'rst') except ImportError: print("warning: pypandoc module not found, could not convert Markdown to RST") read_md = lambda f: open(f, 'r').read() except Exception: read_md = lambda f: open(f, 'r').read() # Reading the .md file try: long_description = read_md(os.path.join(HERE,"README.md")) except: long_description = "" # Creating the application path applicationPath = configuration.getConfigPath() applicationPathDefaults = os.path.join(applicationPath, "default") applicationPathTransforms = os.path.join(applicationPath, "transforms") # Copying the default configuration files. if not os.path.exists(applicationPathDefaults): os.makedirs(applicationPathDefaults) if not os.path.exists(applicationPathTransforms): os.makedirs(applicationPathTransforms) # Launching the setup setup( name="osrframework", version=NEW_VERSION, description="OSRFramework - A set of GPLv3+ OSINT tools developed by i3visio analysts for online research.", author="Felix Brezo and Yaiza Rubio",
from pypandoc import convert read_md = lambda f: convert(f, 'rst') except ImportError: print("[!] pypandoc module not found, could not convert Markdown to RST") read_md = lambda f: open(f, 'r').read() except Exception: read_md = lambda f: open(f, 'r').read() # Reading the .md file try: long_description = read_md(os.path.join(HERE,"README.md")) except: long_description = "" # Creating the application paths paths = configuration.getConfigPath() print "[*] Defining the installation of the osrframework module..." # Launching the setup setup( name="osrframework", version=NEW_VERSION, description="OSRFramework - A set of GPLv3+ OSINT tools developed by i3visio analysts for online research.", author="Felix Brezo and Yaiza Rubio", author_email="*****@*****.**", url="http://github.com/i3visio/osrframework", license="COPYING", keywords = "python osint harvesting profiling maltego username socialmedia forums", scripts= [ "osrframework/alias_generator.py", "osrframework/entify.py", "osrframework/mailfy.py",