def returnListOfCreds():
'''
:return:
A list of tuples containing in the first the name of the platform,
as read from the accounts.cfg file in the application folder. E. g.:
listCreds.append(("<platform>", "<username>", "<password>"))
'''
listCreds = []
# If a accounts.cfg has not been found, creating it by copying from default
configPath = configuration.getConfigPath("accounts.cfg")
# Checking if the configuration file exists
if not os.path.exists(configPath):
try:
# Copy the data from the default folder
defaultConfigPath = configuration.getConfigPath(os.path.join("default", "accounts.cfg"))
with open(configPath, "w") as oF:
with open(defaultConfigPath) as iF:
cont = iF.read()
oF.write(cont)
except Exception, e:
print "WARNING. No configuration file could be found and the default file was not found either, so NO credentials have been loaded."
print str(e)
print
return listCreds
def returnListOfCreds():
'''
:return:
A list of tuples containing in the first the name of the platform,
as read from the accounts.cfg file in the application folder. E. g.:
listCreds.append(("<platform>", "<username>", "<password>"))
'''
listCreds = []
# If a accounts.cfg has not been found, creating it by copying from default
configPath = os.path.join(configuration.getConfigPath()["appPath"],
"accounts.cfg")
# Checking if the configuration file exists
if not os.path.exists(configPath):
# Copy the data from the default folder
defaultConfigPath = os.path.join(
configuration.getConfigPath()["appPathDefaults"], "accounts.cfg")
try:
with open(defaultConfigPath) as iF:
cont = iF.read()
with open(configPath, "w") as oF:
oF.write(cont)
except Exception, e:
raise errors.ConfigurationFileNotFoundError(
configPath, defaultConfigPath)
return listCreds
def returnListOfAPIKeys():
'''
:return: A dictionary containing the API Keys stored in a dictionary depending on the information required by each platform
'''
dictAPIKeys = {}
# If a api_keys.cfg has not been found, creating it by copying from default
configPath = os.path.join(configuration.getConfigPath()["appPath"], "api_keys.cfg")
# Checking if the configuration file exists
if not os.path.exists(configPath):
try:
# Copy the data from the default folder
defaultConfigPath = os.path.join(configuration.getConfigPath()["appPathDefaults"], "api_keys.cfg")
with open(configPath, "w") as oF:
with open(defaultConfigPath) as iF:
cont = iF.read()
oF.write(cont)
except Exception, e:
print "WARNING. No configuration file could be found and the default file was not found either, so NO API keys have been loaded."
print str(e)
print
return dictAPIKeys
def __init__(self):
"""
Recovering an instance of a new Browser.
"""
# Browser
self.br = mechanize.Browser()
# Cookie Jar
self.cj = cookielib.LWPCookieJar()
self.br.set_cookiejar(self.cj)
# Browser options
self.br.set_handle_equiv(True)
self.br.set_handle_gzip(False)
self.br.set_handle_redirect(True)
self.br.set_handle_referer(False)
self.br.set_handle_robots(False)
self.br.set_handled_schemes(['http', 'https'])
# Follows refresh 0 but not hangs on refresh > 0
self.br.set_handle_refresh(mechanize._http.HTTPRefreshProcessor(), max_time=1)
# Defining User Agents
self.userAgents = []
# Handling proxies
self.proxies = {}
self.timeout = 2
# Want debugging messages?
#self.br.set_debug_http(True)
#self.br.set_debug_redirects(True)
#self.br.set_debug_responses(True)
# Trying to read the configuration
# --------------------------------
# If a current.cfg has not been found, creating it by copying from default
configPath = configuration.getConfigPath("browser.cfg")
configPath = os.path.join(configuration.getConfigPath()["appPath"], "browser.cfg")
# Checking if the configuration file exists
if not os.path.exists(configPath):
try:
# Copy the data from the default folder
defaultConfigPath = os.path.join(configuration.getConfigPath()["appPathDefaults"], "browser.cfg")
with open(defaultConfigPath) as iF:
cont = iF.read()
with open(configPath, "w") as oF:
oF.write(cont)
except Exception, e:
print("WARNING. No configuration file could be found and the default file was not found either, so configuration will be set as default.")
print(str(e))
print()
# Storing configuration as default
self.userAgents = ['Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/55.0.2883.87 Chrome/55.0.2883.87 Safari/537.36']
self.proxies = {}
return None
def returnListOfCreds():
'''
:return:
A list of tuples containing in the first the name of the platform,
as read from the accounts.cfg file in the application folder. E. g.:
listCreds.append(("<platform>", "<username>", "<password>"))
'''
listCreds = []
# If a accounts.cfg has not been found, creating it by copying from default
configPath = configuration.getConfigPath("accounts.cfg")
# Checking if the configuration file exists
if not os.path.exists(configPath):
try:
# Copy the data from the default folder
defaultConfigPath = configuration.getConfigPath(
os.path.join("default", "accounts.cfg"))
with open(configPath, "w") as oF:
with open(defaultConfigPath) as iF:
cont = iF.read()
oF.write(cont)
except Exception, e:
print "WARNING. No configuration file could be found and the default file was not found either, so NO credentials have been loaded."
print str(e)
print
return listCreds
def __init__(self):
"""
Recovering an instance of a new Browser.
"""
# Browser
self.br = mechanize.Browser()
# Cookie Jar
self.cj = cookielib.LWPCookieJar()
self.br.set_cookiejar(self.cj)
# Browser options
self.br.set_handle_equiv(True)
self.br.set_handle_gzip(False)
self.br.set_handle_redirect(True)
self.br.set_handle_referer(False)
self.br.set_handle_robots(False)
# Follows refresh 0 but not hangs on refresh > 0
self.br.set_handle_refresh(mechanize._http.HTTPRefreshProcessor(),
max_time=1)
# Want debugging messages?
#self.br.set_debug_http(True)
#self.br.set_debug_redirects(True)
#self.br.set_debug_responses(True)
# Trying to read the configuration
# --------------------------------
# If a current.cfg has not been found, creating it by copying from default
configPath = configuration.getConfigPath("browser.cfg")
# Checking if the configuration file exists
if not os.path.exists(configPath):
try:
# Copy the data from the default folder
defaultConfigPath = configuration.getConfigPath(
os.path.join("default", "browser.cfg"))
with open(configPath, "w") as oF:
with open(defaultConfigPath) as iF:
cont = iF.read()
oF.write(cont)
except Exception, e:
print "WARNING. No configuration file could be found and the default file was not found either, so configuration will be set as default."
print str(e)
print
# Storing configuration as default
self.userAgents = [
'Mozilla/5.0 (Windows NT 5.1; rv:31.0) Gecko/20100101 Firefox/31.0'
]
self.proxies = []
return None
def __init__(self):
"""
Recovering an instance of a new Browser.
"""
# Browser
self.br = mechanize.Browser()
# Cookie Jar
self.cj = cookielib.LWPCookieJar()
self.br.set_cookiejar(self.cj)
# Browser options
self.br.set_handle_equiv(True)
self.br.set_handle_gzip(False)
self.br.set_handle_redirect(True)
self.br.set_handle_referer(False)
self.br.set_handle_robots(False)
# Follows refresh 0 but not hangs on refresh > 0
self.br.set_handle_refresh(mechanize._http.HTTPRefreshProcessor(), max_time=1)
# Want debugging messages?
#self.br.set_debug_http(True)
#self.br.set_debug_redirects(True)
#self.br.set_debug_responses(True)
# Trying to read the configuration
# --------------------------------
# If a current.cfg has not been found, creating it by copying from default
configPath = configuration.getConfigPath("browser.cfg")
configPath = os.path.join(configuration.getConfigPath()["appPath"], "browser.cfg")
# Checking if the configuration file exists
if not os.path.exists(configPath):
try:
# Copy the data from the default folder
defaultConfigPath = os.path.join(configuration.getConfigPath()["appPathDefaults"], "browser.cfg")
with open(configPath, "w") as oF:
with open(defaultConfigPath) as iF:
cont = iF.read()
oF.write(cont)
except Exception, e:
print "WARNING. No configuration file could be found and the default file was not found either, so configuration will be set as default."
print str(e)
print
# Storing configuration as default
self.userAgents = ['Mozilla/5.0 (Windows NT 5.1; rv:31.0) Gecko/20100101 Firefox/31.0']
self.proxies = []
return None
def do_info(self, line):
"""
Command that shows again the general information about the application.
"""
configInfo = "\n Additional configuration files:"
configInfo += "\n -------------------------------"
configInfo += "\n You will be able to find more configuration options in the following files in your system. The relevant paths are the ones that follows:"
# Get the configuration folders in each system
paths = configuration.getConfigPath()
configInfo += "\n\t- '" + os.path.join(
paths["appPath"], "accounts.cfg"
) + "' -> Configuration details about the login credentials already configured in the framework."
configInfo += "\n\t- '" + os.path.join(
paths["appPath"], "api_keys.cfg"
) + "' -> Configuration details about the API credentials already configured."
configInfo += "\n\t- '" + os.path.join(
paths["appPath"], "browser.cfg"
) + "' -> Connection configuration about how the browsers will be connected."
configInfo += "\n\t- '" + os.path.join(
paths["appPath"], "general.cfg"
) + "' -> General configuration of the different utils containing the default options."
configInfo += "\n\t- '" + paths[
"appPathDefaults"] + "/' -> Directory containing default files as a backup."
configInfo += "\n\t- '" + paths[
"appPathPlugins"] + "/' -> Directory containing the details of the user defined plugins."
configInfo += "\n\t- '" + paths[
"appPathPatterns"] + "/' -> Directory containing the user-defined patterns for entify.py."
configInfo += "\n\t- '" + paths[
"appPathWrappers"] + "/' -> Directory containing the user-defined wrappers for usufy, searchfy and phonefy platforms."
configInfo += "\n"
print self.info + configInfo
def do_info(self, line):
"""
Command that shows again the general information about the application
Args:
-----
line: the string of the line typed.
"""
configInfo = """
""" + general.emphasis("""Additional configuration files:
-------------------------------""") + """
You will be able to find more configuration options in the following files
in your system. The relevant paths are the ones that follow:"""
# Get the configuration folders in each system
paths = configuration.getConfigPath()
configInfo += """
- Configuration details about the login credentials in OSRFramework:
""" + general.info(os.path.join(paths["appPath"], "accounts.cfg")) + """
- Configuration details about the API credentials already configured:
""" + general.info(os.path.join(paths["appPath"], "api_keys.cfg")) + """
- Connection configuration about how the browsers will be connected:
""" + general.info(os.path.join(paths["appPath"], "browser.cfg")) + """
- General default configuration of the the utils:
""" + general.info(os.path.join(paths["appPath"], "general.cfg")) + """
- Directory containing default files as a backup:
""" + general.info(paths["appPathDefaults"]) + """
- Directory containing the user-defined patterns for entify.py:
""" + general.info(paths["appPathPatterns"]) + """
- Directory containing the user-defined wrappers for usufy platforms:
""" + general.info(paths["appPathWrappers"])
print(self.info + configInfo)
def __init__(self, platform, *args, **kwargs):
msg = """
[*] Warning:\t{}. Details:
No valid credentials provided for '{}'.
Update the configuration file at: '{}'.
""".format(
self.__class__.__name__, platform,
os.path.join(configuration.getConfigPath()["appPath"],
"accounts.cfg"), general.emphasis("-x " + platform))
OSRFrameworkException.__init__(self, general.warning(msg))
self.generic = "The credentials for some platforms where NOT provided."
def getConfiguration(configuration_file="server.cfg",
package="osrframework_server",
section="osrframework_server"):
"""
Method that recovers the configuration information osrframework_server.
TODO: Grab the default file from the package data instead of storing it in the main folder.
Args:
-----
configuration_file: The configuration file to be read.
package: The package name where the default files are stored.
section: The secction's name that will be read.
Returns:
--------
A dictionary containing the default configuration.
"""
VALUES = {}
# If a api_keys.cfg has not been found, creating it by copying from default
configPath = os.path.join(configuration.getConfigPath()["appPath"],
configuration_file)
# Checking if the configuration file exists
if not os.path.exists(configPath):
print(
warning(
"[*] No configuration files found for OSRFramework Server."))
print(
warning(
"[*] Copying default configuration files to OSRFramework configuration folder at '"
+ configPath + "'."))
# Copy the data from the default folder
defaultConfigPath = pkg_resources.resource_filename(
package, os.path.join('config', configuration_file))
copyfile(defaultConfigPath, configPath)
# Reading the configuration file
config = ConfigParser.ConfigParser()
config.read(configPath)
# Iterating through all the sections, which contain the platforms
for s in config.sections():
incomplete = False
if s.lower() == section.lower():
# Iterating through parameters
for (param, value) in config.items(s):
if value != '':
VALUES[param] = value
return VALUES
def returnListOfAPIKeys():
'''
:return: A dictionary containing the API Keys stored in a dictionary depending on the information required by each platform
'''
dictAPIKeys = {}
# If a api_keys.cfg has not been found, creating it by copying from default
configPath = configuration.getConfigPath("api_keys.cfg")
# Checking if the configuration file exists
if not os.path.exists(configPath):
try:
# Copy the data from the default folder
defaultConfigPath = configuration.getConfigPath(os.path.join("default", "api_keys.cfg"))
with open(configPath, "w") as oF:
with open(defaultConfigPath) as iF:
cont = iF.read()
oF.write(cont)
except Exception, e:
print "WARNING. No configuration file could be found and the default file was not found either, so NO API keys have been loaded."
print str(e)
print
return listCreds
def getAllPlatformObjects(mode = None):
'''
Method that recovers ALL the list of <Platform> classes to be processed....
:param mode: The mode of the search. The following can be chosen: ["phonefy", "usufy", "searchfy"].
:return: Returns a list [] of <Platform> objects.
'''
listAll = []
############################################################################
############################################################################
# --------------------------------------------------------------------------
# Dinamically collecting all the "official" modules
# --------------------------------------------------------------------------
# A list that will contain all of the module names
all_modules = []
# Grabbing all the module names
for _, name, _ in pkgutil.iter_modules(osrframework.wrappers.__path__):
all_modules.append("osrframework.wrappers." + name)
# Iterating through all the module names to grab them
for moduleName in all_modules:
# Importing the module
my_module = importlib.import_module(moduleName)
# Getting all the classNames.
classNames = [m[0] for m in inspect.getmembers(my_module, inspect.isclass) if m[1].__module__ == moduleName]
# Dinamically grabbing the first class of the module. IT SHOULD BE ALONE!
MyClass = getattr(my_module, classNames[0])
# Instantiating the object
newInstance = MyClass()
# Adding to the list!
listAll.append(newInstance)
# --------------------------------------------------------------------------
# Loading user-defined wrappers under [OSRFrameworkHOME]/plugins/wrappers/
# --------------------------------------------------------------------------
# Creating the application paths
paths = configuration.getConfigPath()
newPath = os.path.abspath(paths["appPathWrappers"])
# Inserting in the System Path
if not newPath in sys.path:
sys.path.append(newPath)
userImportedModules = {}
for module in os.listdir(newPath):
if module[-3:] == '.py':
current = module.replace('.py', '')
userImportedModules[current] = __import__(current)
del newPath
userClasses = []
# Iterating through all the files
for userModule in userImportedModules.keys():
my_module = userImportedModules[userModule]
# Getting all the classNames.
classNames = [m[0] for m in inspect.getmembers(my_module, inspect.isclass) if m[1].__module__ == userModule]
# Dinamically grabbing the first class of the module. IT SHOULD BE ALONE!
MyClass = getattr(my_module, classNames[0])
# Instantiating the object
newInstance = MyClass()
# Adding to the list!
userClasses.append(newInstance)
# --------------------------------------------------------------------------
# Overwriting original modules with the user plugins
# --------------------------------------------------------------------------
listToAdd = []
for userClass in userClasses:
overwritten = False
for i, officialClass in enumerate(listAll):
# Checking if the name is the same
if str(userClass) == str(officialClass):
# Replacing the official module if a user module exists for it
listAll[i] = userClass
# We stop iterating this loop
overwritten = True
break
if not overwritten:
# Appending the new class
listToAdd.append(userClass)
# Merging listAll and listToAdd
listAll = listAll + listToAdd
############################################################################
############################################################################
creds = credentials.getCredentials()
for p in listAll:
# Verify if there are credentials to be loaded
if p.platformName.lower() in creds.keys():
p.setCredentials(creds[p.platformName.lower()])
if mode == None:
return listAll
else:
# We are returning only those platforms which are required by the mode.
selected = []
for p in listAll:
if p.isValidMode[mode]:
selected.append(p)
return selected
def getAllRegexp():
'''
Method that recovers ALL the list of <RegexpObject> classes to be processed....
:return: Returns a list [] of <RegexpObject> classes.
'''
listAll = []
############################################################################
############################################################################
# --------------------------------------------------------------------------
# Dinamically collecting all the "official" modules
# --------------------------------------------------------------------------
# A list that will contain all of the module names
all_modules = []
# Grabbing all the module names
for _, name, _ in pkgutil.iter_modules(osrframework.patterns.__path__):
all_modules.append("osrframework.patterns." + name)
# Iterating through all the module names to grab them
for moduleName in all_modules:
# Importing the module
my_module = importlib.import_module(moduleName)
# Getting all the classNames.
classNames = [
m[0] for m in inspect.getmembers(my_module, inspect.isclass)
if m[1].__module__ == moduleName
]
# Dinamically grabbing the first class of the module. IT SHOULD BE ALONE!
MyClass = getattr(my_module, classNames[0])
# Instantiating the object
newInstance = MyClass()
# Adding to the list!
listAll.append(newInstance)
# --------------------------------------------------------------------------
# Loading user-defined wrappers under [OSRFrameworkHOME]/plugins/patterns/
# --------------------------------------------------------------------------
# Creating the application paths
paths = configuration.getConfigPath()
newPath = os.path.abspath(paths["appPathPatterns"])
# Inserting in the System Path
if not newPath in sys.path:
sys.path.append(newPath)
userImportedModules = {}
for module in os.listdir(newPath):
if module[-3:] == '.py':
current = module.replace('.py', '')
userImportedModules[current] = __import__(current)
del newPath
userClasses = []
# Iterating through all the files
for userModule in userImportedModules.keys():
my_module = userImportedModules[userModule]
# Getting all the classNames.
classNames = [
m[0] for m in inspect.getmembers(my_module, inspect.isclass)
if m[1].__module__ == userModule
]
# Dinamically grabbing the first class of the module. IT SHOULD BE ALONE!
MyClass = getattr(my_module, classNames[0])
# Instantiating the object
newInstance = MyClass()
# Adding to the list!
userClasses.append(newInstance)
# --------------------------------------------------------------------------
# Overwriting original modules with the user plugins
# --------------------------------------------------------------------------
listToAdd = []
for userClass in userClasses:
for i, officialClass in enumerate(listAll):
# Checking if the name is the same
if str(userClass) == str(officialClass):
# Replacing the official module if a user module exists for it
listAll[i] = userClass
else:
if userClass not in listToAdd:
# Appending the new class
listToAdd.append(userClass)
# Merging listAll and listToAdd
listAll = listAll + listToAdd
############################################################################
############################################################################
return listAll
groupAbout = parser.add_argument_group(
'About arguments',
'Showing additional information about this program.')
groupAbout.add_argument('-h',
'--help',
action='help',
help='shows this help and exists.')
groupAbout.add_argument(
'--version',
action='version',
version=VERSION,
help='shows the version of the program and exists.')
args = parser.parse_args()
# Creating the application path
applicationPath = configuration.getConfigPath()
applicationPathDefaults = os.path.join(applicationPath, "default")
applicationPathTransforms = os.path.join(applicationPath, "transforms")
# Copying the default configuration files.
if not os.path.exists(applicationPathDefaults):
os.makedirs(applicationPathDefaults)
if not os.path.exists(applicationPathTransforms):
os.makedirs(applicationPathTransforms)
configureMaltego(transformsConfigFolder=applicationPathTransforms,
base=args.base,
debug=args.debug,
backupPath=applicationPathDefaults)
from pypandoc import convert
read_md = lambda f: convert(f, 'rst')
except ImportError:
print("[!] pypandoc module not found, could not convert Markdown to RST")
read_md = lambda f: open(f, 'r').read()
except Exception:
read_md = lambda f: open(f, 'r').read()
# Reading the .md file
try:
long_description = read_md(os.path.join(HERE,"README.md"))
except:
long_description = ""
# Creating the application paths
paths = configuration.getConfigPath()
print("[*] Launching the installation of the osrframework module...")
# Launching the setup
setup(
name="osrframework",
version=NEW_VERSION,
description="OSRFramework - A set of GPLv3+ OSINT tools developed by i3visio analysts for online research.",
author="Felix Brezo and Yaiza Rubio",
author_email="*****@*****.**",
url="http://github.com/i3visio/osrframework",
license="COPYING",
keywords = "python osint harvesting profiling username socialmedia forums",
entry_points={
'console_scripts': [
'alias_generator = osrframework.alias_generator:main',
read_md = lambda f: convert(f, 'rst')
except ImportError:
print("warning: pypandoc module not found, could not convert Markdown to RST")
read_md = lambda f: open(f, 'r').read()
except Exception:
read_md = lambda f: open(f, 'r').read()
# Reading the .md file
try:
long_description = read_md(os.path.join(HERE,"README.md"))
except:
long_description = ""
# Creating the application path
applicationPath = configuration.getConfigPath()
applicationPathDefaults = os.path.join(applicationPath, "default")
applicationPathTransforms = os.path.join(applicationPath, "transforms")
# Copying the default configuration files.
if not os.path.exists(applicationPathDefaults):
os.makedirs(applicationPathDefaults)
if not os.path.exists(applicationPathTransforms):
os.makedirs(applicationPathTransforms)
# Launching the setup
setup( name="osrframework",
version=NEW_VERSION,
description="OSRFramework - A set of GPLv3+ OSINT tools developed by i3visio analysts for online research.",
author="Felix Brezo and Yaiza Rubio",
from pypandoc import convert
read_md = lambda f: convert(f, 'rst')
except ImportError:
print("[!] pypandoc module not found, could not convert Markdown to RST")
read_md = lambda f: open(f, 'r').read()
except Exception:
read_md = lambda f: open(f, 'r').read()
# Reading the .md file
try:
long_description = read_md(os.path.join(HERE,"README.md"))
except:
long_description = ""
# Creating the application paths
paths = configuration.getConfigPath()
print "[*] Defining the installation of the osrframework module..."
# Launching the setup
setup( name="osrframework",
version=NEW_VERSION,
description="OSRFramework - A set of GPLv3+ OSINT tools developed by i3visio analysts for online research.",
author="Felix Brezo and Yaiza Rubio",
author_email="*****@*****.**",
url="http://github.com/i3visio/osrframework",
license="COPYING",
keywords = "python osint harvesting profiling maltego username socialmedia forums",
scripts= [
"osrframework/alias_generator.py",
"osrframework/entify.py",
"osrframework/mailfy.py",
