def do_info(self, line):
"""
Command that shows again the general information about the application
Args:
-----
line: the string of the line typed.
"""
configInfo = """
Additional configuration files:
-------------------------------
You will be able to find more configuration options in the following files
in your system. The relevant paths are the ones that follow:"""
# Get the configuration folders in each system
paths = configuration.getConfigPath()
configInfo += """
- Configuration details about the login credentials in OSRFramework:
""" + os.path.join(paths["appPath"], "accounts.cfg") + """
- Configuration details about the API credentials already configured:
""" + os.path.join(paths["appPath"], "api_keys.cfg") + """
- Connection configuration about how the browsers will be connected:
""" + os.path.join(paths["appPath"], "browser.cfg") + """
- General default configuration of the the utils:
""" + os.path.join(paths["appPath"], "general.cfg") + """
- Directory containing default files as a backup:
""" + paths["appPathDefaults"] + """
- Directory containing the user-defined patterns for entify.py:
""" + paths["appPathPatterns"] + """
- Directory containing the user-defined wrappers for usufy platforms:
""" + paths["appPathWrappers"]
print(general.info(self.info) + general.info(configInfo))
def do_run(self, line):
"""Running the current application. This method should be redefined for each util."""
print
# Checking if all the required parameters have been set
if self._checkIfRequiredAreSet():
print(general.info("Collecting the options set by the user..."))
# Getting the parser...
parser = usufy.getParser()
# Generating the parameters
params = self._getParams()
args = parser.parse_args(params)
print(general.info("Launching " + self.UNAME + " with the following parameters: ") + general.emphashis(str(params)))
try:
usufy.main(args)
except Exception as e:
print(gemeral.error("[!] ERROR. Something happenned when launching the utility. Type 'show options' to check the parameters. "))
print(general.error("Traceback: " + str(e)))
else:
print(general.error("[!] ERROR. There are required parameters which have not been set."))
self.do_show("options")
print(general.success("Execution ended successfully."))
def do_show(self, line):
"""
Showing the information about the module
The things to show are: 'options' and 'command'.
- 'options' will show the current values of each and every
parameter.
- 'command' will show the command needed to launch the module as is
using the cli applications.
Args:
-----
line: the string of the line typed.
"""
if line.lower() == "options":
print(general.emphasis("\n\tOptions\n\t-------\n"))
for key in self.CONFIG.keys():
print("\t- " +
(key + (" (*)." if self.CONFIG[key]["REQUIRED"] else ".")
).ljust(14) + "" + self.CONFIG[key]["DESCRIPTION"])
print(general.emphasis("\n\tCurrent values\n\t-------------\n"))
for key in self.CONFIG.keys():
print("\t- " +
(key + (" (*)" if self.CONFIG[key]["REQUIRED"] else "") +
": ").ljust(14) +
general.info("" if self.CONFIG[key]["CURRENT_VALUE"] ==
None else utils.listToString(
self.CONFIG[key]["CURRENT_VALUE"])))
elif line.lower() == "command":
print(
general.emphasis(
"\n\tTerminal command\n\t----------------\n\t\t") + "$ " +
general.info(self.createCommandLine()))
def main(params=None):
"""
Main function that starts the server
Args:
-----
params: Arguments received in the command line.
Returns:
--------
A list of i3visio entities.
"""
# Grabbing the parser
parser = getParser()
args = parser.parse_args(params)
print(general.title(osrframework.utils.banner.text))
# Starting the server
print(
general.info("[*] OSRFramework Server " +
osrframework_server.__version__ + " started at " +
"http://" + args.host + ":" + str(args.port) +
"... You can access it in your browser."))
print(general.info("[*] Press <Ctrl + C> at any time to stop the server."))
app.run(debug=args.debug, host=args.host, port=args.port)
print(general.info("\n[*] OSRFramework server exited normally."))
def do_run(self, line):
"""
Command that send the order to the framework to launch this util
Args:
-----
line: The string of the line typed.
"""
# Checking if all the required parameters have been set
if self._checkIfRequiredAreSet():
print(general.info("\nCollecting the options set by the user...\n"))
# Getting the parser...
parser = entify.getParser()
# Generating the parameters
params = self._getParams()
args = parser.parse_args(params)
print(general.info("\nLaunching " + self.UNAME + " with the following parameters:") + general.emphasis("\t$ " + self.UNAME + " " + utils.listToString(params) + "\n"))
try:
entify.main(args)
except Exception as e:
print(general.error("\n[!!] ERROR. Something happened when launching the utility. Type 'show options' to check the parameters.\n"))
print(general.error("Traceback: " + str(e)))
else:
print(general.error("\n[!!] ERROR. There are required parameters which have not been set."))
self.do_show("options")
print(general.success("Execution ended successfully."))
def do_show(self, line):
"""Showing the information about the module. The things to show are: 'options' and 'command'.
- ' options' will show the current values of each and every parameter.
- 'command' will show the command needed to launch the module as is using the cli applications."""
if line.lower() == "options":
print(general.info("Defining the different options for util " + self.UNAME + "..."))
for key in self.CONFIG.keys():
print(general.info("\t- " + (key + (" (*)." if self.CONFIG[key]["REQUIRED"] else ".") ).ljust(14) + "" + self.CONFIG[key]["DESCRIPTION"]))
print(general.info("Showing the current state of the options for util " + self.UNAME + "..."))
for key in self.CONFIG.keys():
print(general.info("\t- " + (key + (" (*)" if self.CONFIG[key]["REQUIRED"] else "") + ": ").ljust(14) + ("" if self.CONFIG[key]["CURRENT_VALUE"] == None else str(self.CONFIG[key]["CURRENT_VALUE"]))))
elif line.lower() == "command":
print(general.info("Equivalent command to be launched to imitate the current configuration:\n\t$ ") + general.title(self.createCommandLine()) + "\n")
def enumerateURL(urlDict, outputFolder, startIndex=0, maxErrors=100):
"""
Function that performs the enumeration itself.
"""
for i, url in enumerate(urlDict.keys()):
# Grabbing domain name:
domain = re.findall("://(.*)/", url)[0]
# Defining the starting index
index = startIndex
# The app will stop when this value reaches maxErrors
consecutiveErrors = 0
i3Browser = browser.Browser()
# Main loop that checks if the maximum number of errors has been reached
while consecutiveErrors <= maxErrors:
# creating the new URL to download
newQuery = url.replace("<INDEX>", str(index))
print(newQuery)
# Downloading the file
try:
data = i3Browser.recoverURL(newQuery)
filename = domain.replace(
"/", "|") + "_" + "-profile_" + str(index).rjust(
10, "0") + ".html"
if urlDict[url] != None:
if urlDict[url] in data:
print(
general.info("Storing resource as:\t" + filename +
"..."))
# The profile was found so we will store it:
with open(outputFolder + "/" + filename, "w") as oF:
oF.write(data)
else:
# The profile was found so we will store it:
print(
general.info("Storing resource as:\t" + filename +
"..."))
with open(outputFolder + "/" + filename, "w") as oF:
oF.write(data)
except:
pass
#logger.error("The resource could not be downloaded.")
index += 1
def do_set(self, line):
"""
Setting the variables defined in CONFIG
You can check their values at any time by typing 'show options'.
Args:
-----
line: the string of the line typed.
"""
try:
parameter, value = line.split(" ", 1)
# Setting the parameter
if parameter in self.CONFIG.keys():
# Verifying if the parameter is in the options
if len(self.CONFIG[parameter]["OPTIONS"]) > 0:
splittedValues = value.split(" ")
for s in splittedValues:
if s not in self.CONFIG[parameter]["OPTIONS"]:
raise Exception(
"ERROR: the value provided is not valid.")
# Setting the value
self.CONFIG[parameter]["CURRENT_VALUE"] = value
print(general.info(parameter + "=" + str(value)))
else:
raise Exception("ERROR: parameter not valid.")
except Exception as e:
print(
general.error(
"[!] ERROR: Not enough parameters provided. Usage: set OPTION VALUE."
))
print(general.error(str(e)))
def do_run(self, line):
"""Command that send the order to the framework to launch the current utility."""
if self._checkIfRequiredAreSet():
print(general.info("Launching the util..."))
else:
print(general.error("There are required parameters which have not been set."))
self.do_show("options")
def main(args=None):
print(general.title(banner.text))
saying_hello = f"""
OSRFramework Upgrade Tool | Copyright (C) Yaiza Rubio & Félix Brezo (i3visio) 2014-2020
This program comes with ABSOLUTELY NO WARRANTY. This is free software, and you
are welcome to redistribute it under certain conditions. For additional info,
visit <{general.LICENSE_URL}>.
"""
print(general.info(saying_hello))
startTime = dt.datetime.now()
print(
f"{startTime}\tGrabbing information about local and upstream versions of OSRFramework…\n"
)
package = UpgradablePackage(package_name="osrframework")
print(
f"Details:\n{general.emphasis(json.dumps(package.get_dict(), indent=2))}\n"
)
now = dt.datetime.now()
if package.is_upgradable():
print(
f"{now}\tLocal version of OSRFramework {general.warning('can be upgraded')} to version {package.get_dict()['remote_version']}.\n"
)
if not args.only_check:
now = dt.datetime.now()
cmd = ["pip3", "install", "osrframework", "--upgrade"]
# Installing development versions
if args.use_development:
cmd.append("--pre")
# Make installation for the user
if not sys.platform == 'win32' or os.geteuid() != 0:
cmd.append("--user")
print(
f"{now}\tTrying to upgrade the package running '{' '.join(cmd)}'..."
)
status = call(cmd)
if status:
# Displaying a warning if this is being run in a windows system
if sys.platform == 'win32':
print(f"{now}\t{general.error('Installation failed')}.")
else:
print(
f"{now}\t{general.error('Installation failed')}. Retry the installation as a superuser: '******' '.join(['sudo'] + cmd)}'"
)
sys.exit(1)
else:
print(
f"{now}\t{general.error('Installation sucessful')}. Upgraded to '{general.success(package.get_dict()['remote_version'])}'."
)
else:
print(
f"{now}\tLocal version of OSRFramework ({package.get_dict()['local_version']}) {general.success('is up to date')} with the remote version ({package.get_dict()['remote_version']}).\n"
)
def do_exit(self, line):
"""
This command will exit the osrfconsole normally
Args:
-----
line: the string of the line typed.
"""
print(general.info("\nExiting the program...\n"))
sys.exit()
def do_unset(self, line):
"""Unsetting the variables defined in CONFIG. You can check their values at any time by typing 'show options' and unsetting all the options at once by typing 'unset all'"."""
try:
parameter = line.split(" ")[0]
# Getting the parameter
if parameter in self.CONFIG.keys():
# Unsetting the value
self.CONFIG[parameter]["CURRENT_VALUE"] = self.CONFIG[parameter]["DEFAULT_VALUE"]
print(general.info(parameter + " reseted to '" + str(self.CONFIG[parameter]["DEFAULT_VALUE"]) + "'."))
elif parameter == "all":
for p in self.CONFIG.keys():
# Unsetting all the values
self.CONFIG[p]["CURRENT_VALUE"] = self.CONFIG[p]["DEFAULT_VALUE"]
print(general.info("All parameters reseted to their default values."))
else:
raise Exception("ERROR: parameter not valid.")
except Exception as e:
print(gemeral.error("[!] ERROR: Not enough parameters provided. Usage: unset OPTION"))
print(general.error("Traceback: " + str(e)))
def do_info(self, line):
"""
Shows all the information available about the module.
Args:
-----
line: the string of the line typed.
"""
print(general.info("Displaying module information."))
self.do_show("options")
self.do_show("command")
def main(params=None):
"""
Main loop for the enumeration
Args:
-----
params: A list with the parameters as grabbed by the terminal. It is
None when this is called by an entry_point.
"""
# Grabbing the parser
parser = getParser()
if params != None:
args = parser.parse_args(params)
else:
args = parser.parse_args()
print(general.title(banner.text))
sayingHello = """
Enumeration | Copyright (C) F. Brezo and Y. Rubio (i3visio) 2016-2018
This program comes with ABSOLUTELY NO WARRANTY. This is free software, and you
are welcome to redistribute it under certain conditions. For additional info,
visit """ + general.LICENSE_URL + "\n"
print(general.info(sayingHello))
# Loading URL
urlDict = {}
if args.url != None:
urlDict[str(args.url)] = None
elif args.platforms != None:
for p in args.platforms:
with open(args.config, "r") as iF:
lines = iF.read().splitlines()
for l in lines:
platform = l.split('\t')[0]
url = l.split('\t')[1]
notFound = l.split('\t')[2]
if p == platform:
urlDict[url] = notFound
if not os.path.exists(args.output_folder):
os.makedirs(args.output_folder)
# Making the call
enumerateURL(urlDict,
args.output_folder,
startIndex=args.start_index,
maxErrors=args.max_errors)
def do_run(self, line):
"""
Command that send the order to the framework to launch this util
Args:
-----
line: the string of the line typed.
"""
if self._checkIfRequiredAreSet():
print(general.info("\nLaunching the util...\n"))
else:
print(
general.error(
"\n[!!] ERROR: There are required parameters which have not been set."
))
self.do_show("options")
def do_info(self, line):
"""
Command that shows again the general information about the application.
"""
configInfo = "\n Additional configuration files:"
configInfo += "\n -------------------------------"
configInfo += "\n You will be able to find more configuration options in the following files in your system. The relevant paths are the ones that follows:"
# Get the configuration folders in each system
paths = configuration.getConfigPath()
configInfo += "\n\t- '" + os.path.join(paths["appPath"], "accounts.cfg") + "' -> Configuration details about the login credentials already configured in the framework."
configInfo += "\n\t- '" + os.path.join(paths["appPath"], "api_keys.cfg") + "' -> Configuration details about the API credentials already configured."
configInfo += "\n\t- '" + os.path.join(paths["appPath"], "browser.cfg") + "' -> Connection configuration about how the browsers will be connected."
configInfo += "\n\t- '" + os.path.join(paths["appPath"], "general.cfg") + "' -> General configuration of the different utils containing the default options."
configInfo += "\n\t- '" + paths["appPathDefaults"] + "/' -> Directory containing default files as a backup."
configInfo += "\n\t- '" + paths["appPathPlugins"] + "/' -> Directory containing the details of the user defined plugins."
configInfo += "\n\t- '" + paths["appPathPatterns"] + "/' -> Directory containing the user-defined patterns for entify.py."
configInfo += "\n\t- '" + paths["appPathWrappers"] + "/' -> Directory containing the user-defined wrappers for usufy, searchfy and phonefy platforms."
configInfo += "\n"
print(general.title(self.info) + general.info(configInfo))
def main(args):
'''
Main function. This function is created in this way so as to let other applications make use of the full configuration capabilities of the application.
'''
# Recovering the logger
# Calling the logger when being imported
osrframework.utils.logger.setupLogger(loggerName="osrframework.usufy", verbosity=args.verbose, logFolder=args.logfolder)
# From now on, the logger can be recovered like this:
logger = logging.getLogger("osrframework.usufy")
# Printing the results if requested
if not args.maltego:
print(general.title(banner.text))
sayingHello = """
usufy.py Copyright (C) F. Brezo and Y. Rubio (i3visio) 2014-2017
This program comes with ABSOLUTELY NO WARRANTY. This is free software, and you
are welcome to redistribute it under certain conditions. For additional info,
visit <http://www.gnu.org/licenses/gpl-3.0.txt>."""
logger.info(sayingHello)
print(general.title(sayingHello))
logger.info("Starting usufy.py...")
if args.license:
logger.info("Looking for the license...")
# showing the license
try:
with open ("COPYING", "r") as iF:
contenido = iF.read().splitlines()
for linea in contenido:
print(linea)
except Exception:
try:
# Trying to recover the COPYING file...
with open ("/usr/share/osrframework/COPYING", "r") as iF:
contenido = iF.read().splitlines()
for linea in contenido:
print(linea)
except:
logger.error("ERROR: there has been an error when opening the COPYING file.\n\tThe file contains the terms of the GPLv3 under which this software is distributed.\n\tIn case of doubts, verify the integrity of the files or contact [email protected].")
elif args.fuzz:
logger.info("Performing the fuzzing tasks...")
res = fuzzUsufy(args.fuzz, args.fuzz_config)
logger.info("Recovered platforms:\n" + str(res))
else:
logger.debug("Recovering the list of platforms to be processed...")
# Recovering the list of platforms to be launched
listPlatforms = platform_selection.getPlatformsByName(platformNames=args.platforms, tags=args.tags, mode="usufy", excludePlatformNames=args.exclude)
logger.debug("Platforms recovered.")
if args.info:
# Information actions...
if args.info == 'list_platforms':
infoPlatforms="Listing the platforms:\n"
for p in listPlatforms:
infoPlatforms += "\t\t" + (str(p) + ": ").ljust(16, ' ') + str(p.tags)+"\n"
logger.info(infoPlatforms)
return infoPlatforms
elif args.info == 'list_tags':
logger.info("Listing the tags:")
tags = {}
# Going through all the selected platforms to get their tags
for p in listPlatforms:
for t in p.tags:
if t not in tags.keys():
tags[t] = 1
else:
tags[t] += 1
infoTags = "List of tags:\n"
# Displaying the results in a sorted list
for t in tags.keys():
infoTags += "\t\t" + (t + ": ").ljust(16, ' ') + str(tags[t]) + " time(s)\n"
logger.info(infoTags)
return infoTags
else:
pass
# performing the test
elif args.benchmark:
logger.warning("The benchmark mode may last some minutes as it will be performing similar queries to the ones performed by the program in production. ")
logger.info("Launching the benchmarking tests...")
platforms = platform_selection.getAllPlatformNames("usufy")
res = benchmark.doBenchmark(platforms)
strTimes = ""
for e in sorted(res.keys()):
strTimes += str(e) + "\t" + str(res[e]) + "\n"
logger.info(strTimes)
return strTimes
# showing the tags of the usufy platforms
elif args.show_tags:
logger.info("Collecting the list of tags...")
tags = platform_selection.getAllPlatformNamesByTag("usufy")
logger.info(json.dumps(tags, indent=2))
print(general.info("This is the list of platforms grouped by tag.\n"))
print(json.dumps(tags, indent=2, sort_keys=True))
print(general.info("[Tip] Remember that you can always launch the platform using the -t option followed by any of the aforementioned.\n"))
return tags
# Executing the corresponding process...
else:
# Showing the execution time...
if not args.maltego:
startTime= dt.datetime.now()
print("\n")
print(str(startTime) + "\tStarting search in " + str(len(listPlatforms)) + " platform(s)... Relax!\n")
print(general.emphasis("\tPress <Ctrl + C> to stop...\n"))
# Defining the list of users to monitor
nicks = []
logger.debug("Recovering nicknames to be processed...")
if args.nicks:
for n in args.nicks:
# TO-DO
# A trick to avoid having the processing of the properties when being queried by Maltego
if "properties.i3visio" not in n:
nicks.append(n)
else:
# Reading the nick files
try:
nicks = args.list.read().splitlines()
except:
logger.error("ERROR: there has been an error when opening the file that stores the nicks.\tPlease, check the existence of this file.")
# Definning the results
res = []
if args.output_folder != None:
# if Verifying an output folder was selected
logger.debug("Preparing the output folder...")
if not args.maltego:
if not os.path.exists(args.output_folder):
logger.warning("The output folder \'" + args.output_folder + "\' does not exist. The system will try to create it.")
os.makedirs(args.output_folder)
# Launching the process...
res = processNickList(nicks, listPlatforms, args.output_folder, avoidProcessing = args.avoid_processing, avoidDownload = args.avoid_download, nThreads=args.threads, verbosity= args.verbose, logFolder=args.logfolder)
else:
try:
res = processNickList(nicks, listPlatforms, nThreads=args.threads, verbosity= args.verbose, logFolder=args.logfolder)
except Exception as e:
print(general.error("Exception grabbed when processing the nicks: " + str(e)))
print(general.error(traceback.print_stack()))
logger.info("Listing the results obtained...")
# We are going to iterate over the results...
strResults = "\t"
# Structure returned
"""
[
{
"attributes": [
{
"attributes": [],
"type": "i3visio.uri",
"value": "http://twitter.com/i3visio"
},
{
"attributes": [],
"type": "i3visio.alias",
"value": "i3visio"
},
{
"attributes": [],
"type": "i3visio.platform",
"value": "Twitter"
}
],
"type": "i3visio.profile",
"value": "Twitter - i3visio"
}
,
...
]
"""
for r in res:
# The format of the results (attributes) for a given nick is a list as follows:
for att in r["attributes"]:
# iterating through the attributes
platform = ""
uri = ""
for details in att["attributes"]:
if details["type"] == "i3visio.platform":
platform = details["value"]
if details["type"] == "i3visio.uri":
uri = details["value"]
try:
strResults+= (str(platform) + ":").ljust(16, ' ')+ " "+ str(uri)+"\n\t\t"
except:
pass
logger.info(strResults)
# Generating summary files for each ...
if args.extension:
# Storing the file...
logger.info("Creating output files as requested.")
if not args.maltego:
# Verifying if the outputPath exists
if not os.path.exists (args.output_folder):
logger.warning("The output folder \'" + args.output_folder + "\' does not exist. The system will try to create it.")
os.makedirs(args.output_folder)
# Grabbing the results
fileHeader = os.path.join(args.output_folder, args.file_header)
# Iterating through the given extensions to print its values
for ext in args.extension:
# Generating output files
general.exportUsufy(res, ext, fileHeader)
# Generating the Maltego output
if args.maltego:
general.listToMaltego(res)
# Printing the results if requested
else:
now = dt.datetime.now()
print(str(now) + "\tA summary of the results obtained are shown in the following table:\n")
print(general.success(unicode(general.usufyToTextExport(res))))
if args.web_browser:
general.openResultsInBrowser(res)
print("\n")
now = dt.datetime.now()
print(str(now) + "\tYou can find all the information collected in the following files:")
for ext in args.extension:
# Showing the output files
print("\t-" + general.emphasis(fileHeader + "." + ext))
# Showing the execution time...
endTime= dt.datetime.now()
print("\n" + str(endTime) +"\tFinishing execution...\n")
print("Total time used:\t" + str(endTime-startTime))
print("Average seconds/query:\t" + str((endTime-startTime).total_seconds()/len(listPlatforms)) +" seconds\n")
# Urging users to place an issue on Github...
print("Did something go wrong? Is a platform reporting false positives? Do you need to integrate a new one?")
print("Then, place an issue in the Github project: <https://github.com/i3visio/osrframework/issues>.")
print("Note that otherwise, we won't know about it!\n")
return res
def main(params=None):
"""
Main function to launch phonefy.
The function is created in this way so as to let other applications make
use of the full configuration capabilities of the application. The
parameters received are used as parsed by this modules `getParser()`.
Args:
-----
params: A list with the parameters as grabbed by the terminal. It is
None when this is called by an entry_point. If it is called by osrf
the data is already parsed.
Results:
--------
Returns a list with i3visio entities.
"""
if params == None:
parser = getParser()
args = parser.parse_args(params)
else:
args = params
results = []
# Recovering the logger
# Calling the logger when being imported
logSet.setupLogger(loggerName="osrframework.entify",
verbosity=args.verbose,
logFolder=args.logfolder)
# From now on, the logger can be recovered like this:
logger = logging.getLogger("osrframework.entify")
logger.info("Selecting the regular expressions to be analysed...")
if not args.quiet:
print(general.title(banner.text))
sayingHello = """
Entify | Copyright (C) Yaiza Rubio & Félix Brezo (i3visio) 2014-2018
This program comes with ABSOLUTELY NO WARRANTY. This is free software, and you
are welcome to redistribute it under certain conditions. For additional info,
visit <{}>.
""".format(general.LICENSE_URL)
print(general.info(sayingHello))
if args.license:
general.showLicense()
else:
listRegexp = []
if args.regexp:
listRegexp = regexp_selection.getRegexpsByName(args.regexp)
elif args.new_regexp:
for i, r in enumerate(args.new_regexp):
listRegexp.append(
RegexpObject(name="NewRegexp" + str(i),
reg_exp=args.new_regexp))
if not args.web:
results = scanFolderForRegexp(folder=args.input_folder,
listRegexp=listRegexp,
recursive=args.recursive,
verbosity=args.verbose,
logFolder=args.logfolder,
quiet=args.quiet)
else:
results = scanResource(uri=args.web,
listRegexp=listRegexp,
verbosity=args.verbose,
logFolder=args.logfolder)
logger.info("Logging the results:\n" +
json.dumps(results, indent=2, sort_keys=True))
# Trying to store the information recovered
if args.output_folder != None:
# Verifying an output folder was selected
logger.debug("Preparing the output folder...")
if not os.path.exists(args.output_folder):
logger.warning(
"The output folder \'" + args.output_folder +
"\' does not exist. The system will try to create it.")
os.makedirs(args.output_folder)
# Grabbing the results
fileHeader = os.path.join(args.output_folder, args.file_header)
for ext in args.extension:
# Generating output files
general.exportUsufy(results, ext, fileHeader)
# Showing the information gathered if requested
if not args.quiet:
now = dt.datetime.now()
print("\n{}\tResults obtained:\n".format(str(now)))
print(general.success(general.usufyToTextExport(results)))
now = dt.datetime.now()
print(
str(now) +
"\tYou can find all the information collected in the following files:"
)
for ext in args.extension:
# Showing the output files
print("\t-" + general.emphasis(fileHeader + "." + ext))
# Urging users to place an issue on Github...
print(banner.footer)
if params:
return results
def do_info(self, line):
"""This command shows all the information available about the module."""
print(general.info("Displaying module information."))
self.do_show("options")
self.do_show("command")
class OSRFConsoleMain(cmd.Cmd):
"""
OSRFramework console application to control the different framework utils
The user can type 'help' at any time to find the available commands included
in the framework.
"""
DISCLAIMER = """\tOSRFramework """ + osrframework.__version__ + """ - Copyright (C) F. Brezo and Y. Rubio (i3visio) 2016-2017
This program comes with ABSOLUTELY NO WARRANTY. This software is free
software, and you are really welcome to redistribute it under certain
conditions. For additional information about the terms and conditions of the
AGPLv3+ license, visit <http://www.gnu.org/licenses/agpl-3.0.txt>."""
intro = banner.text + "\n" + DISCLAIMER + "\n"
info = general.info("") + general.emphasis("""
General information
===================""") + """
OSRFConsole is a terminal GUI to interact with OSRFramework utilities.
OSRFramework stands for Open Sources Research Framework. It includes a set
of tools that help the analyst in the task of user profiling making use of
different OSINT tools.
To get additional information about the available commands type 'help'.
""" + general.emphasis("""Modules available:
------------------""") + """
- """ + general.success("usufy") + """ --> the Jewel of the Chrown. A tool that verifies if a
username exists in """ + str(len(platform_selection.getAllPlatformNames("usufy"))) + """ platforms.
- """ + general.success("mailfy") + """ --> a tool to check if a username has been registered in up to
""" + str(len(mailfy.EMAIL_DOMAINS )) + """ email providers.
- """ + general.success("searchfy") + """ --> a tool to look for profiles using full names and other
info in """ + str(len(platform_selection.getAllPlatformNames("searchfy"))) + """ platforms.
- """ + general.success("domainfy") + """ --> a tool to check the existence of a given domain in up to
""" + str(domainfy.getNumberTLD()) + """ different TLDs.
- """ + general.success("phonefy") + """ --> a tool that checks if a phone number has been linked to
spam practices in """ + str(len(platform_selection.getAllPlatformNames("phonefy"))) + """ platforms.
- """ + general.success("entify") + """ --> a util to look for regular expressions using """ + str(len(regexp_selection.getAllRegexpNames())) + """ patterns."""
# Appending the self.info data to the headers...
intro += info
# Defining the prompt
prompt = general.emphasis('\nosrf > ')
ruler = '='
def do_info(self, line):
"""
Command that shows again the general information about the application
Args:
-----
line: the string of the line typed.
"""
configInfo = """
""" + general.emphasis("""Additional configuration files:
-------------------------------""") + """
You will be able to find more configuration options in the following files
in your system. The relevant paths are the ones that follow:"""
# Get the configuration folders in each system
paths = configuration.getConfigPath()
configInfo += """
- Configuration details about the login credentials in OSRFramework:
""" + general.info(os.path.join(paths["appPath"], "accounts.cfg")) + """
- Configuration details about the API credentials already configured:
""" + general.info(os.path.join(paths["appPath"], "api_keys.cfg")) + """
- Connection configuration about how the browsers will be connected:
""" + general.info(os.path.join(paths["appPath"], "browser.cfg")) + """
- General default configuration of the the utils:
""" + general.info(os.path.join(paths["appPath"], "general.cfg")) + """
- Directory containing default files as a backup:
""" + general.info(paths["appPathDefaults"]) + """
- Directory containing the user-defined patterns for entify.py:
""" + general.info(paths["appPathPatterns"]) + """
- Directory containing the user-defined wrappers for usufy platforms:
""" + general.info(paths["appPathWrappers"])
print(self.info + configInfo)
def do_use(self, line):
"""
This command will define which of the framework's utilities will be loaded.
The available options are the following:
- domainfy
- entify
- mailfy
- phonefy
- searchfy
- usufy
For example, type 'use usufy' to load the usufy util. You can always use
the <TAB> to be helped using the autocomplete options.
Args:
-----
line: the string of the line typed.
"""
if line not in UTILS:
print(general.warning("[!] Util is not correct. Try 'help use' to check the available options."))
return False
elif line == "domainfy":
OSRFConsoleDomainfy().cmdloop()
elif line == "entify":
OSRFConsoleEntify().cmdloop()
elif line == "mailfy":
OSRFConsoleMailfy().cmdloop()
elif line == "phonefy":
OSRFConsolePhonefy().cmdloop()
elif line == "searchfy":
OSRFConsoleSearchfy().cmdloop()
elif line == "usufy":
OSRFConsoleUsufy().cmdloop()
else:
print(general.warning("[!] Not implemented yet. Try 'help use' to check the available options."))
def complete_use(self, text, line, begidx, endidx):
if not text:
completions = UTILS
else:
completions = [ f
for f in UTILS
if f.startswith(text.lower())
]
return completions
def do_exit(self, line):
"""
This command will exit the osrfconsole normally
Args:
-----
line: the string of the line typed.
"""
print("\nExiting...\n")
sys.exit()
def main(params=None):
"""Main function to launch mailfy
The function is created in this way so as to let other applications make
use of the full configuration capabilities of the application. The
parameters received are used as parsed by this modules `get_parser()`.
Args:
params: A list with the parameters as grabbed by the terminal. It is
None when this is called by an entry_point. If it is called by osrf
the data is already parsed.
Returns:
list. A list of i3visio entities.
"""
if params is None:
parser = get_parser()
args = parser.parse_args(params)
else:
args = params
results = []
if not args.quiet:
print(general.title(banner.text))
saying_hello = f"""
Mailfy | Copyright (C) Yaiza Rubio & Félix Brezo (i3visio) 2014-2020
This program comes with ABSOLUTELY NO WARRANTY. This is free software, and you
are welcome to redistribute it under certain conditions. For additional info,
visit <{general.LICENSE_URL}>.
"""
print(general.info(saying_hello))
# Displaying a warning if this is being run in a windows system
if sys.platform == 'win32':
print(
general.warning(
"""OSRFramework has detected that you are running mailfy in a Windows system.
As the "emailahoy" library is NOT working properly there, "validate_email" will
be used instead. Verification may be slower though."""))
if args.license:
general.showLicense()
else:
# processing only the given domains and excluding the ones provided
extra_domains = []
for d in args.domains:
if d not in args.exclude and not d == "all":
extra_domains.append(d)
# Two different arrays are mantained since there are some domains that cannot be safely verified
if args.create_emails:
potentially_existing_emails = grab_emails(
nicks_file=args.create_emails,
domains=EMAIL_DOMAINS + extra_domains,
exclude_domains=args.exclude)
potentially_leaked_emails = grab_emails(
nicks_file=args.create_emails,
domains=LEAKED_DOMAINS + extra_domains,
exclude_domains=args.exclude)
else:
potentially_existing_emails = grab_emails(
emails=args.emails,
emails_file=args.emails_file,
nicks=args.nicks,
nicks_file=args.nicks_file,
domains=EMAIL_DOMAINS + extra_domains,
exclude_domains=args.exclude)
potentially_leaked_emails = grab_emails(
emails=args.emails,
emails_file=args.emails_file,
nicks=args.nicks,
nicks_file=args.nicks_file,
domains=LEAKED_DOMAINS + extra_domains,
exclude_domains=args.exclude)
emails = list(
set(potentially_leaked_emails + potentially_existing_emails))
if not args.quiet:
start_time = dt.datetime.now()
print(
f"\n{start_time}\t{general.emphasis('Step 1/5')}. Trying to determine if any of the following {general.emphasis(str(len(potentially_existing_emails)))} emails exist using emailahoy3...\n{general.emphasis(json.dumps(potentially_existing_emails, indent=2))}\n"
)
print(
general.emphasis("\tPress <Ctrl + C> to skip this step...\n"))
# Perform searches, using different Threads
try:
results = verify_with_emailahoy_step_1(potentially_existing_emails,
num_threads=args.threads)
except KeyboardInterrupt:
print(
general.warning("\tStep 1 manually skipped by the user...\n"))
results = []
# Grabbing the <Platform> objects
platforms = platform_selection.get_platforms_by_name(args.platforms,
mode="mailfy")
names = [p.platformName for p in platforms]
if not args.quiet:
now = dt.datetime.now()
print(
f"\n{now}\t{general.emphasis('Step 2/5')}. Checking if the emails have been used to register accounts in {general.emphasis(str(len(platforms)))} platforms...\n{general.emphasis(json.dumps(names, indent=2))}\n"
)
print(
general.emphasis("\tPress <Ctrl + C> to skip this step...\n"))
try:
registered = process_mail_list_step_2(platforms=platforms,
emails=emails)
except KeyboardInterrupt:
print(
general.warning("\tStep 2 manually skipped by the user...\n"))
registered = []
results += registered
if not args.quiet:
if len(results) > 0:
for r in registered:
print(
f"\t[*] Linked account found: {general.success(r['value'])}"
)
else:
print(f"\t[*] No account found.")
now = dt.datetime.now()
print(
f"\n{now}\t{general.emphasis('Step 3/5')}. Verifying if the provided emails have been leaked somewhere using HaveIBeenPwned.com...\n"
)
print(
general.emphasis("\tPress <Ctrl + C> to skip this step...\n"))
all_keys = config_api_keys.get_list_of_api_keys()
try:
# Verify the existence of the mails found as leaked emails.
for query in potentially_leaked_emails:
# Iterate through the different leak platforms
leaks = hibp.check_if_email_was_hacked(
query, api_key=all_keys["haveibeenpwned_com"]["api_key"])
if len(leaks) > 0:
if not args.quiet:
print(
f"\t[*] '{general.success(query)}' has been found in at least {general.success(len(leaks))} different leaks."
)
else:
if not args.quiet:
print(
f"\t[*] '{general.error(query)}' has NOT been found on any leak yet."
)
results += leaks
except KeyError:
# API_Key not found
config_path = os.path.join(
configuration.get_config_path()["appPath"], "api_keys.cfg")
print(
"\t[*] " + general.warning("No API found for HaveIBeenPwned") +
f". Request one at <https://haveibeenpwned.com/API/Key> and add it to '{config_path}'."
)
except KeyboardInterrupt:
print(
general.warning("\tStep 3 manually skipped by the user...\n"))
if not args.quiet:
now = dt.datetime.now()
print(
f"\n{now}\t{general.emphasis('Step 4/5')}. Verifying if the provided emails have been leaked somewhere using Dehashed.com...\n"
)
print(
general.emphasis("\tPress <Ctrl + C> to skip this step...\n"))
try:
# Verify the existence of the mails found as leaked emails.
for query in emails:
try:
# Iterate through the different leak platforms
leaks = dehashed.check_if_email_was_hacked(query)
if len(leaks) > 0:
if not args.quiet:
print(
f"\t[*] '{general.success(query)}' has been found in at least {general.success(len(leaks))} different leaks as shown by Dehashed.com."
)
else:
if not args.quiet:
print(
f"\t[*] '{general.error(query)}' has NOT been found on any leak yet."
)
results += leaks
except Exception as e:
print(
general.warning(
f"Something happened when querying Dehashed.com about '{email}'. Omitting..."
))
except KeyboardInterrupt:
print(
general.warning("\tStep 4 manually skipped by the user...\n"))
if not args.quiet:
now = dt.datetime.now()
print(
f"\n{now}\t{general.emphasis('Step 5/5')}. Verifying if the provided emails have registered a domain using ViewDNS.info...\n"
)
print(
general.emphasis("\tPress <Ctrl + C> to skip this step...\n"))
try:
# Verify the existence of the mails found as leaked emails.
for query in potentially_leaked_emails:
try:
# Iterate through the different leak platforms
domains = viewdns.check_reverse_whois(query)
if len(domains) > 0:
if not args.quiet:
print(
f"\t[*] '{general.success(query)}' has registered at least {general.success(len(domains))} different domains as shown by ViewDNS.info."
)
else:
if not args.quiet:
print(
f"\t[*] '{general.error(query)}' has NOT registered a domain yet."
)
results += domains
except Exception as e:
print(
general.warning(
f"Something happened when querying Viewdns.info about '{query}'. Omitting..."
))
except KeyboardInterrupt:
print(
general.warning("\tStep 5 manually skipped by the user...\n"))
# Trying to store the information recovered
if args.output_folder != None:
if not os.path.exists(args.output_folder):
os.makedirs(args.output_folder)
# Grabbing the results
fileHeader = os.path.join(args.output_folder, args.file_header)
for ext in args.extension:
# Generating output files
general.export_usufy(results, ext, fileHeader)
# Showing the information gathered if requested
if not args.quiet:
now = dt.datetime.now()
print(f"\n{now}\tResults obtained:\n")
print(general.success(general.osrf_to_text_export(results)))
now = dt.datetime.now()
print(
f"\n{now}\tYou can find all the information collected in the following files:"
)
for ext in args.extension:
# Showing the output files
print(general.emphasis("\t" + fileHeader + "." + ext))
# Showing the execution time...
if not args.quiet:
end_time = dt.datetime.now()
print("\n{end_time}\tFinishing execution...\n")
print("Total time used:\t" +
general.emphasis(str(end_time - start_time)))
if not args.quiet:
# Urging users to place an issue on Github...
print(banner.footer)
if params:
return results
def main(params=None):
"""
Main function to launch phonefy.
The function is created in this way so as to let other applications make
use of the full configuration capabilities of the application. The
parameters received are used as parsed by this modules `getParser()`.
Args:
-----
params: A list with the parameters as grabbed by the terminal. It is
None when this is called by an entry_point. If it is called by osrf
the data is already parsed.
Results:
--------
list: Returns a list with i3visio entities.
"""
if params == None:
parser = getParser()
args = parser.parse_args(params)
else:
args = params
results = []
if not args.quiet:
print(general.title(banner.text))
sayingHello = """
Checkfy | Copyright (C) Yaiza Rubio & Félix Brezo (i3visio) 2014-2018
This program comes with ABSOLUTELY NO WARRANTY. This is free software, and you
are welcome to redistribute it under certain conditions. For additional info,
visit <{}>.
""".format(general.LICENSE_URL)
print(general.info(sayingHello))
if args.license:
general.showLicense()
else:
# Processing the options returned to remove the "all" option
if args.nicks:
emails = createEmails(nicks=args.nicks)
else:
# nicks_file
emails = createEmails(nicksFile=args.nicks_file)
# Showing the execution time...
if not args.quiet:
startTime= dt.datetime.now()
print(str(startTime) + "\tTrying to identify possible emails " + general.emphasis(str(len(emails))) + " email(s)... Relax!\n")
print(general.emphasis("\tPress <Ctrl + C> to stop...\n"))
print(args.pattern)
if args.type == "twitter":
pattern = args.pattern.replace(".", "\.")
pattern = pattern.replace("*", ".")
pattern = "^{}$".format(pattern)
elif args.type == "regexp":
pattern = args.pattern
# Perform searches, using different Threads
results = verifyEmails(emails, pattern)
# Sorting list
results.sort()
print("\nProcess finished.")
print("\nValidated emails:\n")
print(general.success(json.dumps(results, indent=2, sort_keys=True)))
print("\nUp to " + general.emphasis(str(len(results))) + " possible emails foundd.\n")
# Trying to store the information recovered
if args.output_folder != None:
if not os.path.exists(args.output_folder):
os.makedirs(args.output_folder)
outputPath = os.path.join(args.output_folder, "possible_emails.txt")
print("Writing the results onto the file:\n\t" + general.emphasis(outputPath))
with open(outputPath, "w") as oF:
for r in results:
oF.write(r+"\n")
# Showing the execution time...
if not args.quiet:
# Showing the execution time...
endTime= dt.datetime.now()
print("\n" + str(endTime) +"\tFinishing execution...\n")
print("Total time used:\t" + general.emphasis(str(endTime-startTime)))
print("Average seconds/query:\t" + general.emphasis(str((endTime-startTime).total_seconds()/len(emails))) +" seconds\n")
# Urging users to place an issue on Github...
print(banner.footer)
if params:
return results
def main(params=None):
"""
Main function to launch phonefy.
The function is created in this way so as to let other applications make
use of the full configuration capabilities of the application. The
parameters received are used as parsed by this modules `getParser()`.
Args:
-----
params: A list with the parameters as grabbed by the terminal. It is
None when this is called by an entry_point. If it is called by osrf
the data is already parsed.
Returns:
--------
A list of i3visio entities.
"""
if params == None:
parser = getParser()
args = parser.parse_args(params)
else:
args = params
results = []
if not args.quiet:
print(general.title(banner.text))
sayingHello = """
Mailfy | Copyright (C) Yaiza Rubio & Félix Brezo (i3visio) 2014-2018
This program comes with ABSOLUTELY NO WARRANTY. This is free software, and you
are welcome to redistribute it under certain conditions. For additional info,
visit <{}>.
""".format(general.LICENSE_URL)
print(general.info(sayingHello))
# Displaying a warning if this is being run in a windows system
if sys.platform == 'win32':
print(
general.warning(
"""OSRFramework has detected that you are running mailfy.py in a Windows system.
As the "emailahoy" library is NOT working properly there, "validate_email" will
be used instead. Verification may be slower though."""))
if args.license:
general.showLicense()
else:
# processing only the given domains and excluding the ones provided
extra_domains = []
for d in args.domains:
if d not in args.exclude and not d == "all":
extra_domains.append(d)
# Two different arrays are mantained since there are some domains that cannot be safely verified
if args.create_emails:
potentially_existing_emails = grabEmails(
nicksFile=args.create_emails,
domains=EMAIL_DOMAINS + extra_domains,
excludeDomains=args.exclude)
potentially_leaked_emails = grabEmails(
nicksFile=args.create_emails,
domains=LEAKED_DOMAINS + extra_domains,
excludeDomains=args.exclude)
else:
potentially_existing_emails = grabEmails(
emails=args.emails,
emailsFile=args.emails_file,
nicks=args.nicks,
nicksFile=args.nicks_file,
domains=EMAIL_DOMAINS + extra_domains,
excludeDomains=args.exclude)
potentially_leaked_emails = grabEmails(emails=args.emails,
emailsFile=args.emails_file,
nicks=args.nicks,
nicksFile=args.nicks_file,
domains=LEAKED_DOMAINS +
extra_domains,
excludeDomains=args.exclude)
emails = list(
set(potentially_leaked_emails + potentially_existing_emails))
# Showing the execution time...
if not args.quiet:
startTime = dt.datetime.now()
print("{}\tStarting search of {} different emails:\n{}\n".format(
str(startTime), general.emphasis(str(len(emails))),
json.dumps(emails, indent=2, sort_keys=True)))
if not args.quiet:
now = dt.datetime.now()
print(
"\n{}\tStep 1. Trying to determine if the emails provided do exist...\n"
.format(str(now)))
print(general.emphasis("\tPress <Ctrl + C> to stop...\n"))
# Perform searches, using different Threads
results = performSearch(potentially_existing_emails,
nThreads=args.threads)
if not args.quiet:
now = dt.datetime.now()
print(
"\n{}\tStep 2. Checking if the emails have been used to register socialmedia accounts...\n"
.format(str(now)))
print(general.emphasis("\tPress <Ctrl + C> to stop...\n"))
registered = processMailList(platformNames=args.platforms,
emails=potentially_existing_emails)
results += registered
if not args.quiet:
if len(results) > 0:
for r in registered:
print("\t[*] Registered account found: {}".format(
general.success(r["value"])))
else:
print("\t[*] Registered account found: {}".format(
general.error("None")))
now = dt.datetime.now()
print(
"\n{}\tStep 3. Verifying if the provided emails have been leaked somewhere?\n"
.format(str(now)))
print(general.emphasis("\tPress <Ctrl + C> to stop...\n"))
# Verify the existence of the mails found as leaked emails.
for query in potentially_leaked_emails:
# Iterate through the different leak platforms
leaks = hibp.checkIfEmailWasHacked(query)
if len(leaks) > 0:
if not args.quiet:
if len(leaks) > 0:
print(
"\t[*] '{}' has been found in at least {} different leaks."
.format(general.success(query),
general.success(str(len(leaks)))))
else:
print("\t[*] '{}' has NOT been found in any leak.".
format(general.error(query)))
else:
if not args.quiet:
print("\t[*] '{}' has NOT been found on any leak yet.".
format(general.error(query)))
results += leaks
# Trying to store the information recovered
if args.output_folder != None:
if not os.path.exists(args.output_folder):
os.makedirs(args.output_folder)
# Grabbing the results
fileHeader = os.path.join(args.output_folder, args.file_header)
for ext in args.extension:
# Generating output files
general.exportUsufy(results, ext, fileHeader)
# Showing the information gathered if requested
if not args.quiet:
now = dt.datetime.now()
print("\n{}\tResults obtained:\n".format(str(now)))
print(general.success(general.usufyToTextExport(results)))
now = dt.datetime.now()
print(
"\n" + str(now) +
"\tYou can find all the information collected in the following files:"
)
for ext in args.extension:
# Showing the output files
print(general.emphasis("\t" + fileHeader + "." + ext))
# Showing the execution time...
if not args.quiet:
endTime = dt.datetime.now()
print("\n" + str(endTime) + "\tFinishing execution...\n")
print("Total time used:\t" +
general.emphasis(str(endTime - startTime)))
print("Average seconds/query:\t" + general.emphasis(
str((endTime - startTime).total_seconds() / len(emails))) +
" seconds\n")
if not args.quiet:
# Urging users to place an issue on Github...
print(banner.footer)
if params:
return results
'choose whether error messages will be deployed. Do NOT use this for production.'
)
groupAbout = parser.add_argument_group(
'About arguments',
'Showing additional information about this program.')
groupAbout.add_argument('-h',
'--help',
action='help',
help='shows this help and exists.')
groupAbout.add_argument(
'--version',
action='version',
version='%(prog)s ' + " " + osrframework.__version__,
help='shows the version of the program and exists.')
# Parse args
args = parser.parse_args()
print(general.title(osrframework.utils.banner.text))
# Starting the server
print(
general.info("[*] Server started at " + "http://" + args.host + ":" +
str(args.port) +
"... You can access it in your browser."))
print(general.info("[*] Press <Ctrl + C> at any time to stop the server."))
app.run(debug=args.debug, host=args.host, port=args.port)
print(general.info("\n[*] OSRFramework server exited normally."))
def main(params=None):
"""ain function to launch usufy
The function is created in this way so as to let other applications make
use of the full configuration capabilities of the application. The
parameters received are used as parsed by this modules `get_parser()`.
Args:
params: A list with the parameters as grabbed by the terminal. It is
None when this is called by an entry_point. If it is called by osrf
the data is already parsed.
Returns:
dict: A Json representing the matching results.
"""
if params is None:
parser = get_parser()
args = parser.parse_args(params)
else:
args = params
print(general.title(banner.text))
saying_hello = f"""
Usufy | Copyright (C) Yaiza Rubio & Félix Brezo (i3visio) 2014-2020
This program comes with ABSOLUTELY NO WARRANTY. This is free software, and you
are welcome to redistribute it under certain conditions. For additional info,
visit <{general.LICENSE_URL}>.
"""
print(general.info(saying_hello))
if args.fuzz:
res = fuzzUsufy(args.fuzz, args.fuzz_config)
else:
# Recovering the list of platforms to be launched
list_platforms = platform_selection.get_platforms_by_name(
platform_names=args.platforms,
tags=args.tags,
mode="usufy",
exclude_platform_names=args.exclude)
if args.info:
# Information actions...
if args.info == 'list_platforms':
info_platforms = "Listing the platforms:\n"
for p in list_platforms:
info_platforms += "\t\t" + (str(p) + ": ").ljust(
16, ' ') + str(p.tags) + "\n"
return info_platforms
elif args.info == 'list_tags':
tags = {}
# Going through all the selected platforms to get their tags
for p in list_platforms:
for t in p.tags:
if t not in tags.keys():
tags[t] = 1
else:
tags[t] += 1
info_tags = "List of tags:\n"
# Displaying the results in a sorted list
for t in tags.keys():
info_tags += "\t\t" + (t + ": ").ljust(16, ' ') + str(
tags[t]) + " time(s)\n"
return info_tags
else:
pass
# performing the test
elif args.benchmark:
platforms = platform_selection.get_all_platform_names("usufy")
res = benchmark.do_benchmark(platforms)
str_times = ""
for e in sorted(res.keys()):
str_times += str(e) + "\t" + str(res[e]) + "\n"
return str_times
# showing the tags of the usufy platforms
elif args.show_tags:
tags = platform_selection.get_all_platform_namesByTag("usufy")
print(
general.info(
"This is the list of platforms grouped by tag.\n"))
print(json.dumps(tags, indent=2, sort_keys=True))
print(
general.info(
"[Tip] Remember that you can always launch the platform using the -t option followed by any of the aforementioned.\n"
))
return tags
# Executing the corresponding process...
else:
# Showing the execution time...
start_time = dt.datetime.now()
print(
f"{start_time}\tStarting search in {general.emphasis(str(len(list_platforms)))} platform(s)... Relax!\n"
)
print(general.emphasis("\tPress <Ctrl + C> to stop...\n"))
# Defining the list of users to monitor
nicks = []
if args.nicks:
for n in args.nicks:
nicks.append(n)
else:
# Reading the nick files
try:
nicks = args.list.read().splitlines()
except:
print(
general.error(
"ERROR: there has been an error when opening the file that stores the nicks.\tPlease, check the existence of this file."
))
# Definning the results
res = []
if args.output_folder != None:
# if Verifying an output folder was selected
if not os.path.exists(args.output_folder):
os.makedirs(args.output_folder)
# Launching the process...
res = process_nick_list(nicks,
list_platforms,
args.output_folder,
avoidProcessing=args.avoid_processing,
avoidDownload=args.avoid_download,
nThreads=args.threads,
verbosity=args.verbose,
logFolder=args.logfolder)
else:
try:
res = process_nick_list(nicks,
list_platforms,
nThreads=args.threads,
verbosity=args.verbose,
logFolder=args.logfolder)
except Exception as e:
print(
general.error(
"Exception grabbed when processing the nicks: " +
str(e)))
print(general.error(traceback.print_stack()))
# We are going to iterate over the results...
str_results = "\t"
# Structure returned
"""
[
{
"attributes": [
{
"attributes": [],
"type": "com.i3visio.URI",
"value": "http://twitter.com/i3visio"
},
{
"attributes": [],
"type": "com.i3visio.Alias",
"value": "i3visio"
},
{
"attributes": [],
"type": "com.i3visio.Platform",
"value": "Twitter"
}
],
"type": "com.i3visio.Profile",
"value": "Twitter - i3visio"
}
,
...
]
"""
for r in res:
# The format of the results (attributes) for a given nick is a list as follows:
for att in r["attributes"]:
# iterating through the attributes
platform = ""
uri = ""
for details in att["attributes"]:
if details["type"] == "com.i3visio.Platform":
platform = details["value"]
if details["type"] == "com.i3visio.URI":
uri = details["value"]
try:
str_results += (str(platform) + ":").ljust(
16, ' ') + " " + str(uri) + "\n\t\t"
except:
pass
# Generating summary files for each ...
if args.extension:
# Verifying if the outputPath exists
if not os.path.exists(args.output_folder):
os.makedirs(args.output_folder)
# Grabbing the results
file_header = os.path.join(args.output_folder,
args.file_header)
# Iterating through the given extensions to print its values
for ext in args.extension:
# Generating output files
general.export_usufy(res, ext, file_header)
now = dt.datetime.now()
print(
f"\n{now}\tResults obtained ({general.emphasis(len(res))}):\n")
print(general.success(general.osrf_to_text_export(res)))
if args.web_browser:
general.open_results_in_browser(res)
now = dt.datetime.now()
print("\n" + str(now) + "\tYou can find all the information here:")
for ext in args.extension:
# Showing the output files
print("\t" + general.emphasis(file_header + "." + ext))
# Showing the execution time...
end_time = dt.datetime.now()
print(f"\n{end_time}\tFinishing execution...\n")
print("Total time consumed:\t" +
general.emphasis(str(end_time - start_time)))
print("Average seconds/query:\t" + general.emphasis(
str((end_time - start_time).total_seconds() /
len(list_platforms))) + " seconds\n")
# Urging users to place an issue on Github...
print(banner.footer)
if params:
return res
def do_exit(self, line):
"""This command will exit the osrfconsole normally."""
print(general.info("Exiting the program..."))
sys.exit()
def main(params=None):
"""Main function to launch searchfy
The function is created in this way so as to let other applications make
use of the full configuration capabilities of the application. The
parameters received are used as parsed by this modules `get_parser()`.
Args:
params (list): A list with the parameters as grabbed by the terminal. It is
None when this is called by an entry_point. If it is called by osrf
the data is already parsed.
Returns:
list. A list of i3visio entities.
"""
if params is None:
parser = get_parser()
args = parser.parse_args(params)
else:
args = params
results = []
print(general.title(banner.text))
saying_hello = f"""
Searchfy | Copyright (C) Yaiza Rubio & Félix Brezo (i3visio) 2014-2020
This program comes with ABSOLUTELY NO WARRANTY. This is free software, and you
are welcome to redistribute it under certain conditions. For additional info,
visit <{general.LICENSE_URL}>.
"""
print(general.info(saying_hello))
if args.license:
general.showLicense()
else:
# Showing the execution time...
start_time = dt.datetime.now()
print(
f"{start_time}\tStarting search in different platform(s)... Relax!\n"
)
print(general.emphasis("\tPress <Ctrl + C> to stop...\n"))
# Performing the search
try:
results = perform_search(platformNames=args.platforms,
queries=args.queries,
exclude_platform_names=args.exclude)
except KeyboardInterrupt:
print(
general.error(
"\n[!] Process manually stopped by the user. Workers terminated without providing any result.\n"
))
results = []
# Generating summary files for each ...
if args.extension:
# Verifying if the outputPath exists
if not os.path.exists(args.output_folder):
os.makedirs(args.output_folder)
# Grabbing the results
fileHeader = os.path.join(args.output_folder, args.file_header)
# Iterating through the given extensions to print its values
for ext in args.extension:
# Generating output files
general.export_usufy(results, ext, fileHeader)
# Printing the results if requested
now = dt.datetime.now()
print(f"\n{now}\tResults obtained:\n")
print(general.success(general.osrf_to_text_export(results)))
if args.web_browser:
general.open_results_in_browser(results)
now = dt.datetime.now()
print(
"\n{date}\tYou can find all the information collected in the following files:"
.format(date=str(now)))
for ext in args.extension:
# Showing the output files
print("\t" + general.emphasis(fileHeader + "." + ext))
# Showing the execution time...
end_time = dt.datetime.now()
print(f"\n{end_time}\tFinishing execution...\n")
print("Total time used:\t" +
general.emphasis(str(end_time - start_time)))
print("Average seconds/query:\t" + general.emphasis(
str((end_time - start_time).total_seconds() /
len(args.platforms))) + " seconds\n")
# Urging users to place an issue on Github...
print(banner.footer)
if params:
return results
def main(params=None):
"""
Main function to launch phonefy.
The function is created in this way so as to let other applications make
use of the full configuration capabilities of the application. The
parameters received are used as parsed by this modules `getParser()`.
Args:
-----
params: A list with the parameters as grabbed by the terminal. It is
None when this is called by an entry_point. If it is called by osrf
the data is already parsed.
Results:
--------
list: Returns a list with i3visio entities.
"""
if params == None:
parser = getParser()
args = parser.parse_args(params)
else:
args = params
results = []
if not args.quiet:
print(general.title(banner.text))
sayingHello = """
Domainfy | Copyright (C) Yaiza Rubio & Félix Brezo (i3visio) 2014-2018
This program comes with ABSOLUTELY NO WARRANTY. This is free software, and you
are welcome to redistribute it under certain conditions. For additional info,
visit <{}>.
""".format(general.LICENSE_URL)
print(general.info(sayingHello))
if args.license:
general.showLicense()
else:
# Processing the options returned to remove the "all" option
tlds = []
if "all" in args.tlds:
for typeTld in TLD.keys():
for tld in TLD[typeTld]:
if tld not in args.exclude:
tlds.append({"tld": tld, "type": typeTld})
elif "none" in args.tlds:
pass
else:
for typeTld in TLD.keys():
if typeTld in args.tlds:
for tld in TLD[typeTld]:
if tld not in args.exclude:
tlds.append({"tld": tld, "type": typeTld})
for new in args.user_defined:
if new not in args.exclude:
tlds.append({"tld": new, "type": "user_defined"})
if args.nicks:
domains = createDomains(tlds, nicks=args.nicks)
else:
# nicks_file
domains = createDomains(tlds, nicksFile=args.nicks_file)
# Showing the execution time...
if not args.quiet:
startTime = dt.datetime.now()
print("{}\tTrying to resolve {} domain(s)…\n".format(
str(startTime), general.emphasis(str(len(domains)))))
if len(domains) > 200:
print(
""" Note that a full '-t all' search may take around 3.5 mins. If that's too
long for you, try narrowing the search using '-t cc' or similar arguments.
Otherwise, just wait and keep calm!
""")
print(general.emphasis("\tPress <Ctrl + C> to stop...\n"))
# Perform searches, using different Threads
results = performSearch(domains, args.threads, args.whois)
# Trying to store the information recovered
if args.output_folder != None:
if not os.path.exists(args.output_folder):
os.makedirs(args.output_folder)
# Grabbing the results
fileHeader = os.path.join(args.output_folder, args.file_header)
for ext in args.extension:
# Generating output files
general.exportUsufy(results, ext, fileHeader)
# Showing the information gathered if requested
if not args.quiet:
now = dt.datetime.now()
print("\n{}\tResults obtained:\n".format(str(now)))
try:
print(general.success(general.usufyToTextExport(results)))
except:
print(
general.warning(
"\nSomething happened when exporting the results. The Json will be shown instead:\n"
))
print(general.warning(json.dumps(results, indent=2)))
now = dt.datetime.now()
print(
"\n" + str(now) +
"\tYou can find all the information collected in the following files:"
)
for ext in args.extension:
# Showing the output files
print("\t" + general.emphasis(fileHeader + "." + ext))
# Showing the execution time...
if not args.quiet:
# Showing the execution time...
endTime = dt.datetime.now()
print("\n{}\tFinishing execution...\n".format(endTime))
print("Total time used:\t" +
general.emphasis(str(endTime - startTime)))
print("Average seconds/query:\t" + general.emphasis(
str((endTime - startTime).total_seconds() / len(domains))) +
" seconds\n")
# Urging users to place an issue on Github...
print(banner.footer)
if params:
return results