def do_set(self, line):
"""
Setting the variables defined in CONFIG
You can check their values at any time by typing 'show options'.
Args:
-----
line: the string of the line typed.
"""
try:
parameter, value = line.split(" ", 1)
# Setting the parameter
if parameter in self.CONFIG.keys():
splittedValues = value.split(" ")
# Verifying if the parameter is in the options
if len(self.CONFIG[parameter]["OPTIONS"]) > 0:
for s in splittedValues:
if s not in self.CONFIG[parameter]["OPTIONS"]:
raise Exception("ERROR: the value '" + s +
"' provided is not valid.")
# Setting the value
self.CONFIG[parameter]["CURRENT_VALUE"] = splittedValues
print(general.success("\n[OK] " + parameter + "=" +
str(value)))
else:
raise Exception("ERROR: parameter not valid.")
except Exception as e:
print(
general.error(
"\n[!!] ERROR: Not enough parameters provided. Usage: set OPTION VALUE."
))
print(general.error(str(e)))
def do_run(self, line):
"""Running the current application. This method should be redefined for each util."""
print
# Checking if all the required parameters have been set
if self._checkIfRequiredAreSet():
print(general.info("Collecting the options set by the user..."))
# Getting the parser...
parser = usufy.getParser()
# Generating the parameters
params = self._getParams()
args = parser.parse_args(params)
print(general.info("Launching " + self.UNAME + " with the following parameters: ") + general.emphashis(str(params)))
try:
usufy.main(args)
except Exception as e:
print(gemeral.error("[!] ERROR. Something happenned when launching the utility. Type 'show options' to check the parameters. "))
print(general.error("Traceback: " + str(e)))
else:
print(general.error("[!] ERROR. There are required parameters which have not been set."))
self.do_show("options")
print(general.success("Execution ended successfully."))
def do_run(self, line):
"""
Command that send the order to the framework to launch this util
Args:
-----
line: The string of the line typed.
"""
# Checking if all the required parameters have been set
if self._checkIfRequiredAreSet():
print(general.info("\nCollecting the options set by the user...\n"))
# Getting the parser...
parser = entify.getParser()
# Generating the parameters
params = self._getParams()
args = parser.parse_args(params)
print(general.info("\nLaunching " + self.UNAME + " with the following parameters:") + general.emphasis("\t$ " + self.UNAME + " " + utils.listToString(params) + "\n"))
try:
entify.main(args)
except Exception as e:
print(general.error("\n[!!] ERROR. Something happened when launching the utility. Type 'show options' to check the parameters.\n"))
print(general.error("Traceback: " + str(e)))
else:
print(general.error("\n[!!] ERROR. There are required parameters which have not been set."))
self.do_show("options")
print(general.success("Execution ended successfully."))
def launchQueryForMode(self, query=None, mode=None):
"""
Method that launches an i3Browser to collect data.
Args:
-----
query: The query to be performed
mode: The mode to be used to build the query.
Return:
-------
A string containing the recovered data or None.
"""
# Creating the query URL for that mode
qURL = self.createURL(word=query, mode=mode)
i3Browser = browser.Browser()
try:
# Check if it needs creds
if self.needsCredentials[mode]:
self._getAuthenticated(i3Browser, qURL)
data = i3Browser.recoverURL(qURL)
else:
# Accessing the resources
data = i3Browser.recoverURL(qURL)
return data
except KeyError:
print(
general.error(
"[*] '{}' is not a valid mode for this wrapper ({}).".
format(mode, self.__class__.__name__)))
return None
def pool_function(args):
"""
A wrapper for being able to launch all the threads.
We will use python-emailahoy library for the verification.
Args:
-----
args: reception of the parameters for getPageWrapper as a tuple.
Returns:
--------
A dictionary representing whether the verification was ended
successfully. The format is as follows:
```
{"platform": "str(domain["value"])", "status": "DONE", "data": aux}
```
"""
is_valid = True
try:
checker = emailahoy.VerifyEmail()
status, message = checker.verify_email_smtp(args, from_host='gmail.com', from_email='*****@*****.**')
if status == 250:
print("\t[*] Verification of '{}' status: {}. Details:\n{}".format(general.success(args), general.success("SUCCESS ({})".format(str(status))), message))
is_valid = True
else:
print("\t[*] Verification of '{}' status: {}. Details:\n{}".format(general.error(args), general.error("FAILED ({})".format(str(status))), message))
is_valid = False
except Exception, e:
print(general.warning("WARNING. An error was found when performing the search. You can omit this message.\n" + str(e)))
is_valid = False
def do_run(self, line):
"""Command that send the order to the framework to launch the current utility."""
if self._checkIfRequiredAreSet():
print(general.info("Launching the util..."))
else:
print(general.error("There are required parameters which have not been set."))
self.do_show("options")
def do_unset(self, line):
"""
Unsetting the variables defined in CONFIG
You can check their values at any time by typing 'show options' and
unsetting all the options at once by typing 'unset all'.
Args:
-----
line: the string of the line typed.
Raises:
-------
ValueError: if the parameter is not valid.
"""
try:
parameter = line.split(" ")[0]
# Getting the parameter
if parameter in self.CONFIG.keys():
# Unsetting the value
self.CONFIG[parameter]["CURRENT_VALUE"] = self.CONFIG[
parameter]["DEFAULT_VALUE"]
print(
general.info(parameter + " reseted to '" +
str(self.CONFIG[parameter]["DEFAULT_VALUE"]) +
"'."))
elif parameter == "all":
for p in self.CONFIG.keys():
# Unsetting all the values
self.CONFIG[p]["CURRENT_VALUE"] = self.CONFIG[p][
"DEFAULT_VALUE"]
print(
general.success(
"\nAll parameters reseted to their default values."))
else:
raise ValueError("ERROR: parameter not valid.")
except Exception as e:
print(
general.error(
"\n[!!] ERROR: Not enough parameters provided. Usage: unset OPTION"
))
print(general.error("Traceback: " + str(e)))
def pool_function(p, nick, rutaDescarga, avoidProcessing = True, avoidDownload = True, outQueue=None):
'''
Wrapper for being able to launch all the threads of getPageWrapper.
:param args: We receive the parameters for getPageWrapper as a tuple.
'''
try:
res = getPageWrapper(p, nick, rutaDescarga, avoidProcessing, avoidDownload, outQueue)
return {"platform" : str(p), "status": "DONE", "data": res}
except Exception as e:
print(general.error("\tERROR: " + str(p)))
return {"platform" : str(p), "status": "ERROR", "data": []}
def getPageWrapper(p,
nick,
rutaDescarga,
avoidProcessing=True,
avoidDownload=True,
outQueue=None):
"""
Method that wraps the call to the getInfo. Before it was getUserPage.
Args:
-----
pName: Platform where the information is stored. It is a string.
nick: Nick to be searched.
rutaDescarga: Local file where saving the obtained information.
avoidProcessing: Boolean var that defines whether the profiles will NOT
be processed (stored in this version).
avoidDownload: Boolean var that defines whether the profiles will NOT be
downloaded (stored in this version).
outQueue: Queue where the information will be stored.
maltego: Parameter to tell usufy.py that he has been invoked by Malego.
Returns:
--------
None if a queue is provided. Note that the values will be stored in the
outQueue or a dictionary is returned.
"""
logger = logging.getLogger("osrframework.usufy")
logger.debug("\tLooking for profiles in " + str(p) + "...")
#res = p.getUserPage(nick, rutaDescarga, avoidProcessing = avoidProcessing, avoidDownload = avoidDownload)
try:
res = p.getInfo(
query=nick, mode="usufy", process=True
) #rutaDescarga, avoidProcessing = avoidProcessing, avoidDownload = avoidDownload)
if res != []:
if outQueue != None:
# Storing in the output queue the values
outQueue.put((res))
else:
# If no queue was given, return the value normally
return res
else:
logger.debug("\t" + str(p) + " - User profile not found...")
return []
except:
print(
general.error(
"ERROR: something happened when processing " + str(p) +
". You may like to deactivate this wrapper if the error persist."
))
return []
def launch_query_for_mode(self, query=None, mode=None):
"""Method that launches an i3Browser to collect data
Args:
query: The query to be performed
mode: The mode to be used to build the query.
Returns:
A string containing the recovered data or None."""
# Creating the query URL for that mode
qURL = self.create_url(word=query, mode=mode)
i3Browser = browser.Browser()
try:
# Check if it needs creds
needs_credentials = False
try:
# Suport for version 2 of wrappers
if self.modes[mode]["needs_credentials"]:
needs_credentials = True
except AttributeError as e:
if self.needsCredentials[mode]:
needs_credentials = True
if needs_credentials:
self._getAuthenticated(i3Browser, qURL)
data = i3Browser.recover_url(qURL)
else:
# Accessing the resources
data = i3Browser.recover_url(qURL)
try:
if self.modes[mode]["debug"]:
with open(
os.path.join(tempfile.gettempdir(),
self.platformName) + ".html",
"w") as file:
print(
f"DEBUG mode is active for '{general.emphasis(self.platformName)}'. Data grabbed saved to: '{general.warning(file.name)}'"
)
file.write(data)
except:
pass
return data
except KeyError:
print(
general.error(
"[*] '{}' is not a valid mode for this wrapper ({}).".
format(mode, self.__class__.__name__)))
return None
def do_run(self, line):
"""
Command that send the order to the framework to launch this util
Args:
-----
line: the string of the line typed.
"""
if self._checkIfRequiredAreSet():
print(general.info("\nLaunching the util...\n"))
else:
print(
general.error(
"\n[!!] ERROR: There are required parameters which have not been set."
))
self.do_show("options")
def launch_query_for_mode(self, query=None, mode=None):
"""Method that launches an i3Browser to collect data
Args:
query: The query to be performed
mode: The mode to be used to build the query.
Returns:
A string containing the recovered data or None."""
# Creating the query URL for that mode
qURL = self.create_url(word=query, mode=mode)
i3Browser = browser.Browser()
try:
# Check if it needs creds
needs_credentials = False
try:
# Suport for version 2 of wrappers
if self.modes[mode]["needs_credentials"]:
needs_credentials = True
except AttributeError as e:
if self.needsCredentials[mode]:
needs_credentials = True
if needs_credentials:
self._getAuthenticated(i3Browser, qURL)
data = i3Browser.recover_url(qURL)
else:
# Accessing the resources
data = i3Browser.recover_url(qURL)
return data
except KeyError:
print(
general.error(
"[*] '{}' is not a valid mode for this wrapper ({}).".
format(mode, self.__class__.__name__)))
return None
def main(params=None):
"""
Main function to launch phonefy.
The function is created in this way so as to let other applications make
use of the full configuration capabilities of the application. The
parameters received are used as parsed by this modules `getParser()`.
Args:
-----
params: A list with the parameters as grabbed by the terminal. It is
None when this is called by an entry_point.
Returns:
--------
A list of i3visio entities.
"""
# Grabbing the parser
parser = getParser()
if params != None:
args = parser.parse_args(params)
else:
args = parser.parse_args()
results = []
if not args.quiet:
print(general.title(banner.text))
sayingHello = """
Phonefy | Copyright (C) F. Brezo and Y. Rubio (i3visio) 2014-2018
This program comes with ABSOLUTELY NO WARRANTY. This is free software, and you
are welcome to redistribute it under certain conditions. For additional info,
visit """ + general.LICENSE_URL + "\n"
print(general.title(sayingHello))
if args.license:
general.showLicense()
else:
# Showing the execution time...
startTime = dt.datetime.now()
#TODO: Get the number searchable platforms in this context
print(
str(startTime) +
"\tStarting search in different platform(s)... Relax!\n")
print(general.emphasis("\tPress <Ctrl + C> to stop...\n"))
try:
results = processPhoneList(platformNames=args.platforms,
numbers=args.numbers,
excludePlatformNames=args.exclude)
except KeyboardInterrupt:
print(
general.error(
"\n[!] Process manually stopped by the user. Workers terminated without providing any result.\n"
))
# Trying to store the information recovered
if args.output_folder != None:
# Verifying an output folder was selected
if not os.path.exists(args.output_folder):
os.makedirs(args.output_folder)
# Grabbing the results
fileHeader = os.path.join(args.output_folder, args.file_header)
for ext in args.extension:
# Generating output files
general.exportUsufy(results, ext, fileHeader)
# Showing the information gathered if requested
if not args.quiet:
now = dt.datetime.now()
print(
str(now) +
"\tA summary of the results obtained is shown in the following table:\n"
)
print(general.success(general.usufyToTextExport(results)))
if args.web_browser:
general.openResultsInBrowser(results)
now = dt.datetime.now()
print(
"\n" + str(now) +
"\tYou can find all the information collected in the following files:"
)
for ext in args.extension:
# Showing the output files
print("\t" + general.emphasis(fileHeader + "." + ext))
# Showing the execution time...
endTime = dt.datetime.now()
print("\n" + str(endTime) + "\tFinishing execution...\n")
print("Total time consumed:\t" +
general.emphasis(str(endTime - startTime)) + "\n")
#TODO: Get the number searchable platforms in this context
#print("Average seconds/query:\t" + general.emphasis(str((endTime-startTime).total_seconds()/len(listPlatforms))) +" seconds\n")
# Urging users to place an issue on Github...
print(banner.footer)
if params:
return results
def main(params=None):
"""Main function to launch searchfy
The function is created in this way so as to let other applications make
use of the full configuration capabilities of the application. The
parameters received are used as parsed by this modules `get_parser()`.
Args:
params (list): A list with the parameters as grabbed by the terminal. It is
None when this is called by an entry_point. If it is called by osrf
the data is already parsed.
Returns:
list. A list of i3visio entities.
"""
if params is None:
parser = get_parser()
args = parser.parse_args(params)
else:
args = params
results = []
print(general.title(banner.text))
saying_hello = f"""
Searchfy | Copyright (C) Yaiza Rubio & Félix Brezo (i3visio) 2014-2020
This program comes with ABSOLUTELY NO WARRANTY. This is free software, and you
are welcome to redistribute it under certain conditions. For additional info,
visit <{general.LICENSE_URL}>.
"""
print(general.info(saying_hello))
if args.license:
general.showLicense()
else:
# Showing the execution time...
start_time = dt.datetime.now()
print(
f"{start_time}\tStarting search in different platform(s)... Relax!\n"
)
print(general.emphasis("\tPress <Ctrl + C> to stop...\n"))
# Performing the search
try:
results = perform_search(platformNames=args.platforms,
queries=args.queries,
exclude_platform_names=args.exclude)
except KeyboardInterrupt:
print(
general.error(
"\n[!] Process manually stopped by the user. Workers terminated without providing any result.\n"
))
results = []
# Generating summary files for each ...
if args.extension:
# Verifying if the outputPath exists
if not os.path.exists(args.output_folder):
os.makedirs(args.output_folder)
# Grabbing the results
fileHeader = os.path.join(args.output_folder, args.file_header)
# Iterating through the given extensions to print its values
for ext in args.extension:
# Generating output files
general.export_usufy(results, ext, fileHeader)
# Printing the results if requested
now = dt.datetime.now()
print(f"\n{now}\tResults obtained:\n")
print(general.success(general.osrf_to_text_export(results)))
if args.web_browser:
general.open_results_in_browser(results)
now = dt.datetime.now()
print(
"\n{date}\tYou can find all the information collected in the following files:"
.format(date=str(now)))
for ext in args.extension:
# Showing the output files
print("\t" + general.emphasis(fileHeader + "." + ext))
# Showing the execution time...
end_time = dt.datetime.now()
print(f"\n{end_time}\tFinishing execution...\n")
print("Total time used:\t" +
general.emphasis(str(end_time - start_time)))
print("Average seconds/query:\t" + general.emphasis(
str((end_time - start_time).total_seconds() /
len(args.platforms))) + " seconds\n")
# Urging users to place an issue on Github...
print(banner.footer)
if params:
return results
def main(params=None):
"""ain function to launch usufy
The function is created in this way so as to let other applications make
use of the full configuration capabilities of the application. The
parameters received are used as parsed by this modules `get_parser()`.
Args:
params: A list with the parameters as grabbed by the terminal. It is
None when this is called by an entry_point. If it is called by osrf
the data is already parsed.
Returns:
dict: A Json representing the matching results.
"""
if params is None:
parser = get_parser()
args = parser.parse_args(params)
else:
args = params
print(general.title(banner.text))
saying_hello = f"""
Usufy | Copyright (C) Yaiza Rubio & Félix Brezo (i3visio) 2014-2020
This program comes with ABSOLUTELY NO WARRANTY. This is free software, and you
are welcome to redistribute it under certain conditions. For additional info,
visit <{general.LICENSE_URL}>.
"""
print(general.info(saying_hello))
if args.fuzz:
res = fuzzUsufy(args.fuzz, args.fuzz_config)
else:
# Recovering the list of platforms to be launched
list_platforms = platform_selection.get_platforms_by_name(
platform_names=args.platforms,
tags=args.tags,
mode="usufy",
exclude_platform_names=args.exclude)
if args.info:
# Information actions...
if args.info == 'list_platforms':
info_platforms = "Listing the platforms:\n"
for p in list_platforms:
info_platforms += "\t\t" + (str(p) + ": ").ljust(
16, ' ') + str(p.tags) + "\n"
return info_platforms
elif args.info == 'list_tags':
tags = {}
# Going through all the selected platforms to get their tags
for p in list_platforms:
for t in p.tags:
if t not in tags.keys():
tags[t] = 1
else:
tags[t] += 1
info_tags = "List of tags:\n"
# Displaying the results in a sorted list
for t in tags.keys():
info_tags += "\t\t" + (t + ": ").ljust(16, ' ') + str(
tags[t]) + " time(s)\n"
return info_tags
else:
pass
# performing the test
elif args.benchmark:
platforms = platform_selection.get_all_platform_names("usufy")
res = benchmark.do_benchmark(platforms)
str_times = ""
for e in sorted(res.keys()):
str_times += str(e) + "\t" + str(res[e]) + "\n"
return str_times
# showing the tags of the usufy platforms
elif args.show_tags:
tags = platform_selection.get_all_platform_namesByTag("usufy")
print(
general.info(
"This is the list of platforms grouped by tag.\n"))
print(json.dumps(tags, indent=2, sort_keys=True))
print(
general.info(
"[Tip] Remember that you can always launch the platform using the -t option followed by any of the aforementioned.\n"
))
return tags
# Executing the corresponding process...
else:
# Showing the execution time...
start_time = dt.datetime.now()
print(
f"{start_time}\tStarting search in {general.emphasis(str(len(list_platforms)))} platform(s)... Relax!\n"
)
print(general.emphasis("\tPress <Ctrl + C> to stop...\n"))
# Defining the list of users to monitor
nicks = []
if args.nicks:
for n in args.nicks:
nicks.append(n)
else:
# Reading the nick files
try:
nicks = args.list.read().splitlines()
except:
print(
general.error(
"ERROR: there has been an error when opening the file that stores the nicks.\tPlease, check the existence of this file."
))
# Definning the results
res = []
if args.output_folder != None:
# if Verifying an output folder was selected
if not os.path.exists(args.output_folder):
os.makedirs(args.output_folder)
# Launching the process...
res = process_nick_list(nicks,
list_platforms,
args.output_folder,
avoidProcessing=args.avoid_processing,
avoidDownload=args.avoid_download,
nThreads=args.threads,
verbosity=args.verbose,
logFolder=args.logfolder)
else:
try:
res = process_nick_list(nicks,
list_platforms,
nThreads=args.threads,
verbosity=args.verbose,
logFolder=args.logfolder)
except Exception as e:
print(
general.error(
"Exception grabbed when processing the nicks: " +
str(e)))
print(general.error(traceback.print_stack()))
# We are going to iterate over the results...
str_results = "\t"
# Structure returned
"""
[
{
"attributes": [
{
"attributes": [],
"type": "com.i3visio.URI",
"value": "http://twitter.com/i3visio"
},
{
"attributes": [],
"type": "com.i3visio.Alias",
"value": "i3visio"
},
{
"attributes": [],
"type": "com.i3visio.Platform",
"value": "Twitter"
}
],
"type": "com.i3visio.Profile",
"value": "Twitter - i3visio"
}
,
...
]
"""
for r in res:
# The format of the results (attributes) for a given nick is a list as follows:
for att in r["attributes"]:
# iterating through the attributes
platform = ""
uri = ""
for details in att["attributes"]:
if details["type"] == "com.i3visio.Platform":
platform = details["value"]
if details["type"] == "com.i3visio.URI":
uri = details["value"]
try:
str_results += (str(platform) + ":").ljust(
16, ' ') + " " + str(uri) + "\n\t\t"
except:
pass
# Generating summary files for each ...
if args.extension:
# Verifying if the outputPath exists
if not os.path.exists(args.output_folder):
os.makedirs(args.output_folder)
# Grabbing the results
file_header = os.path.join(args.output_folder,
args.file_header)
# Iterating through the given extensions to print its values
for ext in args.extension:
# Generating output files
general.export_usufy(res, ext, file_header)
now = dt.datetime.now()
print(
f"\n{now}\tResults obtained ({general.emphasis(len(res))}):\n")
print(general.success(general.osrf_to_text_export(res)))
if args.web_browser:
general.open_results_in_browser(res)
now = dt.datetime.now()
print("\n" + str(now) + "\tYou can find all the information here:")
for ext in args.extension:
# Showing the output files
print("\t" + general.emphasis(file_header + "." + ext))
# Showing the execution time...
end_time = dt.datetime.now()
print(f"\n{end_time}\tFinishing execution...\n")
print("Total time consumed:\t" +
general.emphasis(str(end_time - start_time)))
print("Average seconds/query:\t" + general.emphasis(
str((end_time - start_time).total_seconds() /
len(list_platforms))) + " seconds\n")
# Urging users to place an issue on Github...
print(banner.footer)
if params:
return res
def process_nick_list(nicks,
platforms=None,
rutaDescarga="./",
avoidProcessing=True,
avoidDownload=True,
nThreads=12,
verbosity=1,
logFolder="./logs"):
"""
Process a list of nicks to check whether they exist.
This method receives as a parameter a series of nicks and verifies whether
those nicks have a profile associated in different social networks.
Args:
nicks: List of nicks to process.
platforms: List of <Platform> objects to be processed.
rutaDescarga: Local file where saving the obtained information.
avoidProcessing: A boolean var that defines whether the profiles will
NOT be processed.
avoidDownload: A boolean var that defines whether the profiles will NOT
be downloaded.
verbosity: The level of verbosity to be used.
logFolder: The path to the log folder.
Returns:
A dictionary where the key is the nick and the value another dictionary
where the keys are the social networks and the value is the
corresponding URL.
"""
if platforms is None:
platforms = platform_selection.get_all_platform_names("usufy")
# Defining the output results variable
res = []
# Processing the whole list of terms...
for nick in nicks:
# If the process is executed by the current app, we use the Processes. It is faster than pools.
if nThreads <= 0 or nThreads > len(platforms):
nThreads = len(platforms)
# Using threads in a pool if we are not running the program in main
# Example catched from: https://stackoverflow.com/questions/11312525/catch-ctrlc-sigint-and-exit-multiprocesses-gracefully-in-python
try:
original_sigint_handler = signal.signal(signal.SIGINT,
signal.SIG_IGN)
pool = Pool(nThreads)
signal.signal(signal.SIGINT, original_sigint_handler)
except ValueError:
# To avoid: ValueError: signal only works in main thread
pool = Pool(nThreads)
pool_results = []
try:
def log_result(result):
# This is called whenever foo_pool(i) returns a result.
# result_list is modified only by the main process, not the pool workers.
pool_results.append(result)
for plat in platforms:
# We need to create all the arguments that will be needed
parameters = (plat, nick, rutaDescarga, avoidProcessing,
avoidDownload, verbosity)
pool.apply_async(
pool_function,
args=parameters,
callback=log_result,
)
# Waiting for results to be finished
while len(pool_results) < len(platforms):
time.sleep(1)
# Closing normal termination
pool.close()
except KeyboardInterrupt:
print(
general.warning(
"\n[!] Process manually stopped by the user. Terminating workers.\n"
))
pool.terminate()
print(
general.warning(
"[!] The following platforms were not processed:"))
pending = ""
for p in platforms:
processed = False
for processedPlatform in pool_results:
if str(p) == processedPlatform["platform"]:
processed = True
break
if not processed:
print("\t- " + str(p))
pending += " " + str(p).lower()
print("\n")
print(
general.warning(
"If you want to relaunch the app with these platforms you can always run the command with: "
))
print("\t usufy ... -p " + general.emphasis(pending))
print("\n")
print(
general.warning(
"If you prefer to avoid these platforms you can manually evade them for whatever reason with: "
))
print("\t usufy ... -x " + general.emphasis(pending))
print("\n")
pool.join()
# Collecting the results
profiles = []
errors = {}
warnings = {}
for info in pool_results:
if info["status"] == "Ok":
array = json.loads(info["data"])
for r in array:
if r != "{}":
profiles.append(r)
else:
e = info["status"]
if isinstance(e, OSRFrameworkError):
aux = errors.get(e.__class__.__name__, {})
aux["info"] = info["data"]
aux["counter"] = aux.get("counter", 0) + 1
errors[e.__class__.__name__] = aux
else:
aux = warnings.get(e.__class__.__name__, {})
aux["info"] = info["data"]
aux["counter"] = aux.get("counter", 0) + 1
warnings[e.__class__.__name__] = aux
res += profiles
if errors:
now = dt.datetime.now()
print(f"\n{now}\tSome errors where found in the process:")
for key, value in errors.items():
print(
textwrap.fill("- {} (found: {}). Details:".format(
general.error(key), general.error(value["counter"])),
90,
initial_indent="\t"))
print(
textwrap.fill("\t{}".format(value["info"]),
80,
initial_indent="\t"))
if warnings and verbosity >= 2:
now = dt.datetime.now()
print(
"\n{}\tSome warnings where found in the process:".format(now))
for key, value in warnings.items():
print(
textwrap.fill("- {} (found: {}). Details:".format(
general.warning(key),
general.warning(value["counter"])),
90,
initial_indent="\t"))
print(
textwrap.fill("\t{}".format(value["info"]),
80,
initial_indent="\t"))
return res
def main(args):
"""
Main function to launch usufy.
The function is created in this way so as to let other applications make
use of the full configuration capabilities of the application. The
parameters received are used as parsed by this modules `getParser()`.
Args:
-----
args: Arguments received in the command line.
Returns:
--------
A list of i3visio entities.
"""
results = []
if not args.maltego:
print(general.title(banner.text))
sayingHello = """
searchfy.py Copyright (C) F. Brezo and Y. Rubio (i3visio) 2014-2017
This program comes with ABSOLUTELY NO WARRANTY. This is free software, and you
are welcome to redistribute it under certain conditions. For additional info,
visit """ + general.LICENSE_URL + "\n"
print(general.title(sayingHello))
if args.license:
general.showLicense()
else:
# Showing the execution time...
startTime = dt.datetime.now()
print(
str(startTime) +
"\tStarting search in different platform(s)... Relax!\n")
print(general.emphasis("\tPress <Ctrl + C> to stop...\n"))
# Performing the search
try:
results = performSearch(platformNames=args.platforms,
queries=args.queries,
process=args.process,
excludePlatformNames=args.exclude)
except KeyboardInterrupt:
print(
general.error(
"\n[!] Process manually stopped by the user. Workers terminated without providing any result.\n"
))
results = []
# Generating summary files for each ...
if args.extension:
# Storing the file...
if not args.maltego:
# Verifying if the outputPath exists
if not os.path.exists(args.output_folder):
os.makedirs(args.output_folder)
# Grabbing the results
fileHeader = os.path.join(args.output_folder, args.file_header)
if not args.maltego:
# Iterating through the given extensions to print its values
for ext in args.extension:
# Generating output files
general.exportUsufy(results, ext, fileHeader)
# Generating the Maltego output
if args.maltego:
general.listToMaltego(results)
# Printing the results if requested
if not args.maltego:
now = dt.datetime.now()
print(
str(now) +
"\tA summary of the results obtained are listed in the following table:\n"
)
print(general.success(general.usufyToTextExport(results)))
if args.web_browser:
general.openResultsInBrowser(results)
now = dt.datetime.now()
print(
"\n" + str(now) +
"\tYou can find all the information collected in the following files:"
)
for ext in args.extension:
# Showing the output files
print("\t" + general.emphasis(fileHeader + "." + ext))
# Showing the execution time...
endTime = dt.datetime.now()
print("\n" + str(endTime) + "\tFinishing execution...\n")
print("Total time used:\t" +
general.emphasis(str(endTime - startTime)))
print("Average seconds/query:\t" + general.emphasis(
str((endTime - startTime).total_seconds() /
len(args.platforms))) + " seconds\n")
# Urging users to place an issue on Github...
print(banner.footer)
return results
def main(params=None):
"""
Main function to launch phonefy.
The function is created in this way so as to let other applications make
use of the full configuration capabilities of the application. The
parameters received are used as parsed by this modules `getParser()`.
Args:
-----
params: A list with the parameters as grabbed by the terminal. It is
None when this is called by an entry_point.
Returns:
--------
A list of i3visio entities.
"""
# Grabbing the parser
parser = getParser()
if params != None:
args = parser.parse_args(params)
else:
args = parser.parse_args()
results = []
if not args.quiet:
print(general.title(banner.text))
sayingHello = """
Mailfy | Copyright (C) F. Brezo and Y. Rubio (i3visio) 2016-2018
This program comes with ABSOLUTELY NO WARRANTY. This is free software, and you
are welcome to redistribute it under certain conditions. For additional info,
visit """ + general.LICENSE_URL + "\n"
print(general.title(sayingHello))
# Displaying a warning if this is being run in a windows system
if sys.platform == 'win32':
print(general.warning("""OSRFramework has detected that you are running mailfy.py in a Windows system.
As the "emailahoy" library is NOT working properly there, "validate_email" will
be used instead. Verification may be slower though."""))
if args.license:
general.showLicense()
else:
# Grabbing the list of global domains
if args.verify_emails:
domains = EMAIL_DOMAINS
# Processing the options returned to remove the "all" option
elif "all" in args.domains:
domains = LEAKED_DOMAINS
else:
# processing only the given domains and excluding the ones provided
domains = []
for d in args.domains:
if d not in args.exclude:
domains.append(d)
if args.create_emails:
emails = grabEmails(nicksFile=args.create_emails, domains=domains, excludeDomains=args.exclude)
else:
emails = grabEmails(emails=args.emails, emailsFile=args.emails_file, nicks=args.nicks, nicksFile=args.nicks_file, domains=domains, excludeDomains=args.exclude)
startTime= dt.datetime.now()
# Original functionality. UNSTABLE feature!
if args.verify_emails:
# Showing the execution time...
if not args.quiet:
print(str(startTime) +"\tStarting search in " + general.emphasis(str(len(emails))) + " different emails:\n"+ json.dumps(emails, indent=2, sort_keys=True) + "\n")
print(general.emphasis("\tPress <Ctrl + C> to stop...\n"))
# Perform searches, using different Threads
tmp = performSearch(emails, args.threads)
# We make a strict copy of the object
results = list(tmp)
if not args.quiet:
now = dt.datetime.now()
print("\n{}\tMailfy has found {} existing email(s). Have they been leaked somewhere?\n".format(str(now), general.emphasis(str(len(results)))))
print(general.emphasis("\tPress <Ctrl + C> to stop...\n"))
# Verify the existence of the mails found as leaked emails.
for r in tmp:
# We assume that the first attribute is always the email
query = r["attributes"][0]["value"]
# Iterate through the different leak platforms
leaks = hibp.checkIfEmailWasHacked(query)
if len(leaks) > 0:
if not args.quiet:
if len(leaks) > 0:
print("\t[*] '{}' has been found in at least {} different leaks.".format(general.success(query), general.success(str(len(leaks)))))
else:
print("\t[*] '{}' has NOT been found in any leak.".format(general.error(query)))
email, alias, domain = getMoreInfo(query)
for leak in leaks:
# Creating a new full entity from scratch
new = {}
new["type"] = "i3visio.profile"
new["value"] = leak["value"] + " - " + alias["value"]
new["attributes"] = []
new["attributes"].append(email)
new["attributes"].append(alias)
new["attributes"].append(domain)
# leak contains a i3visio.platform built by HIBP
new["attributes"].append(leak)
results.append(new)
else:
if not args.quiet:
print(general.warning("\t" + query + " has NOT been found on any leak yet."))
else:
if not args.quiet:
print("\n" + str(startTime) +"\tStarting search of " + general.emphasis(str(len(emails))) + " different emails in leaked databases.\n\nNote that this will take between 1 and 2 seconds per query due to the thirdparties API restrictions:\n"+ json.dumps(emails, indent=2, sort_keys=True) + "\n")
print(general.emphasis("\tPress <Ctrl + C> to stop...\n"))
# Perform is_leaked function
results = []
print("Mailfy will use haveibeenpwned.com (HIBP) API to find leaked emails...\n")
for i, e in enumerate(emails):
if not args.quiet:
print("\t" + str(i+1) + "/" + str(len(emails)) + " - Searching if " + e + " has been leaked...")
# Iterate through the different leak platforms
leaks = hibp.checkIfEmailWasHacked(e)
if len(leaks) > 0:
if not args.quiet:
print(general.success("\t" + e + " has been found in at least " + str(len(leaks)) + " different leaks."))
email, alias, domain = getMoreInfo(e)
for leak in leaks:
# Creating a new full entity from scratch
new = {}
new["type"] = "i3visio.profile"
new["value"] = leak["value"] + " - " + alias["value"]
new["attributes"] = []
new["attributes"].append(email)
new["attributes"].append(alias)
new["attributes"].append(domain)
# leak contains a i3visio.platform built by HIBP
new["attributes"].append(leak)
results.append(new)
# Trying to store the information recovered
if args.output_folder != None:
if not os.path.exists(args.output_folder):
os.makedirs(args.output_folder)
# Grabbing the results
fileHeader = os.path.join(args.output_folder, args.file_header)
for ext in args.extension:
# Generating output files
general.exportUsufy(results, ext, fileHeader)
# Showing the information gathered if requested
if not args.quiet:
now = dt.datetime.now()
print("\n" + str(now) + "\tA summary of the results obtained is shown in the following table:\n")
print(general.success(general.usufyToTextExport(results)))
now = dt.datetime.now()
print("\n" + str(now) + "\tYou can find all the information collected in the following files:")
for ext in args.extension:
# Showing the output files
print(general.emphasis("\t" + fileHeader + "." + ext))
# Showing the execution time...
if not args.quiet:
endTime= dt.datetime.now()
print("\n" + str(endTime) +"\tFinishing execution...\n")
print("Total time used:\t" + general.emphasis(str(endTime-startTime)))
print("Average seconds/query:\t" + general.emphasis(str((endTime-startTime).total_seconds()/len(emails))) +" seconds\n")
if not args.quiet:
# Urging users to place an issue on Github...
print(banner.footer)
if params:
return results
def main(args):
'''
Main function. This function is created in this way so as to let other applications make use of the full configuration capabilities of the application.
'''
# Recovering the logger
# Calling the logger when being imported
osrframework.utils.logger.setupLogger(loggerName="osrframework.usufy", verbosity=args.verbose, logFolder=args.logfolder)
# From now on, the logger can be recovered like this:
logger = logging.getLogger("osrframework.usufy")
# Printing the results if requested
if not args.maltego:
print(general.title(banner.text))
sayingHello = """
usufy.py Copyright (C) F. Brezo and Y. Rubio (i3visio) 2014-2017
This program comes with ABSOLUTELY NO WARRANTY. This is free software, and you
are welcome to redistribute it under certain conditions. For additional info,
visit <http://www.gnu.org/licenses/gpl-3.0.txt>."""
logger.info(sayingHello)
print(general.title(sayingHello))
logger.info("Starting usufy.py...")
if args.license:
logger.info("Looking for the license...")
# showing the license
try:
with open ("COPYING", "r") as iF:
contenido = iF.read().splitlines()
for linea in contenido:
print(linea)
except Exception:
try:
# Trying to recover the COPYING file...
with open ("/usr/share/osrframework/COPYING", "r") as iF:
contenido = iF.read().splitlines()
for linea in contenido:
print(linea)
except:
logger.error("ERROR: there has been an error when opening the COPYING file.\n\tThe file contains the terms of the GPLv3 under which this software is distributed.\n\tIn case of doubts, verify the integrity of the files or contact [email protected].")
elif args.fuzz:
logger.info("Performing the fuzzing tasks...")
res = fuzzUsufy(args.fuzz, args.fuzz_config)
logger.info("Recovered platforms:\n" + str(res))
else:
logger.debug("Recovering the list of platforms to be processed...")
# Recovering the list of platforms to be launched
listPlatforms = platform_selection.getPlatformsByName(platformNames=args.platforms, tags=args.tags, mode="usufy", excludePlatformNames=args.exclude)
logger.debug("Platforms recovered.")
if args.info:
# Information actions...
if args.info == 'list_platforms':
infoPlatforms="Listing the platforms:\n"
for p in listPlatforms:
infoPlatforms += "\t\t" + (str(p) + ": ").ljust(16, ' ') + str(p.tags)+"\n"
logger.info(infoPlatforms)
return infoPlatforms
elif args.info == 'list_tags':
logger.info("Listing the tags:")
tags = {}
# Going through all the selected platforms to get their tags
for p in listPlatforms:
for t in p.tags:
if t not in tags.keys():
tags[t] = 1
else:
tags[t] += 1
infoTags = "List of tags:\n"
# Displaying the results in a sorted list
for t in tags.keys():
infoTags += "\t\t" + (t + ": ").ljust(16, ' ') + str(tags[t]) + " time(s)\n"
logger.info(infoTags)
return infoTags
else:
pass
# performing the test
elif args.benchmark:
logger.warning("The benchmark mode may last some minutes as it will be performing similar queries to the ones performed by the program in production. ")
logger.info("Launching the benchmarking tests...")
platforms = platform_selection.getAllPlatformNames("usufy")
res = benchmark.doBenchmark(platforms)
strTimes = ""
for e in sorted(res.keys()):
strTimes += str(e) + "\t" + str(res[e]) + "\n"
logger.info(strTimes)
return strTimes
# showing the tags of the usufy platforms
elif args.show_tags:
logger.info("Collecting the list of tags...")
tags = platform_selection.getAllPlatformNamesByTag("usufy")
logger.info(json.dumps(tags, indent=2))
print(general.info("This is the list of platforms grouped by tag.\n"))
print(json.dumps(tags, indent=2, sort_keys=True))
print(general.info("[Tip] Remember that you can always launch the platform using the -t option followed by any of the aforementioned.\n"))
return tags
# Executing the corresponding process...
else:
# Showing the execution time...
if not args.maltego:
startTime= dt.datetime.now()
print("\n")
print(str(startTime) + "\tStarting search in " + str(len(listPlatforms)) + " platform(s)... Relax!\n")
print(general.emphasis("\tPress <Ctrl + C> to stop...\n"))
# Defining the list of users to monitor
nicks = []
logger.debug("Recovering nicknames to be processed...")
if args.nicks:
for n in args.nicks:
# TO-DO
# A trick to avoid having the processing of the properties when being queried by Maltego
if "properties.i3visio" not in n:
nicks.append(n)
else:
# Reading the nick files
try:
nicks = args.list.read().splitlines()
except:
logger.error("ERROR: there has been an error when opening the file that stores the nicks.\tPlease, check the existence of this file.")
# Definning the results
res = []
if args.output_folder != None:
# if Verifying an output folder was selected
logger.debug("Preparing the output folder...")
if not args.maltego:
if not os.path.exists(args.output_folder):
logger.warning("The output folder \'" + args.output_folder + "\' does not exist. The system will try to create it.")
os.makedirs(args.output_folder)
# Launching the process...
res = processNickList(nicks, listPlatforms, args.output_folder, avoidProcessing = args.avoid_processing, avoidDownload = args.avoid_download, nThreads=args.threads, verbosity= args.verbose, logFolder=args.logfolder)
else:
try:
res = processNickList(nicks, listPlatforms, nThreads=args.threads, verbosity= args.verbose, logFolder=args.logfolder)
except Exception as e:
print(general.error("Exception grabbed when processing the nicks: " + str(e)))
print(general.error(traceback.print_stack()))
logger.info("Listing the results obtained...")
# We are going to iterate over the results...
strResults = "\t"
# Structure returned
"""
[
{
"attributes": [
{
"attributes": [],
"type": "i3visio.uri",
"value": "http://twitter.com/i3visio"
},
{
"attributes": [],
"type": "i3visio.alias",
"value": "i3visio"
},
{
"attributes": [],
"type": "i3visio.platform",
"value": "Twitter"
}
],
"type": "i3visio.profile",
"value": "Twitter - i3visio"
}
,
...
]
"""
for r in res:
# The format of the results (attributes) for a given nick is a list as follows:
for att in r["attributes"]:
# iterating through the attributes
platform = ""
uri = ""
for details in att["attributes"]:
if details["type"] == "i3visio.platform":
platform = details["value"]
if details["type"] == "i3visio.uri":
uri = details["value"]
try:
strResults+= (str(platform) + ":").ljust(16, ' ')+ " "+ str(uri)+"\n\t\t"
except:
pass
logger.info(strResults)
# Generating summary files for each ...
if args.extension:
# Storing the file...
logger.info("Creating output files as requested.")
if not args.maltego:
# Verifying if the outputPath exists
if not os.path.exists (args.output_folder):
logger.warning("The output folder \'" + args.output_folder + "\' does not exist. The system will try to create it.")
os.makedirs(args.output_folder)
# Grabbing the results
fileHeader = os.path.join(args.output_folder, args.file_header)
# Iterating through the given extensions to print its values
for ext in args.extension:
# Generating output files
general.exportUsufy(res, ext, fileHeader)
# Generating the Maltego output
if args.maltego:
general.listToMaltego(res)
# Printing the results if requested
else:
now = dt.datetime.now()
print(str(now) + "\tA summary of the results obtained are shown in the following table:\n")
print(general.success(unicode(general.usufyToTextExport(res))))
if args.web_browser:
general.openResultsInBrowser(res)
print("\n")
now = dt.datetime.now()
print(str(now) + "\tYou can find all the information collected in the following files:")
for ext in args.extension:
# Showing the output files
print("\t-" + general.emphasis(fileHeader + "." + ext))
# Showing the execution time...
endTime= dt.datetime.now()
print("\n" + str(endTime) +"\tFinishing execution...\n")
print("Total time used:\t" + str(endTime-startTime))
print("Average seconds/query:\t" + str((endTime-startTime).total_seconds()/len(listPlatforms)) +" seconds\n")
# Urging users to place an issue on Github...
print("Did something go wrong? Is a platform reporting false positives? Do you need to integrate a new one?")
print("Then, place an issue in the Github project: <https://github.com/i3visio/osrframework/issues>.")
print("Note that otherwise, we won't know about it!\n")
return res
def main(params=None):
"""
Main function to launch phonefy.
The function is created in this way so as to let other applications make
use of the full configuration capabilities of the application. The
parameters received are used as parsed by this modules `getParser()`.
Args:
-----
params: A list with the parameters as grabbed by the terminal. It is
None when this is called by an entry_point. If it is called by osrf
the data is already parsed.
Returns:
--------
A list of i3visio entities.
"""
if params == None:
parser = getParser()
args = parser.parse_args(params)
else:
args = params
results = []
if not args.quiet:
print(general.title(banner.text))
sayingHello = """
Mailfy | Copyright (C) Yaiza Rubio & Félix Brezo (i3visio) 2014-2018
This program comes with ABSOLUTELY NO WARRANTY. This is free software, and you
are welcome to redistribute it under certain conditions. For additional info,
visit <{}>.
""".format(general.LICENSE_URL)
print(general.info(sayingHello))
# Displaying a warning if this is being run in a windows system
if sys.platform == 'win32':
print(
general.warning(
"""OSRFramework has detected that you are running mailfy.py in a Windows system.
As the "emailahoy" library is NOT working properly there, "validate_email" will
be used instead. Verification may be slower though."""))
if args.license:
general.showLicense()
else:
# processing only the given domains and excluding the ones provided
extra_domains = []
for d in args.domains:
if d not in args.exclude and not d == "all":
extra_domains.append(d)
# Two different arrays are mantained since there are some domains that cannot be safely verified
if args.create_emails:
potentially_existing_emails = grabEmails(
nicksFile=args.create_emails,
domains=EMAIL_DOMAINS + extra_domains,
excludeDomains=args.exclude)
potentially_leaked_emails = grabEmails(
nicksFile=args.create_emails,
domains=LEAKED_DOMAINS + extra_domains,
excludeDomains=args.exclude)
else:
potentially_existing_emails = grabEmails(
emails=args.emails,
emailsFile=args.emails_file,
nicks=args.nicks,
nicksFile=args.nicks_file,
domains=EMAIL_DOMAINS + extra_domains,
excludeDomains=args.exclude)
potentially_leaked_emails = grabEmails(emails=args.emails,
emailsFile=args.emails_file,
nicks=args.nicks,
nicksFile=args.nicks_file,
domains=LEAKED_DOMAINS +
extra_domains,
excludeDomains=args.exclude)
emails = list(
set(potentially_leaked_emails + potentially_existing_emails))
# Showing the execution time...
if not args.quiet:
startTime = dt.datetime.now()
print("{}\tStarting search of {} different emails:\n{}\n".format(
str(startTime), general.emphasis(str(len(emails))),
json.dumps(emails, indent=2, sort_keys=True)))
if not args.quiet:
now = dt.datetime.now()
print(
"\n{}\tStep 1. Trying to determine if the emails provided do exist...\n"
.format(str(now)))
print(general.emphasis("\tPress <Ctrl + C> to stop...\n"))
# Perform searches, using different Threads
results = performSearch(potentially_existing_emails,
nThreads=args.threads)
if not args.quiet:
now = dt.datetime.now()
print(
"\n{}\tStep 2. Checking if the emails have been used to register socialmedia accounts...\n"
.format(str(now)))
print(general.emphasis("\tPress <Ctrl + C> to stop...\n"))
registered = processMailList(platformNames=args.platforms,
emails=potentially_existing_emails)
results += registered
if not args.quiet:
if len(results) > 0:
for r in registered:
print("\t[*] Registered account found: {}".format(
general.success(r["value"])))
else:
print("\t[*] Registered account found: {}".format(
general.error("None")))
now = dt.datetime.now()
print(
"\n{}\tStep 3. Verifying if the provided emails have been leaked somewhere?\n"
.format(str(now)))
print(general.emphasis("\tPress <Ctrl + C> to stop...\n"))
# Verify the existence of the mails found as leaked emails.
for query in potentially_leaked_emails:
# Iterate through the different leak platforms
leaks = hibp.checkIfEmailWasHacked(query)
if len(leaks) > 0:
if not args.quiet:
if len(leaks) > 0:
print(
"\t[*] '{}' has been found in at least {} different leaks."
.format(general.success(query),
general.success(str(len(leaks)))))
else:
print("\t[*] '{}' has NOT been found in any leak.".
format(general.error(query)))
else:
if not args.quiet:
print("\t[*] '{}' has NOT been found on any leak yet.".
format(general.error(query)))
results += leaks
# Trying to store the information recovered
if args.output_folder != None:
if not os.path.exists(args.output_folder):
os.makedirs(args.output_folder)
# Grabbing the results
fileHeader = os.path.join(args.output_folder, args.file_header)
for ext in args.extension:
# Generating output files
general.exportUsufy(results, ext, fileHeader)
# Showing the information gathered if requested
if not args.quiet:
now = dt.datetime.now()
print("\n{}\tResults obtained:\n".format(str(now)))
print(general.success(general.usufyToTextExport(results)))
now = dt.datetime.now()
print(
"\n" + str(now) +
"\tYou can find all the information collected in the following files:"
)
for ext in args.extension:
# Showing the output files
print(general.emphasis("\t" + fileHeader + "." + ext))
# Showing the execution time...
if not args.quiet:
endTime = dt.datetime.now()
print("\n" + str(endTime) + "\tFinishing execution...\n")
print("Total time used:\t" +
general.emphasis(str(endTime - startTime)))
print("Average seconds/query:\t" + general.emphasis(
str((endTime - startTime).total_seconds() / len(emails))) +
" seconds\n")
if not args.quiet:
# Urging users to place an issue on Github...
print(banner.footer)
if params:
return results
def __init__(self, msg, *args, **kwargs):
Exception.__init__(self, "{}".format(general.error(msg)))
self.generic = "Generic OSRFramework error."
