def launchQueryForMode(self, query=None, mode=None):
"""
Method that launches an i3Browser to collect data.
Args:
-----
query: The query to be performed
mode: The mode to be used to build the query.
Return:
-------
A string containing the recovered data or None.
"""
# Creating the query URL for that mode
qURL = self.createURL(word=query, mode=mode)
i3Browser = browser.Browser()
try:
# Check if it needs creds
if self.needsCredentials[mode]:
self._getAuthenticated(i3Browser, qURL)
data = i3Browser.recoverURL(qURL)
else:
# Accessing the resources
data = i3Browser.recoverURL(qURL)
return data
except KeyError:
print(
general.error(
"[*] '{}' is not a valid mode for this wrapper ({}).".
format(mode, self.__class__.__name__)))
return None
def launch_query_for_mode(self, query=None, mode=None):
"""Method that launches an i3Browser to collect data
Args:
query: The query to be performed
mode: The mode to be used to build the query.
Returns:
A string containing the recovered data or None."""
# Creating the query URL for that mode
qURL = self.create_url(word=query, mode=mode)
i3Browser = browser.Browser()
try:
# Check if it needs creds
needs_credentials = False
try:
# Suport for version 2 of wrappers
if self.modes[mode]["needs_credentials"]:
needs_credentials = True
except AttributeError as e:
if self.needsCredentials[mode]:
needs_credentials = True
if needs_credentials:
self._getAuthenticated(i3Browser, qURL)
data = i3Browser.recover_url(qURL)
else:
# Accessing the resources
data = i3Browser.recover_url(qURL)
try:
if self.modes[mode]["debug"]:
with open(
os.path.join(tempfile.gettempdir(),
self.platformName) + ".html",
"w") as file:
print(
f"DEBUG mode is active for '{general.emphasis(self.platformName)}'. Data grabbed saved to: '{general.warning(file.name)}'"
)
file.write(data)
except:
pass
return data
except KeyError:
print(
general.error(
"[*] '{}' is not a valid mode for this wrapper ({}).".
format(mode, self.__class__.__name__)))
return None
def enumerateURL(urlDict, outputFolder, startIndex=0, maxErrors=100):
"""
Function that performs the enumeration itself.
"""
for i, url in enumerate(urlDict.keys()):
# Grabbing domain name:
domain = re.findall("://(.*)/", url)[0]
# Defining the starting index
index = startIndex
# The app will stop when this value reaches maxErrors
consecutiveErrors = 0
i3Browser = browser.Browser()
# Main loop that checks if the maximum number of errors has been reached
while consecutiveErrors <= maxErrors:
# creating the new URL to download
newQuery = url.replace("<INDEX>", str(index))
print(newQuery)
# Downloading the file
try:
data = i3Browser.recoverURL(newQuery)
filename = domain.replace(
"/", "|") + "_" + "-profile_" + str(index).rjust(
10, "0") + ".html"
if urlDict[url] != None:
if urlDict[url] in data:
print(
general.info("Storing resource as:\t" + filename +
"..."))
# The profile was found so we will store it:
with open(outputFolder + "/" + filename, "w") as oF:
oF.write(data)
else:
# The profile was found so we will store it:
print(
general.info("Storing resource as:\t" + filename +
"..."))
with open(outputFolder + "/" + filename, "w") as oF:
oF.write(data)
except:
pass
#logger.error("The resource could not be downloaded.")
index += 1
def getResults(uri):
'''
Method that recovers the text for each result in infobel.com
:param uri: Infobel uri
:return: A list of textual information to be processed
'''
# Using i3visio browser to avoid certain issues...
i3Browser = browser.Browser()
data = i3Browser.recoverURL(uri)
# Strings to be searched
regExp = "<!-- Results -->(.*)<!-- /Results -->"
# re.DOTALL is needed to match any character INCLUDING \n
results = re.findall(regExp, data, re.DOTALL)
return results
def launch_query_for_mode(self, query=None, mode=None):
"""Method that launches an i3Browser to collect data
Args:
query: The query to be performed
mode: The mode to be used to build the query.
Returns:
A string containing the recovered data or None."""
# Creating the query URL for that mode
qURL = self.create_url(word=query, mode=mode)
i3Browser = browser.Browser()
try:
# Check if it needs creds
needs_credentials = False
try:
# Suport for version 2 of wrappers
if self.modes[mode]["needs_credentials"]:
needs_credentials = True
except AttributeError as e:
if self.needsCredentials[mode]:
needs_credentials = True
if needs_credentials:
self._getAuthenticated(i3Browser, qURL)
data = i3Browser.recover_url(qURL)
else:
# Accessing the resources
data = i3Browser.recover_url(qURL)
return data
except KeyError:
print(
general.error(
"[*] '{}' is not a valid mode for this wrapper ({}).".
format(mode, self.__class__.__name__)))
return None
def fuzzUsufy(fDomains = None, fFuzzStruct = None):
'''
Method to guess the usufy path against a list of domains or subdomains.
:param fDomains: a list to strings containing the domains and (optionally) a nick.
:param fFuzzStruct: a list to strings containing the transforms to be performed.
:return: Dictionary of {domain: url}.
'''
logger = logging.getLogger("osrframework.usufy")
if fFuzzStruct == None:
# Loading these structures by default
fuzzingStructures = [
"http://<DOMAIN>/<USERNAME>",
"http://<DOMAIN>/~<USERNAME>",
"http://<DOMAIN>/?action=profile;user=<USERNAME>",
"http://<DOMAIN>/causes/author/<USERNAME>",
"http://<DOMAIN>/channel/<USERNAME>",
"http://<DOMAIN>/community/profile/<USERNAME>",
"http://<DOMAIN>/component/comprofiler/userprofiler/<USERNAME>",
"http://<DOMAIN>/details/@<USERNAME>",
"http://<DOMAIN>/foros/member.php?username=<USERNAME>",
"http://<DOMAIN>/forum/member/<USERNAME>",
"http://<DOMAIN>/forum/member.php?username=<USERNAME>",
"http://<DOMAIN>/forum/profile.php?mode=viewprofile&u=<USERNAME>",
"http://<DOMAIN>/home/<USERNAME>",
"http://<DOMAIN>/index.php?action=profile;user=<USERNAME>",
"http://<DOMAIN>/member_profile.php?u=<USERNAME>",
"http://<DOMAIN>/member.php?username=<USERNAME>",
"http://<DOMAIN>/members/?username=<USERNAME>",
"http://<DOMAIN>/members/<USERNAME>",
"http://<DOMAIN>/members/view/<USERNAME>",
"http://<DOMAIN>/mi-espacio/<USERNAME>",
"http://<DOMAIN>/u<USERNAME>",
"http://<DOMAIN>/u/<USERNAME>",
"http://<DOMAIN>/user-<USERNAME>",
"http://<DOMAIN>/user/<USERNAME>",
"http://<DOMAIN>/user/<USERNAME>.html",
"http://<DOMAIN>/users/<USERNAME>",
"http://<DOMAIN>/usr/<USERNAME>",
"http://<DOMAIN>/usuario/<USERNAME>",
"http://<DOMAIN>/usuarios/<USERNAME>",
"http://<DOMAIN>/en/users/<USERNAME>",
"http://<DOMAIN>/people/<USERNAME>",
"http://<DOMAIN>/profil/<USERNAME>",
"http://<DOMAIN>/profile/<USERNAME>",
"http://<DOMAIN>/profile/page/<USERNAME>",
"http://<DOMAIN>/rapidforum/index.php?action=profile;user=<USERNAME>",
"http://<DOMAIN>/social/usuarios/<USERNAME>",
"http://<USERNAME>.<DOMAIN>",
"http://<USERNAME>.<DOMAIN>/user/"
]
else:
try:
fuzzingStructures = fFuzzStruct.read().splitlines()
except:
logger.error("Usufy could NOT open the following file: " + fFuzzStruct )
res = {}
lines = fDomains.read().splitlines()
# Going through all the lines
for l in lines:
domain = l.split()[0]
print "Performing tests for" + domain + "..."
# selecting the number of nicks to be tested in this domain
nick = l.split()[1]
# Choosing the errors from the input file
#errors = l.split('\t')[2:]
# possibleURLs found
possibleURL = []
for struct in fuzzingStructures:
# initiating list
urlToTry = struct.replace("<DOMAIN>", domain)
test = urlToTry.replace("<USERNAME>", nick.lower())
print "Processing "+ test + "..."
i3Browser = browser.Browser()
try:
html = i3Browser.recoverURL(test)
if nick in html:
possibleURL.append(test)
print "\tPossible usufy found!!!\n"
except:
logger.error("The resource could not be downloaded.")
#print possibleURL
res[domain] = possibleURL
print json.dumps(res, indent = 2)
return res
def processData(self, uri=None, data=None, mode=None):
'''
Method to process and extract the entities of a URL of this type.
:param uri: The URI of this platform to be processed.
:param data: The information from which the info will be extracted. This way, info will not be downloaded twice.
:param mode: Mode to be executed.
:return: A list of the entities found.
'''
if data == None:
# Accessing the resource
i3Browser = browser.Browser()
try:
# check if it needs creds
if self.needsCredentials[mode]:
authenticated = self._getAuthenticated(i3Browser)
if authenticated:
# Accessing the resources
data = i3Browser.recoverURL(uri)
else:
# Accessing the resources
data = i3Browser.recoverURL(uri)
except:
# No information was found, then we return a null entity
# TO-DO: i3BrowserException
return json.dumps({})
#else:
# return json.dumps({})
info = []
# Searchfy needs an special treatment to recover the results
if mode != "searchfy":
# Iterating through all the type of fields
for field in self.fieldsRegExp[mode].keys():
# Recovering the RegularExpression
try:
# Using the old approach of "Start" + "End"
regexp = self.fieldsRegExp[mode][field][
"start"] + "([^\)]+)" + self.fieldsRegExp[mode][field][
"end"]
tmp = re.findall(regexp, data)
# Now we are performing an operation just in case the "end" tag is found in the results, which would mean that the tag selected matches something longer in the data.
values = []
for t in tmp:
if self.fieldsRegExp[mode][field]["end"] in t:
values.append(
t.split(
self.fieldsRegExp[mode][field]["end"])[0])
else:
values.append(t)
except:
# Using the compact approach if start and end tags do not exist.
regexp = self.fieldsRegExp[mode][field]
values = re.findall(regexp, data)
for val in values:
aux = {}
aux["type"] = field
aux["value"] = val
aux["attributes"] = []
if aux not in info:
info.append(aux)
# Searchfy results
else:
# Grabbing the results for the search
resultText = re.findall(searchfyAliasRegexp, data)
# Analysing each and every result to parse it...
for resURI in resultText:
r = {}
r["type"] = "i3visio.uri"
r["value"] = resURI
r["attributes"] = []
"""# Iterating through all the type of fields
i3Browser = browser.Browser()
try:
# check if the profile needs credentials in usufy mode
if self.needsCredentials["usufy"]:
authenticated = self._getAuthenticated(i3Browser)
if authenticated:
# Accessing the resources
data = i3Browser.recoverURL(resURI)
else:
# Accessing the resources
data = i3Browser.recoverURL(resURI)
except:
data = ""
for field in self.fieldsRegExp["usufy"].keys():
# Building the regular expression if the format is a "start" and "end" approach... Easier to understand but less compact.
try:
# Using the old approach of "Start" + "End"
regexp = self.fieldsRegExp["usufy"][field]["start"]+"([^\)]+)"+self.fieldsRegExp["usufy"][field]["end"]
# Parsing the result for the text
tmp = re.findall(regexp, data)
# Now we are performing an operation just in case the "end" tag is found in the results, which would mean that the tag selected matches something longer in the data.
values = []
for t in tmp:
if self.fieldsRegExp["usufy"][field]["end"] in t:
values.append(t.split(self.fieldsRegExp["usufy"][field]["end"])[0])
else:
values.append(t)
# In the case of a compact approach being used. This would happen if start and end tags do not exist, but the expected behaviour is the same.
except:
regexp = self.fieldsRegExp["usufy"][field]
values = re.findall(regexp, data)
if field == "i3visio.uri":
for val in values:
r["value"] = val
else:
for val in values:
aux = {}
aux["type"] = field
aux["value"] = val
aux["attributes"] = []
if aux not in r["attributes"]:
r["attributes"].append(aux) """
r["attributes"] = json.loads(
self.getInfo(process=True, mode="usufy", qURI=resURI))
info.append(r)
return json.dumps(info)
def getInfo(self, query=None, process=False, mode="phonefy", qURI=None):
'''
Method that checks the presence of a given query and recovers the first list of complains.
:param query: Query to verify.
:param proces: Calling the processing function.
:param mode: Mode to be executed.
:param qURI: A query to be checked
:return: Python structure for the html processed.
'''
# Defining variables for this process
results = []
data = ""
if not self.modeIsValid(mode=mode):
# TO-DO: InvalidModeException
return json.dumps(results)
# Verrifying if the mode is valid
if not self._isValidQuery(query, mode=mode):
# TO-DO: InvalidQueryException
return json.dumps(results)
# Verifying if the platform has an API defined
try:
if type(self.wrapperAPI) != "<type 'NoneType'>":
if mode == "phonefy":
pass
elif mode == "usufy":
results = self.wrapperAPI.get_user(query)
# Manually appending the URL
for r in results:
aux = {}
aux["type"] = "i3visio.uri"
alias = r["value"].split(' - ')[1]
qURL, query = self.createURL(word=alias, mode="usufy")
aux["value"] = qURL
aux["attributes"] = []
r["attributes"].append(aux)
elif mode == "searchfy":
results = self.wrapperAPI.search_users(query)
# Manually appending the URL
for r in results:
aux = {}
aux["type"] = "i3visio.uri"
alias = r["value"].split(' - ')[1]
qURL, query = self.createURL(word=alias, mode="usufy")
aux["value"] = qURL
aux["attributes"] = []
r["attributes"].append(aux)
else:
# NoneType returned
pass
# The platform does not have a Wrapper defined for its API... Then we will use the traditional approach...
except:
# Creating the query URL for that mode
if qURI != None:
qURL = qURI
else:
qURL, query = self.createURL(word=query, mode=mode)
i3Browser = browser.Browser()
try:
# check if it needs creds
if self.needsCredentials[mode]:
authenticated = self._getAuthenticated(i3Browser)
if authenticated:
# Accessing the resources
data = i3Browser.recoverURL(qURL)
else:
# Accessing the resources
data = i3Browser.recoverURL(qURL)
except:
# No information was found, then we return a null entity
# TO-DO: i3BrowserException
return json.dumps(results)
# Verifying if the platform exists
if self.somethingFound(data, mode=mode):
if mode == "phonefy":
r = {}
r["type"] = "i3visio.phone"
r["value"] = self.platformName + " - " + query
r["attributes"] = []
# Appending platform URI
aux = {}
aux["type"] = "i3visio.uri"
aux["value"] = qURL
aux["attributes"] = []
r["attributes"].append(aux)
# Appending platform name
aux = {}
aux["type"] = "i3visio.platform"
aux["value"] = self.platformName
aux["attributes"] = []
r["attributes"].append(aux)
# Iterating if requested to extract more entities from the URI
if process:
# This function returns a json text!
r["attributes"] += json.loads(
self.processData(data=data, mode=mode))
# Appending the result to results: in this case only one profile will be grabbed
results.append(r)
elif mode == "usufy":
r = {}
r["type"] = "i3visio.profile"
r["value"] = self.platformName + " - " + query
r["attributes"] = []
# Appending platform URI
aux = {}
aux["type"] = "i3visio.uri"
aux["value"] = qURL
aux["attributes"] = []
r["attributes"].append(aux)
# Appending the alias
aux = {}
aux["type"] = "i3visio.alias"
aux["value"] = query
aux["attributes"] = []
r["attributes"].append(aux)
# Appending platform name
aux = {}
aux["type"] = "i3visio.platform"
aux["value"] = self.platformName
aux["attributes"] = []
r["attributes"].append(aux)
# Iterating if requested to extract more entities from the URI
if process:
# This function returns a json text!
r["attributes"] += json.loads(
self.processData(data=data, mode=mode))
# Appending the result to results: in this case only one profile will be grabbed
results.append(r)
elif mode == "searchfy":
# Recovering all the found aliases...
ids = re.findall(self.searchfyAliasRegexp, data, re.DOTALL)
for j, i in enumerate(ids):
r = {}
r["type"] = "i3visio.profile"
r["value"] = self.platformName + " - " + i
r["attributes"] = []
# Appending platform URI
aux = {}
aux["type"] = "i3visio.uri"
# Creating the URI based on the base URL for the new profiles...
uri, alias = self.createURL(word=i, mode="base")
#uri=self.baseURL+i
aux["value"] = uri
aux["attributes"] = []
r["attributes"].append(aux)
# Appending the alias
aux = {}
aux["type"] = "i3visio.alias"
aux["value"] = alias
aux["attributes"] = []
r["attributes"].append(aux)
# Appending platform name
aux = {}
aux["type"] = "i3visio.platform"
aux["value"] = self.platformName
aux["attributes"] = []
r["attributes"].append(aux)
# Appending the query performed to grab this items
aux = {}
aux["type"] = "i3visio.search"
aux["value"] = query
aux["attributes"] = []
r["attributes"].append(aux)
# TO-DO:
# Perform additional procesing
# Iterating the requested profiles to extract more entities from the URI would be slow!
"""if process:
# This function returns a json text in usufy format for the returned objects.
r["attributes"] += json.loads(self.getInfo(process = True, mode="usufy", qURI=uri, query=i))
# Appending the result to results: in this case only one profile will be grabbed"""
results.append(r)
return json.dumps(results)
def fuzzUsufy(fDomains=None, fFuzzStruct=None):
"""
Method to guess the usufy path against a list of domains or subdomains.
Args:
fDomains: A list to strings containing the domains and (optionally) a
nick.
fFuzzStruct: A list to strings containing the transforms to be
performed.
Returns:
dict: A dictionary of the form of `{"domain": "url"}`.
"""
if fFuzzStruct == None:
# Loading these structures by default
fuzzingStructures = [
"http://<DOMAIN>/<USERNAME>", "http://<DOMAIN>/~<USERNAME>",
"http://<DOMAIN>/?action=profile;user=<USERNAME>",
"http://<DOMAIN>/causes/author/<USERNAME>",
"http://<DOMAIN>/channel/<USERNAME>",
"http://<DOMAIN>/community/profile/<USERNAME>",
"http://<DOMAIN>/component/comprofiler/userprofiler/<USERNAME>",
"http://<DOMAIN>/details/@<USERNAME>",
"http://<DOMAIN>/foros/member.php?username=<USERNAME>",
"http://<DOMAIN>/forum/member/<USERNAME>",
"http://<DOMAIN>/forum/member.php?username=<USERNAME>",
"http://<DOMAIN>/forum/profile.php?mode=viewprofile&u=<USERNAME>",
"http://<DOMAIN>/home/<USERNAME>",
"http://<DOMAIN>/index.php?action=profile;user=<USERNAME>",
"http://<DOMAIN>/member_profile.php?u=<USERNAME>",
"http://<DOMAIN>/member.php?username=<USERNAME>",
"http://<DOMAIN>/members/?username=<USERNAME>",
"http://<DOMAIN>/members/<USERNAME>",
"http://<DOMAIN>/members/view/<USERNAME>",
"http://<DOMAIN>/mi-espacio/<USERNAME>",
"http://<DOMAIN>/u<USERNAME>", "http://<DOMAIN>/u/<USERNAME>",
"http://<DOMAIN>/user-<USERNAME>",
"http://<DOMAIN>/user/<USERNAME>",
"http://<DOMAIN>/user/<USERNAME>.html",
"http://<DOMAIN>/users/<USERNAME>",
"http://<DOMAIN>/usr/<USERNAME>",
"http://<DOMAIN>/usuario/<USERNAME>",
"http://<DOMAIN>/usuarios/<USERNAME>",
"http://<DOMAIN>/en/users/<USERNAME>",
"http://<DOMAIN>/people/<USERNAME>",
"http://<DOMAIN>/profil/<USERNAME>",
"http://<DOMAIN>/profile/<USERNAME>",
"http://<DOMAIN>/profile/page/<USERNAME>",
"http://<DOMAIN>/rapidforum/index.php?action=profile;user=<USERNAME>",
"http://<DOMAIN>/social/usuarios/<USERNAME>",
"http://<USERNAME>.<DOMAIN>", "http://<USERNAME>.<DOMAIN>/user/"
]
else:
try:
fuzzingStructures = fFuzzStruct.read().splitlines()
except:
print("Usufy could NOT open the following file: " + fFuzzStruct)
res = {}
lines = fDomains.read().splitlines()
# Going through all the lines
for l in lines:
domain = l.split()[0]
print("Performing tests for" + domain + "...")
# selecting the number of nicks to be tested in this domain
nick = l.split()[1]
# possibleURLs found
possibleURL = []
for struct in fuzzingStructures:
# initiating list
urlToTry = struct.replace("<DOMAIN>", domain)
test = urlToTry.replace("<USERNAME>", nick.lower())
print("Processing " + test + "...")
i3Browser = browser.Browser()
try:
html = i3Browser.recoverURL(test)
if nick in html:
possibleURL.append(test)
print(general.success("\tPossible usufy found!!!\n"))
except:
print("The resource could not be downloaded.")
res[domain] = possibleURL
print(json.dumps(res, indent=2))
return res
def uriToI3visioEntities(argv, platform='all'):
'''
Method that obtains all the entities in a given profile.
:param argv: the uri to be received.
:param platform: a platform string representing the regular expression to be used.
:return: Nothing is returned but the code of the entities is created.
'''
me = MaltegoTransform(argv)
# Recovering the Uri value
try:
uri = argv
except:
uri = me.getVar("@value")
#print uri
newEntities = []
# Defining the main entity
aux = {}
aux["type"] = "i3visio.uri"
aux["value"] = uri
aux["attributes"] = []
newEntities.append(aux)
# Using i3visio browser to avoid certain issues...
i3Browser = browser.Browser()
# Accessing the resources
data = i3Browser.recoverURL(uri)
# Getting the list of <RegExp> objects from entify
lRegexp = regexp_selection.getRegexpsByName([platform])
newEntities = entify.getEntitiesByRegexp(data=data, listRegexp=lRegexp)
# This returns a dictionary like the following:
"""
[{
'attributes': [],
'type': 'i3visio.sha256',
'value': 'a9b8c5d848205db514d4097d2b78f4528d01a79f39601e0f9c5c40ed689471'
}, {
'attributes': [],
'type': 'i3visio.sha256',
'value': 'b28b896e6eeb8d651cacd5f4a4d1490fbe9d05dbc92221609350b0ce7a68e9'
}, {
'attributes': [],
'type': 'i3visio.sha256',
'value': 'd727fed4d969b14b28165c75ad12d7dddd56c0198fa70cedc3fdad7ac395b2'
}, {
'attributes': [],
'type': 'i3visio.sha256',
'value': '3e9a2204fcfc6f7dde250e61ca35353411880024102cba14a0bd45f05f1e74'
}]
"""
# Adding list of entities to be displayed
me.addListOfEntities(newEntities)
#print json.dumps(entities, indent=2)
#for elem in entities:
# newEnt = me.addEntity(elem["type"],elem["value"])
# newEnt.setDisplayInformation("<h3>" + elem["value"] +"</h3><p>"+str(elem["attributes"])+"</p>")
# for extraAtt in elem["attributes"]:
# newEnt.addAdditionalFields(str(extraAtt['type']), str(extraAtt['type']), True, str(extraAtt['value']))
# Returning the output text...
me.returnOutput()
