def launchQueryForMode(self, query=None, mode=None): """ Method that launches an i3Browser to collect data. Args: ----- query: The query to be performed mode: The mode to be used to build the query. Return: ------- A string containing the recovered data or None. """ # Creating the query URL for that mode qURL = self.createURL(word=query, mode=mode) i3Browser = browser.Browser() try: # Check if it needs creds if self.needsCredentials[mode]: self._getAuthenticated(i3Browser, qURL) data = i3Browser.recoverURL(qURL) else: # Accessing the resources data = i3Browser.recoverURL(qURL) return data except KeyError: print( general.error( "[*] '{}' is not a valid mode for this wrapper ({}).". format(mode, self.__class__.__name__))) return None
def launch_query_for_mode(self, query=None, mode=None): """Method that launches an i3Browser to collect data Args: query: The query to be performed mode: The mode to be used to build the query. Returns: A string containing the recovered data or None.""" # Creating the query URL for that mode qURL = self.create_url(word=query, mode=mode) i3Browser = browser.Browser() try: # Check if it needs creds needs_credentials = False try: # Suport for version 2 of wrappers if self.modes[mode]["needs_credentials"]: needs_credentials = True except AttributeError as e: if self.needsCredentials[mode]: needs_credentials = True if needs_credentials: self._getAuthenticated(i3Browser, qURL) data = i3Browser.recover_url(qURL) else: # Accessing the resources data = i3Browser.recover_url(qURL) try: if self.modes[mode]["debug"]: with open( os.path.join(tempfile.gettempdir(), self.platformName) + ".html", "w") as file: print( f"DEBUG mode is active for '{general.emphasis(self.platformName)}'. Data grabbed saved to: '{general.warning(file.name)}'" ) file.write(data) except: pass return data except KeyError: print( general.error( "[*] '{}' is not a valid mode for this wrapper ({}).". format(mode, self.__class__.__name__))) return None
def enumerateURL(urlDict, outputFolder, startIndex=0, maxErrors=100): """ Function that performs the enumeration itself. """ for i, url in enumerate(urlDict.keys()): # Grabbing domain name: domain = re.findall("://(.*)/", url)[0] # Defining the starting index index = startIndex # The app will stop when this value reaches maxErrors consecutiveErrors = 0 i3Browser = browser.Browser() # Main loop that checks if the maximum number of errors has been reached while consecutiveErrors <= maxErrors: # creating the new URL to download newQuery = url.replace("<INDEX>", str(index)) print(newQuery) # Downloading the file try: data = i3Browser.recoverURL(newQuery) filename = domain.replace( "/", "|") + "_" + "-profile_" + str(index).rjust( 10, "0") + ".html" if urlDict[url] != None: if urlDict[url] in data: print( general.info("Storing resource as:\t" + filename + "...")) # The profile was found so we will store it: with open(outputFolder + "/" + filename, "w") as oF: oF.write(data) else: # The profile was found so we will store it: print( general.info("Storing resource as:\t" + filename + "...")) with open(outputFolder + "/" + filename, "w") as oF: oF.write(data) except: pass #logger.error("The resource could not be downloaded.") index += 1
def getResults(uri): ''' Method that recovers the text for each result in infobel.com :param uri: Infobel uri :return: A list of textual information to be processed ''' # Using i3visio browser to avoid certain issues... i3Browser = browser.Browser() data = i3Browser.recoverURL(uri) # Strings to be searched regExp = "<!-- Results -->(.*)<!-- /Results -->" # re.DOTALL is needed to match any character INCLUDING \n results = re.findall(regExp, data, re.DOTALL) return results
def launch_query_for_mode(self, query=None, mode=None): """Method that launches an i3Browser to collect data Args: query: The query to be performed mode: The mode to be used to build the query. Returns: A string containing the recovered data or None.""" # Creating the query URL for that mode qURL = self.create_url(word=query, mode=mode) i3Browser = browser.Browser() try: # Check if it needs creds needs_credentials = False try: # Suport for version 2 of wrappers if self.modes[mode]["needs_credentials"]: needs_credentials = True except AttributeError as e: if self.needsCredentials[mode]: needs_credentials = True if needs_credentials: self._getAuthenticated(i3Browser, qURL) data = i3Browser.recover_url(qURL) else: # Accessing the resources data = i3Browser.recover_url(qURL) return data except KeyError: print( general.error( "[*] '{}' is not a valid mode for this wrapper ({}).". format(mode, self.__class__.__name__))) return None
def fuzzUsufy(fDomains = None, fFuzzStruct = None): ''' Method to guess the usufy path against a list of domains or subdomains. :param fDomains: a list to strings containing the domains and (optionally) a nick. :param fFuzzStruct: a list to strings containing the transforms to be performed. :return: Dictionary of {domain: url}. ''' logger = logging.getLogger("osrframework.usufy") if fFuzzStruct == None: # Loading these structures by default fuzzingStructures = [ "http://<DOMAIN>/<USERNAME>", "http://<DOMAIN>/~<USERNAME>", "http://<DOMAIN>/?action=profile;user=<USERNAME>", "http://<DOMAIN>/causes/author/<USERNAME>", "http://<DOMAIN>/channel/<USERNAME>", "http://<DOMAIN>/community/profile/<USERNAME>", "http://<DOMAIN>/component/comprofiler/userprofiler/<USERNAME>", "http://<DOMAIN>/details/@<USERNAME>", "http://<DOMAIN>/foros/member.php?username=<USERNAME>", "http://<DOMAIN>/forum/member/<USERNAME>", "http://<DOMAIN>/forum/member.php?username=<USERNAME>", "http://<DOMAIN>/forum/profile.php?mode=viewprofile&u=<USERNAME>", "http://<DOMAIN>/home/<USERNAME>", "http://<DOMAIN>/index.php?action=profile;user=<USERNAME>", "http://<DOMAIN>/member_profile.php?u=<USERNAME>", "http://<DOMAIN>/member.php?username=<USERNAME>", "http://<DOMAIN>/members/?username=<USERNAME>", "http://<DOMAIN>/members/<USERNAME>", "http://<DOMAIN>/members/view/<USERNAME>", "http://<DOMAIN>/mi-espacio/<USERNAME>", "http://<DOMAIN>/u<USERNAME>", "http://<DOMAIN>/u/<USERNAME>", "http://<DOMAIN>/user-<USERNAME>", "http://<DOMAIN>/user/<USERNAME>", "http://<DOMAIN>/user/<USERNAME>.html", "http://<DOMAIN>/users/<USERNAME>", "http://<DOMAIN>/usr/<USERNAME>", "http://<DOMAIN>/usuario/<USERNAME>", "http://<DOMAIN>/usuarios/<USERNAME>", "http://<DOMAIN>/en/users/<USERNAME>", "http://<DOMAIN>/people/<USERNAME>", "http://<DOMAIN>/profil/<USERNAME>", "http://<DOMAIN>/profile/<USERNAME>", "http://<DOMAIN>/profile/page/<USERNAME>", "http://<DOMAIN>/rapidforum/index.php?action=profile;user=<USERNAME>", "http://<DOMAIN>/social/usuarios/<USERNAME>", "http://<USERNAME>.<DOMAIN>", "http://<USERNAME>.<DOMAIN>/user/" ] else: try: fuzzingStructures = fFuzzStruct.read().splitlines() except: logger.error("Usufy could NOT open the following file: " + fFuzzStruct ) res = {} lines = fDomains.read().splitlines() # Going through all the lines for l in lines: domain = l.split()[0] print "Performing tests for" + domain + "..." # selecting the number of nicks to be tested in this domain nick = l.split()[1] # Choosing the errors from the input file #errors = l.split('\t')[2:] # possibleURLs found possibleURL = [] for struct in fuzzingStructures: # initiating list urlToTry = struct.replace("<DOMAIN>", domain) test = urlToTry.replace("<USERNAME>", nick.lower()) print "Processing "+ test + "..." i3Browser = browser.Browser() try: html = i3Browser.recoverURL(test) if nick in html: possibleURL.append(test) print "\tPossible usufy found!!!\n" except: logger.error("The resource could not be downloaded.") #print possibleURL res[domain] = possibleURL print json.dumps(res, indent = 2) return res
def processData(self, uri=None, data=None, mode=None): ''' Method to process and extract the entities of a URL of this type. :param uri: The URI of this platform to be processed. :param data: The information from which the info will be extracted. This way, info will not be downloaded twice. :param mode: Mode to be executed. :return: A list of the entities found. ''' if data == None: # Accessing the resource i3Browser = browser.Browser() try: # check if it needs creds if self.needsCredentials[mode]: authenticated = self._getAuthenticated(i3Browser) if authenticated: # Accessing the resources data = i3Browser.recoverURL(uri) else: # Accessing the resources data = i3Browser.recoverURL(uri) except: # No information was found, then we return a null entity # TO-DO: i3BrowserException return json.dumps({}) #else: # return json.dumps({}) info = [] # Searchfy needs an special treatment to recover the results if mode != "searchfy": # Iterating through all the type of fields for field in self.fieldsRegExp[mode].keys(): # Recovering the RegularExpression try: # Using the old approach of "Start" + "End" regexp = self.fieldsRegExp[mode][field][ "start"] + "([^\)]+)" + self.fieldsRegExp[mode][field][ "end"] tmp = re.findall(regexp, data) # Now we are performing an operation just in case the "end" tag is found in the results, which would mean that the tag selected matches something longer in the data. values = [] for t in tmp: if self.fieldsRegExp[mode][field]["end"] in t: values.append( t.split( self.fieldsRegExp[mode][field]["end"])[0]) else: values.append(t) except: # Using the compact approach if start and end tags do not exist. regexp = self.fieldsRegExp[mode][field] values = re.findall(regexp, data) for val in values: aux = {} aux["type"] = field aux["value"] = val aux["attributes"] = [] if aux not in info: info.append(aux) # Searchfy results else: # Grabbing the results for the search resultText = re.findall(searchfyAliasRegexp, data) # Analysing each and every result to parse it... for resURI in resultText: r = {} r["type"] = "i3visio.uri" r["value"] = resURI r["attributes"] = [] """# Iterating through all the type of fields i3Browser = browser.Browser() try: # check if the profile needs credentials in usufy mode if self.needsCredentials["usufy"]: authenticated = self._getAuthenticated(i3Browser) if authenticated: # Accessing the resources data = i3Browser.recoverURL(resURI) else: # Accessing the resources data = i3Browser.recoverURL(resURI) except: data = "" for field in self.fieldsRegExp["usufy"].keys(): # Building the regular expression if the format is a "start" and "end" approach... Easier to understand but less compact. try: # Using the old approach of "Start" + "End" regexp = self.fieldsRegExp["usufy"][field]["start"]+"([^\)]+)"+self.fieldsRegExp["usufy"][field]["end"] # Parsing the result for the text tmp = re.findall(regexp, data) # Now we are performing an operation just in case the "end" tag is found in the results, which would mean that the tag selected matches something longer in the data. values = [] for t in tmp: if self.fieldsRegExp["usufy"][field]["end"] in t: values.append(t.split(self.fieldsRegExp["usufy"][field]["end"])[0]) else: values.append(t) # In the case of a compact approach being used. This would happen if start and end tags do not exist, but the expected behaviour is the same. except: regexp = self.fieldsRegExp["usufy"][field] values = re.findall(regexp, data) if field == "i3visio.uri": for val in values: r["value"] = val else: for val in values: aux = {} aux["type"] = field aux["value"] = val aux["attributes"] = [] if aux not in r["attributes"]: r["attributes"].append(aux) """ r["attributes"] = json.loads( self.getInfo(process=True, mode="usufy", qURI=resURI)) info.append(r) return json.dumps(info)
def getInfo(self, query=None, process=False, mode="phonefy", qURI=None): ''' Method that checks the presence of a given query and recovers the first list of complains. :param query: Query to verify. :param proces: Calling the processing function. :param mode: Mode to be executed. :param qURI: A query to be checked :return: Python structure for the html processed. ''' # Defining variables for this process results = [] data = "" if not self.modeIsValid(mode=mode): # TO-DO: InvalidModeException return json.dumps(results) # Verrifying if the mode is valid if not self._isValidQuery(query, mode=mode): # TO-DO: InvalidQueryException return json.dumps(results) # Verifying if the platform has an API defined try: if type(self.wrapperAPI) != "<type 'NoneType'>": if mode == "phonefy": pass elif mode == "usufy": results = self.wrapperAPI.get_user(query) # Manually appending the URL for r in results: aux = {} aux["type"] = "i3visio.uri" alias = r["value"].split(' - ')[1] qURL, query = self.createURL(word=alias, mode="usufy") aux["value"] = qURL aux["attributes"] = [] r["attributes"].append(aux) elif mode == "searchfy": results = self.wrapperAPI.search_users(query) # Manually appending the URL for r in results: aux = {} aux["type"] = "i3visio.uri" alias = r["value"].split(' - ')[1] qURL, query = self.createURL(word=alias, mode="usufy") aux["value"] = qURL aux["attributes"] = [] r["attributes"].append(aux) else: # NoneType returned pass # The platform does not have a Wrapper defined for its API... Then we will use the traditional approach... except: # Creating the query URL for that mode if qURI != None: qURL = qURI else: qURL, query = self.createURL(word=query, mode=mode) i3Browser = browser.Browser() try: # check if it needs creds if self.needsCredentials[mode]: authenticated = self._getAuthenticated(i3Browser) if authenticated: # Accessing the resources data = i3Browser.recoverURL(qURL) else: # Accessing the resources data = i3Browser.recoverURL(qURL) except: # No information was found, then we return a null entity # TO-DO: i3BrowserException return json.dumps(results) # Verifying if the platform exists if self.somethingFound(data, mode=mode): if mode == "phonefy": r = {} r["type"] = "i3visio.phone" r["value"] = self.platformName + " - " + query r["attributes"] = [] # Appending platform URI aux = {} aux["type"] = "i3visio.uri" aux["value"] = qURL aux["attributes"] = [] r["attributes"].append(aux) # Appending platform name aux = {} aux["type"] = "i3visio.platform" aux["value"] = self.platformName aux["attributes"] = [] r["attributes"].append(aux) # Iterating if requested to extract more entities from the URI if process: # This function returns a json text! r["attributes"] += json.loads( self.processData(data=data, mode=mode)) # Appending the result to results: in this case only one profile will be grabbed results.append(r) elif mode == "usufy": r = {} r["type"] = "i3visio.profile" r["value"] = self.platformName + " - " + query r["attributes"] = [] # Appending platform URI aux = {} aux["type"] = "i3visio.uri" aux["value"] = qURL aux["attributes"] = [] r["attributes"].append(aux) # Appending the alias aux = {} aux["type"] = "i3visio.alias" aux["value"] = query aux["attributes"] = [] r["attributes"].append(aux) # Appending platform name aux = {} aux["type"] = "i3visio.platform" aux["value"] = self.platformName aux["attributes"] = [] r["attributes"].append(aux) # Iterating if requested to extract more entities from the URI if process: # This function returns a json text! r["attributes"] += json.loads( self.processData(data=data, mode=mode)) # Appending the result to results: in this case only one profile will be grabbed results.append(r) elif mode == "searchfy": # Recovering all the found aliases... ids = re.findall(self.searchfyAliasRegexp, data, re.DOTALL) for j, i in enumerate(ids): r = {} r["type"] = "i3visio.profile" r["value"] = self.platformName + " - " + i r["attributes"] = [] # Appending platform URI aux = {} aux["type"] = "i3visio.uri" # Creating the URI based on the base URL for the new profiles... uri, alias = self.createURL(word=i, mode="base") #uri=self.baseURL+i aux["value"] = uri aux["attributes"] = [] r["attributes"].append(aux) # Appending the alias aux = {} aux["type"] = "i3visio.alias" aux["value"] = alias aux["attributes"] = [] r["attributes"].append(aux) # Appending platform name aux = {} aux["type"] = "i3visio.platform" aux["value"] = self.platformName aux["attributes"] = [] r["attributes"].append(aux) # Appending the query performed to grab this items aux = {} aux["type"] = "i3visio.search" aux["value"] = query aux["attributes"] = [] r["attributes"].append(aux) # TO-DO: # Perform additional procesing # Iterating the requested profiles to extract more entities from the URI would be slow! """if process: # This function returns a json text in usufy format for the returned objects. r["attributes"] += json.loads(self.getInfo(process = True, mode="usufy", qURI=uri, query=i)) # Appending the result to results: in this case only one profile will be grabbed""" results.append(r) return json.dumps(results)
def fuzzUsufy(fDomains=None, fFuzzStruct=None): """ Method to guess the usufy path against a list of domains or subdomains. Args: fDomains: A list to strings containing the domains and (optionally) a nick. fFuzzStruct: A list to strings containing the transforms to be performed. Returns: dict: A dictionary of the form of `{"domain": "url"}`. """ if fFuzzStruct == None: # Loading these structures by default fuzzingStructures = [ "http://<DOMAIN>/<USERNAME>", "http://<DOMAIN>/~<USERNAME>", "http://<DOMAIN>/?action=profile;user=<USERNAME>", "http://<DOMAIN>/causes/author/<USERNAME>", "http://<DOMAIN>/channel/<USERNAME>", "http://<DOMAIN>/community/profile/<USERNAME>", "http://<DOMAIN>/component/comprofiler/userprofiler/<USERNAME>", "http://<DOMAIN>/details/@<USERNAME>", "http://<DOMAIN>/foros/member.php?username=<USERNAME>", "http://<DOMAIN>/forum/member/<USERNAME>", "http://<DOMAIN>/forum/member.php?username=<USERNAME>", "http://<DOMAIN>/forum/profile.php?mode=viewprofile&u=<USERNAME>", "http://<DOMAIN>/home/<USERNAME>", "http://<DOMAIN>/index.php?action=profile;user=<USERNAME>", "http://<DOMAIN>/member_profile.php?u=<USERNAME>", "http://<DOMAIN>/member.php?username=<USERNAME>", "http://<DOMAIN>/members/?username=<USERNAME>", "http://<DOMAIN>/members/<USERNAME>", "http://<DOMAIN>/members/view/<USERNAME>", "http://<DOMAIN>/mi-espacio/<USERNAME>", "http://<DOMAIN>/u<USERNAME>", "http://<DOMAIN>/u/<USERNAME>", "http://<DOMAIN>/user-<USERNAME>", "http://<DOMAIN>/user/<USERNAME>", "http://<DOMAIN>/user/<USERNAME>.html", "http://<DOMAIN>/users/<USERNAME>", "http://<DOMAIN>/usr/<USERNAME>", "http://<DOMAIN>/usuario/<USERNAME>", "http://<DOMAIN>/usuarios/<USERNAME>", "http://<DOMAIN>/en/users/<USERNAME>", "http://<DOMAIN>/people/<USERNAME>", "http://<DOMAIN>/profil/<USERNAME>", "http://<DOMAIN>/profile/<USERNAME>", "http://<DOMAIN>/profile/page/<USERNAME>", "http://<DOMAIN>/rapidforum/index.php?action=profile;user=<USERNAME>", "http://<DOMAIN>/social/usuarios/<USERNAME>", "http://<USERNAME>.<DOMAIN>", "http://<USERNAME>.<DOMAIN>/user/" ] else: try: fuzzingStructures = fFuzzStruct.read().splitlines() except: print("Usufy could NOT open the following file: " + fFuzzStruct) res = {} lines = fDomains.read().splitlines() # Going through all the lines for l in lines: domain = l.split()[0] print("Performing tests for" + domain + "...") # selecting the number of nicks to be tested in this domain nick = l.split()[1] # possibleURLs found possibleURL = [] for struct in fuzzingStructures: # initiating list urlToTry = struct.replace("<DOMAIN>", domain) test = urlToTry.replace("<USERNAME>", nick.lower()) print("Processing " + test + "...") i3Browser = browser.Browser() try: html = i3Browser.recoverURL(test) if nick in html: possibleURL.append(test) print(general.success("\tPossible usufy found!!!\n")) except: print("The resource could not be downloaded.") res[domain] = possibleURL print(json.dumps(res, indent=2)) return res
def uriToI3visioEntities(argv, platform='all'): ''' Method that obtains all the entities in a given profile. :param argv: the uri to be received. :param platform: a platform string representing the regular expression to be used. :return: Nothing is returned but the code of the entities is created. ''' me = MaltegoTransform(argv) # Recovering the Uri value try: uri = argv except: uri = me.getVar("@value") #print uri newEntities = [] # Defining the main entity aux = {} aux["type"] = "i3visio.uri" aux["value"] = uri aux["attributes"] = [] newEntities.append(aux) # Using i3visio browser to avoid certain issues... i3Browser = browser.Browser() # Accessing the resources data = i3Browser.recoverURL(uri) # Getting the list of <RegExp> objects from entify lRegexp = regexp_selection.getRegexpsByName([platform]) newEntities = entify.getEntitiesByRegexp(data=data, listRegexp=lRegexp) # This returns a dictionary like the following: """ [{ 'attributes': [], 'type': 'i3visio.sha256', 'value': 'a9b8c5d848205db514d4097d2b78f4528d01a79f39601e0f9c5c40ed689471' }, { 'attributes': [], 'type': 'i3visio.sha256', 'value': 'b28b896e6eeb8d651cacd5f4a4d1490fbe9d05dbc92221609350b0ce7a68e9' }, { 'attributes': [], 'type': 'i3visio.sha256', 'value': 'd727fed4d969b14b28165c75ad12d7dddd56c0198fa70cedc3fdad7ac395b2' }, { 'attributes': [], 'type': 'i3visio.sha256', 'value': '3e9a2204fcfc6f7dde250e61ca35353411880024102cba14a0bd45f05f1e74' }] """ # Adding list of entities to be displayed me.addListOfEntities(newEntities) #print json.dumps(entities, indent=2) #for elem in entities: # newEnt = me.addEntity(elem["type"],elem["value"]) # newEnt.setDisplayInformation("<h3>" + elem["value"] +"</h3><p>"+str(elem["attributes"])+"</p>") # for extraAtt in elem["attributes"]: # newEnt.addAdditionalFields(str(extraAtt['type']), str(extraAtt['type']), True, str(extraAtt['value'])) # Returning the output text... me.returnOutput()