def feeder(message, count=0): if flag_the_hive or flag_misp: tag, path = message.split(';') ## FIXME: remove it if PASTES_FOLDER not in path: path = os.path.join(PASTES_FOLDER, path) try: paste = Paste.Paste(path) except FileNotFoundError: if count < 10: r_serv_db.zincrby('mess_not_saved_export', message, 1) return 0 else: r_serv_db.zrem('mess_not_saved_export', message) print('Error: {} do not exist, tag= {}'.format(path, tag)) return 0 source = '/'.join(paste.p_path.split('/')[-6:]) if HiveApi != False: if int(r_serv_db.get('hive:auto-alerts')) == 1: whitelist_hive = r_serv_db.scard('whitelist_hive') if r_serv_db.sismember('whitelist_hive', tag): create_the_hive_alert(source, path, tag) else: print('hive, auto alerts creation disable') if flag_misp: if int(r_serv_db.get('misp:auto-events')) == 1: if r_serv_db.sismember('whitelist_misp', tag): misp_wrapper.pushToMISP(uuid_ail, path, tag) else: print('misp, auto events creation disable')
def compute(self, message): try: PST = Paste.Paste(message) docpath = message.split(" ", -1)[-1] paste = PST.get_p_content() self.redis_logger.debug(f"Indexing - {self.indexname}: {docpath}") # Avoid calculating the index's size at each message if (time.time() - self.last_refresh > self.TIME_WAIT): self.last_refresh = time.time() if self.check_index_size() >= self.INDEX_SIZE_THRESHOLD * ( 1000 * 1000): timestamp = int(time.time()) self.redis_logger.debug(f"Creating new index {timestamp}") self.indexpath = join(self.baseindexpath, str(timestamp)) self.indexname = str(timestamp) # update all_index with open(self.indexRegister_path, "a") as f: f.write('\n' + str(timestamp)) # create new dir os.mkdir(self.indexpath) self.ix = create_in(self.indexpath, self.schema) if self.indexertype == "whoosh": indexwriter = self.ix.writer() indexwriter.update_document(title=docpath, path=docpath, content=paste) indexwriter.commit() except IOError: self.redis_logger.debug(f"CRC Checksum Failed on: {PST.p_path}") self.redis_logger.error( 'Duplicate;{};{};{};CRC Checksum Failed'.format( PST.p_source, PST.p_date, PST.p_name))
def compute(self, message): paste = Paste.Paste(message) content = paste.get_p_content() # List of the regex results in the Paste, may be null results = self.REG_PHONE.findall(content) # If the list is greater than 4, we consider the Paste may contain a list of phone numbers if len(results) > 4: self.redis_logger.debug(results) self.redis_logger.warning('{} contains PID (phone numbers)'.format( paste.p_name)) msg = 'infoleak:automatic-detection="phone-number";{}'.format( message) self.process.populate_set_out(msg, 'Tags') # Send to duplicate self.process.populate_set_out(message, 'Duplicate') stats = {} for phone_number in results: try: x = phonenumbers.parse(phone_number, None) country_code = x.country_code if stats.get(country_code) is None: stats[country_code] = 1 else: stats[country_code] = stats[country_code] + 1 except: pass for country_code in stats: if stats[country_code] > 4: self.redis_logger.warning( '{} contains Phone numbers with country code {}'. format(paste.p_name, country_code))
def analyse(url, path): if is_sql_injection(url): faup.decode(url) url_parsed = faup.get() paste = Paste.Paste(path) print("Detected SQL in URL: ") print(urllib.request.unquote(url)) to_print = 'SQLInjection;{};{};{};{};{}'.format(paste.p_source, paste.p_date, paste.p_name, "Detected SQL in URL", paste.p_rel_path) publisher.warning(to_print) #Send to duplicate p.populate_set_out(path, 'Duplicate') msg = 'infoleak:automatic-detection="sql-injection";{}'.format(path) p.populate_set_out(msg, 'Tags') #statistics tld = url_parsed['tld'] if tld is not None: ## TODO: # FIXME: remove me try: tld = tld.decode() except: pass date = datetime.datetime.now().strftime("%Y%m") server_statistics.hincrby('SQLInjection_by_tld:'+date, tld, 1)
def search_phone(message): paste = Paste.Paste(message) content = paste.get_p_content() # regex to find phone numbers, may raise many false positives (shalt thou seek optimization, upgrading is required) reg_phone = re.compile(r'(\+\d{1,4}(\(\d\))?\d?|0\d?)(\d{6,8}|([-/\. ]{1}\d{2,3}){3,4})') reg_phone = re.compile(r'(\+\d{1,4}(\(\d\))?\d?|0\d?)(\d{6,8}|([-/\. ]{1}\(?\d{2,4}\)?){3,4})') # list of the regex results in the Paste, may be null results = reg_phone.findall(content) # if the list is greater than 4, we consider the Paste may contain a list of phone numbers if len(results) > 4: print results publisher.warning('{} contains PID (phone numbers)'.format(paste.p_name)) #send to Browse_warning_paste p.populate_set_out('phone;{}'.format(message), 'BrowseWarningPaste') #Send to duplicate p.populate_set_out(message, 'Duplicate') stats = {} for phone_number in results: try: x = phonenumbers.parse(phone_number, None) country_code = x.country_code if stats.get(country_code) is None: stats[country_code] = 1 else: stats[country_code] = stats[country_code] + 1 except: pass for country_code in stats: if stats[country_code] > 4: publisher.warning('{} contains Phone numbers with country code {}'.format(paste.p_name, country_code))
def add_new_object(self, uuid_ail, path, p_source, tag): self.uuid_ail = uuid_ail self.path = path self.p_source = p_source self.paste = Paste.Paste(path) self.p_date = self.date_to_str(self.paste.p_date) self.p_content = self.paste.get_p_content() self.p_tag = tag temp = self.paste._get_p_duplicate() #beautifier if not temp: temp = '' p_duplicate_number = len(temp) if len(temp) >= 0 else 0 to_ret = "" for dup in temp[:10]: dup = dup.replace('\'', '\"').replace('(', '[').replace(')', ']') dup = json.loads(dup) algo = dup[0] path = dup[1].split('/')[-6:] path = '/'.join(path)[:-3] # -3 removes .gz if algo == 'tlsh': perc = 100 - int(dup[2]) else: perc = dup[2] to_ret += "{}: {} [{}%]\n".format(path, algo, perc) p_duplicate = to_ret self.mispObject = AilLeakObject(self.uuid_ail, self.p_source, self.p_date, self.p_content, p_duplicate, p_duplicate_number)
def _setValue(self): try: #Verify that the module have a paste if COMPLETE_PASTE_PATH_PER_PID[current_selected_value] is None: self.label_list[0]._text = "No paste for this module" for i in range(1, self.num_label): self.label_list[i]._text = "" return paste = Paste.Paste( COMPLETE_PASTE_PATH_PER_PID[current_selected_value]) old_content = paste.get_p_content()[ 0:4000] # Limit number of char to be displayed #Replace unprintable char by ? content = "" for i, c in enumerate(old_content): if ord(c) > 127: # Used to avoid printing unprintable char content += '?' elif c == "\t": # Replace tab by 4 spaces content += " " else: content += c #Print in the correct label, END or more to_print = "" i = 0 for line in content.split("\n"): if i > self.num_label - 2: break self.label_list[i]._text = str(i) + ". " + line.replace( "\r", "") i += 1 if i > self.num_label - 2: self.label_list[i]._text = "- ALL PASTE NOT DISPLAYED -" i += 1 else: self.label_list[i]._text = "- END of PASTE -" i += 1 while i < self.num_label: #Clear out remaining lines self.label_list[i]._text = "" i += 1 except OSError as e: self.label_list[ 0]._text = "Error during parsing the filepath. Please, check manually" self.label_list[1]._text = COMPLETE_PASTE_PATH_PER_PID[ current_selected_value] for i in range(2, self.num_label): self.label_list[i]._text = "" except Exception as e: self.label_list[ 0]._text = "Error while displaying the paste: " + COMPLETE_PASTE_PATH_PER_PID[ current_selected_value] self.label_list[1]._text = str(e) for i in range(2, self.num_label): self.label_list[i]._text = ""
def search_game(message): #We recover the paste paste = Paste.Paste(message) content = paste.get_p_content() #We open the file with all game word and the stock for all paste found filetoopen = open("corpus.txt", "r") filetowrite = open("stock.txt", "a") count = 0 #Number of game word found in 1 file for line in filetoopen: linestrip = line.strip( ) #Must do because it takes all the line and not just the word reg = re.compile(r'{}'.format(linestrip)) #we create the regex results = re.findall(reg, content) #we find the occurences if (len(results) > 0): count = count + 1 re.purge() if count > 5: print results publisher.warning( '{} contains Game related conversations+{} occurences of a game related word ' .format(paste.p_name, count)) #warning for the logs filetowrite.write( '{} contains Game related conversations+{} occurences of a game related word \n' .format(paste.p_name, count)) #For stock.txt to_print = 'GameConv;{};{};{};{} Terms related;{}'.format( paste.p_source, paste.p_date, paste.p_name, count, paste.p_path) #To see on the webinterface publisher.warning(to_print) filetoopen.close() filetowrite.close()
def search_api_key(message): filename, score = message.split() paste = Paste.Paste(filename) content = paste.get_p_content() aws_access_key = regex_aws_access_key.findall(content) aws_secret_key = regex_aws_secret_key.findall(content) google_api_key = regex_google_api_key.findall(content) if (len(aws_access_key) > 0 or len(aws_secret_key) > 0 or len(google_api_key) > 0): to_print = 'ApiKey;{};{};{};'.format(paste.p_source, paste.p_date, paste.p_name) if (len(google_api_key) > 0): print('found google api key') print(to_print) publisher.warning('{}Checked {} found Google API Key;{}'.format( to_print, len(google_api_key), paste.p_path)) if (len(aws_access_key) > 0 or len(aws_secret_key) > 0): print('found AWS key') print(to_print) total = len(aws_access_key) + len(aws_secret_key) publisher.warning('{}Checked {} found AWS Key;{}'.format( to_print, total, paste.p_path)) msg = 'apikey;{}'.format(filename) p.populate_set_out(msg, 'alertHandler') #Send to duplicate p.populate_set_out(filename, 'Duplicate')
def search_key(message): paste = Paste.Paste(message) content = paste.get_p_content() find = False if '-----BEGIN PGP MESSAGE-----' in content: publisher.warning('{} has a PGP enc message'.format(paste.p_name)) find = True if '-----BEGIN CERTIFICATE-----' in content: publisher.warning('{} has a certificate message'.format(paste.p_name)) find = True if '-----BEGIN RSA PRIVATE KEY-----' in content: publisher.warning('{} has a RSA key message'.format(paste.p_name)) find = True if '-----BEGIN PRIVATE KEY-----' in content: publisher.warning('{} has a private message'.format(paste.p_name)) find = True if '-----BEGIN ENCRYPTED PRIVATE KEY-----' in content: publisher.warning('{} has an encrypted private message'.format( paste.p_name)) find = True if find: #Send to duplicate p.populate_set_out(message, 'Duplicate') #send to Browse_warning_paste p.populate_set_out('keys;{}'.format(message), 'alertHandler')
def analyse(url, path): faup.decode(url) url_parsed = faup.get() resource_path = url_parsed['resource_path'] query_string = url_parsed['query_string'] result_path = 0 result_query = 0 if resource_path is not None: result_path = is_sql_injection(resource_path) if query_string is not None: result_query = is_sql_injection(query_string) if (result_path > 0) or (result_query > 0): paste = Paste.Paste(path) if (result_path > 1) or (result_query > 1): print "Detected SQL in URL: " print urllib2.unquote(url) to_print = 'SQLInjection;{};{};{};{};{}'.format(paste.p_source, paste.p_date, paste.p_name, "Detected SQL in URL", paste.p_path) publisher.warning(to_print) #Send to duplicate p.populate_set_out(path, 'Duplicate') #send to Browse_warning_paste p.populate_set_out('sqlinjection;{}'.format(path), 'BrowseWarningPaste') else: print "Potential SQL injection:" print urllib2.unquote(url) to_print = 'SQLInjection;{};{};{};{}'.format(paste.p_source, paste.p_date, paste.p_name, "Potential SQL injection") publisher.info(to_print)
def compute(self, message): # Cast message as paste paste = Paste.Paste(message) # Get paste content content = paste.get_p_content() # init categories found is_categ_found = False # Search for pattern categories in paste content for categ, pattern in self.categ_items: found = set(re.findall(pattern, content)) lenfound = len(found) if lenfound >= self.matchingThreshold: is_categ_found = True msg = '{} {}'.format(paste.p_rel_path, lenfound) self.redis_logger.debug('%s;%s %s' % (self.module_name, msg, categ)) # Export message to categ queue self.process.populate_set_out(msg, categ) self.redis_logger.info( 'Categ;{};{};{};Detected {} as {};{}'.format( paste.p_source, paste.p_date, paste.p_name, lenfound, categ, paste.p_rel_path)) if not is_categ_found: self.redis_logger.debug('No %s found in this paste: %s' % (self.module_name, paste.p_name))
def analyse(url, path): faup.decode(url) url_parsed = faup.get() pprint.pprint(url_parsed) resource_path = url_parsed['resource_path'] query_string = url_parsed['query_string'] result_path = {'sqli': False} result_query = {'sqli': False} if resource_path is not None: result_path = pylibinjection.detect_sqli(resource_path) print("path is sqli : {0}".format(result_path)) if query_string is not None: result_query = pylibinjection.detect_sqli(query_string) print("query is sqli : {0}".format(result_query)) if result_path['sqli'] is True or result_query['sqli'] is True: paste = Paste.Paste(path) print("Detected (libinjection) SQL in URL: ") print(urllib.request.unquote(url)) to_print = 'LibInjection;{};{};{};{};{}'.format( paste.p_source, paste.p_date, paste.p_name, "Detected SQL in URL", paste.p_path) publisher.warning(to_print) #Send to duplicate p.populate_set_out(path, 'Duplicate') #send to Browse_warning_paste p.populate_set_out('sqlinjection;{}'.format(path), 'alertHandler') msg = 'infoleak:automatic-detection="sql-injection";{}'.format(path) p.populate_set_out(msg, 'Tags')
def search_ip(message): paste = Paste.Paste(message) content = paste.get_p_content() # regex to find IPs reg_ip = re.compile( r'^(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)', flags=re.MULTILINE) # list of the regex results in the Paste, may be null results = reg_ip.findall(content) matching_ips = [] for res in results: address = IPv4Address(res) for network in ip_networks: if address in network: matching_ips.append(address) if len(matching_ips) > 0: print('{} contains {} IPs'.format(paste.p_name, len(matching_ips))) publisher.warning('{} contains {} IPs'.format(paste.p_name, len(matching_ips))) #Tag message with IP msg = 'infoleak:automatic-detection="ip";{}'.format(message) p.populate_set_out(msg, 'Tags') #Send to duplicate p.populate_set_out(message, 'Duplicate')
def add_new_object(self, moduleName, path): self.moduleName = moduleName self.path = path self.paste = Paste.Paste(path) self.p_date = self.date_to_str(self.paste.p_date) self.p_source = self.paste.p_path self.p_content = self.paste.get_p_content().decode('utf8') temp = self.paste._get_p_duplicate() try: temp = temp.decode('utf8') except AttributeError: pass #beautifier temp = json.loads(temp) self.p_duplicate_number = len(temp) if len(temp) >= 0 else 0 to_ret = "" for dup in temp[:self.maxDuplicateToPushToMISP]: algo = dup[0] path = dup[1].split('/')[-5:] path = '/'.join(path)[:-3] # -3 removes .gz perc = dup[2] to_ret += "{}: {} [{}%]\n".format(path, algo, perc) self.p_duplicate = to_ret self.mispObject = AilleakObject(self.moduleName, self.p_source, self.p_date, self.p_content, self.p_duplicate, self.p_duplicate_number)
def search_gpg(message): paste = Paste.Paste(message) content = paste.get_p_content() if '-----BEGIN PGP MESSAGE-----' in content: publisher.warning('{} has a PGP enc message'.format(paste.p_name)) #Send to duplicate p.populate_set_out(message, 'Duplicate') #send to Browse_warning_paste p.populate_set_out('keys;{}'.format(message), 'BrowseWarningPaste')
def analyse(url, path): faup.decode(url) url_parsed = faup.get() resource_path = url_parsed['resource_path'] query_string = url_parsed['query_string'] result_path = 0 result_query = 0 if resource_path is not None: ## TODO: # FIXME: remove me try: resource_path = resource_path.decode() except: pass result_path = is_sql_injection(resource_path) if query_string is not None: ## TODO: # FIXME: remove me try: query_string = query_string.decode() except: pass result_query = is_sql_injection(query_string) if (result_path > 0) or (result_query > 0): paste = Paste.Paste(path) if (result_path > 1) or (result_query > 1): print("Detected SQL in URL: ") print(urllib.request.unquote(url)) to_print = 'SQLInjection;{};{};{};{};{}'.format(paste.p_source, paste.p_date, paste.p_name, "Detected SQL in URL", paste.p_rel_path) publisher.warning(to_print) #Send to duplicate p.populate_set_out(path, 'Duplicate') msg = 'infoleak:automatic-detection="sql-injection";{}'.format(path) p.populate_set_out(msg, 'Tags') #statistics tld = url_parsed['tld'] if tld is not None: ## TODO: # FIXME: remove me try: tld = tld.decode() except: pass date = datetime.datetime.now().strftime("%Y%m") server_statistics.hincrby('SQLInjection_by_tld:'+date, tld, 1) else: print("Potential SQL injection:") print(urllib.request.unquote(url)) to_print = 'SQLInjection;{};{};{};{};{}'.format(paste.p_source, paste.p_date, paste.p_name, "Potential SQL injection", paste.p_rel_path) publisher.info(to_print)
def compute_provider_info(server_trend, server_pasteName, path): redis_all_provider = 'all_provider_set' paste = Paste.Paste(path) paste_baseName = paste.p_name.split('.')[0] paste_size = paste._get_p_size() paste_provider = paste.p_source paste_date = str(paste._get_p_date()) redis_sum_size_set = 'top_size_set_' + paste_date redis_avg_size_name_set = 'top_avg_size_set_' + paste_date redis_providers_name_set = 'providers_set_' + paste_date # Add/Update in Redis server_pasteName.sadd(paste_baseName, path) server_trend.sadd(redis_all_provider, paste_provider) num_paste = int(server_trend.hincrby(paste_provider+'_num', paste_date, 1)) sum_size = float(server_trend.hincrbyfloat(paste_provider+'_size', paste_date, paste_size)) new_avg = float(sum_size) / float(num_paste) server_trend.hset(paste_provider +'_avg', paste_date, new_avg) # # Compute Most Posted # # Size if server_trend.zcard(redis_sum_size_set) < max_set_cardinality or server_trend.zscore(redis_sum_size_set, paste_provider) != "nil": server_trend.zadd(redis_sum_size_set, float(num_paste), paste_provider) server_trend.zadd(redis_avg_size_name_set, float(new_avg), paste_provider) else: #set full capacity member_set = server_trend.zrangebyscore(redis_sum_size_set, '-inf', '+inf', withscores=True, start=0, num=1) # Member set is a list of (value, score) pairs if float(member_set[0][1]) < new_avg: #remove min from set and add the new one print 'Size - adding ' +paste_provider+ '(' +str(new_avg)+') in set and removing '+member_set[0][0]+'('+str(member_set[0][1])+')' server_trend.zrem(redis_sum_size_set, member_set[0][0]) server_trend.zadd(redis_sum_size_set, float(sum_size), paste_provider) server_trend.zrem(redis_avg_size_name_set, member_set[0][0]) server_trend.zadd(redis_avg_size_name_set, float(new_avg), paste_provider) # Num # if set not full or provider already present if server_trend.zcard(redis_providers_name_set) < max_set_cardinality or server_trend.zscore(redis_providers_name_set, paste_provider) != "nil": server_trend.zadd(redis_providers_name_set, float(num_paste), paste_provider) else: #set at full capacity member_set = server_trend.zrangebyscore(redis_providers_name_set, '-inf', '+inf', withscores=True, start=0, num=1) # Member set is a list of (value, score) pairs if int(member_set[0][1]) < num_paste: #remove min from set and add the new one print 'Num - adding ' +paste_provider+ '(' +str(num_paste)+') in set and removing '+member_set[0][0]+'('+str(member_set[0][1])+')' server_trend.zrem(member_set[0][0]) server_trend.zadd(redis_providers_name_set, float(num_paste), paste_provider)
def search_phone(message): paste = Paste.Paste(message) content = paste.get_p_content() # regex to find phone numbers, may raise many false positives (shalt thou seek optimization, upgrading is required) reg_phone = re.compile(r'(\+\d{1,4}(\(\d\))?\d?|0\d?)(\d{6,8}|([-/\. ]{1}\d{2,3}){3,4})') # list of the regex results in the Paste, may be null results = reg_phone.findall(content) # if the list is greater than 4, we consider the Paste may contain a list of phone numbers if len(results) > 4: print results publisher.warning('{} contains PID (phone numbers)'.format(paste.p_name))
def main(): publisher.port = 6380 publisher.channel = "Script" config_section = 'DomClassifier' p = Process(config_section) addr_dns = p.config.get("DomClassifier", "dns") publisher.info("""ZMQ DomainClassifier is Running""") c = DomainClassifier.domainclassifier.Extract(rawtext="", nameservers=[addr_dns]) cc = p.config.get("DomClassifier", "cc") cc_tld = p.config.get("DomClassifier", "cc_tld") while True: try: message = p.get_from_set() if message is not None: PST = Paste.Paste(message) else: publisher.debug("Script DomClassifier is idling 1s") time.sleep(1) continue paste = PST.get_p_content() mimetype = PST._get_p_encoding() if mimetype == "text/plain": c.text(rawtext=paste) c.potentialdomain() c.validdomain(rtype=['A'], extended=True) localizeddomains = c.include(expression=cc_tld) if localizeddomains: print(localizeddomains) publisher.warning( 'DomainC;{};{};{};Checked {} located in {};{}'.format( PST.p_source, PST.p_date, PST.p_name, localizeddomains, cc_tld, PST.p_path)) localizeddomains = c.localizedomain(cc=cc) if localizeddomains: print(localizeddomains) publisher.warning( 'DomainC;{};{};{};Checked {} located in {};{}'.format( PST.p_source, PST.p_date, PST.p_name, localizeddomains, cc, PST.p_path)) except IOError: print("CRC Checksum Failed on :", PST.p_path) publisher.error('Duplicate;{};{};{};CRC Checksum Failed'.format( PST.p_source, PST.p_date, PST.p_name))
def analyse(url, path): faup.decode(url) url_parsed = faup.get() pprint.pprint(url_parsed) ## TODO: # FIXME: remove me try: resource_path = url_parsed['resource_path'].encode() except: resource_path = url_parsed['resource_path'] ## TODO: # FIXME: remove me try: query_string = url_parsed['query_string'].encode() except: query_string = url_parsed['query_string'] result_path = {'sqli': False} result_query = {'sqli': False} if resource_path is not None: result_path = pylibinjection.detect_sqli(resource_path) print("path is sqli : {0}".format(result_path)) if query_string is not None: result_query = pylibinjection.detect_sqli(query_string) print("query is sqli : {0}".format(result_query)) if result_path['sqli'] is True or result_query['sqli'] is True: paste = Paste.Paste(path) print("Detected (libinjection) SQL in URL: ") print(urllib.request.unquote(url)) to_print = 'LibInjection;{};{};{};{};{}'.format( paste.p_source, paste.p_date, paste.p_name, "Detected SQL in URL", paste.p_rel_path) publisher.warning(to_print) #Send to duplicate p.populate_set_out(path, 'Duplicate') msg = 'infoleak:automatic-detection="sql-injection";{}'.format(path) p.populate_set_out(msg, 'Tags') #statistics ## TODO: # FIXME: remove me try: tld = url_parsed['tld'].decode() except: tld = url_parsed['tld'] if tld is not None: date = datetime.datetime.now().strftime("%Y%m") server_statistics.hincrby('SQLInjection_by_tld:' + date, tld, 1)
def search_cve(message): filepath, count = message.split() paste = Paste.Paste(filepath) content = paste.get_p_content() # regex to find CVE reg_cve = re.compile(r'(CVE-)[1-2]\d{1,4}-\d{1,5}') # list of the regex results in the Paste, may be null results = set(reg_cve.findall(content)) # if the list is greater than 2, we consider the Paste may contain a list of cve if len(results) > 0: print('{} contains CVEs'.format(paste.p_name)) publisher.warning('{} contains CVEs'.format(paste.p_name)) #send to Browse_warning_paste p.populate_set_out('cve;{}'.format(filepath), 'BrowseWarningPaste') #Send to duplicate p.populate_set_out(filepath, 'Duplicate')
def compute(self, message): filepath, count = message.split() paste = Paste.Paste(filepath) content = paste.get_p_content() # list of the regex results in the Paste, may be null results = set(self.reg_cve.findall(content)) # if the list is positive, we consider the Paste may contain a list of cve if len(results) > 0: warning = f'{paste.p_name} contains CVEs' print(warning) self.redis_logger.warning(warning) msg = f'infoleak:automatic-detection="cve";{filepath}' # Send to Tags Queue self.send_message_to_queue(msg, 'Tags') # Send to Duplicate Queue self.send_message_to_queue(filepath, 'Duplicate')
def search_phone(message): paste = Paste.Paste(message) content = paste.get_p_content() # regex to find phone numbers, may raise many false positives (shalt thou seek optimization, upgrading is required) reg_phone = re.compile(r'(\+\d{1,4}(\(\d\))?\d?|0\d?)(\d{6,8}|([-/\. ]{1}\d{2,3}){3,4})') # list of the regex results in the Paste, may be null results = reg_phone.findall(content) # if the list is greater than 4, we consider the Paste may contain a list of phone numbers if len(results) > 4 : print results publisher.warning('{} contains PID (phone numbers)'.format(paste.p_name)) if __name__ == '__main__': # If you wish to use an other port of channel, do not forget to run a subscriber accordingly (see launch_logs.sh) # Port of the redis instance used by pubsublogger publisher.port = 6380 # Script is the default channel used for the modules. publisher.channel = 'Script' # Section name in bin/packages/modules.cfg config_section = 'Phone' # Setup the I/O queues p = Process(config_section) # Sent to the logging a description of the module publisher.info("Run Phone module") # Endless loop getting messages from the input queue while True: # Get one message from the input queue message = p.get_from_set() if message is None: publisher.debug("{} queue is empty, waiting".format(config_section)) time.sleep(1) continue # Do something with the message from the queue search_phone(message)
def analyse(url, path): faup.decode(url) url_parsed = faup.get() resource_path = url_parsed['resource_path'] query_string = url_parsed['query_string'] result_path = 0 result_query = 0 if resource_path is not None: result_path = is_sql_injection(resource_path.decode('utf8')) if query_string is not None: result_query = is_sql_injection(query_string.decode('utf8')) if (result_path > 0) or (result_query > 0): paste = Paste.Paste(path) if (result_path > 1) or (result_query > 1): print("Detected SQL in URL: ") print(urllib.request.unquote(url)) to_print = 'SQLInjection;{};{};{};{};{}'.format( paste.p_source, paste.p_date, paste.p_name, "Detected SQL in URL", paste.p_path) publisher.warning(to_print) #Send to duplicate p.populate_set_out(path, 'Duplicate') #send to Browse_warning_paste p.populate_set_out('sqlinjection;{}'.format(path), 'alertHandler') msg = 'infoleak:automatic-detection="sql-injection";{}'.format( path) p.populate_set_out(msg, 'Tags') else: print("Potential SQL injection:") print(urllib.request.unquote(url)) to_print = 'SQLInjection;{};{};{};{};{}'.format( paste.p_source, paste.p_date, paste.p_name, "Potential SQL injection", paste.p_path) publisher.info(to_print)
def get_dict_cve(list_paste_cve, only_one_same_cve_by_paste=False): dict_keyword = {} for paste_cve in list_paste_cve: paste_content = Paste.Paste(paste_cve).get_p_content() cve_list = reg_cve.findall(paste_content) if only_one_same_cve_by_paste: cve_list = set(cve_list) for cve in reg_cve.findall(paste_content): try: dict_keyword[cve] += 1 except KeyError: dict_keyword[cve] = 1 print('------------------------------------------------') if dict_keyword: res = [(k, dict_keyword[k]) for k in sorted(dict_keyword, key=dict_keyword.get, reverse=True)] for item in res: pass print(item)
def search_geolocation(message): paste = Paste.Paste(message) content = paste.get_p_content() # Load Geoparser geo = Geoparser() geolocation = geo.geoparse(message) # regex ton find latitude and longitude reg_lat = re.compile(r'(\'lat\': \'([-\d.]+)\',)') reg_lon = re.compile(r'(\'lon\': \'([-\d.]+)\',)') #lat = set(reg_lat.findall(content)) #lon = set(reg_lat.findall(content)) lat = reg_lat.search(message).group(2) lon = reg_lon.search(message).group(2) print('latitude: {}'.format(lat)) print('longitude: {}'.format(lon)) print('{} text geolocation'.format(paste.p_name)) publisher.warning('{} contains geolocation'.format(paste.p_name))
cards = [ r'\b4\d{3}(?:[\ \-]?)\d{4}(?:[\ \-]?)\d{4}(?:[\ \-]?)\d{4}\b', # 16-digit VISA, with separators r'\b5[1-5]\d{2}(?:[\ \-]?)\d{4}(?:[\ \-]?)\d{4}(?:[\ \-]?)\d{4}\b', # 16 digits MasterCard r'\b6(?:011|22(?:(?=[\ \-]?(?:2[6-9]|[3-9]))|[2-8]|9(?=[\ \-]?(?:[01]|2[0-5])))|4[4-9]\d|5\d\d)(?:[\ \-]?)\d{4}(?:[\ \-]?)\d{4}(?:[\ \-]?)\d{4}\b', # Discover Card r'\b35(?:2[89]|[3-8]\d)(?:[\ \-]?)\d{4}(?:[\ \-]?)\d{4}(?:[\ \-]?)\d{4}\b', # Japan Credit Bureau (JCB) r'\b3[47]\d\d(?:[\ \-]?)\d{6}(?:[\ \-]?)\d{5}\b', # American Express r'\b(?:5[0678]\d\d|6304|6390|67\d\d)\d{8,15}\b', # Maestro ] regex = re.compile('|'.join(cards)) while True: message = p.get_from_set() if message is not None: filename, score = message.split() paste = Paste.Paste(filename) content = paste.get_p_content() all_cards = re.findall(regex, content) if len(all_cards) > 0: print 'All matching', all_cards creditcard_set = set([]) for card in all_cards: clean_card = re.sub('[^0-9]', '', card) if lib_refine.is_luhn_valid(clean_card): print clean_card, 'is valid' creditcard_set.add(clean_card) paste.__setattr__(channel, creditcard_set) paste.save_attribute_redis(channel, creditcard_set)
if __name__ == '__main__': # If you wish to use an other port of channel, do not forget to run a subscriber accordingly (see launch_logs.sh) # Port of the redis instance used by pubsublogger publisher.port = 6380 # Script is the default channel used for the modules. publisher.channel = 'Script' # Section name in bin/packages/modules.cfg config_section = 'Keys' # Setup the I/O queues p = Process(config_section) # Sent to the logging a description of the module publisher.info("Run Keys module ") # Endless loop getting messages from the input queue while True: # Get one message from the input queue message = p.get_from_set() if message is None: publisher.debug("{} queue is empty, waiting".format(config_section)) time.sleep(1) continue # Do something with the message from the queue paste = Paste.Paste(message) search_key(paste) # (Optional) Send that thing to the next queue
prec_filename = None # Thanks to Faup project for this regex # https://github.com/stricaud/faup url_regex = "((http|https|ftp)\://([a-zA-Z0-9\.\-]+(\:[a-zA-Z0-9\.&%\$\-]+)*@)*((25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9])\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[0-9])|localhost|([a-zA-Z0-9\-]+\.)*[a-zA-Z0-9\-]+\.onion)(\:[0-9]+)*(/($|[a-zA-Z0-9\.\,\?\'\\\+&%\$#\=~_\-]+))*)" while True: if message is not None: print message filename, score = message.split() # "For each new paste" if prec_filename is None or filename != prec_filename: domains_list = [] urls = [] PST = Paste.Paste(filename) for x in PST.get_regex(url_regex): # Extracting url with regex url, s, credential, subdomain, domain, host, port, \ resource_path, query_string, f1, f2, f3, f4 = x domains_list.append(domain) urls.append(url) # Saving the list of extracted onion domains. PST.__setattr__(channel, domains_list) PST.save_attribute_redis(channel, domains_list) to_print = 'Onion;{};{};{};'.format(PST.p_source, PST.p_date, PST.p_name) if len(domains_list) > 0: