def search_phone(message): paste = Paste.Paste(message) content = paste.get_p_content() # regex to find phone numbers, may raise many false positives (shalt thou seek optimization, upgrading is required) reg_phone = re.compile(r'(\+\d{1,4}(\(\d\))?\d?|0\d?)(\d{6,8}|([-/\. ]{1}\d{2,3}){3,4})') reg_phone = re.compile(r'(\+\d{1,4}(\(\d\))?\d?|0\d?)(\d{6,8}|([-/\. ]{1}\(?\d{2,4}\)?){3,4})') # list of the regex results in the Paste, may be null results = reg_phone.findall(content) # if the list is greater than 4, we consider the Paste may contain a list of phone numbers if len(results) > 4: print(results) publisher.warning('{} contains PID (phone numbers)'.format(paste.p_name)) msg = 'infoleak:automatic-detection="phone-number";{}'.format(message) p.populate_set_out(msg, 'Tags') #Send to duplicate p.populate_set_out(message, 'Duplicate') stats = {} for phone_number in results: try: x = phonenumbers.parse(phone_number, None) country_code = x.country_code if stats.get(country_code) is None: stats[country_code] = 1 else: stats[country_code] = stats[country_code] + 1 except: pass for country_code in stats: if stats[country_code] > 4: publisher.warning('{} contains Phone numbers with country code {}'.format(paste.p_name, country_code))
def analyse(url, path): faup.decode(url) url_parsed = faup.get() pprint.pprint(url_parsed) resource_path = url_parsed['resource_path'] query_string = url_parsed['query_string'] result_path = {'sqli': False} result_query = {'sqli': False} if resource_path is not None: result_path = pylibinjection.detect_sqli(resource_path) print("path is sqli : {0}".format(result_path)) if query_string is not None: result_query = pylibinjection.detect_sqli(query_string) print("query is sqli : {0}".format(result_query)) if result_path['sqli'] is True or result_query['sqli'] is True: paste = Paste.Paste(path) print("Detected (libinjection) SQL in URL: ") print(urllib.request.unquote(url)) to_print = 'LibInjection;{};{};{};{};{}'.format( paste.p_source, paste.p_date, paste.p_name, "Detected SQL in URL", paste.p_path) publisher.warning(to_print) #Send to duplicate p.populate_set_out(path, 'Duplicate') #send to Browse_warning_paste p.populate_set_out('sqlinjection;{}'.format(path), 'alertHandler') msg = 'infoleak:automatic-detection="sql-injection";{}'.format(path) p.populate_set_out(msg, 'Tags')
def check_all_iban(l_iban, paste, filename): nb_valid_iban = 0 for iban in l_iban: iban = iban[0] + iban[1] + iban[2] iban = ''.join(e for e in iban if e.isalnum()) #iban = iban.upper() res = iban_regex_verify.findall(iban) date = datetime.datetime.now().strftime("%Y%m") if res: print('checking ' + iban) if is_valid_iban(iban): print('------') nb_valid_iban = nb_valid_iban + 1 server_statistics.hincrby('iban_by_country:' + date, iban[0:2], 1) if (nb_valid_iban > 0): to_print = 'Iban;{};{};{};'.format(paste.p_source, paste.p_date, paste.p_name) publisher.warning('{}Checked found {} IBAN;{}'.format( to_print, nb_valid_iban, paste.p_rel_path)) msg = 'infoleak:automatic-detection="iban";{}'.format(filename) p.populate_set_out(msg, 'Tags') #Send to duplicate p.populate_set_out(filename, 'Duplicate')
def search_api_key(message): filename, score = message.split() paste = Paste.Paste(filename) content = paste.get_p_content() aws_access_key = regex_aws_access_key.findall(content) aws_secret_key = regex_aws_secret_key.findall(content) google_api_key = regex_google_api_key.findall(content) if (len(aws_access_key) > 0 or len(aws_secret_key) > 0 or len(google_api_key) > 0): to_print = 'ApiKey;{};{};{};'.format(paste.p_source, paste.p_date, paste.p_name) if (len(google_api_key) > 0): print('found google api key') print(to_print) publisher.warning('{}Checked {} found Google API Key;{}'.format( to_print, len(google_api_key), paste.p_path)) if (len(aws_access_key) > 0 or len(aws_secret_key) > 0): print('found AWS key') print(to_print) total = len(aws_access_key) + len(aws_secret_key) publisher.warning('{}Checked {} found AWS Key;{}'.format( to_print, total, paste.p_path)) msg = 'apikey;{}'.format(filename) p.populate_set_out(msg, 'alertHandler') #Send to duplicate p.populate_set_out(filename, 'Duplicate')
def search_game(message): #We recover the paste paste = Paste.Paste(message) content = paste.get_p_content() #We open the file with all game word and the stock for all paste found filetoopen = open("corpus.txt", "r") filetowrite = open("stock.txt", "a") count = 0 #Number of game word found in 1 file for line in filetoopen: linestrip = line.strip( ) #Must do because it takes all the line and not just the word reg = re.compile(r'{}'.format(linestrip)) #we create the regex results = re.findall(reg, content) #we find the occurences if (len(results) > 0): count = count + 1 re.purge() if count > 5: print results publisher.warning( '{} contains Game related conversations+{} occurences of a game related word ' .format(paste.p_name, count)) #warning for the logs filetowrite.write( '{} contains Game related conversations+{} occurences of a game related word \n' .format(paste.p_name, count)) #For stock.txt to_print = 'GameConv;{};{};{};{} Terms related;{}'.format( paste.p_source, paste.p_date, paste.p_name, count, paste.p_path) #To see on the webinterface publisher.warning(to_print) filetoopen.close() filetowrite.close()
def search_key(content, message, paste): bitcoin_address = re.findall(regex_bitcoin_public_address, content) bitcoin_private_key = re.findall(regex_bitcoin_private_key, content) validate_address = False key = False if(len(bitcoin_address) >0): #print(message) for address in bitcoin_address: if(check_bc(address)): validate_address = True print('Bitcoin address found : {}'.format(address)) if(len(bitcoin_private_key) > 0): for private_key in bitcoin_private_key: print('Bitcoin private key found : {}'.format(private_key)) key = True if(validate_address): p.populate_set_out(message, 'Duplicate') to_print = 'Bitcoin found: {} address and {} private Keys'.format(len(bitcoin_address), len(bitcoin_private_key)) print(to_print) publisher.warning(to_print) msg = ('bitcoin;{}'.format(message)) p.populate_set_out( msg, 'alertHandler') msg = 'infoleak:automatic-detection="bitcoin-address";{}'.format(message) p.populate_set_out(msg, 'Tags') if(key): msg = 'infoleak:automatic-detection="bitcoin-private-key";{}'.format(message) p.populate_set_out(msg, 'Tags') to_print = 'Bitcoin;{};{};{};'.format(paste.p_source, paste.p_date, paste.p_name) publisher.warning('{}Detected {} Bitcoin private key;{}'.format( to_print, len(bitcoin_private_key),paste.p_path))
def main(): """Main Function""" # CONFIG # cfg = ConfigParser.ConfigParser() cfg.read(configfile) # REDIS # r_serv = redis.StrictRedis( host = cfg.get("Redis_Queues", "host"), port = cfg.getint("Redis_Queues", "port"), db = cfg.getint("Redis_Queues", "db")) # LOGGING # publisher.channel = "Queuing" # ZMQ # Sub = ZMQ_PubSub.ZMQSub(configfile,"PubSub_Categ", "onion_categ", "tor") # FUNCTIONS # publisher.info("""Suscribed to channel {0}""".format("onion_categ")) while True: Sub.get_and_lpush(r_serv) if r_serv.sismember("SHUTDOWN_FLAGS", "Onion_Q"): r_serv.srem("SHUTDOWN_FLAGS", "Onion_Q") print "Shutdown Flag Up: Terminating" publisher.warning("Shutdown Flag Up: Terminating.") break
def search_phone(message): paste = Paste.Paste(message) content = paste.get_p_content() # regex to find phone numbers, may raise many false positives (shalt thou seek optimization, upgrading is required) reg_phone = re.compile(r'(\+\d{1,4}(\(\d\))?\d?|0\d?)(\d{6,8}|([-/\. ]{1}\d{2,3}){3,4})') reg_phone = re.compile(r'(\+\d{1,4}(\(\d\))?\d?|0\d?)(\d{6,8}|([-/\. ]{1}\(?\d{2,4}\)?){3,4})') # list of the regex results in the Paste, may be null results = reg_phone.findall(content) # if the list is greater than 4, we consider the Paste may contain a list of phone numbers if len(results) > 4: print results publisher.warning('{} contains PID (phone numbers)'.format(paste.p_name)) #send to Browse_warning_paste p.populate_set_out('phone;{}'.format(message), 'BrowseWarningPaste') #Send to duplicate p.populate_set_out(message, 'Duplicate') stats = {} for phone_number in results: try: x = phonenumbers.parse(phone_number, None) country_code = x.country_code if stats.get(country_code) is None: stats[country_code] = 1 else: stats[country_code] = stats[country_code] + 1 except: pass for country_code in stats: if stats[country_code] > 4: publisher.warning('{} contains Phone numbers with country code {}'.format(paste.p_name, country_code))
def set_out_paste(decoder_name, message): publisher.warning(decoder_name+' decoded') #Send to duplicate p.populate_set_out(message, 'Duplicate') msg = 'infoleak:automatic-detection="'+decoder_name+'";{}'.format(message) p.populate_set_out(msg, 'Tags')
def analyse(url, path): faup.decode(url) url_parsed = faup.get() resource_path = url_parsed['resource_path'] query_string = url_parsed['query_string'] result_path = 0 result_query = 0 if resource_path is not None: result_path = is_sql_injection(resource_path) if query_string is not None: result_query = is_sql_injection(query_string) if (result_path > 0) or (result_query > 0): paste = Paste.Paste(path) if (result_path > 1) or (result_query > 1): print "Detected SQL in URL: " print urllib2.unquote(url) to_print = 'SQLInjection;{};{};{};{};{}'.format(paste.p_source, paste.p_date, paste.p_name, "Detected SQL in URL", paste.p_path) publisher.warning(to_print) #Send to duplicate p.populate_set_out(path, 'Duplicate') #send to Browse_warning_paste p.populate_set_out('sqlinjection;{}'.format(path), 'BrowseWarningPaste') else: print "Potential SQL injection:" print urllib2.unquote(url) to_print = 'SQLInjection;{};{};{};{}'.format(paste.p_source, paste.p_date, paste.p_name, "Potential SQL injection") publisher.info(to_print)
def analyse(url, path): if is_sql_injection(url): faup.decode(url) url_parsed = faup.get() paste = Paste.Paste(path) print("Detected SQL in URL: ") print(urllib.request.unquote(url)) to_print = 'SQLInjection;{};{};{};{};{}'.format(paste.p_source, paste.p_date, paste.p_name, "Detected SQL in URL", paste.p_rel_path) publisher.warning(to_print) #Send to duplicate p.populate_set_out(path, 'Duplicate') msg = 'infoleak:automatic-detection="sql-injection";{}'.format(path) p.populate_set_out(msg, 'Tags') #statistics tld = url_parsed['tld'] if tld is not None: ## TODO: # FIXME: remove me try: tld = tld.decode() except: pass date = datetime.datetime.now().strftime("%Y%m") server_statistics.hincrby('SQLInjection_by_tld:'+date, tld, 1)
def search_base64(content, message): find = False base64_list = re.findall(regex_base64, content) if (len(base64_list) > 0): for b64 in base64_list: if len(b64) >= 40: decode = base64.b64decode(b64) type = magic.from_buffer(decode, mime=True) #print(type) #print(decode) find = True hash = sha1(decode).hexdigest() data = {} data['name'] = hash data['date'] = datetime.datetime.now().strftime("%d/%m/%y") data['origin'] = message data['estimated type'] = type json_data = json.dumps(data) save_base64_as_file(decode, type, hash, json_data) print('found {} '.format(type)) if (find): publisher.warning('base64 decoded') #Send to duplicate p.populate_set_out(message, 'Duplicate') #send to Browse_warning_paste msg = ('base64;{}'.format(message)) p.populate_set_out(msg, 'alertHandler')
def search_key(message): paste = Paste.Paste(message) content = paste.get_p_content() find = False if '-----BEGIN PGP MESSAGE-----' in content: publisher.warning('{} has a PGP enc message'.format(paste.p_name)) find = True if '-----BEGIN CERTIFICATE-----' in content: publisher.warning('{} has a certificate message'.format(paste.p_name)) find = True if '-----BEGIN RSA PRIVATE KEY-----' in content: publisher.warning('{} has a RSA key message'.format(paste.p_name)) find = True if '-----BEGIN PRIVATE KEY-----' in content: publisher.warning('{} has a private message'.format(paste.p_name)) find = True if '-----BEGIN ENCRYPTED PRIVATE KEY-----' in content: publisher.warning('{} has an encrypted private message'.format( paste.p_name)) find = True if find: #Send to duplicate p.populate_set_out(message, 'Duplicate') #send to Browse_warning_paste p.populate_set_out('keys;{}'.format(message), 'alertHandler')
def set_out_paste(decoder_name, message): publisher.warning(decoder_name+' decoded') #Send to duplicate p.populate_set_out(message, 'Duplicate') msg = 'infoleak:automatic-detection="'+decoder_name+'";{}'.format(message) p.populate_set_out(msg, 'Tags')
def search_ip(message): paste = Paste.Paste(message) content = paste.get_p_content() # regex to find IPs reg_ip = re.compile( r'^(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)', flags=re.MULTILINE) # list of the regex results in the Paste, may be null results = reg_ip.findall(content) matching_ips = [] for res in results: address = IPv4Address(res) for network in ip_networks: if address in network: matching_ips.append(address) if len(matching_ips) > 0: print('{} contains {} IPs'.format(paste.p_name, len(matching_ips))) publisher.warning('{} contains {} IPs'.format(paste.p_name, len(matching_ips))) #Tag message with IP msg = 'infoleak:automatic-detection="ip";{}'.format(message) p.populate_set_out(msg, 'Tags') #Send to duplicate p.populate_set_out(message, 'Duplicate')
def analyse(url, path): faup.decode(url) url_parsed = faup.get() resource_path = url_parsed['resource_path'] query_string = url_parsed['query_string'] result_path = 0 result_query = 0 if resource_path is not None: result_path = is_sql_injection(resource_path) if query_string is not None: result_query = is_sql_injection(query_string) if (result_path > 0) or (result_query > 0): paste = Paste.Paste(path) if (result_path > 1) or (result_query > 1): print "Detected SQL in URL: " print urllib2.unquote(url) to_print = 'SQLInjection;{};{};{};{}'.format(paste.p_source, paste.p_date, paste.p_name, "Detected SQL in URL") publisher.warning(to_print) #Send to duplicate p.populate_set_out(path, 'Duplicate') #send to Browse_warning_paste p.populate_set_out('sqlinjection;{}'.format(path), 'BrowseWarningPaste') else: print "Potential SQL injection:" print urllib2.unquote(url) to_print = 'SQLInjection;{};{};{};{}'.format(paste.p_source, paste.p_date, paste.p_name, "Potential SQL injection") publisher.info(to_print)
def search_key(content, message, paste): bitcoin_address = re.findall(regex_bitcoin_public_address, content) bitcoin_private_key = re.findall(regex_bitcoin_private_key, content) validate_address = False key = False if(len(bitcoin_address) >0): #print(message) for address in bitcoin_address: if(check_bc(address)): validate_address = True print('Bitcoin address found : {}'.format(address)) if(len(bitcoin_private_key) > 0): for private_key in bitcoin_private_key: print('Bitcoin private key found : {}'.format(private_key)) key = True if(validate_address): p.populate_set_out(message, 'Duplicate') to_print = 'Bitcoin found: {} address and {} private Keys'.format(len(bitcoin_address), len(bitcoin_private_key)) print(to_print) publisher.warning(to_print) msg = ('bitcoin;{}'.format(message)) p.populate_set_out( msg, 'alertHandler') msg = 'infoleak:automatic-detection="bitcoin-address";{}'.format(message) p.populate_set_out(msg, 'Tags') if(key): msg = 'infoleak:automatic-detection="bitcoin-private-key";{}'.format(message) p.populate_set_out(msg, 'Tags') to_print = 'Bitcoin;{};{};{};'.format(paste.p_source, paste.p_date, paste.p_name) publisher.warning('{}Detected {} Bitcoin private key;{}'.format( to_print, len(bitcoin_private_key),paste.p_path))
def db_import(filename, day): with open(filename, 'r') as f: entry = '' pipeline = routing_db.pipeline() i = 0 for line in f: # End of block, extracting the information if line == '\n': i += 1 parsed = re.findall('(?:ASPATH|PREFIX): ([^\n{]*)', entry) try: block = parsed[0].strip() # RIPE-NCC-RIS BGP IPv6 Anchor Prefix @RRC00 # RIPE-NCC-RIS BGP Anchor Prefix @ rrc00 - RIPE NCC if block in ['2001:7fb:ff00::/48', '84.205.80.0/24', '2001:7fb:fe00::/48', '84.205.64.0/24']: asn = 12654 else: asn = int(parsed[1].split()[-1].strip()) pipeline.hset(block, day, asn) except: #FIXME: check the cause of the exception publisher.warning(entry) entry = '' if i%10000 == 0: pipeline.execute() pipeline = routing_db.pipeline() else : # append the line to the current block. entry += line pipeline.execute() publisher.info('{f} finished, {nb} entries impported.'.\ format(f=filename, nb = i))
def main(): """Main Function""" # CONFIG # cfg = ConfigParser.ConfigParser() cfg.read(configfile) # REDIS # r_serv = redis.StrictRedis( host = cfg.get("Redis_Queues", "host"), port = cfg.getint("Redis_Queues", "port"), db = cfg.getint("Redis_Queues", "db")) # LOGGING # publisher.channel = "Queuing" # ZMQ # channel = cfg.get("PubSub_Words", "channel_0") subscriber_name = "curve" subscriber_config_section = "PubSub_Words" Sub = ZMQ_PubSub.ZMQSub(configfile, subscriber_config_section, channel, subscriber_name) # FUNCTIONS # publisher.info("""Suscribed to channel {0}""".format(channel)) while True: Sub.get_and_lpush(r_serv) if r_serv.sismember("SHUTDOWN_FLAGS", "Curve_Q"): r_serv.srem("SHUTDOWN_FLAGS", "Curve_Q") print "Shutdown Flag Up: Terminating" publisher.warning("Shutdown Flag Up: Terminating.") break
def set_out_item(decoder_name, item_id): publisher.warning(decoder_name + ' decoded') #Send to duplicate p.populate_set_out(item_id, 'Duplicate') msg = 'infoleak:automatic-detection="' + decoder_name + '";{}'.format( item_id) p.populate_set_out(msg, 'Tags')
def search_gpg(message): paste = Paste.Paste(message) content = paste.get_p_content() if '-----BEGIN PGP MESSAGE-----' in content: publisher.warning('{} has a PGP enc message'.format(paste.p_name)) #Send to duplicate p.populate_set_out(message, 'Duplicate') #send to Browse_warning_paste p.populate_set_out('keys;{}'.format(message), 'BrowseWarningPaste')
def search_gpg(message): paste = Paste.Paste(message) content = paste.get_p_content() if '-----BEGIN PGP MESSAGE-----' in content: publisher.warning('{} has a PGP enc message'.format(paste.p_name)) #Send to duplicate p.populate_set_out(message, 'Duplicate') #send to Browse_warning_paste p.populate_set_out('keys;{}'.format(message), 'BrowseWarningPaste')
def main(): """Main Function""" # CONFIG # cfg = ConfigParser.ConfigParser() cfg.read(configfile) # REDIS # r_serv = redis.StrictRedis( host = cfg.get("Redis_Queues", "host"), port = cfg.getint("Redis_Queues", "port"), db = cfg.getint("Redis_Queues", "db")) # LOGGING # publisher.channel = "Script" # ZMQ # channel = cfg.get("PubSub_Longlines", "channel_1") subscriber_name = "tokenize" subscriber_config_section = "PubSub_Longlines" #Publisher publisher_config_section = "PubSub_Words" publisher_name = "pubtokenize" Sub = ZMQ_PubSub.ZMQSub(configfile, subscriber_config_section, channel, subscriber_name) Pub = ZMQ_PubSub.ZMQPub(configfile, publisher_config_section, publisher_name) channel_0 = cfg.get("PubSub_Words", "channel_0") # FUNCTIONS # publisher.info("Tokeniser subscribed to channel {0}".format(cfg.get("PubSub_Longlines", "channel_1"))) while True: message = Sub.get_msg_from_queue(r_serv) print message if message != None: PST = P.Paste(message.split(" ",-1)[-1]) else: if r_serv.sismember("SHUTDOWN_FLAGS", "Tokenize"): r_serv.srem("SHUTDOWN_FLAGS", "Tokenize") print "Shutdown Flag Up: Terminating" publisher.warning("Shutdown Flag Up: Terminating.") break publisher.debug("Tokeniser is idling 10s") time.sleep(10) print "sleepin" continue for word, score in PST._get_top_words().items(): if len(word) >= 4: msg = channel_0+' '+PST.p_path+' '+str(word)+' '+str(score) Pub.send_message(msg) print msg else: pass
def redis_interbargraph_set(r_serv, year, month, overwrite): """Create a Redis sorted set. :param r_serv: -- connexion to redis database :param year: -- (integer) The year to process :param month: -- (integer) The month to process :param overwrite: -- (bool) trigger the overwrite mode This function create inside redis the intersection of all days in a month two by two. Example: For a month of 31days it will create 30 sorted set between day and day+1 until the last day. The overwrite mode delete the intersets and re-create them. """ a = date(year, month, 01) b = date(year, month, cal.monthrange(year, month)[1]) if overwrite: r_serv.delete("InterSet") for dt in rrule(DAILY, dtstart = a, until = b - timedelta(1)): dayafter = dt+timedelta(1) r_serv.delete(str(dt.strftime("%Y%m%d"))+str(dayafter.strftime("%Y%m%d"))) r_serv.zinterstore( str(dt.strftime("%Y%m%d"))+str(dayafter.strftime("%Y%m%d")), {str(dt.strftime("%Y%m%d")):1, str(dayafter.strftime("%Y%m%d")):-1}) r_serv.zadd( "InterSet", 1, str(dt.strftime("%Y%m%d"))+str(dayafter.strftime("%Y%m%d"))) else: for dt in rrule(DAILY, dtstart = a, until = b - timedelta(1)): dayafter = dt+timedelta(1) if r_serv.zcard(str(dt.strftime("%Y%m%d"))+str(dayafter.strftime("%Y%m%d"))) == 0: r_serv.zinterstore( str(dt.strftime("%Y%m%d"))+str(dayafter.strftime("%Y%m%d")), {str(dt.strftime("%Y%m%d")):1, str(dayafter.strftime("%Y%m%d")):-1}) r_serv.zadd( "InterSet", 1, str(dt.strftime("%Y%m%d"))+str(dayafter.strftime("%Y%m%d"))) publisher.info(str(dt.strftime("%Y%m%d"))+str(dayafter.strftime("%Y%m%d"))+" Intersection Created") else: publisher.warning("Data already exist, operation aborted.")
def analyse(url, path): faup.decode(url) url_parsed = faup.get() resource_path = url_parsed['resource_path'] query_string = url_parsed['query_string'] result_path = 0 result_query = 0 if resource_path is not None: ## TODO: # FIXME: remove me try: resource_path = resource_path.decode() except: pass result_path = is_sql_injection(resource_path) if query_string is not None: ## TODO: # FIXME: remove me try: query_string = query_string.decode() except: pass result_query = is_sql_injection(query_string) if (result_path > 0) or (result_query > 0): paste = Paste.Paste(path) if (result_path > 1) or (result_query > 1): print("Detected SQL in URL: ") print(urllib.request.unquote(url)) to_print = 'SQLInjection;{};{};{};{};{}'.format(paste.p_source, paste.p_date, paste.p_name, "Detected SQL in URL", paste.p_rel_path) publisher.warning(to_print) #Send to duplicate p.populate_set_out(path, 'Duplicate') msg = 'infoleak:automatic-detection="sql-injection";{}'.format(path) p.populate_set_out(msg, 'Tags') #statistics tld = url_parsed['tld'] if tld is not None: ## TODO: # FIXME: remove me try: tld = tld.decode() except: pass date = datetime.datetime.now().strftime("%Y%m") server_statistics.hincrby('SQLInjection_by_tld:'+date, tld, 1) else: print("Potential SQL injection:") print(urllib.request.unquote(url)) to_print = 'SQLInjection;{};{};{};{};{}'.format(paste.p_source, paste.p_date, paste.p_name, "Potential SQL injection", paste.p_rel_path) publisher.info(to_print)
def __query_logging(ip, user_agent, method, q_ip=None, announce_date=None, days_limit=None, level=None): if level == 'warning': publisher.warning(__csv2string([ip, user_agent, method, q_ip, announce_date, days_limit, level])) elif level == 'error': publisher.error(__csv2string([ip, user_agent, method, q_ip, announce_date, days_limit, level])) else: publisher.info(__csv2string([ip, user_agent, method, q_ip, announce_date, days_limit, level]))
def __query_logging(ip, user_agent, method, q_ip=None, announce_date=None, days_limit=None, level=None): if level == 'warning': publisher.warning(__csv2string([ip, user_agent, method, q_ip, announce_date, days_limit, level])) elif level == 'error': publisher.error(__csv2string([ip, user_agent, method, q_ip, announce_date, days_limit, level])) else: publisher.info(__csv2string([ip, user_agent, method, q_ip, announce_date, days_limit, level]))
def search_phone(message): paste = Paste.Paste(message) content = paste.get_p_content() # regex to find phone numbers, may raise many false positives (shalt thou seek optimization, upgrading is required) reg_phone = re.compile(r"(\+\d{1,4}(\(\d\))?\d?|0\d?)(\d{6,8}|([-/\. ]{1}\d{2,3}){3,4})") # list of the regex results in the Paste, may be null results = reg_phone.findall(content) # if the list is greater than 4, we consider the Paste may contain a list of phone numbers if len(results) > 4: print results publisher.warning("{} contains PID (phone numbers)".format(paste.p_name))
def search_phone(message): paste = Paste.Paste(message) content = paste.get_p_content() # regex to find phone numbers, may raise many false positives (shalt thou seek optimization, upgrading is required) reg_phone = re.compile(r'(\+\d{1,4}(\(\d\))?\d?|0\d?)(\d{6,8}|([-/\. ]{1}\d{2,3}){3,4})') # list of the regex results in the Paste, may be null results = reg_phone.findall(content) # if the list is greater than 4, we consider the Paste may contain a list of phone numbers if len(results) > 4: print results publisher.warning('{} contains PID (phone numbers)'.format(paste.p_name))
def sendEmailNotification(recipient, alert_name, content): if not os.path.exists(configfile): raise Exception('Unable to find the configuration file. \ Did you set environment variables? \ Or activate the virtualenv?') cfg = configparser.ConfigParser() cfg.read(configfile) sender = cfg.get("Notifications", "sender") sender_host = cfg.get("Notifications", "sender_host") sender_port = cfg.getint("Notifications", "sender_port") sender_pw = cfg.get("Notifications", "sender_pw") if sender_pw == 'None': sender_pw = None # raise an exception if any of these is None if (sender is None or sender_host is None or sender_port is None): raise Exception( 'SMTP configuration (host, port, sender) is missing or incomplete!' ) try: if sender_pw is not None: try: smtp_server = smtplib.SMTP(sender_host, sender_port) smtp_server.starttls() except smtplib.SMTPNotSupportedError: print("The server does not support the STARTTLS extension.") smtp_server = smtplib.SMTP_SSL(sender_host, sender_port) smtp_server.ehlo() smtp_server.login(sender, sender_pw) else: smtp_server = smtplib.SMTP(sender_host, sender_port) mime_msg = MIMEMultipart() mime_msg['From'] = sender mime_msg['To'] = recipient mime_msg['Subject'] = "AIL Framework " + alert_name + " Alert" body = content mime_msg.attach(MIMEText(body, 'plain')) smtp_server.sendmail(sender, recipient, mime_msg.as_string()) smtp_server.quit() print('Send notification ' + alert_name + ' to ' + recipient) except Exception as err: traceback.print_tb(err.__traceback__) publisher.warning(err)
def main(): publisher.port = 6380 publisher.channel = "Script" config_section = 'DomClassifier' p = Process(config_section) addr_dns = p.config.get("DomClassifier", "dns") publisher.info("""ZMQ DomainClassifier is Running""") c = DomainClassifier.domainclassifier.Extract(rawtext="", nameservers=[addr_dns]) cc = p.config.get("DomClassifier", "cc") cc_tld = p.config.get("DomClassifier", "cc_tld") while True: try: message = p.get_from_set() if message is not None: PST = Paste.Paste(message) else: publisher.debug("Script DomClassifier is idling 1s") time.sleep(1) continue paste = PST.get_p_content() mimetype = PST._get_p_encoding() if mimetype == "text/plain": c.text(rawtext=paste) c.potentialdomain() c.validdomain(rtype=['A'], extended=True) localizeddomains = c.include(expression=cc_tld) if localizeddomains: print(localizeddomains) publisher.warning( 'DomainC;{};{};{};Checked {} located in {};{}'.format( PST.p_source, PST.p_date, PST.p_name, localizeddomains, cc_tld, PST.p_path)) localizeddomains = c.localizedomain(cc=cc) if localizeddomains: print(localizeddomains) publisher.warning( 'DomainC;{};{};{};Checked {} located in {};{}'.format( PST.p_source, PST.p_date, PST.p_name, localizeddomains, cc, PST.p_path)) except IOError: print("CRC Checksum Failed on :", PST.p_path) publisher.error('Duplicate;{};{};{};CRC Checksum Failed'.format( PST.p_source, PST.p_date, PST.p_name))
def test_publisher(self): for i in range(0, 21): if i % 2 == 0: publisher.info('test' + str(i)) elif i % 3 == 0: publisher.warning('test' + str(i)) elif i % 5 == 0: publisher.error('test' + str(i)) elif i % 7 == 0: publisher.critical('test' + str(i)) else: publisher.debug('test' + str(i)) time.sleep(1)
def test_publisher(self): for i in range(0, 21): if i % 2 == 0: publisher.info('test' + str(i)) elif i % 3 == 0: publisher.warning('test' + str(i)) elif i % 5 == 0: publisher.error('test' + str(i)) elif i % 7 == 0: publisher.critical('test' + str(i)) else: publisher.debug('test' + str(i)) time.sleep(1)
def analyse(url, path): faup.decode(url) url_parsed = faup.get() pprint.pprint(url_parsed) ## TODO: # FIXME: remove me try: resource_path = url_parsed['resource_path'].encode() except: resource_path = url_parsed['resource_path'] ## TODO: # FIXME: remove me try: query_string = url_parsed['query_string'].encode() except: query_string = url_parsed['query_string'] result_path = {'sqli': False} result_query = {'sqli': False} if resource_path is not None: result_path = pylibinjection.detect_sqli(resource_path) print("path is sqli : {0}".format(result_path)) if query_string is not None: result_query = pylibinjection.detect_sqli(query_string) print("query is sqli : {0}".format(result_query)) if result_path['sqli'] is True or result_query['sqli'] is True: paste = Paste.Paste(path) print("Detected (libinjection) SQL in URL: ") print(urllib.request.unquote(url)) to_print = 'LibInjection;{};{};{};{};{}'.format( paste.p_source, paste.p_date, paste.p_name, "Detected SQL in URL", paste.p_rel_path) publisher.warning(to_print) #Send to duplicate p.populate_set_out(path, 'Duplicate') msg = 'infoleak:automatic-detection="sql-injection";{}'.format(path) p.populate_set_out(msg, 'Tags') #statistics ## TODO: # FIXME: remove me try: tld = url_parsed['tld'].decode() except: tld = url_parsed['tld'] if tld is not None: date = datetime.datetime.now().strftime("%Y%m") server_statistics.hincrby('SQLInjection_by_tld:' + date, tld, 1)
def sendEmailNotification(recipient, alert_name, content): sender = config_loader.get_config_str("Notifications", "sender") sender_user = config_loader.get_config_str("Notifications", "sender_user") sender_host = config_loader.get_config_str("Notifications", "sender_host") sender_port = config_loader.get_config_int("Notifications", "sender_port") sender_pw = config_loader.get_config_str("Notifications", "sender_pw") if sender_pw == 'None': sender_pw = None # raise an exception if any of these is None if (sender is None or sender_host is None or sender_port is None): raise Exception( 'SMTP configuration (host, port, sender) is missing or incomplete!' ) try: if sender_pw is not None: try: smtp_server = smtplib.SMTP(sender_host, sender_port) smtp_server.starttls() except smtplib.SMTPNotSupportedError: print("The server does not support the STARTTLS extension.") smtp_server = smtplib.SMTP_SSL(sender_host, sender_port) smtp_server.ehlo() if sender_user is not None: smtp_server.login(sender_user, sender_pw) else: smtp_server.login(sender, sender_pw) else: smtp_server = smtplib.SMTP(sender_host, sender_port) mime_msg = MIMEMultipart() mime_msg['From'] = sender mime_msg['To'] = recipient mime_msg['Subject'] = "AIL Framework " + alert_name + " Alert" body = content mime_msg.attach(MIMEText(body, 'plain')) smtp_server.sendmail(sender, recipient, mime_msg.as_string()) smtp_server.quit() print('Send notification ' + alert_name + ' to ' + recipient) except Exception as err: traceback.print_tb(err.__traceback__) publisher.warning(err)
def analyse(url, path): faup.decode(url) url_parsed = faup.get() pprint.pprint(url_parsed) ## TODO: # FIXME: remove me try: resource_path = url_parsed['resource_path'].encode() except: resource_path = url_parsed['resource_path'] ## TODO: # FIXME: remove me try: query_string = url_parsed['query_string'].encode() except: query_string = url_parsed['query_string'] result_path = {'sqli' : False} result_query = {'sqli' : False} if resource_path is not None: result_path = pylibinjection.detect_sqli(resource_path) print("path is sqli : {0}".format(result_path)) if query_string is not None: result_query = pylibinjection.detect_sqli(query_string) print("query is sqli : {0}".format(result_query)) if result_path['sqli'] is True or result_query['sqli'] is True: paste = Paste.Paste(path) print("Detected (libinjection) SQL in URL: ") print(urllib.request.unquote(url)) to_print = 'LibInjection;{};{};{};{};{}'.format(paste.p_source, paste.p_date, paste.p_name, "Detected SQL in URL", paste.p_rel_path) publisher.warning(to_print) #Send to duplicate p.populate_set_out(path, 'Duplicate') msg = 'infoleak:automatic-detection="sql-injection";{}'.format(path) p.populate_set_out(msg, 'Tags') #statistics ## TODO: # FIXME: remove me try: tld = url_parsed['tld'].decode() except: tld = url_parsed['tld'] if tld is not None: date = datetime.datetime.now().strftime("%Y%m") server_statistics.hincrby('SQLInjection_by_tld:'+date, tld, 1)
def search_phone(message): paste = Paste.Paste(message) content = paste.get_p_content() # regex to find phone numbers, may raise many false positives (shalt thou seek optimization, upgrading is required) reg_phone = re.compile(r'(\+\d{1,4}(\(\d\))?\d?|0\d?)(\d{6,8}|([-/\. ]{1}\d{2,3}){3,4})') # list of the regex results in the Paste, may be null results = reg_phone.findall(content) # if the list is greater than 4, we consider the Paste may contain a list of phone numbers if len(results) > 4: print results publisher.warning('{} contains PID (phone numbers)'.format(paste.p_name)) #send to Browse_warning_paste p.populate_set_out('phone;{}'.format(message), 'BrowseWarningPaste') #Send to duplicate p.populate_set_out(message, 'Duplicate')
def main(): publisher.port = 6380 publisher.channel = "Script" config_section = 'DomClassifier' p = Process(config_section) addr_dns = p.config.get("DomClassifier", "dns") publisher.info("""ZMQ DomainClassifier is Running""") c = DomainClassifier.domainclassifier.Extract(rawtext="", nameservers=[addr_dns]) cc = p.config.get("DomClassifier", "cc") cc_tld = p.config.get("DomClassifier", "cc_tld") while True: try: message = p.get_from_set() if message is not None: PST = Paste.Paste(message) else: publisher.debug("Script DomClassifier is idling 1s") time.sleep(1) continue paste = PST.get_p_content() mimetype = PST._get_p_encoding() if mimetype == "text/plain": c.text(rawtext=paste) c.potentialdomain() c.validdomain(rtype=['A'], extended=True) localizeddomains = c.include(expression=cc_tld) if localizeddomains: print(localizeddomains) publisher.warning('DomainC;{};{};{};Checked {} located in {};{}'.format( PST.p_source, PST.p_date, PST.p_name, localizeddomains, cc_tld, PST.p_path)) localizeddomains = c.localizedomain(cc=cc) if localizeddomains: print(localizeddomains) publisher.warning('DomainC;{};{};{};Checked {} located in {};{}'.format( PST.p_source, PST.p_date, PST.p_name, localizeddomains, cc, PST.p_path)) except IOError: print("CRC Checksum Failed on :", PST.p_path) publisher.error('Duplicate;{};{};{};CRC Checksum Failed'.format( PST.p_source, PST.p_date, PST.p_name))
def search_cve(message): filepath, count = message.split() paste = Paste.Paste(filepath) content = paste.get_p_content() # regex to find CVE reg_cve = re.compile(r'(CVE-)[1-2]\d{1,4}-\d{1,5}') # list of the regex results in the Paste, may be null results = set(reg_cve.findall(content)) # if the list is greater than 2, we consider the Paste may contain a list of cve if len(results) > 0: print('{} contains CVEs'.format(paste.p_name)) publisher.warning('{} contains CVEs'.format(paste.p_name)) #send to Browse_warning_paste p.populate_set_out('cve;{}'.format(filepath), 'BrowseWarningPaste') #Send to duplicate p.populate_set_out(filepath, 'Duplicate')
def search_cve(message): filepath, count = message.split() paste = Paste.Paste(filepath) content = paste.get_p_content() # regex to find CVE reg_cve = re.compile(r'(CVE-)[1-2]\d{1,4}-\d{1,5}') # list of the regex results in the Paste, may be null results = set(reg_cve.findall(content)) # if the list is greater than 2, we consider the Paste may contain a list of cve if len(results) > 0: print('{} contains CVEs'.format(paste.p_name)) publisher.warning('{} contains CVEs'.format(paste.p_name)) #send to Browse_warning_paste p.populate_set_out('cve;{}'.format(filepath), 'BrowseWarningPaste') #Send to duplicate p.populate_set_out(filepath, 'Duplicate')
def search_phone(message): paste = Paste.Paste(message) content = paste.get_p_content() # regex to find phone numbers, may raise many false positives (shalt thou seek optimization, upgrading is required) reg_phone = re.compile(r'(\+\d{1,4}(\(\d\))?\d?|0\d?)(\d{6,8}|([-/\. ]{1}\d{2,3}){3,4})') # list of the regex results in the Paste, may be null results = reg_phone.findall(content) # if the list is greater than 4, we consider the Paste may contain a list of phone numbers if len(results) > 4 : print results publisher.warning('{} contains PID (phone numbers)'.format(paste.p_name)) if __name__ == '__main__': # If you wish to use an other port of channel, do not forget to run a subscriber accordingly (see launch_logs.sh) # Port of the redis instance used by pubsublogger publisher.port = 6380 # Script is the default channel used for the modules. publisher.channel = 'Script' # Section name in bin/packages/modules.cfg config_section = 'Phone' # Setup the I/O queues p = Process(config_section) # Sent to the logging a description of the module publisher.info("Run Phone module") # Endless loop getting messages from the input queue while True: # Get one message from the input queue message = p.get_from_set() if message is None: publisher.debug("{} queue is empty, waiting".format(config_section)) time.sleep(1) continue # Do something with the message from the queue search_phone(message)
def analyse(url, path): faup.decode(url) url_parsed = faup.get() resource_path = url_parsed['resource_path'] query_string = url_parsed['query_string'] result_path = 0 result_query = 0 if resource_path is not None: result_path = is_sql_injection(resource_path.decode('utf8')) if query_string is not None: result_query = is_sql_injection(query_string.decode('utf8')) if (result_path > 0) or (result_query > 0): paste = Paste.Paste(path) if (result_path > 1) or (result_query > 1): print("Detected SQL in URL: ") print(urllib.request.unquote(url)) to_print = 'SQLInjection;{};{};{};{};{}'.format( paste.p_source, paste.p_date, paste.p_name, "Detected SQL in URL", paste.p_path) publisher.warning(to_print) #Send to duplicate p.populate_set_out(path, 'Duplicate') #send to Browse_warning_paste p.populate_set_out('sqlinjection;{}'.format(path), 'alertHandler') msg = 'infoleak:automatic-detection="sql-injection";{}'.format( path) p.populate_set_out(msg, 'Tags') else: print("Potential SQL injection:") print(urllib.request.unquote(url)) to_print = 'SQLInjection;{};{};{};{};{}'.format( paste.p_source, paste.p_date, paste.p_name, "Potential SQL injection", paste.p_path) publisher.info(to_print)
def search_api_key(message): filename, score = message.split() paste = Paste.Paste(filename) content = paste.get_p_content() aws_access_key = regex_aws_access_key.findall(content) aws_secret_key = regex_aws_secret_key.findall(content) google_api_key = regex_google_api_key.findall(content) if(len(aws_access_key) > 0 or len(aws_secret_key) > 0 or len(google_api_key) > 0): to_print = 'ApiKey;{};{};{};'.format( paste.p_source, paste.p_date, paste.p_name) if(len(google_api_key) > 0): print('found google api key') print(to_print) publisher.warning('{}Checked {} found Google API Key;{}'.format( to_print, len(google_api_key), paste.p_path)) msg = 'infoleak:automatic-detection="google-api-key";{}'.format(filename) p.populate_set_out(msg, 'Tags') if(len(aws_access_key) > 0 or len(aws_secret_key) > 0): print('found AWS key') print(to_print) total = len(aws_access_key) + len(aws_secret_key) publisher.warning('{}Checked {} found AWS Key;{}'.format( to_print, total, paste.p_path)) msg = 'infoleak:automatic-detection="aws-key";{}'.format(filename) p.populate_set_out(msg, 'Tags') msg = 'infoleak:automatic-detection="api-key";{}'.format(filename) p.populate_set_out(msg, 'Tags') msg = 'apikey;{}'.format(filename) p.populate_set_out(msg, 'alertHandler') #Send to duplicate p.populate_set_out(filename, 'Duplicate')
def downloadURL(url, filename): """ Inconditianilly download the URL in a temporary directory. When finished, the file is moved in the real directory. Like this an other process will not attempt to extract an inclomplete file. """ path_temp_bviewfile = os.path.join(c.raw_data, c.bview_dir, 'tmp', filename) path_bviewfile = os.path.join(c.raw_data, c.bview_dir, filename) try: f = urlopen(url) except: return False if f.getcode() != 200: publisher.warning('{} unavailable, code: {}'.format(url, f.getcode())) return False try: with open(path_temp_bviewfile, 'w') as outfile: outfile.write(f.read()) os.rename(path_temp_bviewfile, path_bviewfile) except: os.remove(path_temp_bviewfile) return False return True
def search_geolocation(message): paste = Paste.Paste(message) content = paste.get_p_content() # Load Geoparser geo = Geoparser() geolocation = geo.geoparse(message) # regex ton find latitude and longitude reg_lat = re.compile(r'(\'lat\': \'([-\d.]+)\',)') reg_lon = re.compile(r'(\'lon\': \'([-\d.]+)\',)') #lat = set(reg_lat.findall(content)) #lon = set(reg_lat.findall(content)) lat = reg_lat.search(message).group(2) lon = reg_lon.search(message).group(2) print('latitude: {}'.format(lat)) print('longitude: {}'.format(lon)) print('{} text geolocation'.format(paste.p_name)) publisher.warning('{} contains geolocation'.format(paste.p_name))
def downloadURL(url, filename): """ Inconditianilly download the URL in a temporary directory. When finished, the file is moved in the real directory. Like this an other process will not attempt to extract an inclomplete file. """ path_temp_bviewfile = os.path.join(c.raw_data, c.bview_dir, 'tmp', filename) path_bviewfile = os.path.join(c.raw_data, c.bview_dir, filename) try: f = urllib.urlopen(url) except: return False if f.getcode() != 200: publisher.warning('{} unavailable, code: {}'.format(url, f.getcode())) return False try: with open(path_temp_bviewfile, 'w') as outfile: outfile.write(f.read()) os.rename(path_temp_bviewfile, path_bviewfile) except: os.remove(path_temp_bviewfile) return False return True
def db_import(filename, day): routing_db = get_redis_connector() with open(filename, 'r') as f: entry = '' pipeline = routing_db.pipeline() i = 0 for line in f: # End of block, extracting the information if line == '\n': i += 1 parsed = re.findall('(?:ASPATH|PREFIX): ([^\n{]*)', entry) try: block = parsed[0].strip() # RIPE-NCC-RIS BGP IPv6 Anchor Prefix @RRC00 # RIPE-NCC-RIS BGP Anchor Prefix @ rrc00 - RIPE NCC if block in [ '2001:7fb:ff00::/48', '84.205.80.0/24', '2001:7fb:fe00::/48', '84.205.64.0/24' ]: asn = 12654 else: asn = int(parsed[1].split()[-1].strip()) pipeline.hset(block, day, asn) except: # FIXME: check the cause of the exception publisher.warning(entry) entry = '' if i % 10000 == 0: pipeline.execute() pipeline = routing_db.pipeline() else: # append the line to the current block. entry += line pipeline.execute() publisher.info('{f} finished, {nb} entries impported.'.format( f=filename, nb=i))
def main(): """Main Function""" # CONFIG # cfg = ConfigParser.ConfigParser() cfg.read(configfile) # REDIS # r_serv = redis.StrictRedis( host=cfg.get("Redis_Queues", "host"), port=cfg.getint("Redis_Queues", "port"), db=cfg.getint("Redis_Queues", "db"), ) # LOGGING # publisher.channel = "Queuing" # ZMQ # channel = cfg.get("PubSub_Global", "channel") subscriber_name = "indexer" Sub = ZMQ_PubSub.ZMQSub(configfile, "PubSub_Global", channel, subscriber_name) publisher.info("""Suscribed to channel {0}""".format(channel)) # Until the service is requested to be shutdown, the service # will get the data from the global ZMQ queue and buffer it in Redis. while True: Sub.get_and_lpush(r_serv) if r_serv.sismember("SHUTDOWN_FLAGS", "Indexer_Q"): r_serv.srem("SHUTDOWN_FLAGS", "Indexer_Q") print "Shutdown Flag Up: Terminating" publisher.warning("Shutdown Flag Up: Terminating.") break
def check_all_iban(l_iban, paste, filename): nb_valid_iban = 0 for iban in l_iban: iban = iban[0]+iban[1]+iban[2] iban = ''.join(e for e in iban if e.isalnum()) #iban = iban.upper() res = iban_regex_verify.findall(iban) date = datetime.datetime.now().strftime("%Y%m") if res: print('checking '+iban) if is_valid_iban(iban): print('------') nb_valid_iban = nb_valid_iban + 1 server_statistics.hincrby('iban_by_country:'+date, iban[0:2], 1) if(nb_valid_iban > 0): to_print = 'Iban;{};{};{};'.format(paste.p_source, paste.p_date, paste.p_name) publisher.warning('{}Checked found {} IBAN;{}'.format( to_print, nb_valid_iban, paste.p_rel_path)) msg = 'infoleak:automatic-detection="iban";{}'.format(filename) p.populate_set_out(msg, 'Tags') #Send to duplicate p.populate_set_out(filename, 'Duplicate')
if message is not None: filename, score = message.split() paste = Paste.Paste(filename) content = paste.get_p_content() all_cards = re.findall(regex, content) if len(all_cards) > 0: print 'All matching', all_cards creditcard_set = set([]) for card in all_cards: clean_card = re.sub('[^0-9]', '', card) if lib_refine.is_luhn_valid(clean_card): print clean_card, 'is valid' creditcard_set.add(clean_card) paste.__setattr__(channel, creditcard_set) paste.save_attribute_redis(channel, creditcard_set) pprint.pprint(creditcard_set) to_print = 'CreditCard;{};{};{};'.format( paste.p_source, paste.p_date, paste.p_name) if (len(creditcard_set) > 0): publisher.warning('{}Checked {} valid number(s)'.format( to_print, len(creditcard_set))) else: publisher.info('{}CreditCard related'.format(to_print)) else: publisher.debug("Script creditcard is idling 1m") print 'Sleeping' time.sleep(10)
def search_key(paste): content = paste.get_p_content() find = False if '-----BEGIN PGP MESSAGE-----' in content: publisher.warning('{} has a PGP enc message'.format(paste.p_name)) msg = 'infoleak:automatic-detection="pgp-message";{}'.format(message) p.populate_set_out(msg, 'Tags') find = True if '-----BEGIN CERTIFICATE-----' in content: publisher.warning('{} has a certificate message'.format(paste.p_name)) msg = 'infoleak:automatic-detection="certificate";{}'.format(message) p.populate_set_out(msg, 'Tags') find = True if '-----BEGIN RSA PRIVATE KEY-----' in content: publisher.warning('{} has a RSA private key message'.format(paste.p_name)) print('rsa private key message found') msg = 'infoleak:automatic-detection="rsa-private-key";{}'.format(message) p.populate_set_out(msg, 'Tags') find = True if '-----BEGIN PRIVATE KEY-----' in content: publisher.warning('{} has a private key message'.format(paste.p_name)) print('private key message found') msg = 'infoleak:automatic-detection="private-key";{}'.format(message) p.populate_set_out(msg, 'Tags') find = True if '-----BEGIN ENCRYPTED PRIVATE KEY-----' in content: publisher.warning('{} has an encrypted private key message'.format(paste.p_name)) print('encrypted private key message found') msg = 'infoleak:automatic-detection="encrypted-private-key";{}'.format(message) p.populate_set_out(msg, 'Tags') find = True if '-----BEGIN OPENSSH PRIVATE KEY-----' in content: publisher.warning('{} has an openssh private key message'.format(paste.p_name)) print('openssh private key message found') msg = 'infoleak:automatic-detection="private-ssh-key";{}'.format(message) p.populate_set_out(msg, 'Tags') find = True if '-----BEGIN OpenVPN Static key V1-----' in content: publisher.warning('{} has an openssh private key message'.format(paste.p_name)) print('OpenVPN Static key message found') msg = 'infoleak:automatic-detection="vpn-static-key";{}'.format(message) p.populate_set_out(msg, 'Tags') find = True if '-----BEGIN DSA PRIVATE KEY-----' in content: publisher.warning('{} has a dsa private key message'.format(paste.p_name)) msg = 'infoleak:automatic-detection="dsa-private-key";{}'.format(message) p.populate_set_out(msg, 'Tags') find = True if '-----BEGIN EC PRIVATE KEY-----' in content: publisher.warning('{} has an ec private key message'.format(paste.p_name)) msg = 'infoleak:automatic-detection="ec-private-key";{}'.format(message) p.populate_set_out(msg, 'Tags') find = True if '-----BEGIN PGP PRIVATE KEY BLOCK-----' in content: publisher.warning('{} has a pgp private key block message'.format(paste.p_name)) msg = 'infoleak:automatic-detection="pgp-private-key";{}'.format(message) p.populate_set_out(msg, 'Tags') find = True if find : #Send to duplicate p.populate_set_out(message, 'Duplicate') #send to Browse_warning_paste msg = ('keys;{}'.format(message)) print(message) p.populate_set_out( msg, 'alertHandler')
updated_descrs = 0 for asn, descr in data: all_descrs = r.hgetall(asn) if len(all_descrs) == 0: p.hset(asn, timestamp, descr) publisher.debug('New asn: {asn}'.format(asn=asn)) new_asns += 1 else: dates = sorted(all_descrs.keys()) last_descr = all_descrs[dates[-1]] if descr != last_descr: p.hset(asn, timestamp, descr) msg = 'New description for {asn}. Was {old}, is {new}'.format( asn=asn, old=last_descr, new=descr) publisher.info(msg) updated_descrs += 1 p.execute() msg = '===== Import finished: {new}, new ASNs:{nb}, Updated:{up} ====='.format( new=timestamp, nb=new_asns, up=updated_descrs) publisher.info(msg) if args.not_new: break else: newfile = False try: newfile = fetch('http://www.cidr-report.org/as2.0/autnums.html', args.directory) except: publisher.warning('Exception in fetching!') if not newfile: time.sleep(sleep_timer)
try: l = client.lookup(ip, qType='IP') except ipaddress.AddressValueError: continue cc = getattr(l, 'cc') asn = getattr(l, 'asn') # EU is not an official ISO 3166 code (but used by RIPE # IP allocation) if cc is not None and cc != "EU": print hostl, asn, cc, \ pycountry.countries.get(alpha2=cc).name if cc == cc_critical: publisher.warning( 'Url;{};{};{};Detected {} {}'.format( PST.p_source, PST.p_date, PST.p_name, hostl, cc)) else: print hostl, asn, cc A_values = lib_refine.checking_A_record(r_serv2, domains_list) if A_values[0] >= 1: PST.__setattr__(channel, A_values) PST.save_attribute_redis(channel, (A_values[0], list(A_values[1]))) pprint.pprint(A_values) publisher.info('Url;{};{};{};Checked {} URL'.format( PST.p_source, PST.p_date, PST.p_name, A_values[0])) prec_filename = filename
sites= re.findall(regex_web, content) #Use to count occurences sites_set = set(re.findall(regex_web, content)) message = 'Checked {} credentials found.'.format(len(creds)) if sites_set: message += ' Related websites: {}'.format( (', '.join(sites_set)) ) to_print = 'Credential;{};{};{};{};{}'.format(paste.p_source, paste.p_date, paste.p_name, message, paste.p_rel_path) print('\n '.join(creds)) #num of creds above tresh, publish an alert if len(creds) > criticalNumberToAlert: print("========> Found more than 10 credentials in this file : {}".format( filepath )) publisher.warning(to_print) #Send to duplicate p.populate_set_out(filepath, 'Duplicate') msg = 'infoleak:automatic-detection="credential";{}'.format(filepath) p.populate_set_out(msg, 'Tags') #Put in form, count occurences, then send to moduleStats creds_sites = {} site_occurence = re.findall(regex_site_for_stats, content) for site in site_occurence: site_domain = site[1:-1] if site_domain in creds_sites.keys(): creds_sites[site_domain] += 1 else: creds_sites[site_domain] = 1
def main(): """Main Function""" # CONFIG # cfg = ConfigParser.ConfigParser() cfg.read(configfile) # REDIS # r_serv = redis.StrictRedis( host = cfg.get("Redis_Queues", "host"), port = cfg.getint("Redis_Queues", "port"), db = cfg.getint("Redis_Queues", "db")) r_serv1 = redis.StrictRedis( host = cfg.get("Redis_Data_Merging", "host"), port = cfg.getint("Redis_Data_Merging", "port"), db = cfg.getint("Redis_Data_Merging", "db")) r_serv2 = redis.StrictRedis( host = cfg.get("Redis_Cache", "host"), port = cfg.getint("Redis_Cache", "port"), db = cfg.getint("Redis_Cache", "db")) # LOGGING # publisher.channel = "Script" # ZMQ # Sub = ZMQ_PubSub.ZMQSub(configfile,"PubSub_Categ", "mails_categ", "emails") # FUNCTIONS # publisher.info("Suscribed to channel mails_categ") message = Sub.get_msg_from_queue(r_serv) prec_filename = None email_regex = "[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,6}" while True: try: if message != None: channel, filename, word, score = message.split() if prec_filename == None or filename != prec_filename: PST = P.Paste(filename) MX_values = lib_refine.checking_MX_record(r_serv2, PST.get_regex(email_regex)) if MX_values[0] >= 1: PST.__setattr__(channel, MX_values) PST.save_attribute_redis(r_serv1, channel, (MX_values[0], list(MX_values[1]))) pprint.pprint(MX_values) if MX_values[0] > 10: publisher.warning('{0};{1};{2};{3};{4}'.format("Mails", PST.p_source, PST.p_date, PST.p_name,"Checked "+ str(MX_values[0])+ " e-mails" )) else: publisher.info('{0};{1};{2};{3};{4}'.format("Mails", PST.p_source, PST.p_date, PST.p_name,"Checked " str(MX_values[0])+ " e-mail(s)" )) prec_filename = filename else: if r_serv.sismember("SHUTDOWN_FLAGS", "Mails"): r_serv.srem("SHUTDOWN_FLAGS", "Mails") print "Shutdown Flag Up: Terminating" publisher.warning("Shutdown Flag Up: Terminating.") break publisher.debug("Script Mails is Idling 10s") time.sleep(10) message = Sub.get_msg_from_queue(r_serv) except dns.exception.Timeout: print "dns.exception.Timeout" pass
creditcard_set = set([]) PST = Paste.Paste(filename) for x in PST.get_regex(creditcard_regex): if lib_refine.is_luhn_valid(x): creditcard_set.add(x) PST.__setattr__(channel, creditcard_set) PST.save_attribute_redis(channel, creditcard_set) pprint.pprint(creditcard_set) to_print = 'CreditCard;{};{};{};'.format( PST.p_source, PST.p_date, PST.p_name) if (len(creditcard_set) > 0): publisher.critical('{}Checked {} valid number(s)'.format( to_print, len(creditcard_set))) else: publisher.info('{}CreditCard related'.format(to_print)) prec_filename = filename else: if h.redis_queue_shutdown(): print "Shutdown Flag Up: Terminating" publisher.warning("Shutdown Flag Up: Terminating.") break publisher.debug("Script creditcard is idling 1m") time.sleep(60) message = h.redis_rpop()
for x in PST.get_regex(url_regex): # Extracting url with regex url, s, credential, subdomain, domain, host, port, \ resource_path, query_string, f1, f2, f3, f4 = x domains_list.append(domain) urls.append(url) # Saving the list of extracted onion domains. PST.__setattr__(channel, domains_list) PST.save_attribute_redis(channel, domains_list) to_print = 'Onion;{};{};{};'.format(PST.p_source, PST.p_date, PST.p_name) if len(domains_list) > 0: publisher.warning('{}Detected {} .onion(s)'.format( to_print, len(domains_list))) now = datetime.datetime.now() path = os.path.join('onions', str(now.year).zfill(4), str(now.month).zfill(2), str(now.day).zfill(2), str(int(time.mktime(now.utctimetuple())))) to_print = 'Onion;{};{};{};'.format(PST.p_source, PST.p_date, PST.p_name) for url in fetch(p, r_cache, urls, domains_list, path): publisher.warning('{}Checked {}'.format(to_print, url)) else: publisher.info('{}Onion related'.format(to_print)) prec_filename = filename else:
def search_gpg(message): paste = Paste.Paste(message) content = paste.get_p_content() if '-----BEGIN PGP MESSAGE-----' in content: publisher.warning('{} has a PGP enc message'.format(paste.p_name))