def search_phone(message):
paste = Paste.Paste(message)
content = paste.get_p_content()
# regex to find phone numbers, may raise many false positives (shalt thou seek optimization, upgrading is required)
reg_phone = re.compile(r'(\+\d{1,4}(\(\d\))?\d?|0\d?)(\d{6,8}|([-/\. ]{1}\d{2,3}){3,4})')
reg_phone = re.compile(r'(\+\d{1,4}(\(\d\))?\d?|0\d?)(\d{6,8}|([-/\. ]{1}\(?\d{2,4}\)?){3,4})')
# list of the regex results in the Paste, may be null
results = reg_phone.findall(content)
# if the list is greater than 4, we consider the Paste may contain a list of phone numbers
if len(results) > 4:
print(results)
publisher.warning('{} contains PID (phone numbers)'.format(paste.p_name))
msg = 'infoleak:automatic-detection="phone-number";{}'.format(message)
p.populate_set_out(msg, 'Tags')
#Send to duplicate
p.populate_set_out(message, 'Duplicate')
stats = {}
for phone_number in results:
try:
x = phonenumbers.parse(phone_number, None)
country_code = x.country_code
if stats.get(country_code) is None:
stats[country_code] = 1
else:
stats[country_code] = stats[country_code] + 1
except:
pass
for country_code in stats:
if stats[country_code] > 4:
publisher.warning('{} contains Phone numbers with country code {}'.format(paste.p_name, country_code))
def analyse(url, path):
faup.decode(url)
url_parsed = faup.get()
pprint.pprint(url_parsed)
resource_path = url_parsed['resource_path']
query_string = url_parsed['query_string']
result_path = {'sqli': False}
result_query = {'sqli': False}
if resource_path is not None:
result_path = pylibinjection.detect_sqli(resource_path)
print("path is sqli : {0}".format(result_path))
if query_string is not None:
result_query = pylibinjection.detect_sqli(query_string)
print("query is sqli : {0}".format(result_query))
if result_path['sqli'] is True or result_query['sqli'] is True:
paste = Paste.Paste(path)
print("Detected (libinjection) SQL in URL: ")
print(urllib.request.unquote(url))
to_print = 'LibInjection;{};{};{};{};{}'.format(
paste.p_source, paste.p_date, paste.p_name, "Detected SQL in URL",
paste.p_path)
publisher.warning(to_print)
#Send to duplicate
p.populate_set_out(path, 'Duplicate')
#send to Browse_warning_paste
p.populate_set_out('sqlinjection;{}'.format(path), 'alertHandler')
msg = 'infoleak:automatic-detection="sql-injection";{}'.format(path)
p.populate_set_out(msg, 'Tags')
def check_all_iban(l_iban, paste, filename):
nb_valid_iban = 0
for iban in l_iban:
iban = iban[0] + iban[1] + iban[2]
iban = ''.join(e for e in iban if e.isalnum())
#iban = iban.upper()
res = iban_regex_verify.findall(iban)
date = datetime.datetime.now().strftime("%Y%m")
if res:
print('checking ' + iban)
if is_valid_iban(iban):
print('------')
nb_valid_iban = nb_valid_iban + 1
server_statistics.hincrby('iban_by_country:' + date, iban[0:2],
1)
if (nb_valid_iban > 0):
to_print = 'Iban;{};{};{};'.format(paste.p_source, paste.p_date,
paste.p_name)
publisher.warning('{}Checked found {} IBAN;{}'.format(
to_print, nb_valid_iban, paste.p_rel_path))
msg = 'infoleak:automatic-detection="iban";{}'.format(filename)
p.populate_set_out(msg, 'Tags')
#Send to duplicate
p.populate_set_out(filename, 'Duplicate')
def search_api_key(message):
filename, score = message.split()
paste = Paste.Paste(filename)
content = paste.get_p_content()
aws_access_key = regex_aws_access_key.findall(content)
aws_secret_key = regex_aws_secret_key.findall(content)
google_api_key = regex_google_api_key.findall(content)
if (len(aws_access_key) > 0 or len(aws_secret_key) > 0
or len(google_api_key) > 0):
to_print = 'ApiKey;{};{};{};'.format(paste.p_source, paste.p_date,
paste.p_name)
if (len(google_api_key) > 0):
print('found google api key')
print(to_print)
publisher.warning('{}Checked {} found Google API Key;{}'.format(
to_print, len(google_api_key), paste.p_path))
if (len(aws_access_key) > 0 or len(aws_secret_key) > 0):
print('found AWS key')
print(to_print)
total = len(aws_access_key) + len(aws_secret_key)
publisher.warning('{}Checked {} found AWS Key;{}'.format(
to_print, total, paste.p_path))
msg = 'apikey;{}'.format(filename)
p.populate_set_out(msg, 'alertHandler')
#Send to duplicate
p.populate_set_out(filename, 'Duplicate')
def search_game(message):
#We recover the paste
paste = Paste.Paste(message)
content = paste.get_p_content()
#We open the file with all game word and the stock for all paste found
filetoopen = open("corpus.txt", "r")
filetowrite = open("stock.txt", "a")
count = 0 #Number of game word found in 1 file
for line in filetoopen:
linestrip = line.strip(
) #Must do because it takes all the line and not just the word
reg = re.compile(r'{}'.format(linestrip)) #we create the regex
results = re.findall(reg, content) #we find the occurences
if (len(results) > 0):
count = count + 1
re.purge()
if count > 5:
print results
publisher.warning(
'{} contains Game related conversations+{} occurences of a game related word '
.format(paste.p_name, count)) #warning for the logs
filetowrite.write(
'{} contains Game related conversations+{} occurences of a game related word \n'
.format(paste.p_name, count)) #For stock.txt
to_print = 'GameConv;{};{};{};{} Terms related;{}'.format(
paste.p_source, paste.p_date, paste.p_name, count,
paste.p_path) #To see on the webinterface
publisher.warning(to_print)
filetoopen.close()
filetowrite.close()
def search_key(content, message, paste):
bitcoin_address = re.findall(regex_bitcoin_public_address, content)
bitcoin_private_key = re.findall(regex_bitcoin_private_key, content)
validate_address = False
key = False
if(len(bitcoin_address) >0):
#print(message)
for address in bitcoin_address:
if(check_bc(address)):
validate_address = True
print('Bitcoin address found : {}'.format(address))
if(len(bitcoin_private_key) > 0):
for private_key in bitcoin_private_key:
print('Bitcoin private key found : {}'.format(private_key))
key = True
if(validate_address):
p.populate_set_out(message, 'Duplicate')
to_print = 'Bitcoin found: {} address and {} private Keys'.format(len(bitcoin_address), len(bitcoin_private_key))
print(to_print)
publisher.warning(to_print)
msg = ('bitcoin;{}'.format(message))
p.populate_set_out( msg, 'alertHandler')
msg = 'infoleak:automatic-detection="bitcoin-address";{}'.format(message)
p.populate_set_out(msg, 'Tags')
if(key):
msg = 'infoleak:automatic-detection="bitcoin-private-key";{}'.format(message)
p.populate_set_out(msg, 'Tags')
to_print = 'Bitcoin;{};{};{};'.format(paste.p_source, paste.p_date,
paste.p_name)
publisher.warning('{}Detected {} Bitcoin private key;{}'.format(
to_print, len(bitcoin_private_key),paste.p_path))
def main():
"""Main Function"""
# CONFIG #
cfg = ConfigParser.ConfigParser()
cfg.read(configfile)
# REDIS #
r_serv = redis.StrictRedis(
host = cfg.get("Redis_Queues", "host"),
port = cfg.getint("Redis_Queues", "port"),
db = cfg.getint("Redis_Queues", "db"))
# LOGGING #
publisher.channel = "Queuing"
# ZMQ #
Sub = ZMQ_PubSub.ZMQSub(configfile,"PubSub_Categ", "onion_categ", "tor")
# FUNCTIONS #
publisher.info("""Suscribed to channel {0}""".format("onion_categ"))
while True:
Sub.get_and_lpush(r_serv)
if r_serv.sismember("SHUTDOWN_FLAGS", "Onion_Q"):
r_serv.srem("SHUTDOWN_FLAGS", "Onion_Q")
print "Shutdown Flag Up: Terminating"
publisher.warning("Shutdown Flag Up: Terminating.")
break
def search_phone(message):
paste = Paste.Paste(message)
content = paste.get_p_content()
# regex to find phone numbers, may raise many false positives (shalt thou seek optimization, upgrading is required)
reg_phone = re.compile(r'(\+\d{1,4}(\(\d\))?\d?|0\d?)(\d{6,8}|([-/\. ]{1}\d{2,3}){3,4})')
reg_phone = re.compile(r'(\+\d{1,4}(\(\d\))?\d?|0\d?)(\d{6,8}|([-/\. ]{1}\(?\d{2,4}\)?){3,4})')
# list of the regex results in the Paste, may be null
results = reg_phone.findall(content)
# if the list is greater than 4, we consider the Paste may contain a list of phone numbers
if len(results) > 4:
print results
publisher.warning('{} contains PID (phone numbers)'.format(paste.p_name))
#send to Browse_warning_paste
p.populate_set_out('phone;{}'.format(message), 'BrowseWarningPaste')
#Send to duplicate
p.populate_set_out(message, 'Duplicate')
stats = {}
for phone_number in results:
try:
x = phonenumbers.parse(phone_number, None)
country_code = x.country_code
if stats.get(country_code) is None:
stats[country_code] = 1
else:
stats[country_code] = stats[country_code] + 1
except:
pass
for country_code in stats:
if stats[country_code] > 4:
publisher.warning('{} contains Phone numbers with country code {}'.format(paste.p_name, country_code))
def set_out_paste(decoder_name, message):
publisher.warning(decoder_name+' decoded')
#Send to duplicate
p.populate_set_out(message, 'Duplicate')
msg = 'infoleak:automatic-detection="'+decoder_name+'";{}'.format(message)
p.populate_set_out(msg, 'Tags')
def analyse(url, path):
faup.decode(url)
url_parsed = faup.get()
resource_path = url_parsed['resource_path']
query_string = url_parsed['query_string']
result_path = 0
result_query = 0
if resource_path is not None:
result_path = is_sql_injection(resource_path)
if query_string is not None:
result_query = is_sql_injection(query_string)
if (result_path > 0) or (result_query > 0):
paste = Paste.Paste(path)
if (result_path > 1) or (result_query > 1):
print "Detected SQL in URL: "
print urllib2.unquote(url)
to_print = 'SQLInjection;{};{};{};{};{}'.format(paste.p_source, paste.p_date, paste.p_name, "Detected SQL in URL", paste.p_path)
publisher.warning(to_print)
#Send to duplicate
p.populate_set_out(path, 'Duplicate')
#send to Browse_warning_paste
p.populate_set_out('sqlinjection;{}'.format(path), 'BrowseWarningPaste')
else:
print "Potential SQL injection:"
print urllib2.unquote(url)
to_print = 'SQLInjection;{};{};{};{}'.format(paste.p_source, paste.p_date, paste.p_name, "Potential SQL injection")
publisher.info(to_print)
def analyse(url, path):
if is_sql_injection(url):
faup.decode(url)
url_parsed = faup.get()
paste = Paste.Paste(path)
print("Detected SQL in URL: ")
print(urllib.request.unquote(url))
to_print = 'SQLInjection;{};{};{};{};{}'.format(paste.p_source, paste.p_date, paste.p_name, "Detected SQL in URL", paste.p_rel_path)
publisher.warning(to_print)
#Send to duplicate
p.populate_set_out(path, 'Duplicate')
msg = 'infoleak:automatic-detection="sql-injection";{}'.format(path)
p.populate_set_out(msg, 'Tags')
#statistics
tld = url_parsed['tld']
if tld is not None:
## TODO: # FIXME: remove me
try:
tld = tld.decode()
except:
pass
date = datetime.datetime.now().strftime("%Y%m")
server_statistics.hincrby('SQLInjection_by_tld:'+date, tld, 1)
def search_base64(content, message):
find = False
base64_list = re.findall(regex_base64, content)
if (len(base64_list) > 0):
for b64 in base64_list:
if len(b64) >= 40:
decode = base64.b64decode(b64)
type = magic.from_buffer(decode, mime=True)
#print(type)
#print(decode)
find = True
hash = sha1(decode).hexdigest()
data = {}
data['name'] = hash
data['date'] = datetime.datetime.now().strftime("%d/%m/%y")
data['origin'] = message
data['estimated type'] = type
json_data = json.dumps(data)
save_base64_as_file(decode, type, hash, json_data)
print('found {} '.format(type))
if (find):
publisher.warning('base64 decoded')
#Send to duplicate
p.populate_set_out(message, 'Duplicate')
#send to Browse_warning_paste
msg = ('base64;{}'.format(message))
p.populate_set_out(msg, 'alertHandler')
def search_key(message):
paste = Paste.Paste(message)
content = paste.get_p_content()
find = False
if '-----BEGIN PGP MESSAGE-----' in content:
publisher.warning('{} has a PGP enc message'.format(paste.p_name))
find = True
if '-----BEGIN CERTIFICATE-----' in content:
publisher.warning('{} has a certificate message'.format(paste.p_name))
find = True
if '-----BEGIN RSA PRIVATE KEY-----' in content:
publisher.warning('{} has a RSA key message'.format(paste.p_name))
find = True
if '-----BEGIN PRIVATE KEY-----' in content:
publisher.warning('{} has a private message'.format(paste.p_name))
find = True
if '-----BEGIN ENCRYPTED PRIVATE KEY-----' in content:
publisher.warning('{} has an encrypted private message'.format(
paste.p_name))
find = True
if find:
#Send to duplicate
p.populate_set_out(message, 'Duplicate')
#send to Browse_warning_paste
p.populate_set_out('keys;{}'.format(message), 'alertHandler')
def set_out_paste(decoder_name, message):
publisher.warning(decoder_name+' decoded')
#Send to duplicate
p.populate_set_out(message, 'Duplicate')
msg = 'infoleak:automatic-detection="'+decoder_name+'";{}'.format(message)
p.populate_set_out(msg, 'Tags')
def search_ip(message):
paste = Paste.Paste(message)
content = paste.get_p_content()
# regex to find IPs
reg_ip = re.compile(
r'^(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)',
flags=re.MULTILINE)
# list of the regex results in the Paste, may be null
results = reg_ip.findall(content)
matching_ips = []
for res in results:
address = IPv4Address(res)
for network in ip_networks:
if address in network:
matching_ips.append(address)
if len(matching_ips) > 0:
print('{} contains {} IPs'.format(paste.p_name, len(matching_ips)))
publisher.warning('{} contains {} IPs'.format(paste.p_name,
len(matching_ips)))
#Tag message with IP
msg = 'infoleak:automatic-detection="ip";{}'.format(message)
p.populate_set_out(msg, 'Tags')
#Send to duplicate
p.populate_set_out(message, 'Duplicate')
def analyse(url, path):
faup.decode(url)
url_parsed = faup.get()
resource_path = url_parsed['resource_path']
query_string = url_parsed['query_string']
result_path = 0
result_query = 0
if resource_path is not None:
result_path = is_sql_injection(resource_path)
if query_string is not None:
result_query = is_sql_injection(query_string)
if (result_path > 0) or (result_query > 0):
paste = Paste.Paste(path)
if (result_path > 1) or (result_query > 1):
print "Detected SQL in URL: "
print urllib2.unquote(url)
to_print = 'SQLInjection;{};{};{};{}'.format(paste.p_source, paste.p_date, paste.p_name, "Detected SQL in URL")
publisher.warning(to_print)
#Send to duplicate
p.populate_set_out(path, 'Duplicate')
#send to Browse_warning_paste
p.populate_set_out('sqlinjection;{}'.format(path), 'BrowseWarningPaste')
else:
print "Potential SQL injection:"
print urllib2.unquote(url)
to_print = 'SQLInjection;{};{};{};{}'.format(paste.p_source, paste.p_date, paste.p_name, "Potential SQL injection")
publisher.info(to_print)
def search_key(content, message, paste):
bitcoin_address = re.findall(regex_bitcoin_public_address, content)
bitcoin_private_key = re.findall(regex_bitcoin_private_key, content)
validate_address = False
key = False
if(len(bitcoin_address) >0):
#print(message)
for address in bitcoin_address:
if(check_bc(address)):
validate_address = True
print('Bitcoin address found : {}'.format(address))
if(len(bitcoin_private_key) > 0):
for private_key in bitcoin_private_key:
print('Bitcoin private key found : {}'.format(private_key))
key = True
if(validate_address):
p.populate_set_out(message, 'Duplicate')
to_print = 'Bitcoin found: {} address and {} private Keys'.format(len(bitcoin_address), len(bitcoin_private_key))
print(to_print)
publisher.warning(to_print)
msg = ('bitcoin;{}'.format(message))
p.populate_set_out( msg, 'alertHandler')
msg = 'infoleak:automatic-detection="bitcoin-address";{}'.format(message)
p.populate_set_out(msg, 'Tags')
if(key):
msg = 'infoleak:automatic-detection="bitcoin-private-key";{}'.format(message)
p.populate_set_out(msg, 'Tags')
to_print = 'Bitcoin;{};{};{};'.format(paste.p_source, paste.p_date,
paste.p_name)
publisher.warning('{}Detected {} Bitcoin private key;{}'.format(
to_print, len(bitcoin_private_key),paste.p_path))
def db_import(filename, day):
with open(filename, 'r') as f:
entry = ''
pipeline = routing_db.pipeline()
i = 0
for line in f:
# End of block, extracting the information
if line == '\n':
i += 1
parsed = re.findall('(?:ASPATH|PREFIX): ([^\n{]*)', entry)
try:
block = parsed[0].strip()
# RIPE-NCC-RIS BGP IPv6 Anchor Prefix @RRC00
# RIPE-NCC-RIS BGP Anchor Prefix @ rrc00 - RIPE NCC
if block in ['2001:7fb:ff00::/48', '84.205.80.0/24',
'2001:7fb:fe00::/48', '84.205.64.0/24']:
asn = 12654
else:
asn = int(parsed[1].split()[-1].strip())
pipeline.hset(block, day, asn)
except:
#FIXME: check the cause of the exception
publisher.warning(entry)
entry = ''
if i%10000 == 0:
pipeline.execute()
pipeline = routing_db.pipeline()
else :
# append the line to the current block.
entry += line
pipeline.execute()
publisher.info('{f} finished, {nb} entries impported.'.\
format(f=filename, nb = i))
def main():
"""Main Function"""
# CONFIG #
cfg = ConfigParser.ConfigParser()
cfg.read(configfile)
# REDIS #
r_serv = redis.StrictRedis(
host = cfg.get("Redis_Queues", "host"),
port = cfg.getint("Redis_Queues", "port"),
db = cfg.getint("Redis_Queues", "db"))
# LOGGING #
publisher.channel = "Queuing"
# ZMQ #
channel = cfg.get("PubSub_Words", "channel_0")
subscriber_name = "curve"
subscriber_config_section = "PubSub_Words"
Sub = ZMQ_PubSub.ZMQSub(configfile, subscriber_config_section, channel, subscriber_name)
# FUNCTIONS #
publisher.info("""Suscribed to channel {0}""".format(channel))
while True:
Sub.get_and_lpush(r_serv)
if r_serv.sismember("SHUTDOWN_FLAGS", "Curve_Q"):
r_serv.srem("SHUTDOWN_FLAGS", "Curve_Q")
print "Shutdown Flag Up: Terminating"
publisher.warning("Shutdown Flag Up: Terminating.")
break
def set_out_item(decoder_name, item_id):
publisher.warning(decoder_name + ' decoded')
#Send to duplicate
p.populate_set_out(item_id, 'Duplicate')
msg = 'infoleak:automatic-detection="' + decoder_name + '";{}'.format(
item_id)
p.populate_set_out(msg, 'Tags')
def search_gpg(message):
paste = Paste.Paste(message)
content = paste.get_p_content()
if '-----BEGIN PGP MESSAGE-----' in content:
publisher.warning('{} has a PGP enc message'.format(paste.p_name))
#Send to duplicate
p.populate_set_out(message, 'Duplicate')
#send to Browse_warning_paste
p.populate_set_out('keys;{}'.format(message), 'BrowseWarningPaste')
def search_gpg(message):
paste = Paste.Paste(message)
content = paste.get_p_content()
if '-----BEGIN PGP MESSAGE-----' in content:
publisher.warning('{} has a PGP enc message'.format(paste.p_name))
#Send to duplicate
p.populate_set_out(message, 'Duplicate')
#send to Browse_warning_paste
p.populate_set_out('keys;{}'.format(message), 'BrowseWarningPaste')
def main():
"""Main Function"""
# CONFIG #
cfg = ConfigParser.ConfigParser()
cfg.read(configfile)
# REDIS #
r_serv = redis.StrictRedis(
host = cfg.get("Redis_Queues", "host"),
port = cfg.getint("Redis_Queues", "port"),
db = cfg.getint("Redis_Queues", "db"))
# LOGGING #
publisher.channel = "Script"
# ZMQ #
channel = cfg.get("PubSub_Longlines", "channel_1")
subscriber_name = "tokenize"
subscriber_config_section = "PubSub_Longlines"
#Publisher
publisher_config_section = "PubSub_Words"
publisher_name = "pubtokenize"
Sub = ZMQ_PubSub.ZMQSub(configfile, subscriber_config_section, channel, subscriber_name)
Pub = ZMQ_PubSub.ZMQPub(configfile, publisher_config_section, publisher_name)
channel_0 = cfg.get("PubSub_Words", "channel_0")
# FUNCTIONS #
publisher.info("Tokeniser subscribed to channel {0}".format(cfg.get("PubSub_Longlines", "channel_1")))
while True:
message = Sub.get_msg_from_queue(r_serv)
print message
if message != None:
PST = P.Paste(message.split(" ",-1)[-1])
else:
if r_serv.sismember("SHUTDOWN_FLAGS", "Tokenize"):
r_serv.srem("SHUTDOWN_FLAGS", "Tokenize")
print "Shutdown Flag Up: Terminating"
publisher.warning("Shutdown Flag Up: Terminating.")
break
publisher.debug("Tokeniser is idling 10s")
time.sleep(10)
print "sleepin"
continue
for word, score in PST._get_top_words().items():
if len(word) >= 4:
msg = channel_0+' '+PST.p_path+' '+str(word)+' '+str(score)
Pub.send_message(msg)
print msg
else:
pass
def redis_interbargraph_set(r_serv, year, month, overwrite):
"""Create a Redis sorted set.
:param r_serv: -- connexion to redis database
:param year: -- (integer) The year to process
:param month: -- (integer) The month to process
:param overwrite: -- (bool) trigger the overwrite mode
This function create inside redis the intersection of all days in
a month two by two.
Example:
For a month of 31days it will create 30 sorted set between day and
day+1 until the last day.
The overwrite mode delete the intersets and re-create them.
"""
a = date(year, month, 01)
b = date(year, month, cal.monthrange(year, month)[1])
if overwrite:
r_serv.delete("InterSet")
for dt in rrule(DAILY, dtstart = a, until = b - timedelta(1)):
dayafter = dt+timedelta(1)
r_serv.delete(str(dt.strftime("%Y%m%d"))+str(dayafter.strftime("%Y%m%d")))
r_serv.zinterstore(
str(dt.strftime("%Y%m%d"))+str(dayafter.strftime("%Y%m%d")),
{str(dt.strftime("%Y%m%d")):1,
str(dayafter.strftime("%Y%m%d")):-1})
r_serv.zadd(
"InterSet",
1,
str(dt.strftime("%Y%m%d"))+str(dayafter.strftime("%Y%m%d")))
else:
for dt in rrule(DAILY, dtstart = a, until = b - timedelta(1)):
dayafter = dt+timedelta(1)
if r_serv.zcard(str(dt.strftime("%Y%m%d"))+str(dayafter.strftime("%Y%m%d"))) == 0:
r_serv.zinterstore(
str(dt.strftime("%Y%m%d"))+str(dayafter.strftime("%Y%m%d")),
{str(dt.strftime("%Y%m%d")):1,
str(dayafter.strftime("%Y%m%d")):-1})
r_serv.zadd(
"InterSet",
1,
str(dt.strftime("%Y%m%d"))+str(dayafter.strftime("%Y%m%d")))
publisher.info(str(dt.strftime("%Y%m%d"))+str(dayafter.strftime("%Y%m%d"))+" Intersection Created")
else:
publisher.warning("Data already exist, operation aborted.")
def analyse(url, path):
faup.decode(url)
url_parsed = faup.get()
resource_path = url_parsed['resource_path']
query_string = url_parsed['query_string']
result_path = 0
result_query = 0
if resource_path is not None:
## TODO: # FIXME: remove me
try:
resource_path = resource_path.decode()
except:
pass
result_path = is_sql_injection(resource_path)
if query_string is not None:
## TODO: # FIXME: remove me
try:
query_string = query_string.decode()
except:
pass
result_query = is_sql_injection(query_string)
if (result_path > 0) or (result_query > 0):
paste = Paste.Paste(path)
if (result_path > 1) or (result_query > 1):
print("Detected SQL in URL: ")
print(urllib.request.unquote(url))
to_print = 'SQLInjection;{};{};{};{};{}'.format(paste.p_source, paste.p_date, paste.p_name, "Detected SQL in URL", paste.p_rel_path)
publisher.warning(to_print)
#Send to duplicate
p.populate_set_out(path, 'Duplicate')
msg = 'infoleak:automatic-detection="sql-injection";{}'.format(path)
p.populate_set_out(msg, 'Tags')
#statistics
tld = url_parsed['tld']
if tld is not None:
## TODO: # FIXME: remove me
try:
tld = tld.decode()
except:
pass
date = datetime.datetime.now().strftime("%Y%m")
server_statistics.hincrby('SQLInjection_by_tld:'+date, tld, 1)
else:
print("Potential SQL injection:")
print(urllib.request.unquote(url))
to_print = 'SQLInjection;{};{};{};{};{}'.format(paste.p_source, paste.p_date, paste.p_name, "Potential SQL injection", paste.p_rel_path)
publisher.info(to_print)
def __query_logging(ip, user_agent, method, q_ip=None, announce_date=None,
days_limit=None, level=None):
if level == 'warning':
publisher.warning(__csv2string([ip, user_agent, method, q_ip,
announce_date, days_limit, level]))
elif level == 'error':
publisher.error(__csv2string([ip, user_agent, method, q_ip,
announce_date, days_limit, level]))
else:
publisher.info(__csv2string([ip, user_agent, method, q_ip,
announce_date, days_limit, level]))
def __query_logging(ip, user_agent, method, q_ip=None, announce_date=None,
days_limit=None, level=None):
if level == 'warning':
publisher.warning(__csv2string([ip, user_agent, method, q_ip,
announce_date, days_limit, level]))
elif level == 'error':
publisher.error(__csv2string([ip, user_agent, method, q_ip,
announce_date, days_limit, level]))
else:
publisher.info(__csv2string([ip, user_agent, method, q_ip,
announce_date, days_limit, level]))
def search_phone(message):
paste = Paste.Paste(message)
content = paste.get_p_content()
# regex to find phone numbers, may raise many false positives (shalt thou seek optimization, upgrading is required)
reg_phone = re.compile(r"(\+\d{1,4}(\(\d\))?\d?|0\d?)(\d{6,8}|([-/\. ]{1}\d{2,3}){3,4})")
# list of the regex results in the Paste, may be null
results = reg_phone.findall(content)
# if the list is greater than 4, we consider the Paste may contain a list of phone numbers
if len(results) > 4:
print results
publisher.warning("{} contains PID (phone numbers)".format(paste.p_name))
def search_phone(message):
paste = Paste.Paste(message)
content = paste.get_p_content()
# regex to find phone numbers, may raise many false positives (shalt thou seek optimization, upgrading is required)
reg_phone = re.compile(r'(\+\d{1,4}(\(\d\))?\d?|0\d?)(\d{6,8}|([-/\. ]{1}\d{2,3}){3,4})')
# list of the regex results in the Paste, may be null
results = reg_phone.findall(content)
# if the list is greater than 4, we consider the Paste may contain a list of phone numbers
if len(results) > 4:
print results
publisher.warning('{} contains PID (phone numbers)'.format(paste.p_name))
def sendEmailNotification(recipient, alert_name, content):
if not os.path.exists(configfile):
raise Exception('Unable to find the configuration file. \
Did you set environment variables? \
Or activate the virtualenv?')
cfg = configparser.ConfigParser()
cfg.read(configfile)
sender = cfg.get("Notifications", "sender")
sender_host = cfg.get("Notifications", "sender_host")
sender_port = cfg.getint("Notifications", "sender_port")
sender_pw = cfg.get("Notifications", "sender_pw")
if sender_pw == 'None':
sender_pw = None
# raise an exception if any of these is None
if (sender is None or sender_host is None or sender_port is None):
raise Exception(
'SMTP configuration (host, port, sender) is missing or incomplete!'
)
try:
if sender_pw is not None:
try:
smtp_server = smtplib.SMTP(sender_host, sender_port)
smtp_server.starttls()
except smtplib.SMTPNotSupportedError:
print("The server does not support the STARTTLS extension.")
smtp_server = smtplib.SMTP_SSL(sender_host, sender_port)
smtp_server.ehlo()
smtp_server.login(sender, sender_pw)
else:
smtp_server = smtplib.SMTP(sender_host, sender_port)
mime_msg = MIMEMultipart()
mime_msg['From'] = sender
mime_msg['To'] = recipient
mime_msg['Subject'] = "AIL Framework " + alert_name + " Alert"
body = content
mime_msg.attach(MIMEText(body, 'plain'))
smtp_server.sendmail(sender, recipient, mime_msg.as_string())
smtp_server.quit()
print('Send notification ' + alert_name + ' to ' + recipient)
except Exception as err:
traceback.print_tb(err.__traceback__)
publisher.warning(err)
def main():
publisher.port = 6380
publisher.channel = "Script"
config_section = 'DomClassifier'
p = Process(config_section)
addr_dns = p.config.get("DomClassifier", "dns")
publisher.info("""ZMQ DomainClassifier is Running""")
c = DomainClassifier.domainclassifier.Extract(rawtext="",
nameservers=[addr_dns])
cc = p.config.get("DomClassifier", "cc")
cc_tld = p.config.get("DomClassifier", "cc_tld")
while True:
try:
message = p.get_from_set()
if message is not None:
PST = Paste.Paste(message)
else:
publisher.debug("Script DomClassifier is idling 1s")
time.sleep(1)
continue
paste = PST.get_p_content()
mimetype = PST._get_p_encoding()
if mimetype == "text/plain":
c.text(rawtext=paste)
c.potentialdomain()
c.validdomain(rtype=['A'], extended=True)
localizeddomains = c.include(expression=cc_tld)
if localizeddomains:
print(localizeddomains)
publisher.warning(
'DomainC;{};{};{};Checked {} located in {};{}'.format(
PST.p_source, PST.p_date, PST.p_name,
localizeddomains, cc_tld, PST.p_path))
localizeddomains = c.localizedomain(cc=cc)
if localizeddomains:
print(localizeddomains)
publisher.warning(
'DomainC;{};{};{};Checked {} located in {};{}'.format(
PST.p_source, PST.p_date, PST.p_name,
localizeddomains, cc, PST.p_path))
except IOError:
print("CRC Checksum Failed on :", PST.p_path)
publisher.error('Duplicate;{};{};{};CRC Checksum Failed'.format(
PST.p_source, PST.p_date, PST.p_name))
def test_publisher(self):
for i in range(0, 21):
if i % 2 == 0:
publisher.info('test' + str(i))
elif i % 3 == 0:
publisher.warning('test' + str(i))
elif i % 5 == 0:
publisher.error('test' + str(i))
elif i % 7 == 0:
publisher.critical('test' + str(i))
else:
publisher.debug('test' + str(i))
time.sleep(1)
def test_publisher(self):
for i in range(0, 21):
if i % 2 == 0:
publisher.info('test' + str(i))
elif i % 3 == 0:
publisher.warning('test' + str(i))
elif i % 5 == 0:
publisher.error('test' + str(i))
elif i % 7 == 0:
publisher.critical('test' + str(i))
else:
publisher.debug('test' + str(i))
time.sleep(1)
def analyse(url, path):
faup.decode(url)
url_parsed = faup.get()
pprint.pprint(url_parsed)
## TODO: # FIXME: remove me
try:
resource_path = url_parsed['resource_path'].encode()
except:
resource_path = url_parsed['resource_path']
## TODO: # FIXME: remove me
try:
query_string = url_parsed['query_string'].encode()
except:
query_string = url_parsed['query_string']
result_path = {'sqli': False}
result_query = {'sqli': False}
if resource_path is not None:
result_path = pylibinjection.detect_sqli(resource_path)
print("path is sqli : {0}".format(result_path))
if query_string is not None:
result_query = pylibinjection.detect_sqli(query_string)
print("query is sqli : {0}".format(result_query))
if result_path['sqli'] is True or result_query['sqli'] is True:
paste = Paste.Paste(path)
print("Detected (libinjection) SQL in URL: ")
print(urllib.request.unquote(url))
to_print = 'LibInjection;{};{};{};{};{}'.format(
paste.p_source, paste.p_date, paste.p_name, "Detected SQL in URL",
paste.p_rel_path)
publisher.warning(to_print)
#Send to duplicate
p.populate_set_out(path, 'Duplicate')
msg = 'infoleak:automatic-detection="sql-injection";{}'.format(path)
p.populate_set_out(msg, 'Tags')
#statistics
## TODO: # FIXME: remove me
try:
tld = url_parsed['tld'].decode()
except:
tld = url_parsed['tld']
if tld is not None:
date = datetime.datetime.now().strftime("%Y%m")
server_statistics.hincrby('SQLInjection_by_tld:' + date, tld, 1)
def sendEmailNotification(recipient, alert_name, content):
sender = config_loader.get_config_str("Notifications", "sender")
sender_user = config_loader.get_config_str("Notifications", "sender_user")
sender_host = config_loader.get_config_str("Notifications", "sender_host")
sender_port = config_loader.get_config_int("Notifications", "sender_port")
sender_pw = config_loader.get_config_str("Notifications", "sender_pw")
if sender_pw == 'None':
sender_pw = None
# raise an exception if any of these is None
if (sender is None or sender_host is None or sender_port is None):
raise Exception(
'SMTP configuration (host, port, sender) is missing or incomplete!'
)
try:
if sender_pw is not None:
try:
smtp_server = smtplib.SMTP(sender_host, sender_port)
smtp_server.starttls()
except smtplib.SMTPNotSupportedError:
print("The server does not support the STARTTLS extension.")
smtp_server = smtplib.SMTP_SSL(sender_host, sender_port)
smtp_server.ehlo()
if sender_user is not None:
smtp_server.login(sender_user, sender_pw)
else:
smtp_server.login(sender, sender_pw)
else:
smtp_server = smtplib.SMTP(sender_host, sender_port)
mime_msg = MIMEMultipart()
mime_msg['From'] = sender
mime_msg['To'] = recipient
mime_msg['Subject'] = "AIL Framework " + alert_name + " Alert"
body = content
mime_msg.attach(MIMEText(body, 'plain'))
smtp_server.sendmail(sender, recipient, mime_msg.as_string())
smtp_server.quit()
print('Send notification ' + alert_name + ' to ' + recipient)
except Exception as err:
traceback.print_tb(err.__traceback__)
publisher.warning(err)
def analyse(url, path):
faup.decode(url)
url_parsed = faup.get()
pprint.pprint(url_parsed)
## TODO: # FIXME: remove me
try:
resource_path = url_parsed['resource_path'].encode()
except:
resource_path = url_parsed['resource_path']
## TODO: # FIXME: remove me
try:
query_string = url_parsed['query_string'].encode()
except:
query_string = url_parsed['query_string']
result_path = {'sqli' : False}
result_query = {'sqli' : False}
if resource_path is not None:
result_path = pylibinjection.detect_sqli(resource_path)
print("path is sqli : {0}".format(result_path))
if query_string is not None:
result_query = pylibinjection.detect_sqli(query_string)
print("query is sqli : {0}".format(result_query))
if result_path['sqli'] is True or result_query['sqli'] is True:
paste = Paste.Paste(path)
print("Detected (libinjection) SQL in URL: ")
print(urllib.request.unquote(url))
to_print = 'LibInjection;{};{};{};{};{}'.format(paste.p_source, paste.p_date, paste.p_name, "Detected SQL in URL", paste.p_rel_path)
publisher.warning(to_print)
#Send to duplicate
p.populate_set_out(path, 'Duplicate')
msg = 'infoleak:automatic-detection="sql-injection";{}'.format(path)
p.populate_set_out(msg, 'Tags')
#statistics
## TODO: # FIXME: remove me
try:
tld = url_parsed['tld'].decode()
except:
tld = url_parsed['tld']
if tld is not None:
date = datetime.datetime.now().strftime("%Y%m")
server_statistics.hincrby('SQLInjection_by_tld:'+date, tld, 1)
def search_phone(message):
paste = Paste.Paste(message)
content = paste.get_p_content()
# regex to find phone numbers, may raise many false positives (shalt thou seek optimization, upgrading is required)
reg_phone = re.compile(r'(\+\d{1,4}(\(\d\))?\d?|0\d?)(\d{6,8}|([-/\. ]{1}\d{2,3}){3,4})')
# list of the regex results in the Paste, may be null
results = reg_phone.findall(content)
# if the list is greater than 4, we consider the Paste may contain a list of phone numbers
if len(results) > 4:
print results
publisher.warning('{} contains PID (phone numbers)'.format(paste.p_name))
#send to Browse_warning_paste
p.populate_set_out('phone;{}'.format(message), 'BrowseWarningPaste')
#Send to duplicate
p.populate_set_out(message, 'Duplicate')
def main():
publisher.port = 6380
publisher.channel = "Script"
config_section = 'DomClassifier'
p = Process(config_section)
addr_dns = p.config.get("DomClassifier", "dns")
publisher.info("""ZMQ DomainClassifier is Running""")
c = DomainClassifier.domainclassifier.Extract(rawtext="", nameservers=[addr_dns])
cc = p.config.get("DomClassifier", "cc")
cc_tld = p.config.get("DomClassifier", "cc_tld")
while True:
try:
message = p.get_from_set()
if message is not None:
PST = Paste.Paste(message)
else:
publisher.debug("Script DomClassifier is idling 1s")
time.sleep(1)
continue
paste = PST.get_p_content()
mimetype = PST._get_p_encoding()
if mimetype == "text/plain":
c.text(rawtext=paste)
c.potentialdomain()
c.validdomain(rtype=['A'], extended=True)
localizeddomains = c.include(expression=cc_tld)
if localizeddomains:
print(localizeddomains)
publisher.warning('DomainC;{};{};{};Checked {} located in {};{}'.format(
PST.p_source, PST.p_date, PST.p_name, localizeddomains, cc_tld, PST.p_path))
localizeddomains = c.localizedomain(cc=cc)
if localizeddomains:
print(localizeddomains)
publisher.warning('DomainC;{};{};{};Checked {} located in {};{}'.format(
PST.p_source, PST.p_date, PST.p_name, localizeddomains, cc, PST.p_path))
except IOError:
print("CRC Checksum Failed on :", PST.p_path)
publisher.error('Duplicate;{};{};{};CRC Checksum Failed'.format(
PST.p_source, PST.p_date, PST.p_name))
def search_cve(message):
filepath, count = message.split()
paste = Paste.Paste(filepath)
content = paste.get_p_content()
# regex to find CVE
reg_cve = re.compile(r'(CVE-)[1-2]\d{1,4}-\d{1,5}')
# list of the regex results in the Paste, may be null
results = set(reg_cve.findall(content))
# if the list is greater than 2, we consider the Paste may contain a list of cve
if len(results) > 0:
print('{} contains CVEs'.format(paste.p_name))
publisher.warning('{} contains CVEs'.format(paste.p_name))
#send to Browse_warning_paste
p.populate_set_out('cve;{}'.format(filepath), 'BrowseWarningPaste')
#Send to duplicate
p.populate_set_out(filepath, 'Duplicate')
def search_cve(message):
filepath, count = message.split()
paste = Paste.Paste(filepath)
content = paste.get_p_content()
# regex to find CVE
reg_cve = re.compile(r'(CVE-)[1-2]\d{1,4}-\d{1,5}')
# list of the regex results in the Paste, may be null
results = set(reg_cve.findall(content))
# if the list is greater than 2, we consider the Paste may contain a list of cve
if len(results) > 0:
print('{} contains CVEs'.format(paste.p_name))
publisher.warning('{} contains CVEs'.format(paste.p_name))
#send to Browse_warning_paste
p.populate_set_out('cve;{}'.format(filepath), 'BrowseWarningPaste')
#Send to duplicate
p.populate_set_out(filepath, 'Duplicate')
def search_phone(message):
paste = Paste.Paste(message)
content = paste.get_p_content()
# regex to find phone numbers, may raise many false positives (shalt thou seek optimization, upgrading is required)
reg_phone = re.compile(r'(\+\d{1,4}(\(\d\))?\d?|0\d?)(\d{6,8}|([-/\. ]{1}\d{2,3}){3,4})')
# list of the regex results in the Paste, may be null
results = reg_phone.findall(content)
# if the list is greater than 4, we consider the Paste may contain a list of phone numbers
if len(results) > 4 :
print results
publisher.warning('{} contains PID (phone numbers)'.format(paste.p_name))
if __name__ == '__main__':
# If you wish to use an other port of channel, do not forget to run a subscriber accordingly (see launch_logs.sh)
# Port of the redis instance used by pubsublogger
publisher.port = 6380
# Script is the default channel used for the modules.
publisher.channel = 'Script'
# Section name in bin/packages/modules.cfg
config_section = 'Phone'
# Setup the I/O queues
p = Process(config_section)
# Sent to the logging a description of the module
publisher.info("Run Phone module")
# Endless loop getting messages from the input queue
while True:
# Get one message from the input queue
message = p.get_from_set()
if message is None:
publisher.debug("{} queue is empty, waiting".format(config_section))
time.sleep(1)
continue
# Do something with the message from the queue
search_phone(message)
def analyse(url, path):
faup.decode(url)
url_parsed = faup.get()
resource_path = url_parsed['resource_path']
query_string = url_parsed['query_string']
result_path = 0
result_query = 0
if resource_path is not None:
result_path = is_sql_injection(resource_path.decode('utf8'))
if query_string is not None:
result_query = is_sql_injection(query_string.decode('utf8'))
if (result_path > 0) or (result_query > 0):
paste = Paste.Paste(path)
if (result_path > 1) or (result_query > 1):
print("Detected SQL in URL: ")
print(urllib.request.unquote(url))
to_print = 'SQLInjection;{};{};{};{};{}'.format(
paste.p_source, paste.p_date, paste.p_name,
"Detected SQL in URL", paste.p_path)
publisher.warning(to_print)
#Send to duplicate
p.populate_set_out(path, 'Duplicate')
#send to Browse_warning_paste
p.populate_set_out('sqlinjection;{}'.format(path), 'alertHandler')
msg = 'infoleak:automatic-detection="sql-injection";{}'.format(
path)
p.populate_set_out(msg, 'Tags')
else:
print("Potential SQL injection:")
print(urllib.request.unquote(url))
to_print = 'SQLInjection;{};{};{};{};{}'.format(
paste.p_source, paste.p_date, paste.p_name,
"Potential SQL injection", paste.p_path)
publisher.info(to_print)
def search_api_key(message):
filename, score = message.split()
paste = Paste.Paste(filename)
content = paste.get_p_content()
aws_access_key = regex_aws_access_key.findall(content)
aws_secret_key = regex_aws_secret_key.findall(content)
google_api_key = regex_google_api_key.findall(content)
if(len(aws_access_key) > 0 or len(aws_secret_key) > 0 or len(google_api_key) > 0):
to_print = 'ApiKey;{};{};{};'.format(
paste.p_source, paste.p_date, paste.p_name)
if(len(google_api_key) > 0):
print('found google api key')
print(to_print)
publisher.warning('{}Checked {} found Google API Key;{}'.format(
to_print, len(google_api_key), paste.p_path))
msg = 'infoleak:automatic-detection="google-api-key";{}'.format(filename)
p.populate_set_out(msg, 'Tags')
if(len(aws_access_key) > 0 or len(aws_secret_key) > 0):
print('found AWS key')
print(to_print)
total = len(aws_access_key) + len(aws_secret_key)
publisher.warning('{}Checked {} found AWS Key;{}'.format(
to_print, total, paste.p_path))
msg = 'infoleak:automatic-detection="aws-key";{}'.format(filename)
p.populate_set_out(msg, 'Tags')
msg = 'infoleak:automatic-detection="api-key";{}'.format(filename)
p.populate_set_out(msg, 'Tags')
msg = 'apikey;{}'.format(filename)
p.populate_set_out(msg, 'alertHandler')
#Send to duplicate
p.populate_set_out(filename, 'Duplicate')
def downloadURL(url, filename):
"""
Inconditianilly download the URL in a temporary directory.
When finished, the file is moved in the real directory.
Like this an other process will not attempt to extract an inclomplete file.
"""
path_temp_bviewfile = os.path.join(c.raw_data, c.bview_dir, 'tmp', filename)
path_bviewfile = os.path.join(c.raw_data, c.bview_dir, filename)
try:
f = urlopen(url)
except:
return False
if f.getcode() != 200:
publisher.warning('{} unavailable, code: {}'.format(url, f.getcode()))
return False
try:
with open(path_temp_bviewfile, 'w') as outfile:
outfile.write(f.read())
os.rename(path_temp_bviewfile, path_bviewfile)
except:
os.remove(path_temp_bviewfile)
return False
return True
def search_geolocation(message):
paste = Paste.Paste(message)
content = paste.get_p_content()
# Load Geoparser
geo = Geoparser()
geolocation = geo.geoparse(message)
# regex ton find latitude and longitude
reg_lat = re.compile(r'(\'lat\': \'([-\d.]+)\',)')
reg_lon = re.compile(r'(\'lon\': \'([-\d.]+)\',)')
#lat = set(reg_lat.findall(content))
#lon = set(reg_lat.findall(content))
lat = reg_lat.search(message).group(2)
lon = reg_lon.search(message).group(2)
print('latitude: {}'.format(lat))
print('longitude: {}'.format(lon))
print('{} text geolocation'.format(paste.p_name))
publisher.warning('{} contains geolocation'.format(paste.p_name))
def downloadURL(url, filename):
"""
Inconditianilly download the URL in a temporary directory.
When finished, the file is moved in the real directory.
Like this an other process will not attempt to extract an inclomplete file.
"""
path_temp_bviewfile = os.path.join(c.raw_data, c.bview_dir, 'tmp', filename)
path_bviewfile = os.path.join(c.raw_data, c.bview_dir, filename)
try:
f = urllib.urlopen(url)
except:
return False
if f.getcode() != 200:
publisher.warning('{} unavailable, code: {}'.format(url, f.getcode()))
return False
try:
with open(path_temp_bviewfile, 'w') as outfile:
outfile.write(f.read())
os.rename(path_temp_bviewfile, path_bviewfile)
except:
os.remove(path_temp_bviewfile)
return False
return True
def db_import(filename, day):
routing_db = get_redis_connector()
with open(filename, 'r') as f:
entry = ''
pipeline = routing_db.pipeline()
i = 0
for line in f:
# End of block, extracting the information
if line == '\n':
i += 1
parsed = re.findall('(?:ASPATH|PREFIX): ([^\n{]*)', entry)
try:
block = parsed[0].strip()
# RIPE-NCC-RIS BGP IPv6 Anchor Prefix @RRC00
# RIPE-NCC-RIS BGP Anchor Prefix @ rrc00 - RIPE NCC
if block in [
'2001:7fb:ff00::/48', '84.205.80.0/24',
'2001:7fb:fe00::/48', '84.205.64.0/24'
]:
asn = 12654
else:
asn = int(parsed[1].split()[-1].strip())
pipeline.hset(block, day, asn)
except:
# FIXME: check the cause of the exception
publisher.warning(entry)
entry = ''
if i % 10000 == 0:
pipeline.execute()
pipeline = routing_db.pipeline()
else:
# append the line to the current block.
entry += line
pipeline.execute()
publisher.info('{f} finished, {nb} entries impported.'.format(
f=filename, nb=i))
def main():
"""Main Function"""
# CONFIG #
cfg = ConfigParser.ConfigParser()
cfg.read(configfile)
# REDIS #
r_serv = redis.StrictRedis(
host=cfg.get("Redis_Queues", "host"),
port=cfg.getint("Redis_Queues", "port"),
db=cfg.getint("Redis_Queues", "db"),
)
# LOGGING #
publisher.channel = "Queuing"
# ZMQ #
channel = cfg.get("PubSub_Global", "channel")
subscriber_name = "indexer"
Sub = ZMQ_PubSub.ZMQSub(configfile, "PubSub_Global", channel, subscriber_name)
publisher.info("""Suscribed to channel {0}""".format(channel))
# Until the service is requested to be shutdown, the service
# will get the data from the global ZMQ queue and buffer it in Redis.
while True:
Sub.get_and_lpush(r_serv)
if r_serv.sismember("SHUTDOWN_FLAGS", "Indexer_Q"):
r_serv.srem("SHUTDOWN_FLAGS", "Indexer_Q")
print "Shutdown Flag Up: Terminating"
publisher.warning("Shutdown Flag Up: Terminating.")
break
def check_all_iban(l_iban, paste, filename):
nb_valid_iban = 0
for iban in l_iban:
iban = iban[0]+iban[1]+iban[2]
iban = ''.join(e for e in iban if e.isalnum())
#iban = iban.upper()
res = iban_regex_verify.findall(iban)
date = datetime.datetime.now().strftime("%Y%m")
if res:
print('checking '+iban)
if is_valid_iban(iban):
print('------')
nb_valid_iban = nb_valid_iban + 1
server_statistics.hincrby('iban_by_country:'+date, iban[0:2], 1)
if(nb_valid_iban > 0):
to_print = 'Iban;{};{};{};'.format(paste.p_source, paste.p_date, paste.p_name)
publisher.warning('{}Checked found {} IBAN;{}'.format(
to_print, nb_valid_iban, paste.p_rel_path))
msg = 'infoleak:automatic-detection="iban";{}'.format(filename)
p.populate_set_out(msg, 'Tags')
#Send to duplicate
p.populate_set_out(filename, 'Duplicate')
if message is not None:
filename, score = message.split()
paste = Paste.Paste(filename)
content = paste.get_p_content()
all_cards = re.findall(regex, content)
if len(all_cards) > 0:
print 'All matching', all_cards
creditcard_set = set([])
for card in all_cards:
clean_card = re.sub('[^0-9]', '', card)
if lib_refine.is_luhn_valid(clean_card):
print clean_card, 'is valid'
creditcard_set.add(clean_card)
paste.__setattr__(channel, creditcard_set)
paste.save_attribute_redis(channel, creditcard_set)
pprint.pprint(creditcard_set)
to_print = 'CreditCard;{};{};{};'.format(
paste.p_source, paste.p_date, paste.p_name)
if (len(creditcard_set) > 0):
publisher.warning('{}Checked {} valid number(s)'.format(
to_print, len(creditcard_set)))
else:
publisher.info('{}CreditCard related'.format(to_print))
else:
publisher.debug("Script creditcard is idling 1m")
print 'Sleeping'
time.sleep(10)
def search_key(paste):
content = paste.get_p_content()
find = False
if '-----BEGIN PGP MESSAGE-----' in content:
publisher.warning('{} has a PGP enc message'.format(paste.p_name))
msg = 'infoleak:automatic-detection="pgp-message";{}'.format(message)
p.populate_set_out(msg, 'Tags')
find = True
if '-----BEGIN CERTIFICATE-----' in content:
publisher.warning('{} has a certificate message'.format(paste.p_name))
msg = 'infoleak:automatic-detection="certificate";{}'.format(message)
p.populate_set_out(msg, 'Tags')
find = True
if '-----BEGIN RSA PRIVATE KEY-----' in content:
publisher.warning('{} has a RSA private key message'.format(paste.p_name))
print('rsa private key message found')
msg = 'infoleak:automatic-detection="rsa-private-key";{}'.format(message)
p.populate_set_out(msg, 'Tags')
find = True
if '-----BEGIN PRIVATE KEY-----' in content:
publisher.warning('{} has a private key message'.format(paste.p_name))
print('private key message found')
msg = 'infoleak:automatic-detection="private-key";{}'.format(message)
p.populate_set_out(msg, 'Tags')
find = True
if '-----BEGIN ENCRYPTED PRIVATE KEY-----' in content:
publisher.warning('{} has an encrypted private key message'.format(paste.p_name))
print('encrypted private key message found')
msg = 'infoleak:automatic-detection="encrypted-private-key";{}'.format(message)
p.populate_set_out(msg, 'Tags')
find = True
if '-----BEGIN OPENSSH PRIVATE KEY-----' in content:
publisher.warning('{} has an openssh private key message'.format(paste.p_name))
print('openssh private key message found')
msg = 'infoleak:automatic-detection="private-ssh-key";{}'.format(message)
p.populate_set_out(msg, 'Tags')
find = True
if '-----BEGIN OpenVPN Static key V1-----' in content:
publisher.warning('{} has an openssh private key message'.format(paste.p_name))
print('OpenVPN Static key message found')
msg = 'infoleak:automatic-detection="vpn-static-key";{}'.format(message)
p.populate_set_out(msg, 'Tags')
find = True
if '-----BEGIN DSA PRIVATE KEY-----' in content:
publisher.warning('{} has a dsa private key message'.format(paste.p_name))
msg = 'infoleak:automatic-detection="dsa-private-key";{}'.format(message)
p.populate_set_out(msg, 'Tags')
find = True
if '-----BEGIN EC PRIVATE KEY-----' in content:
publisher.warning('{} has an ec private key message'.format(paste.p_name))
msg = 'infoleak:automatic-detection="ec-private-key";{}'.format(message)
p.populate_set_out(msg, 'Tags')
find = True
if '-----BEGIN PGP PRIVATE KEY BLOCK-----' in content:
publisher.warning('{} has a pgp private key block message'.format(paste.p_name))
msg = 'infoleak:automatic-detection="pgp-private-key";{}'.format(message)
p.populate_set_out(msg, 'Tags')
find = True
if find :
#Send to duplicate
p.populate_set_out(message, 'Duplicate')
#send to Browse_warning_paste
msg = ('keys;{}'.format(message))
print(message)
p.populate_set_out( msg, 'alertHandler')
updated_descrs = 0
for asn, descr in data:
all_descrs = r.hgetall(asn)
if len(all_descrs) == 0:
p.hset(asn, timestamp, descr)
publisher.debug('New asn: {asn}'.format(asn=asn))
new_asns += 1
else:
dates = sorted(all_descrs.keys())
last_descr = all_descrs[dates[-1]]
if descr != last_descr:
p.hset(asn, timestamp, descr)
msg = 'New description for {asn}. Was {old}, is {new}'.format(
asn=asn, old=last_descr, new=descr)
publisher.info(msg)
updated_descrs += 1
p.execute()
msg = '===== Import finished: {new}, new ASNs:{nb}, Updated:{up} ====='.format(
new=timestamp, nb=new_asns, up=updated_descrs)
publisher.info(msg)
if args.not_new:
break
else:
newfile = False
try:
newfile = fetch('http://www.cidr-report.org/as2.0/autnums.html', args.directory)
except:
publisher.warning('Exception in fetching!')
if not newfile:
time.sleep(sleep_timer)
try:
l = client.lookup(ip, qType='IP')
except ipaddress.AddressValueError:
continue
cc = getattr(l, 'cc')
asn = getattr(l, 'asn')
# EU is not an official ISO 3166 code (but used by RIPE
# IP allocation)
if cc is not None and cc != "EU":
print hostl, asn, cc, \
pycountry.countries.get(alpha2=cc).name
if cc == cc_critical:
publisher.warning(
'Url;{};{};{};Detected {} {}'.format(
PST.p_source, PST.p_date, PST.p_name,
hostl, cc))
else:
print hostl, asn, cc
A_values = lib_refine.checking_A_record(r_serv2,
domains_list)
if A_values[0] >= 1:
PST.__setattr__(channel, A_values)
PST.save_attribute_redis(channel, (A_values[0],
list(A_values[1])))
pprint.pprint(A_values)
publisher.info('Url;{};{};{};Checked {} URL'.format(
PST.p_source, PST.p_date, PST.p_name, A_values[0]))
prec_filename = filename
sites= re.findall(regex_web, content) #Use to count occurences
sites_set = set(re.findall(regex_web, content))
message = 'Checked {} credentials found.'.format(len(creds))
if sites_set:
message += ' Related websites: {}'.format( (', '.join(sites_set)) )
to_print = 'Credential;{};{};{};{};{}'.format(paste.p_source, paste.p_date, paste.p_name, message, paste.p_rel_path)
print('\n '.join(creds))
#num of creds above tresh, publish an alert
if len(creds) > criticalNumberToAlert:
print("========> Found more than 10 credentials in this file : {}".format( filepath ))
publisher.warning(to_print)
#Send to duplicate
p.populate_set_out(filepath, 'Duplicate')
msg = 'infoleak:automatic-detection="credential";{}'.format(filepath)
p.populate_set_out(msg, 'Tags')
#Put in form, count occurences, then send to moduleStats
creds_sites = {}
site_occurence = re.findall(regex_site_for_stats, content)
for site in site_occurence:
site_domain = site[1:-1]
if site_domain in creds_sites.keys():
creds_sites[site_domain] += 1
else:
creds_sites[site_domain] = 1
def main():
"""Main Function"""
# CONFIG #
cfg = ConfigParser.ConfigParser()
cfg.read(configfile)
# REDIS #
r_serv = redis.StrictRedis(
host = cfg.get("Redis_Queues", "host"),
port = cfg.getint("Redis_Queues", "port"),
db = cfg.getint("Redis_Queues", "db"))
r_serv1 = redis.StrictRedis(
host = cfg.get("Redis_Data_Merging", "host"),
port = cfg.getint("Redis_Data_Merging", "port"),
db = cfg.getint("Redis_Data_Merging", "db"))
r_serv2 = redis.StrictRedis(
host = cfg.get("Redis_Cache", "host"),
port = cfg.getint("Redis_Cache", "port"),
db = cfg.getint("Redis_Cache", "db"))
# LOGGING #
publisher.channel = "Script"
# ZMQ #
Sub = ZMQ_PubSub.ZMQSub(configfile,"PubSub_Categ", "mails_categ", "emails")
# FUNCTIONS #
publisher.info("Suscribed to channel mails_categ")
message = Sub.get_msg_from_queue(r_serv)
prec_filename = None
email_regex = "[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,6}"
while True:
try:
if message != None:
channel, filename, word, score = message.split()
if prec_filename == None or filename != prec_filename:
PST = P.Paste(filename)
MX_values = lib_refine.checking_MX_record(r_serv2, PST.get_regex(email_regex))
if MX_values[0] >= 1:
PST.__setattr__(channel, MX_values)
PST.save_attribute_redis(r_serv1, channel, (MX_values[0], list(MX_values[1])))
pprint.pprint(MX_values)
if MX_values[0] > 10:
publisher.warning('{0};{1};{2};{3};{4}'.format("Mails", PST.p_source, PST.p_date, PST.p_name,"Checked "+ str(MX_values[0])+ " e-mails" ))
else:
publisher.info('{0};{1};{2};{3};{4}'.format("Mails", PST.p_source, PST.p_date, PST.p_name,"Checked " str(MX_values[0])+ " e-mail(s)" ))
prec_filename = filename
else:
if r_serv.sismember("SHUTDOWN_FLAGS", "Mails"):
r_serv.srem("SHUTDOWN_FLAGS", "Mails")
print "Shutdown Flag Up: Terminating"
publisher.warning("Shutdown Flag Up: Terminating.")
break
publisher.debug("Script Mails is Idling 10s")
time.sleep(10)
message = Sub.get_msg_from_queue(r_serv)
except dns.exception.Timeout:
print "dns.exception.Timeout"
pass
creditcard_set = set([])
PST = Paste.Paste(filename)
for x in PST.get_regex(creditcard_regex):
if lib_refine.is_luhn_valid(x):
creditcard_set.add(x)
PST.__setattr__(channel, creditcard_set)
PST.save_attribute_redis(channel, creditcard_set)
pprint.pprint(creditcard_set)
to_print = 'CreditCard;{};{};{};'.format(
PST.p_source, PST.p_date, PST.p_name)
if (len(creditcard_set) > 0):
publisher.critical('{}Checked {} valid number(s)'.format(
to_print, len(creditcard_set)))
else:
publisher.info('{}CreditCard related'.format(to_print))
prec_filename = filename
else:
if h.redis_queue_shutdown():
print "Shutdown Flag Up: Terminating"
publisher.warning("Shutdown Flag Up: Terminating.")
break
publisher.debug("Script creditcard is idling 1m")
time.sleep(60)
message = h.redis_rpop()
for x in PST.get_regex(url_regex):
# Extracting url with regex
url, s, credential, subdomain, domain, host, port, \
resource_path, query_string, f1, f2, f3, f4 = x
domains_list.append(domain)
urls.append(url)
# Saving the list of extracted onion domains.
PST.__setattr__(channel, domains_list)
PST.save_attribute_redis(channel, domains_list)
to_print = 'Onion;{};{};{};'.format(PST.p_source, PST.p_date,
PST.p_name)
if len(domains_list) > 0:
publisher.warning('{}Detected {} .onion(s)'.format(
to_print, len(domains_list)))
now = datetime.datetime.now()
path = os.path.join('onions', str(now.year).zfill(4),
str(now.month).zfill(2),
str(now.day).zfill(2),
str(int(time.mktime(now.utctimetuple()))))
to_print = 'Onion;{};{};{};'.format(PST.p_source,
PST.p_date,
PST.p_name)
for url in fetch(p, r_cache, urls, domains_list, path):
publisher.warning('{}Checked {}'.format(to_print, url))
else:
publisher.info('{}Onion related'.format(to_print))
prec_filename = filename
else:
def search_gpg(message):
paste = Paste.Paste(message)
content = paste.get_p_content()
if '-----BEGIN PGP MESSAGE-----' in content:
publisher.warning('{} has a PGP enc message'.format(paste.p_name))