def feeder(message, count=0):
if flag_the_hive or flag_misp:
tag, path = message.split(';')
## FIXME: remove it
if PASTES_FOLDER not in path:
path = os.path.join(PASTES_FOLDER, path)
try:
paste = Paste.Paste(path)
except FileNotFoundError:
if count < 10:
r_serv_db.zincrby('mess_not_saved_export', message, 1)
return 0
else:
r_serv_db.zrem('mess_not_saved_export', message)
print('Error: {} do not exist, tag= {}'.format(path, tag))
return 0
source = '/'.join(paste.p_path.split('/')[-6:])
if HiveApi != False:
if int(r_serv_db.get('hive:auto-alerts')) == 1:
whitelist_hive = r_serv_db.scard('whitelist_hive')
if r_serv_db.sismember('whitelist_hive', tag):
create_the_hive_alert(source, path, tag)
else:
print('hive, auto alerts creation disable')
if flag_misp:
if int(r_serv_db.get('misp:auto-events')) == 1:
if r_serv_db.sismember('whitelist_misp', tag):
misp_wrapper.pushToMISP(uuid_ail, path, tag)
else:
print('misp, auto events creation disable')
def compute(self, message):
try:
PST = Paste.Paste(message)
docpath = message.split(" ", -1)[-1]
paste = PST.get_p_content()
self.redis_logger.debug(f"Indexing - {self.indexname}: {docpath}")
# Avoid calculating the index's size at each message
if (time.time() - self.last_refresh > self.TIME_WAIT):
self.last_refresh = time.time()
if self.check_index_size() >= self.INDEX_SIZE_THRESHOLD * (
1000 * 1000):
timestamp = int(time.time())
self.redis_logger.debug(f"Creating new index {timestamp}")
self.indexpath = join(self.baseindexpath, str(timestamp))
self.indexname = str(timestamp)
# update all_index
with open(self.indexRegister_path, "a") as f:
f.write('\n' + str(timestamp))
# create new dir
os.mkdir(self.indexpath)
self.ix = create_in(self.indexpath, self.schema)
if self.indexertype == "whoosh":
indexwriter = self.ix.writer()
indexwriter.update_document(title=docpath,
path=docpath,
content=paste)
indexwriter.commit()
except IOError:
self.redis_logger.debug(f"CRC Checksum Failed on: {PST.p_path}")
self.redis_logger.error(
'Duplicate;{};{};{};CRC Checksum Failed'.format(
PST.p_source, PST.p_date, PST.p_name))
def compute(self, message):
paste = Paste.Paste(message)
content = paste.get_p_content()
# List of the regex results in the Paste, may be null
results = self.REG_PHONE.findall(content)
# If the list is greater than 4, we consider the Paste may contain a list of phone numbers
if len(results) > 4:
self.redis_logger.debug(results)
self.redis_logger.warning('{} contains PID (phone numbers)'.format(
paste.p_name))
msg = 'infoleak:automatic-detection="phone-number";{}'.format(
message)
self.process.populate_set_out(msg, 'Tags')
# Send to duplicate
self.process.populate_set_out(message, 'Duplicate')
stats = {}
for phone_number in results:
try:
x = phonenumbers.parse(phone_number, None)
country_code = x.country_code
if stats.get(country_code) is None:
stats[country_code] = 1
else:
stats[country_code] = stats[country_code] + 1
except:
pass
for country_code in stats:
if stats[country_code] > 4:
self.redis_logger.warning(
'{} contains Phone numbers with country code {}'.
format(paste.p_name, country_code))
def analyse(url, path):
if is_sql_injection(url):
faup.decode(url)
url_parsed = faup.get()
paste = Paste.Paste(path)
print("Detected SQL in URL: ")
print(urllib.request.unquote(url))
to_print = 'SQLInjection;{};{};{};{};{}'.format(paste.p_source, paste.p_date, paste.p_name, "Detected SQL in URL", paste.p_rel_path)
publisher.warning(to_print)
#Send to duplicate
p.populate_set_out(path, 'Duplicate')
msg = 'infoleak:automatic-detection="sql-injection";{}'.format(path)
p.populate_set_out(msg, 'Tags')
#statistics
tld = url_parsed['tld']
if tld is not None:
## TODO: # FIXME: remove me
try:
tld = tld.decode()
except:
pass
date = datetime.datetime.now().strftime("%Y%m")
server_statistics.hincrby('SQLInjection_by_tld:'+date, tld, 1)
def search_phone(message):
paste = Paste.Paste(message)
content = paste.get_p_content()
# regex to find phone numbers, may raise many false positives (shalt thou seek optimization, upgrading is required)
reg_phone = re.compile(r'(\+\d{1,4}(\(\d\))?\d?|0\d?)(\d{6,8}|([-/\. ]{1}\d{2,3}){3,4})')
reg_phone = re.compile(r'(\+\d{1,4}(\(\d\))?\d?|0\d?)(\d{6,8}|([-/\. ]{1}\(?\d{2,4}\)?){3,4})')
# list of the regex results in the Paste, may be null
results = reg_phone.findall(content)
# if the list is greater than 4, we consider the Paste may contain a list of phone numbers
if len(results) > 4:
print results
publisher.warning('{} contains PID (phone numbers)'.format(paste.p_name))
#send to Browse_warning_paste
p.populate_set_out('phone;{}'.format(message), 'BrowseWarningPaste')
#Send to duplicate
p.populate_set_out(message, 'Duplicate')
stats = {}
for phone_number in results:
try:
x = phonenumbers.parse(phone_number, None)
country_code = x.country_code
if stats.get(country_code) is None:
stats[country_code] = 1
else:
stats[country_code] = stats[country_code] + 1
except:
pass
for country_code in stats:
if stats[country_code] > 4:
publisher.warning('{} contains Phone numbers with country code {}'.format(paste.p_name, country_code))
def add_new_object(self, uuid_ail, path, p_source, tag):
self.uuid_ail = uuid_ail
self.path = path
self.p_source = p_source
self.paste = Paste.Paste(path)
self.p_date = self.date_to_str(self.paste.p_date)
self.p_content = self.paste.get_p_content()
self.p_tag = tag
temp = self.paste._get_p_duplicate()
#beautifier
if not temp:
temp = ''
p_duplicate_number = len(temp) if len(temp) >= 0 else 0
to_ret = ""
for dup in temp[:10]:
dup = dup.replace('\'', '\"').replace('(', '[').replace(')', ']')
dup = json.loads(dup)
algo = dup[0]
path = dup[1].split('/')[-6:]
path = '/'.join(path)[:-3] # -3 removes .gz
if algo == 'tlsh':
perc = 100 - int(dup[2])
else:
perc = dup[2]
to_ret += "{}: {} [{}%]\n".format(path, algo, perc)
p_duplicate = to_ret
self.mispObject = AilLeakObject(self.uuid_ail, self.p_source,
self.p_date, self.p_content,
p_duplicate, p_duplicate_number)
def _setValue(self):
try:
#Verify that the module have a paste
if COMPLETE_PASTE_PATH_PER_PID[current_selected_value] is None:
self.label_list[0]._text = "No paste for this module"
for i in range(1, self.num_label):
self.label_list[i]._text = ""
return
paste = Paste.Paste(
COMPLETE_PASTE_PATH_PER_PID[current_selected_value])
old_content = paste.get_p_content()[
0:4000] # Limit number of char to be displayed
#Replace unprintable char by ?
content = ""
for i, c in enumerate(old_content):
if ord(c) > 127: # Used to avoid printing unprintable char
content += '?'
elif c == "\t": # Replace tab by 4 spaces
content += " "
else:
content += c
#Print in the correct label, END or more
to_print = ""
i = 0
for line in content.split("\n"):
if i > self.num_label - 2:
break
self.label_list[i]._text = str(i) + ". " + line.replace(
"\r", "")
i += 1
if i > self.num_label - 2:
self.label_list[i]._text = "- ALL PASTE NOT DISPLAYED -"
i += 1
else:
self.label_list[i]._text = "- END of PASTE -"
i += 1
while i < self.num_label: #Clear out remaining lines
self.label_list[i]._text = ""
i += 1
except OSError as e:
self.label_list[
0]._text = "Error during parsing the filepath. Please, check manually"
self.label_list[1]._text = COMPLETE_PASTE_PATH_PER_PID[
current_selected_value]
for i in range(2, self.num_label):
self.label_list[i]._text = ""
except Exception as e:
self.label_list[
0]._text = "Error while displaying the paste: " + COMPLETE_PASTE_PATH_PER_PID[
current_selected_value]
self.label_list[1]._text = str(e)
for i in range(2, self.num_label):
self.label_list[i]._text = ""
def search_game(message):
#We recover the paste
paste = Paste.Paste(message)
content = paste.get_p_content()
#We open the file with all game word and the stock for all paste found
filetoopen = open("corpus.txt", "r")
filetowrite = open("stock.txt", "a")
count = 0 #Number of game word found in 1 file
for line in filetoopen:
linestrip = line.strip(
) #Must do because it takes all the line and not just the word
reg = re.compile(r'{}'.format(linestrip)) #we create the regex
results = re.findall(reg, content) #we find the occurences
if (len(results) > 0):
count = count + 1
re.purge()
if count > 5:
print results
publisher.warning(
'{} contains Game related conversations+{} occurences of a game related word '
.format(paste.p_name, count)) #warning for the logs
filetowrite.write(
'{} contains Game related conversations+{} occurences of a game related word \n'
.format(paste.p_name, count)) #For stock.txt
to_print = 'GameConv;{};{};{};{} Terms related;{}'.format(
paste.p_source, paste.p_date, paste.p_name, count,
paste.p_path) #To see on the webinterface
publisher.warning(to_print)
filetoopen.close()
filetowrite.close()
def search_api_key(message):
filename, score = message.split()
paste = Paste.Paste(filename)
content = paste.get_p_content()
aws_access_key = regex_aws_access_key.findall(content)
aws_secret_key = regex_aws_secret_key.findall(content)
google_api_key = regex_google_api_key.findall(content)
if (len(aws_access_key) > 0 or len(aws_secret_key) > 0
or len(google_api_key) > 0):
to_print = 'ApiKey;{};{};{};'.format(paste.p_source, paste.p_date,
paste.p_name)
if (len(google_api_key) > 0):
print('found google api key')
print(to_print)
publisher.warning('{}Checked {} found Google API Key;{}'.format(
to_print, len(google_api_key), paste.p_path))
if (len(aws_access_key) > 0 or len(aws_secret_key) > 0):
print('found AWS key')
print(to_print)
total = len(aws_access_key) + len(aws_secret_key)
publisher.warning('{}Checked {} found AWS Key;{}'.format(
to_print, total, paste.p_path))
msg = 'apikey;{}'.format(filename)
p.populate_set_out(msg, 'alertHandler')
#Send to duplicate
p.populate_set_out(filename, 'Duplicate')
def search_key(message):
paste = Paste.Paste(message)
content = paste.get_p_content()
find = False
if '-----BEGIN PGP MESSAGE-----' in content:
publisher.warning('{} has a PGP enc message'.format(paste.p_name))
find = True
if '-----BEGIN CERTIFICATE-----' in content:
publisher.warning('{} has a certificate message'.format(paste.p_name))
find = True
if '-----BEGIN RSA PRIVATE KEY-----' in content:
publisher.warning('{} has a RSA key message'.format(paste.p_name))
find = True
if '-----BEGIN PRIVATE KEY-----' in content:
publisher.warning('{} has a private message'.format(paste.p_name))
find = True
if '-----BEGIN ENCRYPTED PRIVATE KEY-----' in content:
publisher.warning('{} has an encrypted private message'.format(
paste.p_name))
find = True
if find:
#Send to duplicate
p.populate_set_out(message, 'Duplicate')
#send to Browse_warning_paste
p.populate_set_out('keys;{}'.format(message), 'alertHandler')
def analyse(url, path):
faup.decode(url)
url_parsed = faup.get()
resource_path = url_parsed['resource_path']
query_string = url_parsed['query_string']
result_path = 0
result_query = 0
if resource_path is not None:
result_path = is_sql_injection(resource_path)
if query_string is not None:
result_query = is_sql_injection(query_string)
if (result_path > 0) or (result_query > 0):
paste = Paste.Paste(path)
if (result_path > 1) or (result_query > 1):
print "Detected SQL in URL: "
print urllib2.unquote(url)
to_print = 'SQLInjection;{};{};{};{};{}'.format(paste.p_source, paste.p_date, paste.p_name, "Detected SQL in URL", paste.p_path)
publisher.warning(to_print)
#Send to duplicate
p.populate_set_out(path, 'Duplicate')
#send to Browse_warning_paste
p.populate_set_out('sqlinjection;{}'.format(path), 'BrowseWarningPaste')
else:
print "Potential SQL injection:"
print urllib2.unquote(url)
to_print = 'SQLInjection;{};{};{};{}'.format(paste.p_source, paste.p_date, paste.p_name, "Potential SQL injection")
publisher.info(to_print)
def compute(self, message):
# Cast message as paste
paste = Paste.Paste(message)
# Get paste content
content = paste.get_p_content()
# init categories found
is_categ_found = False
# Search for pattern categories in paste content
for categ, pattern in self.categ_items:
found = set(re.findall(pattern, content))
lenfound = len(found)
if lenfound >= self.matchingThreshold:
is_categ_found = True
msg = '{} {}'.format(paste.p_rel_path, lenfound)
self.redis_logger.debug('%s;%s %s' %
(self.module_name, msg, categ))
# Export message to categ queue
self.process.populate_set_out(msg, categ)
self.redis_logger.info(
'Categ;{};{};{};Detected {} as {};{}'.format(
paste.p_source, paste.p_date, paste.p_name, lenfound,
categ, paste.p_rel_path))
if not is_categ_found:
self.redis_logger.debug('No %s found in this paste: %s' %
(self.module_name, paste.p_name))
def analyse(url, path):
faup.decode(url)
url_parsed = faup.get()
pprint.pprint(url_parsed)
resource_path = url_parsed['resource_path']
query_string = url_parsed['query_string']
result_path = {'sqli': False}
result_query = {'sqli': False}
if resource_path is not None:
result_path = pylibinjection.detect_sqli(resource_path)
print("path is sqli : {0}".format(result_path))
if query_string is not None:
result_query = pylibinjection.detect_sqli(query_string)
print("query is sqli : {0}".format(result_query))
if result_path['sqli'] is True or result_query['sqli'] is True:
paste = Paste.Paste(path)
print("Detected (libinjection) SQL in URL: ")
print(urllib.request.unquote(url))
to_print = 'LibInjection;{};{};{};{};{}'.format(
paste.p_source, paste.p_date, paste.p_name, "Detected SQL in URL",
paste.p_path)
publisher.warning(to_print)
#Send to duplicate
p.populate_set_out(path, 'Duplicate')
#send to Browse_warning_paste
p.populate_set_out('sqlinjection;{}'.format(path), 'alertHandler')
msg = 'infoleak:automatic-detection="sql-injection";{}'.format(path)
p.populate_set_out(msg, 'Tags')
def search_ip(message):
paste = Paste.Paste(message)
content = paste.get_p_content()
# regex to find IPs
reg_ip = re.compile(
r'^(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)',
flags=re.MULTILINE)
# list of the regex results in the Paste, may be null
results = reg_ip.findall(content)
matching_ips = []
for res in results:
address = IPv4Address(res)
for network in ip_networks:
if address in network:
matching_ips.append(address)
if len(matching_ips) > 0:
print('{} contains {} IPs'.format(paste.p_name, len(matching_ips)))
publisher.warning('{} contains {} IPs'.format(paste.p_name,
len(matching_ips)))
#Tag message with IP
msg = 'infoleak:automatic-detection="ip";{}'.format(message)
p.populate_set_out(msg, 'Tags')
#Send to duplicate
p.populate_set_out(message, 'Duplicate')
def add_new_object(self, moduleName, path):
self.moduleName = moduleName
self.path = path
self.paste = Paste.Paste(path)
self.p_date = self.date_to_str(self.paste.p_date)
self.p_source = self.paste.p_path
self.p_content = self.paste.get_p_content().decode('utf8')
temp = self.paste._get_p_duplicate()
try:
temp = temp.decode('utf8')
except AttributeError:
pass
#beautifier
temp = json.loads(temp)
self.p_duplicate_number = len(temp) if len(temp) >= 0 else 0
to_ret = ""
for dup in temp[:self.maxDuplicateToPushToMISP]:
algo = dup[0]
path = dup[1].split('/')[-5:]
path = '/'.join(path)[:-3] # -3 removes .gz
perc = dup[2]
to_ret += "{}: {} [{}%]\n".format(path, algo, perc)
self.p_duplicate = to_ret
self.mispObject = AilleakObject(self.moduleName, self.p_source,
self.p_date, self.p_content,
self.p_duplicate,
self.p_duplicate_number)
def search_gpg(message):
paste = Paste.Paste(message)
content = paste.get_p_content()
if '-----BEGIN PGP MESSAGE-----' in content:
publisher.warning('{} has a PGP enc message'.format(paste.p_name))
#Send to duplicate
p.populate_set_out(message, 'Duplicate')
#send to Browse_warning_paste
p.populate_set_out('keys;{}'.format(message), 'BrowseWarningPaste')
def analyse(url, path):
faup.decode(url)
url_parsed = faup.get()
resource_path = url_parsed['resource_path']
query_string = url_parsed['query_string']
result_path = 0
result_query = 0
if resource_path is not None:
## TODO: # FIXME: remove me
try:
resource_path = resource_path.decode()
except:
pass
result_path = is_sql_injection(resource_path)
if query_string is not None:
## TODO: # FIXME: remove me
try:
query_string = query_string.decode()
except:
pass
result_query = is_sql_injection(query_string)
if (result_path > 0) or (result_query > 0):
paste = Paste.Paste(path)
if (result_path > 1) or (result_query > 1):
print("Detected SQL in URL: ")
print(urllib.request.unquote(url))
to_print = 'SQLInjection;{};{};{};{};{}'.format(paste.p_source, paste.p_date, paste.p_name, "Detected SQL in URL", paste.p_rel_path)
publisher.warning(to_print)
#Send to duplicate
p.populate_set_out(path, 'Duplicate')
msg = 'infoleak:automatic-detection="sql-injection";{}'.format(path)
p.populate_set_out(msg, 'Tags')
#statistics
tld = url_parsed['tld']
if tld is not None:
## TODO: # FIXME: remove me
try:
tld = tld.decode()
except:
pass
date = datetime.datetime.now().strftime("%Y%m")
server_statistics.hincrby('SQLInjection_by_tld:'+date, tld, 1)
else:
print("Potential SQL injection:")
print(urllib.request.unquote(url))
to_print = 'SQLInjection;{};{};{};{};{}'.format(paste.p_source, paste.p_date, paste.p_name, "Potential SQL injection", paste.p_rel_path)
publisher.info(to_print)
def compute_provider_info(server_trend, server_pasteName, path):
redis_all_provider = 'all_provider_set'
paste = Paste.Paste(path)
paste_baseName = paste.p_name.split('.')[0]
paste_size = paste._get_p_size()
paste_provider = paste.p_source
paste_date = str(paste._get_p_date())
redis_sum_size_set = 'top_size_set_' + paste_date
redis_avg_size_name_set = 'top_avg_size_set_' + paste_date
redis_providers_name_set = 'providers_set_' + paste_date
# Add/Update in Redis
server_pasteName.sadd(paste_baseName, path)
server_trend.sadd(redis_all_provider, paste_provider)
num_paste = int(server_trend.hincrby(paste_provider+'_num', paste_date, 1))
sum_size = float(server_trend.hincrbyfloat(paste_provider+'_size', paste_date, paste_size))
new_avg = float(sum_size) / float(num_paste)
server_trend.hset(paste_provider +'_avg', paste_date, new_avg)
#
# Compute Most Posted
#
# Size
if server_trend.zcard(redis_sum_size_set) < max_set_cardinality or server_trend.zscore(redis_sum_size_set, paste_provider) != "nil":
server_trend.zadd(redis_sum_size_set, float(num_paste), paste_provider)
server_trend.zadd(redis_avg_size_name_set, float(new_avg), paste_provider)
else: #set full capacity
member_set = server_trend.zrangebyscore(redis_sum_size_set, '-inf', '+inf', withscores=True, start=0, num=1)
# Member set is a list of (value, score) pairs
if float(member_set[0][1]) < new_avg:
#remove min from set and add the new one
print 'Size - adding ' +paste_provider+ '(' +str(new_avg)+') in set and removing '+member_set[0][0]+'('+str(member_set[0][1])+')'
server_trend.zrem(redis_sum_size_set, member_set[0][0])
server_trend.zadd(redis_sum_size_set, float(sum_size), paste_provider)
server_trend.zrem(redis_avg_size_name_set, member_set[0][0])
server_trend.zadd(redis_avg_size_name_set, float(new_avg), paste_provider)
# Num
# if set not full or provider already present
if server_trend.zcard(redis_providers_name_set) < max_set_cardinality or server_trend.zscore(redis_providers_name_set, paste_provider) != "nil":
server_trend.zadd(redis_providers_name_set, float(num_paste), paste_provider)
else: #set at full capacity
member_set = server_trend.zrangebyscore(redis_providers_name_set, '-inf', '+inf', withscores=True, start=0, num=1)
# Member set is a list of (value, score) pairs
if int(member_set[0][1]) < num_paste:
#remove min from set and add the new one
print 'Num - adding ' +paste_provider+ '(' +str(num_paste)+') in set and removing '+member_set[0][0]+'('+str(member_set[0][1])+')'
server_trend.zrem(member_set[0][0])
server_trend.zadd(redis_providers_name_set, float(num_paste), paste_provider)
def search_phone(message):
paste = Paste.Paste(message)
content = paste.get_p_content()
# regex to find phone numbers, may raise many false positives (shalt thou seek optimization, upgrading is required)
reg_phone = re.compile(r'(\+\d{1,4}(\(\d\))?\d?|0\d?)(\d{6,8}|([-/\. ]{1}\d{2,3}){3,4})')
# list of the regex results in the Paste, may be null
results = reg_phone.findall(content)
# if the list is greater than 4, we consider the Paste may contain a list of phone numbers
if len(results) > 4:
print results
publisher.warning('{} contains PID (phone numbers)'.format(paste.p_name))
def main():
publisher.port = 6380
publisher.channel = "Script"
config_section = 'DomClassifier'
p = Process(config_section)
addr_dns = p.config.get("DomClassifier", "dns")
publisher.info("""ZMQ DomainClassifier is Running""")
c = DomainClassifier.domainclassifier.Extract(rawtext="",
nameservers=[addr_dns])
cc = p.config.get("DomClassifier", "cc")
cc_tld = p.config.get("DomClassifier", "cc_tld")
while True:
try:
message = p.get_from_set()
if message is not None:
PST = Paste.Paste(message)
else:
publisher.debug("Script DomClassifier is idling 1s")
time.sleep(1)
continue
paste = PST.get_p_content()
mimetype = PST._get_p_encoding()
if mimetype == "text/plain":
c.text(rawtext=paste)
c.potentialdomain()
c.validdomain(rtype=['A'], extended=True)
localizeddomains = c.include(expression=cc_tld)
if localizeddomains:
print(localizeddomains)
publisher.warning(
'DomainC;{};{};{};Checked {} located in {};{}'.format(
PST.p_source, PST.p_date, PST.p_name,
localizeddomains, cc_tld, PST.p_path))
localizeddomains = c.localizedomain(cc=cc)
if localizeddomains:
print(localizeddomains)
publisher.warning(
'DomainC;{};{};{};Checked {} located in {};{}'.format(
PST.p_source, PST.p_date, PST.p_name,
localizeddomains, cc, PST.p_path))
except IOError:
print("CRC Checksum Failed on :", PST.p_path)
publisher.error('Duplicate;{};{};{};CRC Checksum Failed'.format(
PST.p_source, PST.p_date, PST.p_name))
def analyse(url, path):
faup.decode(url)
url_parsed = faup.get()
pprint.pprint(url_parsed)
## TODO: # FIXME: remove me
try:
resource_path = url_parsed['resource_path'].encode()
except:
resource_path = url_parsed['resource_path']
## TODO: # FIXME: remove me
try:
query_string = url_parsed['query_string'].encode()
except:
query_string = url_parsed['query_string']
result_path = {'sqli': False}
result_query = {'sqli': False}
if resource_path is not None:
result_path = pylibinjection.detect_sqli(resource_path)
print("path is sqli : {0}".format(result_path))
if query_string is not None:
result_query = pylibinjection.detect_sqli(query_string)
print("query is sqli : {0}".format(result_query))
if result_path['sqli'] is True or result_query['sqli'] is True:
paste = Paste.Paste(path)
print("Detected (libinjection) SQL in URL: ")
print(urllib.request.unquote(url))
to_print = 'LibInjection;{};{};{};{};{}'.format(
paste.p_source, paste.p_date, paste.p_name, "Detected SQL in URL",
paste.p_rel_path)
publisher.warning(to_print)
#Send to duplicate
p.populate_set_out(path, 'Duplicate')
msg = 'infoleak:automatic-detection="sql-injection";{}'.format(path)
p.populate_set_out(msg, 'Tags')
#statistics
## TODO: # FIXME: remove me
try:
tld = url_parsed['tld'].decode()
except:
tld = url_parsed['tld']
if tld is not None:
date = datetime.datetime.now().strftime("%Y%m")
server_statistics.hincrby('SQLInjection_by_tld:' + date, tld, 1)
def search_cve(message):
filepath, count = message.split()
paste = Paste.Paste(filepath)
content = paste.get_p_content()
# regex to find CVE
reg_cve = re.compile(r'(CVE-)[1-2]\d{1,4}-\d{1,5}')
# list of the regex results in the Paste, may be null
results = set(reg_cve.findall(content))
# if the list is greater than 2, we consider the Paste may contain a list of cve
if len(results) > 0:
print('{} contains CVEs'.format(paste.p_name))
publisher.warning('{} contains CVEs'.format(paste.p_name))
#send to Browse_warning_paste
p.populate_set_out('cve;{}'.format(filepath), 'BrowseWarningPaste')
#Send to duplicate
p.populate_set_out(filepath, 'Duplicate')
def compute(self, message):
filepath, count = message.split()
paste = Paste.Paste(filepath)
content = paste.get_p_content()
# list of the regex results in the Paste, may be null
results = set(self.reg_cve.findall(content))
# if the list is positive, we consider the Paste may contain a list of cve
if len(results) > 0:
warning = f'{paste.p_name} contains CVEs'
print(warning)
self.redis_logger.warning(warning)
msg = f'infoleak:automatic-detection="cve";{filepath}'
# Send to Tags Queue
self.send_message_to_queue(msg, 'Tags')
# Send to Duplicate Queue
self.send_message_to_queue(filepath, 'Duplicate')
def search_phone(message):
paste = Paste.Paste(message)
content = paste.get_p_content()
# regex to find phone numbers, may raise many false positives (shalt thou seek optimization, upgrading is required)
reg_phone = re.compile(r'(\+\d{1,4}(\(\d\))?\d?|0\d?)(\d{6,8}|([-/\. ]{1}\d{2,3}){3,4})')
# list of the regex results in the Paste, may be null
results = reg_phone.findall(content)
# if the list is greater than 4, we consider the Paste may contain a list of phone numbers
if len(results) > 4 :
print results
publisher.warning('{} contains PID (phone numbers)'.format(paste.p_name))
if __name__ == '__main__':
# If you wish to use an other port of channel, do not forget to run a subscriber accordingly (see launch_logs.sh)
# Port of the redis instance used by pubsublogger
publisher.port = 6380
# Script is the default channel used for the modules.
publisher.channel = 'Script'
# Section name in bin/packages/modules.cfg
config_section = 'Phone'
# Setup the I/O queues
p = Process(config_section)
# Sent to the logging a description of the module
publisher.info("Run Phone module")
# Endless loop getting messages from the input queue
while True:
# Get one message from the input queue
message = p.get_from_set()
if message is None:
publisher.debug("{} queue is empty, waiting".format(config_section))
time.sleep(1)
continue
# Do something with the message from the queue
search_phone(message)
def analyse(url, path):
faup.decode(url)
url_parsed = faup.get()
resource_path = url_parsed['resource_path']
query_string = url_parsed['query_string']
result_path = 0
result_query = 0
if resource_path is not None:
result_path = is_sql_injection(resource_path.decode('utf8'))
if query_string is not None:
result_query = is_sql_injection(query_string.decode('utf8'))
if (result_path > 0) or (result_query > 0):
paste = Paste.Paste(path)
if (result_path > 1) or (result_query > 1):
print("Detected SQL in URL: ")
print(urllib.request.unquote(url))
to_print = 'SQLInjection;{};{};{};{};{}'.format(
paste.p_source, paste.p_date, paste.p_name,
"Detected SQL in URL", paste.p_path)
publisher.warning(to_print)
#Send to duplicate
p.populate_set_out(path, 'Duplicate')
#send to Browse_warning_paste
p.populate_set_out('sqlinjection;{}'.format(path), 'alertHandler')
msg = 'infoleak:automatic-detection="sql-injection";{}'.format(
path)
p.populate_set_out(msg, 'Tags')
else:
print("Potential SQL injection:")
print(urllib.request.unquote(url))
to_print = 'SQLInjection;{};{};{};{};{}'.format(
paste.p_source, paste.p_date, paste.p_name,
"Potential SQL injection", paste.p_path)
publisher.info(to_print)
def get_dict_cve(list_paste_cve, only_one_same_cve_by_paste=False):
dict_keyword = {}
for paste_cve in list_paste_cve:
paste_content = Paste.Paste(paste_cve).get_p_content()
cve_list = reg_cve.findall(paste_content)
if only_one_same_cve_by_paste:
cve_list = set(cve_list)
for cve in reg_cve.findall(paste_content):
try:
dict_keyword[cve] += 1
except KeyError:
dict_keyword[cve] = 1
print('------------------------------------------------')
if dict_keyword:
res = [(k, dict_keyword[k]) for k in sorted(dict_keyword, key=dict_keyword.get, reverse=True)]
for item in res:
pass
print(item)
def search_geolocation(message):
paste = Paste.Paste(message)
content = paste.get_p_content()
# Load Geoparser
geo = Geoparser()
geolocation = geo.geoparse(message)
# regex ton find latitude and longitude
reg_lat = re.compile(r'(\'lat\': \'([-\d.]+)\',)')
reg_lon = re.compile(r'(\'lon\': \'([-\d.]+)\',)')
#lat = set(reg_lat.findall(content))
#lon = set(reg_lat.findall(content))
lat = reg_lat.search(message).group(2)
lon = reg_lon.search(message).group(2)
print('latitude: {}'.format(lat))
print('longitude: {}'.format(lon))
print('{} text geolocation'.format(paste.p_name))
publisher.warning('{} contains geolocation'.format(paste.p_name))
cards = [
r'\b4\d{3}(?:[\ \-]?)\d{4}(?:[\ \-]?)\d{4}(?:[\ \-]?)\d{4}\b', # 16-digit VISA, with separators
r'\b5[1-5]\d{2}(?:[\ \-]?)\d{4}(?:[\ \-]?)\d{4}(?:[\ \-]?)\d{4}\b', # 16 digits MasterCard
r'\b6(?:011|22(?:(?=[\ \-]?(?:2[6-9]|[3-9]))|[2-8]|9(?=[\ \-]?(?:[01]|2[0-5])))|4[4-9]\d|5\d\d)(?:[\ \-]?)\d{4}(?:[\ \-]?)\d{4}(?:[\ \-]?)\d{4}\b', # Discover Card
r'\b35(?:2[89]|[3-8]\d)(?:[\ \-]?)\d{4}(?:[\ \-]?)\d{4}(?:[\ \-]?)\d{4}\b', # Japan Credit Bureau (JCB)
r'\b3[47]\d\d(?:[\ \-]?)\d{6}(?:[\ \-]?)\d{5}\b', # American Express
r'\b(?:5[0678]\d\d|6304|6390|67\d\d)\d{8,15}\b', # Maestro
]
regex = re.compile('|'.join(cards))
while True:
message = p.get_from_set()
if message is not None:
filename, score = message.split()
paste = Paste.Paste(filename)
content = paste.get_p_content()
all_cards = re.findall(regex, content)
if len(all_cards) > 0:
print 'All matching', all_cards
creditcard_set = set([])
for card in all_cards:
clean_card = re.sub('[^0-9]', '', card)
if lib_refine.is_luhn_valid(clean_card):
print clean_card, 'is valid'
creditcard_set.add(clean_card)
paste.__setattr__(channel, creditcard_set)
paste.save_attribute_redis(channel, creditcard_set)
if __name__ == '__main__':
# If you wish to use an other port of channel, do not forget to run a subscriber accordingly (see launch_logs.sh)
# Port of the redis instance used by pubsublogger
publisher.port = 6380
# Script is the default channel used for the modules.
publisher.channel = 'Script'
# Section name in bin/packages/modules.cfg
config_section = 'Keys'
# Setup the I/O queues
p = Process(config_section)
# Sent to the logging a description of the module
publisher.info("Run Keys module ")
# Endless loop getting messages from the input queue
while True:
# Get one message from the input queue
message = p.get_from_set()
if message is None:
publisher.debug("{} queue is empty, waiting".format(config_section))
time.sleep(1)
continue
# Do something with the message from the queue
paste = Paste.Paste(message)
search_key(paste)
# (Optional) Send that thing to the next queue
prec_filename = None
# Thanks to Faup project for this regex
# https://github.com/stricaud/faup
url_regex = "((http|https|ftp)\://([a-zA-Z0-9\.\-]+(\:[a-zA-Z0-9\.&%\$\-]+)*@)*((25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9])\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[0-9])|localhost|([a-zA-Z0-9\-]+\.)*[a-zA-Z0-9\-]+\.onion)(\:[0-9]+)*(/($|[a-zA-Z0-9\.\,\?\'\\\+&%\$#\=~_\-]+))*)"
while True:
if message is not None:
print message
filename, score = message.split()
# "For each new paste"
if prec_filename is None or filename != prec_filename:
domains_list = []
urls = []
PST = Paste.Paste(filename)
for x in PST.get_regex(url_regex):
# Extracting url with regex
url, s, credential, subdomain, domain, host, port, \
resource_path, query_string, f1, f2, f3, f4 = x
domains_list.append(domain)
urls.append(url)
# Saving the list of extracted onion domains.
PST.__setattr__(channel, domains_list)
PST.save_attribute_redis(channel, domains_list)
to_print = 'Onion;{};{};{};'.format(PST.p_source, PST.p_date,
PST.p_name)
if len(domains_list) > 0:
