ipv4s = common_functions.pull_ipv4_addresses(es, es_collection_name, body)
# Remove recently-seen items from the result set
for i in xrange(len(ipv4s) - 1, -1, -1):
if external_lookups.check_newobserved(ipv4s[i]):
del ipv4s[i]
# Exclude previously seen addresses from the past 30 days
### Everything after this is to write the HTML page
report_contents += preskel
for ipv4 in ipv4s:
meta = common_functions.describesources(es, es_collection_name, 'ipv4:'+ipv4)
report_contents += '<tr>'
v4hash = hashlib.sha224(ipv4).hexdigest()
report_contents += '<th scope="row"><a href="#'+str(v4hash)+'">'+ipv4+'</a></th>'
report_contents += '<td>'
seensubjects = []
for subject in meta[0]:
subject = subject.strip()
if subject in seensubjects: continue
seensubjects.append(subject)
if re.search('^RE$', subject): continue
subject = re.sub('\[cw warroom\] ', '', subject)
subject = re.sub('\[cw general\] ', '', subject)
report_contents += subject+'</br/>'
report_contents += '</td>'
report_contents += '<th>'
print 'Raw count of domains: ' + str(len(domains))
# Remove recently-seen items from the result set
for i in xrange(len(domains) - 1, -1, -1):
if re.search('^\-', domains[i]):
del domains[i]
if external_lookups.check_newobserved(domains[i]):
del domains[i]
print 'Scrubbed count of domains: ' + str(len(domains))
### Everything after this is to write the HTML page
report_contents += preskel
for domain in domains:
meta = common_functions.describesources(es, es_collection_name,
'domain:' + domain)
report_contents += '<tr>'
v4hash = hashlib.sha224(domain).hexdigest()
report_contents += '<th scope="row"><a href="#' + str(
v4hash) + '">' + domain + '</a></th>'
report_contents += '<td>'
seensubjects = []
for subject in meta[0]:
subject = subject.strip()
if subject in seensubjects: continue
if re.search('^RE$', subject): continue
seensubjects.append(subject)
subject = re.sub('\[cw warroom\] ', '', subject)
subject = re.sub('\[cw general\] ', '', subject)
report_contents += subject + '</br/>'
report_contents += '</td>'
md5s = common_functions.pull_md5s(es, es_collection_name, body)
print 'Raw count of md5s: '+str(len(md5s))
# Remove recently-seen items from the result set
for i in xrange(len(md5s) - 1, -1, -1):
if external_lookups.check_newobserved(md5s[i]):
del md5s[i]
print 'Scrubbed count of md5s: '+str(len(md5s))
### Everything after this is to write the HTML page
report_contents += preskel
for md5 in md5s:
meta = common_functions.describesources(es, es_collection_name, 'md5:'+md5)
report_contents += '<tr>'
report_contents += '<th scope="row"><a href="#'+str(md5)+'">'+md5+'</a></th>'
report_contents += '<td>'
seensubjects = []
for subject in meta[0]:
subject = subject.strip()
if subject in seensubjects: continue
if re.search('^RE$', subject): continue
seensubjects.append(subject)
subject = re.sub('\[cw warroom\] ', '', subject)
subject = re.sub('\[cw general\] ', '', subject)
report_contents += subject+'</br/>'
report_contents += '</td>'
report_contents += '<th>'
seenlists = []
ipv4s = common_functions.pull_ipv4_addresses(es, es_collection_name, body)
# Remove recently-seen items from the result set
for i in xrange(len(ipv4s) - 1, -1, -1):
if external_lookups.check_newobserved(ipv4s[i]):
del ipv4s[i]
# Exclude previously seen addresses from the past 30 days
### Everything after this is to write the HTML page
report_contents += preskel
for ipv4 in ipv4s:
meta = common_functions.describesources(es, es_collection_name,
'ipv4:' + ipv4)
report_contents += '<tr>'
v4hash = hashlib.sha224(ipv4).hexdigest()
report_contents += '<th scope="row"><a href="#' + str(
v4hash) + '">' + ipv4 + '</a></th>'
report_contents += '<td>'
seensubjects = []
for subject in meta[0]:
subject = subject.strip()
if subject in seensubjects: continue
seensubjects.append(subject)
if re.search('^RE$', subject): continue
subject = re.sub('\[cw warroom\] ', '', subject)
subject = re.sub('\[cw general\] ', '', subject)
report_contents += subject + '</br/>'
report_contents += '</td>'
md5s = common_functions.pull_md5s(es, es_collection_name, body)
print 'Raw count of md5s: ' + str(len(md5s))
# Remove recently-seen items from the result set
for i in xrange(len(md5s) - 1, -1, -1):
if external_lookups.check_newobserved(md5s[i]):
del md5s[i]
print 'Scrubbed count of md5s: ' + str(len(md5s))
### Everything after this is to write the HTML page
report_contents += preskel
for md5 in md5s:
meta = common_functions.describesources(es, es_collection_name,
'md5:' + md5)
report_contents += '<tr>'
report_contents += '<th scope="row"><a href="#' + str(
md5) + '">' + md5 + '</a></th>'
report_contents += '<td>'
seensubjects = []
for subject in meta[0]:
subject = subject.strip()
if subject in seensubjects: continue
if re.search('^RE$', subject): continue
seensubjects.append(subject)
subject = re.sub('\[cw warroom\] ', '', subject)
subject = re.sub('\[cw general\] ', '', subject)
report_contents += subject + '</br/>'
report_contents += '</td>'
report_contents += '<th>'
print 'Raw count of domains: '+str(len(domains))
# Remove recently-seen items from the result set
for i in xrange(len(domains) - 1, -1, -1):
if re.search('^\-', domains[i]):
del domains[i]
if external_lookups.check_newobserved(domains[i]):
del domains[i]
print 'Scrubbed count of domains: '+str(len(domains))
### Everything after this is to write the HTML page
report_contents += preskel
for domain in domains:
meta = common_functions.describesources(es, es_collection_name, 'domain:'+domain)
report_contents += '<tr>'
v4hash = hashlib.sha224(domain).hexdigest()
report_contents += '<th scope="row"><a href="#'+str(v4hash)+'">'+domain+'</a></th>'
report_contents += '<td>'
seensubjects = []
for subject in meta[0]:
subject = subject.strip()
if subject in seensubjects: continue
if re.search('^RE$', subject): continue
seensubjects.append(subject)
subject = re.sub('\[cw warroom\] ', '', subject)
subject = re.sub('\[cw general\] ', '', subject)
report_contents += subject+'</br/>'
report_contents += '</td>'
report_contents += '<th>'
