def analyze(observable, results):
links = set()
parts = extract(observable.value)
if parts.subdomain == '':
data = DomainToolsApi.get("/{}/whois/history".format(observable.value), results.settings)
results.update(raw=json.dumps(data, indent=2))
for record in data['response']['history']:
created = datetime.strptime(record['whois']['registration']['created'], "%Y-%m-%d")
expires = datetime.strptime(record['whois']['registration']['expires'], "%Y-%m-%d")
registrar = Company.get_or_create(name=record['whois']['registration']['registrar'])
registrant = Text.get_or_create(value=record['whois']['registrant'])
links.update(observable.link_to(registrar, 'Registrar', 'DomainTools', created, expires))
links.update(observable.link_to(registrant, 'Registrant', 'DomainTools', created, expires))
parsed = parse_raw_whois([record['whois']['record']], normalized=True)
email = get_value_at(parsed, 'contacts.registrant.email')
if email:
email = Email.get_or_create(value=email)
links.update(observable.link_to(email, 'Registrant Email', 'DomainTools', created, expires))
return list(links)
def analyze(ip, results):
links = set()
r = IPWhois(ip.value)
result = r.lookup_whois()
results.update(raw=pformat(result))
# Let's focus on the most specific information
# Which should be in the smallest subnet
n = 0
smallest_subnet = None
for network in result['nets']:
cidr_bits = int(network['cidr'].split('/')[1].split(',')[0])
if cidr_bits > n:
n = cidr_bits
smallest_subnet = network
if smallest_subnet:
# Create the company
company = Company.get_or_create(
name=smallest_subnet['description'].split("\n")[0])
links.update(ip.active_link_to(company, 'hosting',
'Network Whois'))
# Link it to every email address referenced
if smallest_subnet['emails']:
for email_address in smallest_subnet['emails']:
email = Email.get_or_create(value=email_address)
links.update(company.link_to(email, None, 'Network Whois'))
# Copy the subnet info into the main dict
for key in smallest_subnet:
if smallest_subnet[key]:
result["net_{}".format(key)] = smallest_subnet[key]
# Add the network whois to the context if not already present
for context in ip.context:
if context['source'] == 'network_whois':
break
else:
# Remove the nets info (the main one was copied)
result.pop("nets", None)
result.pop("raw", None)
result.pop("raw_referral", None)
result.pop("referral", None)
result.pop("query", None)
result['source'] = 'network_whois'
ip.add_context(result)
return list(links)
def analyze(ip, results):
links = set()
r = IPWhois(ip.value)
result = r.lookup_whois()
results.update(raw=pformat(result))
# Let's focus on the most specific information
# Which should be in the smallest subnet
n = 0
smallest_subnet = None
for network in result['nets']:
cidr_bits = int(network['cidr'].split('/')[1].split(',')[0])
if cidr_bits > n:
n = cidr_bits
smallest_subnet = network
if smallest_subnet:
# Create the company
company = Company.get_or_create(
name=smallest_subnet['description'].split("\n")[0])
links.update(ip.active_link_to(company, 'hosting', 'Network Whois'))
# Link it to every email address referenced
if smallest_subnet['emails']:
for email_address in smallest_subnet['emails']:
email = Email.get_or_create(value=email_address)
links.update(company.link_to(email, None, 'Network Whois'))
# Copy the subnet info into the main dict
for key in smallest_subnet:
if smallest_subnet[key]:
result["net_{}".format(key)] = smallest_subnet[key]
# Add the network whois to the context if not already present
for context in ip.context:
if context['source'] == 'network_whois':
break
else:
# Remove the nets info (the main one was copied)
result.pop("nets", None)
result.pop("raw", None)
result.pop("raw_referral", None)
result.pop("referral", None)
result.pop("query", None)
result['source'] = 'network_whois'
ip.add_context(result)
return list(links)
def analyze(observable, results):
links = set()
parts = tldextract_parser(observable.value)
if parts.subdomain == "":
data = DomainToolsApi.get(
"/{}/whois/history".format(observable.value), results.settings
)
results.update(raw=json.dumps(data, indent=2))
for record in data["response"]["history"]:
created = datetime.strptime(
record["whois"]["registration"]["created"], "%Y-%m-%d"
)
expires = datetime.strptime(
record["whois"]["registration"]["expires"], "%Y-%m-%d"
)
registrar = Company.get_or_create(
name=record["whois"]["registration"]["registrar"]
)
registrant = Text.get_or_create(value=record["whois"]["registrant"])
links.update(
observable.link_to(
registrar, "Registrar", "DomainTools", created, expires
)
)
links.update(
observable.link_to(
registrant, "Registrant", "DomainTools", created, expires
)
)
parsed = parse_raw_whois([record["whois"]["record"]], normalized=True)
email = get_value_at(parsed, "contacts.registrant.email")
if email:
email = Email.get_or_create(value=email)
links.update(
observable.link_to(
email, "Registrant Email", "DomainTools", created, expires
)
)
return list(links)
def analyze(observable, results):
links = set()
params = {
'query': observable.value,
'field': 'nameserver'
}
data = PassiveTotalApi.get('/whois/search', results.settings, params)
for record in data['results']:
domain = Hostname.get_or_create(value=record['domain'])
links.update(domain.active_link_to(observable, "NS record", 'PassiveTotal'))
registrant_email = get_value_at(record, 'registrant.email')
if registrant_email:
registrant = Email.get_or_create(value=registrant_email)
links.update(domain.active_link_to(registrant, "Registrant Email", 'PassiveTotal'))
return list(links)
def whois_links(observable, data):
links = set()
created_date = data.get("createdDate")
expires_date = data.get("expiresDate")
first_seen = convert_to_datetime(created_date)
last_seen = convert_to_datetime(expires_date)
registrar_name = data.get("registrarName")
if registrar_name is not None:
node = Text.get_or_create(value=registrar_name)
try:
links.update(
observable.link_to(node, "Registrar name", SOURCE, first_seen,
last_seen))
except Exception as e:
logger.error(e.message)
contacts = data.get("contact", [])
for contact in contacts:
email = contact.get("email")
if email is not None:
node = Email.get_or_create(value=email)
try:
links.update(observable.link_to(node, "Contact email", SOURCE),
first_seen, last_seen)
except Exception as e:
logger.error(e.message)
name_servers = data.get("nameServers", [])
name_servers = name_servers if name_servers is not None else []
for name_server in name_servers:
node = Hostname.get_or_create(value=name_server)
try:
links.update(observable.link_to(node, "Name server", SOURCE),
first_seen, last_seen)
except Exception as e:
logger.error(e.message)
return list(links)
def analyze(ip):
links = []
results = IPWhois(ip.value)
results = results.lookup_rdap()
for entity in results['objects']:
entity = results['objects'][entity]
if entity['contact']['kind'] != 'individual':
# Create the company
company = Company.get_or_create(name=entity['contact']['name'], rdap=entity)
link = Link.connect(ip, company)
link.add_history('hosting')
links.append(link)
# Link it to every email address referenced
for email_info in entity['contact']['email']:
email = Email.get_or_create(value=email_info['value'])
link = Link.connect(company, email)
links.append(link)
return links
def analyze(observable, results):
links = set()
json_result = ThreatCrowdAPI.fetch(observable)
json_string = json.dumps(
json_result, sort_keys=True, indent=4, separators=(",", ": ")
)
results.update(raw=json_string)
result = {}
if isinstance(observable, Hostname):
if "resolutions" in json_result:
result["ip on this domains"] = 0
for ip in json_result["resolutions"]:
if ip["ip_address"].strip() != observable.value:
if ip["last_resolved"] != "0000-00-00":
last_resolved = datetime.datetime.strptime(
ip["last_resolved"], "%Y-%m-%d"
)
try:
new_ip = Ip.get_or_create(
value=ip["ip_address"].strip()
)
links.update(
new_ip.active_link_to(
observable, "IP", "ThreatCrowd", last_resolved
)
)
result["ip on this domains"] += 1
except ObservableValidationError:
logging.error(
"An error occurred when trying to add subdomain {} to the database".format(
ip["ip_address"]
)
)
if "emails" in json_result:
result["nb emails"] = 0
for email in json_result["emails"]:
try:
new_email = Email.get_or_create(value=email)
links.update(
new_email.active_link_to(
observable, "Used by", "ThreatCrowd"
)
)
result["nb emails"] += 1
except ObservableValidationError:
logging.error(
"An error occurred when trying to add email {} to the database".format(
email
)
)
if "subdomains" in json_result:
result["nb subdomains"] = 0
for subdomain in json_result["subdomains"]:
try:
new_domain = Hostname.get_or_create(value=subdomain)
links.update(
observable.active_link_to(
new_domain, "subdomain", "ThreatCrowd"
)
)
result["nb subdomains"] += 1
except ObservableValidationError:
logging.error(
"An error occurred when trying to add subdomain {} to the database".format(
subdomain
)
)
if isinstance(observable, Ip):
if "resolutions" in json_result:
result["domains resolved"] = 0
for domain in json_result["resolutions"]:
if domain["domain"].strip() != observable.value:
try:
last_resolved = datetime.datetime.strptime(
domain["last_resolved"], "%Y-%m-%d"
)
new_domain = Hostname.get_or_create(
value=domain["domain"].strip()
)
links.update(
new_domain.active_link_to(
observable, "A Record", "ThreatCrowd", last_resolved
)
)
result["domains resolved"] += 1
except ObservableValidationError:
logging.error(
"An error occurred when trying to add domain {} to the database".format(
domain["domain"]
)
)
if "hashes" in json_result and len(json_result["hashes"]) > 0:
result["malwares"] = 0
for h in json_result["hashes"]:
new_hash = Hash.get_or_create(value=h)
links.update(
new_hash.active_link_to(observable, "hash", "ThreatCrowd")
)
result["malwares"] += 1
if isinstance(observable, Email):
if "domains" in json_result and len(json_result) > 0:
result["domains recorded by email"] = 0
for domain in json_result["domains"]:
new_domain = Hostname.get_or_create(value=domain)
links.update(
new_domain.active_link_to(
observable, "recorded by", "ThreatCrowd"
)
)
result["domains recorded by email"] += 1
if isinstance(observable, Hash):
result["nb c2"] = 0
if "md5" in json_result:
new_hash = Hash.get_or_create(value=json_result["md5"])
links.update(new_hash.active_link_to(observable, "md5", "ThreadCrowd"))
if "sha1" in json_result:
new_hash = Hash.get_or_create(value=json_result["sha1"])
links.update(new_hash.active_link_to(observable, "sha1", "ThreadCrowd"))
if "sha256" in json_result:
new_hash = Hash.get_or_create(value=json_result["sha256"])
links.update(
new_hash.active_link_to(observable, "sha256", "ThreadCrowd")
)
if "domains" in json_result and len(json_result["domains"]):
for domain in json_result["domains"]:
new_domain = Hostname.get_or_create(value=domain)
links.update(
observable.active_link_to(new_domain, "c2", "ThreatCrowd")
)
result["nb c2"] += 1
if "ips" in json_result and len(json_result["ips"]):
for ip in json_result["ips"]:
new_ip = Ip.get_or_create(value=ip.strip())
links.update(observable.active_link_to(new_ip, "c2", "ThreatCrowd"))
result["nb c2"] += 1
if "permalink" in json_result:
result["permalink"] = json_result["permalink"]
result["source"] = "threatcrowd_query"
result["raw"] = json_string
observable.add_context(result)
return list(links)
def _add_events_nodes(self, events, context, tags):
log.debug('_add_events_nodes on {nb} events'.format(nb=len(events)))
attach_unsupported = dict([
(_, 0) for _ in ['UNSUPPORTED_TYPE', 'TOO_SMALL', None]
])
event_nodes = list()
for msg in events:
create_t = datetime.strptime(msg['messageTime'],
"%Y-%m-%dT%H:%M:%S.%fZ")
# PPS unique value
guid = Text.get_or_create(value='proofpoint://%s' % msg['GUID'],
created=create_t,
context=[context])
log.debug('Event {msg}'.format(msg=msg['messageID']))
message_contents = list()
src_ip = Ip.get_or_create(value=msg['senderIP'],
created=create_t,
context=[context])
src_ip.tag(['MTA'])
guid.active_link_to([src_ip], "MTA src ip", self.name)
# new event
event_nodes.append(guid)
#
if self.config['import_email_metadata']:
# email details
# messageID
message_id = Email.get_or_create(value=msg['messageID'],
created=create_t,
context=[context])
guid.active_link_to([message_id], "seen in", self.name)
# sender
_s1 = Email.get_or_create(value=msg['sender'],
created=create_t,
context=[context])
_s1.tag(['sender'])
guid.active_link_to([_s1], "sender", self.name)
if 'headerFrom' in msg:
# header From
_s2 = Email.get_or_create(value=msg['headerFrom'],
created=create_t,
context=[context])
_s2.tag(['sender'])
guid.active_link_to([_s2], "headerFrom", self.name)
# FIXME is that a duplicate of attachment-malware ?
# attachment events
for attach in msg['messageParts']:
if attach['sandboxStatus'] in ['THREAT']:
md5 = Hash.get_or_create(value=attach['md5'],
created=create_t,
context=[context])
md5.tag([t['name'] for t in tags])
fname = File.get_or_create(value=attach['filename'],
created=create_t,
context=[context])
fname.tag([t['name'] for t in tags])
# this should be a DUP from threat_nodes in analyse()
sha_threat = Hash.get_or_create(value=attach['sha256'],
created=create_t,
context=[context])
sha_threat.active_link_to([md5, fname], "relates",
self.name)
sha_threat.tag([t['name'] for t in tags])
message_contents.append(sha_threat)
# link the 3 together
elif attach['sandboxStatus'] in [
'UNSUPPORTED_TYPE', 'TOO_SMALL', None
]:
attach_unsupported[attach['sandboxStatus']] += 1
log.debug(pprint.pformat(attach))
# add context to the hashes
guid.active_link_to(message_contents, "delivers", self.name)
_stats = ', '.join("%s: %d" % (k, v)
for k, v in attach_unsupported.items())
log.warning('Ignored unsupported attachments: %s', _stats)
for o in event_nodes:
o.tag([t['name'] for t in tags])
return event_nodes
def analyze(observable, results):
links = set()
json_result = ThreatCrowdAPI.fetch(observable)
json_string = json.dumps(json_result,
sort_keys=True,
indent=4,
separators=(',', ': '))
results.update(raw=json_string)
result = {}
if isinstance(observable, Hostname):
if 'resolutions' in json_result:
result['ip on this domains'] = 0
for ip in json_result['resolutions']:
if ip['ip_address'].strip() != observable.value:
if ip['last_resolved'] != '0000-00-00':
last_resolved = datetime.datetime.strptime(
ip['last_resolved'], "%Y-%m-%d")
try:
new_ip = Ip.get_or_create(
value=ip['ip_address'].strip())
links.update(
new_ip.active_link_to(
observable, 'IP', 'ThreatCrowd',
last_resolved))
result['ip on this domains'] += 1
except ObservableValidationError:
logging.error(
"An error occurred when trying to add subdomain {} to the database"
.format(ip['ip_address']))
if 'emails' in json_result:
result['nb emails'] = 0
for email in json_result['emails']:
try:
new_email = Email.get_or_create(value=email)
links.update(
new_email.active_link_to(observable, 'Used by',
'ThreatCrowd'))
result['nb emails'] += 1
except ObservableValidationError:
logging.error(
"An error occurred when trying to add email {} to the database"
.format(email))
if 'subdomains' in json_result:
result['nb subdomains'] = 0
for subdomain in json_result['subdomains']:
try:
new_domain = Hostname.get_or_create(value=subdomain)
links.update(
observable.active_link_to(new_domain, 'subdomain',
'ThreatCrowd'))
result['nb subdomains'] += 1
except ObservableValidationError:
logging.error(
"An error occurred when trying to add subdomain {} to the database"
.format(subdomain))
if isinstance(observable, Ip):
if 'resolutions' in json_result:
result['domains resolved'] = 0
for domain in json_result['resolutions']:
if domain['domain'].strip() != observable.value:
try:
last_resolved = datetime.datetime.strptime(
domain['last_resolved'], "%Y-%m-%d")
new_domain = Hostname.get_or_create(
value=domain['domain'].strip())
links.update(
new_domain.active_link_to(
observable, 'A Record', 'ThreatCrowd',
last_resolved))
result['domains resolved'] += 1
except ObservableValidationError:
logging.error(
"An error occurred when trying to add domain {} to the database"
.format(domain['domain']))
if 'hashes' in json_result and len(json_result['hashes']) > 0:
result['malwares'] = 0
for h in json_result['hashes']:
new_hash = Hash.get_or_create(value=h)
links.update(
new_hash.active_link_to(observable, 'hash',
'ThreatCrowd'))
result['malwares'] += 1
if isinstance(observable, Email):
if 'domains' in json_result and len(json_result) > 0:
result['domains recorded by email'] = 0
for domain in json_result['domains']:
new_domain = Hostname.get_or_create(value=domain)
links.update(
new_domain.active_link_to(observable, 'recorded by',
'ThreatCrowd'))
result['domains recorded by email'] += 1
if isinstance(observable, Hash):
result['nb c2'] = 0
if 'md5' in json_result:
new_hash = Hash.get_or_create(value=json_result['md5'])
links.update(
new_hash.active_link_to(observable, 'md5', 'ThreadCrowd'))
if 'sha1' in json_result:
new_hash = Hash.get_or_create(value=json_result['sha1'])
links.update(
new_hash.active_link_to(observable, 'sha1', 'ThreadCrowd'))
if 'sha256' in json_result:
new_hash = Hash.get_or_create(value=json_result['sha256'])
links.update(
new_hash.active_link_to(observable, 'sha256',
'ThreadCrowd'))
if 'domains' in json_result and len(json_result['domains']):
for domain in json_result['domains']:
new_domain = Hostname.get_or_create(value=domain)
links.update(
observable.active_link_to(new_domain, 'c2',
'ThreatCrowd'))
result['nb c2'] += 1
if 'ips' in json_result and len(json_result['ips']):
for ip in json_result['ips']:
new_ip = Ip.get_or_create(value=ip.strip())
links.update(
observable.active_link_to(new_ip, 'c2', 'ThreatCrowd'))
result['nb c2'] += 1
if 'permalink' in json_result:
result['permalink'] = json_result['permalink']
result['source'] = 'threatcrowd_query'
result['raw'] = json_string
observable.add_context(result)
return list(links)
def analyze(hostname, results):
links = set()
data = whois.whois(hostname.value)
if not data["domain_name"]:
return list(links)
should_add_context = False
for context in hostname.context:
if context["source"] == "whois":
break
else:
should_add_context = True
context = {"source": "whois"}
context["whois_server"] = data["whois_server"]
if data["dnssec"]:
context["dnssec"] = data["dnssec"]
if isinstance(data["creation_date"], list):
context["creation_date"] = sorted(data["creation_date"])[0]
else:
context["creation_date"] = data["creation_date"]
if isinstance(data["updated_date"], list):
context["updated_date"] = sorted(data["updated_date"],
reverse=True)[0]
else:
context["updated_date"] = data["updated_date"]
if isinstance(data["expiration_date"], list):
context["expiration_date"] = sorted(data["expiration_date"],
reverse=True)[0]
else:
context["expiration_date"] = data["expiration_date"]
name_servers = data["name_servers"]
if isinstance(name_servers, list):
for ns in name_servers:
ns_obs = Hostname.get_or_create(value=ns)
links.update(
ns_obs.active_link_to(hostname, "NS", context["source"]))
else:
ns_obs = Hostname.get_or_create(value=name_servers)
links.update(
ns_obs.active_link_to(hostname, "NS", context["source"]))
for email in data["emails"]:
email_obs = Email.get_or_create(value=email)
links.update(
email_obs.active_link_to(hostname, "email registrar",
context["source"]))
if data["org"]:
company_org = Company.get_or_create(name=data["org"])
links.update(
company_org.active_link_to(hostname, "Org", context["source"]))
if data["registrar"]:
company_registrar = Company.get_or_create(name=data["registrar"])
links.update(
company_registrar.active_link_to(hostname, "registrar",
context["source"]))
if should_add_context:
hostname.add_context(context)
else:
hostname.save()
return list(links)
def analyze(observable, results):
links = set()
json_result = ThreatCrowdAPI.fetch(observable)
json_string = json.dumps(
json_result, sort_keys=True, indent=4, separators=(',', ': '))
results.update(raw=json_string)
result = {}
if isinstance(observable, Hostname):
if 'resolutions' in json_result:
result['ip on this domains'] = 0
for ip in json_result['resolutions']:
if ip['ip_address'].strip() != observable.value:
if ip['last_resolved'] != '0000-00-00':
last_resolved = datetime.datetime.strptime(
ip['last_resolved'], "%Y-%m-%d")
try:
new_ip = Ip.get_or_create(
value=ip['ip_address'].strip())
links.update(
new_ip.active_link_to(
observable, 'IP', 'ThreatCrowd',
last_resolved))
result['ip on this domains'] += 1
except ObservableValidationError:
logging.error(
"An error occurred when trying to add subdomain {} to the database".
format(ip['ip_address']))
if 'emails' in json_result:
result['nb emails'] = 0
for email in json_result['emails']:
try:
new_email = Email.get_or_create(value=email)
links.update(
new_email.active_link_to(
observable, 'Used by', 'ThreatCrowd'))
result['nb emails'] += 1
except ObservableValidationError:
logging.error(
"An error occurred when trying to add email {} to the database".
format(email))
if 'subdomains' in json_result:
result['nb subdomains'] = 0
for subdomain in json_result['subdomains']:
try:
new_domain = Hostname.get_or_create(value=subdomain)
links.update(
observable.active_link_to(
new_domain, 'subdomain', 'ThreatCrowd'))
result['nb subdomains'] += 1
except ObservableValidationError:
logging.error(
"An error occurred when trying to add subdomain {} to the database".
format(subdomain))
if isinstance(observable, Ip):
if 'resolutions' in json_result:
result['domains resolved'] = 0
for domain in json_result['resolutions']:
if domain['domain'].strip() != observable.value:
try:
last_resolved = datetime.datetime.strptime(
domain['last_resolved'], "%Y-%m-%d")
new_domain = Hostname.get_or_create(
value=domain['domain'].strip())
links.update(
new_domain.active_link_to(
observable, 'A Record', 'ThreatCrowd',
last_resolved))
result['domains resolved'] += 1
except ObservableValidationError:
logging.error(
"An error occurred when trying to add domain {} to the database".
format(domain['domain']))
if 'hashes' in json_result and len(json_result['hashes']) > 0:
result['malwares'] = 0
for h in json_result['hashes']:
new_hash = Hash.get_or_create(value=h)
links.update(
new_hash.active_link_to(
observable, 'hash', 'ThreatCrowd'))
result['malwares'] += 1
if isinstance(observable, Email):
if 'domains' in json_result and len(json_result) > 0:
result['domains recorded by email'] = 0
for domain in json_result['domains']:
new_domain = Hostname.get_or_create(value=domain)
links.update(
new_domain.active_link_to(
observable, 'recorded by', 'ThreatCrowd'))
result['domains recorded by email'] += 1
if isinstance(observable, Hash):
result['nb c2'] = 0
if 'md5' in json_result:
new_hash = Hash.get_or_create(value=json_result['md5'])
links.update(
new_hash.active_link_to(observable, 'md5', 'ThreadCrowd'))
if 'sha1' in json_result:
new_hash = Hash.get_or_create(value=json_result['sha1'])
links.update(
new_hash.active_link_to(observable, 'sha1', 'ThreadCrowd'))
if 'sha256' in json_result:
new_hash = Hash.get_or_create(value=json_result['sha256'])
links.update(
new_hash.active_link_to(
observable, 'sha256', 'ThreadCrowd'))
if 'domains' in json_result and len(json_result['domains']):
for domain in json_result['domains']:
new_domain = Hostname.get_or_create(value=domain)
links.update(
observable.active_link_to(
new_domain, 'c2', 'ThreatCrowd'))
result['nb c2'] += 1
if 'ips' in json_result and len(json_result['ips']):
for ip in json_result['ips']:
new_ip = Ip.get_or_create(value=ip.strip())
links.update(
observable.active_link_to(new_ip, 'c2', 'ThreatCrowd'))
result['nb c2'] += 1
if 'permalink' in json_result:
result['permalink'] = json_result['permalink']
result['source'] = 'threatcrowd_query'
result['raw'] = json_string
observable.add_context(result)
return list(links)
def _add_events_nodes(self, events, context, tags):
log.debug('_add_events_nodes on {nb} events'.format(nb=len(events)))
attach_unsupported = dict(
[(_, 0) for _ in ['UNSUPPORTED_TYPE', 'TOO_SMALL', None]])
event_nodes = list()
for msg in events:
create_t = datetime.strptime(
msg['messageTime'], "%Y-%m-%dT%H:%M:%S.%fZ")
# PPS unique value
guid = Text.get_or_create(
value='proofpoint://%s' % msg['GUID'],
created=create_t,
context=[context])
log.debug('Event {msg}'.format(msg=msg['messageID']))
message_contents = list()
src_ip = Ip.get_or_create(
value=msg['senderIP'], created=create_t, context=[context])
src_ip.tag(['MTA'])
guid.active_link_to([src_ip], "MTA src ip", self.name)
# new event
event_nodes.append(guid)
#
if self.config['import_email_metadata']:
# email details
# messageID
message_id = Email.get_or_create(
value=msg['messageID'], created=create_t, context=[context])
guid.active_link_to([message_id], "seen in", self.name)
# sender
_s1 = Email.get_or_create(
value=msg['sender'], created=create_t, context=[context])
_s1.tag(['sender'])
guid.active_link_to([_s1], "sender", self.name)
if 'headerFrom' in msg:
# header From
_s2 = Email.get_or_create(
value=msg['headerFrom'],
created=create_t,
context=[context])
_s2.tag(['sender'])
guid.active_link_to([_s2], "headerFrom", self.name)
# FIXME is that a duplicate of attachment-malware ?
# attachment events
for attach in msg['messageParts']:
if attach['sandboxStatus'] in ['THREAT']:
md5 = Hash.get_or_create(
value=attach['md5'],
created=create_t,
context=[context])
md5.tag([t['name'] for t in tags])
fname = File.get_or_create(
value=attach['filename'],
created=create_t,
context=[context])
fname.tag([t['name'] for t in tags])
# this should be a DUP from threat_nodes in analyse()
sha_threat = Hash.get_or_create(
value=attach['sha256'],
created=create_t,
context=[context])
sha_threat.active_link_to([md5, fname], "relates",
self.name)
sha_threat.tag([t['name'] for t in tags])
message_contents.append(sha_threat)
# link the 3 together
elif attach['sandboxStatus'] in ['UNSUPPORTED_TYPE',
'TOO_SMALL', None]:
attach_unsupported[attach['sandboxStatus']] += 1
log.debug(pprint.pformat(attach))
# add context to the hashes
guid.active_link_to(message_contents, "delivers", self.name)
_stats = ', '.join(
"%s: %d" % (k, v) for k, v in attach_unsupported.items())
log.warning('Ignored unsupported attachments: %s', _stats)
for o in event_nodes:
o.tag([t['name'] for t in tags])
return event_nodes