def delete_whois(domain, date, analyst):
"""
Remove whois information for a domain.
:param domain: The domain for the whois entry.
:type domain: str
:param date: The date for the whois data.
:type date: datetime.datetime
:param analyst: The user editing the domain name.
:type analyst: str
:returns: dict with keys:
"success" (boolean),
"message" (str) if failed.
"""
domain = Domain.objects(domain=domain).first()
if not domain:
return {'success': False,
'message': "No matching domain found."}
try:
domain.delete_whois(date)
domain.save(username=analyst)
return {'success': True}
except ValidationError, e:
return {'success': False, 'message': e}
def __init__(self, *args, **kwargs):
#populate date choices
self.domain = ""
allow_adding = True
if 'domain' in kwargs:
self.domain = kwargs['domain']
del kwargs['domain'] #keep the default __init__ from erroring out
if 'allow_adding' in kwargs:
allow_adding = kwargs['allow_adding']
del kwargs['allow_adding']
super(UpdateWhoisForm, self).__init__(*args, **kwargs)
if self.domain:
if allow_adding:
date_choices = [("", "Add New")]
else:
date_choices = []
dmain = Domain.objects(domain=self.domain).first()
if dmain:
whois = dmain.whois
whois.sort(key=lambda w: w['date'], reverse=True)
for w in dmain.whois:
date = datetime.strftime(w['date'],
settings.PY_DATETIME_FORMAT)
date_choices.append((date, date))
self.fields['date'].choices = date_choices
def edit_whois(domain, data, date, analyst):
"""
Edit whois information for a domain.
:param domain: The domain for the whois entry.
:type domain: str
:param data: The whois data.
:type data: str
:param date: The date for the whois data.
:type date: datetime.datetime
:param analyst: The user editing the domain name.
:type analyst: str
:returns: dict with keys:
"success" (boolean),
"message" (str) if failed.
"""
domain = Domain.objects(domain=domain).first()
if not domain:
return {'success': False, 'message': "No matching domain found."}
try:
domain.edit_whois(data, date)
domain.save(username=analyst)
return {'success': True}
except ValidationError, e:
return {'success': False, 'message': e}
def add_whois(domain, data, date, analyst, editable):
"""
Add whois information to a domain.
:param domain: The domain for the whois entry.
:type domain: str
:param data: The whois data.
:type data: str
:param date: The date for the whois data.
:type date: datetime.datetime
:param analyst: The user editing the domain name.
:type analyst: str
:param editable: If this entry can be modified.
:type editable: boolean
:returns: dict with keys:
"success" (boolean),
"whois" (str) if successful,
"message" (str) if failed.
"""
domain = Domain.objects(domain=domain).first()
if not domain:
return {'success': False, 'message': "No matching domain found."}
try:
whois_entry = domain.add_whois(data, analyst, date, editable)
domain.save(username=analyst)
return {"success": True, 'whois': whois_entry}
except ValidationError, e:
return {"success": False, "message": e}
def __init__(self, *args, **kwargs):
#populate date choices
self.domain = ""
allow_adding = True
if 'domain' in kwargs:
self.domain = kwargs['domain']
del kwargs['domain'] #keep the default __init__ from erroring out
if 'allow_adding' in kwargs:
allow_adding = kwargs['allow_adding']
del kwargs['allow_adding']
super(UpdateWhoisForm, self).__init__(*args, **kwargs)
if self.domain:
if allow_adding:
date_choices = [("","Add New")]
else:
date_choices = []
dmain = Domain.objects(domain=self.domain).first()
if dmain:
whois = dmain.whois
whois.sort(key=lambda w: w['date'], reverse=True)
for w in dmain.whois:
date = datetime.strftime(w['date'],
settings.PY_DATETIME_FORMAT)
date_choices.append((date,date))
self.fields['date'].choices = date_choices
def edit_domain_name(domain, new_domain, analyst):
"""
Edit domain name for an entry.
:param domain: The domain name to edit.
:type domain: str
:param new_domain: The new domain name.
:type new_domain: str
:param analyst: The user editing the domain name.
:type analyst: str
:returns: boolean
"""
# validate new domain
(root, validated_domain, error) = get_valid_root_domain(new_domain)
if error:
return False
domain = Domain.objects(domain=domain).first()
if not domain:
return False
try:
domain.domain = validated_domain
domain.save(username=analyst)
return True
except ValidationError:
return False
def edit_domain_name(domain, new_domain, analyst):
"""
Edit domain name for an entry.
:param domain: The domain name to edit.
:type domain: str
:param new_domain: The new domain name.
:type new_domain: str
:param analyst: The user editing the domain name.
:type analyst: str
:returns: boolean
"""
# validate new domain
(root, validated_domain, error) = get_valid_root_domain(new_domain)
if error:
return False
domain = Domain.objects(domain=domain).first()
if not domain:
return False
try:
domain.domain = validated_domain
domain.save(username=analyst)
return True
except ValidationError:
return False
def add_whois(domain, data, date, analyst, editable):
"""
Add whois information to a domain.
:param domain: The domain for the whois entry.
:type domain: str
:param data: The whois data.
:type data: str
:param date: The date for the whois data.
:type date: datetime.datetime
:param analyst: The user editing the domain name.
:type analyst: str
:param editable: If this entry can be modified.
:type editable: boolean
:returns: dict with keys:
"success" (boolean),
"whois" (str) if successful,
"message" (str) if failed.
"""
domain = Domain.objects(domain=domain).first()
if not domain:
return {'success': False,
'message': "No matching domain found."}
try:
whois_entry = domain.add_whois(data, analyst, date, editable)
domain.save(username=analyst)
return {"success": True, 'whois': whois_entry}
except ValidationError, e:
return {"success": False, "message": e}
def create_domain_context(self, identifier, username):
domain = Domain.objects(id=identifier).first()
if not domain:
raise ValueError("Domain not found in database")
return DomainContext(username=username,
_id=identifier,
domain_dict=domain.to_dict())
def create_domain_context(self, identifier, username):
domain = Domain.objects(id=identifier).first()
if not domain:
raise ValueError("Domain not found in database")
return DomainContext(username=username,
_id=identifier,
domain_dict=domain.to_dict())
def process_bulk_add_domain(request, formdict):
"""
Performs the bulk add of domains by parsing the request data. Batches
some data into a cache object for performance by reducing large
amounts of single database queries.
:param request: Django request.
:type request: :class:`django.http.HttpRequest`
:param formdict: The form representing the bulk uploaded data.
:type formdict: dict
:returns: :class:`django.http.HttpResponse`
"""
domain_names = []
ip_addresses = []
cached_domain_results = {}
cached_ip_results = {}
cleanedRowsData = convert_handsontable_to_rows(request)
for rowData in cleanedRowsData:
if rowData != None:
if rowData.get(form_consts.Domain.DOMAIN_NAME) != None:
domain = rowData.get(
form_consts.Domain.DOMAIN_NAME).strip().lower()
(root_domain, full_domain,
error) = get_valid_root_domain(domain)
domain_names.append(full_domain)
if domain != root_domain:
domain_names.append(root_domain)
if rowData.get(form_consts.Domain.IP_ADDRESS) != None:
ip_addr = rowData.get(form_consts.Domain.IP_ADDRESS)
ip_type = rowData.get(form_consts.Domain.IP_TYPE)
(ip_addr, error) = validate_and_normalize_ip(ip_addr, ip_type)
ip_addresses.append(ip_addr)
domain_results = Domain.objects(domain__in=domain_names)
ip_results = IP.objects(ip__in=ip_addresses)
for domain_result in domain_results:
cached_domain_results[domain_result.domain] = domain_result
for ip_result in ip_results:
cached_ip_results[ip_result.ip] = ip_result
cache = {
form_consts.Domain.CACHED_RESULTS: cached_domain_results,
form_consts.IP.CACHED_RESULTS: cached_ip_results,
'cleaned_rows_data': cleanedRowsData
}
response = parse_bulk_upload(request, parse_row_to_bound_domain_form,
add_new_domain_via_bulk, formdict, cache)
return response
def clean_date(self):
date = self.cleaned_data['date']
if date:
date_obj = datetime.strptime(self.cleaned_data['date'],
settings.PY_DATETIME_FORMAT)
domain = Domain.objects(domain=self.domain,
whois__date=date_obj).first()
if not domain:
raise forms.ValidationError(u'%s is not a valid date.' % date)
def clean_date(self):
date = self.cleaned_data['date']
if date:
date_obj = datetime.strptime(self.cleaned_data['date'],
settings.PY_DATETIME_FORMAT)
domain = Domain.objects(domain=self.domain,
whois__date=date_obj).first()
if not domain:
raise forms.ValidationError(u'%s is not a valid date.' % date)
def class_from_value(type_, value):
"""
Return an instantiated class object.
:param type_: The CRITs top-level object type.
:type type_: str
:param value: The value to search for.
:type value: str
:returns: class which inherits from
:class:`crits.core.crits_mongoengine.CritsBaseAttributes`
"""
# doing this to avoid circular imports
from crits.campaigns.campaign import Campaign
from crits.certificates.certificate import Certificate
from crits.comments.comment import Comment
from crits.domains.domain import Domain
from crits.emails.email import Email
from crits.events.event import Event
from crits.indicators.indicator import Indicator
from crits.ips.ip import IP
from crits.pcaps.pcap import PCAP
from crits.raw_data.raw_data import RawData
from crits.samples.sample import Sample
from crits.screenshots.screenshot import Screenshot
from crits.targets.target import Target
if type_ == 'Campaign':
return Campaign.objects(name=value).first()
elif type_ == 'Certificate':
return Certificate.objects(md5=value).first()
elif type_ == 'Comment':
return Comment.objects(id=value).first()
elif type_ == 'Domain':
return Domain.objects(domain=value).first()
elif type_ == 'Email':
return Email.objects(id=value).first()
elif type_ == 'Event':
return Event.objects(id=value).first()
elif type_ == 'Indicator':
return Indicator.objects(id=value).first()
elif type_ == 'IP':
return IP.objects(ip=value).first()
elif type_ == 'PCAP':
return PCAP.objects(md5=value).first()
elif type_ == 'RawData':
return RawData.objects(md5=value).first()
elif type_ == 'Sample':
return Sample.objects(md5=value).first()
elif type_ == 'Screenshot':
return Screenshot.objects(id=value).first()
elif type_ == 'Target':
return Target.objects(email_address=value).first()
else:
return None
def class_from_value(type_, value):
"""
Return an instantiated class object.
:param type_: The CRITs top-level object type.
:type type_: str
:param value: The value to search for.
:type value: str
:returns: class which inherits from
:class:`crits.core.crits_mongoengine.CritsBaseAttributes`
"""
# doing this to avoid circular imports
from crits.campaigns.campaign import Campaign
from crits.certificates.certificate import Certificate
from crits.comments.comment import Comment
from crits.domains.domain import Domain
from crits.emails.email import Email
from crits.events.event import Event
from crits.indicators.indicator import Indicator
from crits.ips.ip import IP
from crits.pcaps.pcap import PCAP
from crits.raw_data.raw_data import RawData
from crits.samples.sample import Sample
from crits.screenshots.screenshot import Screenshot
from crits.targets.target import Target
if type_ == 'Campaign':
return Campaign.objects(name=value).first()
elif type_ == 'Certificate':
return Certificate.objects(md5=value).first()
elif type_ == 'Comment':
return Comment.objects(id=value).first()
elif type_ == 'Domain':
return Domain.objects(domain=value).first()
elif type_ == 'Email':
return Email.objects(id=value).first()
elif type_ == 'Event':
return Event.objects(id=value).first()
elif type_ == 'Indicator':
return Indicator.objects(id=value).first()
elif type_ == 'IP':
return IP.objects(ip=value).first()
elif type_ == 'PCAP':
return PCAP.objects(md5=value).first()
elif type_ == 'RawData':
return RawData.objects(md5=value).first()
elif type_ == 'Sample':
return Sample.objects(md5=value).first()
elif type_ == 'Screenshot':
return Screenshot.objects(id=value).first()
elif type_ == 'Target':
return Target.objects(email_address=value).first()
else:
return None
def process_bulk_add_domain(request, formdict):
"""
Performs the bulk add of domains by parsing the request data. Batches
some data into a cache object for performance by reducing large
amounts of single database queries.
:param request: Django request.
:type request: :class:`django.http.HttpRequest`
:param formdict: The form representing the bulk uploaded data.
:type formdict: dict
:returns: :class:`django.http.HttpResponse`
"""
domain_names = []
ip_addresses = []
cached_domain_results = {}
cached_ip_results = {}
cleanedRowsData = convert_handsontable_to_rows(request)
for rowData in cleanedRowsData:
if rowData != None:
if rowData.get(form_consts.Domain.DOMAIN_NAME) != None:
domain = rowData.get(form_consts.Domain.DOMAIN_NAME).strip().lower()
(root_domain, full_domain, error) = get_valid_root_domain(domain)
domain_names.append(full_domain)
if domain != root_domain:
domain_names.append(root_domain)
if rowData.get(form_consts.Domain.IP_ADDRESS) != None:
ip_addr = rowData.get(form_consts.Domain.IP_ADDRESS)
ip_type = rowData.get(form_consts.Domain.IP_TYPE)
(ip_addr, error) = validate_and_normalize_ip(ip_addr, ip_type)
ip_addresses.append(ip_addr)
domain_results = Domain.objects(domain__in=domain_names)
ip_results = IP.objects(ip__in=ip_addresses)
for domain_result in domain_results:
cached_domain_results[domain_result.domain] = domain_result
for ip_result in ip_results:
cached_ip_results[ip_result.ip] = ip_result
cache = {
form_consts.Domain.CACHED_RESULTS: cached_domain_results,
form_consts.IP.CACHED_RESULTS: cached_ip_results,
"cleaned_rows_data": cleanedRowsData,
}
response = parse_bulk_upload(request, parse_row_to_bound_domain_form, add_new_domain_via_bulk, formdict, cache)
return response
def _delete_all_analysis_results(self, md5_digest, service_name):
"""
Delete all analysis results for this service.
"""
obj = Sample.objects(md5=md5_digest).first()
if obj:
obj.analysis[:] = [
a for a in obj.analysis if a.service_name != service_name
]
obj.save()
obj = PCAP.objects(md5=md5_digest).first()
if obj:
obj.analysis[:] = [
a for a in obj.analysis if a.service_name != service_name
]
obj.save()
obj = Certificate.objects(md5=md5_digest).first()
if obj:
obj.analysis[:] = [
a for a in obj.analysis if a.service_name != service_name
]
obj.save()
obj = RawData.objects(id=md5_digest).first()
if obj:
obj.analysis[:] = [
a for a in obj.analysis if a.service_name != service_name
]
obj.save()
obj = Event.objects(id=md5_digest).first()
if obj:
obj.analysis[:] = [
a for a in obj.analysis if a.service_name != service_name
]
obj.save()
obj = Indicator.objects(id=md5_digest).first()
if obj:
obj.analysis[:] = [
a for a in obj.analysis if a.service_name != service_name
]
obj.save()
obj = Domain.objects(id=md5_digest).first()
if obj:
obj.analysis[:] = [
a for a in obj.analysis if a.service_name != service_name
]
obj.save()
obj = IP.objects(id=md5_digest).first()
if obj:
obj.analysis[:] = [
a for a in obj.analysis if a.service_name != service_name
]
obj.save()
def whois_diff(domain, from_date, to_date):
"""
Diff whois entries.
:param domain: The domain to get whois information for.
:type domain: str
:param from_date: The date for the first whois entry.
:type from_date: datetime.datetime
:param to_date: The date for the second whois entry.
:type to_date: datetime.datetime
:returns: dict if failed, str on success.
"""
domain = Domain.objects(domain=domain).first()
if not domain:
return {'success': False, 'message': "No matching domain found."}
return domain.whois_diff(from_date, to_date)
def whois_diff(domain, from_date, to_date):
"""
Diff whois entries.
:param domain: The domain to get whois information for.
:type domain: str
:param from_date: The date for the first whois entry.
:type from_date: datetime.datetime
:param to_date: The date for the second whois entry.
:type to_date: datetime.datetime
:returns: dict if failed, str on success.
"""
domain = Domain.objects(domain=domain).first()
if not domain:
return {"success": False, "message": "No matching domain found."}
return domain.whois_diff(from_date, to_date)
def __init__(self, *args, **kwargs):
#populate date choices
domain = ""
if 'domain' in kwargs:
domain = kwargs['domain']
del kwargs['domain'] #keep the default __init__ from erroring out
super(DiffWhoisForm, self).__init__(*args, **kwargs)
if domain:
date_choices = [("","Select Date To Compare")]
dmain = Domain.objects(domain=domain).first()
if dmain:
whois = dmain.whois
whois.sort(key=lambda w: w['date'], reverse=True)
for w in dmain.whois:
date = datetime.strftime(w['date'],
settings.PY_DATETIME_FORMAT)
date_choices.append((date,date))
self.fields['from_date'].choices = self.fields['to_date'].choices = date_choices
def __init__(self, *args, **kwargs):
#populate date choices
domain = ""
if 'domain' in kwargs:
domain = kwargs['domain']
del kwargs['domain'] #keep the default __init__ from erroring out
super(DiffWhoisForm, self).__init__(*args, **kwargs)
if domain:
date_choices = [("", "Select Date To Compare")]
dmain = Domain.objects(domain=domain).first()
if dmain:
whois = dmain.whois
whois.sort(key=lambda w: w['date'], reverse=True)
for w in dmain.whois:
date = datetime.strftime(w['date'],
settings.PY_DATETIME_FORMAT)
date_choices.append((date, date))
self.fields['from_date'].choices = self.fields[
'to_date'].choices = date_choices
def retrieve_domain(domain, cache):
"""
Retrieves a domain by checking cache first. If not in cache
then queries mongo for the domain.
:param domain: The domain name.
:type domain: str
:param cache: Cached data, typically for performance enhancements
during bulk uperations.
:type cache: dict
:returns: :class:`crits.domains.domain.Domain`
"""
domain_obj = None
cached_results = cache.get(form_consts.Domain.CACHED_RESULTS)
if cached_results != None:
domain_obj = cached_results.get(domain.lower())
else:
domain_obj = Domain.objects(domain_iexact=domain).first()
return domain_obj
def retrieve_domain(domain, cache):
"""
Retrieves a domain by checking cache first. If not in cache
then queries mongo for the domain.
:param domain: The domain name.
:type domain: str
:param cache: Cached data, typically for performance enhancements
during bulk uperations.
:type cache: dict
:returns: :class:`crits.domains.domain.Domain`
"""
domain_obj = None
cached_results = cache.get(form_consts.Domain.CACHED_RESULTS)
if cached_results != None:
domain_obj = cached_results.get(domain.lower())
else:
domain_obj = Domain.objects(domain_iexact=domain).first()
return domain_obj
def _delete_all_analysis_results(self, md5_digest, service_name):
"""
Delete all analysis results for this service.
"""
obj = Sample.objects(md5=md5_digest).first()
if obj:
obj.analysis[:] = [a for a in obj.analysis if a.service_name != service_name]
obj.save()
obj = PCAP.objects(md5=md5_digest).first()
if obj:
obj.analysis[:] = [a for a in obj.analysis if a.service_name != service_name]
obj.save()
obj = Certificate.objects(md5=md5_digest).first()
if obj:
obj.analysis[:] = [a for a in obj.analysis if a.service_name != service_name]
obj.save()
obj = RawData.objects(id=md5_digest).first()
if obj:
obj.analysis[:] = [a for a in obj.analysis if a.service_name != service_name]
obj.save()
obj = Event.objects(id=md5_digest).first()
if obj:
obj.analysis[:] = [a for a in obj.analysis if a.service_name != service_name]
obj.save()
obj = Indicator.objects(id=md5_digest).first()
if obj:
obj.analysis[:] = [a for a in obj.analysis if a.service_name != service_name]
obj.save()
obj = Domain.objects(id=md5_digest).first()
if obj:
obj.analysis[:] = [a for a in obj.analysis if a.service_name != service_name]
obj.save()
obj = IP.objects(id=md5_digest).first()
if obj:
obj.analysis[:] = [a for a in obj.analysis if a.service_name != service_name]
obj.save()
def get_domain_details(domain, analyst):
"""
Generate the data to render the Domain details template.
:param domain: The name of the Domain to get details for.
:type domain: str
:param analyst: The user requesting this information.
:type analyst: str
:returns: template (str), arguments (dict)
"""
template = None
allowed_sources = user_sources(analyst)
dmain = Domain.objects(domain=domain, source__name__in=allowed_sources).first()
if not dmain:
error = "Either no data exists for this domain" " or you do not have permission to view it."
template = "error.html"
args = {"error": error}
return template, args
dmain.sanitize_sources(username="******" % analyst, sources=allowed_sources)
# remove pending notifications for user
remove_user_from_notification("%s" % analyst, dmain.id, "Domain")
# subscription
subscription = {
"type": "Domain",
"id": dmain.id,
"subscribed": is_user_subscribed("%s" % analyst, "Domain", dmain.id),
}
# objects
objects = dmain.sort_objects()
# relationships
relationships = dmain.sort_relationships("%s" % analyst, meta=True)
# relationship
relationship = {"type": "Domain", "value": dmain.id}
# comments
comments = {"comments": dmain.get_comments(), "url_key": dmain.domain}
# screenshots
screenshots = dmain.get_screenshots(analyst)
# favorites
favorite = is_user_favorite("%s" % analyst, "Domain", dmain.id)
# services
service_list = get_supported_services("Domain")
# analysis results
service_results = dmain.get_analysis_results()
args = {
"objects": objects,
"relationships": relationships,
"comments": comments,
"favorite": favorite,
"relationship": relationship,
"subscription": subscription,
"screenshots": screenshots,
"domain": dmain,
"service_list": service_list,
"service_results": service_results,
}
return template, args
def get_domain_details(domain, analyst):
"""
Generate the data to render the Domain details template.
:param domain: The name of the Domain to get details for.
:type domain: str
:param analyst: The user requesting this information.
:type analyst: str
:returns: template (str), arguments (dict)
"""
template = None
allowed_sources = user_sources(analyst)
dmain = Domain.objects(domain=domain,
source__name__in=allowed_sources).first()
if not dmain:
error = ("Either no data exists for this domain"
" or you do not have permission to view it.")
template = "error.html"
args = {'error': error}
return template, args
dmain.sanitize_sources(username="******" % analyst,
sources=allowed_sources)
# remove pending notifications for user
remove_user_from_notification("%s" % analyst, dmain.id, 'Domain')
# subscription
subscription = {
'type': 'Domain',
'id': dmain.id,
'subscribed': is_user_subscribed("%s" % analyst,
'Domain',
dmain.id),
}
#objects
objects = dmain.sort_objects()
#relationships
relationships = dmain.sort_relationships("%s" % analyst, meta=True)
# relationship
relationship = {
'type': 'Domain',
'value': dmain.id
}
#comments
comments = {'comments': dmain.get_comments(),
'url_key':dmain.domain}
#screenshots
screenshots = dmain.get_screenshots(analyst)
# favorites
favorite = is_user_favorite("%s" % analyst, 'Domain', dmain.id)
# services
service_list = get_supported_services('Domain')
# analysis results
service_results = dmain.get_analysis_results()
args = {'objects': objects,
'relationships': relationships,
'comments': comments,
'favorite': favorite,
'relationship': relationship,
'subscription': subscription,
'screenshots': screenshots,
'domain': dmain,
'service_list': service_list,
'service_results': service_results}
return template, args
def upsert_domain(domain, source, username=None, campaign=None,
confidence=None, bucket_list=None, ticket=None, cache={}):
"""
Add or update a domain/FQDN. Campaign is assumed to be a list of campaign
dictionary objects.
:param domain: The domain to add/update.
:type domain: str
:param source: The name of the source.
:type source: str
:param username: The user adding/updating the domain.
:type username: str
:param campaign: The campaign to attribute to this domain.
:type campaign: list, str
:param confidence: Confidence for the campaign attribution.
:type confidence: str
:param bucket_list: List of buckets to add to this domain.
:type bucket_list: list, str
:param ticket: The ticket for this domain.
:type ticket: str
:param cache: Cached data, typically for performance enhancements
during bulk uperations.
:type cache: dict
:returns: dict with keys:
"success" (boolean),
"object" the domain that was added,
"is_domain_new" (boolean)
"""
# validate domain and grab root domain
(root, domain, error) = get_valid_root_domain(domain)
if error:
return {'success': False, 'message': error}
is_fqdn_domain_new = False
is_root_domain_new = False
if not campaign:
campaign = []
# assume it's a list, but check if it's a string
elif isinstance(campaign, basestring):
c = EmbeddedCampaign(name=campaign, confidence=confidence, analyst=username)
campaign = [c]
# assume it's a list, but check if it's a string
if isinstance(source, basestring):
s = EmbeddedSource()
s.name = source
instance = EmbeddedSource.SourceInstance()
instance.reference = ''
instance.method = ''
instance.analyst = username
instance.date = datetime.datetime.now()
s.instances = [instance]
source = [s]
fqdn_domain = None
root_domain = None
cached_results = cache.get(form_consts.Domain.CACHED_RESULTS)
if cached_results != None:
if domain != root:
fqdn_domain = cached_results.get(domain)
root_domain = cached_results.get(root)
else:
root_domain = cached_results.get(root)
else:
#first find the domain(s) if it/they already exist
root_domain = Domain.objects(domain=root).first()
if domain != root:
fqdn_domain = Domain.objects(domain=domain).first()
#if they don't exist, create them
if not root_domain:
root_domain = Domain()
root_domain.domain = root
root_domain.source = []
root_domain.record_type = 'A'
is_root_domain_new = True
if cached_results != None:
cached_results[root] = root_domain
if domain != root and not fqdn_domain:
fqdn_domain = Domain()
fqdn_domain.domain = domain
fqdn_domain.source = []
fqdn_domain.record_type = 'A'
is_fqdn_domain_new = True
if cached_results != None:
cached_results[domain] = fqdn_domain
# if new or found, append the new source(s)
for s in source:
if root_domain:
root_domain.add_source(s)
if fqdn_domain:
fqdn_domain.add_source(s)
#campaigns
#both root and fqdn get campaigns updated
for c in campaign:
if root_domain:
root_domain.add_campaign(c)
if fqdn_domain:
fqdn_domain.add_campaign(c)
if username:
if root_domain:
root_domain.analyst = username
if fqdn_domain:
fqdn_domain.analyst = username
if bucket_list:
if root_domain:
root_domain.add_bucket_list(bucket_list, username)
if fqdn_domain:
fqdn_domain.add_bucket_list(bucket_list, username)
if ticket:
if root_domain:
root_domain.add_ticket(ticket, username)
if fqdn_domain:
fqdn_domain.add_ticket(ticket, username)
# save
try:
if root_domain:
root_domain.save(username=username)
if fqdn_domain:
fqdn_domain.save(username=username)
except Exception, e:
return {'success': False, 'message': e}
def class_from_id(type_, _id):
"""
Return an instantiated class object.
:param type_: The CRITs top-level object type.
:type type_: str
:param _id: The ObjectId to search for.
:type _id: str
:returns: class which inherits from
:class:`crits.core.crits_mongoengine.CritsBaseAttributes`
"""
# Quick fail
if not _id or not type_:
return None
# doing this to avoid circular imports
from crits.actors.actor import ActorThreatIdentifier, Actor
from crits.backdoors.backdoor import Backdoor
from crits.campaigns.campaign import Campaign
from crits.certificates.certificate import Certificate
from crits.comments.comment import Comment
from crits.core.crits_mongoengine import Action
from crits.core.source_access import SourceAccess
from crits.core.user_role import UserRole
from crits.domains.domain import Domain
from crits.emails.email import Email
from crits.events.event import Event
from crits.exploits.exploit import Exploit
from crits.indicators.indicator import Indicator
from crits.ips.ip import IP
from crits.pcaps.pcap import PCAP
from crits.raw_data.raw_data import RawData, RawDataType
from crits.samples.sample import Sample
from crits.screenshots.screenshot import Screenshot
from crits.signatures.signature import Signature, SignatureType, SignatureDependency
from crits.targets.target import Target
# make sure it's a string
_id = str(_id)
# Use bson.ObjectId to make sure this is a valid ObjectId, otherwise
# the queries below will raise a ValidationError exception.
if not ObjectId.is_valid(_id.decode("utf8")):
return None
if type_ == "Actor":
return Actor.objects(id=_id).first()
elif type_ == "Backdoor":
return Backdoor.objects(id=_id).first()
elif type_ == "ActorThreatIdentifier":
return ActorThreatIdentifier.objects(id=_id).first()
elif type_ == "Campaign":
return Campaign.objects(id=_id).first()
elif type_ == "Certificate":
return Certificate.objects(id=_id).first()
elif type_ == "Comment":
return Comment.objects(id=_id).first()
elif type_ == "Domain":
return Domain.objects(id=_id).first()
elif type_ == "Email":
return Email.objects(id=_id).first()
elif type_ == "Event":
return Event.objects(id=_id).first()
elif type_ == "Exploit":
return Exploit.objects(id=_id).first()
elif type_ == "Indicator":
return Indicator.objects(id=_id).first()
elif type_ == "Action":
return Action.objects(id=_id).first()
elif type_ == "IP":
return IP.objects(id=_id).first()
elif type_ == "PCAP":
return PCAP.objects(id=_id).first()
elif type_ == "RawData":
return RawData.objects(id=_id).first()
elif type_ == "RawDataType":
return RawDataType.objects(id=_id).first()
elif type_ == "Sample":
return Sample.objects(id=_id).first()
elif type_ == "Signature":
return Signature.objects(id=_id).first()
elif type_ == "SignatureType":
return SignatureType.objects(id=_id).first()
elif type_ == "SignatureDependency":
return SignatureDependency.objects(id=_id).first()
elif type_ == "SourceAccess":
return SourceAccess.objects(id=_id).first()
elif type_ == "Screenshot":
return Screenshot.objects(id=_id).first()
elif type_ == "Target":
return Target.objects(id=_id).first()
elif type_ == "UserRole":
return UserRole.objects(id=_id).first()
else:
return None
def get_domain_details(domain, analyst):
"""
Generate the data to render the Domain details template.
:param domain: The name of the Domain to get details for.
:type domain: str
:param analyst: The user requesting this information.
:type analyst: str
:returns: template (str), arguments (dict)
"""
template = None
allowed_sources = user_sources(analyst)
dmain = Domain.objects(domain=domain, source__name__in=allowed_sources).first()
if not dmain:
error = "Either no data exists for this domain" " or you do not have permission to view it."
template = "error.html"
args = {"error": error}
return template, args
forms = {}
# populate whois data into whois form
# and create data object (keyed on date) for updating form on date select
whois_data = {"": ""} # blank info for "Add New" option
initial_data = {"data": " "}
raw_data = {}
whois = getattr(dmain, "whois", None)
if whois:
for w in whois:
# build data as a display-friendly string
w.date = datetime.datetime.strftime(w.date, settings.PY_DATETIME_FORMAT)
from whois_parser import WhoisEntry
# prettify the whois data
w.data = unicode(WhoisEntry.from_dict(w.data))
if "text" not in w: # whois data was added with old data format
w.text = w.data
# also save our text blob for easy viewing of the original data
whois_data[w.date] = (w.data, w.text)
# show most recent entry first
initial_data = {"data": whois[-1].data, "date": whois[-1].date}
raw_data = {"data": whois[-1].text, "date": whois[-1].date}
whois_len = len(whois_data) - 1 # subtract one to account for blank "Add New" entry
whois_data = json.dumps(whois_data)
dmain.sanitize_sources(username="******" % analyst, sources=allowed_sources)
forms["whois"] = UpdateWhoisForm(initial_data, domain=domain)
forms["raw_whois"] = UpdateWhoisForm(raw_data, domain=domain, allow_adding=False)
forms["diff_whois"] = DiffWhoisForm(domain=domain)
# remove pending notifications for user
remove_user_from_notification("%s" % analyst, dmain.id, "Domain")
# subscription
subscription = {
"type": "Domain",
"id": dmain.id,
"subscribed": is_user_subscribed("%s" % analyst, "Domain", dmain.id),
}
# objects
objects = dmain.sort_objects()
# relationships
relationships = dmain.sort_relationships("%s" % analyst, meta=True)
# relationship
relationship = {"type": "Domain", "value": dmain.id}
# comments
comments = {"comments": dmain.get_comments(), "url_key": dmain.domain}
# screenshots
screenshots = dmain.get_screenshots(analyst)
# favorites
favorite = is_user_favorite("%s" % analyst, "Domain", dmain.id)
# services
service_list = get_supported_services("Domain")
# analysis results
service_results = dmain.get_analysis_results()
args = {
"objects": objects,
"relationships": relationships,
"comments": comments,
"favorite": favorite,
"relationship": relationship,
"subscription": subscription,
"screenshots": screenshots,
"domain": dmain,
"forms": forms,
"whois_data": whois_data,
"service_list": service_list,
"service_results": service_results,
"whois_len": whois_len,
}
return template, args
def class_from_id(type_, _id):
"""
Return an instantiated class object.
:param type_: The CRITs top-level object type.
:type type_: str
:param _id: The ObjectId to search for.
:type _id: str
:returns: class which inherits from
:class:`crits.core.crits_mongoengine.CritsBaseAttributes`
"""
# doing this to avoid circular imports
from crits.actors.actor import ActorThreatIdentifier, Actor
from crits.backdoors.backdoor import Backdoor
from crits.campaigns.campaign import Campaign
from crits.certificates.certificate import Certificate
from crits.comments.comment import Comment
from crits.core.crits_mongoengine import Action
from crits.core.source_access import SourceAccess
from crits.core.user_role import UserRole
from crits.domains.domain import Domain
from crits.emails.email import Email
from crits.events.event import Event
from crits.exploits.exploit import Exploit
from crits.indicators.indicator import Indicator
from crits.ips.ip import IP
from crits.pcaps.pcap import PCAP
from crits.raw_data.raw_data import RawData, RawDataType
from crits.samples.sample import Sample
from crits.screenshots.screenshot import Screenshot
from crits.signatures.signature import Signature, SignatureType, SignatureDependency
from crits.targets.target import Target
if not _id:
return None
# make sure it's a string
_id = str(_id)
# Use bson.ObjectId to make sure this is a valid ObjectId, otherwise
# the queries below will raise a ValidationError exception.
if not ObjectId.is_valid(_id.decode('utf8')):
return None
if type_ == 'Actor':
return Actor.objects(id=_id).first()
elif type_ == 'Backdoor':
return Backdoor.objects(id=_id).first()
elif type_ == 'ActorThreatIdentifier':
return ActorThreatIdentifier.objects(id=_id).first()
elif type_ == 'Campaign':
return Campaign.objects(id=_id).first()
elif type_ == 'Certificate':
return Certificate.objects(id=_id).first()
elif type_ == 'Comment':
return Comment.objects(id=_id).first()
elif type_ == 'Domain':
return Domain.objects(id=_id).first()
elif type_ == 'Email':
return Email.objects(id=_id).first()
elif type_ == 'Event':
return Event.objects(id=_id).first()
elif type_ == 'Exploit':
return Exploit.objects(id=_id).first()
elif type_ == 'Indicator':
return Indicator.objects(id=_id).first()
elif type_ == 'Action':
return Action.objects(id=_id).first()
elif type_ == 'IP':
return IP.objects(id=_id).first()
elif type_ == 'PCAP':
return PCAP.objects(id=_id).first()
elif type_ == 'RawData':
return RawData.objects(id=_id).first()
elif type_ == 'RawDataType':
return RawDataType.objects(id=_id).first()
elif type_ == 'Sample':
return Sample.objects(id=_id).first()
elif type_ == 'Signature':
return Signature.objects(id=_id).first()
elif type_ == 'SignatureType':
return SignatureType.objects(id=_id).first()
elif type_ == 'SignatureDependency':
return SignatureDependency.objects(id=_id).first()
elif type_ == 'SourceAccess':
return SourceAccess.objects(id=_id).first()
elif type_ == 'Screenshot':
return Screenshot.objects(id=_id).first()
elif type_ == 'Target':
return Target.objects(id=_id).first()
elif type_ == 'UserRole':
return UserRole.objects(id=_id).first()
else:
return None
def class_from_value(type_, value):
"""
Return an instantiated class object.
:param type_: The CRITs top-level object type.
:type type_: str
:param value: The value to search for.
:type value: str
:returns: class which inherits from
:class:`crits.core.crits_mongoengine.CritsBaseAttributes`
"""
# doing this to avoid circular imports
from crits.actors.actor import ActorThreatIdentifier, Actor
from crits.backdoors.backdoor import Backdoor
from crits.campaigns.campaign import Campaign
from crits.certificates.certificate import Certificate
from crits.comments.comment import Comment
from crits.domains.domain import Domain
from crits.emails.email import Email
from crits.events.event import Event
from crits.exploits.exploit import Exploit
from crits.indicators.indicator import Indicator
from crits.ips.ip import IP
from crits.pcaps.pcap import PCAP
from crits.raw_data.raw_data import RawData
from crits.samples.sample import Sample
from crits.screenshots.screenshot import Screenshot
from crits.signatures.signature import Signature
from crits.targets.target import Target
# Make sure value is a string...
value = str(value)
# Use bson.ObjectId to make sure this is a valid ObjectId, otherwise
# the queries below will raise a ValidationError exception.
if (type_ in [
'Backdoor', 'Comment', 'Event', 'Exploit', 'Indicator',
'Screenshot'
] and not ObjectId.is_valid(value.decode('utf8'))):
return None
if type_ == 'Actor':
return Actor.objects(name=value).first()
if type_ == 'Backdoor':
return Backdoor.objects(id=value).first()
elif type_ == 'ActorThreatIdentifier':
return ActorThreatIdentifier.objects(name=value).first()
elif type_ == 'Campaign':
return Campaign.objects(name=value).first()
elif type_ == 'Certificate':
return Certificate.objects(md5=value).first()
elif type_ == 'Comment':
return Comment.objects(id=value).first()
elif type_ == 'Domain':
return Domain.objects(domain=value).first()
elif type_ == 'Email':
return Email.objects(message_id=value).first()
elif type_ == 'Event':
return Event.objects(id=value).first()
elif type_ == 'Exploit':
return Exploit.objects(id=value).first()
elif type_ == 'Indicator':
return Indicator.objects(id=value).first()
elif type_ == 'IP':
return IP.objects(ip=value).first()
elif type_ == 'PCAP':
return PCAP.objects(md5=value).first()
elif type_ == 'RawData':
return RawData.objects(md5=value).first()
elif type_ == 'Sample':
return Sample.objects(md5=value).first()
elif type_ == 'Screenshot':
return Screenshot.objects(id=value).first()
elif type_ == 'Signature':
return Signature.objects(md5=value).first()
elif type_ == 'Target':
target = Target.objects(email_address=value).first()
if target:
return target
else:
return Target.objects(email_address__iexact=value).first()
else:
return None
def class_from_id(type_, _id):
"""
Return an instantiated class object.
:param type_: The CRITs top-level object type.
:type type_: str
:param _id: The ObjectId to search for.
:type _id: str
:returns: class which inherits from
:class:`crits.core.crits_mongoengine.CritsBaseAttributes`
"""
# doing this to avoid circular imports
from crits.actors.actor import ActorThreatIdentifier, Actor
from crits.backdoors.backdoor import Backdoor
from crits.campaigns.campaign import Campaign
from crits.certificates.certificate import Certificate
from crits.comments.comment import Comment
from crits.core.source_access import SourceAccess
from crits.core.user_role import UserRole
from crits.domains.domain import Domain
from crits.emails.email import Email
from crits.events.event import Event
from crits.exploits.exploit import Exploit
from crits.indicators.indicator import Indicator, IndicatorAction
from crits.ips.ip import IP
from crits.pcaps.pcap import PCAP
from crits.raw_data.raw_data import RawData, RawDataType
from crits.samples.sample import Sample
from crits.screenshots.screenshot import Screenshot
from crits.targets.target import Target
if not _id:
return None
# make sure it's a string
_id = str(_id)
# Use bson.ObjectId to make sure this is a valid ObjectId, otherwise
# the queries below will raise a ValidationError exception.
if not ObjectId.is_valid(_id.decode('utf8')):
return None
if type_ == 'Actor':
return Actor.objects(id=_id).first()
elif type_ == 'Backdoor':
return Backdoor.objects(id=_id).first()
elif type_ == 'ActorThreatIdentifier':
return ActorThreatIdentifier.objects(id=_id).first()
elif type_ == 'Campaign':
return Campaign.objects(id=_id).first()
elif type_ == 'Certificate':
return Certificate.objects(id=_id).first()
elif type_ == 'Comment':
return Comment.objects(id=_id).first()
elif type_ == 'Domain':
return Domain.objects(id=_id).first()
elif type_ == 'Email':
return Email.objects(id=_id).first()
elif type_ == 'Event':
return Event.objects(id=_id).first()
elif type_ == 'Exploit':
return Exploit.objects(id=_id).first()
elif type_ == 'Indicator':
return Indicator.objects(id=_id).first()
elif type_ == 'IndicatorAction':
return IndicatorAction.objects(id=_id).first()
elif type_ == 'IP':
return IP.objects(id=_id).first()
elif type_ == 'PCAP':
return PCAP.objects(id=_id).first()
elif type_ == 'RawData':
return RawData.objects(id=_id).first()
elif type_ == 'RawDataType':
return RawDataType.objects(id=_id).first()
elif type_ == 'Sample':
return Sample.objects(id=_id).first()
elif type_ == 'SourceAccess':
return SourceAccess.objects(id=_id).first()
elif type_ == 'Screenshot':
return Screenshot.objects(id=_id).first()
elif type_ == 'Target':
return Target.objects(id=_id).first()
elif type_ == 'UserRole':
return UserRole.objects(id=_id).first()
else:
return None
def class_from_value(type_, value):
"""
Return an instantiated class object.
:param type_: The CRITs top-level object type.
:type type_: str
:param value: The value to search for.
:type value: str
:returns: class which inherits from
:class:`crits.core.crits_mongoengine.CritsBaseAttributes`
"""
# doing this to avoid circular imports
from crits.actors.actor import ActorThreatIdentifier, Actor
from crits.backdoors.backdoor import Backdoor
from crits.campaigns.campaign import Campaign
from crits.certificates.certificate import Certificate
from crits.comments.comment import Comment
from crits.domains.domain import Domain
from crits.emails.email import Email
from crits.events.event import Event
from crits.exploits.exploit import Exploit
from crits.indicators.indicator import Indicator
from crits.ips.ip import IP
from crits.pcaps.pcap import PCAP
from crits.raw_data.raw_data import RawData
from crits.samples.sample import Sample
from crits.screenshots.screenshot import Screenshot
from crits.targets.target import Target
# Make sure value is a string...
value = str(value)
# Use bson.ObjectId to make sure this is a valid ObjectId, otherwise
# the queries below will raise a ValidationError exception.
if (type_ in ['Backdoor', 'Comment', 'Email', 'Event', 'Exploit',
'Indicator', 'Screenshot'] and
not ObjectId.is_valid(value.decode('utf8'))):
return None
if type_ == 'Actor':
return Actor.objects(name=value).first()
if type_ == 'Backdoor':
return Backdoor.objects(id=value).first()
elif type_ == 'ActorThreatIdentifier':
return ActorThreatIdentifier.objects(name=value).first()
elif type_ == 'Campaign':
return Campaign.objects(name=value).first()
elif type_ == 'Certificate':
return Certificate.objects(md5=value).first()
elif type_ == 'Comment':
return Comment.objects(id=value).first()
elif type_ == 'Domain':
return Domain.objects(domain=value).first()
elif type_ == 'Email':
return Email.objects(id=value).first()
elif type_ == 'Event':
return Event.objects(id=value).first()
elif type_ == 'Exploit':
return Exploit.objects(id=value).first()
elif type_ == 'Indicator':
return Indicator.objects(id=value).first()
elif type_ == 'IP':
return IP.objects(ip=value).first()
elif type_ == 'PCAP':
return PCAP.objects(md5=value).first()
elif type_ == 'RawData':
return RawData.objects(md5=value).first()
elif type_ == 'Sample':
return Sample.objects(md5=value).first()
elif type_ == 'Screenshot':
return Screenshot.objects(id=value).first()
elif type_ == 'Target':
target = Target.objects(email_address=value).first()
if target:
return target
else:
return Target.objects(email_address__iexact=value).first()
else:
return None
def get_domain_details(domain, analyst):
"""
Generate the data to render the Domain details template.
:param domain: The name of the Domain to get details for.
:type domain: str
:param analyst: The user requesting this information.
:type analyst: str
:returns: template (str), arguments (dict)
"""
template = None
allowed_sources = user_sources(analyst)
dmain = Domain.objects(domain=domain,
source__name__in=allowed_sources).first()
if not dmain:
error = ("Either no data exists for this domain"
" or you do not have permission to view it.")
template = "error.html"
args = {'error': error}
return template, args
forms = {}
#populate whois data into whois form
# and create data object (keyed on date) for updating form on date select
whois_data = {'':''} #blank info for "Add New" option
initial_data = {'data':' '}
raw_data = {}
whois = getattr(dmain, 'whois', None)
if whois:
for w in whois:
#build data as a display-friendly string
w.date = datetime.datetime.strftime(w.date,
settings.PY_DATETIME_FORMAT)
from whois_parser import WhoisEntry
#prettify the whois data
w.data = unicode(WhoisEntry.from_dict(w.data))
if 'text' not in w: #whois data was added with old data format
w.text = w.data
#also save our text blob for easy viewing of the original data
whois_data[w.date] = (w.data, w.text)
#show most recent entry first
initial_data = {'data':whois[-1].data, 'date': whois[-1].date}
raw_data = {'data':whois[-1].text, 'date': whois[-1].date}
whois_len = len(whois_data)-1 #subtract one to account for blank "Add New" entry
whois_data = json.dumps(whois_data)
dmain.sanitize_sources(username="******" % analyst,
sources=allowed_sources)
forms['whois'] = UpdateWhoisForm(initial_data,
domain=domain)
forms['raw_whois'] = UpdateWhoisForm(raw_data,
domain=domain,
allow_adding=False)
forms['diff_whois'] = DiffWhoisForm(domain=domain)
# remove pending notifications for user
remove_user_from_notification("%s" % analyst, dmain.id, 'Domain')
# subscription
subscription = {
'type': 'Domain',
'id': dmain.id,
'subscribed': is_user_subscribed("%s" % analyst,
'Domain',
dmain.id),
}
#objects
objects = dmain.sort_objects()
#relationships
relationships = dmain.sort_relationships("%s" % analyst, meta=True)
# relationship
relationship = {
'type': 'Domain',
'value': dmain.id
}
#comments
comments = {'comments': dmain.get_comments(),
'url_key':dmain.domain}
#screenshots
screenshots = dmain.get_screenshots(analyst)
# favorites
favorite = is_user_favorite("%s" % analyst, 'Domain', dmain.id)
# services
manager = crits.service_env.manager
service_list = manager.get_supported_services('Domain', True)
args = {'objects': objects,
'relationships': relationships,
'comments': comments,
'favorite': favorite,
'relationship': relationship,
'subscription': subscription,
'screenshots': screenshots,
'domain': dmain,
'forms': forms,
'whois_data': whois_data,
'service_list': service_list,
'whois_len': whois_len}
return template, args
def class_from_id(type_, _id):
"""
Return an instantiated class object.
:param type_: The CRITs top-level object type.
:type type_: str
:param _id: The ObjectId to search for.
:type _id: str
:returns: class which inherits from
:class:`crits.core.crits_mongoengine.CritsBaseAttributes`
"""
# doing this to avoid circular imports
from crits.campaigns.campaign import Campaign
from crits.certificates.certificate import Certificate
from crits.comments.comment import Comment
from crits.core.crits_mongoengine import RelationshipType
from crits.core.source_access import SourceAccess
from crits.core.user_role import UserRole
from crits.domains.domain import Domain
from crits.emails.email import Email
from crits.events.event import Event, EventType
from crits.indicators.indicator import Indicator, IndicatorAction
from crits.ips.ip import IP
from crits.objects.object_type import ObjectType
from crits.pcaps.pcap import PCAP
from crits.raw_data.raw_data import RawData, RawDataType
from crits.samples.backdoor import Backdoor
from crits.samples.exploit import Exploit
from crits.samples.sample import Sample
from crits.screenshots.screenshot import Screenshot
from crits.targets.target import Target
if not _id:
return None
# make sure it's a string
_id = str(_id)
if type_ == 'Backdoor':
return Backdoor.objects(id=_id).first()
if type_ == 'Campaign':
return Campaign.objects(id=_id).first()
elif type_ == 'Certificate':
return Certificate.objects(id=_id).first()
elif type_ == 'Comment':
return Comment.objects(id=_id).first()
elif type_ == 'Domain':
return Domain.objects(id=_id).first()
elif type_ == 'Email':
return Email.objects(id=_id).first()
elif type_ == 'Event':
return Event.objects(id=_id).first()
elif type_ == 'EventType':
return EventType.objects(id=_id).first()
elif type_ == 'Exploit':
return Exploit.objects(id=_id).first()
elif type_ == 'Indicator':
return Indicator.objects(id=_id).first()
elif type_ == 'IndicatorAction':
return IndicatorAction.objects(id=_id).first()
elif type_ == 'IP':
return IP.objects(id=_id).first()
elif type_ == 'ObjectType':
return ObjectType.objects(id=_id).first()
elif type_ == 'PCAP':
return PCAP.objects(id=_id).first()
elif type_ == 'RawData':
return RawData.objects(id=_id).first()
elif type_ == 'RawDataType':
return RawDataType.objects(id=_id).first()
elif type_ == 'RelationshipType':
return RelationshipType.objects(id=_id).first()
elif type_ == 'Sample':
return Sample.objects(id=_id).first()
elif type_ == 'SourceAccess':
return SourceAccess.objects(id=_id).first()
elif type_ == 'Screenshot':
return Screenshot.objects(id=_id).first()
elif type_ == 'Target':
return Target.objects(id=_id).first()
elif type_ == 'UserRole':
return UserRole.objects(id=_id).first()
else:
return None
def get_domain_details(domain, analyst):
"""
Generate the data to render the Domain details template.
:param domain: The name of the Domain to get details for.
:type domain: str
:param analyst: The user requesting this information.
:type analyst: str
:returns: template (str), arguments (dict)
"""
template = None
allowed_sources = user_sources(analyst)
dmain = Domain.objects(domain=domain,
source__name__in=allowed_sources).first()
if not dmain:
error = ("Either no data exists for this domain"
" or you do not have permission to view it.")
template = "error.html"
args = {'error': error}
return template, args
forms = {}
#populate whois data into whois form
# and create data object (keyed on date) for updating form on date select
whois_data = {'': ''} #blank info for "Add New" option
initial_data = {'data': ' '}
raw_data = {}
whois = getattr(dmain, 'whois', None)
if whois:
for w in whois:
#build data as a display-friendly string
w.date = datetime.datetime.strftime(w.date,
settings.PY_DATETIME_FORMAT)
from whois_parser import WhoisEntry
#prettify the whois data
w.data = unicode(WhoisEntry.from_dict(w.data))
if 'text' not in w: #whois data was added with old data format
w.text = w.data
#also save our text blob for easy viewing of the original data
whois_data[w.date] = (w.data, w.text)
#show most recent entry first
initial_data = {'data': whois[-1].data, 'date': whois[-1].date}
raw_data = {'data': whois[-1].text, 'date': whois[-1].date}
whois_len = len(
whois_data) - 1 #subtract one to account for blank "Add New" entry
whois_data = json.dumps(whois_data)
dmain.sanitize_sources(username="******" % analyst, sources=allowed_sources)
forms['whois'] = UpdateWhoisForm(initial_data, domain=domain)
forms['raw_whois'] = UpdateWhoisForm(raw_data,
domain=domain,
allow_adding=False)
forms['diff_whois'] = DiffWhoisForm(domain=domain)
# remove pending notifications for user
remove_user_from_notification("%s" % analyst, dmain.id, 'Domain')
# subscription
subscription = {
'type': 'Domain',
'id': dmain.id,
'subscribed': is_user_subscribed("%s" % analyst, 'Domain', dmain.id),
}
#objects
objects = dmain.sort_objects()
#relationships
relationships = dmain.sort_relationships("%s" % analyst, meta=True)
# relationship
relationship = {'type': 'Domain', 'value': dmain.id}
#comments
comments = {'comments': dmain.get_comments(), 'url_key': dmain.domain}
#screenshots
screenshots = dmain.get_screenshots(analyst)
# favorites
favorite = is_user_favorite("%s" % analyst, 'Domain', dmain.id)
# services
manager = crits.service_env.manager
service_list = manager.get_supported_services('Domain', True)
args = {
'objects': objects,
'relationships': relationships,
'comments': comments,
'favorite': favorite,
'relationship': relationship,
'subscription': subscription,
'screenshots': screenshots,
'domain': dmain,
'forms': forms,
'whois_data': whois_data,
'service_list': service_list,
'whois_len': whois_len
}
return template, args
def run(self, argv):
parser = OptionParser()
parser.add_option("-d",
"--domain",
action="store",
dest="domain",
type="string",
help="Domain to use (if not provided, do all)")
parser.add_option("-c",
"--config",
dest="config",
default={},
help="Service configuration")
parser.add_option("-v",
"--verbose",
action="store_true",
dest="verbose",
default=False,
help="Be verbose")
parser.add_option("-n",
"--dry_run",
action="store_true",
dest="dry_run",
default=False,
help="Dry run, just show what would happen.")
(opts, args) = parser.parse_args(argv)
if opts.domain:
if opts.verbose:
print "[+] Using domain: %s" % opts.domain
domain = opts.domain
else:
if opts.verbose:
print "[+] Using ALL domains"
domain = None
config = {}
if opts.config:
config = ast.literal_eval(opts.config)
if not config:
print "No config provided, defaulting to live only."
config['live_query'] = True
else:
print "Using config: %s" % config
query = {
'$or': [{
'whois': {
'$exists': True,
'$not': {
'$size': 0
}
}
}, {
'unsupported_attrs.whois': {
'$exists': True,
'$not': {
'$size': 0
}
}
}]
}
if domain:
query['domain'] = domain
doms = Domain.objects(__raw__=query)
for dom in doms:
print "Executing whois for %s" % dom.domain
if opts.dry_run:
continue
try:
result = run_service('whois',
'Domain',
dom.id,
self.username,
obj=dom,
custom_config=config)
dom.save()
except ServiceAnalysisError as e:
print "Service error: %s" % e
except ValidationError as e:
print "Validation error: %s" % e
def handle_indicator_insert(ind,
source,
reference='',
analyst='',
method='',
add_domain=False,
add_relationship=False,
cache={}):
"""
Insert an individual indicator into the database.
NOTE: Setting add_domain to True will always create a relationship as well.
However, to create a relationship with an object that already exists before
this function was called, set add_relationship to True. This will assume
that the domain or IP object to create the relationship with already exists
and will avoid infinite mutual calls between, for example, add_update_ip
and this function. add domain/IP objects.
:param ind: Information about the indicator.
:type ind: dict
:param source: The source for this indicator.
:type source: list, str, :class:`crits.core.crits_mongoengine.EmbeddedSource`
:param reference: The reference to the data.
:type reference: str
:param analyst: The user adding this indicator.
:type analyst: str
:param method: Method of acquiring this indicator.
:type method: str
:param add_domain: If this indicator is also a top-level object, try to add
it.
:type add_domain: boolean
:param add_relationship: Attempt to add relationships if applicable.
:type add_relationship: boolean
:param cache: Cached data, typically for performance enhancements
during bulk uperations.
:type cache: dict
:returns: dict with keys:
"success" (boolean),
"message" str) if failed,
"objectid" (str) if successful,
"is_new_indicator" (boolean) if successful.
"""
is_new_indicator = False
rank = {'unknown': 0, 'benign': 1, 'low': 2, 'medium': 3, 'high': 4}
indicator = Indicator.objects(ind_type=ind['type'],
value=ind['value']).first()
if not indicator:
indicator = Indicator()
indicator.ind_type = ind['type']
indicator.value = ind['value']
indicator.created = datetime.datetime.now()
indicator.confidence = EmbeddedConfidence(analyst=analyst)
indicator.impact = EmbeddedImpact(analyst=analyst)
is_new_indicator = True
ec = None
if 'campaign' in ind:
confidence = 'low'
if 'campaign_confidence' in ind:
confidence = ind['campaign_confidence']
ec = EmbeddedCampaign(name=ind['campaign'],
confidence=confidence,
description="",
analyst=analyst,
date=datetime.datetime.now())
if 'confidence' in ind and rank.get(ind['confidence'], 0) > rank.get(
indicator.confidence.rating, 0):
indicator.confidence.rating = ind['confidence']
indicator.confidence.analyst = analyst
if 'impact' in ind and rank.get(ind['impact'], 0) > rank.get(
indicator.impact.rating, 0):
indicator.impact.rating = ind['impact']
indicator.impact.analyst = analyst
bucket_list = None
if form_consts.Common.BUCKET_LIST_VARIABLE_NAME in ind:
bucket_list = ind[form_consts.Common.BUCKET_LIST_VARIABLE_NAME]
indicator.add_bucket_list(bucket_list, analyst)
ticket = None
if form_consts.Common.TICKET_VARIABLE_NAME in ind:
ticket = ind[form_consts.Common.TICKET_VARIABLE_NAME]
indicator.add_ticket(ticket, analyst)
if isinstance(source, list):
for s in source:
indicator.add_source(source_item=s)
elif isinstance(source, EmbeddedSource):
indicator.add_source(source_item=source)
elif isinstance(source, basestring):
s = EmbeddedSource()
s.name = source
instance = EmbeddedSource.SourceInstance()
instance.reference = reference
instance.method = method
instance.analyst = analyst
instance.date = datetime.datetime.now()
s.instances = [instance]
indicator.add_source(s)
if ec:
indicator.add_campaign(ec)
indicator.save(username=analyst)
if add_domain or add_relationship:
ind_type = indicator.ind_type
ind_value = indicator.value
if ind_type in ("URI - Domain Name", "URI - URL"):
if ind_type == "URI - URL":
domain = ind_value.split("/")[2]
elif ind_type == "URI - Domain Name":
domain = ind_value
#try:
(sdomain, fqdn) = get_domain(domain)
success = None
if add_domain:
success = upsert_domain(sdomain,
fqdn,
indicator.source,
'%s' % analyst,
None,
bucket_list=bucket_list,
cache=cache)
if not success['success']:
return {'success': False, 'message': success['message']}
if not success or not 'object' in success:
dmain = Domain.objects(domain=domain).first()
else:
dmain = success['object']
if dmain:
dmain.add_relationship(rel_item=indicator,
rel_type='Related_To',
analyst="%s" % analyst,
get_rels=False)
dmain.save(username=analyst)
indicator.save(username=analyst)
elif ind_type.startswith(
"Address - ip") or ind_type == "Address - cidr":
success = None
if add_domain:
success = ip_add_update(indicator.value,
ind_type,
source=indicator.source,
campaign=indicator.campaign,
analyst=analyst,
bucket_list=bucket_list,
ticket=ticket,
indicator_reference=reference,
cache=cache)
if not success['success']:
return {'success': False, 'message': success['message']}
if not success or not 'object' in success:
ip = IP.objects(ip=indicator.value).first()
else:
ip = success['object']
if ip:
ip.add_relationship(rel_item=indicator,
rel_type='Related_To',
analyst="%s" % analyst,
get_rels=False)
ip.save(username=analyst)
indicator.save(username=analyst)
# run indicator triage
if is_new_indicator:
indicator.reload()
run_triage(None, indicator, analyst)
return {
'success': True,
'objectid': indicator.id,
'is_new_indicator': is_new_indicator,
'object': indicator
}
def handle_indicator_insert(
ind, source, reference="", analyst="", method="", add_domain=False, add_relationship=False, cache={}
):
"""
Insert an individual indicator into the database.
NOTE: Setting add_domain to True will always create a relationship as well.
However, to create a relationship with an object that already exists before
this function was called, set add_relationship to True. This will assume
that the domain or IP object to create the relationship with already exists
and will avoid infinite mutual calls between, for example, add_update_ip
and this function. add domain/IP objects.
:param ind: Information about the indicator.
:type ind: dict
:param source: The source for this indicator.
:type source: list, str, :class:`crits.core.crits_mongoengine.EmbeddedSource`
:param reference: The reference to the data.
:type reference: str
:param analyst: The user adding this indicator.
:type analyst: str
:param method: Method of acquiring this indicator.
:type method: str
:param add_domain: If this indicator is also a top-level object, try to add
it.
:type add_domain: boolean
:param add_relationship: Attempt to add relationships if applicable.
:type add_relationship: boolean
:param cache: Cached data, typically for performance enhancements
during bulk uperations.
:type cache: dict
:returns: dict with keys:
"success" (boolean),
"message" (str) if failed,
"objectid" (str) if successful,
"is_new_indicator" (boolean) if successful.
"""
if ind["type"] not in IndicatorTypes.values():
return {"success": False, "message": "Not a valid Indicator Type: %s" % ind["type"]}
if ind["threat_type"] not in IndicatorThreatTypes.values():
return {"success": False, "message": "Not a valid Indicator Threat Type: %s" % ind["threat_type"]}
if ind["attack_type"] not in IndicatorAttackTypes.values():
return {"success": False, "message": "Not a valid Indicator Attack Type: " % ind["attack_type"]}
(ind["value"], error) = validate_indicator_value(ind["value"], ind["type"])
if error:
return {"success": False, "message": error}
is_new_indicator = False
dmain = None
ip = None
rank = {"unknown": 0, "benign": 1, "low": 2, "medium": 3, "high": 4}
if ind.get("status", None) is None or len(ind.get("status", "")) < 1:
ind["status"] = Status.NEW
indicator = Indicator.objects(
ind_type=ind["type"], lower=ind["lower"], threat_type=ind["threat_type"], attack_type=ind["attack_type"]
).first()
if not indicator:
indicator = Indicator()
indicator.ind_type = ind["type"]
indicator.threat_type = ind["threat_type"]
indicator.attack_type = ind["attack_type"]
indicator.value = ind["value"]
indicator.lower = ind["lower"]
indicator.description = ind["description"]
indicator.created = datetime.datetime.now()
indicator.confidence = EmbeddedConfidence(analyst=analyst)
indicator.impact = EmbeddedImpact(analyst=analyst)
indicator.status = ind["status"]
is_new_indicator = True
else:
if ind["status"] != Status.NEW:
indicator.status = ind["status"]
add_desc = "\nSeen on %s as: %s" % (str(datetime.datetime.now()), ind["value"])
if indicator.description is None:
indicator.description = add_desc
else:
indicator.description += add_desc
if "campaign" in ind:
if isinstance(ind["campaign"], basestring) and len(ind["campaign"]) > 0:
confidence = ind.get("campaign_confidence", "low")
ind["campaign"] = EmbeddedCampaign(
name=ind["campaign"],
confidence=confidence,
description="",
analyst=analyst,
date=datetime.datetime.now(),
)
if isinstance(ind["campaign"], EmbeddedCampaign):
indicator.add_campaign(ind["campaign"])
elif isinstance(ind["campaign"], list):
for campaign in ind["campaign"]:
if isinstance(campaign, EmbeddedCampaign):
indicator.add_campaign(campaign)
if "confidence" in ind and rank.get(ind["confidence"], 0) > rank.get(indicator.confidence.rating, 0):
indicator.confidence.rating = ind["confidence"]
indicator.confidence.analyst = analyst
if "impact" in ind and rank.get(ind["impact"], 0) > rank.get(indicator.impact.rating, 0):
indicator.impact.rating = ind["impact"]
indicator.impact.analyst = analyst
bucket_list = None
if form_consts.Common.BUCKET_LIST_VARIABLE_NAME in ind:
bucket_list = ind[form_consts.Common.BUCKET_LIST_VARIABLE_NAME]
if bucket_list:
indicator.add_bucket_list(bucket_list, analyst)
ticket = None
if form_consts.Common.TICKET_VARIABLE_NAME in ind:
ticket = ind[form_consts.Common.TICKET_VARIABLE_NAME]
if ticket:
indicator.add_ticket(ticket, analyst)
if isinstance(source, list):
for s in source:
indicator.add_source(source_item=s, method=method, reference=reference)
elif isinstance(source, EmbeddedSource):
indicator.add_source(source_item=source, method=method, reference=reference)
elif isinstance(source, basestring):
s = EmbeddedSource()
s.name = source
instance = EmbeddedSource.SourceInstance()
instance.reference = reference
instance.method = method
instance.analyst = analyst
instance.date = datetime.datetime.now()
s.instances = [instance]
indicator.add_source(s)
if add_domain or add_relationship:
ind_type = indicator.ind_type
ind_value = indicator.lower
url_contains_ip = False
if ind_type in (IndicatorTypes.DOMAIN, IndicatorTypes.URI):
if ind_type == IndicatorTypes.URI:
domain_or_ip = urlparse.urlparse(ind_value).hostname
try:
validate_ipv46_address(domain_or_ip)
url_contains_ip = True
except DjangoValidationError:
pass
else:
domain_or_ip = ind_value
if not url_contains_ip:
success = None
if add_domain:
success = upsert_domain(
domain_or_ip,
indicator.source,
username="******" % analyst,
campaign=indicator.campaign,
bucket_list=bucket_list,
cache=cache,
)
if not success["success"]:
return {"success": False, "message": success["message"]}
if not success or not "object" in success:
dmain = Domain.objects(domain=domain_or_ip).first()
else:
dmain = success["object"]
if ind_type in IPTypes.values() or url_contains_ip:
if url_contains_ip:
ind_value = domain_or_ip
try:
validate_ipv4_address(domain_or_ip)
ind_type = IndicatorTypes.IPV4_ADDRESS
except DjangoValidationError:
ind_type = IndicatorTypes.IPV6_ADDRESS
success = None
if add_domain:
success = ip_add_update(
ind_value,
ind_type,
source=indicator.source,
campaign=indicator.campaign,
analyst=analyst,
bucket_list=bucket_list,
ticket=ticket,
indicator_reference=reference,
cache=cache,
)
if not success["success"]:
return {"success": False, "message": success["message"]}
if not success or not "object" in success:
ip = IP.objects(ip=indicator.value).first()
else:
ip = success["object"]
indicator.save(username=analyst)
if dmain:
dmain.add_relationship(indicator, RelationshipTypes.RELATED_TO, analyst="%s" % analyst, get_rels=False)
dmain.save(username=analyst)
if ip:
ip.add_relationship(indicator, RelationshipTypes.RELATED_TO, analyst="%s" % analyst, get_rels=False)
ip.save(username=analyst)
# run indicator triage
if is_new_indicator:
indicator.reload()
run_triage(indicator, analyst)
return {"success": True, "objectid": str(indicator.id), "is_new_indicator": is_new_indicator, "object": indicator}
def handle_indicator_insert(ind, source, reference='', analyst='', method='',
add_domain=False, add_relationship=False, cache={}):
"""
Insert an individual indicator into the database.
NOTE: Setting add_domain to True will always create a relationship as well.
However, to create a relationship with an object that already exists before
this function was called, set add_relationship to True. This will assume
that the domain or IP object to create the relationship with already exists
and will avoid infinite mutual calls between, for example, add_update_ip
and this function. add domain/IP objects.
:param ind: Information about the indicator.
:type ind: dict
:param source: The source for this indicator.
:type source: list, str, :class:`crits.core.crits_mongoengine.EmbeddedSource`
:param reference: The reference to the data.
:type reference: str
:param analyst: The user adding this indicator.
:type analyst: str
:param method: Method of acquiring this indicator.
:type method: str
:param add_domain: If this indicator is also a top-level object, try to add
it.
:type add_domain: boolean
:param add_relationship: Attempt to add relationships if applicable.
:type add_relationship: boolean
:param cache: Cached data, typically for performance enhancements
during bulk uperations.
:type cache: dict
:returns: dict with keys:
"success" (boolean),
"message" str) if failed,
"objectid" (str) if successful,
"is_new_indicator" (boolean) if successful.
"""
if ind['type'] == "URI - URL" and "://" not in ind['value'].split('.')[0]:
return {"success": False, "message": "URI - URL must contain protocol prefix (e.g. http://, https://, ftp://) "}
is_new_indicator = False
dmain = None
ip = None
rank = {
'unknown': 0,
'benign': 1,
'low': 2,
'medium': 3,
'high': 4,
}
indicator = Indicator.objects(ind_type=ind['type'],
value=ind['value']).first()
if not indicator:
indicator = Indicator()
indicator.ind_type = ind['type']
indicator.value = ind['value']
indicator.created = datetime.datetime.now()
indicator.confidence = EmbeddedConfidence(analyst=analyst)
indicator.impact = EmbeddedImpact(analyst=analyst)
is_new_indicator = True
if 'campaign' in ind:
if isinstance(ind['campaign'], basestring) and len(ind['campaign']) > 0:
confidence = ind.get('campaign_confidence', 'low')
ind['campaign'] = EmbeddedCampaign(name=ind['campaign'],
confidence=confidence,
description="",
analyst=analyst,
date=datetime.datetime.now())
if isinstance(ind['campaign'], EmbeddedCampaign):
indicator.add_campaign(ind['campaign'])
elif isinstance(ind['campaign'], list):
for campaign in ind['campaign']:
if isinstance(campaign, EmbeddedCampaign):
indicator.add_campaign(campaign)
if 'confidence' in ind and rank.get(ind['confidence'], 0) > rank.get(indicator.confidence.rating, 0):
indicator.confidence.rating = ind['confidence']
indicator.confidence.analyst = analyst
if 'impact' in ind and rank.get(ind['impact'], 0) > rank.get(indicator.impact.rating, 0):
indicator.impact.rating = ind['impact']
indicator.impact.analyst = analyst
bucket_list = None
if form_consts.Common.BUCKET_LIST_VARIABLE_NAME in ind:
bucket_list = ind[form_consts.Common.BUCKET_LIST_VARIABLE_NAME]
if bucket_list:
indicator.add_bucket_list(bucket_list, analyst)
ticket = None
if form_consts.Common.TICKET_VARIABLE_NAME in ind:
ticket = ind[form_consts.Common.TICKET_VARIABLE_NAME]
if ticket:
indicator.add_ticket(ticket, analyst)
if isinstance(source, list):
for s in source:
indicator.add_source(source_item=s, method=method, reference=reference)
elif isinstance(source, EmbeddedSource):
indicator.add_source(source_item=source, method=method, reference=reference)
elif isinstance(source, basestring):
s = EmbeddedSource()
s.name = source
instance = EmbeddedSource.SourceInstance()
instance.reference = reference
instance.method = method
instance.analyst = analyst
instance.date = datetime.datetime.now()
s.instances = [instance]
indicator.add_source(s)
if add_domain or add_relationship:
ind_type = indicator.ind_type
ind_value = indicator.value
url_contains_ip = False
if ind_type in ("URI - Domain Name", "URI - URL"):
if ind_type == "URI - URL":
domain_or_ip = urlparse.urlparse(ind_value).hostname
elif ind_type == "URI - Domain Name":
domain_or_ip = ind_value
(sdomain, fqdn) = get_domain(domain_or_ip)
if sdomain == "no_tld_found_error" and ind_type == "URI - URL":
try:
validate_ipv46_address(domain_or_ip)
url_contains_ip = True
except DjangoValidationError:
pass
if not url_contains_ip:
success = None
if add_domain:
success = upsert_domain(sdomain, fqdn, indicator.source,
'%s' % analyst, None,
bucket_list=bucket_list, cache=cache)
if not success['success']:
return {'success': False, 'message': success['message']}
if not success or not 'object' in success:
dmain = Domain.objects(domain=domain_or_ip).first()
else:
dmain = success['object']
if ind_type.startswith("Address - ip") or ind_type == "Address - cidr" or url_contains_ip:
if url_contains_ip:
ind_value = domain_or_ip
try:
validate_ipv4_address(domain_or_ip)
ind_type = 'Address - ipv4-addr'
except DjangoValidationError:
ind_type = 'Address - ipv6-addr'
success = None
if add_domain:
success = ip_add_update(ind_value,
ind_type,
source=indicator.source,
campaign=indicator.campaign,
analyst=analyst,
bucket_list=bucket_list,
ticket=ticket,
indicator_reference=reference,
cache=cache)
if not success['success']:
return {'success': False, 'message': success['message']}
if not success or not 'object' in success:
ip = IP.objects(ip=indicator.value).first()
else:
ip = success['object']
indicator.save(username=analyst)
if dmain:
dmain.add_relationship(rel_item=indicator,
rel_type='Related_To',
analyst="%s" % analyst,
get_rels=False)
dmain.save(username=analyst)
if ip:
ip.add_relationship(rel_item=indicator,
rel_type='Related_To',
analyst="%s" % analyst,
get_rels=False)
ip.save(username=analyst)
indicator.save(username=analyst)
# run indicator triage
if is_new_indicator:
indicator.reload()
run_triage(indicator, analyst)
return {'success': True, 'objectid': str(indicator.id),
'is_new_indicator': is_new_indicator, 'object': indicator}
def get_domain_details(domain, analyst):
"""
Generate the data to render the Domain details template.
:param domain: The name of the Domain to get details for.
:type domain: str
:param analyst: The user requesting this information.
:type analyst: str
:returns: template (str), arguments (dict)
"""
template = None
allowed_sources = user_sources(analyst)
dmain = Domain.objects(domain=domain,
source__name__in=allowed_sources).first()
if not dmain:
error = ("Either no data exists for this domain"
" or you do not have permission to view it.")
template = "error.html"
args = {'error': error}
return template, args
dmain.sanitize_sources(username="******" % analyst, sources=allowed_sources)
# remove pending notifications for user
remove_user_from_notification("%s" % analyst, dmain.id, 'Domain')
# subscription
subscription = {
'type': 'Domain',
'id': dmain.id,
'subscribed': is_user_subscribed("%s" % analyst, 'Domain', dmain.id),
}
#objects
objects = dmain.sort_objects()
#relationships
relationships = dmain.sort_relationships("%s" % analyst, meta=True)
# relationship
relationship = {'type': 'Domain', 'value': dmain.id}
#comments
comments = {'comments': dmain.get_comments(), 'url_key': dmain.domain}
#screenshots
screenshots = dmain.get_screenshots(analyst)
# favorites
favorite = is_user_favorite("%s" % analyst, 'Domain', dmain.id)
# services
service_list = get_supported_services('Domain')
# analysis results
service_results = dmain.get_analysis_results()
args = {
'objects': objects,
'relationships': relationships,
'comments': comments,
'favorite': favorite,
'relationship': relationship,
'subscription': subscription,
'screenshots': screenshots,
'domain': dmain,
'service_list': service_list,
'service_results': service_results
}
return template, args
def upsert_domain(sdomain,
domain,
source,
username=None,
campaign=None,
confidence=None,
bucket_list=None,
ticket=None,
cache={}):
"""
Add or update a domain/FQDN. Campaign is assumed to be a list of campaign
dictionary objects.
:param sdomain: Response from parsing the domain for a root domain. Will
either be an error message or the root domain itself.
:type sdomain: str
:param domain: The domain to add/update.
:type domain: str
:param source: The name of the source.
:type source: str
:param username: The user adding/updating the domain.
:type username: str
:param campaign: The campaign to attribute to this domain.
:type campaign: list, str
:param confidence: Confidence for the campaign attribution.
:type confidence: str
:param bucket_list: List of buckets to add to this domain.
:type bucket_list: list, str
:param ticket: The ticket for this domain.
:type ticket: str
:param cache: Cached data, typically for performance enhancements
during bulk uperations.
:type cache: dict
:returns: dict with keys:
"success" (boolean),
"object" the domain that was added,
"is_domain_new" (boolean)
"""
if sdomain == "no_tld_found_error": #oops...
return {'success': False, 'message': "Invalid domain: %s " % sdomain}
is_fqdn_domain_new = False
is_root_domain_new = False
if not campaign:
campaign = []
# assume it's a list, but check if it's a string
elif isinstance(campaign, basestring):
c = EmbeddedCampaign(name=campaign,
confidence=confidence,
analyst=username)
campaign = [c]
# assume it's a list, but check if it's a string
if isinstance(source, basestring):
s = EmbeddedSource()
s.name = source
instance = EmbeddedSource.SourceInstance()
instance.reference = ''
instance.method = ''
instance.analyst = username
instance.date = datetime.datetime.now()
s.instances = [instance]
source = [s]
fqdn_domain = None
root_domain = None
cached_results = cache.get(form_consts.Domain.CACHED_RESULTS)
if cached_results != None:
if domain != sdomain:
fqdn_domain = cached_results.get(domain)
root_domain = cached_results.get(sdomain)
else:
root_domain = cached_results.get(sdomain)
else:
#first find the domain(s) if it/they already exist
root_domain = Domain.objects(domain=sdomain).first()
if domain != sdomain:
fqdn_domain = Domain.objects(domain=domain).first()
#if they don't exist, create them
if not root_domain:
root_domain = Domain()
root_domain.domain = sdomain.strip()
root_domain.source = []
root_domain.record_type = 'A'
is_root_domain_new = True
if cached_results != None:
cached_results[sdomain] = root_domain
if domain != sdomain and not fqdn_domain:
fqdn_domain = Domain()
fqdn_domain.domain = domain.strip()
fqdn_domain.source = []
fqdn_domain.record_type = 'A'
is_fqdn_domain_new = True
if cached_results != None:
cached_results[domain] = fqdn_domain
# if new or found, append the new source(s)
for s in source:
if root_domain:
root_domain.add_source(s)
if fqdn_domain:
fqdn_domain.add_source(s)
#campaigns
#both root and fqdn get campaigns updated
for c in campaign:
if root_domain:
root_domain.add_campaign(c)
if fqdn_domain:
fqdn_domain.add_campaign(c)
if username:
if root_domain:
root_domain.analyst = username
if fqdn_domain:
fqdn_domain.analyst = username
if bucket_list:
if root_domain:
root_domain.add_bucket_list(bucket_list, username)
if fqdn_domain:
fqdn_domain.add_bucket_list(bucket_list, username)
if ticket:
if root_domain:
root_domain.add_ticket(ticket, username)
if fqdn_domain:
fqdn_domain.add_ticket(ticket, username)
# save
try:
if root_domain:
root_domain.save(username=username)
if fqdn_domain:
fqdn_domain.save(username=username)
except Exception, e:
return {'success': False, 'message': e}
def run(self, argv):
parser = OptionParser()
parser.add_option("-d", "--domain", action="store", dest="domain",
type="string",
help="Domain to use (if not provided, do all)")
parser.add_option("-c", "--config", dest="config", default={},
help="Service configuration")
parser.add_option("-v", "--verbose", action="store_true",
dest="verbose", default=False, help="Be verbose")
parser.add_option("-n", "--dry_run", action="store_true",
dest="dry_run", default=False, help="Dry run, just show what would happen.")
(opts, args) = parser.parse_args(argv)
if opts.domain:
if opts.verbose:
print "[+] Using domain: %s" % opts.domain
domain = opts.domain
else:
if opts.verbose:
print "[+] Using ALL domains"
domain = None
config = {}
if opts.config:
config = ast.literal_eval(opts.config)
if not config:
print "No config provided, defaulting to live only."
config['live_query'] = True
else:
print "Using config: %s" % config
query = {
'$or': [
{
'whois':
{
'$exists': True,
'$not': {'$size': 0}
}
},
{
'unsupported_attrs.whois':
{
'$exists': True,
'$not': {'$size': 0}
}
}
]
}
if domain:
query['domain'] = domain
doms = Domain.objects(__raw__=query)
for dom in doms:
print "Executing whois for %s" % dom.domain
if opts.dry_run:
continue
try:
result = run_service('whois',
'Domain',
dom.id,
self.username,
obj=dom,
custom_config=config)
dom.save()
except ServiceAnalysisError as e:
print "Service error: %s" % e
except ValidationError as e:
print "Validation error: %s" % e
def upsert_domain(domain, source, username=None, campaign=None,
confidence=None, bucket_list=None, ticket=None, cache={}, related_id=None, related_type=None, relationship_type=None):
"""
Add or update a domain/FQDN. Campaign is assumed to be a list of campaign
dictionary objects.
:param domain: The domain to add/update.
:type domain: str
:param source: The name of the source.
:type source: str
:param username: The user adding/updating the domain.
:type username: str
:param campaign: The campaign to attribute to this domain.
:type campaign: list, str
:param confidence: Confidence for the campaign attribution.
:type confidence: str
:param bucket_list: List of buckets to add to this domain.
:type bucket_list: list, str
:param ticket: The ticket for this domain.
:type ticket: str
:param cache: Cached data, typically for performance enhancements
during bulk uperations.
:type cache: dict
:param related_id: ID of object to create relationship with
:type related_id: str
:param related_type: Type of object to create relationship with
:type related_id: str
:param relationship_type: Type of relationship to create.
:type relationship_type: str
:returns: dict with keys:
"success" (boolean),
"object" the domain that was added,
"is_domain_new" (boolean)
"""
# validate domain and grab root domain
(root, domain, error) = get_valid_root_domain(domain)
if error:
return {'success': False, 'message': error}
is_fqdn_domain_new = False
is_root_domain_new = False
if not campaign:
campaign = []
# assume it's a list, but check if it's a string
elif isinstance(campaign, basestring):
c = EmbeddedCampaign(name=campaign, confidence=confidence, analyst=username)
campaign = [c]
# assume it's a list, but check if it's a string
if isinstance(source, basestring):
s = EmbeddedSource()
s.name = source
instance = EmbeddedSource.SourceInstance()
instance.reference = ''
instance.method = ''
instance.analyst = username
instance.date = datetime.datetime.now()
s.instances = [instance]
source = [s]
fqdn_domain = None
root_domain = None
cached_results = cache.get(form_consts.Domain.CACHED_RESULTS)
if cached_results != None:
if domain != root:
fqdn_domain = cached_results.get(domain)
root_domain = cached_results.get(root)
else:
root_domain = cached_results.get(root)
else:
#first find the domain(s) if it/they already exist
root_domain = Domain.objects(domain=root).first()
if domain != root:
fqdn_domain = Domain.objects(domain=domain).first()
#if they don't exist, create them
if not root_domain:
root_domain = Domain()
root_domain.domain = root
root_domain.source = []
root_domain.record_type = 'A'
is_root_domain_new = True
if cached_results != None:
cached_results[root] = root_domain
if domain != root and not fqdn_domain:
fqdn_domain = Domain()
fqdn_domain.domain = domain
fqdn_domain.source = []
fqdn_domain.record_type = 'A'
is_fqdn_domain_new = True
if cached_results != None:
cached_results[domain] = fqdn_domain
# if new or found, append the new source(s)
for s in source:
if root_domain:
root_domain.add_source(s)
if fqdn_domain:
fqdn_domain.add_source(s)
#campaigns
#both root and fqdn get campaigns updated
for c in campaign:
if root_domain:
root_domain.add_campaign(c)
if fqdn_domain:
fqdn_domain.add_campaign(c)
if username:
if root_domain:
root_domain.analyst = username
if fqdn_domain:
fqdn_domain.analyst = username
if bucket_list:
if root_domain:
root_domain.add_bucket_list(bucket_list, username)
if fqdn_domain:
fqdn_domain.add_bucket_list(bucket_list, username)
if ticket:
if root_domain:
root_domain.add_ticket(ticket, username)
if fqdn_domain:
fqdn_domain.add_ticket(ticket, username)
related_obj = None
if related_id:
related_obj = class_from_id(related_type, related_id)
if not related_obj:
retVal['success'] = False
retVal['message'] = 'Related Object not found.'
return retVal
# save
try:
if root_domain:
root_domain.save(username=username)
if fqdn_domain:
fqdn_domain.save(username=username)
except Exception, e:
return {'success': False, 'message': e}