def save_submission(self, request):
current_user = User.objects.get(email__exact=request.user)
lookup_time = time_jump(minutes=2)
set_interval = 24
for domain in self.valid_domains:
try:
new_monitor = DomainMonitor(owner=current_user,
domain_name=domain,
lookup_interval=set_interval,
next_lookup=lookup_time)
new_monitor = self.update_monitor(new_monitor)
new_monitor.save()
except:
LOGGER.exception("Error saving domain monitor from %s for %s", current_user, domain)
for ip_address in self.valid_ips:
try:
new_monitor = IpMonitor(owner=current_user,
ip_address=ip_address,
lookup_interval=set_interval,
next_lookup=lookup_time)
new_monitor = self.update_monitor(new_monitor)
new_monitor.save()
except:
LOGGER.exception("Error saving IP monitor from %s for %s", current_user, ip_address)
pass
def post(self, request):
task_tracking = {}
submitted_form = SubmissionForm(request.POST)
current_time = datetime.datetime.utcnow()
desired_time = time_jump(hours=-24)
if submitted_form.is_valid():
recent_tasks = submitted_form.check_recent_tasks(desired_time)
# If a recent task exists, use that one instead
if recent_tasks:
task_tracking['id'] = recent_tasks.group_id
else:
new_task = submitted_form.create_new_task(current_time)
if new_task:
task_tracking['id'] = new_task.id
else:
task_tracking["errors"] = "Unexpected Failure"
else: # pass form errors back to user from async request
task_tracking["errors"] = submitted_form.errors
json_response = json.dumps(task_tracking)
return HttpResponse(json_response, content_type="application/json")
def post(self, request):
task_tracking = {}
submitted_form = SubmissionForm(request.POST)
current_time = datetime.datetime.utcnow()
desired_time = time_jump(hours=-24)
if submitted_form.is_valid():
recent_tasks = submitted_form.check_recent_tasks(desired_time)
# If a recent task exists, use that one instead
if recent_tasks:
task_tracking['id'] = recent_tasks.group_id
else:
new_task = submitted_form.create_new_task(current_time)
if new_task:
task_tracking['id'] = new_task.id
else:
task_tracking["errors"] = "Unexpected Failure"
else: # pass form errors back to user from async request
task_tracking["errors"] = submitted_form.errors
json_response = json.dumps(task_tracking)
return HttpResponse(json_response, content_type="application/json")
def save_submission(self, request):
current_user = User.objects.get(email__exact=request.user)
lookup_time = time_jump(minutes=2)
set_interval = 24
for domain in self.valid_domains:
try:
new_monitor = DomainMonitor(owner=current_user,
domain_name=domain,
lookup_interval=set_interval,
next_lookup=lookup_time)
new_monitor = self.update_monitor(new_monitor)
new_monitor.save()
except:
LOGGER.exception("Error saving domain monitor from %s for %s",
current_user, domain)
for ip_address in self.valid_ips:
try:
new_monitor = IpMonitor(owner=current_user,
ip_address=ip_address,
lookup_interval=set_interval,
next_lookup=lookup_time)
new_monitor = self.update_monitor(new_monitor)
new_monitor.save()
except:
LOGGER.exception("Error saving IP monitor from %s for %s",
current_user, ip_address)
def save_submission(self, request):
current_user = User.objects.get(email__exact=request.user)
lookup_time = time_jump(minutes=2)
set_interval = 24
for domain in self.valid_domains:
try:
new_monitor = DomainMonitor(owner=current_user,
domain_name=domain,
lookup_interval=set_interval,
next_lookup=lookup_time)
new_monitor.save()
except:
pass
for ip_address in self.valid_ips:
try:
new_monitor = IpMonitor(owner=current_user,
ip_address=ip_address,
lookup_interval=set_interval,
next_lookup=lookup_time)
new_monitor.save()
except:
pass
def get_context_data(self, **kwargs):
listing = User.objects.get(email__exact=self.request.user).ip_list
context = super(IpMonitorPanel, self).get_context_data(**kwargs)
context["alerts"] = IpAlert.objects.filter(alert_time__gte=time_jump(hours=-72), ip_address__in=listing)
context["form"] = IpMonitorAddition.form_class
return context
def get_context_data(self, **kwargs):
listing = User.objects.get(email__exact=self.request.user).domain_list
context = super(DomainMonitorPanel, self).get_context_data(**kwargs)
context['alerts'] = DomainAlert.objects.filter(alert_time__gte=time_jump(hours=-72),
domain_name__in=listing)
context['form'] = DomainMonitorAddition.form_class
return context
def test_check_recent_tasks_none(self):
post = {}
post['indicator'] = self.indicator
post['record_type'] = "Search"
form = SubmissionForm(post)
# Required for normalizing form data in Django.
form.is_valid()
recent_tasks = form.check_recent_tasks(time_jump(hours=-24))
self.assertIsNone(recent_tasks)
def test_check_recent_tasks_none(self):
post = {}
post['indicator'] = self.indicator
post['record_type'] = "Search"
form = SubmissionForm(post)
# Required for normalizing form data in Django.
form.is_valid()
recent_tasks = form.check_recent_tasks(time_jump(hours=-24))
self.assertIsNone(recent_tasks)
def post(self, request):
desired_time = time_jump(hours=-24)
task = request.POST['task_id']
res = GroupResult.restore(task)
if res and not res.ready():
return HttpResponse(json.dumps({"status": "loading"}), content_type="application/json")
# Task completion allows for origin information to be pulled
try:
task_origin = TaskTracker.objects.get(group_id=task)
record_type = task_origin.type
indicator = task_origin.keyword
except MultipleObjectsReturned:
task_origin = TaskTracker.objects.filter(group_id=task).latest('date')
record_type = task_origin.type
indicator = task_origin.keyword
except ObjectDoesNotExist:
record_type = None
indicator = None
# Pull data according to the record type
if record_type == "current":
# Collect whois record for current records
whois_record = WhoisRecord.objects.recent_record(indicator)
self.template_vars["whois_record"] = whois_record
# Collect host records for current records
host_record = HostRecord.objects.current_hosts(indicator, desired_time)
self.template_vars["host_record"] = host_record
self.template_name = "pivoteer/CurrentRecords.html"
elif record_type == "passive":
host_records = HostRecord.objects.passive_records(indicator, request)
self.template_vars["passive_records"] = host_records
self.template_name = "pivoteer/PassiveRecords.html"
elif record_type == "malware":
malware_records = MalwareRecord.objects.malware_records(indicator)
self.template_vars["malware_records"] = malware_records
self.template_name = "pivoteer/MalwareRecords.html"
elif record_type == "other":
google_search = SearchEngineHits.objects.recent_record(indicator)
self.template_vars["google_search"] = google_search
self.template_name = "pivoteer/OtherRecords.html"
return render(request, self.template_name, self.template_vars)
def export_current(self, indicator):
desired_time = time_jump(hours=-24)
hosts = HostRecord.objects.current_hosts(indicator, desired_time)
self.writer.writerow(['Resolution Date', 'Domain', 'IP Address', 'Source'])
for host in hosts:
record = [host.resolution_date, host.domain_name, host.ip_address, host.resolution_source]
self.writer.writerow(record)
self.line_separator()
whois = WhoisRecord.objects.recent_record(indicator)
self.writer.writerow(['Whois Record'])
self.writer.writerow([whois])
self.line_separator()
def save_submission(self, request):
"""
Save a CertificateMonitor based upon the contents of this form.
:param request: The request being processed
:return: This method returns no values
"""
indicator = self.cleaned_data.get("fragment")
if indicator is None:
LOGGER.debug("No certificate specified")
return
user = User.objects.get(email__exact=request.user)
lookup_time = time_jump(minutes=2)
interval = 24
# Build the 'resolutions' monitor member. We're not actually going to do full resolutions now (which would
# entail doing geo-location and domain lookup for each IP host). Rather, we'll just use placeholder values
# until the monitor runs for the first time (as a periodic task). This is important because it means that we
# will have a set of hosts already saved for this monitor. Therefore, when this monitor runs for the first time
# it will have a previous set of hosts to which to compare. (In other words, the first time it runs not every
# host will necessarily be new, and there may be some missing hosts on the first run.)
resolutions = dict()
for host in self.valid_hosts:
if host not in resolutions:
resolutions[host] = dict()
resolution = resolutions[host]
resolution[GEOLOCATION_KEY] = PENDING
resolution[DOMAIN_KEY] = [PENDING]
# Finally, we can construct the actual monitor object and save it to the database.
monitor = CertificateMonitor(owner=user,
certificate_value=indicator,
lookup_interval=interval,
next_lookup=lookup_time,
last_hosts=self.valid_hosts,
resolutions=resolutions)
monitor = self.update_monitor(monitor)
try:
monitor.save()
LOGGER.info("New certificate monitor from %s for '%s' (initial hosts: %s)",
user,
indicator,
self.valid_hosts)
except:
LOGGER.exception("Error saving certificate monitor")
def save_submission(self, request):
"""
Save a CertificateMonitor based upon the contents of this form.
:param request: The request being processed
:return: This method returns no values
"""
indicator = self.cleaned_data.get("fragment")
if indicator is None:
LOGGER.debug("No certificate specified")
return
user = User.objects.get(email__exact=request.user)
lookup_time = time_jump(minutes=2)
interval = 24
# Build the 'resolutions' monitor member. We're not actually going to do full resolutions now (which would
# entail doing geo-location and domain lookup for each IP host). Rather, we'll just use placeholder values
# until the monitor runs for the first time (as a periodic task). This is important because it means that we
# will have a set of hosts already saved for this monitor. Therefore, when this monitor runs for the first time
# it will have a previous set of hosts to which to compare. (In other words, the first time it runs not every
# host will necessarily be new, and there may be some missing hosts on the first run.)
resolutions = dict()
for host in self.valid_hosts:
if host not in resolutions:
resolutions[host] = dict()
resolution = resolutions[host]
resolution[GEOLOCATION_KEY] = PENDING
resolution[DOMAIN_KEY] = [PENDING]
# Finally, we can construct the actual monitor object and save it to the database.
monitor = CertificateMonitor(owner=user,
certificate_value=indicator,
lookup_interval=interval,
next_lookup=lookup_time,
last_hosts=self.valid_hosts,
resolutions=resolutions)
monitor = self.update_monitor(monitor)
try:
monitor.save()
LOGGER.info(
"New certificate monitor from %s for '%s' (initial hosts: %s)",
user, indicator, self.valid_hosts)
except:
LOGGER.exception("Error saving certificate monitor")
def test_check_recent_tasks_many(self):
post = {}
post['indicator'] = self.indicator
post['record_type'] = "Search"
form = SubmissionForm(post)
time_old = datetime.datetime.utcnow()
time_new = datetime.datetime.utcnow()
# Required for normalizing form data in Django.
form.is_valid()
# Create the task and the TaskTracker.
form.create_new_task(time_old)
form.create_new_task(time_new)
# Request tasks in the last 24 hours.
lookup = form.check_recent_tasks(time_jump(hours=-24))
self.assertIsNotNone(lookup)
self.assertEquals(lookup.date, time_new)
def test_check_recent_tasks_many(self):
post = {}
post['indicator'] = self.indicator
post['record_type'] = "Search"
form = SubmissionForm(post)
time_old = datetime.datetime.utcnow()
time_new = datetime.datetime.utcnow()
# Required for normalizing form data in Django.
form.is_valid()
# Create the task and the TaskTracker.
form.create_new_task(time_old)
form.create_new_task(time_new)
# Request tasks in the last 24 hours.
lookup = form.check_recent_tasks(time_jump(hours=-24))
self.assertIsNotNone(lookup)
self.assertEquals(lookup.date, time_new)
def save_submission(self, request):
ips = self.cleaned_data.get('ips')
current_user = User.objects.get(email__exact=request.user)
for ip in ips:
try:
IpMonitor.objects.get(ip_address__exact=ip)
except ObjectDoesNotExist:
lookup_time = time_jump(minutes=1)
new_monitor = IpMonitor(ip_address=ip,
lookup_interval='24',
next_lookup=lookup_time)
new_monitor.save()
if ip not in current_user.ip_list:
current_user.ip_list.append(ip)
current_user.save()
def save_submission(self, request):
domains = self.cleaned_data.get('domains')
current_user = User.objects.get(email__exact=request.user)
for domain in domains:
try:
DomainMonitor.objects.get(domain_name__exact=domain)
except ObjectDoesNotExist:
lookup_time = time_jump(minutes=1)
new_monitor = DomainMonitor(domain_name=domain,
lookup_interval='24',
next_lookup=lookup_time)
new_monitor.save()
if domain not in current_user.domain_list:
current_user.domain_list.append(domain)
current_user.save()
def save_submission(self, request):
"""
Save a CertificateMonitor based upon the contents of this form.
:param request: The request being processed
:return: This method returns no values
"""
request.success = True
request.msg = "Indicator(s) added for monitoring"
#indicator = self.cleaned_data.get("fragment").replace('','').replace('"','').strip()
indicator = re.sub(r'[,\';"|\n\r ]+', '',
self.cleaned_data.get("fragment")).strip()
if indicator is None:
LOGGER.debug("No certificate specified")
return
user = User.objects.get(email__exact=request.user)
lookup_time = time_jump(minutes=2)
interval = 1
current_time = datetime.datetime.utcnow()
# Build the 'resolutions' monitor member. We're not actually going to do full resolutions now (which would
# entail doing geo-location and domain lookup for each IP host). Rather, we'll just use placeholder values
# until the monitor runs for the first time (as a periodic task). This is important because it means that we
# will have a set of hosts already saved for this monitor. Therefore, when this monitor runs for the first time
# it will have a previous set of hosts to which to compare. (In other words, the first time it runs not every
# host will necessarily be new, and there may be some missing hosts on the first run.)
resolutions = dict()
for host in self.valid_hosts:
if host not in resolutions:
resolutions[host] = dict()
resolution = resolutions[host]
resolution[GEOLOCATION_KEY] = PENDING
resolution[DOMAIN_KEY] = [PENDING]
# for indicator in self.valid_certificates:
# added by LNguyen on 10jan2017 to retrieve current owner and cert values from the database
# Need to first check if there is an existing owner with the given cert in the database
# If an existing owner does not exist, then set existing owner to NONE.
# If there are multiple existing owners, then set existing owner to the 1st owner
try:
current_owner = CertificateSubscription.objects.get(
certificate=indicator).owner
except CertificateSubscription.DoesNotExist:
current_owner = None
except CertificateSubscription.MultipleObjectsReturned:
try:
current_owner = CertificateSubscription.objects.get(
certificate=indicator, owner=user).owner
except CertificateSubscription.DoesNotExist:
current_owner = CertificateSubscription.objects.filter(
certificate=indicator)[0].owner
#current_owner = CertificateSubscription.objects.filter(certificate=indicator, owner=user)[0].owner
# added by LNguyen on 10jan2017
# Need to check if the given cert exists in the database
# If the cert does not exist, then set the cert to NONE.
# If there are multiple entries for the given cert, then set the cert to the 1st entry
try:
current_cert = CertificateMonitor.objects.get(
certificate_value=indicator).certificate_value
except CertificateMonitor.DoesNotExist:
current_cert = None
except CertificateMonitor.MultipleObjectsReturned:
current_cert = CertificateMonitor.objects.filter(
certificate_value=indicator)[0].certificate_value
# Finally, we can construct the actual monitor object for the certificate info and save it to the database.
# updated by LNguyen on 29nov2016
# added created field
#print("saving Certificate Monitor...")
monitor = CertificateMonitor(owner=user,
created=current_time,
certificate_value=indicator,
lookup_interval=interval,
next_lookup=lookup_time,
last_hosts=self.valid_hosts,
resolutions=resolutions)
monitor = self.update_monitor(monitor)
# Construct the subscription to store the new certificate and owner relationship
subscription = CertificateSubscription(certificate=monitor, owner=user)
try:
# If there is no existing owner and no existing cert in the database, then perform an initial save for the new values in
# the certificate monitor and certificate subscription tables
if not current_owner and not current_cert:
monitor.save()
subscription.save()
# If the cert exists but there is no existing owner, then save the new owner info in the certificate subscription table and update the submission for new hosts info
elif not current_owner and current_cert == indicator:
subscription.save()
self.update_submission(request)
# If the cert exists for a different owner, then save the new owner info in the certificate subscription table and update the submission for new hosts info
elif current_owner != user and current_cert == indicator:
subscription.save()
self.update_submission(request)
# Else if no condition is satisfied, then set success flag to False and do nothing
else:
request.success = False
request.msg = "No indicator added for monitoring because duplicate certificate exists"
# Updates the last_hosts and resoultions fields for the Certificate Monitor record
# CertificateMonitor.objects.filter(certificate_value=indicator, owner=user).update(last_hosts=self.valid_hosts,resolutions=resolutions)
# Enable the following lines to test save for CertificateMontior and CertificateSubscription tables
# monitor.save()
# subscription.save()
LOGGER.info(
"New certificate monitor from %s for '%s' (initial hosts: %s)",
user, indicator, self.valid_hosts)
except Exception as err:
LOGGER.exception("Error saving certificate monitor: ", str(err))
request.success = False
request.msg = "Error saving certificate monitor submission: " + str(
err)
def update_submission(self, request):
"""
Created by: LNguyen
Date: 1/7/2017
Update a CertificateMonitor based upon the contents of this form.
:param request: The request being processed
:return: This method returns no values
"""
request.success = True
request.msg = "Indicator has been updated"
#indicator = self.cleaned_data.get("fragment").strip()
indicator = re.sub(r'[,\';"|\n\r ]+', '',
self.cleaned_data.get("fragment")).strip()
if indicator is None:
LOGGER.debug("No certificate specified")
return
user = User.objects.get(email__exact=request.user)
lookup_time = time_jump(minutes=2)
interval = 1
current_time = datetime.datetime.utcnow()
# Build the 'resolutions' monitor member. We're not actually going to do full resolutions now (which would
# entail doing geo-location and domain lookup for each IP host). Rather, we'll just use placeholder values
# until the monitor runs for the first time (as a periodic task). This is important because it means that we
# will have a set of hosts already fsaved for this monitor. Therefore, when this monitor runs for the first time
# it will have a previous set of hosts to which to compare. (In other words, the first time it runs not every
# host will necessarily be new, and there may be some missing hosts on the first run.)
resolutions = dict()
for host in self.valid_hosts:
if host not in resolutions:
resolutions[host] = dict()
resolution = resolutions[host]
resolution[DOMAIN_KEY] = [PENDING]
resolution[GEOLOCATION_KEY] = PENDING
# Retrieve current certificate hosts info from the database and compare it to new hosts data.
# If the new hosts data does not exist in the database, then append the data to the list of current hosts data and save the updates in the database
# If the new hosts data exists in the database, then set the success flag to False and do not save the updates.
try:
# Retrieve the current certificate hosts from the database for the given cert
# lasthosts = CertificateMonitor.objects.get(
# certificatesubscription=CertificateSubscription.objects.get(certificate=indicator,
# owner=user)).last_hosts
lasthosts = CertificateMonitor.objects.get(
certificate_value=indicator).last_hosts
# For every item in the list of new hosts, check to see if it exists in the list of current hosts
for host in self.valid_hosts:
# If there is no match, then perform the updates
if host not in lasthosts:
# dbresolutions = CertificateMonitor.objects.get(
# certificatesubscription=CertificateSubscription.objects.get(certificate=indicator,
# owner=user)).resolutions
dbresolutions = CertificateMonitor.objects.get(
certificate_value=indicator).resolutions
# Combine the current and new hosts to the current list
lasthosts.extend(list(self.valid_hosts))
# Combine the current and new resolutions and store to temp variable - dbtmp
dbtmp = {**resolutions, **dbresolutions}
# Updates the last_hosts and resolutions fields for the Certificate Monitor record
CertificateMonitor.objects.filter(
certificate_value=indicator).update(
last_hosts=lasthosts, resolutions=dbtmp)
# CertificateMonitor.objects.filter(certificatesubscription=CertificateSubscription.objects.get(certificate=indicator,
# owner=user)).update(last_hosts=lasthosts, resolutions=dbtmp)
# Else if there is no match, then set the success Flag to False and do nothing
else:
request.success = False
request.msg = "Indicator has not been updated because duplicate value exists"
LOGGER.info(
"New certificate monitor from %s for '%s' (initial hosts: %s)",
user, indicator, self.valid_hosts)
except Exception as err:
LOGGER.exception("Error updating certificate monitor: ", str(err))
request.success = False
request.msg = "Error updating certificate monitor : " + str(err)
def save_submission(self, request):
request.success = True
request.msg = "Indicator has been added for submission"
current_user = User.objects.get(email__exact=request.user)
lookup_time = time_jump(minutes=2)
#set_interval = 24
set_interval = 1
for domain in self.valid_domains:
try:
current_owner = DomainSubscription.objects.get(
domain_name=domain).owner
except DomainSubscription.DoesNotExist:
current_owner = None
except DomainSubscription.MultipleObjectsReturned:
current_owner = DomainSubscription.objects.filter(
domain_name=domain, owner=current_user)[0].owner
try:
current_domain = DomainMonitor.objects.get(
domain_name=domain).domain_name
except DomainMonitor.DoesNotExist:
current_domain = None
except DomainMonitor.MultipleObjectsReturned:
current_domain = DomainMonitor.objects.filter(
domain_name=domain)[0].domain_name
try:
new_monitor = DomainMonitor(owner=current_user,
domain_name=domain,
lookup_interval=set_interval,
next_lookup=lookup_time)
new_monitor = self.update_monitor(new_monitor)
# new_monitor.save()
new_subscription = DomainSubscription(domain_name=new_monitor,
owner=current_user)
# IF there is no existing owner and no existing domain in the database, then perform an initial save for the new values
if not current_owner and not current_domain:
new_monitor.save()
new_subscription.save()
# IF the domain exists but there is no existing user, then save the new owner info in the domain subscription table
elif not current_owner and current_domain == domain:
new_subscription.save()
# IF the domain exists for a different user, then save the owner info in the domain subscription table
elif current_owner != current_user and current_domain == domain:
new_subscription.save()
# IF no condition is satisified, then set success flag to False and do nothing
else:
request.success = False
request.msg = "No indicator added for monitoring because duplicate domain exists"
except:
LOGGER.exception("Error saving domain monitor from %s for %s",
current_user, domain)
request.success = False
request.msg = "Error saving domain monitor for " + domain
for ip_address in self.valid_ips:
try:
current_owner = IpSubscription.objects.get(
ip_address=ip_address).owner
except IpSubscription.DoesNotExist:
current_owner = None
except IpSubscription.MultipleObjectsReturned:
current_owner = IpSubscription.objects.filter(
ip_address=ip_address, owner=current_user)[0].owner
try:
current_IP = IpMonitor.objects.get(
ip_address=ip_address).ip_address
except IpMonitor.DoesNotExist:
current_IP = None
except IpMonitor.MultipleObjectsReturned:
current_IP = IpMonitor.objects.filter(
ip_address=ip_address)[0].ip_address
try:
# current_owner = IpSubscription.objects.get(domain_name=domain).owner
# current_IP = IpSubscription.objects.get(domain_name=domain).domain_name
new_monitor = IpMonitor(owner=current_user,
ip_address=ip_address,
lookup_interval=set_interval,
next_lookup=lookup_time)
new_monitor = self.update_monitor(new_monitor)
# new_monitor.save()
new_subscription = IpSubscription(ip_address=new_monitor,
owner=current_user)
#IF there is no existing owner and no existing IP in the database, then perform an initial save for the new values
if not current_owner and not current_IP:
new_monitor.save()
new_subscription.save()
# IF the IP exists but there is no existing user, then save the new owner info in the IP subscription table
elif not current_owner and current_IP == ip_address:
new_subscription.save()
# IF the IP exists for a different user, then save the owner info in the IP subscription table
elif current_owner != current_user and current_IP == ip_address:
new_subscription.save()
# IF no condition is satisified, then set success flag to False and do nothing
else:
request.success = False
request.msg = "No indicator added for monitoring because duplicate IP address exists"
except:
LOGGER.exception("Error saving IP monitor from %s for %s",
current_user, ip_address)
request.success = False
request.msg = "Error saving IP monitor for " + ip_address
