def historical_whois(self, indicator):
record_type = 'WR'
time_frame = datetime.datetime.utcnow() + datetime.timedelta(hours=-24)
if check_domain_valid(indicator):
indicator = get_base_domain(indicator)
raw_records = self.get_queryset().filter(Q(record_type=record_type),
Q(info_date__lt=time_frame),
Q(info__at_query__endswith=indicator) |
Q(info__at_domain_name__endswith=indicator)).values('info_hash',
'info_date')
tracking = []
unique_records = []
annotated_records = raw_records.annotate(latest=Max('info_date')).annotate(earliest=Min('info_date'))
for record in annotated_records:
hash_value = record['info_hash']
if hash_value not in tracking:
record_info = self.get_queryset().filter(info_hash=hash_value).values('info')[0]['info']
new_record = {'latest': record['latest'], 'earliest': record['earliest'], 'info': record_info}
unique_records.append(new_record)
tracking.append(hash_value)
return unique_records
def get_search_records(self, indicator):
"""
Retrieve any search records from within the last 24 hours for an indicator from the database.
:param indicator: The indicator value
:return: The search records for the indicator
"""
record_type = RecordType.SR
time_frame = datetime.datetime.utcnow() + datetime.timedelta(hours=-24)
value = indicator
if check_domain_valid(indicator):
value = get_base_domain(indicator)
LOGGER.debug("Using search value: %s", value)
records = self.get_queryset().filter(Q(record_type=record_type.name),
Q(info_date__gte=time_frame),
Q(info__at_indicator__exact=value)).values('info', 'info_date')
if LOGGER.isEnabledFor(logging.INFO):
rank = 0
msg = "Found %d search record(s):" % len(records)
for record in records:
info = record['info']
results = info['results']
for result in results:
rank += 1
url = result['url']
msg += "\n\t%d - %s" % (rank, url)
LOGGER.info(msg)
return records
def get_search_records(self, indicator):
"""
Retrieve any search records from within the last 24 hours for an indicator from the database.
:param indicator: The indicator value
:return: The search records for the indicator
"""
record_type = RecordType.SR
time_frame = datetime.datetime.utcnow() + datetime.timedelta(hours=-24)
value = indicator
if check_domain_valid(indicator):
value = get_base_domain(indicator)
LOGGER.debug("Using search value: %s", value)
records = self.get_queryset().filter(
Q(record_type=record_type.name), Q(info_date__gte=time_frame),
Q(info__at_indicator__exact=value)).values('info', 'info_date')
if LOGGER.isEnabledFor(logging.INFO):
rank = 0
msg = "Found %d search record(s):" % len(records)
for record in records:
info = record['info']
results = info['results']
for result in results:
rank += 1
url = result['url']
msg += "\n\t%d - %s" % (rank, url)
LOGGER.info(msg)
return records
def historical_whois(self, indicator):
record_type = RecordType.WR
time_frame = datetime.datetime.utcnow() + datetime.timedelta(hours=-24)
if check_domain_valid(indicator):
indicator = get_base_domain(indicator)
raw_records = self.get_queryset().filter(
Q(record_type=record_type.name), Q(info_date__lt=time_frame),
Q(info__at_query__endswith=indicator)
| Q(info__at_domain_name__endswith=indicator)).values(
'info_hash', 'info_date')
tracking = []
unique_records = []
annotated_records = raw_records.annotate(
latest=Max('info_date')).annotate(earliest=Min('info_date'))
for record in annotated_records:
hash_value = record['info_hash']
if hash_value not in tracking:
record_info = self.get_queryset().filter(
info_hash=hash_value).values('info')[0]['info']
span = str(record['earliest']) + " / " + str(record['latest'])
new_record = {
'latest': record['latest'],
'earliest': record['earliest'],
'info_date': span,
'info': record_info
}
unique_records.append(new_record)
tracking.append(hash_value)
return unique_records
def whois_records(self, indicator):
record_type = 'WR'
if check_domain_valid(indicator):
indicator = get_base_domain(indicator)
records = self.get_queryset().filter(Q(record_type=record_type),
Q(info__at_query__endswith=indicator) |
Q(info__at_domain_name__endswith=indicator)).values('info', 'info_date')
return records
def whois_records(self, indicator):
record_type = RecordType.WR
if check_domain_valid(indicator):
indicator = get_base_domain(indicator)
records = self.get_queryset().filter(
Q(record_type=record_type.name),
Q(info__at_query__endswith=indicator)
| Q(info__at_domain_name__endswith=indicator)).values(
'info', 'info_date')
return records
def verify_type(value, validator):
if validator == "ip":
return check_ip_valid(value)
elif validator == "domain":
return check_domain_valid(value)
elif validator == "email":
return check_email_valid(value)
else:
return False
def clean_domains(self):
submission = self.cleaned_data.get('domains')
domain_list = re.split(r'[,;|\n\r ]+', submission)
validated_submissions = []
for domain in domain_list:
domain = domain.rstrip().lower()
if check_domain_valid(domain):
validated_submissions.append(domain)
return validated_submissions
def recent_whois(self, indicator):
record_type = 'WR'
time_frame = datetime.datetime.utcnow() + datetime.timedelta(hours=-24)
if check_domain_valid(indicator):
indicator = get_base_domain(indicator)
record = self.get_queryset().filter(Q(record_type=record_type),
Q(info_date__gte=time_frame),
Q(info__at_query__endswith=indicator) |
Q(info__at_domain_name__endswith=indicator)).values('info', 'info_date')
if record:
return record.latest('info_date')
return record
def recent_whois(self, indicator):
record_type = RecordType.WR
time_frame = datetime.datetime.utcnow() + datetime.timedelta(hours=-24)
if check_domain_valid(indicator):
indicator = get_base_domain(indicator)
record = self.get_queryset().filter(
Q(record_type=record_type.name), Q(info_date__gte=time_frame),
Q(info__at_query__endswith=indicator)
| Q(info__at_domain_name__endswith=indicator)).values(
'info', 'info_date')
if record:
return record.latest('info_date')
return record
