def process_domain(registered_domain, updated_domains, now=None): """Update the statistics for all fuzz results for this domain.""" if now is None: now = datetime.datetime.now() updated_domains = set(updated_domains) delta_report = repository.get_delta_report(registered_domain) if delta_report is None: return updated_domains for domain in delta_reports.extract_domains(delta_report): if domain in updated_domains: continue updated = statistics_repository.noise_stat_last_updated(domain) if updated is not None and (now - updated) < FREQUENCY: continue stat = statistics_repository.get_noise_stat(domain) if stat is None: stat = NoiseStatistic(domain, deltas=1) else: stat.increment() stat.update_window() statistics_repository.set_noise_stat(stat) statistics_repository.mark_noise_stat_as_updated(domain) updated_domains.add(domain) return updated_domains
def process_domain(registered_domain, now=None): """Update the statistics for all fuzz results for this domain.""" if now is None: now = datetime.datetime.now() delta_report = repository.get_delta_report(registered_domain) if delta_report is None: return for domain in delta_reports.extract_domains(delta_report): updated = statistics_repository.noise_stat_last_updated(domain) if updated is not None and (now - updated) < FREQUENCY: continue stat = statistics_repository.get_noise_stat(domain) if stat is None: stat = NoiseStatistic(domain, deltas=1) else: stat.increment() stat.update_window() statistics_repository.set_noise_stat(stat) statistics_repository.mark_noise_stat_as_updated(domain)
def process_sub(sub_id, detail): """Process a subscription.""" domain = detail['domain'] email_address = detail['email_address'] # Ensure the domain is registered for reporting, register if not. repository.register_domain(domain) # Mark delta report as "read" so it's not unsubscribed. repository.mark_delta_report_as_read(domain) # Don't send more than once every 24 hours last_sent = repository.email_last_send_for_sub(sub_id) if last_sent is not None: age_last_sent = datetime.datetime.now() - last_sent if age_last_sent < datetime.timedelta(seconds=PERIOD): print 'Skipping {} + {}, < 24h hours'.format( email_address, domain ) return # Grab the delta delta = repository.get_delta_report(domain) if delta is None: print 'Skipping {} + {}, no delta report yet'.format( email_address, domain ) return # Grab the delta report update time. delta_updated = repository.delta_report_updated(domain) # If the delta report was updated > 23 hours ago, we're too close to the # next delta report. This means we should hold off so we don't send the # same delta report twice. if delta_updated is not None: age_delta_updated = datetime.datetime.now() - delta_updated if age_delta_updated > datetime.timedelta(hours=23): print 'Skipping {} + {}, delta > 23h hours old'.format( email_address, domain ) return # Don't email if no changes new = delta['new'] if len(delta['new']) > 0 else None updated = delta['updated'] if len(delta['updated']) > 0 else None deleted = delta['deleted'] if len(delta['deleted']) > 0 else None if new is updated is deleted is None: print 'Skipping {} + {}, no changes'.format( email_address, domain ) return # Add analysis links if new is not None: new = [(dom, ip, ANALYSIS_ROOT.format(binascii.hexlify(dom))) for (dom, ip) in new] if updated is not None: updated = [(dom, old_ip, new_ip, ANALYSIS_ROOT.format(binascii.hexlify(dom))) for (dom, old_ip, new_ip) in updated] # Get noisy domains delta_domains = delta_reports.extract_domains(delta) noisy_domains = get_noisy_domains(delta_domains) # Email body = email_tools.render_email( 'report.html', domain=domain, new=new, updated=updated, deleted=deleted, noisy=noisy_domains, unsubscribe_link='https://dnstwister.report/email/unsubscribe/{}'.format(sub_id) ) # Mark as emailed to ensure we don't flood if there's an error after the # actual email has been sent. repository.update_last_email_sub_sent_date(sub_id) emailer.send( email_address, 'dnstwister report for {}'.format(domain), body ) print 'Emailed delta for {} to {}'.format(domain, email_address)
def process_sub(sub_id, detail): """Process a subscription.""" domain = detail['domain'] email_address = detail['email_address'] # Ensure the domain is registered for reporting, register if not. repository.register_domain(domain) # Mark delta report as "read" so it's not unsubscribed. repository.mark_delta_report_as_read(domain) # Don't send more than once every 24 hours last_sent = repository.email_last_send_for_sub(sub_id) if last_sent is not None: age_last_sent = datetime.datetime.now() - last_sent if age_last_sent < datetime.timedelta(seconds=PERIOD): print 'Skipping {} + {}, < 24h hours'.format(email_address, domain) return delta_report = repository.get_delta_report(domain) if delta_report is None: print 'Skipping {} + {}, no delta report yet'.format( email_address, domain) return delta_updated = repository.delta_report_updated(domain) # If the delta report was updated > 23 hours ago, we're too close to the # next delta report. This means we should hold off so we don't send the # same delta report twice. if delta_updated is not None: age_delta_updated = datetime.datetime.now() - delta_updated if age_delta_updated > datetime.timedelta(hours=23): print 'Skipping {} + {}, delta > 23h hours old'.format( email_address, domain) return delta_domains = delta_reports.extract_domains(delta_report) noisy_domains = get_noisy_domains(delta_domains) report = EmailReport(delta_report, noisy_domains, include_noisy_domains=include_noisy_domains()) if not report.has_results(): print 'Skipping {} + {}, no results to put in email'.format( email_address, domain) return try: send_email(domain, email_address, sub_id, report) except: print 'Failed to send email for {}:\n {}'.format( domain, traceback.format_exc()) finally: # Mark as emailed to ensure we don't flood if there's an error after # the actual email has been sent. repository.update_last_email_sub_sent_date(sub_id)