def addreport():
form = AddReportForm()
report = Report()
context = {
'title': "GecMonitor | Add a Report",
'report': report,
'form': form,
'sites': current_user.sites
}
if request.method == 'POST' and form.validate():
report_name = form.report_name.data
selected_sites = form.current_user_sites.data
description = form.description.data
new_report = Report(report_name=report_name,
sites=selected_sites,
description=description)
current_user.reports.append(new_report)
db.session.add(new_report)
db.session.commit()
sites_list = [[site.id, site.gt_global_id]
for site in new_report.sites]
initial_site_scan = InitialSiteScan(sites_list, new_report.id)
initial_site_scan.start()
flash(f"{report_name} has been added to your sites!")
return redirect(url_for("index"))
return render_template("add_report.html", **context)
def run(date):
if date is None:
date = datetime.datetime.strftime(datetime.datetime.now(), "%Y-%m-%d")
# Reset the database.
print("Clearing the database.")
models.clear_database()
Report.create(date)
# Read in domains and agencies from domains.csv.
# Returns dicts of values ready for saving as Domain and Agency objects.
domains, agencies = load_domain_data()
# Read in domain-scan CSV data.
scan_data = load_scan_data(domains)
# Pull out a few inspect.csv fields as general domain metadata.
for domain_name in scan_data.keys():
inspect = scan_data[domain_name].get('inspect', None)
if inspect is None:
# generally means scan was on different domains.csv, but
# invalid domains can hit this (e.g. fed.us).
print("[%s][WARNING] No inspect data for domain!" % domain_name)
# Remove the domain from further consideration.
del domains[domain_name]
else:
# print("[%s] Updating with inspection metadata." % domain_name)
domains[domain_name]['live'] = boolean_for(inspect['Live'])
domains[domain_name]['redirect'] = boolean_for(inspect['Redirect'])
domains[domain_name]['canonical'] = inspect['Canonical']
# Save what we've got to the database so far.
for domain_name in domains.keys():
Domain.create(domains[domain_name])
print("[%s] Created." % domain_name)
for agency_name in agencies.keys():
Agency.create(agencies[agency_name])
# print("[%s] Created." % agency_name)
# Calculate high-level per-domain conclusions for each report.
domain_reports = process_domains(domains, agencies, scan_data)
# Save them in the database.
for report_type in domain_reports.keys():
for domain_name in domain_reports[report_type].keys():
print("[%s][%s] Adding report." % (report_type, domain_name))
Domain.add_report(domain_name, report_type, domain_reports[report_type][domain_name])
# Calculate agency-level summaries.
update_agency_totals()
# Create top-level summaries.
reports = latest_reports()
for report in reports:
Report.update(report)
print_report()
def create_report():
data = request.get_json() or {}
if 'guid' not in data or 'latitude' not in data or 'longitude' not in data:
return bad_request('must include guid, latitude, and longitude fields')
report = Report()
report.from_dict(data)
db.session.add(report)
db.session.commit()
response = jsonify(report.to_dict())
response.status_code = 201
return response
def report(user_id):
user = User.query.get(user_id)
form = ReportForm()
if form.validate_on_submit():
report = Report(title = form.title.data, username = current_user.username, body = form.body.data)
db.session.add(report)
db.session.commit()
report.add_report(user, current_user)
db.session.commit()
flash('Your report has been recorded.')
return redirect('/clear_search')
return render_template('report.html', title='Report User', form=form)
def handle_request():
if request.method == 'POST':
if request.files:
username = '******'
filename = images.save(request.files['image'])
url = images.url(filename)
#---------------------
result = predict(filename)
#---------------------
user = User.query.filter_by(username=username).first()
if user is None: user = User(username=username)
report = Report(user=user, data=json.dumps(result))
image = Image(report=report)
image.image_filename = filename
image.image_url = url
db.session.add(user)
db.session.add(report)
db.session.add(image)
db.session.commit()
return 'Report Generated'
return 'No Files Recieved'
return 'Method not POST'
def AssignTask(task_id):
task = Task.query.filter().first()
if task is None:
flash('Такой задачи не существует.')
return redirect(url_for('main.ShowTasks'))
form = AssignmentForm()
if current_user.role == UserRoles.manager:
users = User.query.filter(User.dep_id == current_user.dep_id, User.id != current_user.id, User.role == UserRoles.employee).order_by(User.name).all()
users_list = [(u.id, u.name) for u in users]
else:
users_list = [(current_user.id,current_user.name)]
form.user.choices = users_list
if form.validate_on_submit():
report = Report.query.filter(Report.user_id == form.user.data, Report.date == form.date.data).first()
if report is None:
report = Report(user_id = form.user.data, date = form.date.data)
db.session.add(report)
db.session.commit()
report_task = ReportTask(name = task.name, metric = task.metric, measurement = form.measurement.data, completed = form.completed.data)
if form.comment.data is not None:
report_task.comment = form.comment.data.strip()
db.session.add(report_task)
report_task.report = report
db.session.commit()
flash('Задача успешно назначена сотруднику.')
else:
for error in form.user.errors + form.date.errors + form.measurement.errors + form.completed.errors + form.comment.errors:
flash(error)
return redirect(url_for('main.ShowTasks'))
def reportUser(username):
user = User.query.filter_by(username=username).first()
form = ReportForm()
if form.validate_on_submit():
report = Report(reasonReported=form.reasons.data, report_recipient_id = user.id)
score = user.reportTally #gets report numbers
if form.reasons.data == 'Harrasment':
score += 1
user.reportTally = score
db.session.commit()
elif form.reasons.data == 'Illegal Activities':
score += 3
user.reportTally = score
db.session.commit()
elif form.reasons.data == 'Blackmailing':
score += 2
user.reportTally = score
db.session.commit()
elif form.reasons.data == 'Harmful/Racist messages':
score += 5
user.reportTally = score
db.session.commit()
elif form.reasons.data == 'Stalking':
score += 2
user.reportTally = score
db.session.commit()
db.session.add(report)
db.session.commit()
#accountCheck(user.username)
flash('User has been Reported!')
return redirect(url_for('user', username=username))
return render_template('Report.html', user = user, form=form, title = 'Reporting')
def test_find_reports(self):
"""Test that the reports that a loan belongs to are found correctly."""
u, v, l, r = self.test_add_user_loan_report()
# Create reports, and assign them to loan l2 except the fifth one
l2 = Loan(balance=100, currency='USD', debtor=u)
db.session.add(l2)
rep = []
for i in range(5):
rep.append(
Report(title="report" + str(i),
body="This is the body of the report",
author=v))
db.session.add(rep[i])
db.session.commit()
for i in range(4):
l2.refer(rep[i])
# Check that all but the fifth are referring l2
for i in range(5):
if i != 4:
self.assertTrue(l2.is_referred(rep[i]))
else:
self.assertFalse(l2.is_referred(rep[i]))
# Check that the rep list is NOT l2.reports
self.assertFalse(l2.reports == rep)
# Check that the first 4 items in rep are in the list of l2.reports
for i in range(len(rep[:4])):
self.assertTrue(rep[i] in l2.reports)
def publish():
if request.method == "GET":
admin_id = session.get('admin_id')
admin_name = Admin.query.filter(Admin.id == admin_id).first().name
contexts = {"admin_name": admin_name}
return render_template("admin/publish.html", **contexts)
else:
#每次写新报告之前删除之前的报告,根据登录id删除之前的记录
admin_id = session.get('admin_id')
old_record = Report.query.filter(Report.id == admin_id).first()
try:
db.session.delete(old_record)
db.session.commit()
#获取新的report并添加进数据库
except:
pass
name = Admin.query.filter(Admin.id == admin_id).first().name
content = request.form.get('content')
report = Report(id=admin_id,
content=content,
name=name,
admin_id=admin_id)
db.session.add(report)
db.session.commit()
return redirect('/report_show')
def add_report():
"""
Add report to the database
"""
add_report = True
form = ReportForm()
if form.validate_on_submit():
report = Report(name=form.name.data,
report_file=secure_filename(
form.report_file.data.filename),
employee=current_user)
f = form.report_file.data
filename = secure_filename(f.filename)
f.save(os.path.join(current_app.config['UPLOAD_FOLDER'], filename))
try:
# add report to the database
db.session.add(report)
db.session.commit()
flash('Đã thêm báo cáo thành công.')
except:
# in case report file already exists
flash('Lỗi: Tên file báo cáo đã tồn tại.')
# redirect to the reports page
return redirect(url_for('home.list_reports'))
# load report template
return render_template('home/report.html',
add_report=add_report,
form=form,
title='Thêm báo cáo')
def add_report_to_db(data):
province=Province.query.filter_by(id=int(data['province'])).first()
district=District.query.filter_by(id=int(data['district'])).first()
localbody=LocalBody.query.filter_by(id=int(data['localbody'])).first()
report=Report(province=province.name,district=district.name,localbody=localbody.name,customer_id=data['customer_id'],latitude=data['latitude'],longitude=data['longitude'],ward=data['ward_number'],status="Reported")
db.session.add(report)
db.session.commit()
def test_report(self):
self.assertEqual(Report.query.count(), 0)
report = Report(reporter_id='66666666',
reported_id='11111',
report_id=gen_id(),
body='Terrible mate.')
db.session.add(report)
db.session.commit()
self.assertEqual(Report.query.count(), 1)
def report(cert_id):
"""
Return template for report page
"""
cert = Cert.query.get(cert_id)
if cert.file_id is None:
urls = [url_for('static', filename=os.path.join('img', "missing.png"))]
else:
if not cert.file.converted:
cert.file.convert()
urls = cert.file.pngs
form = ReportForm(request.form)
if request.method == 'POST':
if form.validate_on_submit():
form_completed = False
for key, value in form.data.items():
if value != "" and key is not 'csrf_token' and key is not 'submit':
form_completed = True
if not form_completed:
flash("Please complete at least one form field.",
category="warning")
return render_template('report_issue.html',
form=form,
cert=cert,
urls=urls)
else:
form_fields = {
name: data
for name, data in {
'county': form.county.data,
'month': form.month.data,
'day': form.day.data,
'year': form.year.data,
'age': form.age.data,
'number': form.number.data,
'soundex': form.soundex.data,
'first_name': form.first_name.data,
'last_name': form.last_name.data,
'comments': form.comments.data
}.items() if data
}
report = Report(cert_id=cert_id,
user_id=current_user.id,
values=form_fields)
db.session.add(report)
db.session.commit()
flash("Your report has been submitted.", category="success")
return redirect('/')
else:
flash("An error has occurred.")
print(form.errors)
return render_template('report_issue.html',
form=form,
cert=cert,
urls=urls)
def one_report(severity, email, days, symptoms):
user = User.query.filter_by(email=email).first()
user_id = user.id
days = datetime.today() - timedelta(days=days)
new_report = Report(severity=severity,
user_id=user_id,
school_id=user.school.id,
date=days)
new_report.symptoms = []
new_report.buildings = []
symptoms = symptoms.split(",")
for symptom in symptoms:
s = Symptom.query.filter_by(name=symptom).first()
if s:
new_report.symptoms.append(s)
for building in user.buildings:
new_report.buildings.append(
Building.query.filter_by(name=building.name).first())
db.session.add(new_report)
db.session.commit()
def test_report(self):
seller = User(username="******",
email="*****@*****.**",
first_name="SellerFirst",
last_name="SellerLast",
address="Seller Address",
card_number="1111111111111111",
security_code="111",
is_seller=True)
seller.set_password('Seller')
db.session.add(seller)
db.session.commit()
buyer = User(username="******",
email="*****@*****.**",
first_name="BuyerFirst",
last_name="BuyerLast",
address="Buyer Address",
card_number="2222222222222222",
security_code="222",
is_seller=False)
buyer.set_password("Buyer")
db.session.add(buyer)
db.session.commit()
report = Report(body='This is a test report for a test seller!')
db.session.add(report)
db.session.commit()
self.assertEqual(report.body,
'This is a test report for a test seller!')
self.assertEqual(seller.reports.all(), [])
self.assertFalse(report in seller.reports)
report.add_report(seller, buyer)
self.assertEqual(seller.reports.count(), 1)
self.assertEqual(seller.reports.first().username, 'ABuyer')
self.assertEqual(seller.reports.first().body,
'This is a test report for a test seller!')
self.assertEqual(seller.reports.first().reported_id, seller.id)
def multiavscan(sha256):
"""Task to run the AV scanners
:param sha256: File hash
"""
scanned = Sample.query.filter_by(sha256=sha256).first()
file_path = os.path.join(current_app.config['APP_UPLOADS_SAMPLES'], sha256)
av = Scanner(current_app.config['AVSCAN_CONFIG'])
av_report = av.scan(file_path)
scanned.reports.append(Report(type_id=2, report=json.dumps(av_report)))
db.session.add(scanned)
db.session.commit()
def checkin_employee():
id = request.form['id'].strip()
observation = 'Prezent'
employee = Employee.query.get_or_404(id)
report = Report(employee=employee, observations=observation, check_in=True)
db.session.add(report)
db.session.commit()
return json.dumps({'success': True}), 200, {
'ContentType': 'application/json'
}
def test_add_user_loan_report(self):
"""Test that the test worked."""
u = User(name="Uu")
v = User(name="Vv")
l = Loan(balance=10, currency='GBP', debtor=u)
r = Report(title="The Report",
body="This is the body of the report",
author=v)
db.session.add(u)
db.session.add(v)
db.session.add(r)
db.session.add(l)
db.session.commit()
return u, v, l, r
def specific_report(id):
form = ReportForm()
if form.validate_on_submit():
prev_url = request.args.get('prev')
report_type = request.args.get('type')
if url_parse(prev_url).netloc != '':
return redirect('index')
if report_type.find('post') != -1:
report = Report(reason=form.reason.data,
author=current_user,
post_id=id)
elif report_type.find('comment') != -1:
report = Report(reason=form.reason.data,
author=current_user,
comment_id=id)
else:
report = Report(reason=form.reason.data,
author=current_user,
profile_id=id)
db.session.add(report)
db.session.commit()
flash('Your report has been submitted. Thank you for your feedback!')
return redirect(prev_url)
return render_template('specific_report.html', title='Report', form=form)
def new_report():
rawTime = time.time()
startDate = datetime.datetime.fromtimestamp(rawTime).strftime(
'%Y-%m-%d %H:%M:%S')
substance = request.args.get('substance', "null", type=str)
dosage = request.args.get('dosage', 0, type=int)
dosagelabel = request.args.get('dosagelabel', "null", type=str)
roa = request.args.get('roa', "null", type=str)
source = request.args.get('source', "null", type=str)
tester = startDate + substance + str(dosage) + roa + dosagelabel + source
newReport = Report(StartDate=startDate,
Substance=substance,
Dosage=dosage,
DosageLabel=dosagelabel,
ROA=roa,
Source=source)
db.session.add(newReport)
db.session.commit()
#print("ID pulled from object: " + str(newReport.id), file=sys.stderr)
return jsonify(result=newReport.serialize())
def print_report():
print()
report = Report.latest()
for report_type in report.keys():
if report_type == "report_date":
continue
print("[%s]" % report_type)
eligible = report[report_type]["eligible"]
for key in report[report_type].keys():
if key == "eligible":
continue
print("%s: %i" % (key, percent(report[report_type][key], eligible)))
print()
def report():
if not session['logged_in']:
return redirect(url_for('index'))
form = ReportForm()
if request.method == "POST":
# alpha = request.form['by_cat']
# alpha = request.form['startdate']
# beta = request.form['enddate']
#
is_checked = request.form.get('by_cat')
single_cat_id = request.form.get('single_cat_id')
if is_checked:
# return render_template('404.html')
cat_list = Report.expenses_by_cat(request.form['startdate'], request.form['enddate'])
return render_template('report_cats.html', cats=cat_list, header_text="Category totals")
elif single_cat_id:
single_list = Report.expenses_single_cat(request.form['startdate'], request.form['enddate'], single_cat_id)
# return render_template('report_cats.html', cats=cat_list, header_text="Category totals")
return render_template('expenses.html', expense_list=single_list,
header_text="Expenses for cat {}".format(single_cat_id))
else:
expense_list = Report.expense_report(request.form['startdate'], request.form['enddate'])
#
# r = Report(
# request.form['startdate'],
# request.form['enddate'],
# request.form['by_cat']
# )
# cats = [{"cat":"alpha", "total":"111.11"}, {"cat":"beta", "total":"222.2"}]
# cats = r.exp_report_by_cat()
return render_template('report_results.html', expense_list=expense_list, bankitems=bankitems,
header_text="Expenses")
return render_template('report.html', form=form, header_text="Reports")
def print_report():
print()
report = Report.latest()
for report_type in report.keys():
if report_type == "report_date":
continue
print("[%s]" % report_type)
eligible = report[report_type]["eligible"]
for key in report[report_type].keys():
if key == "eligible":
continue
print("%s: %i" % (key, percent(report[report_type][key], eligible)))
print()
def print_report():
print()
report = Report.latest()
for report_type in report.keys():
# The a11y report has a very different use than the others
if report_type == "report_date" or report_type == "a11y":
continue
print("[%s]" % report_type)
eligible = report[report_type]["eligible"]
for key in report[report_type].keys():
if key == "eligible":
continue
print("%s: %i" % (key, percent(report[report_type][key], eligible)))
print()
def print_report():
print()
report = Report.latest()
for report_type in report.keys():
# The a11y report has a very different use than the others
if report_type == "report_date" or report_type == "a11y":
continue
print("[%s]" % report_type)
eligible = report[report_type]["eligible"]
for key in report[report_type].keys():
if key == "eligible":
continue
print("%s: %i" %
(key, percent(report[report_type][key], eligible)))
print()
def general_report():
form0 = SearchProfileForm()
form = ReportForm()
if form.validate_on_submit():
prev_url = request.args.get('prev')
if url_parse(prev_url).netloc != '':
return redirect(url_for('index'))
report = Report(reason=form.reason.data,
page_of_report=prev_url,
author=current_user)
db.session.add(report)
db.session.commit()
flash('Your report has been submitted. Thank you for your feedback!')
return redirect(prev_url)
return render_template('general_report.html',
form=form,
form0=form0,
title='Report')
def index():
# Build start and end dates
(start, end) = __build_dates_from_request(request)
# Gather filters
filter = request.args.get('group',
default=Filter.no_filter(),
type=__to_attribute_filter)
# Build Repo models
repos = Repo.all_display()
# Group Repos by filter
groups = Group.group_repos(repos, filter, start, end)
# Generate report
report = Report(groups, filter, start, end)
# Build filters
filters = [Filter.no_filter()] + Repo.filterable_attributes()
# Render from template
rendered = render_template("report.html", report=report, filters=filters)
# Build response
return __build_response_with_dates_cookies(rendered, 200, start, end)
def upload():
form = UploadForm()
if request.method == 'POST':
if form.validate_on_submit():
username = form.username.data
filename = images.save(request.files['image'])
url = images.url(filename)
#---------------------
result = predict(filename)
#---------------------
user = User.query.filter_by(username=username).first()
if user is None: user = User(username=username)
report = Report(user=user, data=json.dumps(result))
image = Image(report=report)
image.image_filename = filename
image.image_url = url
db.session.add(user)
db.session.add(report)
db.session.add(image)
db.session.commit()
flash(
'User Report for username: {} generated. '.format(
form.username.data), 'success')
return redirect(url_for('report', report_id=report.id))
else:
flash_errors(form)
flash('ERROR! Report was not generated.', 'error')
return render_template('upload.html', title='Upload', form=form)
def addreport(id):
form = AddVisitReport()
visit = Visit.query.filter_by(id=id).first()
partner = Partner.query.filter_by(id=visit.partner).first()
author = current_user.id
if form.validate_on_submit():
report = Report(content=form.report.data, visit_id=id, author=author)
db.session.add(report)
db.session.commit()
flash('Visit report added.')
return redirect(url_for('visitdetails', id=partner.id))
return render_template('addreport.html',
form=form,
visit=visit,
partner=partner)
def setup_db():
with app.app_context():
db.drop_all()
db.create_all()
db.session.add(
Report(id=4,
content=json.dumps({
"organization":
"Flowers Inc.",
"reported_at":
"2017-11-19",
"created_at":
"2017-11-23",
"inventory": [{
"name": "Flower pot",
"price": "2.00"
}, {
"name": "Roses, 24",
"price": "50.00"
}]
})))
db.session.commit()
def create_report():
form = CreateReportForm()
if form.validate_on_submit():
lat = float(form.coordinates_lat.data)
lng = float(form.coordinates_lng.data)
zone = int(form.zone.data)
report = Report(user_id=current_user.id,
date=form.date.data,
crime_id=form.crime.data,
crime_hour_id=form.hour.data,
latitude=lat,
longitude=lng,
zone=zone,
details=form.details.data,
reference_id=form.reference.data)
db.session.add(report)
db.session.commit()
return redirect(url_for('user_reports'))
return render_template('create_report.html',
title='Crear Reporte',
active='add_report',
form=form)
def report():
if Configuration.query.count() == 0:
#TODO return 404 somehow
pass
form = ReportForm()
form.config_id.choices =[(c.id, c.config_name) for c in Configuration.\
query.with_entities(Configuration.id,
Configuration.config_name).all()]
if form.validate_on_submit():
report = Report(report_name=form.report_name.data,
config_id=form.config_id.data,
hour=form.hour.data,
day=form.day.data,
minute=form.minute.data,
query_text=form.query_text.data)
db.session.add(report)
db.session.commit()
# keep this redirect here for a bit till I figure out what to do
# with it
return redirect(url_for('index'))
return render_template('report.html', form=form)
def submit_report():
"""
Create and submit report
"""
form = ReportForm()
form.main_teacher.choices = Teacher.get_teachers_for_choices()
form.assistant_teacher.choices = Teacher.get_teachers_for_choices()
form.submitted_by.choices = Teacher.get_teachers_for_choices()
if form.validate_on_submit():
report = Report(main_teacher_id=form.main_teacher.data,
assistant_teacher_id=form.assistant_teacher.data,
teaching_date=form.teaching_date.data,
teaching_content=form.teaching_content.data,
student_attendance=form.student_attendance.data,
submitted_by_id=form.submitted_by.data)
db.session.add(report)
db.session.commit()
flash(message='Report is successfully submitted.', category='success')
return redirect(url_for('index'))
return render_template('report_submit.html',
title='Submit a Report',
form=form)
def __init__(self, file, description, user_id=None, admin_submission=True, **kwargs):
print("{0:<35}{1}".format("Process", "Time (s)"))
print("--------------------------------------------")
t_total = time()
# unzip the file and create the submission objects
t = time()
if admin_submission:
filehandler = FileHandler(file, description)
else:
filehandler = FileHandler(file, description, user_id, batch=False)
submission_list = filehandler.submissions
t = time()-t
print("{0:<35}{1:.5f}".format("Extracted "+str(len(submission_list))+" files", t))
# fill out the fingerprints and submission_id's of the submissions
t = time()
self.submission_id = self.get_submission_id() if admin_submission else -1
for submission in submission_list:
submission.submission_id = self.submission_id
fingerprinter = Fingerprinter(submission.file_contents, submission.filename, **kwargs)
submission.fingerprint = fingerprinter.fingerprint
t = time()-t
print("{0:<35}{1:.5f}".format("Generated fingerprints", t))
# save the submissions for later use
t = time()
if admin_submission:
Submission.objects.bulk_create(submission_list)
# retrieve same submissions now that the db has given them all primary keys
submission_list = Submission.objects.filter(submission_id=self.submission_id)
# eval all the fingerprints from string to list
for submission in submission_list:
submission.fingerprint = eval(submission.fingerprint)
t = time()-t
print("{0:<35}{1:.5f}".format("Saved submissions with ID "+str(self.submission_id), t))
# do the comparison and get the report
t = time()
comparator = Comparator(submission_list, compare_history=True, **kwargs)
t = time()-t
print("{0:<35}{1:.5f}".format("Performed comparison", t))
# create the report object and save it
t = time()
self.report = Report()
self.report.submission_id = self.submission_id
self.report.description = description
self.report.match_list = comparator.report
if admin_submission:
self.report.save()
t = time()-t
print("{0:<35}{1:.5f}".format("Saved report with ID "+str(self.report.id), t))
t_total = time()-t_total
print("{0:<35}{1:.5f}".format("TOTAL", t_total))
def run(date):
if date is None:
date = datetime.datetime.strftime(datetime.datetime.now(), "%Y-%m-%d")
# Reset the database.
print("Clearing the database.")
models.clear_database()
Report.create(date)
# Read in domains and agencies from domains.csv.
# Returns dicts of values ready for saving as Domain and Agency objects.
domains, agencies = load_domain_data()
# Read in domain-scan CSV data.
scan_data = load_scan_data(domains)
# Load in some manual exclusion data.
analytics_ineligible = yaml.safe_load(open(os.path.join(this_dir, "ineligible/analytics.yml")))
analytics_ineligible_map = {}
for domain in analytics_ineligible:
analytics_ineligible_map[domain] = True
# Pull out a few pshtt.csv fields as general domain metadata.
for domain_name in scan_data.keys():
analytics = scan_data[domain_name].get('analytics', None)
if analytics:
ineligible = analytics_ineligible_map.get(domain_name, False)
domains[domain_name]['exclude']['analytics'] = ineligible
pshtt = scan_data[domain_name].get('pshtt', None)
if pshtt is None:
# generally means scan was on different domains.csv, but
# invalid domains can hit this.
print("[%s][WARNING] No pshtt data for domain!" % domain_name)
# Remove the domain from further consideration.
# Destructive, so have this done last.
del domains[domain_name]
else:
# print("[%s] Updating with pshtt metadata." % domain_name)
domains[domain_name]['live'] = boolean_for(pshtt['Live'])
domains[domain_name]['redirect'] = boolean_for(pshtt['Redirect'])
domains[domain_name]['canonical'] = pshtt['Canonical URL']
# Save what we've got to the database so far.
for domain_name in domains.keys():
Domain.create(domains[domain_name])
print("[%s] Created." % domain_name)
for agency_name in agencies.keys():
Agency.create(agencies[agency_name])
# print("[%s] Created." % agency_name)
# Calculate high-level per-domain conclusions for each report.
domain_reports = process_domains(domains, agencies, scan_data)
# Save them in the database.
for report_type in domain_reports.keys():
for domain_name in domain_reports[report_type].keys():
print("[%s][%s] Adding report." % (report_type, domain_name))
Domain.add_report(domain_name, report_type, domain_reports[report_type][domain_name])
# Calculate agency-level summaries.
update_agency_totals()
# Create top-level summaries.
reports = latest_reports()
for report in reports:
Report.update(report)
print_report()
