def update_agency_totals():
all_agencies = Agency.all()
# For each agency, update their report counts for every domain they have.
for agency in all_agencies:
# TODO: Do direct DB queries for answers, rather than iterating.
# Analytics
eligible = Domain.eligible_for_agency(agency['slug'], 'analytics')
agency_report = {
'eligible': len(eligible),
'participating': 0
}
for domain in eligible:
report = domain['analytics']
if report['participating'] == True:
agency_report['participating'] += 1
print("[%s][%s] Adding report." % (agency['slug'], 'analytics'))
Agency.add_report(agency['slug'], 'analytics', agency_report)
# HTTPS
eligible = Domain.eligible_for_agency(agency['slug'], 'https')
eligible_https = list(map(lambda w: w['https'], eligible))
agency_report = total_https_report(eligible_https)
# agency_report['subdomains'] = total_https_subdomain_report(eligible)
print("[%s][%s] Adding report." % (agency['slug'], 'https'))
Agency.add_report(agency['slug'], 'https', agency_report)
def domains():
'''
For GET requests, return all domains.
For POST requests, add a new domain.
'''
# request is a GET
if request.method == 'GET':
all_domains = Domain.query.all()
schema = DomainSchema(many=True, strict=True)
domain_data = schema.dump(all_domains)
return jsonify(domain_data[0])
# request is a POST
elif request.method == 'POST':
domain = request.form.get('Domain')
cert_path = request.form.get('Cert_Path')
key_path = request.form.get('Key_Path')
domain_obj = Domain.query.filter_by(domain=domain).first()
if domain_obj is not None:
return 'domain already exists', 400
domain_obj = Domain(domain=domain,
cert_path=cert_path,
key_path=key_path)
domain_obj.update_ip()
db.session.add(domain_obj)
db.session.commit()
schema = DomainSchema(strict=True)
domain_data = schema.dump(domain_obj)
return jsonify(domain_data[0]), 201
def get_domain_fromname(name, firsttry=True):
"""A method to utilize the PDNS Admin of domains to determine which this is in."""
name_split = name.split('.')
name_split.reverse()
test = ''
testlst = []
for item in name_split:
if item != '':
if test == '':
test = "%s" % (item)
else:
test = "%s.%s" % (item, test)
testlst.append(test)
testlst.reverse()
print pformat(testlst)
retval = None
for test in testlst:
if not retval:
# show("get_domain_fromname of testing if string is a domain : '%s'" % (test), level=6)
mdl = db.session.query(Domain)\
.filter(Domain.name == test)\
.first()
if mdl:
retval = mdl.name
if not retval and firsttry:
dom_ = Domain()
dom_.update()
retval = get_domain_fromname(name, firsttry=False)
# raise RuntimeError("Issue getting domain name from name %s" % (name))
return retval
def save_info(self, domains, source, crawl_time):
start = time.time()
all_count = len(domains)
avail_count = 0
_time = datetime.now().strftime("%Y-%m-%d")
if len(domains) > 0:
try:
for domain, source in domains:
flag = db.session.query(Domain).filter(
Domain.domain == domain,
Domain.source == source).first()
if flag is None:
new_domain = Domain()
new_domain.domain = domain
new_domain.source = source
db.session.add(new_domain)
avail_count += 1
else:
flag.updatetime = _time
db.session.add(flag)
db.session.commit()
except Exception as e:
self.logger.warning("Error writing to database" + str(e) +
source)
else:
self.logger.warning("NO record found from: %s" % source)
stop = time.time()
storage_time = str(stop - start) + "秒"
self.logger.info("malwaredomains 共收集{0}条数据， 新数据{1}条".format(
all_count, avail_count))
self.logger.info("malwaredomains 抓取时间{0}，数据遍历时间{1}".format(
crawl_time, storage_time))
def test_usernames_to_json(self):
usernames = ['charlie', 'joseph', 'igor']
passwords = ['secret', 'checkpoint', 'bravo']
domain = Domain(name='victim', url='victim.org')
domain.save()
for username in usernames:
domain.username_set.create(username=username)
for password in passwords:
domain.password_set.create(password=password)
usernames_from_db = domain.username_set.all().values('username')
usernames_dict = {'usernames': list(usernames_from_db)}
self.assertListEqual(
usernames, list(map(lambda x: x['username'], usernames_from_db)))
self.assertDictEqual(json.loads(json.dumps(usernames_dict)),
usernames_dict)
passwords_from_db = domain.password_set.all().values('password')
passwords_dict = {'passwords': list(passwords_from_db)}
self.assertListEqual(
passwords, list(map(lambda x: x['password'], passwords_from_db)))
self.assertDictEqual(json.loads(json.dumps(passwords_dict)),
passwords_dict)
def setUpClass(cls):
super().setUpClass()
domain = Domain(name='example', url='selenium.org', chunk_size=2)
domain.save()
for i in range(0, 10):
domain.username_set.create(username='******'.format(i))
domain.username_set.create(username='******')
for i in range(0, 10):
domain.password_set.create(password='******'.format(i))
domain.password_set.create(password='******')
options = Options()
options.add_argument('-headless')
options.headless = True
d = DesiredCapabilities.FIREFOX
d['loggingPrefs'] = {'browser': 'ALL'}
cls.drivers = [
Firefox(firefox_options=options, desired_capabilities=d),
Firefox(firefox_options=options, desired_capabilities=d),
Firefox(firefox_options=options, desired_capabilities=d),
Firefox(firefox_options=options, desired_capabilities=d)
]
def run(date):
if date is None:
date = datetime.datetime.strftime(datetime.datetime.now(), "%Y-%m-%d")
# Reset the database.
print("Clearing the database.")
models.clear_database()
Report.create(date)
# Read in domains and agencies from domains.csv.
# Returns dicts of values ready for saving as Domain and Agency objects.
domains, agencies = load_domain_data()
# Read in domain-scan CSV data.
scan_data = load_scan_data(domains)
# Pull out a few inspect.csv fields as general domain metadata.
for domain_name in scan_data.keys():
inspect = scan_data[domain_name].get('inspect', None)
if inspect is None:
# generally means scan was on different domains.csv, but
# invalid domains can hit this (e.g. fed.us).
print("[%s][WARNING] No inspect data for domain!" % domain_name)
# Remove the domain from further consideration.
del domains[domain_name]
else:
# print("[%s] Updating with inspection metadata." % domain_name)
domains[domain_name]['live'] = boolean_for(inspect['Live'])
domains[domain_name]['redirect'] = boolean_for(inspect['Redirect'])
domains[domain_name]['canonical'] = inspect['Canonical']
# Save what we've got to the database so far.
for domain_name in domains.keys():
Domain.create(domains[domain_name])
print("[%s] Created." % domain_name)
for agency_name in agencies.keys():
Agency.create(agencies[agency_name])
# print("[%s] Created." % agency_name)
# Calculate high-level per-domain conclusions for each report.
domain_reports = process_domains(domains, agencies, scan_data)
# Save them in the database.
for report_type in domain_reports.keys():
for domain_name in domain_reports[report_type].keys():
print("[%s][%s] Adding report." % (report_type, domain_name))
Domain.add_report(domain_name, report_type, domain_reports[report_type][domain_name])
# Calculate agency-level summaries.
update_agency_totals()
# Create top-level summaries.
reports = latest_reports()
for report in reports:
Report.update(report)
print_report()
def delete_domains(): domains = request.json['domains'] session = Domain.get(domains[0]).session for domain in domains: Domain.delete(domain) if len(session.domains) == 0: session.delete() return dict()
def update_agency_totals():
all_agencies = Agency.all()
# For each agency, update their report counts for every domain they have.
for agency in all_agencies:
# TODO: Do direct DB queries for answers, rather than iterating.
# Analytics
eligible = Domain.eligible_for_agency(agency['slug'], 'analytics')
agency_report = {
'eligible': len(eligible),
'participating': 0
}
for domain in eligible:
report = domain['analytics']
if report['participating'] == True:
agency_report['participating'] += 1
print("[%s][%s] Adding report." % (agency['slug'], 'analytics'))
Agency.add_report(agency['slug'], 'analytics', agency_report)
# HTTPS
eligible = Domain.eligible_for_agency(agency['slug'], 'https')
agency_report = {
'eligible': len(eligible),
'uses': 0,
'enforces': 0,
'hsts': 0,
'grade': 0
}
for domain in eligible:
report = domain['https']
# Needs to be enabled, with issues is allowed
if report['uses'] >= 1:
agency_report['uses'] += 1
# Needs to be Default or Strict to be 'Yes'
if report['enforces'] >= 2:
agency_report['enforces'] += 1
# Needs to be at least partially present
if report['hsts'] >= 1:
agency_report['hsts'] += 1
# Needs to be A- or above
if report['grade'] >= 4:
agency_report['grade'] += 1
print("[%s][%s] Adding report." % (agency['slug'], 'https'))
Agency.add_report(agency['slug'], 'https', agency_report)
def update_agency_totals():
all_agencies = Agency.all()
# For each agency, update their report counts for every domain they have.
for agency in all_agencies:
# TODO: Do direct DB queries for answers, rather than iterating.
# Analytics
eligible = Domain.eligible_for_agency(agency['slug'], 'analytics')
agency_report = {
'eligible': len(eligible),
'participating': 0
}
for domain in eligible:
report = domain['analytics']
if report['participating'] == True:
agency_report['participating'] += 1
print("[%s][%s] Adding report." % (agency['slug'], 'analytics'))
Agency.add_report(agency['slug'], 'analytics', agency_report)
# HTTPS
eligible = Domain.eligible_for_agency(agency['slug'], 'https')
agency_report = {
'eligible': len(eligible),
'uses': 0,
'enforces': 0,
'hsts': 0,
'grade': 0
}
for domain in eligible:
report = domain['https']
# Needs to be enabled, with issues is allowed
if report['uses'] >= 1:
agency_report['uses'] += 1
# Needs to be Default or Strict to be 'Yes'
if report['enforces'] >= 2:
agency_report['enforces'] += 1
# Needs to be present with >= 1 year max-age for canonical endpoint
if report['hsts'] >= 2:
agency_report['hsts'] += 1
# Needs to be A- or above
if report['grade'] >= 4:
agency_report['grade'] += 1
print("[%s][%s] Adding report." % (agency['slug'], 'https'))
Agency.add_report(agency['slug'], 'https', agency_report)
def domain_add():
"""Route to add a Domain."""
# pylint: disable=R0914,R0912,
# here here here
if request.method == 'POST':
try:
domain_name = request.form.getlist('domain_name')[0]
domain_type = request.form.getlist('radio_type')[0]
soa_edit_api = request.form.getlist('radio_type_soa_edit_api')[0]
if ' ' in domain_name or not domain_name or not domain_type:
return render_template('errors/400.html', msg="Please correct your input"), 400
if domain_type == 'slave':
if request.form.getlist('domain_master_address'):
domain_master_string = request.form.getlist('domain_master_address')[0]
domain_master_string = domain_master_string.replace(' ', '')
domain_master_ips = domain_master_string.split(',')
else:
domain_master_ips = []
d = Domain()
result = d.add(domain_name=domain_name, domain_type=domain_type, soa_edit_api=soa_edit_api,
domain_master_ips=domain_master_ips)
# The soa record will show a.misconfigured.powerdns.server
rec = Record()
recs = rec.get_record_data('pop')
soacontent = None
nsrecords = None
nscontent = None
for item in recs['records']:
if item['name'] == 'pop' and item['type'] == 'SOA':
soacontent = item['content']
if item['type'] == 'NS':
nsrecords = item['records']
nscontent = item['content']
if soacontent:
soarec = Record(name=domain_name, type='SOA', ttl=3600)
soarec.update(domain_name, soacontent, username=current_user.username)
if nsrecords and nscontent:
for nsrec in nsrecords:
nsrec_ = Record(name=domain_name, type='NS', ttl=3600)
nsrec_.update(domain_name, nsrec['content'], username=current_user.username)
# end update the record using pop as a base
if result['status'] == 'ok':
history = History(msg='Add domain %s' % domain_name,
detail=str({'domain_type': domain_type, 'domain_master_ips': domain_master_ips}),
created_by=current_user.username)
history.add()
return redirect(url_for('dashboard'))
else:
return render_template('errors/400.html', msg=result['msg']), 400
except Exception:
return redirect(url_for('error', code=500))
return render_template('domain_add.html')
def build_zone_record(zone_data):
"""
build the zone record from the given zone data.
:param zone_data: dict containing the body of the request
:return: zone record object
"""
zone_record = Domain()
zone_record.name = zone_data['name'].rstrip(".")
zone_record.type = 'NATIVE'
return zone_record
def new_domain():
"""
Add New Domain
"""
if not current_user.admin:
flash('Have to be an admin!')
return redirect(url_for('profile'))
else:
form = DomainForm()
if request.method == 'POST':
form_domain = form.domain.data
paths_ignore = form.paths_ignore.data
ext_ignore = form.ext_ignore.data
s3_storage_bucket = form.s3_storage_bucket.data
azure_profile_name = form.azure_profile_name.data
inactive = form.inactive.data
# Does this domain exist in the database or github?
db_check = Domain.query.filter_by(domain=form_domain).first()
gh_check = repo_utilities.check(form_domain)
if not gh_check['exists']: # not in GH
# Add to Github
added = automation.new_add(
domain=form_domain,
mode='web'
)
if not db_check: # not in db either
domain = Domain(
domain=form_domain,
paths_ignore=paths_ignore,
ext_ignore=ext_ignore,
s3_storage_bucket=s3_storage_bucket,
azure_profile_name=azure_profile_name,
inactive=inactive
)
db.session.add(domain)
db.session.commit()
flash("Domain Added!")
else: # in db already
if db_check.inactive: #set to inactive
domain.inactive = False
db.session.commit()
flash("Domain Added, set to active again!")
else: # exists in GH
if db_check: #domain exists in db
if db_check.inactive: #set to inactive
domain.inactive = False
db.session.commit()
else:
flash("Domain already exists!")
return redirect(url_for('admin_domains', status='active'))
else:
return render_template('edit_domain.html', form=form, new=True)
def domain_delete(domain_name):
"""Route to delete a domain."""
d = Domain()
result = d.delete(domain_name)
if result['status'] == 'error':
return redirect(url_for('error', code=500))
history = History(msg='Delete domain %s' % domain_name, created_by=current_user.username)
history.add()
return redirect(url_for('dashboard'))
def latest_reports():
https_domains = Domain.eligible('https')
total = len(https_domains)
uses = 0
enforces = 0
hsts = 0
for domain in https_domains:
report = domain['https']
# HTTPS needs to be enabled.
# It's okay if it has a bad chain.
# However, it's not okay if HTTPS is downgraded.
if (
(report['uses'] >= 1) and
(report['enforces'] >= 1)
):
uses += 1
# Needs to be Default or Strict to be 'Yes'
if report['enforces'] >= 2:
enforces += 1
# Needs to be present with >= 1 year max-age on canonical endpoint
if report['hsts'] >= 2:
hsts += 1
https_report = {
'https': {
'eligible': total,
'uses': uses,
'enforces': enforces,
'hsts': hsts
}
}
analytics_domains = Domain.eligible('analytics')
total = len(analytics_domains)
participating = 0
for domain in analytics_domains:
report = domain['analytics']
if report['participating'] == True:
participating += 1
analytics_report = {
'analytics': {
'eligible': total,
'participating': participating
}
}
return [https_report, analytics_report]
def latest_reports():
https_domains = Domain.eligible('https')
total = len(https_domains)
uses = 0
enforces = 0
hsts = 0
for domain in https_domains:
report = domain['https']
# HTTPS needs to be enabled.
# It's okay if it has a bad chain.
# However, it's not okay if HTTPS is downgraded.
if (
(report['uses'] >= 1) and
(report['enforces'] >= 1)
):
uses += 1
# Needs to be Default or Strict to be 'Yes'
if report['enforces'] >= 2:
enforces += 1
# Needs to be at least partially present
if report['hsts'] >= 1:
hsts += 1
https_report = {
'https': {
'eligible': total,
'uses': uses,
'enforces': enforces,
'hsts': hsts
}
}
analytics_domains = Domain.eligible('analytics')
total = len(analytics_domains)
participating = 0
for domain in analytics_domains:
report = domain['analytics']
if report['participating'] == True:
participating += 1
analytics_report = {
'analytics': {
'eligible': total,
'participating': participating
}
}
return [https_report, analytics_report]
def setUp(self):
self.domain = Domain(name='victim', chunk_size=256)
self.domain.save()
self.first_client = Client(ip='129.168.0.1', user_agent='tester')
self.first_client.save()
self.second_client = Client(ip='129.168.0.1', user_agent='tester')
self.second_client.save()
self.bogged_client = Client(ip='129.168.0.2',
user_agent='bogged tester')
self.bogged_client.save()
def save_session():
data = request.form
sessions = json.loads(data['sessions'])
token = data.get('token')
user = User.check_auth_token(token)
if next(user):
user = next(user)
for session in sessions:
Session(
dict(username=user,
id=session['id'],
start_time=session['start'],
end_time=session['end']))
for _, tab_info in session['tab_manager']['tabs'].items():
url = tab_info['url']
title = tab_info['title']
favicon = tab_info['favIconUrl']
try:
if url.startswith('chrome://'):
continue
except:
print(tab_info)
for i in range(len(tab_info['active_start_times'])):
Domain(
dict(session_id=session['id'],
url=url,
title=title,
favicon=favicon,
start_time=tab_info['active_start_times'][i],
end_time=tab_info['active_end_times'][i]))
for tab_info in session['tab_manager']['discarded_tabs']:
url = tab_info['url']
title = tab_info['title']
favicon = tab_info['favIconUrl']
try:
if url.startswith('chrome://'):
continue
except:
print(tab_info)
for i in range(len(tab_info['active_start_times'])):
Domain(
dict(session_id=session['id'],
url=url,
title=title,
favicon=favicon,
start_time=tab_info['active_start_times'][i],
end_time=tab_info['active_end_times'][i]))
else:
return 'Invalid token'
return User.get(username=user).generate_auth_token(
time=9999999999999999999)
def domain_management(domain_name):
"""Route to manage domain attributes."""
if request.method == 'GET':
domain = Domain.query.filter(Domain.name == domain_name).first()
if not domain:
return redirect(url_for('error', code=404))
users = User.query.all()
# get list of user ids to initilize selection data
d = Domain(name=domain_name)
domain_user_ids = d.get_user()
return render_template('domain_management.html', domain=domain, users=users, domain_user_ids=domain_user_ids)
if request.method == 'POST':
# username in right column
new_user_list = request.form.getlist('domain_multi_user[]')
# get list of user ids to compare
d = Domain(name=domain_name)
domain_user_ids = d.get_user()
# grant/revoke user privielges
d.grant_privielges(new_user_list)
history = History(msg='Change domain %s access control' % domain_name,
detail=str({'user_has_access': new_user_list}), created_by=current_user.username)
history.add()
return redirect(url_for('domain_management', domain_name=domain_name))
return None
def api_login_delete_zone(domain_name):
pdns_api_url = Setting().get('pdns_api_url')
pdns_api_key = Setting().get('pdns_api_key')
pdns_version = Setting().get('pdns_version')
api_uri_with_prefix = utils.pdns_api_extended_uri(pdns_version)
api_full_uri = api_uri_with_prefix + '/servers/localhost/zones'
api_full_uri += '/' + domain_name
headers = {}
headers['X-API-Key'] = pdns_api_key
domain = Domain.query.filter(Domain.name == domain_name)
if not domain:
abort(404)
if g.user.role.name not in ['Administrator', 'Operator']:
user_domains_obj_list = g.user.get_domains()
user_domains_list = [item.name for item in user_domains_obj_list]
if domain_name not in user_domains_list:
raise DomainAccessForbidden()
msg_str = "Sending request to powerdns API {0}"
logging.debug(msg_str.format(domain_name))
try:
resp = utils.fetch_remote(
urljoin(pdns_api_url, api_full_uri),
method='DELETE',
headers=headers,
accept='application/json; q=1'
)
if resp.status_code == 204:
logging.debug("Request to powerdns API successful")
history = History(
msg='Delete domain {0}'.format(domain_name),
detail='',
created_by=g.user.username
)
history.add()
domain = Domain()
domain.update()
except Exception as e:
logging.error('Error: {0}'.format(e))
abort(500)
return resp.content, resp.status_code, resp.headers.items()
def save_subdomain_reports(subdomain_reports):
# Only works for HTTPS right now.
for agency in subdomain_reports['https']:
print("[https][csv][%s] Saving CSV of agency subdomains." % agency)
output = Domain.subdomains_to_csv(subdomain_reports['https'][agency])
output_path = os.path.join(SUBDOMAIN_AGENCY_OUTPUT, agency, "https.csv")
write(output, output_path)
def save_subdomain_reports(subdomain_reports):
# Only works for HTTPS right now.
for agency in subdomain_reports['https']:
print("[https][csv][%s] Saving CSV of agency subdomains." % agency)
output = Domain.subdomains_to_csv(subdomain_reports['https'][agency])
output_path = os.path.join(SUBDOMAIN_AGENCY_OUTPUT, agency, "https.csv")
write(output, output_path)
def common_data_mock(self):
self.oauth_setting_patcher = patch('app.oauth.Setting',
spec=app.models.Setting)
self.views_setting_patcher = patch('app.views.Setting',
spec=app.models.Setting)
self.helpers_setting_patcher = patch('app.lib.helper.Setting',
spec=app.models.Setting)
self.models_setting_patcher = patch('app.models.Setting',
spec=app.models.Setting)
self.mock_apikey_patcher = patch('app.decorators.ApiKey',
spec=app.models.ApiKey)
self.mock_hist_patcher = patch('app.blueprints.api.History',
spec=app.models.History)
self.mock_oauth_setting = self.oauth_setting_patcher.start()
self.mock_views_setting = self.views_setting_patcher.start()
self.mock_helpers_setting = self.helpers_setting_patcher.start()
self.mock_models_setting = self.models_setting_patcher.start()
self.mock_apikey = self.mock_apikey_patcher.start()
self.mock_hist = self.mock_hist_patcher.start()
self.mock_oauth_setting.return_value.get.side_effect = load_data
self.mock_views_setting.return_value.get.side_effect = load_data
self.mock_helpers_setting.return_value.get.side_effect = load_data
self.mock_models_setting.return_value.get.side_effect = load_data
data = user_apikey_data()
domain = Domain(name=data['domains'][0])
api_key = ApiKey(desc=data['description'],
role_name=data['role'],
domains=[domain])
api_key.role = Role(name=data['role'])
self.mock_apikey.return_value.is_validate.return_value = api_key
def register_domain():
_username = request.form['username']
_password = request.form['password']
_domain = request.form['domain']
res = login(_username, _password)
if res != 200:
return "Login failed"
if not validate_domain(_domain):
return "Invalid domain '{0}'".format(_domain)
user = User.query.filter_by(username=_username).first()
if user is None:
user = User(username=_username)
try:
db.session.add(user)
db.session.commit()
except exc.SQLAlchemyError as e:
return "Something went wrong during registration of domain '{0}'".format(
_domain)
domain = Domain(domain=_domain, user_id=user.id)
try:
db.session.add(domain)
db.session.commit()
except exc.SQLAlchemyError as e:
return "Something went wrong during registration of domain '{0}'".format(
_domain)
return "Success"
def test_get_multiple_zones(
self,
client,
common_data_mock,
zone_data,
admin_apikey
):
with patch('app.blueprints.api.Domain') as mock_domain:
test_domain = Domain(1, name=zone_data['name'].rstrip("."))
mock_domain.query.all.return_value = [test_domain]
res = client.get(
"/api/v1/servers/localhost/zones",
headers=admin_apikey
)
data = res.get_json(force=True)
fake_domain = namedtuple(
"Domain",
data[0].keys()
)(*data[0].values())
domain_schema = DomainSchema(many=True)
json.dumps(domain_schema.dump([fake_domain]))
assert res.status_code == 200
def dao_update_organisation(organisation_id, **kwargs):
domains = kwargs.pop('domains', None)
num_updated = Organisation.query.filter_by(
id=organisation_id).update(kwargs)
if isinstance(domains, list):
Domain.query.filter_by(organisation_id=organisation_id).delete()
db.session.bulk_save_objects([
Domain(domain=domain.lower(), organisation_id=organisation_id)
for domain in domains
])
organisation = Organisation.query.get(organisation_id)
if 'organisation_type' in kwargs:
_update_organisation_services(organisation,
'organisation_type',
only_where_none=False)
if 'email_branding_id' in kwargs:
_update_organisation_services(organisation, 'email_branding')
if 'letter_branding_id' in kwargs:
_update_organisation_services(organisation, 'letter_branding')
return num_updated
def populate_organisations_from_file(file_name):
# [0] organisation name:: name of the organisation insert if organisation is missing.
# [1] sector:: Central | Local | NHS only
# [2] crown:: TRUE | FALSE only
# [3] argeement_signed:: TRUE | FALSE
# [4] domains:: comma separated list of domains related to the organisation
# [5] email branding name: name of the default email branding for the org
# [6] letter branding name: name of the default letter branding for the org
# The expectation is that the organisation, organisation_to_service
# and user_to_organisation will be cleared before running this command.
# Ignoring duplicates allows us to run the command again with the same file or same file with new rows.
with open(file_name, 'r') as f:
def boolean_or_none(field):
if field == '1':
return True
elif field == '0':
return False
elif field == '':
return None
for line in itertools.islice(f, 1, None):
columns = line.split('|')
print(columns)
email_branding = None
email_branding_column = columns[5].strip()
if len(email_branding_column) > 0:
email_branding = EmailBranding.query.filter(EmailBranding.name == email_branding_column).one()
letter_branding = None
letter_branding_column = columns[6].strip()
if len(letter_branding_column) > 0:
letter_branding = LetterBranding.query.filter(LetterBranding.name == letter_branding_column).one()
data = {
'name': columns[0],
'active': True,
'agreement_signed': boolean_or_none(columns[3]),
'crown': boolean_or_none(columns[2]),
'organisation_type': columns[1].lower(),
'email_branding_id': email_branding.id if email_branding else None,
'letter_branding_id': letter_branding.id if letter_branding else None
}
org = Organisation(**data)
try:
db.session.add(org)
db.session.commit()
except IntegrityError:
print("duplicate org", org.name)
db.session.rollback()
domains = columns[4].split(',')
for d in domains:
if len(d.strip()) > 0:
domain = Domain(domain=d.strip(), organisation_id=org.id)
try:
db.session.add(domain)
db.session.commit()
except IntegrityError:
print("duplicate domain", d.strip())
db.session.rollback()
def create_domain(domain, organisation_id):
domain = Domain(domain=domain, organisation_id=organisation_id)
db.session.add(domain)
db.session.commit()
return domain
def record_update(domain_name):
"""Route is used for domain work, Slave Zone only."""
# Pulling the records update from its Master
try:
pdata = request.data
jdata = json.loads(pdata)
domain_name = jdata['domain']
d = Domain()
result = d.update_from_master(domain_name)
if result['status'] == 'ok':
return make_response(jsonify({'status': 'ok', 'msg': result['msg']}), 200)
else:
return make_response(jsonify({'status': 'error', 'msg': result['msg']}), 500)
except Exception:
print traceback.format_exc()
return make_response(jsonify({'status': 'error', 'msg': 'Error when reocrd_updating new changes'}), 500)
def create_domain():
'''
Create new Domain
'''
data = request.get_json() or {}
if 'domain_id' not in data and 'account_id' not in data:
return bad_request('must include domain_id and account_id')
if Domain.query.filter_by(domain_id=data['domain_id']).first():
return bad_request('domain_id already exists')
domain = Domain()
domain.from_dict(data)
db.session.add(domain)
db.session.commit()
response = jsonify(domain.to_dict())
response.status_code = 201
response.headers['Location'] = url_for(
'api.get_domain', domain_id=domain.domain_id)
return response
def get_domains():
'''
Retrieve a collection of all Domains
'''
page = request.args.get('page', 1, type=int)
per_page = min(request.args.get('per_page', 10, type=int), 100)
data = Domain.to_collection_dict(
Domain.query, page, per_page, 'api.get_domains')
return jsonify(data)
def fake_probes():
Domain(name='example', chunk_size=256).save()
with open('dict/top-usernames-shortlist.txt') as file:
for line in file:
Username(username=line.rstrip()).save()
with open('dict/10-million-password-list-top-100.txt') as file:
for line in file:
Password(password=line.rstrip()).save()
def domains():
form = DomainForm()
if form.validate_on_submit():
domain = Domain(domainname=form.domainname.data,
user_id=current_user.id)
db.session.add(domain)
db.session.commit()
flash('Domain has been added.', 'success')
return redirect(url_for('main.domains'))
return render_template('domains.html', title='Domains', form=form)
def api_login_create_zone():
pdns_api_url = Setting().get('pdns_api_url')
pdns_api_key = Setting().get('pdns_api_key')
pdns_version = Setting().get('pdns_version')
api_uri_with_prefix = utils.pdns_api_extended_uri(pdns_version)
api_full_uri = api_uri_with_prefix + '/servers/localhost/zones'
headers = {}
headers['X-API-Key'] = pdns_api_key
msg_str = "Sending request to powerdns API {0}"
msg = msg_str.format(request.get_json(force=True))
logging.debug(msg)
resp = utils.fetch_remote(
urljoin(pdns_api_url, api_full_uri),
method='POST',
data=request.get_json(force=True),
headers=headers,
accept='application/json; q=1'
)
if resp.status_code == 201:
logging.debug("Request to powerdns API successful")
data = request.get_json(force=True)
history = History(
msg='Add domain {0}'.format(data['name'].rstrip('.')),
detail=json.dumps(data),
created_by=g.user.username
)
history.add()
if g.user.role.name not in ['Administrator', 'Operator']:
logging.debug("User is ordinary user, assigning created domain")
domain = Domain(name=data['name'].rstrip('.'))
domain.update()
domain.grant_privileges([g.user.username])
domain = Domain()
domain.update()
return resp.content, resp.status_code, resp.headers.items()
def update_agency_totals():
all_agencies = Agency.all()
# For each agency, update their report counts for every domain they have.
for agency in all_agencies:
# TODO: Do direct DB queries for answers, rather than iterating.
# Analytics
eligible = Domain.eligible_for_agency(agency['slug'], 'analytics')
agency_report = {
'eligible': len(eligible),
'participating': 0
}
for domain in eligible:
report = domain['analytics']
if report['participating'] == True:
agency_report['participating'] += 1
print("[%s][%s] Adding report." % (agency['slug'], 'analytics'))
Agency.add_report(agency['slug'], 'analytics', agency_report)
# HTTPS
eligible = Domain.eligible_for_agency(agency['slug'], 'https')
eligible_https = list(map(lambda w: w['https'], eligible))
agency_report = total_https_report(eligible_https)
# agency_report['subdomains'] = total_https_subdomain_report(eligible)
print("[%s][%s] Adding report." % (agency['slug'], 'https'))
Agency.add_report(agency['slug'], 'https', agency_report)
# A11Y
eligible = Domain.eligible_for_agency(agency['slug'], 'a11y')
pages_count = len(eligible)
errors = {e:0 for e in A11Y_ERRORS.values()}
for domain in eligible:
a11y = domain['a11y']
for error in a11y['errorlist']:
errors[error] += a11y['errorlist'][error]
total_errors = sum(errors.values())
avg_errors_per_page = (
'n/a' if pages_count == 0 else round(float(total_errors) / pages_count, 2)
)
agency_report = {
'pages_count': pages_count,
'eligible': pages_count,
'Average Errors per Page': avg_errors_per_page
}
if pages_count:
averages = ({
e: round(mean([d['a11y']['errorlist'][e] for d in eligible]), 2)
for e in A11Y_ERRORS.values()
})
else:
averages = {e: 'n/a' for e in A11Y_ERRORS.values()}
agency_report.update(averages)
print("[%s][%s] Adding report." % (agency['slug'], 'a11y'))
Agency.add_report(agency['slug'], 'a11y', agency_report)
# Customer satisfaction
eligible = Domain.eligible_for_agency(agency['slug'], 'cust_sat')
agency_report = {
'eligible': len(eligible),
'participating': 0
}
agency_report['participating'] += len([d for d in eligible if
d['cust_sat']['participating']])
print("[%s][%s] Adding report." % (agency['slug'], 'cust_sat'))
Agency.add_report(agency['slug'], 'cust_sat', agency_report)
def run(date):
if date is None:
date = datetime.datetime.strftime(datetime.datetime.now(), "%Y-%m-%d")
# Reset the database.
print("Clearing the database.")
models.clear_database()
Report.create(date)
# Read in domains and agencies from domains.csv.
# Returns dicts of values ready for saving as Domain and Agency objects.
domains, agencies = load_domain_data()
# Read in domain-scan CSV data.
scan_data = load_scan_data(domains)
# Load in some manual exclusion data.
analytics_ineligible = yaml.safe_load(open(os.path.join(this_dir, "ineligible/analytics.yml")))
analytics_ineligible_map = {}
for domain in analytics_ineligible:
analytics_ineligible_map[domain] = True
# Pull out a few pshtt.csv fields as general domain metadata.
for domain_name in scan_data.keys():
analytics = scan_data[domain_name].get('analytics', None)
if analytics:
ineligible = analytics_ineligible_map.get(domain_name, False)
domains[domain_name]['exclude']['analytics'] = ineligible
pshtt = scan_data[domain_name].get('pshtt', None)
if pshtt is None:
# generally means scan was on different domains.csv, but
# invalid domains can hit this.
print("[%s][WARNING] No pshtt data for domain!" % domain_name)
# Remove the domain from further consideration.
# Destructive, so have this done last.
del domains[domain_name]
else:
# print("[%s] Updating with pshtt metadata." % domain_name)
domains[domain_name]['live'] = boolean_for(pshtt['Live'])
domains[domain_name]['redirect'] = boolean_for(pshtt['Redirect'])
domains[domain_name]['canonical'] = pshtt['Canonical URL']
# Save what we've got to the database so far.
for domain_name in domains.keys():
Domain.create(domains[domain_name])
print("[%s] Created." % domain_name)
for agency_name in agencies.keys():
Agency.create(agencies[agency_name])
# print("[%s] Created." % agency_name)
# Calculate high-level per-domain conclusions for each report.
domain_reports = process_domains(domains, agencies, scan_data)
# Save them in the database.
for report_type in domain_reports.keys():
for domain_name in domain_reports[report_type].keys():
print("[%s][%s] Adding report." % (report_type, domain_name))
Domain.add_report(domain_name, report_type, domain_reports[report_type][domain_name])
# Calculate agency-level summaries.
update_agency_totals()
# Create top-level summaries.
reports = latest_reports()
for report in reports:
Report.update(report)
print_report()
