def setup(self):
super(TestEngine, self).setup()
round_time_sleep_obj = Setting.get_setting('round_time_sleep')
round_time_sleep_obj.value = 0
self.session.add(round_time_sleep_obj)
worker_refresh_time_obj = Setting.get_setting('worker_refresh_time')
worker_refresh_time_obj.value = 0
self.session.add(worker_refresh_time_obj)
self.session.commit()
def setup(self):
super(TestEngine, self).setup()
target_round_time_obj = Setting.get_setting("target_round_time")
target_round_time_obj.value = 0
self.session.add(target_round_time_obj)
worker_refresh_time_obj = Setting.get_setting("worker_refresh_time")
worker_refresh_time_obj.value = 0
self.session.add(worker_refresh_time_obj)
self.session.commit()
def settings():
if current_user.is_white_team:
about_page_content = Setting.get_setting('about_page_content').value
welcome_page_content = Setting.get_setting(
'welcome_page_content').value
blue_teams = Team.get_all_blue_teams()
return render_template('admin/settings.html',
blue_teams=blue_teams,
about_page_content=about_page_content,
welcome_page_content=welcome_page_content)
else:
return redirect(url_for('auth.unauthorized'))
def permissions():
if current_user.is_white_team:
blue_teams = Team.get_all_blue_teams()
return render_template(
'admin/permissions.html',
blue_teams=blue_teams,
blue_team_update_hostname=Setting.get_setting('blue_team_update_hostname').value,
blue_team_update_port=Setting.get_setting('blue_team_update_port').value,
blue_team_update_account_usernames=Setting.get_setting('blue_team_update_account_usernames').value,
blue_team_update_account_passwords=Setting.get_setting('blue_team_update_account_passwords').value,
)
else:
return redirect(url_for('auth.unauthorized'))
def test_get_setting(self):
setting_old = Setting(name='test_setting', value='test value example')
self.session.add(setting_old)
setting_new = Setting(name='test_setting', value='updated example')
self.session.add(setting_new)
self.session.commit()
assert Setting.get_setting('test_setting').value == 'updated example'
def settings():
if current_user.is_white_team:
about_page_content = Setting.get_setting('about_page_content').value
welcome_page_content = Setting.get_setting('welcome_page_content').value
round_time_sleep = Setting.get_setting('round_time_sleep').value
worker_refresh_time = Setting.get_setting('worker_refresh_time').value
blue_teams = Team.get_all_blue_teams()
return render_template(
'admin/settings.html',
blue_teams=blue_teams,
round_time_sleep=round_time_sleep,
worker_refresh_time=worker_refresh_time,
about_page_content=about_page_content,
welcome_page_content=welcome_page_content
)
else:
return redirect(url_for('auth.unauthorized'))
def settings():
if current_user.is_white_team:
about_page_content = Setting.get_setting("about_page_content").value
welcome_page_content = Setting.get_setting(
"welcome_page_content").value
target_round_time = Setting.get_setting("target_round_time").value
worker_refresh_time = Setting.get_setting("worker_refresh_time").value
blue_teams = Team.get_all_blue_teams()
return render_template(
"admin/settings.html",
blue_teams=blue_teams,
target_round_time=target_round_time,
worker_refresh_time=worker_refresh_time,
about_page_content=about_page_content,
welcome_page_content=welcome_page_content,
)
else:
return redirect(url_for("auth.unauthorized"))
def verify_settings(self):
settings = [
'round_time_sleep',
'worker_refresh_time'
]
for setting_name in settings:
if not Setting.get_setting(setting_name):
logger.error("Must have " + setting_name + " setting.")
exit(1)
def service(id):
service = session.query(Service).get(id)
if service is None or not current_user.team == service.team:
return redirect(url_for('auth.unauthorized'))
modify_hostname_setting = Setting.get_setting('blue_team_update_hostname').value
modify_port_setting = Setting.get_setting('blue_team_update_port').value
modify_account_usernames_setting = Setting.get_setting('blue_team_update_account_usernames').value
modify_account_passwords_setting = Setting.get_setting('blue_team_update_account_passwords').value
return render_template(
'service.html',
id=id,
service=service,
modify_hostname_setting=modify_hostname_setting,
modify_port_setting=modify_port_setting,
modify_account_passwords_setting=modify_account_passwords_setting,
modify_account_usernames_setting=modify_account_usernames_setting
)
def permissions():
if current_user.is_white_team:
blue_teams = Team.get_all_blue_teams()
return render_template(
"admin/permissions.html",
blue_teams=blue_teams,
blue_team_update_hostname=Setting.get_setting(
"blue_team_update_hostname").value,
blue_team_update_port=Setting.get_setting(
"blue_team_update_port").value,
blue_team_update_account_usernames=Setting.get_setting(
"blue_team_update_account_usernames").value,
blue_team_update_account_passwords=Setting.get_setting(
"blue_team_update_account_passwords").value,
blue_team_view_check_output=Setting.get_setting(
"blue_team_view_check_output").value,
)
else:
return redirect(url_for("auth.unauthorized"))
def admin_update_blueteam_edit_account_usernames():
if current_user.is_white_team:
setting = Setting.get_setting('blue_team_update_account_usernames')
if setting.value is True:
setting.value = False
else:
setting.value = True
session.add(setting)
session.commit()
return redirect(url_for('admin.permissions'))
return {'status': 'Unauthorized'}, 403
def admin_update_blueteam_edit_account_passwords():
if current_user.is_white_team:
setting = Setting.get_setting("blue_team_update_account_passwords")
if setting.value is True:
setting.value = False
else:
setting.value = True
session.add(setting)
session.commit()
return redirect(url_for("admin.permissions"))
return {"status": "Unauthorized"}, 403
def admin_update_welcome_page_content():
if current_user.is_white_team:
if 'welcome_page_content' in request.form:
setting = Setting.get_setting('welcome_page_content')
setting.value = request.form['welcome_page_content']
db.save(setting)
flash('Welcome Page Content Successfully Updated.', 'success')
return redirect(url_for('admin.settings'))
flash('Error: welcome_page_content not specified.', 'danger')
return redirect(url_for('admin.manage'))
return {'status': 'Unauthorized'}, 403
def admin_update_blueteam_view_check_output():
if current_user.is_white_team:
setting = Setting.get_setting("blue_team_view_check_output")
print(setting.__dict__)
if setting.value is True:
setting.value = False
else:
setting.value = True
session.add(setting)
session.commit()
return redirect(url_for("admin.permissions"))
return {"status": "Unauthorized"}, 403
def admin_update_about_page_content():
if current_user.is_white_team:
if 'about_page_content' in request.form:
setting = Setting.get_setting('about_page_content')
setting.value = request.form['about_page_content']
session.add(setting)
session.commit()
flash('About Page Content Successfully Updated.', 'success')
return redirect(url_for('admin.settings'))
flash('Error: about_page_content not specified.', 'danger')
return redirect(url_for('admin.manage'))
return {'status': 'Unauthorized'}, 403
def admin_update_welcome_page_content():
if current_user.is_white_team:
if "welcome_page_content" in request.form:
setting = Setting.get_setting("welcome_page_content")
setting.value = request.form["welcome_page_content"]
session.add(setting)
session.commit()
flash("Welcome Page Content Successfully Updated.", "success")
return redirect(url_for("admin.settings"))
flash("Error: welcome_page_content not specified.", "danger")
return redirect(url_for("admin.manage"))
return {"status": "Unauthorized"}, 403
def update_service_account_info():
if current_user.is_white_team or current_user.is_blue_team:
if 'name' in request.form and 'value' in request.form and 'pk' in request.form:
account = session.query(Account).get(int(request.form['pk']))
if current_user.team == account.service.team or current_user.is_white_team:
if account:
if request.form['name'] == 'username':
modify_usernames_setting = Setting.get_setting(
'blue_team_update_account_usernames')
if modify_usernames_setting.value is True:
account.username = html.escape(
request.form['value'])
elif request.form['name'] == 'password':
modify_password_setting = Setting.get_setting(
'blue_team_update_account_passwords')
if modify_password_setting.value is True:
account.password = html.escape(
request.form['value'])
session.add(account)
session.commit()
return jsonify({'status': 'Updated Account Information'})
return jsonify({'error': 'Incorrect permissions'})
return jsonify({'error': 'Incorrect permissions'})
return jsonify({'error': 'Incorrect permissions'})
def update_service_account_info():
if current_user.is_white_team or current_user.is_blue_team:
if "name" in request.form and "value" in request.form and "pk" in request.form:
account = session.query(Account).get(int(request.form["pk"]))
if current_user.team == account.service.team or current_user.is_white_team:
if account:
if request.form["name"] == "username":
modify_usernames_setting = Setting.get_setting(
"blue_team_update_account_usernames")
if modify_usernames_setting.value is True:
account.username = html.escape(
request.form["value"])
elif request.form["name"] == "password":
modify_password_setting = Setting.get_setting(
"blue_team_update_account_passwords")
if modify_password_setting.value is True:
account.password = html.escape(
request.form["value"])
session.add(account)
session.commit()
return jsonify({"status": "Updated Account Information"})
return jsonify({"error": "Incorrect permissions"})
return jsonify({"error": "Incorrect permissions"})
return jsonify({"error": "Incorrect permissions"})
def admin_update_round_time_sleep():
if current_user.is_white_team:
if 'round_time_sleep' in request.form:
setting = Setting.get_setting('round_time_sleep')
input_time = request.form['round_time_sleep']
if not input_time.isdigit():
flash('Error: Round Sleep Time must be an integer.', 'danger')
return redirect(url_for('admin.settings'))
setting.value = input_time
session.add(setting)
session.commit()
flash('Round Sleep Time Successfully Updated.', 'success')
return redirect(url_for('admin.settings'))
flash('Error: round_time_sleep not specified.', 'danger')
return redirect(url_for('admin.settings'))
return {'status': 'Unauthorized'}, 403
def admin_update_target_round_time():
if current_user.is_white_team:
if "target_round_time" in request.form:
setting = Setting.get_setting("target_round_time")
input_time = request.form["target_round_time"]
if not input_time.isdigit():
flash("Error: Target Round Time must be an integer.", "danger")
return redirect(url_for("admin.settings"))
setting.value = input_time
session.add(setting)
session.commit()
flash("Target Round Time Successfully Updated.", "success")
return redirect(url_for("admin.settings"))
flash("Error: target_round_time not specified.", "danger")
return redirect(url_for("admin.settings"))
return {"status": "Unauthorized"}, 403
def admin_update_worker_refresh_time():
if current_user.is_white_team:
if 'worker_refresh_time' in request.form:
setting = Setting.get_setting('worker_refresh_time')
input_time = request.form['worker_refresh_time']
if not input_time.isdigit():
flash('Error: Worker Refresh Time must be an integer.',
'danger')
return redirect(url_for('admin.settings'))
setting.value = input_time
session.add(setting)
session.commit()
flash('Worker Refresh Time Successfully Updated.', 'success')
return redirect(url_for('admin.settings'))
flash('Error: worker_refresh_time not specified.', 'danger')
return redirect(url_for('admin.settings'))
return {'status': 'Unauthorized'}, 403
def update_port():
if current_user.is_blue_team:
if 'name' in request.form and 'value' in request.form and 'pk' in request.form:
service = session.query(Service).get(int(request.form['pk']))
if service:
if service.team == current_user.team and request.form[
'name'] == 'port':
modify_port_setting = Setting.get_setting(
'blue_team_update_port').value
if modify_port_setting is not True:
return jsonify({'error': 'Incorrect permissions'})
service.port = int(html.escape(request.form['value']))
session.add(service)
session.commit()
update_overview_data()
update_services_data(service.team.id)
update_service_data(service.id)
return jsonify({'status': 'Updated Service Information'})
return jsonify({'error': 'Incorrect permissions'})
def update_host():
if current_user.is_blue_team:
if "name" in request.form and "value" in request.form and "pk" in request.form:
service = session.query(Service).get(int(request.form["pk"]))
if service:
if service.team == current_user.team and request.form[
"name"] == "host":
modify_hostname_setting = Setting.get_setting(
"blue_team_update_hostname").value
if modify_hostname_setting is not True:
return jsonify({"error": "Incorrect permissions"})
service.host = html.escape(request.form["value"])
session.add(service)
session.commit()
update_overview_data()
update_services_data(service.team.id)
update_service_data(service.id)
return jsonify({"status": "Updated Service Information"})
return jsonify({"error": "Incorrect permissions"})
def service_get_checks(service_id):
service = session.query(Service).get(service_id)
if service is None or not (current_user.team == service.team
or current_user.team.is_white_team):
return jsonify({"status": "Unauthorized"}), 403
data = []
check_output = (session.query(Check, Round.number).join(Round).filter(
Check.service_id == service_id).order_by(Check.id.desc()).all())
data = [{
"id": check[0].id,
"round": check[1],
"result": check[0].result,
"timestamp": check[0].local_completed_timestamp,
"reason": check[0].reason,
"output": check[0].output,
"command": check[0].command,
} for check in check_output]
if (Setting.get_setting("blue_team_view_check_output").value is False
and current_user.is_blue_team):
for check in data:
check["output"] = "REDACTED"
return jsonify(data=data)
def test_get_setting(self):
setting_old = Setting(name='test_setting', value='test value example')
self.db.save(setting_old)
setting_new = Setting(name='test_setting', value='updated example')
self.db.save(setting_new)
assert Setting.get_setting('test_setting').value == 'updated example'
def home():
welcome_content = Setting.get_setting('welcome_page_content').value
return render_template('welcome.html', welcome_content=welcome_content)
def about():
about_content = Setting.get_setting('about_page_content').value
return render_template('about.html',
version=version,
about_content=about_content)
def run(self):
if self.total_rounds == 0:
logger.info("Running engine for unlimited rounds")
else:
logger.info("Running engine for {0} round(s)".format(self.total_rounds))
while not self.is_last_round():
self.current_round += 1
logger.info("Running round: " + str(self.current_round))
self.round_running = True
self.rounds_run += 1
services = self.session.query(Service).all()[:]
random.shuffle(services)
task_ids = {}
for service in services:
check_class = self.check_name_to_obj(service.check_name)
if check_class is None:
raise LookupError("Unable to map service to check code for " + str(service.check_name))
logger.debug("Adding " + service.team.name + ' - ' + service.name + " check to queue")
environment = random.choice(service.environments)
check_obj = check_class(environment)
command_str = check_obj.command()
job = Job(environment_id=environment.id, command=command_str)
task = execute_command.apply_async(args=[job], queue=service.worker_queue)
team_name = environment.service.team.name
if team_name not in task_ids:
task_ids[team_name] = []
task_ids[team_name].append(task.id)
# This array keeps track of all current round objects
# incase we need to backout any changes to prevent
# inconsistent check results
cleanup_items = []
try:
# We store the list of tasks in the db, so that the web app
# can consume them and can dynamically update a progress bar
task_ids_str = json.dumps(task_ids)
latest_kb = KB(name='task_ids', value=task_ids_str, round_num=self.current_round)
cleanup_items.append(latest_kb)
self.session.add(latest_kb)
self.session.commit()
pending_tasks = self.all_pending_tasks(task_ids)
while pending_tasks:
worker_refresh_time = int(Setting.get_setting('worker_refresh_time').value)
waiting_info = "Waiting for all jobs to finish (sleeping " + str(worker_refresh_time) + " seconds)"
waiting_info += " " + str(len(pending_tasks)) + " left in queue."
logger.info(waiting_info)
self.sleep(worker_refresh_time)
pending_tasks = self.all_pending_tasks(task_ids)
logger.info("All jobs have finished for this round")
logger.info("Determining check results and saving to db")
round_obj = Round(number=self.current_round)
cleanup_items.append(round_obj)
self.session.add(round_obj)
self.session.commit()
# We keep track of the number of passed and failed checks per round
# so we can report a little bit at the end of each round
teams = {}
# Used so we import the finished checks at the end of the round
finished_checks = []
for team_name, task_ids in task_ids.items():
for task_id in task_ids:
task = execute_command.AsyncResult(task_id)
environment = self.session.query(Environment).get(task.result['environment_id'])
if task.result['errored_out']:
result = False
reason = CHECK_TIMED_OUT_TEXT
else:
if re.search(environment.matching_content, task.result['output']):
result = True
reason = CHECK_SUCCESS_TEXT
else:
result = False
reason = CHECK_FAILURE_TEXT
if environment.service.team.name not in teams:
teams[environment.service.team.name] = {
"Success": [],
"Failed": [],
}
if result:
teams[environment.service.team.name]['Success'].append(environment.service.name)
else:
teams[environment.service.team.name]['Failed'].append(environment.service.name)
check = Check(service=environment.service, round=round_obj)
# Grab the first 35,000 characters of output so it'll fit into our TEXT column,
# which maxes at 2^32 (65536) characters
check.finished(result=result, reason=reason, output=task.result['output'][:35000], command=task.result['command'])
finished_checks.append(check)
for finished_check in finished_checks:
cleanup_items.append(finished_check)
self.session.add(finished_check)
self.session.commit()
except Exception as e:
# We got an error while writing to db (could be normal docker stop command)
# but we gotta clean up any trace of the current round so when we startup
# again, we're at a consistent state
logger.error('Error received while writing check results to db')
logger.exception(e)
logger.error('Ending round and cleaning up the db')
for cleanup_item in cleanup_items:
try:
self.session.delete(cleanup_item)
self.session.commit()
except Exception:
pass
sys.exit(1)
logger.info("Finished Round " + str(self.current_round))
logger.info("Round Stats:")
for team_name in sorted(teams):
stat_string = " " + team_name
stat_string += " Success: " + str(len(teams[team_name]['Success']))
stat_string += ", Failed: " + str(len(teams[team_name]['Failed']))
if len(teams[team_name]['Failed']) > 0:
stat_string += ' (' + ', '.join(teams[team_name]['Failed']) + ')'
logger.info(stat_string)
logger.info("Updating Caches")
update_all_cache()
self.round_running = False
if not self.is_last_round():
round_time_sleep = int(Setting.get_setting('round_time_sleep').value)
logger.info("Sleeping in between rounds (" + str(round_time_sleep) + " seconds)")
self.sleep(round_time_sleep)
logger.info("Engine finished running")
def verify_settings(self):
settings = ["target_round_time", "worker_refresh_time"]
for setting_name in settings:
if not Setting.get_setting(setting_name):
logger.error("Must have " + setting_name + " setting.")
exit(1)
