def all_pending_tasks(self, tasks):
pending_tasks = []
for team_name, task_ids in tasks.items():
for task_id in task_ids:
task = execute_command.AsyncResult(task_id)
if task.state == 'PENDING':
pending_tasks.append(task_id)
return pending_tasks
def get_check_progress_total():
if current_user.is_white_team:
task_id_settings = KB.query.filter_by(name='task_ids').order_by(
KB.round_num.desc()).first()
total_stats = {}
total_stats['finished'] = 0
total_stats['pending'] = 0
team_stats = {}
if task_id_settings:
task_dict = json.loads(task_id_settings.value)
for team_name, task_ids in task_dict.items():
for task_id in task_ids:
task = execute_command.AsyncResult(task_id)
if team_name not in team_stats:
team_stats[team_name] = {}
team_stats[team_name]['pending'] = 0
team_stats[team_name]['finished'] = 0
if task.state == 'PENDING':
team_stats[team_name]['pending'] += 1
total_stats['pending'] += 1
else:
team_stats[team_name]['finished'] += 1
total_stats['finished'] += 1
total_percentage = 0
total_tasks = total_stats['finished'] + total_stats['pending']
if total_stats['finished'] == 0:
total_percentage = 0
elif total_tasks == 0:
total_percentage = 100
elif total_stats and total_stats['finished']:
total_percentage = int(
(total_stats['finished'] / total_tasks) * 100)
output_dict = {'Total': total_percentage}
for team_name, team_stat in team_stats.items():
team_total_percentage = 0
team_total_tasks = team_stat['finished'] + team_stat['pending']
if team_stat['finished'] == 0:
team_total_percentage = 0
elif team_total_tasks == 0:
team_total_percentage = 100
elif team_stat and team_stat['finished']:
team_total_percentage = int(
(team_stat['finished'] / team_total_tasks) * 100)
output_dict[team_name] = team_total_percentage
return json.dumps(output_dict)
else:
return {'status': 'Unauthorized'}, 403
def get_check_progress_total():
if current_user.is_white_team:
task_id_settings = (session.query(KB).filter_by(
name="task_ids").order_by(KB.round_num.desc()).first())
total_stats = {}
total_stats["finished"] = 0
total_stats["pending"] = 0
team_stats = {}
if task_id_settings:
task_dict = json.loads(task_id_settings.value)
for team_name, task_ids in task_dict.items():
for task_id in task_ids:
task = execute_command.AsyncResult(task_id)
if team_name not in team_stats:
team_stats[team_name] = {}
team_stats[team_name]["pending"] = 0
team_stats[team_name]["finished"] = 0
if task.state == "PENDING":
team_stats[team_name]["pending"] += 1
total_stats["pending"] += 1
else:
team_stats[team_name]["finished"] += 1
total_stats["finished"] += 1
total_percentage = 0
total_tasks = total_stats["finished"] + total_stats["pending"]
if total_stats["finished"] == 0:
total_percentage = 0
elif total_tasks == 0:
total_percentage = 100
elif total_stats and total_stats["finished"]:
total_percentage = int(
(total_stats["finished"] / total_tasks) * 100)
output_dict = {"Total": total_percentage}
for team_name, team_stat in team_stats.items():
team_total_percentage = 0
team_total_tasks = team_stat["finished"] + team_stat["pending"]
if team_stat["finished"] == 0:
team_total_percentage = 0
elif team_total_tasks == 0:
team_total_percentage = 100
elif team_stat and team_stat["finished"]:
team_total_percentage = int(
(team_stat["finished"] / team_total_tasks) * 100)
output_dict[team_name] = team_total_percentage
return json.dumps(output_dict)
else:
return {"status": "Unauthorized"}, 403
def run(self):
while (not self.last_round) and (self.rounds_run < self.total_rounds
or self.total_rounds == 0):
self.current_round += 1
logger.info("Running round: " + str(self.current_round))
self.round_running = True
self.rounds_run += 1
services = self.db.session.query(Service).all()[:]
random.shuffle(services)
task_ids = {}
for service in services:
check_class = self.check_name_to_obj(service.check_name)
if check_class is None:
raise LookupError(
"Unable to map service to check code for " +
str(service.check_name))
logger.info("Adding " + service.team.name + ' - ' +
service.name + " check to queue")
environment = random.choice(service.environments)
check_obj = check_class(environment)
command_str = check_obj.command()
job = Job(environment_id=environment.id, command=command_str)
task = execute_command.delay(job)
team_name = environment.service.team.name
if team_name not in task_ids:
task_ids[team_name] = []
task_ids[team_name].append(task.id)
# We store the list of tasks in the db, so that the web app
# can consume them and can dynamically update a progress bar
task_ids_str = json.dumps(task_ids)
latest_kb = KB(name='task_ids',
value=task_ids_str,
round_num=self.current_round)
self.db.save(latest_kb)
pending_tasks = self.all_pending_tasks(task_ids)
while pending_tasks:
waiting_info = "Waiting for all jobs to finish (sleeping " + str(
self.worker_wait_time) + " seconds)"
waiting_info += " " + str(
len(pending_tasks)) + " left in queue."
logger.info(waiting_info)
self.sleep(self.worker_wait_time)
pending_tasks = self.all_pending_tasks(task_ids)
logger.info("All jobs have finished for this round")
logger.info("Determining check results and saving to db")
round_obj = Round(number=self.current_round)
self.db.save(round_obj)
# We keep track of the number of passed and failed checks per round
# so we can report a little bit at the end of each round
teams = {}
for team_name, task_ids in task_ids.items():
for task_id in task_ids:
task = execute_command.AsyncResult(task_id)
environment = self.db.session.query(Environment).get(
task.result['environment_id'])
if task.result['errored_out']:
result = False
reason = 'Task Timed Out'
else:
if re.search(environment.matching_regex,
task.result['output']):
result = True
reason = "Successful Content Match"
else:
result = False
reason = 'Unsuccessful Content Match'
if environment.service.team.name not in teams:
teams[environment.service.team.name] = {
"Success": [],
"Failed": [],
}
if result:
teams[environment.service.team.name]['Success'].append(
environment.service.name)
else:
teams[environment.service.team.name]['Failed'].append(
environment.service.name)
check = Check(service=environment.service, round=round_obj)
check.finished(result=result,
reason=reason,
output=task.result['output'],
command=task.result['command'])
self.db.save(check)
logger.info("Finished Round " + str(self.current_round))
logger.info("Round Stats:")
for team_name in sorted(teams):
stat_string = " " + team_name
stat_string += " Success: " + str(
len(teams[team_name]['Success']))
stat_string += ", Failed: " + str(
len(teams[team_name]['Failed']))
if len(teams[team_name]['Failed']) > 0:
stat_string += ' ' + str(teams[team_name]['Failed'])
logger.info(stat_string)
self.round_running = False
if not self.last_round:
logger.info("Sleeping in between rounds (" +
str(self.round_time_sleep) + " seconds)")
self.sleep(self.round_time_sleep)
def run(self):
if self.total_rounds == 0:
logger.info("Running engine for unlimited rounds")
else:
logger.info("Running engine for {0} round(s)".format(self.total_rounds))
while not self.is_last_round():
self.current_round += 1
logger.info("Running round: " + str(self.current_round))
self.round_running = True
self.rounds_run += 1
services = self.session.query(Service).all()[:]
random.shuffle(services)
task_ids = {}
for service in services:
check_class = self.check_name_to_obj(service.check_name)
if check_class is None:
raise LookupError("Unable to map service to check code for " + str(service.check_name))
logger.debug("Adding " + service.team.name + ' - ' + service.name + " check to queue")
environment = random.choice(service.environments)
check_obj = check_class(environment)
command_str = check_obj.command()
job = Job(environment_id=environment.id, command=command_str)
task = execute_command.apply_async(args=[job], queue=service.worker_queue)
team_name = environment.service.team.name
if team_name not in task_ids:
task_ids[team_name] = []
task_ids[team_name].append(task.id)
# This array keeps track of all current round objects
# incase we need to backout any changes to prevent
# inconsistent check results
cleanup_items = []
try:
# We store the list of tasks in the db, so that the web app
# can consume them and can dynamically update a progress bar
task_ids_str = json.dumps(task_ids)
latest_kb = KB(name='task_ids', value=task_ids_str, round_num=self.current_round)
cleanup_items.append(latest_kb)
self.session.add(latest_kb)
self.session.commit()
pending_tasks = self.all_pending_tasks(task_ids)
while pending_tasks:
worker_refresh_time = int(Setting.get_setting('worker_refresh_time').value)
waiting_info = "Waiting for all jobs to finish (sleeping " + str(worker_refresh_time) + " seconds)"
waiting_info += " " + str(len(pending_tasks)) + " left in queue."
logger.info(waiting_info)
self.sleep(worker_refresh_time)
pending_tasks = self.all_pending_tasks(task_ids)
logger.info("All jobs have finished for this round")
logger.info("Determining check results and saving to db")
round_obj = Round(number=self.current_round)
cleanup_items.append(round_obj)
self.session.add(round_obj)
self.session.commit()
# We keep track of the number of passed and failed checks per round
# so we can report a little bit at the end of each round
teams = {}
# Used so we import the finished checks at the end of the round
finished_checks = []
for team_name, task_ids in task_ids.items():
for task_id in task_ids:
task = execute_command.AsyncResult(task_id)
environment = self.session.query(Environment).get(task.result['environment_id'])
if task.result['errored_out']:
result = False
reason = CHECK_TIMED_OUT_TEXT
else:
if re.search(environment.matching_content, task.result['output']):
result = True
reason = CHECK_SUCCESS_TEXT
else:
result = False
reason = CHECK_FAILURE_TEXT
if environment.service.team.name not in teams:
teams[environment.service.team.name] = {
"Success": [],
"Failed": [],
}
if result:
teams[environment.service.team.name]['Success'].append(environment.service.name)
else:
teams[environment.service.team.name]['Failed'].append(environment.service.name)
check = Check(service=environment.service, round=round_obj)
# Grab the first 35,000 characters of output so it'll fit into our TEXT column,
# which maxes at 2^32 (65536) characters
check.finished(result=result, reason=reason, output=task.result['output'][:35000], command=task.result['command'])
finished_checks.append(check)
for finished_check in finished_checks:
cleanup_items.append(finished_check)
self.session.add(finished_check)
self.session.commit()
except Exception as e:
# We got an error while writing to db (could be normal docker stop command)
# but we gotta clean up any trace of the current round so when we startup
# again, we're at a consistent state
logger.error('Error received while writing check results to db')
logger.exception(e)
logger.error('Ending round and cleaning up the db')
for cleanup_item in cleanup_items:
try:
self.session.delete(cleanup_item)
self.session.commit()
except Exception:
pass
sys.exit(1)
logger.info("Finished Round " + str(self.current_round))
logger.info("Round Stats:")
for team_name in sorted(teams):
stat_string = " " + team_name
stat_string += " Success: " + str(len(teams[team_name]['Success']))
stat_string += ", Failed: " + str(len(teams[team_name]['Failed']))
if len(teams[team_name]['Failed']) > 0:
stat_string += ' (' + ', '.join(teams[team_name]['Failed']) + ')'
logger.info(stat_string)
logger.info("Updating Caches")
update_all_cache()
self.round_running = False
if not self.is_last_round():
round_time_sleep = int(Setting.get_setting('round_time_sleep').value)
logger.info("Sleeping in between rounds (" + str(round_time_sleep) + " seconds)")
self.sleep(round_time_sleep)
logger.info("Engine finished running")
