def check_job_status(job_id: str):
plays = JobSaver.query_play(job_id=job_id)
job_success = True
if plays:
for play in plays:
job_success = job_success and play.f_status == PlayStatus.SUCCESS
else:
job_success = False
return job_success
def query_play():
plays = JobSaver.query_play(**request.json)
if not plays:
return get_json_result(retcode=101,
retmsg='Query play failed, no play found.')
play_filters = [
'job_id', 'play_id', 'create_time', 'start_time', 'end_time', 'status',
'elapsed'
]
data = plays[0].to_json(filters=play_filters)
tasks = JobSaver.query_task(reverse=False, **request.json)
if tasks:
task_filters = [
'play_id', 'task_id', 'task_name', 'role', 'create_time',
'start_time', 'end_time', 'status', 'elapsed'
]
data['f_tasks'] = [task.to_json(task_filters) for task in tasks]
return get_json_result(retmsg="Query play successfully.", data=data)
def query_job():
jobs = JobSaver.query_job(**request.json)
if not jobs:
return get_json_result(retcode=101,
retmsg='Query job failed, no job found.')
data = jobs[0].to_json(filters=[
'job_id', 'create_time', 'start_time', 'end_time', 'status', 'elapsed'
])
plays = JobSaver.query_play(reverse=False,
order_by='play_id',
**request.json)
if plays:
play_filters = [
'job_id', 'play_id', 'create_time', 'start_time', 'end_time',
'status', 'elapsed'
]
data['f_plays'] = [
play.to_json(filters=play_filters) for play in plays
]
return get_json_result(retmsg="Query job successfully.", data=data)
def stop_job(job_id, status=JobStatus.CANCELED):
jobs = JobSaver.query_job(job_id=job_id)
if jobs:
plays = JobSaver.query_play(job_id=job_id)
stop_result = {}
final_status = True
for play in [item for item in plays if not EndStatus.contains(item.f_status)]:
stop_status = JobController.stop_play(job_id=job_id, play_id=play.f_play_id, status=status)
stop_result[play.f_play_id] = 'stopped successfully' if stop_status else 'stopped failed'
final_status = final_status & stop_status
if final_status:
update_info = {
'job_id': job_id,
'end_time': current_timestamp(),
'status': JobStatus.CANCELED
}
JobSaver.update_job(update_info)
JobSaver.update_job_status(update_info)
return final_status, stop_result
else:
return False, {job_id: f"Cannot found job {job_id}"}
def stop_play(job_id, play_id, status=PlayStatus.CANCELED):
plays = JobSaver.query_play(play_id=play_id)
if plays:
play = plays[0]
kill_status = job_utils.kill_play_process_execution(play)
if kill_status:
if OngoingStatus.contains(play.f_status):
play_info = {
'job_id': job_id,
'play_id': play_id,
'end_time': current_timestamp(),
'status': status,
}
JobSaver.update_play_status(play_info)
if not StandbyStatus.contains(play.f_status):
JobSaver.update_play(play_info)
return True
else:
return False
else:
schedule_logger(job_id).info(f"cannot find and kill process of play {play_id}")
return False
def retry_play(job_id, play_id, test_mode=False):
plays = JobSaver.query_play(play_id=play_id)
if not plays:
return 100, f"Retry play {play_id} failed, can not find such play in database."
# copy play conf into package dir
play_conf_path_dict = file_utils.get_play_conf_path(play_id)
with open(file_utils.get_job_conf_path(job_id), 'r') as f:
job_conf = json.loads(f.read())
package_dir = get_package_dir_by_version(job_conf.get('version'))
play_conf_path_dict['conf_path'] = shutil.copy2(
src=play_conf_path_dict['conf_path'], dst=package_dir)
update_info = {
'job_id': job_id,
'play_id': play_id,
'status': PlayStatus.WAITING,
}
JobSaver.update_play_status(update_info)
JobSaver.update_job_status(update_info)
# clean task records
JobSaver.clean_task(play_id=play_id)
# execute run_play method
try:
play_retry_executor_pool.submit(
PlayController.run_play,
job_id=job_id,
play_id=play_id,
play_conf_path=play_conf_path_dict['conf_path'],
play_hosts_path=play_conf_path_dict['hosts_path'],
test_mode=test_mode,
retry_mode=True)
return 0, f"Start retrying play {play_id}"
except Exception as e:
stat_logger.exception(e)
return 100, f"Retry play {play_id} failed, details: {str(e)}"
def run_do(self):
try:
running_plays = JobSaver.query_play(status='running')
stop_job_ids = set()
for play in running_plays:
try:
process_exist = job_utils.check_job_process(int(
play.f_pid))
if not process_exist:
detect_logger.info(
'job {} play {} process does not exist'.format(
play.f_job_id, play.f_pid))
stop_job_ids.add(play.f_job_id)
detect_logger.info(
f'start to stop play {play.f_play_id}')
JobController.stop_play(job_id=play.f_job_id,
play_id=play.f_play_id)
except Exception as e:
detect_logger.exception(e)
# ready_plays = JobSaver.query_play(status='ready')
# for play in ready_plays:
# try:
if stop_job_ids:
schedule_logger().info(
'start to stop jobs: {}'.format(stop_job_ids))
for job_id in stop_job_ids:
jobs = JobSaver.query_job(job_id=job_id)
if jobs:
if not EndStatus.contains(jobs[0].f_status):
JobController.stop_job(job_id=job_id)
except Exception as e:
detect_logger.exception(e)
finally:
detect_logger.info('finish detect running job')
def run_job(job_id):
job_data = job_utils.get_job_configuration(job_id=job_id)
stat_logger.info(
f"in play controller run job func, get job data: {json.dumps(job_data, indent=4)}"
)
schedule_logger(job_id).info(
f"in play controller, func run job: {json.dumps(job_data, indent=4)}"
)
play_conf_path_dict = PlayController.initialize_plays(
job_id=job_id, job_data=job_data)
stat_logger.info(
f"in play controller run job func after initialize play\n get play conf path dict: {play_conf_path_dict}"
)
# TODO get package dir by version
package_dir = get_package_dir_by_version(job_data.get('version'))
if not os.path.exists(package_dir) and not os.path.isdir(package_dir):
raise Exception(
f'Local package directory {package_dir} not exists.')
job_info = {
'job_id': job_id,
'status': JobStatus.RUNNING,
'start_time': current_timestamp()
}
JobSaver.update_job_status(job_info)
JobSaver.update_job(job_info)
for play_id, conf_dict in play_conf_path_dict.items():
conf_dict['conf_path'] = shutil.copy2(src=conf_dict['conf_path'],
dst=package_dir)
PlayController.run_play(
job_id=job_id,
play_id=play_id,
play_conf_path=conf_dict.get('conf_path'),
play_hosts_path=conf_dict.get('hosts_path'),
test_mode=TEST_MODE)
if os.path.exists(conf_dict['conf_path']):
os.remove(conf_dict['conf_path'])
plays = JobSaver.query_play(play_id=play_id)
if plays:
play = plays[0]
status = play.f_status
if status != PlayStatus.SUCCESS:
if status in [
PlayStatus.CANCELED, PlayStatus.FAILED,
PlayStatus.TIMEOUT
]:
update_info = {
'job_id': job_id,
'play_id': play_id,
'status': status,
'end_time': current_timestamp()
}
JobSaver.update_play_status(update_info)
JobSaver.update_play(update_info)
JobSaver.update_job_status(update_info)
JobSaver.update_job(update_info)
else:
update_info = {
'job_id': job_id,
'play_id': play_id,
'status': PlayStatus.FAILED,
'end_time': current_timestamp()
}
schedule_logger(job_id).error(
f'Unexpected error occured on play {play_id}, job {job_id} failed, previous status of play: {play.f_status}'
)
stat_logger.error(
f'Unexpected error occured on play {play_id}, job {job_id} failed, previous status of play: {play.f_status}'
)
JobSaver.update_play_status(update_info)
JobSaver.update_play(update_info)
JobSaver.update_job_status(update_info)
JobSaver.update_job(update_info)
schedule_logger(job_id).info(
f"job {job_id} finished, status is {update_info.get('status')}"
)
break
else:
update_info = {
'job_id': job_id,
'play_id': play_id,
'status': PlayStatus.SUCCESS,
'end_time': current_timestamp()
}
JobSaver.update_play_status(update_info)
JobSaver.update_play(update_info)
else:
raise Exception(f'can not find play {play_id}')
else:
update_info = {
'job_id': job_id,
'status': JobStatus.SUCCESS,
'end_time': current_timestamp()
}
JobSaver.update_job(update_info)
JobSaver.update_job_status(update_info)
schedule_logger(job_id).info(
f"job {job_id} finished, status is {update_info.get('status')}"
)
if not TEST_MODE:
plays = JobSaver.query_play(job_id=job_id,
status=PlayStatus.SUCCESS)
modules = []
module_names = []
for play in plays:
module_name = play.f_roles.strip('[]').replace('_', '')
module_names.append(module_name)
modules.append({
'name':
module_name,
'ips':
job_data.get('modules', {}).get(module_name,
{}).get('ips', []),
'port':
job_data.get('modules', {}).get(module_name,
{}).get('port', None)
})
# parties = PartyInfo.get_or_none(f_version=job_data.get('version'), f_party_id=job_data.get('party_id'))
parties = PartyInfo.get_or_none(
f_party_id=job_data.get('party_id'))
if parties:
module_mapping = dict(zip(module_names, modules))
stored_modules = parties.f_modules.get("data", [])
name_map = {}
for offset, item in enumerate(stored_modules):
name_map[item.get('name')] = offset
for key, value in module_mapping.items():
if key in name_map:
schedule_logger(job_id).info(
f"{key} in name map, in replace process")
stored_modules[name_map[key]] = value
else:
schedule_logger(job_id).info(
f"{key} not in name map, in append process ")
stored_modules.append(value)
# update_status = False
# for offset, module_info in enumerate(stored_modules):
# if module_info['name'] in module_mapping:
# stored_modules[offset] = module_mapping[module_info['name']]
# update_status = True
for key in ['role', 'version']:
# if parties[key] != job_data[key]:
# parties[key] = job_data[key]
if getattr(parties, f'f_{key}') != job_data[key]:
setattr(parties, f'f_{key}', job_data[key])
# update_status = True
# if update_status:
parties.f_modules = {'data': stored_modules}
parties.save()
DB.commit()
else:
party_info = PartyInfo()
# party_info.f_job_id = job_id
party_info.f_role = job_data.get('role')
party_info.f_version = job_data.get('version')
party_info.f_party_id = job_data.get('party_id')
party_info.f_modules = {'data': modules}
party_info.save(force_insert=True)