def v2_runner_on_failed(self, result, ignore_errors=False): # schedule_logger(self._job_id).error(f"Run on failed, result: {result._task._uuid}") # schedule_logger(self._job_id).error(f"Run on failed, details: {result.__dict__}") # stat_logger.info(f'>>>>>>>>>run on ok failed: {json.dumps(result._result, indent=4)}') schedule_logger(self._job_id).error( f"<TASK FAILED> Host {result._host} executes task {result._task._uuid} {result._task._attributes.get('name')} failed. Details: {json.dumps(result._result, indent=4)}" ) update_info = { 'job_id': self._job_id, 'play_id': self._play_id, 'task_id': result._task._uuid, 'end_time': current_timestamp(), 'status': TaskStatus.FAILED, } JobSaver.update_task(update_info) JobSaver.update_task_status(update_info) JobSaver.update_play(update_info) JobSaver.update_play_status(update_info) JobSaver.update_job(update_info) JobSaver.update_job_status(update_info) delegated_vars = result._result.get('_ansible_delegated_vars', None) self._clean_results(result._result, result._task.action) if self._last_task_banner != result._task._uuid: self._print_task_banner(result._task) self._handle_exception(result._result, use_stderr=self.display_failed_stderr) self._handle_warnings(result._result) if result._task.loop and 'results' in result._result: self._process_items(result) else: if delegated_vars: self._display.display( "fatal: [%s -> %s]: FAILED! => %s" % (result._host.get_name(), delegated_vars['ansible_host'], self._dump_results(result._result)), color=C.COLOR_ERROR, stderr=self.display_failed_stderr) else: self._display.display("fatal: [%s]: FAILED! => %s" % (result._host.get_name(), self._dump_results(result._result)), color=C.COLOR_ERROR, stderr=self.display_failed_stderr) if ignore_errors: self._display.display("...ignoring", color=C.COLOR_SKIP)
def v2_playbook_on_no_hosts_matched(self): schedule_logger( self._job_id).info("<No host matched> Skipping: no hosts matched.") update_info = { 'job_id': self._job_id, 'play_id': self._play_id, 'end_time': current_timestamp(), 'status': PlayStatus.FAILED } JobSaver.update_play(update_info) JobSaver.update_play_status(update_info) JobSaver.update_job(update_info) JobSaver.update_job_status(update_info) self._display.display("skipping: no hosts matched", color=C.COLOR_SKIP)
def stop_play(job_id, play_id, status=PlayStatus.CANCELED): plays = JobSaver.query_play(play_id=play_id) if plays: play = plays[0] kill_status = job_utils.kill_play_process_execution(play) if kill_status: if OngoingStatus.contains(play.f_status): play_info = { 'job_id': job_id, 'play_id': play_id, 'end_time': current_timestamp(), 'status': status, } JobSaver.update_play_status(play_info) if not StandbyStatus.contains(play.f_status): JobSaver.update_play(play_info) return True else: return False else: schedule_logger(job_id).info(f"cannot find and kill process of play {play_id}") return False
def v2_runner_on_unreachable(self, result): try: schedule_logger(self._job_id).error( f"<HOST UNREACHABLE> Host {result._host} executes task {result._task._uuid} {result._task._attributes.get('name')} unreachable. \nDetails: {json.dumps(result._result, indent=4)}" ) # schedule_logger(self._job_id).info(f'>>>>>>>>>in runner on unreachable: {json.dumps(result._result, indent=4)}') # stat_logger.info(f'>>>>>>>>>in runner on unreachable: {json.dumps(result._result, indent=4)}') update_info = { 'job_id': self._job_id, 'play_id': self._play_id, 'task_id': result._task._uuid, 'end_time': current_timestamp(), 'status': TaskStatus.FAILED, } JobSaver.update_task(update_info) JobSaver.update_task_status(update_info) JobSaver.update_play(update_info) JobSaver.update_play_status(update_info) except Exception: schedule_logger(self._job_id).warning( f"In v2_runner_on_unreachable, details: {traceback.format_exc()}" ) if self._last_task_banner != result._task._uuid: self._print_task_banner(result._task) delegated_vars = result._result.get('_ansible_delegated_vars', None) if delegated_vars: msg = "fatal: [%s -> %s]: UNREACHABLE! => %s" % ( result._host.get_name(), delegated_vars['ansible_host'], self._dump_results(result._result)) else: msg = "fatal: [%s]: UNREACHABLE! => %s" % ( result._host.get_name(), self._dump_results(result._result)) self._display.display(msg, color=C.COLOR_UNREACHABLE, stderr=self.display_failed_stderr)
def retry_play(job_id, play_id, test_mode=False): plays = JobSaver.query_play(play_id=play_id) if not plays: return 100, f"Retry play {play_id} failed, can not find such play in database." # copy play conf into package dir play_conf_path_dict = file_utils.get_play_conf_path(play_id) with open(file_utils.get_job_conf_path(job_id), 'r') as f: job_conf = json.loads(f.read()) package_dir = get_package_dir_by_version(job_conf.get('version')) play_conf_path_dict['conf_path'] = shutil.copy2( src=play_conf_path_dict['conf_path'], dst=package_dir) update_info = { 'job_id': job_id, 'play_id': play_id, 'status': PlayStatus.WAITING, } JobSaver.update_play_status(update_info) JobSaver.update_job_status(update_info) # clean task records JobSaver.clean_task(play_id=play_id) # execute run_play method try: play_retry_executor_pool.submit( PlayController.run_play, job_id=job_id, play_id=play_id, play_conf_path=play_conf_path_dict['conf_path'], play_hosts_path=play_conf_path_dict['hosts_path'], test_mode=test_mode, retry_mode=True) return 0, f"Start retrying play {play_id}" except Exception as e: stat_logger.exception(e) return 100, f"Retry play {play_id} failed, details: {str(e)}"
def run_job(job_id): job_data = job_utils.get_job_configuration(job_id=job_id) stat_logger.info( f"in play controller run job func, get job data: {json.dumps(job_data, indent=4)}" ) schedule_logger(job_id).info( f"in play controller, func run job: {json.dumps(job_data, indent=4)}" ) play_conf_path_dict = PlayController.initialize_plays( job_id=job_id, job_data=job_data) stat_logger.info( f"in play controller run job func after initialize play\n get play conf path dict: {play_conf_path_dict}" ) # TODO get package dir by version package_dir = get_package_dir_by_version(job_data.get('version')) if not os.path.exists(package_dir) and not os.path.isdir(package_dir): raise Exception( f'Local package directory {package_dir} not exists.') job_info = { 'job_id': job_id, 'status': JobStatus.RUNNING, 'start_time': current_timestamp() } JobSaver.update_job_status(job_info) JobSaver.update_job(job_info) for play_id, conf_dict in play_conf_path_dict.items(): conf_dict['conf_path'] = shutil.copy2(src=conf_dict['conf_path'], dst=package_dir) PlayController.run_play( job_id=job_id, play_id=play_id, play_conf_path=conf_dict.get('conf_path'), play_hosts_path=conf_dict.get('hosts_path'), test_mode=TEST_MODE) if os.path.exists(conf_dict['conf_path']): os.remove(conf_dict['conf_path']) plays = JobSaver.query_play(play_id=play_id) if plays: play = plays[0] status = play.f_status if status != PlayStatus.SUCCESS: if status in [ PlayStatus.CANCELED, PlayStatus.FAILED, PlayStatus.TIMEOUT ]: update_info = { 'job_id': job_id, 'play_id': play_id, 'status': status, 'end_time': current_timestamp() } JobSaver.update_play_status(update_info) JobSaver.update_play(update_info) JobSaver.update_job_status(update_info) JobSaver.update_job(update_info) else: update_info = { 'job_id': job_id, 'play_id': play_id, 'status': PlayStatus.FAILED, 'end_time': current_timestamp() } schedule_logger(job_id).error( f'Unexpected error occured on play {play_id}, job {job_id} failed, previous status of play: {play.f_status}' ) stat_logger.error( f'Unexpected error occured on play {play_id}, job {job_id} failed, previous status of play: {play.f_status}' ) JobSaver.update_play_status(update_info) JobSaver.update_play(update_info) JobSaver.update_job_status(update_info) JobSaver.update_job(update_info) schedule_logger(job_id).info( f"job {job_id} finished, status is {update_info.get('status')}" ) break else: update_info = { 'job_id': job_id, 'play_id': play_id, 'status': PlayStatus.SUCCESS, 'end_time': current_timestamp() } JobSaver.update_play_status(update_info) JobSaver.update_play(update_info) else: raise Exception(f'can not find play {play_id}') else: update_info = { 'job_id': job_id, 'status': JobStatus.SUCCESS, 'end_time': current_timestamp() } JobSaver.update_job(update_info) JobSaver.update_job_status(update_info) schedule_logger(job_id).info( f"job {job_id} finished, status is {update_info.get('status')}" ) if not TEST_MODE: plays = JobSaver.query_play(job_id=job_id, status=PlayStatus.SUCCESS) modules = [] module_names = [] for play in plays: module_name = play.f_roles.strip('[]').replace('_', '') module_names.append(module_name) modules.append({ 'name': module_name, 'ips': job_data.get('modules', {}).get(module_name, {}).get('ips', []), 'port': job_data.get('modules', {}).get(module_name, {}).get('port', None) }) # parties = PartyInfo.get_or_none(f_version=job_data.get('version'), f_party_id=job_data.get('party_id')) parties = PartyInfo.get_or_none( f_party_id=job_data.get('party_id')) if parties: module_mapping = dict(zip(module_names, modules)) stored_modules = parties.f_modules.get("data", []) name_map = {} for offset, item in enumerate(stored_modules): name_map[item.get('name')] = offset for key, value in module_mapping.items(): if key in name_map: schedule_logger(job_id).info( f"{key} in name map, in replace process") stored_modules[name_map[key]] = value else: schedule_logger(job_id).info( f"{key} not in name map, in append process ") stored_modules.append(value) # update_status = False # for offset, module_info in enumerate(stored_modules): # if module_info['name'] in module_mapping: # stored_modules[offset] = module_mapping[module_info['name']] # update_status = True for key in ['role', 'version']: # if parties[key] != job_data[key]: # parties[key] = job_data[key] if getattr(parties, f'f_{key}') != job_data[key]: setattr(parties, f'f_{key}', job_data[key]) # update_status = True # if update_status: parties.f_modules = {'data': stored_modules} parties.save() DB.commit() else: party_info = PartyInfo() # party_info.f_job_id = job_id party_info.f_role = job_data.get('role') party_info.f_version = job_data.get('version') party_info.f_party_id = job_data.get('party_id') party_info.f_modules = {'data': modules} party_info.save(force_insert=True)
def run_play(job_id, play_id, play_conf_path, play_hosts_path, test_mode=False, retry_mode=False): schedule_logger(job_id).info( f'Trying to start to run play with id: {play_id}') # task_process_start_status = False process_cmd = [ 'python3', sys.modules[PlayExecutor.__module__].__file__, '--job_id', job_id, '--play_id', play_id, '--conf_path', play_conf_path, '--hosts_path', play_hosts_path, ] if test_mode: process_cmd.append('--test') if retry_mode: process_cmd.append('--retry') schedule_logger(job_id).info( f"Trying to start job {job_id}, play {play_id} subprocess.") try: config_dir = file_utils.get_play_directory(play_id) std_dir = file_utils.get_job_log_directory(job_id) p = job_utils.run_subprocess(config_dir=config_dir, process_cmd=process_cmd, log_dir=std_dir) if p: # task_process_start_status = True play_info = { 'pid': p.pid, 'job_id': job_id, 'play_id': play_id, 'status': PlayStatus.RUNNING, 'start_time': current_timestamp() } JobSaver.update_play_status(play_info=play_info) JobSaver.update_play(play_info=play_info) p.wait() else: raise Exception(f'play {play_id} start subprocess failed') except Exception as e: play_info = { 'job_id': job_id, 'play_id': play_id, 'status': PlayStatus.FAILED, 'end_time': current_timestamp() } JobSaver.update_play_status(play_info) JobSaver.update_play(play_info) schedule_logger(job_id).exception(e) finally: if retry_mode: job_info = { 'job_id': job_id, 'play_id': play_id, 'end_time': current_timestamp(), 'status': JobStatus.SUCCESS if PlayController.check_job_status( job_id=job_id) else JobStatus.FAILED } JobSaver.update_job(job_info) JobSaver.update_job_status(job_info)
def v2_playbook_on_stats(self, stats): self._display.banner("PLAY RECAP") hosts = sorted(stats.processed.keys()) failed_count = 0 for h in hosts: t = stats.summarize(h) # stat_logger.info(f'>>>>>>>>PLAY RECAP, {t}') schedule_logger(self._job_id).info( f"in playbook on stats: RECAP: {t}, host: {h}") failed_count += t['failures'] failed_count += t['unreachable'] # self._display.display( # u"%s : %s %s %s %s %s %s %s" % ( # hostcolor(h, t), # colorize(u'ok', t['ok'], C.COLOR_OK), # colorize(u'changed', t['changed'], C.COLOR_CHANGED), # colorize(u'unreachable', t['unreachable'], C.COLOR_UNREACHABLE), # colorize(u'failed', t['failures'], C.COLOR_ERROR), # colorize(u'skipped', t['skipped'], C.COLOR_SKIP), # colorize(u'rescued', t['rescued'], C.COLOR_OK), # colorize(u'ignored', t['ignored'], C.COLOR_WARN), # ), # screen_only=True # ) self._display.display(u"%s : %s %s %s %s %s %s %s" % ( hostcolor(h, t, False), colorize(u'ok', t['ok'], None), colorize(u'changed', t['changed'], None), colorize(u'unreachable', t['unreachable'], None), colorize(u'failed', t['failures'], None), colorize(u'skipped', t['skipped'], None), colorize(u'rescued', t['rescued'], None), colorize(u'ignored', t['ignored'], None), ), log_only=True) if not failed_count: update_info = { 'job_id': self._job_id, 'play_id': self._play_id, 'end_time': current_timestamp(), 'status': PlayStatus.SUCCESS } JobSaver.update_play(update_info) JobSaver.update_play_status(update_info) self._display.display("", screen_only=True) # print custom stats if required if stats.custom and self.show_custom_stats: self._display.banner("CUSTOM STATS: ") # per host # TODO: come up with 'pretty format' for k in sorted(stats.custom.keys()): if k == '_run': continue self._display.display( '\t%s: %s' % (k, self._dump_results(stats.custom[k], indent=1).replace( '\n', ''))) # print per run custom stats if '_run' in stats.custom: self._display.display("", screen_only=True) self._display.display('\tRUN: %s' % self._dump_results( stats.custom['_run'], indent=1).replace('\n', '')) self._display.display("", screen_only=True) if context.CLIARGS['check'] and self.check_mode_markers: self._display.banner("DRY RUN")