コード例 #1
0
ファイル: callback.py プロジェクト: cold-code/FATE-Cloud
    def v2_runner_on_failed(self, result, ignore_errors=False):
        # schedule_logger(self._job_id).error(f"Run on failed, result: {result._task._uuid}")
        # schedule_logger(self._job_id).error(f"Run on failed, details: {result.__dict__}")
        # stat_logger.info(f'>>>>>>>>>run on ok failed: {json.dumps(result._result, indent=4)}')
        schedule_logger(self._job_id).error(
            f"<TASK FAILED> Host {result._host} executes task {result._task._uuid} {result._task._attributes.get('name')} failed. Details: {json.dumps(result._result, indent=4)}"
        )
        update_info = {
            'job_id': self._job_id,
            'play_id': self._play_id,
            'task_id': result._task._uuid,
            'end_time': current_timestamp(),
            'status': TaskStatus.FAILED,
        }
        JobSaver.update_task(update_info)
        JobSaver.update_task_status(update_info)
        JobSaver.update_play(update_info)
        JobSaver.update_play_status(update_info)
        JobSaver.update_job(update_info)
        JobSaver.update_job_status(update_info)

        delegated_vars = result._result.get('_ansible_delegated_vars', None)
        self._clean_results(result._result, result._task.action)

        if self._last_task_banner != result._task._uuid:
            self._print_task_banner(result._task)

        self._handle_exception(result._result,
                               use_stderr=self.display_failed_stderr)
        self._handle_warnings(result._result)

        if result._task.loop and 'results' in result._result:
            self._process_items(result)

        else:
            if delegated_vars:
                self._display.display(
                    "fatal: [%s -> %s]: FAILED! => %s" %
                    (result._host.get_name(), delegated_vars['ansible_host'],
                     self._dump_results(result._result)),
                    color=C.COLOR_ERROR,
                    stderr=self.display_failed_stderr)
            else:
                self._display.display("fatal: [%s]: FAILED! => %s" %
                                      (result._host.get_name(),
                                       self._dump_results(result._result)),
                                      color=C.COLOR_ERROR,
                                      stderr=self.display_failed_stderr)

        if ignore_errors:
            self._display.display("...ignoring", color=C.COLOR_SKIP)
コード例 #2
0
ファイル: callback.py プロジェクト: cold-code/FATE-Cloud
 def v2_playbook_on_no_hosts_matched(self):
     schedule_logger(
         self._job_id).info("<No host matched> Skipping: no hosts matched.")
     update_info = {
         'job_id': self._job_id,
         'play_id': self._play_id,
         'end_time': current_timestamp(),
         'status': PlayStatus.FAILED
     }
     JobSaver.update_play(update_info)
     JobSaver.update_play_status(update_info)
     JobSaver.update_job(update_info)
     JobSaver.update_job_status(update_info)
     self._display.display("skipping: no hosts matched", color=C.COLOR_SKIP)
コード例 #3
0
 def stop_play(job_id, play_id, status=PlayStatus.CANCELED):
     plays = JobSaver.query_play(play_id=play_id)
     if plays:
         play = plays[0]
         kill_status = job_utils.kill_play_process_execution(play)
         if kill_status:
             if OngoingStatus.contains(play.f_status):
                 play_info = {
                     'job_id': job_id,
                     'play_id': play_id,
                     'end_time': current_timestamp(),
                     'status': status,
                 }
                 JobSaver.update_play_status(play_info)
                 if not StandbyStatus.contains(play.f_status):
                     JobSaver.update_play(play_info)
             return True
         else:
             return False
     else:
         schedule_logger(job_id).info(f"cannot find and kill process of play {play_id}")
         return False
コード例 #4
0
ファイル: callback.py プロジェクト: cold-code/FATE-Cloud
    def v2_runner_on_unreachable(self, result):
        try:
            schedule_logger(self._job_id).error(
                f"<HOST UNREACHABLE> Host {result._host} executes task {result._task._uuid} {result._task._attributes.get('name')} unreachable. \nDetails: {json.dumps(result._result, indent=4)}"
            )
            # schedule_logger(self._job_id).info(f'>>>>>>>>>in runner on unreachable: {json.dumps(result._result, indent=4)}')
            # stat_logger.info(f'>>>>>>>>>in runner on unreachable: {json.dumps(result._result, indent=4)}')

            update_info = {
                'job_id': self._job_id,
                'play_id': self._play_id,
                'task_id': result._task._uuid,
                'end_time': current_timestamp(),
                'status': TaskStatus.FAILED,
            }
            JobSaver.update_task(update_info)
            JobSaver.update_task_status(update_info)
            JobSaver.update_play(update_info)
            JobSaver.update_play_status(update_info)
        except Exception:
            schedule_logger(self._job_id).warning(
                f"In v2_runner_on_unreachable, details: {traceback.format_exc()}"
            )

        if self._last_task_banner != result._task._uuid:
            self._print_task_banner(result._task)

        delegated_vars = result._result.get('_ansible_delegated_vars', None)
        if delegated_vars:
            msg = "fatal: [%s -> %s]: UNREACHABLE! => %s" % (
                result._host.get_name(), delegated_vars['ansible_host'],
                self._dump_results(result._result))
        else:
            msg = "fatal: [%s]: UNREACHABLE! => %s" % (
                result._host.get_name(), self._dump_results(result._result))
        self._display.display(msg,
                              color=C.COLOR_UNREACHABLE,
                              stderr=self.display_failed_stderr)
コード例 #5
0
    def retry_play(job_id, play_id, test_mode=False):
        plays = JobSaver.query_play(play_id=play_id)
        if not plays:
            return 100, f"Retry play {play_id} failed, can not find such play in database."

        # copy play conf into package dir
        play_conf_path_dict = file_utils.get_play_conf_path(play_id)
        with open(file_utils.get_job_conf_path(job_id), 'r') as f:
            job_conf = json.loads(f.read())
        package_dir = get_package_dir_by_version(job_conf.get('version'))
        play_conf_path_dict['conf_path'] = shutil.copy2(
            src=play_conf_path_dict['conf_path'], dst=package_dir)

        update_info = {
            'job_id': job_id,
            'play_id': play_id,
            'status': PlayStatus.WAITING,
        }
        JobSaver.update_play_status(update_info)
        JobSaver.update_job_status(update_info)

        # clean task records
        JobSaver.clean_task(play_id=play_id)
        # execute run_play method
        try:
            play_retry_executor_pool.submit(
                PlayController.run_play,
                job_id=job_id,
                play_id=play_id,
                play_conf_path=play_conf_path_dict['conf_path'],
                play_hosts_path=play_conf_path_dict['hosts_path'],
                test_mode=test_mode,
                retry_mode=True)
            return 0, f"Start retrying play {play_id}"
        except Exception as e:
            stat_logger.exception(e)
            return 100, f"Retry play {play_id} failed, details: {str(e)}"
コード例 #6
0
    def run_job(job_id):
        job_data = job_utils.get_job_configuration(job_id=job_id)
        stat_logger.info(
            f"in play controller run job func, get job data: {json.dumps(job_data, indent=4)}"
        )
        schedule_logger(job_id).info(
            f"in play controller, func run job: {json.dumps(job_data, indent=4)}"
        )

        play_conf_path_dict = PlayController.initialize_plays(
            job_id=job_id, job_data=job_data)
        stat_logger.info(
            f"in play controller run job func after initialize play\n get play conf path dict: {play_conf_path_dict}"
        )

        # TODO get package dir by version
        package_dir = get_package_dir_by_version(job_data.get('version'))
        if not os.path.exists(package_dir) and not os.path.isdir(package_dir):
            raise Exception(
                f'Local package directory {package_dir} not exists.')

        job_info = {
            'job_id': job_id,
            'status': JobStatus.RUNNING,
            'start_time': current_timestamp()
        }
        JobSaver.update_job_status(job_info)
        JobSaver.update_job(job_info)

        for play_id, conf_dict in play_conf_path_dict.items():
            conf_dict['conf_path'] = shutil.copy2(src=conf_dict['conf_path'],
                                                  dst=package_dir)
            PlayController.run_play(
                job_id=job_id,
                play_id=play_id,
                play_conf_path=conf_dict.get('conf_path'),
                play_hosts_path=conf_dict.get('hosts_path'),
                test_mode=TEST_MODE)
            if os.path.exists(conf_dict['conf_path']):
                os.remove(conf_dict['conf_path'])
            plays = JobSaver.query_play(play_id=play_id)
            if plays:
                play = plays[0]
                status = play.f_status
                if status != PlayStatus.SUCCESS:
                    if status in [
                            PlayStatus.CANCELED, PlayStatus.FAILED,
                            PlayStatus.TIMEOUT
                    ]:
                        update_info = {
                            'job_id': job_id,
                            'play_id': play_id,
                            'status': status,
                            'end_time': current_timestamp()
                        }
                        JobSaver.update_play_status(update_info)
                        JobSaver.update_play(update_info)
                        JobSaver.update_job_status(update_info)
                        JobSaver.update_job(update_info)
                    else:
                        update_info = {
                            'job_id': job_id,
                            'play_id': play_id,
                            'status': PlayStatus.FAILED,
                            'end_time': current_timestamp()
                        }
                        schedule_logger(job_id).error(
                            f'Unexpected error occured on play {play_id}, job {job_id} failed, previous status of play: {play.f_status}'
                        )
                        stat_logger.error(
                            f'Unexpected error occured on play {play_id}, job {job_id} failed, previous status of play: {play.f_status}'
                        )

                        JobSaver.update_play_status(update_info)
                        JobSaver.update_play(update_info)
                        JobSaver.update_job_status(update_info)
                        JobSaver.update_job(update_info)

                        schedule_logger(job_id).info(
                            f"job {job_id} finished, status is {update_info.get('status')}"
                        )
                    break
                else:
                    update_info = {
                        'job_id': job_id,
                        'play_id': play_id,
                        'status': PlayStatus.SUCCESS,
                        'end_time': current_timestamp()
                    }
                    JobSaver.update_play_status(update_info)
                    JobSaver.update_play(update_info)
            else:
                raise Exception(f'can not find play {play_id}')
        else:
            update_info = {
                'job_id': job_id,
                'status': JobStatus.SUCCESS,
                'end_time': current_timestamp()
            }
            JobSaver.update_job(update_info)
            JobSaver.update_job_status(update_info)
            schedule_logger(job_id).info(
                f"job {job_id} finished, status is {update_info.get('status')}"
            )

        if not TEST_MODE:
            plays = JobSaver.query_play(job_id=job_id,
                                        status=PlayStatus.SUCCESS)
            modules = []
            module_names = []
            for play in plays:
                module_name = play.f_roles.strip('[]').replace('_', '')
                module_names.append(module_name)
                modules.append({
                    'name':
                    module_name,
                    'ips':
                    job_data.get('modules', {}).get(module_name,
                                                    {}).get('ips', []),
                    'port':
                    job_data.get('modules', {}).get(module_name,
                                                    {}).get('port', None)
                })

            # parties = PartyInfo.get_or_none(f_version=job_data.get('version'), f_party_id=job_data.get('party_id'))
            parties = PartyInfo.get_or_none(
                f_party_id=job_data.get('party_id'))
            if parties:
                module_mapping = dict(zip(module_names, modules))
                stored_modules = parties.f_modules.get("data", [])

                name_map = {}
                for offset, item in enumerate(stored_modules):
                    name_map[item.get('name')] = offset

                for key, value in module_mapping.items():
                    if key in name_map:
                        schedule_logger(job_id).info(
                            f"{key} in name map, in replace process")
                        stored_modules[name_map[key]] = value
                    else:
                        schedule_logger(job_id).info(
                            f"{key} not in name map, in append process ")
                        stored_modules.append(value)

                # update_status = False
                # for offset, module_info in enumerate(stored_modules):
                #     if module_info['name'] in module_mapping:
                #         stored_modules[offset] = module_mapping[module_info['name']]
                #         update_status = True
                for key in ['role', 'version']:
                    # if parties[key] != job_data[key]:
                    #     parties[key] = job_data[key]
                    if getattr(parties, f'f_{key}') != job_data[key]:
                        setattr(parties, f'f_{key}', job_data[key])
                        # update_status = True
                # if update_status:
                parties.f_modules = {'data': stored_modules}
                parties.save()
                DB.commit()
            else:
                party_info = PartyInfo()
                # party_info.f_job_id = job_id
                party_info.f_role = job_data.get('role')
                party_info.f_version = job_data.get('version')
                party_info.f_party_id = job_data.get('party_id')
                party_info.f_modules = {'data': modules}
                party_info.save(force_insert=True)
コード例 #7
0
 def run_play(job_id,
              play_id,
              play_conf_path,
              play_hosts_path,
              test_mode=False,
              retry_mode=False):
     schedule_logger(job_id).info(
         f'Trying to start to run play with id: {play_id}')
     # task_process_start_status = False
     process_cmd = [
         'python3',
         sys.modules[PlayExecutor.__module__].__file__,
         '--job_id',
         job_id,
         '--play_id',
         play_id,
         '--conf_path',
         play_conf_path,
         '--hosts_path',
         play_hosts_path,
     ]
     if test_mode:
         process_cmd.append('--test')
     if retry_mode:
         process_cmd.append('--retry')
     schedule_logger(job_id).info(
         f"Trying to start job {job_id}, play {play_id} subprocess.")
     try:
         config_dir = file_utils.get_play_directory(play_id)
         std_dir = file_utils.get_job_log_directory(job_id)
         p = job_utils.run_subprocess(config_dir=config_dir,
                                      process_cmd=process_cmd,
                                      log_dir=std_dir)
         if p:
             # task_process_start_status = True
             play_info = {
                 'pid': p.pid,
                 'job_id': job_id,
                 'play_id': play_id,
                 'status': PlayStatus.RUNNING,
                 'start_time': current_timestamp()
             }
             JobSaver.update_play_status(play_info=play_info)
             JobSaver.update_play(play_info=play_info)
             p.wait()
         else:
             raise Exception(f'play {play_id} start subprocess failed')
     except Exception as e:
         play_info = {
             'job_id': job_id,
             'play_id': play_id,
             'status': PlayStatus.FAILED,
             'end_time': current_timestamp()
         }
         JobSaver.update_play_status(play_info)
         JobSaver.update_play(play_info)
         schedule_logger(job_id).exception(e)
     finally:
         if retry_mode:
             job_info = {
                 'job_id':
                 job_id,
                 'play_id':
                 play_id,
                 'end_time':
                 current_timestamp(),
                 'status':
                 JobStatus.SUCCESS if PlayController.check_job_status(
                     job_id=job_id) else JobStatus.FAILED
             }
             JobSaver.update_job(job_info)
             JobSaver.update_job_status(job_info)
コード例 #8
0
ファイル: callback.py プロジェクト: cold-code/FATE-Cloud
    def v2_playbook_on_stats(self, stats):

        self._display.banner("PLAY RECAP")

        hosts = sorted(stats.processed.keys())
        failed_count = 0
        for h in hosts:
            t = stats.summarize(h)
            # stat_logger.info(f'>>>>>>>>PLAY RECAP, {t}')
            schedule_logger(self._job_id).info(
                f"in playbook on stats: RECAP: {t}, host: {h}")
            failed_count += t['failures']
            failed_count += t['unreachable']

            # self._display.display(
            #     u"%s : %s %s %s %s %s %s %s" % (
            #         hostcolor(h, t),
            #         colorize(u'ok', t['ok'], C.COLOR_OK),
            #         colorize(u'changed', t['changed'], C.COLOR_CHANGED),
            #         colorize(u'unreachable', t['unreachable'], C.COLOR_UNREACHABLE),
            #         colorize(u'failed', t['failures'], C.COLOR_ERROR),
            #         colorize(u'skipped', t['skipped'], C.COLOR_SKIP),
            #         colorize(u'rescued', t['rescued'], C.COLOR_OK),
            #         colorize(u'ignored', t['ignored'], C.COLOR_WARN),
            #     ),
            #     screen_only=True
            # )

            self._display.display(u"%s : %s %s %s %s %s %s %s" % (
                hostcolor(h, t, False),
                colorize(u'ok', t['ok'], None),
                colorize(u'changed', t['changed'], None),
                colorize(u'unreachable', t['unreachable'], None),
                colorize(u'failed', t['failures'], None),
                colorize(u'skipped', t['skipped'], None),
                colorize(u'rescued', t['rescued'], None),
                colorize(u'ignored', t['ignored'], None),
            ),
                                  log_only=True)

        if not failed_count:
            update_info = {
                'job_id': self._job_id,
                'play_id': self._play_id,
                'end_time': current_timestamp(),
                'status': PlayStatus.SUCCESS
            }

            JobSaver.update_play(update_info)
            JobSaver.update_play_status(update_info)

        self._display.display("", screen_only=True)

        # print custom stats if required
        if stats.custom and self.show_custom_stats:
            self._display.banner("CUSTOM STATS: ")
            # per host
            # TODO: come up with 'pretty format'
            for k in sorted(stats.custom.keys()):
                if k == '_run':
                    continue
                self._display.display(
                    '\t%s: %s' %
                    (k, self._dump_results(stats.custom[k], indent=1).replace(
                        '\n', '')))

            # print per run custom stats
            if '_run' in stats.custom:
                self._display.display("", screen_only=True)
                self._display.display('\tRUN: %s' % self._dump_results(
                    stats.custom['_run'], indent=1).replace('\n', ''))
            self._display.display("", screen_only=True)

        if context.CLIARGS['check'] and self.check_mode_markers:
            self._display.banner("DRY RUN")