Пример #1
0
    def collect_artifacts(self):

        tmp_dir = None
        for i, cmd_part in enumerate(self.cmd):
            if cmd_part in ('--tmp', '-t'):
                tmp_dir = self.cmd[i + 1]
                break

        if not tmp_dir:
            logger.info('Failed to determine tmp directory')
            return {}

        exec_state_path = os.path.join(tmp_dir, 'exec_state')

        logger.info('Parsing exec state: {}'.format(exec_state_path))

        exec_state = {}

        try:
            with open(exec_state_path, 'rb') as f:
                exec_state = json.load(f).get('status', {})
        except Exception:
            logger.exception(
                'Failed to parse exec state file {}'.format(exec_state_path)
            )
            pass

        return exec_state
Пример #2
0
    def execute(self):
        group_base_path = self.params['group_base_path'].rstrip('/')

        if not os.path.exists(group_base_path):
            raise RuntimeError('Group dir {path} does not exist'.format(
                path=group_base_path, ))

        dst_base_path, basename = os.path.split(group_base_path)

        remove_path = os.path.join(dst_base_path,
                                   self.removed_basename(basename))
        logger.info(
            'Renaming group base dir {tmp_dir} to destination dir {dest_dir}'.
            format(
                tmp_dir=group_base_path,
                dest_dir=remove_path,
            ))
        try:
            os.rename(group_base_path, remove_path)
        except OSError as e:
            if e.errno == 2:
                # errno == 2: No such file or directory
                if os.path.exists(remove_path):
                    # group_base_path was already renamed, not an error
                    pass
                else:
                    raise
        except Exception:
            logger.exception('Failed to rename tmp dir to dest dir')
            raise
Пример #3
0
    def run(self):

        # create db record
        s = Session()
        s.begin()
        command = minion.db.commands.Command(
            uid=self.uid,
            pid=None,
            command=self.cmd_str,
            start_ts=int(time.time()),
            task_id=self.params.get('task_id'))
        # TODO: what about group_id, node, node_backend ?

        s.add(command)
        s.commit()

        try:
            self.execute()
        except Exception as e:
            self.error = e
            # TODO: raise?
        self.finish_ts = int(time.time())

        s.begin()
        try:
            command.progress = 1.0
            command.exit_code = 1 if self.error else 0
            command.command_code = 1 if self.error else 0
            command.finish_ts = self.finish_ts
            s.add(command)
            s.commit()
        except Exception as e:
            logger.exception('Failed to update db command')
            s.rollback()
Пример #4
0
    def execute(self):
        group_base_path = self.params['group_base_path'].rstrip('/')

        if not os.path.exists(group_base_path):
            raise RuntimeError(
                'Group dir {path} does not exist'.format(
                    path=group_base_path,
                )
            )

        dst_base_path, basename = os.path.split(group_base_path)

        remove_path = os.path.join(
            dst_base_path,
            self.removed_basename(basename)
        )
        logger.info(
            'Renaming group base dir {tmp_dir} to destination dir {dest_dir}'.format(
                tmp_dir=group_base_path,
                dest_dir=remove_path,
            )
        )
        try:
            os.rename(group_base_path, remove_path)
        except OSError as e:
            if e.errno == 2:
                # errno == 2: No such file or directory
                if os.path.exists(remove_path):
                    # group_base_path was already renamed, not an error
                    pass
                else:
                    raise
        except Exception:
            logger.exception('Failed to rename tmp dir to dest dir')
            raise
Пример #5
0
    def on_command_completed(self):

        self.finish_ts = int(time.time())
        self.artifacts = self.collect_artifacts()

        self.on_update_progress()

        if not self._apply_postprocessors():
            # TODO: add status codes
            logger.info('Command failed, no post processors will be applied')
            return

        for post_processor in self.POST_PROCESSORS:
            params_supplied = all(
                param in self.params
                for param in post_processor.REQUIRED_PARAMS
            )
            if params_supplied:
                # TODO: replace by required params? possibly not
                logger.info('Running post processor {}'.format(post_processor.__name__))
                uid = uuid.uuid4().hex
                command = post_processor(uid, params=self.params)
                try:
                    # NOTE: when running as a post processor command is not
                    # dumped to database, therefore 'execute' method is called
                    # instead of 'run'
                    command.execute()
                except:
                    logger.exception('Post processor {} failed, skipped'.format(
                        post_processor.__name__
                    ))
                    continue
Пример #6
0
 def update_broken_commands(self):
     s = Session()
     s.begin()
     try:
         for c in s.query(Command).filter_by(exit_code=None):
             log_extra = {'task_id': c.task_id, 'job_id': c.job_id}
             if not self.pid_exists(c.pid):
                 c.progress = 1.0
                 c.exit_code = 666
                 c.finish_ts = int(time.time())
                 s.add(c)
                 cmd_logger.info(
                     'Command {}, pid {} is considered broken, will be marked as '
                     'finished'.format(c.uid, c.pid),
                     extra=log_extra,
                 )
             else:
                 cmd_logger.warn(
                     'Command {}, pid {} is considered broken, but process is running'
                     .format(c.uid, c.pid),
                     extra=log_extra,
                 )
         s.commit()
     except Exception:
         logger.exception('Failed to update broken commands')
         s.rollback()
         raise
Пример #7
0
 def update_broken_commands(self):
     s = Session()
     s.begin()
     try:
         for c in s.query(Command).filter_by(exit_code=None):
             if not self.pid_exists(c.pid):
                 c.progress = 1.0
                 c.exit_code = 666
                 c.finish_ts = int(time.time())
                 s.add(c)
                 logger.info(
                     'Command {}, pid {} is considered broken, will be marked as '
                     'finished'.format(
                         c.uid,
                         c.pid
                     )
                 )
             else:
                 logger.warn(
                     'Command {}, pid {} is considered broken, but process is running'.format(
                         c.uid,
                         c.pid
                     )
                 )
         s.commit()
     except Exception:
         logger.exception('Failed to update broken commands')
         s.rollback()
         raise
Пример #8
0
 def execute(self):
     try:
         shutil.rmtree(self.params['remove_path'])
     except Exception:
         logger.exception('Failed to remove path {}'.format(self.params['remove_path']))
         raise
     logger.info('Successfully removed path {}'.format(self.params['remove_path']))
Пример #9
0
    def run(self):

        # create db record
        s = Session()
        s.begin()
        command = minion.db.commands.Command(
            uid=self.uid,
            pid=None,
            command=self.cmd_str,
            start_ts=int(time.time()),
            task_id=self.params.get('task_id')
        )
        # TODO: what about group_id, node, node_backend ?

        s.add(command)
        s.commit()

        try:
            self.execute()
        except Exception as e:
            self.error = e
            # TODO: raise?
        self.finish_ts = int(time.time())

        s.begin()
        try:
            command.progress = 1.0
            command.exit_code = 1 if self.error else 0
            command.command_code = 1 if self.error else 0
            command.finish_ts = self.finish_ts
            s.add(command)
            s.commit()
        except Exception as e:
            logger.exception('Failed to update db command')
            s.rollback()
Пример #10
0
def setup_session(uri):
    engine = sqlalchemy.create_engine(uri)
    try:
        engine.connect()
        logger.info('Successfully connected to db by uri {0}'.format(uri))
    except Exception as e:
        logger.exception('Failed to connect to db engine by uri {0}'.format(uri))
        raise

    return sessionmaker(bind=engine, autocommit=True)
Пример #11
0
def setup_session(uri):
    engine = sqlalchemy.create_engine(uri)
    try:
        engine.connect()
        logger.info('Successfully connected to db by uri {0}'.format(uri))
    except Exception as e:
        logger.exception(
            'Failed to connect to db engine by uri {0}'.format(uri))
        raise

    return sessionmaker(bind=engine, autocommit=True)
Пример #12
0
 def feed(self, s):
     prev_progress = self.progress
     self.output.append(s)
     if abs(self.progress - prev_progress) > 0.01:
         try:
             s = Session()
             s.begin()
             self.command.progress = self.progress
             s.add(self.command)
             s.commit()
         except Exception as e:
             logger.exception('pid {0}: failed to update db command'.format(
                 self.subprocess.pid))
             pass
Пример #13
0
 def update_db_command(self):
     s = Session()
     s.begin()
     try:
         command = self.command
         command.progress = self.progress
         command.exit_code = self.exit_code
         command.command_code = self.command_code
         command.finish_ts = self.finish_ts
         s.add(command)
         s.commit()
     except Exception as e:
         logger.exception('Failed to update db command: {0}'.format(e))
         s.rollback()
Пример #14
0
 def execute(self):
     ids_file = self.params['ids']
     logger.info('Generating ids file {}'.format(ids_file))
     if os.path.exists(ids_file):
         logger.info('Ids file {} already exists'.format(ids_file))
     else:
         try:
             with open(ids_file, 'wb') as f:
                 f.write(os.urandom(64))
         except Exception:
             logger.exception('Failed to create ids file {}'.format(
                 ids_file)
             )
             raise
     logger.info('Successfully created ids file {}'.format(ids_file))
Пример #15
0
 def execute(self):
     group = str(int(self.params['group']))
     path = self.params['group_file'].format(group_id=group)
     try:
         if os.path.exists(path):
             os.rename(path, path + '.bak')
         dirname = os.path.dirname(path)
         if not os.path.exists(dirname):
             os.makedirs(dirname, 0755)
         with open(path, 'w') as f:
             f.write(group)
     except Exception:
         logger.exception('Failed to create group file')
         raise
     logger.info('Successfully created group file {} for group {}'.format(path, group))
Пример #16
0
 def create_group_file(self):
     if self.params.get('group_file'):
         try:
             group = str(int(self.params.get('group')))
             path = self.params.get('group_file')
             if os.path.exists(path):
                 os.rename(path, path + '.bak')
             dirname = os.path.dirname(path)
             if not os.path.exists(dirname):
                 os.makedirs(dirname, 0755)
             with open(path, 'w') as f:
                 f.write(group + '\n')
         except Exception as e:
             logger.exception('Failed to create group file: {0}'.format(e))
         else:
             logger.info('Successfully created group file '
                         'for group {0}'.format(group))
Пример #17
0
 def on_update_progress(self):
     s = Session()
     s.begin()
     try:
         command = self.command
         command.progress = self.watcher.progress
         command.exit_code = self.watcher.exit_code
         command.command_code = self.command_code
         command.stdout = self.watcher.get_stdout()
         command.stderr = self.watcher.get_stderr()
         if command.exit_code is not None:
             command.artifacts = json.dumps(self.artifacts)
             command.finish_ts = self.finish_ts
         s.add(command)
         s.commit()
     except Exception:
         logger.exception('Failed to update db command')
         s.rollback()
Пример #18
0
 def update_broken_commands(self):
     s = Session()
     s.begin()
     try:
         for c in s.query(Command).filter_by(exit_code=None):
             if not self.pid_exists(c.pid):
                 c.progress = 1.0
                 c.exit_code = 666
                 c.finish_ts = int(time.time())
                 s.add(c)
                 logger.info('Command {0}, pid {1} is considered broken, '
                             'will be marked as finished'.format(
                                 c.uid, c.pid))
             else:
                 logger.warn('Command {0}, pid {1} is considered broken, '
                             'but process is running'.format(c.uid, c.pid))
         s.commit()
     except Exception as e:
         logger.exception('Failed to update broken commands')
         s.rollback()
         raise
Пример #19
0
    def execute(self):
        group_base_path_root_dir = self.params['group_base_path_root_dir'].rstrip('/')

        basename = self.get_vacant_basename(group_base_path_root_dir)
        tmp_basename = self.tmp_basename(basename)

        tmp_dir = os.path.join(group_base_path_root_dir, tmp_basename)
        logger.info('Creating tmp dir for new group: {}'.format(tmp_dir))
        try:
            os.mkdir(tmp_dir, 0755)
        except Exception:
            logger.exception('Failed to create tmp dir for new group')
            raise

        logger.info('Adding group files')
        for filename, body in self.params['files'].iteritems():
            logger.info('Adding file {}'.format(filename))
            filename = os.path.join(
                tmp_dir,
                filename
            )
            dirname, basefname = os.path.split(filename)
            if not os.path.exists(dirname):
                os.makedirs(dirname)
            with open(filename, 'wb') as f:
                f.write(body)

        dest_dir = os.path.join(group_base_path_root_dir, basename)
        logger.info(
            'Renaming tmp dir {tmp_dir} to destination dir {dest_dir}'.format(
                tmp_dir=tmp_dir,
                dest_dir=dest_dir,
            )
        )
        try:
            os.rename(tmp_dir, dest_dir)
        except Exception:
            logger.exception('Failed to rename tmp dir to dest dir')
            raise
Пример #20
0
    def execute(self):
        group_base_path_root_dir = self.params[
            'group_base_path_root_dir'].rstrip('/')

        basename = self.get_vacant_basename(group_base_path_root_dir)
        tmp_basename = self.tmp_basename(basename)

        tmp_dir = os.path.join(group_base_path_root_dir, tmp_basename)
        logger.info('Creating tmp dir for new group: {}'.format(tmp_dir))
        try:
            os.mkdir(tmp_dir, 0755)
        except Exception:
            logger.exception('Failed to create tmp dir for new group')
            raise

        logger.info('Adding group files')
        for filename, body in self.params['files'].iteritems():
            logger.info('Adding file {}'.format(filename))
            filename = os.path.join(tmp_dir, filename)
            dirname, basefname = os.path.split(filename)
            if not os.path.exists(dirname):
                os.makedirs(dirname)
            with open(filename, 'wb') as f:
                f.write(body)

        dest_dir = os.path.join(group_base_path_root_dir, basename)
        logger.info(
            'Renaming tmp dir {tmp_dir} to destination dir {dest_dir}'.format(
                tmp_dir=tmp_dir,
                dest_dir=dest_dir,
            ))
        try:
            os.rename(tmp_dir, dest_dir)
        except Exception:
            logger.exception('Failed to rename tmp dir to dest dir')
            raise
Пример #21
0
    def collect_artifacts(self):

        commands_stats_path = self.params.get('commands_stats_path')
        if not commands_stats_path:
            logger.info('Commands stats path was not supplied')
            return {}

        logger.info('Parsing commands stats path: {}'.format(commands_stats_path))

        commands_stats = {}

        try:
            with open(commands_stats_path, 'rb') as f:
                commands_stats = json.load(f).get('commands', {})
        except Exception:
            logger.exception(
                'Failed to parse commands stats file {}'.format(commands_stats_path)
            )

        parsed_stats = self._parse_commands_stats(commands_stats)

        # NOTE: temporary backward compatibility
        self.commands_stats = parsed_stats
        return parsed_stats
Пример #22
0
    def create_group_file(self):
        if self.params.get('group_file'):
            try:
                group = str(int(self.params.get('group')))
                path = self.params['group_file'].format(group_id=group)
                if os.path.exists(path):
                    os.rename(path, path + '.bak')
                dirname = os.path.dirname(path)
                if not os.path.exists(dirname):
                    os.makedirs(dirname, 0755)
                with open(path, 'w') as f:
                    f.write(group)
            except Exception as e:
                logger.exception('Failed to create group file: {0}'.format(e))
            else:
                logger.info('Successfully created group file '
                            'for group {0}'.format(group))
        else:
            logger.info('Group file creation was not requested for '
                        'group {0}'.format(self.params.get('group')))

        if self.params.get('remove_path'):
            try:
                shutil.rmtree(self.params['remove_path'])
            except Exception as e:
                logger.exception('Failed to remove path {0}: {1}'.format(
                    self.params['remove_path'], e))
            else:
                logger.info('Successfully removed path {0} '
                            'for group {1}'.format(self.params['remove_path'],
                                                   group))
        else:
            logger.info('Path removal was not requested for '
                        'group {0}'.format(self.params.get('group')))

        if self.params.get('ids'):
            ids_file = self.params['ids']
            logger.info('Generating ids file {} required'.format(ids_file))
            if os.path.exists(ids_file):
                logger.info('Ids file {} already exists'.format(ids_file))
            else:
                try:
                    with open(ids_file, 'wb') as f:
                        f.write(os.urandom(64))
                except Exception as e:
                    logger.exception(
                        'Failed to create ids file {}'.format(ids_file))
                else:
                    logger.info('Successfully created ids file {} '
                                'for group {1}'.format(ids_file, group))
Пример #23
0
    def create_group_file(self):
        if self.params.get('group_file'):
            try:
                group = str(int(self.params.get('group')))
                path = self.params['group_file'].format(group_id=group)
                if os.path.exists(path):
                    os.rename(path, path + '.bak')
                dirname = os.path.dirname(path)
                if not os.path.exists(dirname):
                    os.makedirs(dirname, 0755)
                with open(path, 'w') as f:
                    f.write(group)
            except Exception as e:
                logger.exception('Failed to create group file: {0}'.format(e))
            else:
                logger.info('Successfully created group file '
                            'for group {0}'.format(group))
        else:
            logger.info('Group file creation was not requested for '
                'group {0}'.format(self.params.get('group')))

        if self.params.get('remove_path'):
            try:
                shutil.rmtree(self.params['remove_path'])
            except Exception as e:
                logger.exception('Failed to remove path {0}: {1}'.format(
                    self.params['remove_path'], e))
            else:
                logger.info('Successfully removed path {0} '
                    'for group {1}'.format(self.params['remove_path'], group))
        else:
            logger.info('Path removal was not requested for '
                'group {0}'.format(self.params.get('group')))

        if self.params.get('ids'):
            ids_file = self.params['ids']
            logger.info('Generating ids file {} required'.format(ids_file))
            if os.path.exists(ids_file):
                logger.info('Ids file {} already exists'.format(ids_file))
            else:
                try:
                    with open(ids_file, 'wb') as f:
                        f.write(os.urandom(64))
                except Exception as e:
                    logger.exception('Failed to create ids file {}'.format(
                        ids_file))
                else:
                    logger.info('Successfully created ids file {} '
                        'for group {1}'.format(ids_file, group))
Пример #24
0
    def create_group_file_marker(self):
        if self.params.get('group_file_marker'):
            try:
                group = (str(int(self.params.get('group')))
                         if self.params.get('group') else '')
                path = self.params['group_file_marker'].format(group_id=group)
                dirname = os.path.dirname(path)
                if not os.path.exists(dirname):
                    os.makedirs(dirname, 0755)
                with open(path, 'w') as f:
                    f.write(group)
            except Exception as e:
                logger.exception(
                    'Failed to create group file marker: {0}'.format(e))
            else:
                logger.info('Successfully created group file marker '
                            'for group {0}'.format(group))
        else:
            logger.info('Group file marker creation was not requested for '
                        'group {0}'.format(self.params.get('group')))

        if self.params.get('remove_group_file'):
            logger.info('Removing group file {0}'.format(
                self.params['remove_group_file']))
            try:
                os.remove(self.params['remove_group_file'])
            except Exception as e:
                logger.exception('Failed to remove group file: {0}'.format(e))
            else:
                logger.info('Successfully removed group '
                            'file {0}'.format(
                                self.params['remove_group_file']))
        else:
            logger.info('Group file removal was not requested for '
                        'group {0}'.format(self.params.get('group')))

        if self.params.get('move_dst') and self.params.get('move_src'):
            try:
                os.rename(self.params['move_src'], self.params['move_dst'])
            except Exception as e:
                logger.exception(
                    'Failed to execute move task: {0} to {1}'.format(
                        self.params['move_src'], self.params['move_dst']))
            else:
                logger.info(
                    'Successfully performed move task: {0} to {1}'.format(
                        self.params['move_src'], self.params['move_dst']))

        if self.params.get('mark_backend'):
            marker = self.params['mark_backend']
            try:
                open(marker, 'w').close()
            except Exception as e:
                logger.error(
                    'Failed to create backend down marker: {0}'.format(e))
            else:
                logger.info(
                    'Successfully created backend down marker: {0}'.format(
                        self.params['mark_backend']))

        if self.params.get('unmark_backend'):
            marker = self.params['unmark_backend']
            try:
                os.remove(self.params['unmark_backend'])
            except Exception as e:
                logger.error(
                    'Failed to remove backend down marker: {0}'.format(e))
            else:
                logger.info(
                    'Successfully removed backend down marker: {0}'.format(
                        self.params['unmark_backend']))
Пример #25
0
    def create_group_file_marker(self):
        if self.params.get('group_file_marker'):
            try:
                group = (str(int(self.params.get('group')))
                         if self.params.get('group') else
                         '')
                path = self.params['group_file_marker'].format(group_id=group)
                dirname = os.path.dirname(path)
                if not os.path.exists(dirname):
                    os.makedirs(dirname, 0755)
                with open(path, 'w') as f:
                    f.write(group)
            except Exception as e:
                logger.exception('Failed to create group file marker: {0}'.format(e))
            else:
                logger.info('Successfully created group file marker '
                            'for group {0}'.format(group))
        else:
            logger.info('Group file marker creation was not requested for '
                'group {0}'.format(self.params.get('group')))

        if self.params.get('remove_group_file'):
            logger.info('Removing group file {0}'.format(self.params['remove_group_file']))
            try:
                os.remove(self.params['remove_group_file'])
            except Exception as e:
                logger.exception('Failed to remove group file: {0}'.format(e))
            else:
                logger.info('Successfully removed group '
                            'file {0}'.format(self.params['remove_group_file']))
        else:
            logger.info('Group file removal was not requested for '
                'group {0}'.format(self.params.get('group')))

        if self.params.get('move_dst') and self.params.get('move_src'):
            try:
                os.rename(self.params['move_src'], self.params['move_dst'])
            except Exception as e:
                logger.exception('Failed to execute move task: {0} to {1}'.format(
                    self.params['move_src'], self.params['move_dst']))
            else:
                logger.info('Successfully performed move task: {0} to {1}'.format(
                    self.params['move_src'], self.params['move_dst']))

        if self.params.get('mark_backend'):
            marker = self.params['mark_backend']
            try:
                open(marker, 'w').close()
            except Exception as e:
                logger.error('Failed to create backend down marker: {0}'.format(e))
            else:
                logger.info('Successfully created backend down marker: {0}'.format(
                    self.params['mark_backend']))

        if self.params.get('unmark_backend'):
            marker = self.params['unmark_backend']
            try:
                os.remove(self.params['unmark_backend'])
            except Exception as e:
                logger.error('Failed to remove backend down marker: {0}'.format(e))
            else:
                logger.info('Successfully removed backend down marker: {0}'.format(
                    self.params['unmark_backend']))