Beispiel #1
0
    def collect_artifacts(self):

        tmp_dir = None
        for i, cmd_part in enumerate(self.cmd):
            if cmd_part in ('--tmp', '-t'):
                tmp_dir = self.cmd[i + 1]
                break

        if not tmp_dir:
            logger.info('Failed to determine tmp directory')
            return {}

        exec_state_path = os.path.join(tmp_dir, 'exec_state')

        logger.info('Parsing exec state: {}'.format(exec_state_path))

        exec_state = {}

        try:
            with open(exec_state_path, 'rb') as f:
                exec_state = json.load(f).get('status', {})
        except Exception:
            logger.exception(
                'Failed to parse exec state file {}'.format(exec_state_path)
            )
            pass

        return exec_state
Beispiel #2
0
    def execute(self):
        group_base_path = self.params['group_base_path'].rstrip('/')

        if not os.path.exists(group_base_path):
            raise RuntimeError(
                'Group dir {path} does not exist'.format(
                    path=group_base_path,
                )
            )

        dst_base_path, basename = os.path.split(group_base_path)

        remove_path = os.path.join(
            dst_base_path,
            self.removed_basename(basename)
        )
        logger.info(
            'Renaming group base dir {tmp_dir} to destination dir {dest_dir}'.format(
                tmp_dir=group_base_path,
                dest_dir=remove_path,
            )
        )
        try:
            os.rename(group_base_path, remove_path)
        except OSError as e:
            if e.errno == 2:
                # errno == 2: No such file or directory
                if os.path.exists(remove_path):
                    # group_base_path was already renamed, not an error
                    pass
                else:
                    raise
        except Exception:
            logger.exception('Failed to rename tmp dir to dest dir')
            raise
Beispiel #3
0
    def _prepare_command(self, cmd):
        cmd_str = ' '.join(cmd)

        if 'cmd_tpl' in self.params:
            # TODO: remove this backward-compatibility hack
            cmd_str = self.params['cmd_tpl']

        missing_parameters = self.PLACEHOLDERS_TPL.findall(cmd_str)

        for param in missing_parameters:
            if param in self.params:
                continue
            elif param == 'backend_id':
                if 'group' not in self.params:
                    raise ValueError('Parameter "group" is required')
                config_path = self.params['config_path']
                logger.info(
                    'backend id not found in request params, '
                    'getting backend id from elliptics config {config}, '
                    'group {group}'.format(
                        config=config_path,
                        group=self.params['group'],
                    )
                )
                config = EllipticsConfig(self.params['config_path'])
                self.params['backend_id'] = config.get_group_backend(int(self.params['group']))
            else:
                raise ValueError('Cannot process command: unknown parameter "{}"'.format(param))

        cmd_str = cmd_str.format(**self.params)
        logger.info('Prepared command: {}'.format(cmd_str))
        return shlex.split(cmd_str)
Beispiel #4
0
    def execute(self):
        group_base_path = self.params['group_base_path'].rstrip('/')

        if not os.path.exists(group_base_path):
            raise RuntimeError('Group dir {path} does not exist'.format(
                path=group_base_path, ))

        dst_base_path, basename = os.path.split(group_base_path)

        remove_path = os.path.join(dst_base_path,
                                   self.removed_basename(basename))
        logger.info(
            'Renaming group base dir {tmp_dir} to destination dir {dest_dir}'.
            format(
                tmp_dir=group_base_path,
                dest_dir=remove_path,
            ))
        try:
            os.rename(group_base_path, remove_path)
        except OSError as e:
            if e.errno == 2:
                # errno == 2: No such file or directory
                if os.path.exists(remove_path):
                    # group_base_path was already renamed, not an error
                    pass
                else:
                    raise
        except Exception:
            logger.exception('Failed to rename tmp dir to dest dir')
            raise
    def is_success(self):

        logger.info('pid {0}: exit code {1}'.format(self.subprocess.pid,
                                                    self.exit_code))

        if self.exit_code == 0:
            return True

        output = ''.join(self.output)
        try:
            data = json.loads(output)
        except Exception as e:
            logger.error('pid {0}: failed to parse output json: {1}'.format(
                self.subprocess.pid, output))
            return False

        if not 'error' in data:
            logger.error('pid {0}: no "error" key in response data'.format(
                self.subprocess.pid))
            return False
        if not 'code' in data['error']:
            logger.error(
                'pid {0}: no "code" key in response error data'.format(
                    self.subprocess.pid))
            return False

        logger.info('pid {0}: operation error code {1}'.format(
            self.subprocess.pid, data['error']['code']))

        return data['error']['code'] in self.success_codes
Beispiel #6
0
    def _parse_command_code(self):
        if self.watcher.exit_code == 0:
            return self.watcher.exit_code

        output = self.watcher.get_stdout()
        try:
            data = json.loads(output)
        except Exception:
            logger.error('pid {}: failed to parse output json: {}'.format(
                self.pid,
                output
            ))
            return self.watcher.exit_code

        if 'error' not in data:
            logger.error('pid {}: no "error" key in response data'.format(
                self.pid
            ))
            return self.watcher.exit_code
        if 'code' not in data['error']:
            logger.error('pid {}: no "code" key in response error data'.format(
                self.pid
            ))
            return self.watcher.exit_code

        logger.info('pid {}: operation error code {}'.format(
            self.pid,
            data['error']['code']
        ))

        return data['error']['code']
    def is_success(self):

        logger.info('pid {0}: exit code {1}'.format(
            self.subprocess.pid, self.exit_code))

        if self.exit_code == 0:
            return True

        output = ''.join(self.output)
        try:
            data = json.loads(output)
        except Exception as e:
            logger.error('pid {0}: failed to parse output json: {1}'.format(
                self.subprocess.pid, output))
            return False

        if not 'error' in data:
            logger.error('pid {0}: no "error" key in response data'.format(
                self.subprocess.pid))
            return False
        if not 'code' in data['error']:
            logger.error('pid {0}: no "code" key in response error data'.format(
                self.subprocess.pid))
            return False

        logger.info('pid {0}: operation error code {1}'.format(
            self.subprocess.pid, data['error']['code']))

        return data['error']['code'] in self.success_codes
Beispiel #8
0
    def on_command_completed(self):

        self.finish_ts = int(time.time())
        self.artifacts = self.collect_artifacts()

        self.on_update_progress()

        if not self._apply_postprocessors():
            # TODO: add status codes
            logger.info('Command failed, no post processors will be applied')
            return

        for post_processor in self.POST_PROCESSORS:
            params_supplied = all(
                param in self.params
                for param in post_processor.REQUIRED_PARAMS
            )
            if params_supplied:
                # TODO: replace by required params? possibly not
                logger.info('Running post processor {}'.format(post_processor.__name__))
                uid = uuid.uuid4().hex
                command = post_processor(uid, params=self.params)
                try:
                    # NOTE: when running as a post processor command is not
                    # dumped to database, therefore 'execute' method is called
                    # instead of 'run'
                    command.execute()
                except:
                    logger.exception('Post processor {} failed, skipped'.format(
                        post_processor.__name__
                    ))
                    continue
Beispiel #9
0
 def execute(self):
     try:
         shutil.rmtree(self.params['remove_path'])
     except Exception:
         logger.exception('Failed to remove path {}'.format(self.params['remove_path']))
         raise
     logger.info('Successfully removed path {}'.format(self.params['remove_path']))
Beispiel #10
0
 def update_broken_commands(self):
     s = Session()
     s.begin()
     try:
         for c in s.query(Command).filter_by(exit_code=None):
             if not self.pid_exists(c.pid):
                 c.progress = 1.0
                 c.exit_code = 666
                 c.finish_ts = int(time.time())
                 s.add(c)
                 logger.info(
                     'Command {}, pid {} is considered broken, will be marked as '
                     'finished'.format(
                         c.uid,
                         c.pid
                     )
                 )
             else:
                 logger.warn(
                     'Command {}, pid {} is considered broken, but process is running'.format(
                         c.uid,
                         c.pid
                     )
                 )
         s.commit()
     except Exception:
         logger.exception('Failed to update broken commands')
         s.rollback()
         raise
 def execute(self):
     marker = self.params["unmark_backend"]
     try:
         os.remove(marker)
     except Exception as e:
         logger.error("Failed to remove backend marker: {}".format(e))
         raise
     logger.info("Successfully removed backend marker: {}".format(marker))
Beispiel #12
0
 def execute(self):
     logger.info('Removing group file {0}'.format(self.params['remove_group_file']))
     try:
         os.remove(self.params['remove_group_file'])
     except Exception as e:
         logger.error('Failed to remove group file: {}'.format(e))
         raise
     logger.info('Successfully removed group file {}'.format(self.params['remove_group_file']))
Beispiel #13
0
 def execute(self):
     stop_file = self.params['create_stop_file']
     try:
         open(stop_file, 'w').close()
     except Exception as e:
         logger.error('Failed to create backend stop marker: {}'.format(e))
         raise
     logger.info('Successfully created backend stop marker: {}'.format(stop_file))
 def execute(self):
     stop_file = self.params["remove_stop_file"]
     try:
         os.remove(stop_file)
     except Exception as e:
         logger.error("Failed to remove backend stop marker: {}".format(e))
         raise
     logger.info("Successfully removed backend stop marker: {}".format(stop_file))
Beispiel #15
0
 def execute(self):
     marker = self.params['mark_backend']
     try:
         open(marker, 'w').close()
     except Exception as e:
         logger.error('Failed to create backend marker: {}'.format(e))
         raise
     logger.info('Successfully created backend marker: {}'.format(marker))
Beispiel #16
0
def setup_session(uri):
    engine = sqlalchemy.create_engine(uri)
    try:
        engine.connect()
        logger.info('Successfully connected to db by uri {0}'.format(uri))
    except Exception as e:
        logger.exception('Failed to connect to db engine by uri {0}'.format(uri))
        raise

    return sessionmaker(bind=engine, autocommit=True)
Beispiel #17
0
def setup_session(uri):
    engine = sqlalchemy.create_engine(uri)
    try:
        engine.connect()
        logger.info('Successfully connected to db by uri {0}'.format(uri))
    except Exception as e:
        logger.exception(
            'Failed to connect to db engine by uri {0}'.format(uri))
        raise

    return sessionmaker(bind=engine, autocommit=True)
Beispiel #18
0
    def terminate(self, uid):
        if not uid in self.subprocesses:
            raise ValueError('Unknown command uid: {0}'.format(uid))
        logger.info('terminating command {0}, pid: {1}'.format(
            uid, self.subprocesses[uid].process.pid))

        # result, error, sub = self.subprocesses[uid].terminate().result()
        code = self.subprocesses[uid].terminate()
        if code:
            raise RuntimeError('Failed to terminate command {0}, '
                               'exit code: {1}'.format(uid, code))
Beispiel #19
0
    def _parse_commands_stats(self, commands_stats):
        op_statuses_count = {}
        for operation_status, count in commands_stats.iteritems():
            operation, status = operation_status.split('.', 1)
            statuses_count = op_statuses_count.setdefault(operation, {})
            statuses_count.setdefault(status, 0)
            statuses_count[status] += count

        logger.info('Parsed command statuses: {}'.format(op_statuses_count))

        return op_statuses_count
Beispiel #20
0
 def __init__(self, uid, cmd=None, params=None, env=None, success_codes=None, io_loop=IOLoop.instance()):
     cmd = self._make_command(params)
     logger.info('Constructed command: {}'.format(cmd))
     super(DnetClientCmd, self).__init__(
         uid,
         cmd,
         params=params,
         env=env,
         success_codes=success_codes,
         io_loop=io_loop
     )
Beispiel #21
0
    def terminate(self, uid):
        if uid not in self.subprocesses:
            raise ValueError('Unknown command uid: {0}'.format(uid))
        logger.info(
            'terminating command {}, pid: {}'.format(uid, self.subprocesses[uid].process.pid)
        )

        # result, error, sub = self.subprocesses[uid].terminate().result()
        code = self.subprocesses[uid].terminate()
        if code:
            raise RuntimeError('Failed to terminate command {}, exit code: {}'.format(uid, code))
 def execute(self):
     group = str(self.params.get("group")) if "group" in self.params.get("group") else ""
     try:
         path = self.params["group_file_marker"].format(group_id=group)
         dirname = os.path.dirname(path)
         if not os.path.exists(dirname):
             os.makedirs(dirname, 0755)
         with open(path, "w") as f:
             f.write(group)
     except Exception as e:
         logger.error("Failed to create group file marker: {}".format(e))
         raise
     logger.info("Successfully created group file marker for group {}".format(group))
Beispiel #23
0
 def execute(self):
     try:
         os.rename(self.params['move_src'], self.params['move_dst'])
     except Exception as e:
         logger.error('Failed to execute move path command: {} to {}: {}'.format(
             self.params['move_src'],
             self.params['move_dst'],
             e
         ))
     logger.info('Successfully performed move task: {} to {}'.format(
         self.params['move_src'],
         self.params['move_dst']
     ))
Beispiel #24
0
    def _apply_postprocessors(self):
        if self.watcher.exit_code == 0:
            return True

        logger.info('Checking success codes: command code {}, success codes {}'.format(
            self.command_code,
            self.success_codes,
        ))

        if self.success_codes and self.command_code in self.success_codes:
            return True

        return False
Beispiel #25
0
    def run(self, command, params, env=None, success_codes=None):
        logger.info('command to execute: {0}'.format(command))
        logger.info(
            'parameters supplied: {params}, env variables: {env}'.format(
                params=params,
                env=env,
            ))
        if isinstance(command, unicode):
            command = command.encode('utf-8')
        cmd = (shlex.split(command)
               if isinstance(command, basestring) else command)

        if params.get('task_id'):
            running_uid = self.try_find_running_subprocess(params['task_id'])
            if running_uid:
                running_sub = self.subprocesses[running_uid]
                logger.info(
                    'command execution is not required, '
                    'process for task {0} is already running: {1}'.format(
                        params['task_id'], running_sub.status()))
                return running_uid
        Subprocess = self.get_subprocess(cmd, params)
        uid = uuid.uuid4().hex
        sub = Subprocess(uid,
                         cmd,
                         params=params,
                         env=env,
                         success_codes=success_codes)
        sub.run()

        logger.info('command execution started successfully, pid: {0}'.format(
            sub.pid))
        self.subprocesses[uid] = sub
        return uid
Beispiel #26
0
    def run(self, command, params, env=None, success_codes=None):
        logger.info('command to execute: {0}'.format(command))
        logger.info('parameters supplied: {params}, env variables: {env}'.format(
            params=params,
            env=env,
        ))
        if isinstance(command, unicode):
            command = command.encode('utf-8')
        cmd = (shlex.split(command)
               if isinstance(command, basestring) else
               command)

        if params.get('task_id'):
            running_uid = self.try_find_running_subprocess(params['task_id'])
            if running_uid:
                running_sub = self.subprocesses[running_uid]
                logger.info(
                    'command execution is not required, process for task {} is already running: '
                    '{}'.format(
                        params['task_id'],
                        running_sub.status()
                    )
                )
                return running_uid
        Subprocess = self.get_subprocess(cmd, params)
        uid = uuid.uuid4().hex
        if issubclass(Subprocess, BaseSubprocess):
            sub = Subprocess(uid, cmd, params=params, env=env, success_codes=success_codes)
        else:
            sub = Subprocess(uid, params=params)
        sub.run()
        logger.info('command execution started successfully: {}'.format(sub))

        self.subprocesses[uid] = sub
        return uid
Beispiel #27
0
    def execute(self, subprocess, cmd=None, params=None):
        logger.info('Execute subprocess: {subprocess}, cmd={cmd}'.format(
            subprocess=subprocess,
            cmd=cmd,
        ))
        cmd = (shlex.split(cmd) if isinstance(cmd, basestring) else cmd)

        uid = uuid.uuid4().hex
        sub = subprocess(uid, cmd or [], params=params or {})
        sub.run()

        logger.info('Subprocess executed successfully')
        self.subprocesses[uid] = sub
        return uid
Beispiel #28
0
 def execute(self):
     group = str(int(self.params['group']))
     path = self.params['group_file'].format(group_id=group)
     try:
         if os.path.exists(path):
             os.rename(path, path + '.bak')
         dirname = os.path.dirname(path)
         if not os.path.exists(dirname):
             os.makedirs(dirname, 0755)
         with open(path, 'w') as f:
             f.write(group)
     except Exception:
         logger.exception('Failed to create group file')
         raise
     logger.info('Successfully created group file {} for group {}'.format(path, group))
Beispiel #29
0
    def _make_command(self, params):
        cmd_tpl = params['cmd_tpl']
        if 'backend_id' not in params:
            # need to get backend id from config by group id
            config_path = params['config_path']
            logger.info(
                'backend id not found in request params, '
                'getting backend id from elliptics config {config}, '
                'group {group}'.format(
                    config=config_path,
                    group=params['group'],
                )
            )
            config = EllipticsConfig(params['config_path'])
            params['backend_id'] = config.get_group_backend(int(params['group']))

        return shlex.split(cmd_tpl.format(**params))
Beispiel #30
0
 def create_group_file(self):
     if self.params.get('group_file'):
         try:
             group = str(int(self.params.get('group')))
             path = self.params.get('group_file')
             if os.path.exists(path):
                 os.rename(path, path + '.bak')
             dirname = os.path.dirname(path)
             if not os.path.exists(dirname):
                 os.makedirs(dirname, 0755)
             with open(path, 'w') as f:
                 f.write(group + '\n')
         except Exception as e:
             logger.exception('Failed to create group file: {0}'.format(e))
         else:
             logger.info('Successfully created group file '
                         'for group {0}'.format(group))
Beispiel #31
0
    def execute(self, subprocess, cmd=None, params=None):
        logger.info(
            'Execute subprocess: {subprocess}, cmd={cmd}'.format(
                subprocess=subprocess,
                cmd=cmd,
            )
        )
        cmd = (shlex.split(cmd)
               if isinstance(cmd, basestring) else
               cmd)

        uid = uuid.uuid4().hex
        sub = subprocess(uid, cmd or [], params=params or {})
        sub.run()

        logger.info('Subprocess executed successfully')
        self.subprocesses[uid] = sub
        return uid
Beispiel #32
0
 def update_broken_commands(self):
     s = Session()
     s.begin()
     try:
         for c in s.query(Command).filter_by(exit_code=None):
             if not self.pid_exists(c.pid):
                 c.progress = 1.0
                 c.exit_code = 666
                 c.finish_ts = int(time.time())
                 s.add(c)
                 logger.info('Command {0}, pid {1} is considered broken, '
                             'will be marked as finished'.format(
                                 c.uid, c.pid))
             else:
                 logger.warn('Command {0}, pid {1} is considered broken, '
                             'but process is running'.format(c.uid, c.pid))
         s.commit()
     except Exception as e:
         logger.exception('Failed to update broken commands')
         s.rollback()
         raise
Beispiel #33
0
    def ensure_exit_cb(self):

        def set_false_exit_code():
            logger.warn('pid {0}: executing false exit callback'.format(
                self.subprocess.pid))
            self.exit_cb_timeout = None
            if self.exit:
                return
            logger.warn('pid {0}: setting exit code to 999'.format(
                self.subprocess.pid))
            self.exit = True
            self.exit_code = 999
            self.progress = 1.0
            self.finish_ts = int(time.time())

            self.update_db_command()

        if self.exit:
            return
        logger.info('pid {0}: setting false exit callback'.format(
            self.subprocess.pid))
        self.exit_cb_timeout = self.subprocess.io_loop.add_timeout(
            datetime.timedelta(seconds=10), set_false_exit_code)
Beispiel #34
0
    def _ensure_exit_cb(self):

        def set_false_exit_code():
            logger.warn('pid {0}: executing false exit callback'.format(
                self.subprocess.pid))
            self._exit_cb_timeout = None
            if self._exit:
                return
            logger.warn('pid {0}: setting exit code to 999'.format(
                self.subprocess.pid))
            self._exit = True
            self.exit_code = 999
            self.progress = 1.0
            self.set_command_code()

            self.command.on_command_completed()

        if self._exit:
            return
        logger.info('pid {0}: setting false exit callback'.format(
            self.subprocess.pid))
        self._exit_cb_timeout = self.subprocess.io_loop.add_timeout(
            datetime.timedelta(seconds=10), set_false_exit_code)
Beispiel #35
0
    def collect_artifacts(self):

        commands_stats_path = self.params.get('commands_stats_path')
        if not commands_stats_path:
            logger.info('Commands stats path was not supplied')
            return {}

        logger.info('Parsing commands stats path: {}'.format(commands_stats_path))

        commands_stats = {}

        try:
            with open(commands_stats_path, 'rb') as f:
                commands_stats = json.load(f).get('commands', {})
        except Exception:
            logger.exception(
                'Failed to parse commands stats file {}'.format(commands_stats_path)
            )

        parsed_stats = self._parse_commands_stats(commands_stats)

        # NOTE: temporary backward compatibility
        self.commands_stats = parsed_stats
        return parsed_stats
Beispiel #36
0
    def execute(self):
        group_base_path_root_dir = self.params['group_base_path_root_dir'].rstrip('/')

        basename = self.get_vacant_basename(group_base_path_root_dir)
        tmp_basename = self.tmp_basename(basename)

        tmp_dir = os.path.join(group_base_path_root_dir, tmp_basename)
        logger.info('Creating tmp dir for new group: {}'.format(tmp_dir))
        try:
            os.mkdir(tmp_dir, 0755)
        except Exception:
            logger.exception('Failed to create tmp dir for new group')
            raise

        logger.info('Adding group files')
        for filename, body in self.params['files'].iteritems():
            logger.info('Adding file {}'.format(filename))
            filename = os.path.join(
                tmp_dir,
                filename
            )
            dirname, basefname = os.path.split(filename)
            if not os.path.exists(dirname):
                os.makedirs(dirname)
            with open(filename, 'wb') as f:
                f.write(body)

        dest_dir = os.path.join(group_base_path_root_dir, basename)
        logger.info(
            'Renaming tmp dir {tmp_dir} to destination dir {dest_dir}'.format(
                tmp_dir=tmp_dir,
                dest_dir=dest_dir,
            )
        )
        try:
            os.rename(tmp_dir, dest_dir)
        except Exception:
            logger.exception('Failed to rename tmp dir to dest dir')
            raise
Beispiel #37
0
 def execute(self):
     ids_file = self.params['ids']
     logger.info('Generating ids file {}'.format(ids_file))
     if os.path.exists(ids_file):
         logger.info('Ids file {} already exists'.format(ids_file))
     else:
         try:
             with open(ids_file, 'wb') as f:
                 f.write(os.urandom(64))
         except Exception:
             logger.exception('Failed to create ids file {}'.format(
                 ids_file)
             )
             raise
     logger.info('Successfully created ids file {}'.format(ids_file))
Beispiel #38
0
    def exit_cb(self, code):
        self.exit = True
        logger.info('pid {0}: exit callback'.format(self.subprocess.pid))
        if self.exit_cb_timeout:
            logger.info('pid {0}: removing false exit callback'.format(
                self.subprocess.pid))
            self.subprocess.io_loop.remove_timeout(self.exit_cb_timeout)
            self.exit_cb_timeout = None
        self.exit_code = code
        self.progress = 1.0
        self.finish_ts = int(time.time())
        self.set_command_code()

        logger.info('pid {0}: exit code {1}, command code {2}'.format(
            self.subprocess.pid, self.exit_code, self.command_code))

        self.update_db_command()

        if self.success_cb:
            if self.exit_code == 0 or self.command_code in self.success_codes:
                logger.info('pid {0}: executing success callback'.format(
                    self.subprocess.pid))
                self.success_cb()
Beispiel #39
0
    def _exit_cb(self, code):
        self._exit = True
        logger.info('pid {0}: exit callback'.format(self.subprocess.pid))
        if self._exit_cb_timeout:
            logger.info('pid {0}: removing false exit callback'.format(
                self.subprocess.pid))
            self.subprocess.io_loop.remove_timeout(self._exit_cb_timeout)
            self._exit_cb_timeout = None
        self.exit_code = code
        self.progress = 1.0
        self.set_command_code()

        logger.info('pid {0}: exit code {1}, command code {2}'.format(
            self.subprocess.pid, self.exit_code, self.command_code))

        self.command.on_command_completed()
Beispiel #40
0
    def execute(self):
        group_base_path_root_dir = self.params[
            'group_base_path_root_dir'].rstrip('/')

        basename = self.get_vacant_basename(group_base_path_root_dir)
        tmp_basename = self.tmp_basename(basename)

        tmp_dir = os.path.join(group_base_path_root_dir, tmp_basename)
        logger.info('Creating tmp dir for new group: {}'.format(tmp_dir))
        try:
            os.mkdir(tmp_dir, 0755)
        except Exception:
            logger.exception('Failed to create tmp dir for new group')
            raise

        logger.info('Adding group files')
        for filename, body in self.params['files'].iteritems():
            logger.info('Adding file {}'.format(filename))
            filename = os.path.join(tmp_dir, filename)
            dirname, basefname = os.path.split(filename)
            if not os.path.exists(dirname):
                os.makedirs(dirname)
            with open(filename, 'wb') as f:
                f.write(body)

        dest_dir = os.path.join(group_base_path_root_dir, basename)
        logger.info(
            'Renaming tmp dir {tmp_dir} to destination dir {dest_dir}'.format(
                tmp_dir=tmp_dir,
                dest_dir=dest_dir,
            ))
        try:
            os.rename(tmp_dir, dest_dir)
        except Exception:
            logger.exception('Failed to rename tmp dir to dest dir')
            raise
Beispiel #41
0
    def create_group_file(self):
        if self.params.get('group_file'):
            try:
                group = str(int(self.params.get('group')))
                path = self.params['group_file'].format(group_id=group)
                if os.path.exists(path):
                    os.rename(path, path + '.bak')
                dirname = os.path.dirname(path)
                if not os.path.exists(dirname):
                    os.makedirs(dirname, 0755)
                with open(path, 'w') as f:
                    f.write(group)
            except Exception as e:
                logger.exception('Failed to create group file: {0}'.format(e))
            else:
                logger.info('Successfully created group file '
                            'for group {0}'.format(group))
        else:
            logger.info('Group file creation was not requested for '
                        'group {0}'.format(self.params.get('group')))

        if self.params.get('remove_path'):
            try:
                shutil.rmtree(self.params['remove_path'])
            except Exception as e:
                logger.exception('Failed to remove path {0}: {1}'.format(
                    self.params['remove_path'], e))
            else:
                logger.info('Successfully removed path {0} '
                            'for group {1}'.format(self.params['remove_path'],
                                                   group))
        else:
            logger.info('Path removal was not requested for '
                        'group {0}'.format(self.params.get('group')))

        if self.params.get('ids'):
            ids_file = self.params['ids']
            logger.info('Generating ids file {} required'.format(ids_file))
            if os.path.exists(ids_file):
                logger.info('Ids file {} already exists'.format(ids_file))
            else:
                try:
                    with open(ids_file, 'wb') as f:
                        f.write(os.urandom(64))
                except Exception as e:
                    logger.exception(
                        'Failed to create ids file {}'.format(ids_file))
                else:
                    logger.info('Successfully created ids file {} '
                                'for group {1}'.format(ids_file, group))
Beispiel #42
0
    def create_group_file_marker(self):
        if self.params.get('group_file_marker'):
            try:
                group = (str(int(self.params.get('group')))
                         if self.params.get('group') else '')
                path = self.params['group_file_marker'].format(group_id=group)
                dirname = os.path.dirname(path)
                if not os.path.exists(dirname):
                    os.makedirs(dirname, 0755)
                with open(path, 'w') as f:
                    f.write(group)
            except Exception as e:
                logger.exception(
                    'Failed to create group file marker: {0}'.format(e))
            else:
                logger.info('Successfully created group file marker '
                            'for group {0}'.format(group))
        else:
            logger.info('Group file marker creation was not requested for '
                        'group {0}'.format(self.params.get('group')))

        if self.params.get('remove_group_file'):
            logger.info('Removing group file {0}'.format(
                self.params['remove_group_file']))
            try:
                os.remove(self.params['remove_group_file'])
            except Exception as e:
                logger.exception('Failed to remove group file: {0}'.format(e))
            else:
                logger.info('Successfully removed group '
                            'file {0}'.format(
                                self.params['remove_group_file']))
        else:
            logger.info('Group file removal was not requested for '
                        'group {0}'.format(self.params.get('group')))

        if self.params.get('move_dst') and self.params.get('move_src'):
            try:
                os.rename(self.params['move_src'], self.params['move_dst'])
            except Exception as e:
                logger.exception(
                    'Failed to execute move task: {0} to {1}'.format(
                        self.params['move_src'], self.params['move_dst']))
            else:
                logger.info(
                    'Successfully performed move task: {0} to {1}'.format(
                        self.params['move_src'], self.params['move_dst']))

        if self.params.get('mark_backend'):
            marker = self.params['mark_backend']
            try:
                open(marker, 'w').close()
            except Exception as e:
                logger.error(
                    'Failed to create backend down marker: {0}'.format(e))
            else:
                logger.info(
                    'Successfully created backend down marker: {0}'.format(
                        self.params['mark_backend']))

        if self.params.get('unmark_backend'):
            marker = self.params['unmark_backend']
            try:
                os.remove(self.params['unmark_backend'])
            except Exception as e:
                logger.error(
                    'Failed to remove backend down marker: {0}'.format(e))
            else:
                logger.info(
                    'Successfully removed backend down marker: {0}'.format(
                        self.params['unmark_backend']))