def collect_artifacts(self): tmp_dir = None for i, cmd_part in enumerate(self.cmd): if cmd_part in ('--tmp', '-t'): tmp_dir = self.cmd[i + 1] break if not tmp_dir: logger.info('Failed to determine tmp directory') return {} exec_state_path = os.path.join(tmp_dir, 'exec_state') logger.info('Parsing exec state: {}'.format(exec_state_path)) exec_state = {} try: with open(exec_state_path, 'rb') as f: exec_state = json.load(f).get('status', {}) except Exception: logger.exception( 'Failed to parse exec state file {}'.format(exec_state_path) ) pass return exec_state
def execute(self): group_base_path = self.params['group_base_path'].rstrip('/') if not os.path.exists(group_base_path): raise RuntimeError( 'Group dir {path} does not exist'.format( path=group_base_path, ) ) dst_base_path, basename = os.path.split(group_base_path) remove_path = os.path.join( dst_base_path, self.removed_basename(basename) ) logger.info( 'Renaming group base dir {tmp_dir} to destination dir {dest_dir}'.format( tmp_dir=group_base_path, dest_dir=remove_path, ) ) try: os.rename(group_base_path, remove_path) except OSError as e: if e.errno == 2: # errno == 2: No such file or directory if os.path.exists(remove_path): # group_base_path was already renamed, not an error pass else: raise except Exception: logger.exception('Failed to rename tmp dir to dest dir') raise
def _prepare_command(self, cmd): cmd_str = ' '.join(cmd) if 'cmd_tpl' in self.params: # TODO: remove this backward-compatibility hack cmd_str = self.params['cmd_tpl'] missing_parameters = self.PLACEHOLDERS_TPL.findall(cmd_str) for param in missing_parameters: if param in self.params: continue elif param == 'backend_id': if 'group' not in self.params: raise ValueError('Parameter "group" is required') config_path = self.params['config_path'] logger.info( 'backend id not found in request params, ' 'getting backend id from elliptics config {config}, ' 'group {group}'.format( config=config_path, group=self.params['group'], ) ) config = EllipticsConfig(self.params['config_path']) self.params['backend_id'] = config.get_group_backend(int(self.params['group'])) else: raise ValueError('Cannot process command: unknown parameter "{}"'.format(param)) cmd_str = cmd_str.format(**self.params) logger.info('Prepared command: {}'.format(cmd_str)) return shlex.split(cmd_str)
def execute(self): group_base_path = self.params['group_base_path'].rstrip('/') if not os.path.exists(group_base_path): raise RuntimeError('Group dir {path} does not exist'.format( path=group_base_path, )) dst_base_path, basename = os.path.split(group_base_path) remove_path = os.path.join(dst_base_path, self.removed_basename(basename)) logger.info( 'Renaming group base dir {tmp_dir} to destination dir {dest_dir}'. format( tmp_dir=group_base_path, dest_dir=remove_path, )) try: os.rename(group_base_path, remove_path) except OSError as e: if e.errno == 2: # errno == 2: No such file or directory if os.path.exists(remove_path): # group_base_path was already renamed, not an error pass else: raise except Exception: logger.exception('Failed to rename tmp dir to dest dir') raise
def is_success(self): logger.info('pid {0}: exit code {1}'.format(self.subprocess.pid, self.exit_code)) if self.exit_code == 0: return True output = ''.join(self.output) try: data = json.loads(output) except Exception as e: logger.error('pid {0}: failed to parse output json: {1}'.format( self.subprocess.pid, output)) return False if not 'error' in data: logger.error('pid {0}: no "error" key in response data'.format( self.subprocess.pid)) return False if not 'code' in data['error']: logger.error( 'pid {0}: no "code" key in response error data'.format( self.subprocess.pid)) return False logger.info('pid {0}: operation error code {1}'.format( self.subprocess.pid, data['error']['code'])) return data['error']['code'] in self.success_codes
def _parse_command_code(self): if self.watcher.exit_code == 0: return self.watcher.exit_code output = self.watcher.get_stdout() try: data = json.loads(output) except Exception: logger.error('pid {}: failed to parse output json: {}'.format( self.pid, output )) return self.watcher.exit_code if 'error' not in data: logger.error('pid {}: no "error" key in response data'.format( self.pid )) return self.watcher.exit_code if 'code' not in data['error']: logger.error('pid {}: no "code" key in response error data'.format( self.pid )) return self.watcher.exit_code logger.info('pid {}: operation error code {}'.format( self.pid, data['error']['code'] )) return data['error']['code']
def is_success(self): logger.info('pid {0}: exit code {1}'.format( self.subprocess.pid, self.exit_code)) if self.exit_code == 0: return True output = ''.join(self.output) try: data = json.loads(output) except Exception as e: logger.error('pid {0}: failed to parse output json: {1}'.format( self.subprocess.pid, output)) return False if not 'error' in data: logger.error('pid {0}: no "error" key in response data'.format( self.subprocess.pid)) return False if not 'code' in data['error']: logger.error('pid {0}: no "code" key in response error data'.format( self.subprocess.pid)) return False logger.info('pid {0}: operation error code {1}'.format( self.subprocess.pid, data['error']['code'])) return data['error']['code'] in self.success_codes
def on_command_completed(self): self.finish_ts = int(time.time()) self.artifacts = self.collect_artifacts() self.on_update_progress() if not self._apply_postprocessors(): # TODO: add status codes logger.info('Command failed, no post processors will be applied') return for post_processor in self.POST_PROCESSORS: params_supplied = all( param in self.params for param in post_processor.REQUIRED_PARAMS ) if params_supplied: # TODO: replace by required params? possibly not logger.info('Running post processor {}'.format(post_processor.__name__)) uid = uuid.uuid4().hex command = post_processor(uid, params=self.params) try: # NOTE: when running as a post processor command is not # dumped to database, therefore 'execute' method is called # instead of 'run' command.execute() except: logger.exception('Post processor {} failed, skipped'.format( post_processor.__name__ )) continue
def execute(self): try: shutil.rmtree(self.params['remove_path']) except Exception: logger.exception('Failed to remove path {}'.format(self.params['remove_path'])) raise logger.info('Successfully removed path {}'.format(self.params['remove_path']))
def update_broken_commands(self): s = Session() s.begin() try: for c in s.query(Command).filter_by(exit_code=None): if not self.pid_exists(c.pid): c.progress = 1.0 c.exit_code = 666 c.finish_ts = int(time.time()) s.add(c) logger.info( 'Command {}, pid {} is considered broken, will be marked as ' 'finished'.format( c.uid, c.pid ) ) else: logger.warn( 'Command {}, pid {} is considered broken, but process is running'.format( c.uid, c.pid ) ) s.commit() except Exception: logger.exception('Failed to update broken commands') s.rollback() raise
def execute(self): marker = self.params["unmark_backend"] try: os.remove(marker) except Exception as e: logger.error("Failed to remove backend marker: {}".format(e)) raise logger.info("Successfully removed backend marker: {}".format(marker))
def execute(self): logger.info('Removing group file {0}'.format(self.params['remove_group_file'])) try: os.remove(self.params['remove_group_file']) except Exception as e: logger.error('Failed to remove group file: {}'.format(e)) raise logger.info('Successfully removed group file {}'.format(self.params['remove_group_file']))
def execute(self): stop_file = self.params['create_stop_file'] try: open(stop_file, 'w').close() except Exception as e: logger.error('Failed to create backend stop marker: {}'.format(e)) raise logger.info('Successfully created backend stop marker: {}'.format(stop_file))
def execute(self): stop_file = self.params["remove_stop_file"] try: os.remove(stop_file) except Exception as e: logger.error("Failed to remove backend stop marker: {}".format(e)) raise logger.info("Successfully removed backend stop marker: {}".format(stop_file))
def execute(self): marker = self.params['mark_backend'] try: open(marker, 'w').close() except Exception as e: logger.error('Failed to create backend marker: {}'.format(e)) raise logger.info('Successfully created backend marker: {}'.format(marker))
def setup_session(uri): engine = sqlalchemy.create_engine(uri) try: engine.connect() logger.info('Successfully connected to db by uri {0}'.format(uri)) except Exception as e: logger.exception('Failed to connect to db engine by uri {0}'.format(uri)) raise return sessionmaker(bind=engine, autocommit=True)
def setup_session(uri): engine = sqlalchemy.create_engine(uri) try: engine.connect() logger.info('Successfully connected to db by uri {0}'.format(uri)) except Exception as e: logger.exception( 'Failed to connect to db engine by uri {0}'.format(uri)) raise return sessionmaker(bind=engine, autocommit=True)
def terminate(self, uid): if not uid in self.subprocesses: raise ValueError('Unknown command uid: {0}'.format(uid)) logger.info('terminating command {0}, pid: {1}'.format( uid, self.subprocesses[uid].process.pid)) # result, error, sub = self.subprocesses[uid].terminate().result() code = self.subprocesses[uid].terminate() if code: raise RuntimeError('Failed to terminate command {0}, ' 'exit code: {1}'.format(uid, code))
def _parse_commands_stats(self, commands_stats): op_statuses_count = {} for operation_status, count in commands_stats.iteritems(): operation, status = operation_status.split('.', 1) statuses_count = op_statuses_count.setdefault(operation, {}) statuses_count.setdefault(status, 0) statuses_count[status] += count logger.info('Parsed command statuses: {}'.format(op_statuses_count)) return op_statuses_count
def __init__(self, uid, cmd=None, params=None, env=None, success_codes=None, io_loop=IOLoop.instance()): cmd = self._make_command(params) logger.info('Constructed command: {}'.format(cmd)) super(DnetClientCmd, self).__init__( uid, cmd, params=params, env=env, success_codes=success_codes, io_loop=io_loop )
def terminate(self, uid): if uid not in self.subprocesses: raise ValueError('Unknown command uid: {0}'.format(uid)) logger.info( 'terminating command {}, pid: {}'.format(uid, self.subprocesses[uid].process.pid) ) # result, error, sub = self.subprocesses[uid].terminate().result() code = self.subprocesses[uid].terminate() if code: raise RuntimeError('Failed to terminate command {}, exit code: {}'.format(uid, code))
def execute(self): group = str(self.params.get("group")) if "group" in self.params.get("group") else "" try: path = self.params["group_file_marker"].format(group_id=group) dirname = os.path.dirname(path) if not os.path.exists(dirname): os.makedirs(dirname, 0755) with open(path, "w") as f: f.write(group) except Exception as e: logger.error("Failed to create group file marker: {}".format(e)) raise logger.info("Successfully created group file marker for group {}".format(group))
def execute(self): try: os.rename(self.params['move_src'], self.params['move_dst']) except Exception as e: logger.error('Failed to execute move path command: {} to {}: {}'.format( self.params['move_src'], self.params['move_dst'], e )) logger.info('Successfully performed move task: {} to {}'.format( self.params['move_src'], self.params['move_dst'] ))
def _apply_postprocessors(self): if self.watcher.exit_code == 0: return True logger.info('Checking success codes: command code {}, success codes {}'.format( self.command_code, self.success_codes, )) if self.success_codes and self.command_code in self.success_codes: return True return False
def run(self, command, params, env=None, success_codes=None): logger.info('command to execute: {0}'.format(command)) logger.info( 'parameters supplied: {params}, env variables: {env}'.format( params=params, env=env, )) if isinstance(command, unicode): command = command.encode('utf-8') cmd = (shlex.split(command) if isinstance(command, basestring) else command) if params.get('task_id'): running_uid = self.try_find_running_subprocess(params['task_id']) if running_uid: running_sub = self.subprocesses[running_uid] logger.info( 'command execution is not required, ' 'process for task {0} is already running: {1}'.format( params['task_id'], running_sub.status())) return running_uid Subprocess = self.get_subprocess(cmd, params) uid = uuid.uuid4().hex sub = Subprocess(uid, cmd, params=params, env=env, success_codes=success_codes) sub.run() logger.info('command execution started successfully, pid: {0}'.format( sub.pid)) self.subprocesses[uid] = sub return uid
def run(self, command, params, env=None, success_codes=None): logger.info('command to execute: {0}'.format(command)) logger.info('parameters supplied: {params}, env variables: {env}'.format( params=params, env=env, )) if isinstance(command, unicode): command = command.encode('utf-8') cmd = (shlex.split(command) if isinstance(command, basestring) else command) if params.get('task_id'): running_uid = self.try_find_running_subprocess(params['task_id']) if running_uid: running_sub = self.subprocesses[running_uid] logger.info( 'command execution is not required, process for task {} is already running: ' '{}'.format( params['task_id'], running_sub.status() ) ) return running_uid Subprocess = self.get_subprocess(cmd, params) uid = uuid.uuid4().hex if issubclass(Subprocess, BaseSubprocess): sub = Subprocess(uid, cmd, params=params, env=env, success_codes=success_codes) else: sub = Subprocess(uid, params=params) sub.run() logger.info('command execution started successfully: {}'.format(sub)) self.subprocesses[uid] = sub return uid
def execute(self, subprocess, cmd=None, params=None): logger.info('Execute subprocess: {subprocess}, cmd={cmd}'.format( subprocess=subprocess, cmd=cmd, )) cmd = (shlex.split(cmd) if isinstance(cmd, basestring) else cmd) uid = uuid.uuid4().hex sub = subprocess(uid, cmd or [], params=params or {}) sub.run() logger.info('Subprocess executed successfully') self.subprocesses[uid] = sub return uid
def execute(self): group = str(int(self.params['group'])) path = self.params['group_file'].format(group_id=group) try: if os.path.exists(path): os.rename(path, path + '.bak') dirname = os.path.dirname(path) if not os.path.exists(dirname): os.makedirs(dirname, 0755) with open(path, 'w') as f: f.write(group) except Exception: logger.exception('Failed to create group file') raise logger.info('Successfully created group file {} for group {}'.format(path, group))
def _make_command(self, params): cmd_tpl = params['cmd_tpl'] if 'backend_id' not in params: # need to get backend id from config by group id config_path = params['config_path'] logger.info( 'backend id not found in request params, ' 'getting backend id from elliptics config {config}, ' 'group {group}'.format( config=config_path, group=params['group'], ) ) config = EllipticsConfig(params['config_path']) params['backend_id'] = config.get_group_backend(int(params['group'])) return shlex.split(cmd_tpl.format(**params))
def create_group_file(self): if self.params.get('group_file'): try: group = str(int(self.params.get('group'))) path = self.params.get('group_file') if os.path.exists(path): os.rename(path, path + '.bak') dirname = os.path.dirname(path) if not os.path.exists(dirname): os.makedirs(dirname, 0755) with open(path, 'w') as f: f.write(group + '\n') except Exception as e: logger.exception('Failed to create group file: {0}'.format(e)) else: logger.info('Successfully created group file ' 'for group {0}'.format(group))
def execute(self, subprocess, cmd=None, params=None): logger.info( 'Execute subprocess: {subprocess}, cmd={cmd}'.format( subprocess=subprocess, cmd=cmd, ) ) cmd = (shlex.split(cmd) if isinstance(cmd, basestring) else cmd) uid = uuid.uuid4().hex sub = subprocess(uid, cmd or [], params=params or {}) sub.run() logger.info('Subprocess executed successfully') self.subprocesses[uid] = sub return uid
def update_broken_commands(self): s = Session() s.begin() try: for c in s.query(Command).filter_by(exit_code=None): if not self.pid_exists(c.pid): c.progress = 1.0 c.exit_code = 666 c.finish_ts = int(time.time()) s.add(c) logger.info('Command {0}, pid {1} is considered broken, ' 'will be marked as finished'.format( c.uid, c.pid)) else: logger.warn('Command {0}, pid {1} is considered broken, ' 'but process is running'.format(c.uid, c.pid)) s.commit() except Exception as e: logger.exception('Failed to update broken commands') s.rollback() raise
def ensure_exit_cb(self): def set_false_exit_code(): logger.warn('pid {0}: executing false exit callback'.format( self.subprocess.pid)) self.exit_cb_timeout = None if self.exit: return logger.warn('pid {0}: setting exit code to 999'.format( self.subprocess.pid)) self.exit = True self.exit_code = 999 self.progress = 1.0 self.finish_ts = int(time.time()) self.update_db_command() if self.exit: return logger.info('pid {0}: setting false exit callback'.format( self.subprocess.pid)) self.exit_cb_timeout = self.subprocess.io_loop.add_timeout( datetime.timedelta(seconds=10), set_false_exit_code)
def _ensure_exit_cb(self): def set_false_exit_code(): logger.warn('pid {0}: executing false exit callback'.format( self.subprocess.pid)) self._exit_cb_timeout = None if self._exit: return logger.warn('pid {0}: setting exit code to 999'.format( self.subprocess.pid)) self._exit = True self.exit_code = 999 self.progress = 1.0 self.set_command_code() self.command.on_command_completed() if self._exit: return logger.info('pid {0}: setting false exit callback'.format( self.subprocess.pid)) self._exit_cb_timeout = self.subprocess.io_loop.add_timeout( datetime.timedelta(seconds=10), set_false_exit_code)
def collect_artifacts(self): commands_stats_path = self.params.get('commands_stats_path') if not commands_stats_path: logger.info('Commands stats path was not supplied') return {} logger.info('Parsing commands stats path: {}'.format(commands_stats_path)) commands_stats = {} try: with open(commands_stats_path, 'rb') as f: commands_stats = json.load(f).get('commands', {}) except Exception: logger.exception( 'Failed to parse commands stats file {}'.format(commands_stats_path) ) parsed_stats = self._parse_commands_stats(commands_stats) # NOTE: temporary backward compatibility self.commands_stats = parsed_stats return parsed_stats
def execute(self): group_base_path_root_dir = self.params['group_base_path_root_dir'].rstrip('/') basename = self.get_vacant_basename(group_base_path_root_dir) tmp_basename = self.tmp_basename(basename) tmp_dir = os.path.join(group_base_path_root_dir, tmp_basename) logger.info('Creating tmp dir for new group: {}'.format(tmp_dir)) try: os.mkdir(tmp_dir, 0755) except Exception: logger.exception('Failed to create tmp dir for new group') raise logger.info('Adding group files') for filename, body in self.params['files'].iteritems(): logger.info('Adding file {}'.format(filename)) filename = os.path.join( tmp_dir, filename ) dirname, basefname = os.path.split(filename) if not os.path.exists(dirname): os.makedirs(dirname) with open(filename, 'wb') as f: f.write(body) dest_dir = os.path.join(group_base_path_root_dir, basename) logger.info( 'Renaming tmp dir {tmp_dir} to destination dir {dest_dir}'.format( tmp_dir=tmp_dir, dest_dir=dest_dir, ) ) try: os.rename(tmp_dir, dest_dir) except Exception: logger.exception('Failed to rename tmp dir to dest dir') raise
def execute(self): ids_file = self.params['ids'] logger.info('Generating ids file {}'.format(ids_file)) if os.path.exists(ids_file): logger.info('Ids file {} already exists'.format(ids_file)) else: try: with open(ids_file, 'wb') as f: f.write(os.urandom(64)) except Exception: logger.exception('Failed to create ids file {}'.format( ids_file) ) raise logger.info('Successfully created ids file {}'.format(ids_file))
def exit_cb(self, code): self.exit = True logger.info('pid {0}: exit callback'.format(self.subprocess.pid)) if self.exit_cb_timeout: logger.info('pid {0}: removing false exit callback'.format( self.subprocess.pid)) self.subprocess.io_loop.remove_timeout(self.exit_cb_timeout) self.exit_cb_timeout = None self.exit_code = code self.progress = 1.0 self.finish_ts = int(time.time()) self.set_command_code() logger.info('pid {0}: exit code {1}, command code {2}'.format( self.subprocess.pid, self.exit_code, self.command_code)) self.update_db_command() if self.success_cb: if self.exit_code == 0 or self.command_code in self.success_codes: logger.info('pid {0}: executing success callback'.format( self.subprocess.pid)) self.success_cb()
def _exit_cb(self, code): self._exit = True logger.info('pid {0}: exit callback'.format(self.subprocess.pid)) if self._exit_cb_timeout: logger.info('pid {0}: removing false exit callback'.format( self.subprocess.pid)) self.subprocess.io_loop.remove_timeout(self._exit_cb_timeout) self._exit_cb_timeout = None self.exit_code = code self.progress = 1.0 self.set_command_code() logger.info('pid {0}: exit code {1}, command code {2}'.format( self.subprocess.pid, self.exit_code, self.command_code)) self.command.on_command_completed()
def execute(self): group_base_path_root_dir = self.params[ 'group_base_path_root_dir'].rstrip('/') basename = self.get_vacant_basename(group_base_path_root_dir) tmp_basename = self.tmp_basename(basename) tmp_dir = os.path.join(group_base_path_root_dir, tmp_basename) logger.info('Creating tmp dir for new group: {}'.format(tmp_dir)) try: os.mkdir(tmp_dir, 0755) except Exception: logger.exception('Failed to create tmp dir for new group') raise logger.info('Adding group files') for filename, body in self.params['files'].iteritems(): logger.info('Adding file {}'.format(filename)) filename = os.path.join(tmp_dir, filename) dirname, basefname = os.path.split(filename) if not os.path.exists(dirname): os.makedirs(dirname) with open(filename, 'wb') as f: f.write(body) dest_dir = os.path.join(group_base_path_root_dir, basename) logger.info( 'Renaming tmp dir {tmp_dir} to destination dir {dest_dir}'.format( tmp_dir=tmp_dir, dest_dir=dest_dir, )) try: os.rename(tmp_dir, dest_dir) except Exception: logger.exception('Failed to rename tmp dir to dest dir') raise
def create_group_file(self): if self.params.get('group_file'): try: group = str(int(self.params.get('group'))) path = self.params['group_file'].format(group_id=group) if os.path.exists(path): os.rename(path, path + '.bak') dirname = os.path.dirname(path) if not os.path.exists(dirname): os.makedirs(dirname, 0755) with open(path, 'w') as f: f.write(group) except Exception as e: logger.exception('Failed to create group file: {0}'.format(e)) else: logger.info('Successfully created group file ' 'for group {0}'.format(group)) else: logger.info('Group file creation was not requested for ' 'group {0}'.format(self.params.get('group'))) if self.params.get('remove_path'): try: shutil.rmtree(self.params['remove_path']) except Exception as e: logger.exception('Failed to remove path {0}: {1}'.format( self.params['remove_path'], e)) else: logger.info('Successfully removed path {0} ' 'for group {1}'.format(self.params['remove_path'], group)) else: logger.info('Path removal was not requested for ' 'group {0}'.format(self.params.get('group'))) if self.params.get('ids'): ids_file = self.params['ids'] logger.info('Generating ids file {} required'.format(ids_file)) if os.path.exists(ids_file): logger.info('Ids file {} already exists'.format(ids_file)) else: try: with open(ids_file, 'wb') as f: f.write(os.urandom(64)) except Exception as e: logger.exception( 'Failed to create ids file {}'.format(ids_file)) else: logger.info('Successfully created ids file {} ' 'for group {1}'.format(ids_file, group))
def create_group_file_marker(self): if self.params.get('group_file_marker'): try: group = (str(int(self.params.get('group'))) if self.params.get('group') else '') path = self.params['group_file_marker'].format(group_id=group) dirname = os.path.dirname(path) if not os.path.exists(dirname): os.makedirs(dirname, 0755) with open(path, 'w') as f: f.write(group) except Exception as e: logger.exception( 'Failed to create group file marker: {0}'.format(e)) else: logger.info('Successfully created group file marker ' 'for group {0}'.format(group)) else: logger.info('Group file marker creation was not requested for ' 'group {0}'.format(self.params.get('group'))) if self.params.get('remove_group_file'): logger.info('Removing group file {0}'.format( self.params['remove_group_file'])) try: os.remove(self.params['remove_group_file']) except Exception as e: logger.exception('Failed to remove group file: {0}'.format(e)) else: logger.info('Successfully removed group ' 'file {0}'.format( self.params['remove_group_file'])) else: logger.info('Group file removal was not requested for ' 'group {0}'.format(self.params.get('group'))) if self.params.get('move_dst') and self.params.get('move_src'): try: os.rename(self.params['move_src'], self.params['move_dst']) except Exception as e: logger.exception( 'Failed to execute move task: {0} to {1}'.format( self.params['move_src'], self.params['move_dst'])) else: logger.info( 'Successfully performed move task: {0} to {1}'.format( self.params['move_src'], self.params['move_dst'])) if self.params.get('mark_backend'): marker = self.params['mark_backend'] try: open(marker, 'w').close() except Exception as e: logger.error( 'Failed to create backend down marker: {0}'.format(e)) else: logger.info( 'Successfully created backend down marker: {0}'.format( self.params['mark_backend'])) if self.params.get('unmark_backend'): marker = self.params['unmark_backend'] try: os.remove(self.params['unmark_backend']) except Exception as e: logger.error( 'Failed to remove backend down marker: {0}'.format(e)) else: logger.info( 'Successfully removed backend down marker: {0}'.format( self.params['unmark_backend']))