def on_command_completed(self):
self.finish_ts = int(time.time())
self.artifacts = self.collect_artifacts()
self.on_update_progress()
if not self._apply_postprocessors():
# TODO: add status codes
cmd_logger.info('Command failed, no post processors will be applied', extra=self.log_extra)
return
for post_processor in self.POST_PROCESSORS:
params_supplied = all(
param in self.params
for param in post_processor.REQUIRED_PARAMS
)
if params_supplied:
# TODO: replace by required params? possibly not
cmd_logger.info('Running post processor {}'.format(post_processor.__name__), extra=self.log_extra)
uid = uuid.uuid4().hex
command = post_processor(uid, params=self.params)
try:
# NOTE: when running as a post processor command is not
# dumped to database, therefore 'execute' method is called
# instead of 'run'
command.execute()
except:
cmd_logger.exception(
'Post processor {} failed, skipped'.format(
post_processor.__name__
),
extra=self.log_extra,
)
continue
def collect_artifacts(self):
tmp_dir = None
for i, cmd_part in enumerate(self.cmd):
if cmd_part in ('--tmp', '-t'):
tmp_dir = self.cmd[i + 1]
break
if not tmp_dir:
cmd_logger.info('Failed to determine tmp directory', extra=self.log_extra)
return {}
exec_state_path = os.path.join(tmp_dir, 'exec_state')
cmd_logger.info('Parsing exec state: {}'.format(exec_state_path), extra=self.log_extra)
exec_state = {}
try:
with open(exec_state_path, 'rb') as f:
exec_state = json.load(f).get('status', {})
except Exception:
cmd_logger.exception(
'Failed to parse exec state file {}'.format(exec_state_path),
extra=self.log_extra,
)
pass
return exec_state
def execute(self):
try:
shutil.rmtree(self.params['remove_path'])
except Exception:
cmd_logger.exception('Failed to remove path {}'.format(self.params['remove_path']), extra=self.log_extra)
raise
cmd_logger.info('Successfully removed path {}'.format(self.params['remove_path']), extra=self.log_extra)
def execute(self):
group_base_path = self.params['group_base_path'].rstrip('/')
if not os.path.exists(group_base_path):
raise RuntimeError('Group dir {path} does not exist'.format(
path=group_base_path, ))
dst_base_path, basename = os.path.split(group_base_path)
remove_path = os.path.join(dst_base_path,
self.removed_basename(basename))
cmd_logger.info(
'Renaming group base dir {tmp_dir} to destination dir {dest_dir}'.
format(
tmp_dir=group_base_path,
dest_dir=remove_path,
),
extra=self.log_extra,
)
try:
os.rename(group_base_path, remove_path)
except OSError as e:
if e.errno == 2:
# errno == 2: No such file or directory
if os.path.exists(remove_path):
# group_base_path was already renamed, not an error
pass
else:
raise
else:
raise
except Exception:
cmd_logger.exception('Failed to rename tmp dir to dest dir',
extra=self.log_extra)
raise
def collect_artifacts(self):
commands_stats_path = self.params.get('commands_stats_path')
if not commands_stats_path:
cmd_logger.info('Commands stats path was not supplied', extra=self.log_extra)
return {}
cmd_logger.info('Parsing commands stats path: {}'.format(commands_stats_path), extra=self.log_extra)
commands_stats = {}
try:
with open(commands_stats_path, 'rb') as f:
commands_stats = json.load(f).get('commands', {})
except Exception:
cmd_logger.exception(
'Failed to parse commands stats file {}'.format(commands_stats_path),
extra=self.log_extra,
)
parsed_stats = self._parse_commands_stats(commands_stats)
# NOTE: temporary backward compatibility
self.commands_stats = parsed_stats
return parsed_stats
def collect_artifacts(self):
commands_stats_path = self.params.get('commands_stats_path')
if not commands_stats_path:
cmd_logger.info('Commands stats path was not supplied',
extra=self.log_extra)
return {}
cmd_logger.info(
'Parsing commands stats path: {}'.format(commands_stats_path),
extra=self.log_extra)
commands_stats = {}
try:
with open(commands_stats_path, 'rb') as f:
commands_stats = json.load(f).get('commands', {})
except Exception:
cmd_logger.exception(
'Failed to parse commands stats file {}'.format(
commands_stats_path),
extra=self.log_extra,
)
parsed_stats = self._parse_commands_stats(commands_stats)
# NOTE: temporary backward compatibility
self.commands_stats = parsed_stats
return parsed_stats
def execute(self):
try:
shutil.rmtree(self.params['remove_path'])
except Exception:
cmd_logger.exception('Failed to remove path {}'.format(
self.params['remove_path']),
extra=self.log_extra)
raise
cmd_logger.info('Successfully removed path {}'.format(
self.params['remove_path']),
extra=self.log_extra)
def execute(self):
group = str(int(self.params['group']))
path = self.params['group_file'].format(group_id=group)
try:
if os.path.exists(path):
os.rename(path, path + '.bak')
dirname = os.path.dirname(path)
if not os.path.exists(dirname):
os.makedirs(dirname, 0755)
with open(path, 'w') as f:
f.write(group)
except Exception:
cmd_logger.exception('Failed to create group file', extra=self.log_extra)
raise
cmd_logger.info('Successfully created group file {} for group {}'.format(path, group), extra=self.log_extra)
def execute(self):
ids_file = self.params['ids']
cmd_logger.info('Generating ids file {}'.format(ids_file), extra=self.log_extra)
if os.path.exists(ids_file):
cmd_logger.info('Ids file {} already exists'.format(ids_file), extra=self.log_extra)
else:
try:
with open(ids_file, 'wb') as f:
f.write(os.urandom(64))
except Exception:
cmd_logger.exception(
'Failed to create ids file {}'.format(ids_file),
extra=self.log_extra,
)
raise
cmd_logger.info('Successfully created ids file {}'.format(ids_file), extra=self.log_extra)
def execute(self):
group_base_path_root_dir = self.params[
'group_base_path_root_dir'].rstrip('/')
basename = self.get_vacant_basename(group_base_path_root_dir)
tmp_basename = self.tmp_basename(basename)
tmp_dir = os.path.join(group_base_path_root_dir, tmp_basename)
cmd_logger.info('Creating tmp dir for new group: {}'.format(tmp_dir),
extra=self.log_extra)
try:
os.mkdir(tmp_dir, 0755)
except Exception:
cmd_logger.exception('Failed to create tmp dir for new group',
extra=self.log_extra)
raise
cmd_logger.info('Adding group files', extra=self.log_extra)
for filename, body in self.params['files'].iteritems():
cmd_logger.info('Adding file {}'.format(filename),
extra=self.log_extra)
filename = os.path.join(tmp_dir, filename)
dirname, basefname = os.path.split(filename)
if not os.path.exists(dirname):
os.makedirs(dirname)
with open(filename, 'wb') as f:
f.write(body)
dest_dir = os.path.join(group_base_path_root_dir, basename)
cmd_logger.info(
'Renaming tmp dir {tmp_dir} to destination dir {dest_dir}'.format(
tmp_dir=tmp_dir,
dest_dir=dest_dir,
),
extra=self.log_extra,
)
try:
os.rename(tmp_dir, dest_dir)
except Exception:
cmd_logger.exception('Failed to rename tmp dir to dest dir',
extra=self.log_extra)
raise
backend_path = dest_dir
if not backend_path.endswith('/'):
backend_path += '/'
self.artifacts['backend_path'] = backend_path
def execute(self):
group_base_path_root_dir = self.params['group_base_path_root_dir'].rstrip('/')
basename = self.get_vacant_basename(group_base_path_root_dir)
tmp_basename = self.tmp_basename(basename)
tmp_dir = os.path.join(group_base_path_root_dir, tmp_basename)
cmd_logger.info('Creating tmp dir for new group: {}'.format(tmp_dir), extra=self.log_extra)
try:
os.mkdir(tmp_dir, 0755)
except Exception:
cmd_logger.exception('Failed to create tmp dir for new group', extra=self.log_extra)
raise
cmd_logger.info('Adding group files', extra=self.log_extra)
for filename, body in self.params['files'].iteritems():
cmd_logger.info('Adding file {}'.format(filename), extra=self.log_extra)
filename = os.path.join(
tmp_dir,
filename
)
dirname, basefname = os.path.split(filename)
if not os.path.exists(dirname):
os.makedirs(dirname)
with open(filename, 'wb') as f:
f.write(body)
dest_dir = os.path.join(group_base_path_root_dir, basename)
cmd_logger.info(
'Renaming tmp dir {tmp_dir} to destination dir {dest_dir}'.format(
tmp_dir=tmp_dir,
dest_dir=dest_dir,
),
extra=self.log_extra,
)
try:
os.rename(tmp_dir, dest_dir)
except Exception:
cmd_logger.exception('Failed to rename tmp dir to dest dir', extra=self.log_extra)
raise
backend_path = dest_dir
if not backend_path.endswith('/'):
backend_path += '/'
self.artifacts['backend_path'] = backend_path
def on_update_progress(self):
s = Session()
s.begin()
try:
command = self.command
command.progress = self.watcher.progress
command.exit_code = self.watcher.exit_code
command.command_code = self.command_code
command.stdout = self.watcher.get_stdout()
command.stderr = self.watcher.get_stderr()
if command.exit_code is not None:
command.artifacts = json.dumps(self.artifacts)
command.finish_ts = self.finish_ts
s.add(command)
s.commit()
except Exception:
cmd_logger.exception('Failed to update db command', extra=self.log_extra)
s.rollback()
def execute(self):
group = str(int(self.params['group']))
path = self.params['group_file'].format(group_id=group)
try:
if os.path.exists(path):
os.rename(path, path + '.bak')
dirname = os.path.dirname(path)
if not os.path.exists(dirname):
os.makedirs(dirname, 0755)
with open(path, 'w') as f:
f.write(group)
except Exception:
cmd_logger.exception('Failed to create group file',
extra=self.log_extra)
raise
cmd_logger.info(
'Successfully created group file {} for group {}'.format(
path, group),
extra=self.log_extra)
def execute(self):
ids_file = self.params['ids']
cmd_logger.info('Generating ids file {}'.format(ids_file),
extra=self.log_extra)
if os.path.exists(ids_file):
cmd_logger.info('Ids file {} already exists'.format(ids_file),
extra=self.log_extra)
else:
try:
with open(ids_file, 'wb') as f:
f.write(os.urandom(64))
except Exception:
cmd_logger.exception(
'Failed to create ids file {}'.format(ids_file),
extra=self.log_extra,
)
raise
cmd_logger.info('Successfully created ids file {}'.format(ids_file),
extra=self.log_extra)
def run(self):
self.start_ts = int(time.time())
# create db record
s = Session()
s.begin()
command = minion.db.commands.Command(
uid=self.uid,
pid=None,
command=self.COMMAND,
start_ts=self.start_ts,
task_id=self.params.get('task_id'),
job_id=self.params.get('job_id'),
)
s.update_ts = int(time.time())
s.add(command)
s.commit()
try:
yield self.execute()
except Exception as e:
cmd_logger.exception('Command execution failed',
extra=self.log_extra)
self.error = e
self.finish_ts = int(time.time())
s.begin()
try:
command.progress = 1.0
command.exit_code = 1 if self.error else 0
command.command_code = 1 if self.error else 0
command.finish_ts = self.finish_ts
command.artifacts = json.dumps(self.artifacts)
s.add(command)
s.commit()
except Exception as e:
cmd_logger.exception('Failed to update db command',
extra=self.log_extra)
s.rollback()
def run(self):
self.start_ts = int(time.time())
# create db record
s = Session()
s.begin()
command = minion.db.commands.Command(
uid=self.uid,
pid=None,
command=self.COMMAND,
start_ts=self.start_ts,
task_id=self.params.get('task_id'),
job_id=self.params.get('job_id'),
)
s.update_ts = int(time.time())
s.add(command)
s.commit()
try:
yield self.execute()
except Exception as e:
cmd_logger.exception('Command execution failed', extra=self.log_extra)
self.error = e
self.finish_ts = int(time.time())
s.begin()
try:
command.progress = 1.0
command.exit_code = 1 if self.error else 0
command.command_code = 1 if self.error else 0
command.finish_ts = self.finish_ts
command.artifacts = json.dumps(self.artifacts)
s.add(command)
s.commit()
except Exception as e:
cmd_logger.exception('Failed to update db command', extra=self.log_extra)
s.rollback()
def execute(self):
group_base_path = self.params['group_base_path'].rstrip('/')
if not os.path.exists(group_base_path):
raise RuntimeError(
'Group dir {path} does not exist'.format(
path=group_base_path,
)
)
dst_base_path, basename = os.path.split(group_base_path)
remove_path = os.path.join(
dst_base_path,
self.removed_basename(basename)
)
cmd_logger.info(
'Renaming group base dir {tmp_dir} to destination dir {dest_dir}'.format(
tmp_dir=group_base_path,
dest_dir=remove_path,
),
extra=self.log_extra,
)
try:
os.rename(group_base_path, remove_path)
except OSError as e:
if e.errno == 2:
# errno == 2: No such file or directory
if os.path.exists(remove_path):
# group_base_path was already renamed, not an error
pass
else:
raise
else:
raise
except Exception:
cmd_logger.exception('Failed to rename tmp dir to dest dir', extra=self.log_extra)
raise
