def collect_artifacts(self):
tmp_dir = None
for i, cmd_part in enumerate(self.cmd):
if cmd_part in ('--tmp', '-t'):
tmp_dir = self.cmd[i + 1]
break
if not tmp_dir:
logger.info('Failed to determine tmp directory')
return {}
exec_state_path = os.path.join(tmp_dir, 'exec_state')
logger.info('Parsing exec state: {}'.format(exec_state_path))
exec_state = {}
try:
with open(exec_state_path, 'rb') as f:
exec_state = json.load(f).get('status', {})
except Exception:
logger.exception(
'Failed to parse exec state file {}'.format(exec_state_path)
)
pass
return exec_state
def execute(self):
group_base_path = self.params['group_base_path'].rstrip('/')
if not os.path.exists(group_base_path):
raise RuntimeError('Group dir {path} does not exist'.format(
path=group_base_path, ))
dst_base_path, basename = os.path.split(group_base_path)
remove_path = os.path.join(dst_base_path,
self.removed_basename(basename))
logger.info(
'Renaming group base dir {tmp_dir} to destination dir {dest_dir}'.
format(
tmp_dir=group_base_path,
dest_dir=remove_path,
))
try:
os.rename(group_base_path, remove_path)
except OSError as e:
if e.errno == 2:
# errno == 2: No such file or directory
if os.path.exists(remove_path):
# group_base_path was already renamed, not an error
pass
else:
raise
except Exception:
logger.exception('Failed to rename tmp dir to dest dir')
raise
def run(self):
# create db record
s = Session()
s.begin()
command = minion.db.commands.Command(
uid=self.uid,
pid=None,
command=self.cmd_str,
start_ts=int(time.time()),
task_id=self.params.get('task_id'))
# TODO: what about group_id, node, node_backend ?
s.add(command)
s.commit()
try:
self.execute()
except Exception as e:
self.error = e
# TODO: raise?
self.finish_ts = int(time.time())
s.begin()
try:
command.progress = 1.0
command.exit_code = 1 if self.error else 0
command.command_code = 1 if self.error else 0
command.finish_ts = self.finish_ts
s.add(command)
s.commit()
except Exception as e:
logger.exception('Failed to update db command')
s.rollback()
def execute(self):
group_base_path = self.params['group_base_path'].rstrip('/')
if not os.path.exists(group_base_path):
raise RuntimeError(
'Group dir {path} does not exist'.format(
path=group_base_path,
)
)
dst_base_path, basename = os.path.split(group_base_path)
remove_path = os.path.join(
dst_base_path,
self.removed_basename(basename)
)
logger.info(
'Renaming group base dir {tmp_dir} to destination dir {dest_dir}'.format(
tmp_dir=group_base_path,
dest_dir=remove_path,
)
)
try:
os.rename(group_base_path, remove_path)
except OSError as e:
if e.errno == 2:
# errno == 2: No such file or directory
if os.path.exists(remove_path):
# group_base_path was already renamed, not an error
pass
else:
raise
except Exception:
logger.exception('Failed to rename tmp dir to dest dir')
raise
def on_command_completed(self):
self.finish_ts = int(time.time())
self.artifacts = self.collect_artifacts()
self.on_update_progress()
if not self._apply_postprocessors():
# TODO: add status codes
logger.info('Command failed, no post processors will be applied')
return
for post_processor in self.POST_PROCESSORS:
params_supplied = all(
param in self.params
for param in post_processor.REQUIRED_PARAMS
)
if params_supplied:
# TODO: replace by required params? possibly not
logger.info('Running post processor {}'.format(post_processor.__name__))
uid = uuid.uuid4().hex
command = post_processor(uid, params=self.params)
try:
# NOTE: when running as a post processor command is not
# dumped to database, therefore 'execute' method is called
# instead of 'run'
command.execute()
except:
logger.exception('Post processor {} failed, skipped'.format(
post_processor.__name__
))
continue
def update_broken_commands(self):
s = Session()
s.begin()
try:
for c in s.query(Command).filter_by(exit_code=None):
log_extra = {'task_id': c.task_id, 'job_id': c.job_id}
if not self.pid_exists(c.pid):
c.progress = 1.0
c.exit_code = 666
c.finish_ts = int(time.time())
s.add(c)
cmd_logger.info(
'Command {}, pid {} is considered broken, will be marked as '
'finished'.format(c.uid, c.pid),
extra=log_extra,
)
else:
cmd_logger.warn(
'Command {}, pid {} is considered broken, but process is running'
.format(c.uid, c.pid),
extra=log_extra,
)
s.commit()
except Exception:
logger.exception('Failed to update broken commands')
s.rollback()
raise
def update_broken_commands(self):
s = Session()
s.begin()
try:
for c in s.query(Command).filter_by(exit_code=None):
if not self.pid_exists(c.pid):
c.progress = 1.0
c.exit_code = 666
c.finish_ts = int(time.time())
s.add(c)
logger.info(
'Command {}, pid {} is considered broken, will be marked as '
'finished'.format(
c.uid,
c.pid
)
)
else:
logger.warn(
'Command {}, pid {} is considered broken, but process is running'.format(
c.uid,
c.pid
)
)
s.commit()
except Exception:
logger.exception('Failed to update broken commands')
s.rollback()
raise
def execute(self):
try:
shutil.rmtree(self.params['remove_path'])
except Exception:
logger.exception('Failed to remove path {}'.format(self.params['remove_path']))
raise
logger.info('Successfully removed path {}'.format(self.params['remove_path']))
def run(self):
# create db record
s = Session()
s.begin()
command = minion.db.commands.Command(
uid=self.uid,
pid=None,
command=self.cmd_str,
start_ts=int(time.time()),
task_id=self.params.get('task_id')
)
# TODO: what about group_id, node, node_backend ?
s.add(command)
s.commit()
try:
self.execute()
except Exception as e:
self.error = e
# TODO: raise?
self.finish_ts = int(time.time())
s.begin()
try:
command.progress = 1.0
command.exit_code = 1 if self.error else 0
command.command_code = 1 if self.error else 0
command.finish_ts = self.finish_ts
s.add(command)
s.commit()
except Exception as e:
logger.exception('Failed to update db command')
s.rollback()
def setup_session(uri):
engine = sqlalchemy.create_engine(uri)
try:
engine.connect()
logger.info('Successfully connected to db by uri {0}'.format(uri))
except Exception as e:
logger.exception('Failed to connect to db engine by uri {0}'.format(uri))
raise
return sessionmaker(bind=engine, autocommit=True)
def setup_session(uri):
engine = sqlalchemy.create_engine(uri)
try:
engine.connect()
logger.info('Successfully connected to db by uri {0}'.format(uri))
except Exception as e:
logger.exception(
'Failed to connect to db engine by uri {0}'.format(uri))
raise
return sessionmaker(bind=engine, autocommit=True)
def feed(self, s):
prev_progress = self.progress
self.output.append(s)
if abs(self.progress - prev_progress) > 0.01:
try:
s = Session()
s.begin()
self.command.progress = self.progress
s.add(self.command)
s.commit()
except Exception as e:
logger.exception('pid {0}: failed to update db command'.format(
self.subprocess.pid))
pass
def update_db_command(self):
s = Session()
s.begin()
try:
command = self.command
command.progress = self.progress
command.exit_code = self.exit_code
command.command_code = self.command_code
command.finish_ts = self.finish_ts
s.add(command)
s.commit()
except Exception as e:
logger.exception('Failed to update db command: {0}'.format(e))
s.rollback()
def execute(self):
ids_file = self.params['ids']
logger.info('Generating ids file {}'.format(ids_file))
if os.path.exists(ids_file):
logger.info('Ids file {} already exists'.format(ids_file))
else:
try:
with open(ids_file, 'wb') as f:
f.write(os.urandom(64))
except Exception:
logger.exception('Failed to create ids file {}'.format(
ids_file)
)
raise
logger.info('Successfully created ids file {}'.format(ids_file))
def execute(self):
group = str(int(self.params['group']))
path = self.params['group_file'].format(group_id=group)
try:
if os.path.exists(path):
os.rename(path, path + '.bak')
dirname = os.path.dirname(path)
if not os.path.exists(dirname):
os.makedirs(dirname, 0755)
with open(path, 'w') as f:
f.write(group)
except Exception:
logger.exception('Failed to create group file')
raise
logger.info('Successfully created group file {} for group {}'.format(path, group))
def create_group_file(self):
if self.params.get('group_file'):
try:
group = str(int(self.params.get('group')))
path = self.params.get('group_file')
if os.path.exists(path):
os.rename(path, path + '.bak')
dirname = os.path.dirname(path)
if not os.path.exists(dirname):
os.makedirs(dirname, 0755)
with open(path, 'w') as f:
f.write(group + '\n')
except Exception as e:
logger.exception('Failed to create group file: {0}'.format(e))
else:
logger.info('Successfully created group file '
'for group {0}'.format(group))
def on_update_progress(self):
s = Session()
s.begin()
try:
command = self.command
command.progress = self.watcher.progress
command.exit_code = self.watcher.exit_code
command.command_code = self.command_code
command.stdout = self.watcher.get_stdout()
command.stderr = self.watcher.get_stderr()
if command.exit_code is not None:
command.artifacts = json.dumps(self.artifacts)
command.finish_ts = self.finish_ts
s.add(command)
s.commit()
except Exception:
logger.exception('Failed to update db command')
s.rollback()
def update_broken_commands(self):
s = Session()
s.begin()
try:
for c in s.query(Command).filter_by(exit_code=None):
if not self.pid_exists(c.pid):
c.progress = 1.0
c.exit_code = 666
c.finish_ts = int(time.time())
s.add(c)
logger.info('Command {0}, pid {1} is considered broken, '
'will be marked as finished'.format(
c.uid, c.pid))
else:
logger.warn('Command {0}, pid {1} is considered broken, '
'but process is running'.format(c.uid, c.pid))
s.commit()
except Exception as e:
logger.exception('Failed to update broken commands')
s.rollback()
raise
def execute(self):
group_base_path_root_dir = self.params['group_base_path_root_dir'].rstrip('/')
basename = self.get_vacant_basename(group_base_path_root_dir)
tmp_basename = self.tmp_basename(basename)
tmp_dir = os.path.join(group_base_path_root_dir, tmp_basename)
logger.info('Creating tmp dir for new group: {}'.format(tmp_dir))
try:
os.mkdir(tmp_dir, 0755)
except Exception:
logger.exception('Failed to create tmp dir for new group')
raise
logger.info('Adding group files')
for filename, body in self.params['files'].iteritems():
logger.info('Adding file {}'.format(filename))
filename = os.path.join(
tmp_dir,
filename
)
dirname, basefname = os.path.split(filename)
if not os.path.exists(dirname):
os.makedirs(dirname)
with open(filename, 'wb') as f:
f.write(body)
dest_dir = os.path.join(group_base_path_root_dir, basename)
logger.info(
'Renaming tmp dir {tmp_dir} to destination dir {dest_dir}'.format(
tmp_dir=tmp_dir,
dest_dir=dest_dir,
)
)
try:
os.rename(tmp_dir, dest_dir)
except Exception:
logger.exception('Failed to rename tmp dir to dest dir')
raise
def execute(self):
group_base_path_root_dir = self.params[
'group_base_path_root_dir'].rstrip('/')
basename = self.get_vacant_basename(group_base_path_root_dir)
tmp_basename = self.tmp_basename(basename)
tmp_dir = os.path.join(group_base_path_root_dir, tmp_basename)
logger.info('Creating tmp dir for new group: {}'.format(tmp_dir))
try:
os.mkdir(tmp_dir, 0755)
except Exception:
logger.exception('Failed to create tmp dir for new group')
raise
logger.info('Adding group files')
for filename, body in self.params['files'].iteritems():
logger.info('Adding file {}'.format(filename))
filename = os.path.join(tmp_dir, filename)
dirname, basefname = os.path.split(filename)
if not os.path.exists(dirname):
os.makedirs(dirname)
with open(filename, 'wb') as f:
f.write(body)
dest_dir = os.path.join(group_base_path_root_dir, basename)
logger.info(
'Renaming tmp dir {tmp_dir} to destination dir {dest_dir}'.format(
tmp_dir=tmp_dir,
dest_dir=dest_dir,
))
try:
os.rename(tmp_dir, dest_dir)
except Exception:
logger.exception('Failed to rename tmp dir to dest dir')
raise
def collect_artifacts(self):
commands_stats_path = self.params.get('commands_stats_path')
if not commands_stats_path:
logger.info('Commands stats path was not supplied')
return {}
logger.info('Parsing commands stats path: {}'.format(commands_stats_path))
commands_stats = {}
try:
with open(commands_stats_path, 'rb') as f:
commands_stats = json.load(f).get('commands', {})
except Exception:
logger.exception(
'Failed to parse commands stats file {}'.format(commands_stats_path)
)
parsed_stats = self._parse_commands_stats(commands_stats)
# NOTE: temporary backward compatibility
self.commands_stats = parsed_stats
return parsed_stats
def create_group_file(self):
if self.params.get('group_file'):
try:
group = str(int(self.params.get('group')))
path = self.params['group_file'].format(group_id=group)
if os.path.exists(path):
os.rename(path, path + '.bak')
dirname = os.path.dirname(path)
if not os.path.exists(dirname):
os.makedirs(dirname, 0755)
with open(path, 'w') as f:
f.write(group)
except Exception as e:
logger.exception('Failed to create group file: {0}'.format(e))
else:
logger.info('Successfully created group file '
'for group {0}'.format(group))
else:
logger.info('Group file creation was not requested for '
'group {0}'.format(self.params.get('group')))
if self.params.get('remove_path'):
try:
shutil.rmtree(self.params['remove_path'])
except Exception as e:
logger.exception('Failed to remove path {0}: {1}'.format(
self.params['remove_path'], e))
else:
logger.info('Successfully removed path {0} '
'for group {1}'.format(self.params['remove_path'],
group))
else:
logger.info('Path removal was not requested for '
'group {0}'.format(self.params.get('group')))
if self.params.get('ids'):
ids_file = self.params['ids']
logger.info('Generating ids file {} required'.format(ids_file))
if os.path.exists(ids_file):
logger.info('Ids file {} already exists'.format(ids_file))
else:
try:
with open(ids_file, 'wb') as f:
f.write(os.urandom(64))
except Exception as e:
logger.exception(
'Failed to create ids file {}'.format(ids_file))
else:
logger.info('Successfully created ids file {} '
'for group {1}'.format(ids_file, group))
def create_group_file(self):
if self.params.get('group_file'):
try:
group = str(int(self.params.get('group')))
path = self.params['group_file'].format(group_id=group)
if os.path.exists(path):
os.rename(path, path + '.bak')
dirname = os.path.dirname(path)
if not os.path.exists(dirname):
os.makedirs(dirname, 0755)
with open(path, 'w') as f:
f.write(group)
except Exception as e:
logger.exception('Failed to create group file: {0}'.format(e))
else:
logger.info('Successfully created group file '
'for group {0}'.format(group))
else:
logger.info('Group file creation was not requested for '
'group {0}'.format(self.params.get('group')))
if self.params.get('remove_path'):
try:
shutil.rmtree(self.params['remove_path'])
except Exception as e:
logger.exception('Failed to remove path {0}: {1}'.format(
self.params['remove_path'], e))
else:
logger.info('Successfully removed path {0} '
'for group {1}'.format(self.params['remove_path'], group))
else:
logger.info('Path removal was not requested for '
'group {0}'.format(self.params.get('group')))
if self.params.get('ids'):
ids_file = self.params['ids']
logger.info('Generating ids file {} required'.format(ids_file))
if os.path.exists(ids_file):
logger.info('Ids file {} already exists'.format(ids_file))
else:
try:
with open(ids_file, 'wb') as f:
f.write(os.urandom(64))
except Exception as e:
logger.exception('Failed to create ids file {}'.format(
ids_file))
else:
logger.info('Successfully created ids file {} '
'for group {1}'.format(ids_file, group))
def create_group_file_marker(self):
if self.params.get('group_file_marker'):
try:
group = (str(int(self.params.get('group')))
if self.params.get('group') else '')
path = self.params['group_file_marker'].format(group_id=group)
dirname = os.path.dirname(path)
if not os.path.exists(dirname):
os.makedirs(dirname, 0755)
with open(path, 'w') as f:
f.write(group)
except Exception as e:
logger.exception(
'Failed to create group file marker: {0}'.format(e))
else:
logger.info('Successfully created group file marker '
'for group {0}'.format(group))
else:
logger.info('Group file marker creation was not requested for '
'group {0}'.format(self.params.get('group')))
if self.params.get('remove_group_file'):
logger.info('Removing group file {0}'.format(
self.params['remove_group_file']))
try:
os.remove(self.params['remove_group_file'])
except Exception as e:
logger.exception('Failed to remove group file: {0}'.format(e))
else:
logger.info('Successfully removed group '
'file {0}'.format(
self.params['remove_group_file']))
else:
logger.info('Group file removal was not requested for '
'group {0}'.format(self.params.get('group')))
if self.params.get('move_dst') and self.params.get('move_src'):
try:
os.rename(self.params['move_src'], self.params['move_dst'])
except Exception as e:
logger.exception(
'Failed to execute move task: {0} to {1}'.format(
self.params['move_src'], self.params['move_dst']))
else:
logger.info(
'Successfully performed move task: {0} to {1}'.format(
self.params['move_src'], self.params['move_dst']))
if self.params.get('mark_backend'):
marker = self.params['mark_backend']
try:
open(marker, 'w').close()
except Exception as e:
logger.error(
'Failed to create backend down marker: {0}'.format(e))
else:
logger.info(
'Successfully created backend down marker: {0}'.format(
self.params['mark_backend']))
if self.params.get('unmark_backend'):
marker = self.params['unmark_backend']
try:
os.remove(self.params['unmark_backend'])
except Exception as e:
logger.error(
'Failed to remove backend down marker: {0}'.format(e))
else:
logger.info(
'Successfully removed backend down marker: {0}'.format(
self.params['unmark_backend']))
def create_group_file_marker(self):
if self.params.get('group_file_marker'):
try:
group = (str(int(self.params.get('group')))
if self.params.get('group') else
'')
path = self.params['group_file_marker'].format(group_id=group)
dirname = os.path.dirname(path)
if not os.path.exists(dirname):
os.makedirs(dirname, 0755)
with open(path, 'w') as f:
f.write(group)
except Exception as e:
logger.exception('Failed to create group file marker: {0}'.format(e))
else:
logger.info('Successfully created group file marker '
'for group {0}'.format(group))
else:
logger.info('Group file marker creation was not requested for '
'group {0}'.format(self.params.get('group')))
if self.params.get('remove_group_file'):
logger.info('Removing group file {0}'.format(self.params['remove_group_file']))
try:
os.remove(self.params['remove_group_file'])
except Exception as e:
logger.exception('Failed to remove group file: {0}'.format(e))
else:
logger.info('Successfully removed group '
'file {0}'.format(self.params['remove_group_file']))
else:
logger.info('Group file removal was not requested for '
'group {0}'.format(self.params.get('group')))
if self.params.get('move_dst') and self.params.get('move_src'):
try:
os.rename(self.params['move_src'], self.params['move_dst'])
except Exception as e:
logger.exception('Failed to execute move task: {0} to {1}'.format(
self.params['move_src'], self.params['move_dst']))
else:
logger.info('Successfully performed move task: {0} to {1}'.format(
self.params['move_src'], self.params['move_dst']))
if self.params.get('mark_backend'):
marker = self.params['mark_backend']
try:
open(marker, 'w').close()
except Exception as e:
logger.error('Failed to create backend down marker: {0}'.format(e))
else:
logger.info('Successfully created backend down marker: {0}'.format(
self.params['mark_backend']))
if self.params.get('unmark_backend'):
marker = self.params['unmark_backend']
try:
os.remove(self.params['unmark_backend'])
except Exception as e:
logger.error('Failed to remove backend down marker: {0}'.format(e))
else:
logger.info('Successfully removed backend down marker: {0}'.format(
self.params['unmark_backend']))