def status(self):
cmd = (shlex.split(self.command.encode('utf-8'))
if isinstance(self.command, basestring) else
self.command)
Subprocess = subprocess.subprocess_factory(cmd)
if issubclass(Subprocess, BaseSubprocess):
Watcher = Subprocess.watcher_base
exit_message = Watcher.exit_messages.get(self.exit_code, 'Unknown')
else:
exit_message = 'Success' if self.exit_code == 0 else ''
artifacts = {}
try:
artifacts = json.loads(self.artifacts)
except ValueError as e:
logger.error('Dumped command (uid {}) has mailformed "artifacts" field: {}'.format(
self.uid,
self.artifacts
))
pass
return {
'pid': self.pid,
'command': self.command,
'task_id': self.task_id,
'progress': self.progress,
'exit_code': self.exit_code,
'exit_message': exit_message,
'command_code': self.command_code,
'start_ts': self.start_ts,
'finish_ts': self.finish_ts,
'output': self.stdout,
'error_output': self.stderr,
'artifacts': artifacts,
}
def execute(self):
marker = self.params["unmark_backend"]
try:
os.remove(marker)
except Exception as e:
logger.error("Failed to remove backend marker: {}".format(e))
raise
logger.info("Successfully removed backend marker: {}".format(marker))
def execute(self):
stop_file = self.params['create_stop_file']
try:
open(stop_file, 'w').close()
except Exception as e:
logger.error('Failed to create backend stop marker: {}'.format(e))
raise
logger.info('Successfully created backend stop marker: {}'.format(stop_file))
def execute(self):
marker = self.params['mark_backend']
try:
open(marker, 'w').close()
except Exception as e:
logger.error('Failed to create backend marker: {}'.format(e))
raise
logger.info('Successfully created backend marker: {}'.format(marker))
def execute(self):
stop_file = self.params["remove_stop_file"]
try:
os.remove(stop_file)
except Exception as e:
logger.error("Failed to remove backend stop marker: {}".format(e))
raise
logger.info("Successfully removed backend stop marker: {}".format(stop_file))
def execute(self):
logger.info('Removing group file {0}'.format(self.params['remove_group_file']))
try:
os.remove(self.params['remove_group_file'])
except Exception as e:
logger.error('Failed to remove group file: {}'.format(e))
raise
logger.info('Successfully removed group file {}'.format(self.params['remove_group_file']))
def execute(self):
group = str(self.params.get("group")) if "group" in self.params.get("group") else ""
try:
path = self.params["group_file_marker"].format(group_id=group)
dirname = os.path.dirname(path)
if not os.path.exists(dirname):
os.makedirs(dirname, 0755)
with open(path, "w") as f:
f.write(group)
except Exception as e:
logger.error("Failed to create group file marker: {}".format(e))
raise
logger.info("Successfully created group file marker for group {}".format(group))
def execute(self):
try:
os.rename(self.params['move_src'], self.params['move_dst'])
except Exception as e:
logger.error('Failed to execute move path command: {} to {}: {}'.format(
self.params['move_src'],
self.params['move_dst'],
e
))
logger.info('Successfully performed move task: {} to {}'.format(
self.params['move_src'],
self.params['move_dst']
))
def post(self):
if config['common'].get('base_path') is None:
logger.error(
'base path is not set, remove group cannot be performed')
self.set_status(500)
raise RuntimeError('group creation is not allowed')
params = {k: v[0] for k, v in self.request.arguments.iteritems()}
params['group_base_path'] = os.path.normpath(params['group_base_path'])
if not params['group_base_path'].startswith(
config['common']['base_path']):
self.set_status(403)
raise RuntimeError(
'Group path {path} is not under common base path'.format(
path=params['group_base_path'], ))
uid = manager.execute(GroupRemover, cmd='remove_group', params=params)
self.set_status(302)
self.add_header('Location', self.reverse_url('status', uid))
def post(self):
if config['common'].get('base_path') is None:
logger.error('base path is not set, remove group cannot be performed')
self.set_status(500)
raise RuntimeError('group creation is not allowed')
params = {
k: v[0]
for k, v in self.request.arguments.iteritems()
}
params['group_base_path'] = os.path.normpath(params['group_base_path'])
if not params['group_base_path'].startswith(config['common']['base_path']):
self.set_status(403)
raise RuntimeError(
'Group path {path} is not under common base path'.format(
path=params['group_base_path'],
)
)
uid = manager.execute(GroupRemover, cmd='remove_group', params=params)
self.set_status(302)
self.add_header('Location', self.reverse_url('status', uid))
def _parse_command_code(self):
if self.watcher.exit_code == 0:
return self.watcher.exit_code
output = self.watcher.get_stdout()
try:
data = json.loads(output)
except Exception:
logger.error('pid {}: failed to parse output json: {}'.format(
self.pid,
output
))
return self.watcher.exit_code
if 'error' not in data:
logger.error('pid {}: no "error" key in response data'.format(
self.pid
))
return self.watcher.exit_code
if 'code' not in data['error']:
logger.error('pid {}: no "code" key in response error data'.format(
self.pid
))
return self.watcher.exit_code
logger.info('pid {}: operation error code {}'.format(
self.pid,
data['error']['code']
))
return data['error']['code']
def is_success(self):
logger.info('pid {0}: exit code {1}'.format(
self.subprocess.pid, self.exit_code))
if self.exit_code == 0:
return True
output = ''.join(self.output)
try:
data = json.loads(output)
except Exception as e:
logger.error('pid {0}: failed to parse output json: {1}'.format(
self.subprocess.pid, output))
return False
if not 'error' in data:
logger.error('pid {0}: no "error" key in response data'.format(
self.subprocess.pid))
return False
if not 'code' in data['error']:
logger.error('pid {0}: no "code" key in response error data'.format(
self.subprocess.pid))
return False
logger.info('pid {0}: operation error code {1}'.format(
self.subprocess.pid, data['error']['code']))
return data['error']['code'] in self.success_codes
def is_success(self):
logger.info('pid {0}: exit code {1}'.format(self.subprocess.pid,
self.exit_code))
if self.exit_code == 0:
return True
output = ''.join(self.output)
try:
data = json.loads(output)
except Exception as e:
logger.error('pid {0}: failed to parse output json: {1}'.format(
self.subprocess.pid, output))
return False
if not 'error' in data:
logger.error('pid {0}: no "error" key in response data'.format(
self.subprocess.pid))
return False
if not 'code' in data['error']:
logger.error(
'pid {0}: no "code" key in response error data'.format(
self.subprocess.pid))
return False
logger.info('pid {0}: operation error code {1}'.format(
self.subprocess.pid, data['error']['code']))
return data['error']['code'] in self.success_codes
def post(self):
if config['common'].get('base_path') is None:
logger.error('base path is not set, create group cannot be performed')
self.set_status(500)
raise RuntimeError('group creation is not allowed')
files = {}
for filename, http_files in self.request.files.iteritems():
norm_filename = os.path.normpath(filename)
if norm_filename.startswith('..') or norm_filename.startswith('/'):
logger.error(
'Cannot create file {filename}, '
'normalized path {norm_filename} is not allowed'.format(
filename=filename,
norm_filename=norm_filename,
)
)
self.set_status(403)
raise RuntimeError(
'File {filename} is forbidden, path should be relative '
'to group base directory'.format(filename=filename)
)
http_file = http_files[0]
files[norm_filename] = http_file.body
params = {
k: v[0]
for k, v in self.request.arguments.iteritems()
}
params['group_base_path_root_dir'] = os.path.normpath(params['group_base_path_root_dir'])
if not params['group_base_path_root_dir'].startswith(config['common']['base_path']):
self.set_status(403)
raise RuntimeError(
'Group base path {path} is not under common base path'.format(
path=params['group_base_path_root_dir'],
)
)
params['files'] = files
uid = manager.execute(GroupCreator, cmd='create_group', params=params)
self.set_status(302)
self.add_header('Location', self.reverse_url('status', uid))
def wrapped(self, *args, **kwargs):
try:
self.add_header('Content-Type', 'text/json')
try:
res = method(self, *args, **kwargs)
except Exception as e:
logger.error('{0}: {1}'.format(e, traceback.format_exc(e)))
response = {'status': 'error',
'error': str(e)}
else:
response = {'status': 'success',
'response': res}
try:
self.write(json.dumps(response))
except Exception as e:
logger.error('Failed to dump json response: {0}\n{1}'.format(e,
traceback.format_exc(e)))
response = {'status': 'error',
'error': 'failed to construct response, see log file'}
self.write(json.dumps(response))
finally:
self.finish()
def wrapped(self, *args, **kwargs):
try:
self.add_header('Content-Type', 'text/json')
try:
res = method(self, *args, **kwargs)
except Exception as e:
logger.error('{0}: {1}'.format(e, traceback.format_exc(e)))
response = {'status': 'error', 'error': str(e)}
else:
response = {'status': 'success', 'response': res}
try:
self.write(json.dumps(response))
except Exception as e:
logger.error('Failed to dump json response: {0}\n{1}'.format(
e, traceback.format_exc(e)))
response = {
'status': 'error',
'error': 'failed to construct response, see log file'
}
self.write(json.dumps(response))
finally:
self.finish()
def post(self):
if config['common'].get('base_path') is None:
logger.error(
'base path is not set, create group cannot be performed')
self.set_status(500)
raise RuntimeError('group creation is not allowed')
files = {}
for filename, http_files in self.request.files.iteritems():
norm_filename = os.path.normpath(filename)
if norm_filename.startswith('..') or norm_filename.startswith('/'):
logger.error(
'Cannot create file {filename}, '
'normalized path {norm_filename} is not allowed'.format(
filename=filename,
norm_filename=norm_filename,
))
self.set_status(403)
raise RuntimeError(
'File {filename} is forbidden, path should be relative '
'to group base directory'.format(filename=filename))
http_file = http_files[0]
files[norm_filename] = http_file.body
params = {k: v[0] for k, v in self.request.arguments.iteritems()}
params['group_base_path_root_dir'] = os.path.normpath(
params['group_base_path_root_dir'])
if not params['group_base_path_root_dir'].startswith(
config['common']['base_path']):
self.set_status(403)
raise RuntimeError(
'Group base path {path} is not under common base path'.format(
path=params['group_base_path_root_dir'], ))
params['files'] = files
uid = manager.execute(GroupCreator, cmd='create_group', params=params)
self.set_status(302)
self.add_header('Location', self.reverse_url('status', uid))
def create_group_file_marker(self):
if self.params.get('group_file_marker'):
try:
group = (str(int(self.params.get('group')))
if self.params.get('group') else '')
path = self.params['group_file_marker'].format(group_id=group)
dirname = os.path.dirname(path)
if not os.path.exists(dirname):
os.makedirs(dirname, 0755)
with open(path, 'w') as f:
f.write(group)
except Exception as e:
logger.exception(
'Failed to create group file marker: {0}'.format(e))
else:
logger.info('Successfully created group file marker '
'for group {0}'.format(group))
else:
logger.info('Group file marker creation was not requested for '
'group {0}'.format(self.params.get('group')))
if self.params.get('remove_group_file'):
logger.info('Removing group file {0}'.format(
self.params['remove_group_file']))
try:
os.remove(self.params['remove_group_file'])
except Exception as e:
logger.exception('Failed to remove group file: {0}'.format(e))
else:
logger.info('Successfully removed group '
'file {0}'.format(
self.params['remove_group_file']))
else:
logger.info('Group file removal was not requested for '
'group {0}'.format(self.params.get('group')))
if self.params.get('move_dst') and self.params.get('move_src'):
try:
os.rename(self.params['move_src'], self.params['move_dst'])
except Exception as e:
logger.exception(
'Failed to execute move task: {0} to {1}'.format(
self.params['move_src'], self.params['move_dst']))
else:
logger.info(
'Successfully performed move task: {0} to {1}'.format(
self.params['move_src'], self.params['move_dst']))
if self.params.get('mark_backend'):
marker = self.params['mark_backend']
try:
open(marker, 'w').close()
except Exception as e:
logger.error(
'Failed to create backend down marker: {0}'.format(e))
else:
logger.info(
'Successfully created backend down marker: {0}'.format(
self.params['mark_backend']))
if self.params.get('unmark_backend'):
marker = self.params['unmark_backend']
try:
os.remove(self.params['unmark_backend'])
except Exception as e:
logger.error(
'Failed to remove backend down marker: {0}'.format(e))
else:
logger.info(
'Successfully removed backend down marker: {0}'.format(
self.params['unmark_backend']))
def create_group_file_marker(self):
if self.params.get('group_file_marker'):
try:
group = (str(int(self.params.get('group')))
if self.params.get('group') else
'')
path = self.params['group_file_marker'].format(group_id=group)
dirname = os.path.dirname(path)
if not os.path.exists(dirname):
os.makedirs(dirname, 0755)
with open(path, 'w') as f:
f.write(group)
except Exception as e:
logger.exception('Failed to create group file marker: {0}'.format(e))
else:
logger.info('Successfully created group file marker '
'for group {0}'.format(group))
else:
logger.info('Group file marker creation was not requested for '
'group {0}'.format(self.params.get('group')))
if self.params.get('remove_group_file'):
logger.info('Removing group file {0}'.format(self.params['remove_group_file']))
try:
os.remove(self.params['remove_group_file'])
except Exception as e:
logger.exception('Failed to remove group file: {0}'.format(e))
else:
logger.info('Successfully removed group '
'file {0}'.format(self.params['remove_group_file']))
else:
logger.info('Group file removal was not requested for '
'group {0}'.format(self.params.get('group')))
if self.params.get('move_dst') and self.params.get('move_src'):
try:
os.rename(self.params['move_src'], self.params['move_dst'])
except Exception as e:
logger.exception('Failed to execute move task: {0} to {1}'.format(
self.params['move_src'], self.params['move_dst']))
else:
logger.info('Successfully performed move task: {0} to {1}'.format(
self.params['move_src'], self.params['move_dst']))
if self.params.get('mark_backend'):
marker = self.params['mark_backend']
try:
open(marker, 'w').close()
except Exception as e:
logger.error('Failed to create backend down marker: {0}'.format(e))
else:
logger.info('Successfully created backend down marker: {0}'.format(
self.params['mark_backend']))
if self.params.get('unmark_backend'):
marker = self.params['unmark_backend']
try:
os.remove(self.params['unmark_backend'])
except Exception as e:
logger.error('Failed to remove backend down marker: {0}'.format(e))
else:
logger.info('Successfully removed backend down marker: {0}'.format(
self.params['unmark_backend']))
