def status(self): cmd = (shlex.split(self.command.encode('utf-8')) if isinstance(self.command, basestring) else self.command) Subprocess = subprocess.subprocess_factory(cmd) if issubclass(Subprocess, BaseSubprocess): Watcher = Subprocess.watcher_base exit_message = Watcher.exit_messages.get(self.exit_code, 'Unknown') else: exit_message = 'Success' if self.exit_code == 0 else '' artifacts = {} try: artifacts = json.loads(self.artifacts) except ValueError as e: logger.error('Dumped command (uid {}) has mailformed "artifacts" field: {}'.format( self.uid, self.artifacts )) pass return { 'pid': self.pid, 'command': self.command, 'task_id': self.task_id, 'progress': self.progress, 'exit_code': self.exit_code, 'exit_message': exit_message, 'command_code': self.command_code, 'start_ts': self.start_ts, 'finish_ts': self.finish_ts, 'output': self.stdout, 'error_output': self.stderr, 'artifacts': artifacts, }
def execute(self): marker = self.params["unmark_backend"] try: os.remove(marker) except Exception as e: logger.error("Failed to remove backend marker: {}".format(e)) raise logger.info("Successfully removed backend marker: {}".format(marker))
def execute(self): stop_file = self.params['create_stop_file'] try: open(stop_file, 'w').close() except Exception as e: logger.error('Failed to create backend stop marker: {}'.format(e)) raise logger.info('Successfully created backend stop marker: {}'.format(stop_file))
def execute(self): marker = self.params['mark_backend'] try: open(marker, 'w').close() except Exception as e: logger.error('Failed to create backend marker: {}'.format(e)) raise logger.info('Successfully created backend marker: {}'.format(marker))
def execute(self): stop_file = self.params["remove_stop_file"] try: os.remove(stop_file) except Exception as e: logger.error("Failed to remove backend stop marker: {}".format(e)) raise logger.info("Successfully removed backend stop marker: {}".format(stop_file))
def execute(self): logger.info('Removing group file {0}'.format(self.params['remove_group_file'])) try: os.remove(self.params['remove_group_file']) except Exception as e: logger.error('Failed to remove group file: {}'.format(e)) raise logger.info('Successfully removed group file {}'.format(self.params['remove_group_file']))
def execute(self): group = str(self.params.get("group")) if "group" in self.params.get("group") else "" try: path = self.params["group_file_marker"].format(group_id=group) dirname = os.path.dirname(path) if not os.path.exists(dirname): os.makedirs(dirname, 0755) with open(path, "w") as f: f.write(group) except Exception as e: logger.error("Failed to create group file marker: {}".format(e)) raise logger.info("Successfully created group file marker for group {}".format(group))
def execute(self): try: os.rename(self.params['move_src'], self.params['move_dst']) except Exception as e: logger.error('Failed to execute move path command: {} to {}: {}'.format( self.params['move_src'], self.params['move_dst'], e )) logger.info('Successfully performed move task: {} to {}'.format( self.params['move_src'], self.params['move_dst'] ))
def post(self): if config['common'].get('base_path') is None: logger.error( 'base path is not set, remove group cannot be performed') self.set_status(500) raise RuntimeError('group creation is not allowed') params = {k: v[0] for k, v in self.request.arguments.iteritems()} params['group_base_path'] = os.path.normpath(params['group_base_path']) if not params['group_base_path'].startswith( config['common']['base_path']): self.set_status(403) raise RuntimeError( 'Group path {path} is not under common base path'.format( path=params['group_base_path'], )) uid = manager.execute(GroupRemover, cmd='remove_group', params=params) self.set_status(302) self.add_header('Location', self.reverse_url('status', uid))
def post(self): if config['common'].get('base_path') is None: logger.error('base path is not set, remove group cannot be performed') self.set_status(500) raise RuntimeError('group creation is not allowed') params = { k: v[0] for k, v in self.request.arguments.iteritems() } params['group_base_path'] = os.path.normpath(params['group_base_path']) if not params['group_base_path'].startswith(config['common']['base_path']): self.set_status(403) raise RuntimeError( 'Group path {path} is not under common base path'.format( path=params['group_base_path'], ) ) uid = manager.execute(GroupRemover, cmd='remove_group', params=params) self.set_status(302) self.add_header('Location', self.reverse_url('status', uid))
def _parse_command_code(self): if self.watcher.exit_code == 0: return self.watcher.exit_code output = self.watcher.get_stdout() try: data = json.loads(output) except Exception: logger.error('pid {}: failed to parse output json: {}'.format( self.pid, output )) return self.watcher.exit_code if 'error' not in data: logger.error('pid {}: no "error" key in response data'.format( self.pid )) return self.watcher.exit_code if 'code' not in data['error']: logger.error('pid {}: no "code" key in response error data'.format( self.pid )) return self.watcher.exit_code logger.info('pid {}: operation error code {}'.format( self.pid, data['error']['code'] )) return data['error']['code']
def is_success(self): logger.info('pid {0}: exit code {1}'.format( self.subprocess.pid, self.exit_code)) if self.exit_code == 0: return True output = ''.join(self.output) try: data = json.loads(output) except Exception as e: logger.error('pid {0}: failed to parse output json: {1}'.format( self.subprocess.pid, output)) return False if not 'error' in data: logger.error('pid {0}: no "error" key in response data'.format( self.subprocess.pid)) return False if not 'code' in data['error']: logger.error('pid {0}: no "code" key in response error data'.format( self.subprocess.pid)) return False logger.info('pid {0}: operation error code {1}'.format( self.subprocess.pid, data['error']['code'])) return data['error']['code'] in self.success_codes
def is_success(self): logger.info('pid {0}: exit code {1}'.format(self.subprocess.pid, self.exit_code)) if self.exit_code == 0: return True output = ''.join(self.output) try: data = json.loads(output) except Exception as e: logger.error('pid {0}: failed to parse output json: {1}'.format( self.subprocess.pid, output)) return False if not 'error' in data: logger.error('pid {0}: no "error" key in response data'.format( self.subprocess.pid)) return False if not 'code' in data['error']: logger.error( 'pid {0}: no "code" key in response error data'.format( self.subprocess.pid)) return False logger.info('pid {0}: operation error code {1}'.format( self.subprocess.pid, data['error']['code'])) return data['error']['code'] in self.success_codes
def post(self): if config['common'].get('base_path') is None: logger.error('base path is not set, create group cannot be performed') self.set_status(500) raise RuntimeError('group creation is not allowed') files = {} for filename, http_files in self.request.files.iteritems(): norm_filename = os.path.normpath(filename) if norm_filename.startswith('..') or norm_filename.startswith('/'): logger.error( 'Cannot create file {filename}, ' 'normalized path {norm_filename} is not allowed'.format( filename=filename, norm_filename=norm_filename, ) ) self.set_status(403) raise RuntimeError( 'File {filename} is forbidden, path should be relative ' 'to group base directory'.format(filename=filename) ) http_file = http_files[0] files[norm_filename] = http_file.body params = { k: v[0] for k, v in self.request.arguments.iteritems() } params['group_base_path_root_dir'] = os.path.normpath(params['group_base_path_root_dir']) if not params['group_base_path_root_dir'].startswith(config['common']['base_path']): self.set_status(403) raise RuntimeError( 'Group base path {path} is not under common base path'.format( path=params['group_base_path_root_dir'], ) ) params['files'] = files uid = manager.execute(GroupCreator, cmd='create_group', params=params) self.set_status(302) self.add_header('Location', self.reverse_url('status', uid))
def wrapped(self, *args, **kwargs): try: self.add_header('Content-Type', 'text/json') try: res = method(self, *args, **kwargs) except Exception as e: logger.error('{0}: {1}'.format(e, traceback.format_exc(e))) response = {'status': 'error', 'error': str(e)} else: response = {'status': 'success', 'response': res} try: self.write(json.dumps(response)) except Exception as e: logger.error('Failed to dump json response: {0}\n{1}'.format(e, traceback.format_exc(e))) response = {'status': 'error', 'error': 'failed to construct response, see log file'} self.write(json.dumps(response)) finally: self.finish()
def wrapped(self, *args, **kwargs): try: self.add_header('Content-Type', 'text/json') try: res = method(self, *args, **kwargs) except Exception as e: logger.error('{0}: {1}'.format(e, traceback.format_exc(e))) response = {'status': 'error', 'error': str(e)} else: response = {'status': 'success', 'response': res} try: self.write(json.dumps(response)) except Exception as e: logger.error('Failed to dump json response: {0}\n{1}'.format( e, traceback.format_exc(e))) response = { 'status': 'error', 'error': 'failed to construct response, see log file' } self.write(json.dumps(response)) finally: self.finish()
def post(self): if config['common'].get('base_path') is None: logger.error( 'base path is not set, create group cannot be performed') self.set_status(500) raise RuntimeError('group creation is not allowed') files = {} for filename, http_files in self.request.files.iteritems(): norm_filename = os.path.normpath(filename) if norm_filename.startswith('..') or norm_filename.startswith('/'): logger.error( 'Cannot create file {filename}, ' 'normalized path {norm_filename} is not allowed'.format( filename=filename, norm_filename=norm_filename, )) self.set_status(403) raise RuntimeError( 'File {filename} is forbidden, path should be relative ' 'to group base directory'.format(filename=filename)) http_file = http_files[0] files[norm_filename] = http_file.body params = {k: v[0] for k, v in self.request.arguments.iteritems()} params['group_base_path_root_dir'] = os.path.normpath( params['group_base_path_root_dir']) if not params['group_base_path_root_dir'].startswith( config['common']['base_path']): self.set_status(403) raise RuntimeError( 'Group base path {path} is not under common base path'.format( path=params['group_base_path_root_dir'], )) params['files'] = files uid = manager.execute(GroupCreator, cmd='create_group', params=params) self.set_status(302) self.add_header('Location', self.reverse_url('status', uid))
def create_group_file_marker(self): if self.params.get('group_file_marker'): try: group = (str(int(self.params.get('group'))) if self.params.get('group') else '') path = self.params['group_file_marker'].format(group_id=group) dirname = os.path.dirname(path) if not os.path.exists(dirname): os.makedirs(dirname, 0755) with open(path, 'w') as f: f.write(group) except Exception as e: logger.exception( 'Failed to create group file marker: {0}'.format(e)) else: logger.info('Successfully created group file marker ' 'for group {0}'.format(group)) else: logger.info('Group file marker creation was not requested for ' 'group {0}'.format(self.params.get('group'))) if self.params.get('remove_group_file'): logger.info('Removing group file {0}'.format( self.params['remove_group_file'])) try: os.remove(self.params['remove_group_file']) except Exception as e: logger.exception('Failed to remove group file: {0}'.format(e)) else: logger.info('Successfully removed group ' 'file {0}'.format( self.params['remove_group_file'])) else: logger.info('Group file removal was not requested for ' 'group {0}'.format(self.params.get('group'))) if self.params.get('move_dst') and self.params.get('move_src'): try: os.rename(self.params['move_src'], self.params['move_dst']) except Exception as e: logger.exception( 'Failed to execute move task: {0} to {1}'.format( self.params['move_src'], self.params['move_dst'])) else: logger.info( 'Successfully performed move task: {0} to {1}'.format( self.params['move_src'], self.params['move_dst'])) if self.params.get('mark_backend'): marker = self.params['mark_backend'] try: open(marker, 'w').close() except Exception as e: logger.error( 'Failed to create backend down marker: {0}'.format(e)) else: logger.info( 'Successfully created backend down marker: {0}'.format( self.params['mark_backend'])) if self.params.get('unmark_backend'): marker = self.params['unmark_backend'] try: os.remove(self.params['unmark_backend']) except Exception as e: logger.error( 'Failed to remove backend down marker: {0}'.format(e)) else: logger.info( 'Successfully removed backend down marker: {0}'.format( self.params['unmark_backend']))
def create_group_file_marker(self): if self.params.get('group_file_marker'): try: group = (str(int(self.params.get('group'))) if self.params.get('group') else '') path = self.params['group_file_marker'].format(group_id=group) dirname = os.path.dirname(path) if not os.path.exists(dirname): os.makedirs(dirname, 0755) with open(path, 'w') as f: f.write(group) except Exception as e: logger.exception('Failed to create group file marker: {0}'.format(e)) else: logger.info('Successfully created group file marker ' 'for group {0}'.format(group)) else: logger.info('Group file marker creation was not requested for ' 'group {0}'.format(self.params.get('group'))) if self.params.get('remove_group_file'): logger.info('Removing group file {0}'.format(self.params['remove_group_file'])) try: os.remove(self.params['remove_group_file']) except Exception as e: logger.exception('Failed to remove group file: {0}'.format(e)) else: logger.info('Successfully removed group ' 'file {0}'.format(self.params['remove_group_file'])) else: logger.info('Group file removal was not requested for ' 'group {0}'.format(self.params.get('group'))) if self.params.get('move_dst') and self.params.get('move_src'): try: os.rename(self.params['move_src'], self.params['move_dst']) except Exception as e: logger.exception('Failed to execute move task: {0} to {1}'.format( self.params['move_src'], self.params['move_dst'])) else: logger.info('Successfully performed move task: {0} to {1}'.format( self.params['move_src'], self.params['move_dst'])) if self.params.get('mark_backend'): marker = self.params['mark_backend'] try: open(marker, 'w').close() except Exception as e: logger.error('Failed to create backend down marker: {0}'.format(e)) else: logger.info('Successfully created backend down marker: {0}'.format( self.params['mark_backend'])) if self.params.get('unmark_backend'): marker = self.params['unmark_backend'] try: os.remove(self.params['unmark_backend']) except Exception as e: logger.error('Failed to remove backend down marker: {0}'.format(e)) else: logger.info('Successfully removed backend down marker: {0}'.format( self.params['unmark_backend']))