def run(self, base_dir):
"""
Compute and launch command line.
"""
self._setup(base_dir)
args = self.get_process_args()
args = [str(arg) for arg in args]
cmd = ' '.join(args)
self.log.info('Running command: %s', cmd)
self.command = Command(base_dir)
self.command.run(args)
self.is_running = True
def run(self, base_dir):
"""
Compute and launch command line.
"""
self._setup(base_dir)
args = self.get_process_args()
args = [str(arg) for arg in args]
cmd = " ".join(args)
self.log.info("Running command: %s", cmd)
self.command = Command(base_dir)
self.command.memory_limit = self.memory_limit
self.command.kill_timeout = self.kill_timeout
self.command.run(args)
self.is_running = True
def run(self, base_dir):
"""
Compute and launch command line.
"""
self._setup(base_dir)
args = self.get_process_args()
args = [str(arg) for arg in args]
cmd = ' '.join(args)
self.log.info('Running command: %s', cmd)
self.command = Command(base_dir)
self.command.run(args)
self.is_running = True
class Worker(object):
class File(object):
def __init__(self, path, params=None):
self.path = path
self.params = params or {}
def get_args(self):
return [self.path]
class InputFile(File):
pass
class OutputFile(File):
pass
def __init__(self, log, params):
self.log = log
self.params = params or {}
self.command = None
self.tool = None
self.is_running = False
self.input_files = []
self.output_files = []
self.time = 0
self.timeleft = 0
self.progress = 0
def add_input_file(self, path, params=None):
"""
Add an input file with associated parameters.
"""
self.input_files.append(self.InputFile(path, params))
def add_output_file(self, path, params=None):
"""
Add an output file with associated parameters.
"""
self.output_files.append(self.OutputFile(path, params))
def _handle_output(self, stdout, stderr):
"""
Virtual method to handle stdout and stderr from running process.
Compute progress and timeleft here.
"""
raise NotImplementedError
def get_args(self):
"""
Return command line parameters as a string list.
"""
args = []
for key, value in self.params.iteritems():
args.append(key)
if not value is None:
args.append(value)
return args
def get_process_args(self):
"""
Return command line as a string list.
"""
args = [self.tool]
args += self.get_args()
return args
def get_error(self):
"""
Virtual method to fetch and return error from exited process.
"""
raise NotImplementedError
def _setup(self, base_dir):
"""
Virtual method called by run method.
"""
raise NotImplementedError
def run(self, base_dir):
"""
Compute and launch command line.
"""
self._setup(base_dir)
args = self.get_process_args()
args = [str(arg) for arg in args]
cmd = ' '.join(args)
self.log.info('Running command: %s', cmd)
self.command = Command(base_dir)
self.command.run(args)
self.is_running = True
def wait(self, read=4096, timeout=3600):
"""
Wait running process. If an error occurs raise a WorkerException,
otherwise returns 0
"""
ret = self.command.wait(self._handle_output, read, timeout)
if ret != 0:
error = self.get_error()
raise WorkerException(error)
return ret
def wait_noloop(self, read=4096, timeout=3600):
"""
Wait (non-blocking) running process and return its exit code.
If process has not exited yet, this method returns None otherwise,
it returns its exit code.
"""
return self.command.wait(self._handle_output,
read,
timeout,
loop=False)
class Worker(object):
class File(object):
def __init__(self, path, params=None):
self.path = path
self.params = params or {}
def get_args(self):
return [self.path]
class InputFile(File):
pass
class OutputFile(File):
pass
def __init__(self, log, params=None):
"""
Create a new worker.
:param log: logger instance to use
:type log: logging.Logger
:param params: worker parameters
:type params: dict
"""
self.log = log
self.params = params or {}
self.command = None
self.tool = None
self.is_running = False
self.input_files = []
self.output_files = []
self.time = 0
self.timeleft = 0
self.progress = 0
self.memory_limit = 0
self.kill_timeout = COMMAND_DEFAULT_KILL_TIMEOUT
self.stdout = ''
self.stderr = ''
self.error_lines = 1
def add_input_file(self, path, params=None):
"""
Add an input file with associated parameters.
:param path: absolute path of the input file.
:type path: string
:param params: parameters associated with input file
:type params: dict
"""
self.input_files.append(self.InputFile(path, params))
def add_output_file(self, path, params=None):
"""
Add an output file with associated parameters.
:param path: absolute path of the input file.
:type path: string
:param params: parameters associated with input file
:type params: dict
"""
self.output_files.append(self.OutputFile(path, params))
def _handle_output(self, stdout, stderr):
"""
Store stdout and stderr from command line.
"""
self.stdout += stdout
self.stderr += stderr
def get_args(self):
"""
Return command line parameters as a string list.
"""
args = []
for key, value in self.params.iteritems():
args.append(key)
if value != None and value != '':
args.append(value)
return args
def get_process_args(self):
"""
Return command line as a string list.
"""
args = [self.tool]
args += self.get_args()
return args
def get_error(self):
"""
Return the last lines from stderr. The number of lines returned
could be configured with Worker.error_lines attribute.
"""
lines = self.stderr.split('\n')
lines.reverse()
i = 0
error_lines = []
for line in lines:
if not line == '':
i += 1
error_lines.append(line)
if i >= self.error_lines:
break
error_lines.reverse()
return '\n'.join(error_lines)
def _setup(self, base_dir):
"""
Initialize the worker. This method should be implemented by subclasses
if needed.
"""
pass
def _finalize(self):
"""
Do some processing at the end of the worker job. This method
should be implemented by subclasses.
"""
pass
def run(self, base_dir):
"""
Compute and launch command line.
"""
self._setup(base_dir)
args = self.get_process_args()
args = [str(arg) for arg in args]
cmd = ' '.join(args)
self.log.info('Running command: %s', cmd)
self.command = Command(base_dir)
self.command.memory_limit = self.memory_limit
self.command.kill_timeout = self.kill_timeout
self.command.run(args)
self.is_running = True
def wait(self):
"""
Wait running process. If an error occurs raise a WorkerException,
otherwise returns 0
"""
ret = self.command.wait(self._handle_output)
if ret != 0:
error = self.get_error()
raise WorkerException(error)
self._finalize()
return ret
def wait_noloop(self):
"""
Wait (non-blocking) running process and return its exit code.
If process has not exited yet, this method returns None otherwise,
it returns its exit code.
"""
ret = self.command.wait(self._handle_output, loop=False)
if ret == 0:
self._finalize()
return ret
class Worker(object):
class File(object):
def __init__(self, path, params=None):
self.path = path
self.params = params or {}
def get_args(self):
return [self.path]
class InputFile(File):
pass
class OutputFile(File):
pass
def __init__(self, log, params):
self.log = log
self.params = params or {}
self.command = None
self.tool = None
self.is_running = False
self.input_files = []
self.output_files = []
self.time = 0
self.timeleft = 0
self.progress = 0
def add_input_file(self, path, params=None):
"""
Add an input file with associated parameters.
"""
self.input_files.append(self.InputFile(path, params))
def add_output_file(self, path, params=None):
"""
Add an output file with associated parameters.
"""
self.output_files.append(self.OutputFile(path, params))
def _handle_output(self, stdout, stderr):
"""
Virtual method to handle stdout and stderr from running process.
Compute progress and timeleft here.
"""
raise NotImplementedError
def get_args(self):
"""
Return command line parameters as a string list.
"""
args = []
for key, value in self.params.iteritems():
args.append(key)
if not value is None:
args.append(value)
return args
def get_process_args(self):
"""
Return command line as a string list.
"""
args = [self.tool]
args += self.get_args()
return args
def get_error(self):
"""
Virtual method to fetch and return error from exited process.
"""
raise NotImplementedError
def _setup(self, base_dir):
"""
Virtual method called by run method.
"""
raise NotImplementedError
def run(self, base_dir):
"""
Compute and launch command line.
"""
self._setup(base_dir)
args = self.get_process_args()
args = [str(arg) for arg in args]
cmd = ' '.join(args)
self.log.info('Running command: %s', cmd)
self.command = Command(base_dir)
self.command.run(args)
self.is_running = True
def wait(self, read=4096, timeout=3600):
"""
Wait running process. If an error occurs raise a WorkerException,
otherwise returns 0
"""
ret = self.command.wait(self._handle_output, read, timeout)
if ret != 0:
error = self.get_error()
raise WorkerException(error)
return ret
def wait_noloop(self, read=4096, timeout=3600):
"""
Wait (non-blocking) running process and return its exit code.
If process has not exited yet, this method returns None otherwise,
it returns its exit code.
"""
return self.command.wait(self._handle_output, read, timeout, loop=False)
class Worker(object):
class File(object):
def __init__(self, path, params=None):
self.path = path
self.params = params or {}
def get_args(self):
return [self.path]
class InputFile(File):
pass
class OutputFile(File):
pass
def __init__(self, log, params=None):
"""
Create a new worker.
:param log: logger instance to use
:type log: logging.Logger
:param params: worker parameters
:type params: dict
"""
self.log = log
self.params = params or {}
self.command = None
self.tool = None
self.is_running = False
self.input_files = []
self.output_files = []
self.time = 0
self.timeleft = 0
self.progress = 0
self.memory_limit = 0
self.kill_timeout = COMMAND_DEFAULT_KILL_TIMEOUT
self.stdout = ""
self.stderr = ""
self.error_lines = 1
def add_input_file(self, path, params=None):
"""
Add an input file with associated parameters.
:param path: absolute path of the input file.
:type path: string
:param params: parameters associated with input file
:type params: dict
"""
self.input_files.append(self.InputFile(path, params))
def add_output_file(self, path, params=None):
"""
Add an output file with associated parameters.
:param path: absolute path of the input file.
:type path: string
:param params: parameters associated with input file
:type params: dict
"""
self.output_files.append(self.OutputFile(path, params))
def _handle_output(self, stdout, stderr):
"""
Store stdout and stderr from command line.
"""
self.stdout += stdout
self.stderr += stderr
def get_args(self):
"""
Return command line parameters as a string list.
"""
args = []
for key, value in list(self.params.items()):
args.append(key)
if value != None and value != "":
args.append(value)
return args
def get_process_args(self):
"""
Return command line as a string list.
"""
args = [self.tool]
args += self.get_args()
return args
def get_error(self):
"""
Return the last lines from stderr. The number of lines returned
could be configured with Worker.error_lines attribute.
"""
lines = self.stderr.split("\n")
lines.reverse()
i = 0
error_lines = []
for line in lines:
if not line == "":
i += 1
error_lines.append(line)
if i >= self.error_lines:
break
error_lines.reverse()
return "\n".join(error_lines)
def _setup(self, base_dir):
"""
Initialize the worker. This method should be implemented by subclasses
if needed.
"""
pass
def _finalize(self):
"""
Do some processing at the end of the worker job. This method
should be implemented by subclasses.
"""
pass
def run(self, base_dir):
"""
Compute and launch command line.
"""
self._setup(base_dir)
args = self.get_process_args()
args = [str(arg) for arg in args]
cmd = " ".join(args)
self.log.info("Running command: %s", cmd)
self.command = Command(base_dir)
self.command.memory_limit = self.memory_limit
self.command.kill_timeout = self.kill_timeout
self.command.run(args)
self.is_running = True
def wait(self):
"""
Wait running process. If an error occurs raise a WorkerException,
otherwise returns 0
"""
ret = self.command.wait(self._handle_output)
if ret != 0:
error = self.get_error()
raise WorkerException(error)
self._finalize()
return ret
def cancel(self):
"""Cancel a running command"""
self.command.cancel()
def wait_noloop(self):
"""
Wait (non-blocking) running process and return its exit code.
If process has not exited yet, this method returns None otherwise,
it returns its exit code.
"""
ret = self.command.wait(self._handle_output, loop=False)
if ret == 0:
self._finalize()
return ret
