def checkJob(self, module, id, handle):
""" checkJob(module: VistrailsModule, id: str, handle: object)
Callback, checks if job has completed.
"""
workflow = self.jobMonitor.currentWorkflow()
if not workflow:
if not handle or not self.jobMonitor.isDone(handle):
raise ModuleSuspended(module, 'Job is running', handle=handle)
workflow_item = self.workflowItems[workflow.id]
item = workflow_item.jobs.get(id, None)
item.setText(0, item.job.name)
# we should check the status using the JobHandle and show dialog
# get current view progress bar and hijack it
if handle:
item.handle = handle
workflow = self.jobMonitor.currentWorkflow()
workflow_item = self.workflowItems.get(workflow.id, None)
workflow_item.updateJobs()
progress = self.controller.progress
conf = configuration.get_vistrails_configuration()
interval = conf.jobCheckInterval
if interval and not conf.jobAutorun and not progress.suspended:
# we should keep checking the job
if handle:
# wait for module to complete
labelText = (("Running external job %s\n"
"Started %s\n"
"Press Cancel to suspend") %
(item.job.name, item.job.start))
progress.setLabelText(labelText)
while not self.jobMonitor.isDone(handle):
i = 0
while i < interval:
i += 1
time.sleep(1)
QtCore.QCoreApplication.processEvents()
if progress.wasCanceled():
# this does not work, need to create a new progress dialog
#progress.goOn()
new_progress = progress.__class__(
progress.parent())
new_progress.setMaximum(progress.maximum())
new_progress.setValue(progress.value())
new_progress.setLabelText(labelText)
new_progress.setMinimumDuration(0)
new_progress.suspended = True
self.controller.progress = new_progress
progress.hide()
progress.deleteLater()
progress = new_progress
progress.show()
QtCore.QCoreApplication.processEvents()
raise ModuleSuspended(module,
'Interrupted by user, job'
' is still running',
handle=handle)
return
if not handle or not self.jobMonitor.isDone(handle):
raise ModuleSuspended(module, 'Job is running', handle=handle)
def checkJob(self, module, id, handle):
""" checkJob(module: VistrailsModule, id: str, handle: object) -> None
Starts monitoring the job for the current running workflow
module - the module to suspend
id - the job identifier
handle - an object following the JobHandle interface, i.e. with a
finished method for checking if the job has completed
"""
if not self.currentWorkflow():
if not handle or not self.isDone(handle):
raise ModuleSuspended(module, 'Job is running', handle=handle)
job = self.getJob(id)
if self.callback:
self.callback.checkJob(module, id, handle)
return
conf = get_vistrails_configuration()
interval = conf.jobCheckInterval
if interval and not conf.jobAutorun:
if handle:
# wait for module to complete
try:
while not self.isDone(handle):
time.sleep(interval)
print("Waiting for job: %s,"
"press Ctrl+C to suspend") % job.name
except KeyboardInterrupt:
raise ModuleSuspended(module, 'Interrupted by user, job'
' is still running',
handle=handle)
else:
if not handle or not self.isDone(handle):
raise ModuleSuspended(module, 'Job is running', handle=handle)
def call_hadoop(self, arguments, workdir, identifier, machine):
self.is_cacheable = lambda *args, **kwargs: False
config = self.get_hadoop_config(machine)
argList = [config['hadoop']]
if type(arguments) in [str, unicode]:
argList += arguments.split(' ')
elif type(arguments) == list:
argList += arguments
else:
raise ModuleError(self, 'Invalid argument types to hadoop')
# 1. this version returns when finished
#return subprocess.call(argList)
# 2. this version reads the results incrementally
# expect = machine.remote._expect_token
# machine.remote.push_expect(None) # Do not wait for call to finish
# result = machine.remote.send_command(" ".join(argList)).strip()
# machine.remote.pop_expect() # restore expect
# # We could show the output in a gui
# print "**** hadoop streaming running ****"
# print result,
# while not expect in result:
# output = machine.remote.consume_output()
# if output:
# print output,
# result += output
# 3. The final version should detach the process on the server
use_machine(machine)
cdir = CreateDirectory("remote", workdir)
job = Subshell("remote",
command=" ".join(argList),
working_directory=workdir,
identifier=identifier,
dependencies=[cdir])
job.run()
finished = job.finished()
if not finished:
status = job.status()
# The Subshell class provides the JobHandle interface, i.e.
# finished()
raise ModuleSuspended(self, '%s' % status, handle=job)
self.is_cacheable = lambda *args, **kwargs: True
return job.standard_error()
def compute(self):
queue = self.get_input('queue')
job_id = self.get_input('id')
# Check job status
try:
with ServerLogger.hide_output():
status, target, arg = queue.status(job_id)
except tej.JobNotFound:
raise ModuleError(self, "Job not found")
# Create job object
job = RemoteJob(queue=queue, job_id=job_id)
if status == tej.RemoteQueue.JOB_DONE:
self.set_output('job', job)
self.set_output('exitcode', int(arg))
elif status == tej.RemoteQueue.JOB_RUNNING:
raise ModuleSuspended(self, "Remote job is running", handle=job)
else:
raise ModuleError(self, "Invalid job status %r" % status)
was_suspended = e
except ModuleSuspended, e:
suspended.append(e)
else:
for connector in connector_list:
try:
connector.obj.update()
except ModuleWasSuspended, e:
was_suspended = e
except ModuleSuspended, e:
suspended.append(e)
if len(suspended) == 1:
raise suspended[0]
elif suspended:
raise ModuleSuspended(
self,
"multiple suspended upstream modules",
children=suspended)
elif was_suspended is not None:
raise was_suspended
for port_name, connectorList in list(self.inputPorts.items()):
if port_name != 'FunctionPort':
for connector in connectorList:
if connector.obj.get_output(connector.port) is \
InvalidOutput: # pragma: no cover
self.remove_input_connector(port_name, connector)
def compute(self):
name_output = self.get_input('OutputPort')
name_condition = self.force_get_input('ConditionPort')
name_state_input = self.force_get_input('StateInputPorts')
name_state_output = self.force_get_input('StateOutputPorts')
was_suspended = e
except ModuleSuspended, e:
suspended.append(e)
else:
for connector in connector_list:
try:
connector.obj.update()
except ModuleWasSuspended, e:
was_suspended = e
except ModuleSuspended, e:
suspended.append(e)
if len(suspended) == 1:
raise suspended[0]
elif suspended:
raise ModuleSuspended(self,
"multiple suspended upstream modules",
children=suspended)
elif was_suspended is not None:
raise was_suspended
for port_name, connectorList in list(self.inputPorts.items()):
if port_name != 'FunctionPort':
for connector in connectorList:
if connector.obj.get_output(connector.port) is \
InvalidOutput: # pragma: no cover
self.remove_input_connector(port_name, connector)
def updateFunctionPort(self):
"""
Function to be used inside the updateUsptream method of the
FoldWithModule module. It updates the modules connected to the
FunctionPort port.
def compute(self):
# Check required attributes
if not hasattr(self, 'pipeline') or self.pipeline is None:
raise VistrailsInternalError(
"%s cannot execute -- pipeline doesn't exist" %
self.__class__.__name__)
elif (not hasattr(self, 'output_remap') or self.output_remap is None or
not hasattr(self, 'input_remap') or self.input_remap is None):
raise VistrailsInternalError(
"%s cannot execute -- remap dictionaries don't exist" %
self.__class__.__name__)
# Setup pipeline for execution
res = self.interpreter.setup_pipeline(self.pipeline)
self.persistent_modules = res[0].values()
if len(res[5]) > 0:
raise ModuleError(self, "Error(s) inside group:\n" +
"\n".join(me.msg for me in res[5].itervalues()))
tmp_id_to_module_map = res[0]
# Connect Group's external input ports to internal InputPort modules
for iport_name, conn in self.inputPorts.iteritems():
iport_module = self.input_remap[iport_name]
iport_obj = tmp_id_to_module_map[iport_module.id]
iport_obj.set_input_port('ExternalPipe', conn[0])
# Execute pipeline
kwargs = {'logger': self.logging.log.recursing(self),
'clean_pipeline': True,
'current_version': self.moduleInfo['version']}
module_info_args = set(['locator', 'reason', 'extra_info', 'actions'])
for arg in module_info_args:
if arg in self.moduleInfo:
kwargs[arg] = self.moduleInfo[arg]
res = self.interpreter.execute_pipeline(self.pipeline,
*res[:2],
**kwargs)
# Check and propagate errors
if len(res[2]) > 0:
raise ModuleError(self, "Error(s) inside group:\n" +
"\n".join("%s: %s" % (
me.module.__class__.__name__, me.msg)
for me in res[2].itervalues()))
# Check and propagate ModuleSuspended exceptions
if res[4]:
message = "\n".join([ms.msg for ms in res[4].itervalues()])
children = list(res[4].values())
raise ModuleSuspended(self, message, children=children)
# Connect internal OutputPort modules to Group's external output ports
for oport_name, oport_module in self.output_remap.iteritems():
if oport_name is not 'self':
oport_obj = tmp_id_to_module_map[oport_module.id]
self.set_output(oport_name,
oport_obj.get_output('ExternalPipe'))
self.interpreter.finalize_pipeline(self.pipeline, *res[:-1],
reset_computed=False)
def compute(self):
machine = self.get_machine()
if not self.has_input('command'):
raise ModuleError(self, "No command specified")
command = self.get_input('command').strip()
working_directory = self.get_input('working_directory') \
if self.has_input('working_directory') else '.'
if not self.has_input('input_directory'):
raise ModuleError(self, "No input directory specified")
input_directory = self.get_input('input_directory').strip()
additional_arguments = {
'processes': 1,
'time': -1,
'mpi': False,
'threads': 1,
'memory': -1,
'diskspace': -1
}
for k in additional_arguments:
if self.has_input(k):
additional_arguments[k] = self.get_input(k)
## This indicates that the coming commands submitted on the machine
# trick to select machine without initializing every time
use_machine(machine)
cdir = CreateDirectory("remote", working_directory)
trans = TransferFiles("remote",
input_directory,
working_directory,
dependencies=[cdir])
job = PBS("remote",
command,
working_directory,
dependencies=[trans],
**additional_arguments)
job.run()
ret = job._ret
if ret:
try:
job_id = int(ret)
except ValueError:
end_machine()
raise ModuleError(self, "Error submitting job: %s" % ret)
finished = job.finished()
job_info = job.get_job_info()
if job_info:
self.annotate({'job_info': job.get_job_info()})
if not finished:
status = job.status()
# try to get more detailed information about the job
# this only seems to work on some versions of torque
if job_info:
comment = [
line for line in job_info.split('\n')
if line.startswith('comment =')
]
if comment:
status += ': ' + comment[10:]
end_machine()
# The PBS class provides the JobHandle interface, i.e. finished()
raise ModuleSuspended(self, '%s' % status, handle=job)
# copies the created files to the client
get_result = TransferFiles("local",
input_directory,
working_directory,
dependencies=[cdir])
get_result.run()
## Popping from the machine stack
end_machine()
self.set_output("stdout", job.standard_output())
self.set_output("stderr", job.standard_error())
files = machine.local.send_command("ls -l %s" % input_directory)
self.set_output("file_list",
[f.split(' ')[-1] for f in files.split('\n')[1:]])
def checkJob(self, module, id, handle):
""" checkJob(module: VistrailsModule, id: str, handle: object)
Callback, checks if job has completed.
"""
workflow = self.jobMonitor.currentWorkflow()
if not workflow:
if not self.jobMonitor.isDone(handle):
raise ModuleSuspended(module, 'Job is running', handle=handle)
workflow_item = self.workflowItems[workflow.id]
item = workflow_item.jobs.get(id, None)
item.setText(0, item.job.name)
# we should check the status using the JobHandle and show dialog
# get current view progress bar and hijack it
item.handle = handle
workflow = self.jobMonitor.currentWorkflow()
workflow_item = self.workflowItems.get(workflow.id, None)
workflow_item.updateJobs()
conf = get_vistrails_configuration()
interval = conf.jobCheckInterval
is_done = self.jobMonitor.isDone(handle)
old_progress = self.controller.progress
if (interval and not conf.jobAutorun and not old_progress.suspended
and not is_done):
# wait for module to complete
old_progress.hide()
progress = QJobProgressDialog(item.job.name, item.job.start,
old_progress.parent(),
old_progress.value(),
old_progress.maximum())
progress.show()
while not is_done:
dieTime = QtCore.QDateTime.currentDateTime().addSecs(interval)
while QtCore.QDateTime.currentDateTime(
) < dieTime and not is_done:
QtCore.QCoreApplication.processEvents(
QtCore.QEventLoop.AllEvents, 100)
if progress.was_checked:
progress.check_now_button.setText('Checking')
progress.check_now_button.setEnabled(False)
progress.cancel_button.setEnabled(False)
is_done = self.jobMonitor.isDone(handle)
progress.check_now_button.setText('Check &Now')
progress.check_now_button.setEnabled(True)
progress.cancel_button.setEnabled(True)
progress.was_checked = False
progress.updateLabel()
elif progress.was_cancelled:
# this does not work, need to create a new progress dialog
#old_progress.goOn()
new_progress = old_progress.__class__(
old_progress.parent())
new_progress.setMaximum(old_progress.maximum())
new_progress.setValue(old_progress.value())
new_progress.setLabelText(old_progress.labelText())
new_progress.setMinimumDuration(0)
new_progress.suspended = True
self.controller.progress = new_progress
old_progress.deleteLater()
progress.hide()
new_progress.show()
QtCore.QCoreApplication.processEvents()
raise ModuleSuspended(module,
'Interrupted by user, job'
' is still running',
handle=handle)
is_done = is_done or self.jobMonitor.isDone(handle)
# is_done!
new_progress = old_progress.__class__(old_progress.parent())
new_progress.setMaximum(old_progress.maximum())
new_progress.setValue(old_progress.value())
new_progress.setLabelText(old_progress.labelText())
new_progress.setMinimumDuration(0)
new_progress.suspended = True
self.controller.progress = new_progress
old_progress.deleteLater()
progress.hide()
new_progress.show()
QtCore.QCoreApplication.processEvents()
return
if not is_done:
raise ModuleSuspended(module, 'Job is running', handle=handle)