def compute(self):
machine = self.get_machine()
jm = self.job_monitor()
cache = jm.getCache(self.signature)
if not cache:
if not self.has_input('local_directory'):
raise ModuleError(self, "No local directory specified")
local_directory = self.get_input('local_directory').strip()
if not self.has_input('remote_directory'):
raise ModuleError(self, "No remote directory specified")
remote_directory = self.get_input('remote_directory').strip()
whereto = 'remote'
if self.has_input('to_local') and self.get_input('to_local'):
whereto = 'local'
use_machine(machine)
to_dir = local_directory if whereto=='local' else remote_directory
cdir = CreateDirectory(whereto, to_dir)
job = TransferFiles(whereto, local_directory, remote_directory,
dependencies = [cdir])
job.run()
end_machine()
d = {}
self.set_job_machine(d, machine)
cache = jm.setCache(self.signature, d, self.job_name())
self.set_output("machine", machine)
def compute(self):
machine = self.get_machine()
jm = self.job_monitor()
cache = jm.getCache(self.signature)
if not cache:
if not self.has_input('local_directory'):
raise ModuleError(self, "No local directory specified")
local_directory = self.get_input('local_directory').strip()
if not self.has_input('remote_directory'):
raise ModuleError(self, "No remote directory specified")
remote_directory = self.get_input('remote_directory').strip()
whereto = 'remote'
if self.has_input('to_local') and self.get_input('to_local'):
whereto = 'local'
use_machine(machine)
to_dir = local_directory if whereto == 'local' else remote_directory
cdir = CreateDirectory(whereto, to_dir)
job = TransferFiles(whereto,
local_directory,
remote_directory,
dependencies=[cdir])
job.run()
end_machine()
d = {}
self.set_job_machine(d, machine)
cache = jm.setCache(self.signature, d, self.job_name())
self.set_output("machine", machine)
def compute(self):
machine = self.get_machine()
if not self.has_input('command'):
raise ModuleError(self, "No command specified")
command = self.get_input('command').strip()
working_directory = self.get_input('working_directory') \
if self.has_input('working_directory') else '.'
if not self.has_input('input_directory'):
raise ModuleError(self, "No input directory specified")
input_directory = self.get_input('input_directory').strip()
additional_arguments = {'processes': 1, 'time': -1, 'mpi': False,
'threads': 1, 'memory':-1, 'diskspace': -1}
for k in additional_arguments:
if self.has_input(k):
additional_arguments[k] = self.get_input(k)
## This indicates that the coming commands submitted on the machine
# trick to select machine without initializing every time
use_machine(machine)
cdir = CreateDirectory("remote", working_directory)
trans = TransferFiles("remote", input_directory, working_directory,
dependencies = [cdir])
job = PBS("remote", command, working_directory, dependencies = [trans],
**additional_arguments)
job.run()
ret = job._ret
if ret:
try:
job_id = int(ret)
except ValueError:
end_machine()
raise ModuleError(self, "Error submitting job: %s" % ret)
finished = job.finished()
job_info = job.get_job_info()
if job_info:
self.annotate({'job_info': job.get_job_info()})
if not finished:
status = job.status()
# try to get more detailed information about the job
# this only seems to work on some versions of torque
if job_info:
comment = [line for line in job_info.split('\n') if line.startswith('comment =')]
if comment:
status += ': ' + comment[10:]
end_machine()
# The PBS class provides the JobHandle interface, i.e. finished()
raise ModuleSuspended(self, '%s' % status, handle=job)
# copies the created files to the client
get_result = TransferFiles("local", input_directory, working_directory,
dependencies = [cdir])
get_result.run()
## Popping from the machine stack
end_machine()
self.set_output("stdout", job.standard_output())
self.set_output("stderr", job.standard_error())
files = machine.local.send_command("ls -l %s" % input_directory)
self.set_output("file_list",
[f.split(' ')[-1] for f in files.split('\n')[1:]])
def job_finish(self, params):
job_info = self.job.get_job_info()
if job_info:
self.annotate({'job_info': job_info})
# copies the created files to the client
get_result = TransferFiles("local", params['input_directory'],
params['working_directory'],
dependencies = [self.cdir])
get_result.run()
end_machine()
params['stdout'] = self.job.standard_output()
params['stderr'] = self.job.standard_error()
return params
def job_finish(self, params):
job_info = self.job.get_job_info()
if job_info:
self.annotate({'job_info': job_info})
# copies the created files to the client
get_result = TransferFiles("local",
params['input_directory'],
params['working_directory'],
dependencies=[self.cdir])
get_result.run()
end_machine()
params['stdout'] = self.job.standard_output()
params['stderr'] = self.job.standard_error()
return params
def compute(self):
machine = self.get_machine()
if not self.has_input('command'):
raise ModuleError(self, "No command specified")
command = self.get_input('command').strip()
working_directory = self.get_input('working_directory') \
if self.has_input('working_directory') else '.'
if not self.has_input('input_directory'):
raise ModuleError(self, "No input directory specified")
input_directory = self.get_input('input_directory').strip()
additional_arguments = {
'processes': 1,
'time': -1,
'mpi': False,
'threads': 1,
'memory': -1,
'diskspace': -1
}
for k in additional_arguments:
if self.has_input(k):
additional_arguments[k] = self.get_input(k)
## This indicates that the coming commands submitted on the machine
# trick to select machine without initializing every time
use_machine(machine)
cdir = CreateDirectory("remote", working_directory)
trans = TransferFiles("remote",
input_directory,
working_directory,
dependencies=[cdir])
job = PBS("remote",
command,
working_directory,
dependencies=[trans],
**additional_arguments)
job.run()
ret = job._ret
if ret:
try:
job_id = int(ret)
except ValueError:
end_machine()
raise ModuleError(self, "Error submitting job: %s" % ret)
finished = job.finished()
job_info = job.get_job_info()
if job_info:
self.annotate({'job_info': job.get_job_info()})
if not finished:
status = job.status()
# try to get more detailed information about the job
# this only seems to work on some versions of torque
if job_info:
comment = [
line for line in job_info.split('\n')
if line.startswith('comment =')
]
if comment:
status += ': ' + comment[10:]
end_machine()
# The PBS class provides the JobHandle interface, i.e. finished()
raise ModuleSuspended(self, '%s' % status, handle=job)
# copies the created files to the client
get_result = TransferFiles("local",
input_directory,
working_directory,
dependencies=[cdir])
get_result.run()
## Popping from the machine stack
end_machine()
self.set_output("stdout", job.standard_output())
self.set_output("stderr", job.standard_error())
files = machine.local.send_command("ls -l %s" % input_directory)
self.set_output("file_list",
[f.split(' ')[-1] for f in files.split('\n')[1:]])