def compute(self):
machine = self.get_machine()
jm = self.job_monitor()
cache = jm.getCache(self.signature)
if not cache:
if not self.has_input('local_directory'):
raise ModuleError(self, "No local directory specified")
local_directory = self.get_input('local_directory').strip()
if not self.has_input('remote_directory'):
raise ModuleError(self, "No remote directory specified")
remote_directory = self.get_input('remote_directory').strip()
whereto = 'remote'
if self.has_input('to_local') and self.get_input('to_local'):
whereto = 'local'
use_machine(machine)
to_dir = local_directory if whereto=='local' else remote_directory
cdir = CreateDirectory(whereto, to_dir)
job = TransferFiles(whereto, local_directory, remote_directory,
dependencies = [cdir])
job.run()
end_machine()
d = {}
self.set_job_machine(d, machine)
cache = jm.setCache(self.signature, d, self.job_name())
self.set_output("machine", machine)
def compute(self):
machine = self.get_machine()
jm = self.job_monitor()
cache = jm.getCache(self.signature)
if cache:
result = cache.parameters['result']
else:
if not self.has_input('command'):
raise ModuleError(self, "No command specified")
command = self.get_input('command').strip()
## This indicates that the coming commands submitted on the machine
# trick to select machine without initializing every time
use_machine(machine)
m = current_machine()
result = m.remote.send_command(command)
exitcode = m.remote.last_exitcode()
end_machine()
if exitcode != 0:
raise ModuleError(self,
"Command failed with exit code %s: %s" %
(exitcode, result))
d = {'result':result}
self.set_job_machine(d, machine)
jm.setCache(self.signature, d, self.job_name())
self.set_output("output", result)
self.set_output("machine", machine)
def compute(self):
machine = self.get_machine()
jm = self.job_monitor()
cache = jm.getCache(self.signature)
if cache:
result = cache.parameters['result']
else:
if not self.has_input('command'):
raise ModuleError(self, "No command specified")
command = self.get_input('command').strip()
## This indicates that the coming commands submitted on the machine
# trick to select machine without initializing every time
use_machine(machine)
m = current_machine()
result = m.remote.send_command(command)
exitcode = m.remote.last_exitcode()
end_machine()
if exitcode != 0:
raise ModuleError(
self, "Command failed with exit code %s: %s" %
(exitcode, result))
d = {'result': result}
self.set_job_machine(d, machine)
jm.setCache(self.signature, d, self.job_name())
self.set_output("output", result)
self.set_output("machine", machine)
def compute(self):
machine = self.get_machine()
jm = self.job_monitor()
cache = jm.getCache(self.signature)
if not cache:
if not self.has_input('local_directory'):
raise ModuleError(self, "No local directory specified")
local_directory = self.get_input('local_directory').strip()
if not self.has_input('remote_directory'):
raise ModuleError(self, "No remote directory specified")
remote_directory = self.get_input('remote_directory').strip()
whereto = 'remote'
if self.has_input('to_local') and self.get_input('to_local'):
whereto = 'local'
use_machine(machine)
to_dir = local_directory if whereto == 'local' else remote_directory
cdir = CreateDirectory(whereto, to_dir)
job = TransferFiles(whereto,
local_directory,
remote_directory,
dependencies=[cdir])
job.run()
end_machine()
d = {}
self.set_job_machine(d, machine)
cache = jm.setCache(self.signature, d, self.job_name())
self.set_output("machine", machine)
def createJob(self, p):
self.job_machine = self.get_machine()
use_machine(self.job_machine)
self.job = Subshell("remote",
command="%s",
working_directory=p['workdir'],
identifier=p['job_identifier'])
def compute(self):
machine = self.get_machine()
if not self.has_input('command'):
raise ModuleError(self, "No command specified")
command = self.get_input('command').strip()
working_directory = self.get_input('working_directory') \
if self.has_input('working_directory') else '.'
if not self.has_input('input_directory'):
raise ModuleError(self, "No input directory specified")
input_directory = self.get_input('input_directory').strip()
additional_arguments = {'processes': 1, 'time': -1, 'mpi': False,
'threads': 1, 'memory':-1, 'diskspace': -1}
for k in additional_arguments:
if self.has_input(k):
additional_arguments[k] = self.get_input(k)
## This indicates that the coming commands submitted on the machine
# trick to select machine without initializing every time
use_machine(machine)
cdir = CreateDirectory("remote", working_directory)
trans = TransferFiles("remote", input_directory, working_directory,
dependencies = [cdir])
job = PBS("remote", command, working_directory, dependencies = [trans],
**additional_arguments)
job.run()
ret = job._ret
if ret:
try:
job_id = int(ret)
except ValueError:
end_machine()
raise ModuleError(self, "Error submitting job: %s" % ret)
finished = job.finished()
job_info = job.get_job_info()
if job_info:
self.annotate({'job_info': job.get_job_info()})
if not finished:
status = job.status()
# try to get more detailed information about the job
# this only seems to work on some versions of torque
if job_info:
comment = [line for line in job_info.split('\n') if line.startswith('comment =')]
if comment:
status += ': ' + comment[10:]
end_machine()
# The PBS class provides the JobHandle interface, i.e. finished()
raise ModuleSuspended(self, '%s' % status, handle=job)
# copies the created files to the client
get_result = TransferFiles("local", input_directory, working_directory,
dependencies = [cdir])
get_result.run()
## Popping from the machine stack
end_machine()
self.set_output("stdout", job.standard_output())
self.set_output("stderr", job.standard_error())
files = machine.local.send_command("ls -l %s" % input_directory)
self.set_output("file_list",
[f.split(' ')[-1] for f in files.split('\n')[1:]])
def job_start(self, params):
work_dir = params['working_directory']
self.machine = self.get_machine()
use_machine(self.machine)
self.job = Subshell("remote", params['command'], work_dir)
self.job.run()
ret = self.job._ret
if ret:
try:
job_id = int(ret.split('\n')[0])
except ValueError:
end_machine()
raise ModuleError(self, "Error submitting job: %s" % ret)
self.set_job_machine(params, self.machine)
return params
def job_start(self, params):
work_dir = params['working_directory']
self.machine = self.get_machine()
use_machine(self.machine)
self.job = Subshell("remote", params['command'], work_dir)
self.job.run()
ret = self.job._ret
if ret:
try:
job_id = int(ret.split('\n')[0])
except ValueError:
end_machine()
raise ModuleError(self, "Error submitting job: %s" % ret)
self.set_job_machine(params, self.machine)
return params
def job_start(self, params):
work_dir = params['working_directory']
self.machine = self.get_machine()
use_machine(self.machine)
self.cdir = CreateDirectory("remote", work_dir)
trans = TransferFiles("remote", params['input_directory'], work_dir,
dependencies = [self.cdir])
self.job = PBSScript("remote", params['command'], work_dir,
dependencies = [trans], **params['additional_arguments'])
self.job.run()
ret = self.job._ret
if ret:
try:
job_id = int(ret.split('\n')[0])
except ValueError:
end_machine()
raise ModuleError(self, "Error submitting job: %s" % ret)
self.set_job_machine(params, self.machine)
return params
def call_hadoop(self, arguments, workdir, identifier, machine):
self.is_cacheable = lambda *args, **kwargs: False
config = self.get_hadoop_config(machine)
argList = [config['hadoop']]
if type(arguments) in [str, unicode]:
argList += arguments.split(' ')
elif type(arguments) == list:
argList += arguments
else:
raise ModuleError(self, 'Invalid argument types to hadoop')
# 1. this version returns when finished
#return subprocess.call(argList)
# 2. this version reads the results incrementally
# expect = machine.remote._expect_token
# machine.remote.push_expect(None) # Do not wait for call to finish
# result = machine.remote.send_command(" ".join(argList)).strip()
# machine.remote.pop_expect() # restore expect
# # We could show the output in a gui
# print "**** hadoop streaming running ****"
# print result,
# while not expect in result:
# output = machine.remote.consume_output()
# if output:
# print output,
# result += output
# 3. The final version should detach the process on the server
use_machine(machine)
cdir = CreateDirectory("remote", workdir)
job = Subshell("remote",
command=" ".join(argList),
working_directory=workdir,
identifier=identifier,
dependencies=[cdir])
job.run()
finished = job.finished()
if not finished:
status = job.status()
# The Subshell class provides the JobHandle interface, i.e.
# finished()
raise ModuleSuspended(self, '%s' % status, handle=job)
self.is_cacheable = lambda *args, **kwargs: True
return job.standard_error()
def compute(self):
machine = self.get_machine()
if self.cache:
result = self.cache.parameters['result']
else:
if not self.has_input('command'):
raise ModuleError(self, "No command specified")
command = self.get_input('command').strip()
## This indicates that the coming commands submitted on the machine
# trick to select machine without initializing every time
use_machine(machine)
m = current_machine()
result = m.remote.send_command(command)
end_machine()
jm = JobMonitor.getInstance()
d = {'result':result}
self.set_job_machine(d, machine)
jm.setCache(self.signature, d, self.getName())
self.set_output("output", result)
self.set_output("machine", machine)
def call_hadoop(self, arguments, workdir, identifier, machine):
self.is_cacheable = lambda *args, **kwargs: False
config = self.get_hadoop_config(machine)
argList = [config['hadoop']]
if type(arguments) in [str, unicode]:
argList += arguments.split(' ')
elif type(arguments)==list:
argList += arguments
else:
raise ModuleError(self, 'Invalid argument types to hadoop')
# 1. this version returns when finished
#return subprocess.call(argList)
# 2. this version reads the results incrementally
# expect = machine.remote._expect_token
# machine.remote.push_expect(None) # Do not wait for call to finish
# result = machine.remote.send_command(" ".join(argList)).strip()
# machine.remote.pop_expect() # restore expect
# # We could show the output in a gui
# print "**** hadoop streaming running ****"
# print result,
# while not expect in result:
# output = machine.remote.consume_output()
# if output:
# print output,
# result += output
# 3. The final version should detach the process on the server
use_machine(machine)
cdir = CreateDirectory("remote", workdir)
job = Subshell("remote", command=" ".join(argList),
working_directory=workdir, identifier=identifier,
dependencies=[cdir])
job.run()
finished = job.finished()
if not finished:
status = job.status()
# The Subshell class provides the BaseMonitor interface, i.e.
# finished()
raise ModuleSuspended(self, '%s' % status, monitor=job)
self.is_cacheable = lambda *args, **kwargs: True
return job.standard_error()
def job_start(self, params):
work_dir = params['working_directory']
self.machine = self.get_machine()
use_machine(self.machine)
self.cdir = CreateDirectory("remote", work_dir)
trans = TransferFiles("remote",
params['input_directory'],
work_dir,
dependencies=[self.cdir])
self.job = PBSScript("remote",
params['command'],
work_dir,
dependencies=[trans],
**params['additional_arguments'])
self.job.run()
ret = self.job._ret
if ret:
try:
job_id = int(ret.split('\n')[0])
except ValueError:
end_machine()
raise ModuleError(self, "Error submitting job: %s" % ret)
self.set_job_machine(params, self.machine)
return params
def compute(self):
machine = self.get_machine()
if not self.has_input('command'):
raise ModuleError(self, "No command specified")
command = self.get_input('command').strip()
working_directory = self.get_input('working_directory') \
if self.has_input('working_directory') else '.'
if not self.has_input('input_directory'):
raise ModuleError(self, "No input directory specified")
input_directory = self.get_input('input_directory').strip()
additional_arguments = {
'processes': 1,
'time': -1,
'mpi': False,
'threads': 1,
'memory': -1,
'diskspace': -1
}
for k in additional_arguments:
if self.has_input(k):
additional_arguments[k] = self.get_input(k)
## This indicates that the coming commands submitted on the machine
# trick to select machine without initializing every time
use_machine(machine)
cdir = CreateDirectory("remote", working_directory)
trans = TransferFiles("remote",
input_directory,
working_directory,
dependencies=[cdir])
job = PBS("remote",
command,
working_directory,
dependencies=[trans],
**additional_arguments)
job.run()
ret = job._ret
if ret:
try:
job_id = int(ret)
except ValueError:
end_machine()
raise ModuleError(self, "Error submitting job: %s" % ret)
finished = job.finished()
job_info = job.get_job_info()
if job_info:
self.annotate({'job_info': job.get_job_info()})
if not finished:
status = job.status()
# try to get more detailed information about the job
# this only seems to work on some versions of torque
if job_info:
comment = [
line for line in job_info.split('\n')
if line.startswith('comment =')
]
if comment:
status += ': ' + comment[10:]
end_machine()
# The PBS class provides the JobHandle interface, i.e. finished()
raise ModuleSuspended(self, '%s' % status, handle=job)
# copies the created files to the client
get_result = TransferFiles("local",
input_directory,
working_directory,
dependencies=[cdir])
get_result.run()
## Popping from the machine stack
end_machine()
self.set_output("stdout", job.standard_output())
self.set_output("stderr", job.standard_error())
files = machine.local.send_command("ls -l %s" % input_directory)
self.set_output("file_list",
[f.split(' ')[-1] for f in files.split('\n')[1:]])
def createJob(self, p):
self.job_machine = self.get_machine()
use_machine(self.job_machine)
self.job = Subshell("remote", command="%s",
working_directory=p['workdir'],
identifier=p['job_identifier'])
