def compute(self): machine = self.get_machine() jm = self.job_monitor() cache = jm.getCache(self.signature) if cache: result = cache.parameters['result'] else: if not self.has_input('command'): raise ModuleError(self, "No command specified") command = self.get_input('command').strip() ## This indicates that the coming commands submitted on the machine # trick to select machine without initializing every time use_machine(machine) m = current_machine() result = m.remote.send_command(command) exitcode = m.remote.last_exitcode() end_machine() if exitcode != 0: raise ModuleError(self, "Command failed with exit code %s: %s" % (exitcode, result)) d = {'result':result} self.set_job_machine(d, machine) jm.setCache(self.signature, d, self.job_name()) self.set_output("output", result) self.set_output("machine", machine)
def compute(self): machine = self.get_machine() jm = self.job_monitor() cache = jm.getCache(self.signature) if not cache: if not self.has_input('local_directory'): raise ModuleError(self, "No local directory specified") local_directory = self.get_input('local_directory').strip() if not self.has_input('remote_directory'): raise ModuleError(self, "No remote directory specified") remote_directory = self.get_input('remote_directory').strip() whereto = 'remote' if self.has_input('to_local') and self.get_input('to_local'): whereto = 'local' use_machine(machine) to_dir = local_directory if whereto == 'local' else remote_directory cdir = CreateDirectory(whereto, to_dir) job = TransferFiles(whereto, local_directory, remote_directory, dependencies=[cdir]) job.run() end_machine() d = {} self.set_job_machine(d, machine) cache = jm.setCache(self.signature, d, self.job_name()) self.set_output("machine", machine)
def compute(self): machine = self.get_machine() jm = self.job_monitor() cache = jm.getCache(self.signature) if cache: result = cache.parameters['result'] else: if not self.has_input('command'): raise ModuleError(self, "No command specified") command = self.get_input('command').strip() ## This indicates that the coming commands submitted on the machine # trick to select machine without initializing every time use_machine(machine) m = current_machine() result = m.remote.send_command(command) exitcode = m.remote.last_exitcode() end_machine() if exitcode != 0: raise ModuleError( self, "Command failed with exit code %s: %s" % (exitcode, result)) d = {'result': result} self.set_job_machine(d, machine) jm.setCache(self.signature, d, self.job_name()) self.set_output("output", result) self.set_output("machine", machine)
def compute(self): machine = self.get_machine() jm = self.job_monitor() cache = jm.getCache(self.signature) if not cache: if not self.has_input('local_directory'): raise ModuleError(self, "No local directory specified") local_directory = self.get_input('local_directory').strip() if not self.has_input('remote_directory'): raise ModuleError(self, "No remote directory specified") remote_directory = self.get_input('remote_directory').strip() whereto = 'remote' if self.has_input('to_local') and self.get_input('to_local'): whereto = 'local' use_machine(machine) to_dir = local_directory if whereto=='local' else remote_directory cdir = CreateDirectory(whereto, to_dir) job = TransferFiles(whereto, local_directory, remote_directory, dependencies = [cdir]) job.run() end_machine() d = {} self.set_job_machine(d, machine) cache = jm.setCache(self.signature, d, self.job_name()) self.set_output("machine", machine)
def create_machine(server, username, password, port): machine = BQMachine(server, username, password, port, accept_fingerprint=True) machine.params = {} machine.params['server'] = server machine.params['port'] = port machine.params['username'] = username machine.params['password'] = bool(password) # force creation of server-side help files select_machine(machine) end_machine() return machine
def job_finish(self, params): job_info = self.job.get_job_info() if job_info: self.annotate({'job_info': job_info}) # copies the created files to the client get_result = TransferFiles("local", params['input_directory'], params['working_directory'], dependencies = [self.cdir]) get_result.run() end_machine() params['stdout'] = self.job.standard_output() params['stderr'] = self.job.standard_error() return params
def job_finish(self, params): job_info = self.job.get_job_info() if job_info: self.annotate({'job_info': job_info}) # copies the created files to the client get_result = TransferFiles("local", params['input_directory'], params['working_directory'], dependencies=[self.cdir]) get_result.run() end_machine() params['stdout'] = self.job.standard_output() params['stderr'] = self.job.standard_error() return params
def job_finish(self, params): params['stdout'] = self.job.standard_output() params['stderr'] = self.job.standard_error() if self.job.failed(): self.job._pushw() code = self.job.terminal.cat("%s.failed" % self.job._identifier_filename) self.job._popw() end_machine() raise ModuleError( self, "Command failed with exit code %s: %s" % (code.strip(), params['stderr'].strip())) end_machine() return params
def job_finish(self, params): params['stdout'] = self.job.standard_output() params['stderr'] = self.job.standard_error() if self.job.failed(): self.job._pushw() code = self.job.terminal.cat("%s.failed" % self.job._identifier_filename) self.job._popw() end_machine() raise ModuleError(self, "Command failed with exit code %s: %s" % (code.strip(), params['stderr'].strip())) end_machine() return params
def job_start(self, params): work_dir = params['working_directory'] self.machine = self.get_machine() use_machine(self.machine) self.job = Subshell("remote", params['command'], work_dir) self.job.run() ret = self.job._ret if ret: try: job_id = int(ret.split('\n')[0]) except ValueError: end_machine() raise ModuleError(self, "Error submitting job: %s" % ret) self.set_job_machine(params, self.machine) return params
def job_start(self, params): work_dir = params['working_directory'] self.machine = self.get_machine() use_machine(self.machine) self.cdir = CreateDirectory("remote", work_dir) trans = TransferFiles("remote", params['input_directory'], work_dir, dependencies = [self.cdir]) self.job = PBSScript("remote", params['command'], work_dir, dependencies = [trans], **params['additional_arguments']) self.job.run() ret = self.job._ret if ret: try: job_id = int(ret.split('\n')[0]) except ValueError: end_machine() raise ModuleError(self, "Error submitting job: %s" % ret) self.set_job_machine(params, self.machine) return params
def compute(self): machine = self.get_machine() if self.cache: result = self.cache.parameters['result'] else: if not self.has_input('command'): raise ModuleError(self, "No command specified") command = self.get_input('command').strip() ## This indicates that the coming commands submitted on the machine # trick to select machine without initializing every time use_machine(machine) m = current_machine() result = m.remote.send_command(command) end_machine() jm = JobMonitor.getInstance() d = {'result':result} self.set_job_machine(d, machine) jm.setCache(self.signature, d, self.getName()) self.set_output("output", result) self.set_output("machine", machine)
def compute(self): machine = self.get_machine() if not self.has_input('command'): raise ModuleError(self, "No command specified") command = self.get_input('command').strip() working_directory = self.get_input('working_directory') \ if self.has_input('working_directory') else '.' if not self.has_input('input_directory'): raise ModuleError(self, "No input directory specified") input_directory = self.get_input('input_directory').strip() additional_arguments = {'processes': 1, 'time': -1, 'mpi': False, 'threads': 1, 'memory':-1, 'diskspace': -1} for k in additional_arguments: if self.has_input(k): additional_arguments[k] = self.get_input(k) ## This indicates that the coming commands submitted on the machine # trick to select machine without initializing every time use_machine(machine) cdir = CreateDirectory("remote", working_directory) trans = TransferFiles("remote", input_directory, working_directory, dependencies = [cdir]) job = PBS("remote", command, working_directory, dependencies = [trans], **additional_arguments) job.run() ret = job._ret if ret: try: job_id = int(ret) except ValueError: end_machine() raise ModuleError(self, "Error submitting job: %s" % ret) finished = job.finished() job_info = job.get_job_info() if job_info: self.annotate({'job_info': job.get_job_info()}) if not finished: status = job.status() # try to get more detailed information about the job # this only seems to work on some versions of torque if job_info: comment = [line for line in job_info.split('\n') if line.startswith('comment =')] if comment: status += ': ' + comment[10:] end_machine() # The PBS class provides the JobHandle interface, i.e. finished() raise ModuleSuspended(self, '%s' % status, handle=job) # copies the created files to the client get_result = TransferFiles("local", input_directory, working_directory, dependencies = [cdir]) get_result.run() ## Popping from the machine stack end_machine() self.set_output("stdout", job.standard_output()) self.set_output("stderr", job.standard_error()) files = machine.local.send_command("ls -l %s" % input_directory) self.set_output("file_list", [f.split(' ')[-1] for f in files.split('\n')[1:]])
def job_start(self, params): work_dir = params['working_directory'] self.machine = self.get_machine() use_machine(self.machine) self.cdir = CreateDirectory("remote", work_dir) trans = TransferFiles("remote", params['input_directory'], work_dir, dependencies=[self.cdir]) self.job = PBSScript("remote", params['command'], work_dir, dependencies=[trans], **params['additional_arguments']) self.job.run() ret = self.job._ret if ret: try: job_id = int(ret.split('\n')[0]) except ValueError: end_machine() raise ModuleError(self, "Error submitting job: %s" % ret) self.set_job_machine(params, self.machine) return params
def compute(self): machine = self.get_machine() if not self.has_input('command'): raise ModuleError(self, "No command specified") command = self.get_input('command').strip() working_directory = self.get_input('working_directory') \ if self.has_input('working_directory') else '.' if not self.has_input('input_directory'): raise ModuleError(self, "No input directory specified") input_directory = self.get_input('input_directory').strip() additional_arguments = { 'processes': 1, 'time': -1, 'mpi': False, 'threads': 1, 'memory': -1, 'diskspace': -1 } for k in additional_arguments: if self.has_input(k): additional_arguments[k] = self.get_input(k) ## This indicates that the coming commands submitted on the machine # trick to select machine without initializing every time use_machine(machine) cdir = CreateDirectory("remote", working_directory) trans = TransferFiles("remote", input_directory, working_directory, dependencies=[cdir]) job = PBS("remote", command, working_directory, dependencies=[trans], **additional_arguments) job.run() ret = job._ret if ret: try: job_id = int(ret) except ValueError: end_machine() raise ModuleError(self, "Error submitting job: %s" % ret) finished = job.finished() job_info = job.get_job_info() if job_info: self.annotate({'job_info': job.get_job_info()}) if not finished: status = job.status() # try to get more detailed information about the job # this only seems to work on some versions of torque if job_info: comment = [ line for line in job_info.split('\n') if line.startswith('comment =') ] if comment: status += ': ' + comment[10:] end_machine() # The PBS class provides the JobHandle interface, i.e. finished() raise ModuleSuspended(self, '%s' % status, handle=job) # copies the created files to the client get_result = TransferFiles("local", input_directory, working_directory, dependencies=[cdir]) get_result.run() ## Popping from the machine stack end_machine() self.set_output("stdout", job.standard_output()) self.set_output("stderr", job.standard_error()) files = machine.local.send_command("ls -l %s" % input_directory) self.set_output("file_list", [f.split(' ')[-1] for f in files.split('\n')[1:]])