def compute(self): machine = self.get_machine() jm = self.job_monitor() cache = jm.getCache(self.signature) if not cache: if not self.has_input('local_directory'): raise ModuleError(self, "No local directory specified") local_directory = self.get_input('local_directory').strip() if not self.has_input('remote_directory'): raise ModuleError(self, "No remote directory specified") remote_directory = self.get_input('remote_directory').strip() whereto = 'remote' if self.has_input('to_local') and self.get_input('to_local'): whereto = 'local' use_machine(machine) to_dir = local_directory if whereto == 'local' else remote_directory cdir = CreateDirectory(whereto, to_dir) job = TransferFiles(whereto, local_directory, remote_directory, dependencies=[cdir]) job.run() end_machine() d = {} self.set_job_machine(d, machine) cache = jm.setCache(self.signature, d, self.job_name()) self.set_output("machine", machine)
def call_hadoop(self, arguments, workdir, identifier, machine): self.is_cacheable = lambda *args, **kwargs: False config = self.get_hadoop_config(machine) argList = [config['hadoop']] if type(arguments) in [str, unicode]: argList += arguments.split(' ') elif type(arguments) == list: argList += arguments else: raise ModuleError(self, 'Invalid argument types to hadoop') # 1. this version returns when finished #return subprocess.call(argList) # 2. this version reads the results incrementally # expect = machine.remote._expect_token # machine.remote.push_expect(None) # Do not wait for call to finish # result = machine.remote.send_command(" ".join(argList)).strip() # machine.remote.pop_expect() # restore expect # # We could show the output in a gui # print "**** hadoop streaming running ****" # print result, # while not expect in result: # output = machine.remote.consume_output() # if output: # print output, # result += output # 3. The final version should detach the process on the server use_machine(machine) cdir = CreateDirectory("remote", workdir) job = Subshell("remote", command=" ".join(argList), working_directory=workdir, identifier=identifier, dependencies=[cdir]) job.run() finished = job.finished() if not finished: status = job.status() # The Subshell class provides the JobHandle interface, i.e. # finished() raise ModuleSuspended(self, '%s' % status, handle=job) self.is_cacheable = lambda *args, **kwargs: True return job.standard_error()
def job_start(self, params): work_dir = params['working_directory'] self.machine = self.get_machine() use_machine(self.machine) self.cdir = CreateDirectory("remote", work_dir) trans = TransferFiles("remote", params['input_directory'], work_dir, dependencies=[self.cdir]) self.job = PBSScript("remote", params['command'], work_dir, dependencies=[trans], **params['additional_arguments']) self.job.run() ret = self.job._ret if ret: try: job_id = int(ret.split('\n')[0]) except ValueError: end_machine() raise ModuleError(self, "Error submitting job: %s" % ret) self.set_job_machine(params, self.machine) return params
def compute(self): machine = self.get_machine() if not self.has_input('command'): raise ModuleError(self, "No command specified") command = self.get_input('command').strip() working_directory = self.get_input('working_directory') \ if self.has_input('working_directory') else '.' if not self.has_input('input_directory'): raise ModuleError(self, "No input directory specified") input_directory = self.get_input('input_directory').strip() additional_arguments = { 'processes': 1, 'time': -1, 'mpi': False, 'threads': 1, 'memory': -1, 'diskspace': -1 } for k in additional_arguments: if self.has_input(k): additional_arguments[k] = self.get_input(k) ## This indicates that the coming commands submitted on the machine # trick to select machine without initializing every time use_machine(machine) cdir = CreateDirectory("remote", working_directory) trans = TransferFiles("remote", input_directory, working_directory, dependencies=[cdir]) job = PBS("remote", command, working_directory, dependencies=[trans], **additional_arguments) job.run() ret = job._ret if ret: try: job_id = int(ret) except ValueError: end_machine() raise ModuleError(self, "Error submitting job: %s" % ret) finished = job.finished() job_info = job.get_job_info() if job_info: self.annotate({'job_info': job.get_job_info()}) if not finished: status = job.status() # try to get more detailed information about the job # this only seems to work on some versions of torque if job_info: comment = [ line for line in job_info.split('\n') if line.startswith('comment =') ] if comment: status += ': ' + comment[10:] end_machine() # The PBS class provides the JobHandle interface, i.e. finished() raise ModuleSuspended(self, '%s' % status, handle=job) # copies the created files to the client get_result = TransferFiles("local", input_directory, working_directory, dependencies=[cdir]) get_result.run() ## Popping from the machine stack end_machine() self.set_output("stdout", job.standard_output()) self.set_output("stderr", job.standard_error()) files = machine.local.send_command("ls -l %s" % input_directory) self.set_output("file_list", [f.split(' ')[-1] for f in files.split('\n')[1:]])