Exemple #1
0
    def compute(self):
        machine = self.get_machine()
        jm = self.job_monitor()
        cache = jm.getCache(self.signature)
        if not cache:
            if not self.has_input('local_directory'):
                raise ModuleError(self, "No local directory specified")
            local_directory = self.get_input('local_directory').strip()
            if not self.has_input('remote_directory'):
                raise ModuleError(self, "No remote directory specified")
            remote_directory = self.get_input('remote_directory').strip()
            whereto = 'remote'
            if self.has_input('to_local') and self.get_input('to_local'):
                whereto = 'local'
            use_machine(machine)
            to_dir = local_directory if whereto=='local' else remote_directory
            cdir = CreateDirectory(whereto, to_dir)
            job = TransferFiles(whereto, local_directory, remote_directory,
                              dependencies = [cdir])
            job.run()
            end_machine()
            d = {}
            self.set_job_machine(d, machine)
            cache = jm.setCache(self.signature, d, self.job_name())

        self.set_output("machine", machine)
Exemple #2
0
    def compute(self):
        machine = self.get_machine()
        jm = self.job_monitor()
        cache = jm.getCache(self.signature)
        if not cache:
            if not self.has_input('local_directory'):
                raise ModuleError(self, "No local directory specified")
            local_directory = self.get_input('local_directory').strip()
            if not self.has_input('remote_directory'):
                raise ModuleError(self, "No remote directory specified")
            remote_directory = self.get_input('remote_directory').strip()
            whereto = 'remote'
            if self.has_input('to_local') and self.get_input('to_local'):
                whereto = 'local'
            use_machine(machine)
            to_dir = local_directory if whereto == 'local' else remote_directory
            cdir = CreateDirectory(whereto, to_dir)
            job = TransferFiles(whereto,
                                local_directory,
                                remote_directory,
                                dependencies=[cdir])
            job.run()
            end_machine()
            d = {}
            self.set_job_machine(d, machine)
            cache = jm.setCache(self.signature, d, self.job_name())

        self.set_output("machine", machine)
Exemple #3
0
    def compute(self):
        machine = self.get_machine()
        if not self.has_input('command'):
            raise ModuleError(self, "No command specified")
        command = self.get_input('command').strip()
        working_directory = self.get_input('working_directory') \
              if self.has_input('working_directory') else '.'
        if not self.has_input('input_directory'):
            raise ModuleError(self, "No input directory specified")
        input_directory = self.get_input('input_directory').strip()
        additional_arguments = {'processes': 1, 'time': -1, 'mpi': False,
                                'threads': 1, 'memory':-1, 'diskspace': -1}
        for k in additional_arguments:
            if self.has_input(k):
                additional_arguments[k] = self.get_input(k)
        ## This indicates that the coming commands submitted on the machine
        # trick to select machine without initializing every time

        use_machine(machine)
        cdir = CreateDirectory("remote", working_directory)
        trans = TransferFiles("remote", input_directory, working_directory,
                              dependencies = [cdir])
        job = PBS("remote", command, working_directory, dependencies = [trans],
                  **additional_arguments)
        job.run()
        ret = job._ret
        if ret:
            try:
                job_id = int(ret)
            except ValueError:
                end_machine()
                raise ModuleError(self, "Error submitting job: %s" % ret)
        finished = job.finished()
        job_info = job.get_job_info()
        if job_info:
            self.annotate({'job_info': job.get_job_info()})
        if not finished:
            status = job.status()
            # try to get more detailed information about the job
            # this only seems to work on some versions of torque
            if job_info:
                comment = [line for line in job_info.split('\n') if line.startswith('comment =')]
                if comment:
                    status += ': ' + comment[10:]
            end_machine()
            # The PBS class provides the JobHandle interface, i.e. finished()
            raise ModuleSuspended(self, '%s' % status, handle=job)
        # copies the created files to the client
        get_result = TransferFiles("local", input_directory, working_directory,
                              dependencies = [cdir])
        get_result.run()
        ## Popping from the machine stack                                                                                                                                     
        end_machine()
        self.set_output("stdout", job.standard_output())
        self.set_output("stderr", job.standard_error())
        files = machine.local.send_command("ls -l %s" % input_directory)
        self.set_output("file_list",
                       [f.split(' ')[-1] for f in files.split('\n')[1:]])
Exemple #4
0
 def job_finish(self, params):
     job_info = self.job.get_job_info()
     if job_info:
         self.annotate({'job_info': job_info})
     # copies the created files to the client
     get_result = TransferFiles("local", params['input_directory'],
                                params['working_directory'],
                                dependencies = [self.cdir])
     get_result.run()
     end_machine()
     params['stdout'] = self.job.standard_output()
     params['stderr'] = self.job.standard_error()
     return params
Exemple #5
0
 def job_finish(self, params):
     job_info = self.job.get_job_info()
     if job_info:
         self.annotate({'job_info': job_info})
     # copies the created files to the client
     get_result = TransferFiles("local",
                                params['input_directory'],
                                params['working_directory'],
                                dependencies=[self.cdir])
     get_result.run()
     end_machine()
     params['stdout'] = self.job.standard_output()
     params['stderr'] = self.job.standard_error()
     return params
Exemple #6
0
    def compute(self):
        machine = self.get_machine()
        if not self.has_input('command'):
            raise ModuleError(self, "No command specified")
        command = self.get_input('command').strip()
        working_directory = self.get_input('working_directory') \
              if self.has_input('working_directory') else '.'
        if not self.has_input('input_directory'):
            raise ModuleError(self, "No input directory specified")
        input_directory = self.get_input('input_directory').strip()
        additional_arguments = {
            'processes': 1,
            'time': -1,
            'mpi': False,
            'threads': 1,
            'memory': -1,
            'diskspace': -1
        }
        for k in additional_arguments:
            if self.has_input(k):
                additional_arguments[k] = self.get_input(k)
        ## This indicates that the coming commands submitted on the machine
        # trick to select machine without initializing every time

        use_machine(machine)
        cdir = CreateDirectory("remote", working_directory)
        trans = TransferFiles("remote",
                              input_directory,
                              working_directory,
                              dependencies=[cdir])
        job = PBS("remote",
                  command,
                  working_directory,
                  dependencies=[trans],
                  **additional_arguments)
        job.run()
        ret = job._ret
        if ret:
            try:
                job_id = int(ret)
            except ValueError:
                end_machine()
                raise ModuleError(self, "Error submitting job: %s" % ret)
        finished = job.finished()
        job_info = job.get_job_info()
        if job_info:
            self.annotate({'job_info': job.get_job_info()})
        if not finished:
            status = job.status()
            # try to get more detailed information about the job
            # this only seems to work on some versions of torque
            if job_info:
                comment = [
                    line for line in job_info.split('\n')
                    if line.startswith('comment =')
                ]
                if comment:
                    status += ': ' + comment[10:]
            end_machine()
            # The PBS class provides the JobHandle interface, i.e. finished()
            raise ModuleSuspended(self, '%s' % status, handle=job)
        # copies the created files to the client
        get_result = TransferFiles("local",
                                   input_directory,
                                   working_directory,
                                   dependencies=[cdir])
        get_result.run()
        ## Popping from the machine stack
        end_machine()
        self.set_output("stdout", job.standard_output())
        self.set_output("stderr", job.standard_error())
        files = machine.local.send_command("ls -l %s" % input_directory)
        self.set_output("file_list",
                        [f.split(' ')[-1] for f in files.split('\n')[1:]])