예제 #1
0
class RunPBSScript(JobMixin,Module):
    _input_ports = [('machine', Machine),
                    ('command', '(edu.utah.sci.vistrails.basic:String)', True),
                    ('working_directory', '(edu.utah.sci.vistrails.basic:String)'),
                    ('input_directory', '(edu.utah.sci.vistrails.basic:String)'),
                    ('processes', '(edu.utah.sci.vistrails.basic:Integer)', True),
                    ('time', '(edu.utah.sci.vistrails.basic:String)', True),
                    ('mpi', '(edu.utah.sci.vistrails.basic:Boolean)', True),
                    ('threads', '(edu.utah.sci.vistrails.basic:Integer)', True),
                    ('memory', '(edu.utah.sci.vistrails.basic:String)', True),
                    ('diskspace', '(edu.utah.sci.vistrails.basic:String)', True),
                   ]
    
    _output_ports = [('stdout', '(edu.utah.sci.vistrails.basic:String)'),
                     ('stderr', '(edu.utah.sci.vistrails.basic:String)'),
                    ]
    
    def getId(self, params):
        return hashlib.md5(params['input_directory'] +
                           params['command'] +
                           params['working_directory']).hexdigest()
        
    def readInputs(self):
        self.job = None
        d = {}
        if not self.hasInputFromPort('machine'):
            raise ModuleError(self, "No machine specified")
        self.machine = self.getInputFromPort('machine').machine
        if not self.hasInputFromPort('command'):
            raise ModuleError(self, "No command specified")
        d['command'] = self.getInputFromPort('command').strip()
        d['working_directory'] = self.getInputFromPort('working_directory') \
              if self.hasInputFromPort('working_directory') else '.'
        if not self.hasInputFromPort('input_directory'):
            raise ModuleError(self, "No input directory specified")
        d['input_directory'] = self.getInputFromPort('input_directory').strip()
        d['additional_arguments'] = {'processes': 1, 'time': -1, 'mpi': False,
                                'threads': 1, 'memory':-1, 'diskspace': -1}
        for k in d['additional_arguments']:
            if self.hasInputFromPort(k):
                d['additional_arguments'][k] = self.getInputFromPort(k)
        return d

    def startJob(self, params):
        work_dir = params['working_directory']
        use_machine(self.machine)
        self.cdir = CreateDirectory("remote", work_dir)
        trans = TransferFiles("remote", params['input_directory'], work_dir,
                              dependencies = [self.cdir])
        self.job = PBSScript("remote", params['command'], work_dir,
                      dependencies = [trans], **params['additional_arguments'])
        self.job.run()
        try:
            ret = self.job._ret
            if ret:
                job_id = int(ret)
        except ValueError:
            end_machine()
            raise ModuleError(self, "Error submitting job: %s" % ret)
        return params
        
    def getMonitor(self, params):
        if not self.job:
            self.startJob(params)
        return self.job

    def finishJob(self, params):
        job_info = self.job.get_job_info()
        if job_info:
            self.annotate({'job_info': job_info})
        # copies the created files to the client
        get_result = TransferFiles("local", params['input_directory'],
                                   params['working_directory'],
                                   dependencies = [self.cdir])
        get_result.run()
        end_machine()
        stdout = self.job.standard_output()
        stderr = self.job.standard_error()
        return {'stdout':stdout, 'stderr':stderr}

    def setResults(self, params):
        self.setResult('stdout', params['stdout'])
        self.setResult('stderr', params['stderr'])
예제 #2
0
파일: init.py 프로젝트: afcarl/BatchQ-PBS
class RunPBSScript(JobMixin, Module):
    _input_ports = [
        ('machine', Machine),
        ('command', '(edu.utah.sci.vistrails.basic:String)', True),
        ('working_directory', '(edu.utah.sci.vistrails.basic:String)'),
        ('input_directory', '(edu.utah.sci.vistrails.basic:String)'),
        ('processes', '(edu.utah.sci.vistrails.basic:Integer)', True),
        ('time', '(edu.utah.sci.vistrails.basic:String)', True),
        ('mpi', '(edu.utah.sci.vistrails.basic:Boolean)', True),
        ('threads', '(edu.utah.sci.vistrails.basic:Integer)', True),
        ('memory', '(edu.utah.sci.vistrails.basic:String)', True),
        ('diskspace', '(edu.utah.sci.vistrails.basic:String)', True),
    ]

    _output_ports = [
        ('stdout', '(edu.utah.sci.vistrails.basic:String)'),
        ('stderr', '(edu.utah.sci.vistrails.basic:String)'),
    ]

    def getId(self, params):
        return hashlib.md5(params['input_directory'] + params['command'] +
                           params['working_directory']).hexdigest()

    def readInputs(self):
        self.job = None
        d = {}
        if not self.hasInputFromPort('machine'):
            raise ModuleError(self, "No machine specified")
        self.machine = self.getInputFromPort('machine').machine
        if not self.hasInputFromPort('command'):
            raise ModuleError(self, "No command specified")
        d['command'] = self.getInputFromPort('command').strip()
        d['working_directory'] = self.getInputFromPort('working_directory') \
              if self.hasInputFromPort('working_directory') else '.'
        if not self.hasInputFromPort('input_directory'):
            raise ModuleError(self, "No input directory specified")
        d['input_directory'] = self.getInputFromPort('input_directory').strip()
        d['additional_arguments'] = {
            'processes': 1,
            'time': -1,
            'mpi': False,
            'threads': 1,
            'memory': -1,
            'diskspace': -1
        }
        for k in d['additional_arguments']:
            if self.hasInputFromPort(k):
                d['additional_arguments'][k] = self.getInputFromPort(k)
        return d

    def startJob(self, params):
        work_dir = params['working_directory']
        use_machine(self.machine)
        self.cdir = CreateDirectory("remote", work_dir)
        trans = TransferFiles("remote",
                              params['input_directory'],
                              work_dir,
                              dependencies=[self.cdir])
        self.job = PBSScript("remote",
                             params['command'],
                             work_dir,
                             dependencies=[trans],
                             **params['additional_arguments'])
        self.job.run()
        try:
            ret = self.job._ret
            if ret:
                job_id = int(ret)
        except ValueError:
            end_machine()
            raise ModuleError(self, "Error submitting job: %s" % ret)
        return params

    def getMonitor(self, params):
        if not self.job:
            self.startJob(params)
        return self.job

    def finishJob(self, params):
        job_info = self.job.get_job_info()
        if job_info:
            self.annotate({'job_info': job_info})
        # copies the created files to the client
        get_result = TransferFiles("local",
                                   params['input_directory'],
                                   params['working_directory'],
                                   dependencies=[self.cdir])
        get_result.run()
        end_machine()
        stdout = self.job.standard_output()
        stderr = self.job.standard_error()
        return {'stdout': stdout, 'stderr': stderr}

    def setResults(self, params):
        self.setResult('stdout', params['stdout'])
        self.setResult('stderr', params['stderr'])