def status(self, job): """Get the status of a jobset.""" if not os.path.exists(job.inputdata.ui_working_dir): raise CRABServerError('Workdir "%s" not found.' % job.inputdata.ui_working_dir) cmd = 'crab -status -c %s' % job.inputdata.ui_working_dir self._send_with_retry(cmd, 'status', job.backend.crab_env) return True
def submit(self, job): """Submit an already created jobset.""" if not os.path.exists(job.inputdata.ui_working_dir): raise CRABServerError('Workdir "%s" not found.' % job.inputdata.ui_working_dir) cmd = 'crab -submit -c %s' % job.inputdata.ui_working_dir self._send_with_retry(cmd, 'submit', job.backend.crab_env) return True
def postMortem(self, job): """Retrieves the postmortem information.""" if not os.path.exists(job.inputdata.ui_working_dir): raise CRABServerError('Workdir %s not found.' % job.inputdata.ui_working_dir) cmd = 'crab -postMortem %d -c %s' % (int(job.id) + 1, job.inputdata.ui_working_dir) self._send_with_retry(cmd, 'postMortem', job.backend.crab_env) return True
def resubmit(self, job): """Resubmit an already created job.""" if not os.path.exists(job.inputdata.ui_working_dir): raise CRABServerError('Workdir "%s" not found.' % job.inputdata.ui_working_dir) cmd = 'crab -resubmit %d -c %s' % (int(job.id) + 1, job.inputdata.ui_working_dir) self._send_with_retry(cmd, 'resubmit', job.backend.crab_env) return True
def create(self, job): """Create a new CRAB jobset.""" cfgfile = os.path.join(job.inputdir, 'crab.cfg') if not os.path.isfile(cfgfile): raise CRABServerError('File "%s" not found.' % cfgfile) # Clean up the working dir for the CRAB UI. shutil.rmtree(job.inputdata.ui_working_dir, ignore_errors=True) cmd = 'crab -create -cfg %s' % cfgfile self._send_with_retry(cmd, 'create', job.backend.crab_env) return True
def kill(self, job): """Kill all the jobs on the task.""" if not os.path.exists(job.inputdata.ui_working_dir): raise CRABServerError('Workdir "%s" not found.' % job.inputdata.ui_working_dir) if not job.master: cmd = 'crab -kill all -c %s' % job.inputdata.ui_working_dir else: cmd = 'crab -kill %d -c %s' % (int(job.id) + 1, job.inputdata.ui_working_dir) self._send_with_retry(cmd, 'kill', job.backend.crab_env) return True
def _send_with_retry(self, cmd, operation, env, retries=3, delay=60): """Wrapper to add some retries to the CRAB command launching.""" assert retries > 0 assert delay >= 0 for _ in range(retries): try: self._send(cmd, operation, env) return except CRABServerError: time.sleep(delay) raise CRABServerError('CRAB %s failed %d times' % (operation, retries))
def _send(self, cmd, operation, env): """Launches a command and waits for output.""" try: logger.debug('Launching a CRAB command: %s' % cmd) init = datetime.datetime.now() p = subprocess.Popen(shlex.split(cmd), stdout=subprocess.PIPE, stderr=subprocess.PIPE, bufsize=-1, env=env) stdout, stderr = p.communicate() p.wait() end = datetime.datetime.now() logger.debug('Finished CRAB command: %s' % cmd) logger.info('%s took %d sec.' % (operation, (end - init).seconds)) if p.returncode != 0: raise CRABServerError('CRAB %s returned %s' % (operation, p.returncode)) except OSError as e: logger.error(stdout) logger.error(stderr) raise CRABServerError(e, 'OSError %s crab job(s).' % operation)
def getOutput(self, job): """Retrieve the output of the job.""" if not os.path.exists(job.inputdata.ui_working_dir): raise CRABServerError('Workdir "%s" not found.' % job.inputdata.ui_working_dir) cmd = 'crab -getoutput %d -c %s' % (int(job.id) + 1, job.inputdata.ui_working_dir) self._send_with_retry(cmd, 'getoutput', job.backend.crab_env) # Make output files coming from the WMS readable. for root, _, files in os.walk( os.path.join(job.inputdata.ui_working_dir, 'res')): # Just 'res'. for f in files: os.chmod(os.path.join(root, f), 0o644) return True