def _getJDLData(self, task, jobNumList, queryArguments): jdlData = self._getBaseJDLData(task, queryArguments) jdlData.extend([ 'Executable = %s' % self.parentPool._getSandboxFilesIn(task)[0][1], ]) try: for authFile in parentPool.proxy.getAuthFiles(): jdlData.extend([ 'x509userproxy = %s' % authFile, 'use_x509userproxy = True', ]) except Exception: clear_current_exception() for jobNum in jobNumList: jdlData.extend(self._getRequirementJdlData(task, jobNum)) jobStageDir = self.getStagingDir( htcID=HTCJobID(gcJobNum=jobNum, gcTaskID=task.taskID)) jdlData.extend([ '+GcJobNum = "%s"' % jobNum, 'arguments = %s' % jobNum, 'initialdir = %s' % jobStageDir, 'Output = %s' % os.path.join(jobStageDir, 'gc.stdout'), 'Error = %s' % os.path.join(jobStageDir, 'gc.stderr'), # HACK: ignore executable (In[0]), stdout (Out[0]) and stderr (Out[1]) 'transfer_input_files = %s' % ','.join([ src for descr, src, trg in self.parentPool._getSandboxFilesIn(task)[1:] ] + [self.parentPool.getJobCfgPath(jobNum)[0]]), 'transfer_output_files = %s' % ','.join([ src for descr, src, trg in self.parentPool._getSandboxFilesOut(task)[2:] ]), '+rawID = "%s"' % HTCJobID(gcJobNum=jobNum, gcTaskID=task.taskID, clusterID='$(Cluster)', procID='$(Process)', scheddURI=self.getURI(), typed=False).rawID, 'Queue', ]) return jdlData
def _digestQueryInfoMap(self, queryInfoMaps, queryArguments): """Digest raw queryInfoMaps to maps of HTCjobID : infoMap""" dataMap = {} for infoMap in queryInfoMaps: htcID = HTCJobID(rawID=infoMap['rawID']) dataMap[htcID] = {} for key in infoMap: if key in queryArguments: dataMap[htcID][key] = infoMap[key] return dataMap
def _getJDLData(self, task, jobNumList, queryArguments): taskFiles, proxyFile, jobFileMap = self._getSubmitFileMap( task, jobNumList) jdlData = self._getBaseJDLData(task, queryArguments) jdlData.extend([ 'Executable = %s' % taskFiles[0][2], ]) if proxyFile: jdlData.extend([ 'use_x509userproxy = True', 'x509userproxy = %s' % proxyFile[2], ]) for jobNum in jobNumList: jdlData.extend(self._getRequirementJdlData(task, jobNum)) jobStageDir = self.getStagingDir( htcID=HTCJobID(gcJobNum=jobNum, gcTaskID=task.taskID)) jdlData.extend([ '+GcJobNum = "%s"' % jobNum, 'arguments = %s' % jobNum, 'initialdir = %s' % jobStageDir, 'Output = %s' % os.path.join(jobStageDir, 'gc.stdout'), 'Error = %s' % os.path.join(jobStageDir, 'gc.stderr'), # HACK: ignore executable (In[0]), stdout (Out[0]) and stderr (Out[1]) 'transfer_input_files = %s' % ','.join([ schd for descr, gc, schd in taskFiles[1:] + jobFileMap[jobNum] ]), 'transfer_output_files = %s' % ','.join([ src for descr, src, trg in self.parentPool._getSandboxFilesOut(task)[2:] ]), '+rawID = "%s"' % HTCJobID(gcJobNum=jobNum, gcTaskID=task.taskID, clusterID='$(Cluster)', procID='$(Process)', scheddURI=self.getURI(), typed=False).rawID, ]) return jdlData
def _stageSubmitFiles(self, task, jobNumList): """ Stage submission files at scheduler. """ taskFiles, proxyFile, jobFileMap = self._getSubmitFileMap( task, jobNumList) self._log(logging.DEBUG1, "Staging task files.") stagedJobs = [] if proxyFile: taskFiles.append(proxyFile) for index, fileInfoBlob in enumerate(taskFiles): self._log( logging.DEBUG3, "Staging task files (%d/%d): %s" % (index, len(taskFiles), fileInfoBlob[0])) putProcess = self._adapter.LoggedPut(fileInfoBlob[1], fileInfoBlob[2]) if putProcess.wait(timeout=self._adapterMaxWait): putProcess.logError(self.parentPool.errorLog, brief=True) self._log( logging.INFO1, "Staging failure. Aborting submit." % (fileInfoBlob[0])) return stagedJobs for jobNum, jobFiles in jobFileMap: try: for fileInfoBlob in jobFiles: self._log(logging.DEBUG3, "Staging job files: %s" % (fileInfoBlob[0])) putProcess = self._adapter.LoggedPut( fileInfoBlob[1], fileInfoBlob[2]) if putProcess.wait(timeout=self._adapterMaxWait): putProcess.logError(self.parentPool.errorLog, brief=True) try: self.cleanStagingDir( htcID=HTCJobID(jobNum, task.taskID)) except Exception: self._log(logging.INFO1, 'unable to clean staging dir') raise BackendError except BackendError: continue else: stagedJobs.append(jobNum) return stagedJobs
def _getBaseJDLData(self, task, queryArguments): """Create a sequence of default attributes for a submission JDL""" jdlData = [ '+submitTool = "GridControl (version %s)"' % utils.getVersion(), 'should_transfer_files = YES', 'when_to_transfer_output = ON_EXIT', 'periodic_remove = (JobStatus == 5 && HoldReasonCode != 16)', 'environment = CONDOR_WMS_DASHID=https://%s:/$(Cluster).$(Process)' % self.parentPool.wmsName, 'Universe = %s' % self.parentPool._jobSettings["Universe"], # TODO: Unhack me '+GcID = "%s"' % self.parentPool._createGcId( HTCJobID(gcJobNum='$(GcJobNum)', gcTaskID=task.taskID, clusterID='$(Cluster)', procID='$(Process)', scheddURI=self.getURI(), typed=False)), '+GcJobNumToWmsID = "$(GcJobNum)@$(Cluster).$(Process)"', '+GcJobNumToGcID = "$(GcJobNum)@$(GcID)"', 'Log = %s' % os.path.join(self.getStagingDir(), 'gcJobs.log'), 'job_ad_information_attrs = %s' % ','.join( [arg for arg in queryArguments if arg not in ['JobStatus']]), ] for key in queryArguments: try: # is this a match string? '+JOB_GLIDEIN_Entry_Name = "$$(GLIDEIN_Entry_Name:Unknown)"' -> MATCH_GLIDEIN_Entry_Name = "CMS_T2_DE_RWTH_grid-ce2" && MATCH_EXP_JOB_GLIDEIN_Entry_Name = "CMS_T2_DE_RWTH_grid-ce2" matchKey = re.match("(?:MATCH_EXP_JOB_|MATCH_|JOB_)(.*)", key).group(1) jdlData['Head']['+JOB_%s' % matchKey] = "$$(%s:Unknown)" % matchKey except AttributeError: clear_current_exception() for line in self.parentPool._jobSettings["ClassAd"]: jdlData.append('+' + line) for line in self.parentPool._jobSettings["JDL"]: jdlData.append(line) return jdlData return jdlData
def _splitGcId(self, gcId): """Split a GcId, returning wmsName and htcJobID""" wmsName, rawId = self._splitId(gcId) return (wmsName, HTCJobID(rawID=rawId))