def addSubRequest(self,requestDict,type): """ Add a new sub-requests of specified type """ # Initialise the sub-request index = self.initiateSubRequest(type)['Value'] # Stuff the sub-request with the attributes attributeDict = {'Status':'Waiting','SubRequestID':makeGuid(), 'CreationTime': str(datetime.datetime.utcnow()), 'ExecutionOrder':0} for attr,value in requestDict['Attributes'].items(): attributeDict[attr] = value self.setSubRequestAttributes(index,type,attributeDict) if requestDict.has_key('Files'): files = [] for file in requestDict['Files']: fileDict = {'Status':'Waiting','FileID':makeGuid(),'Attempt':1} for attr,value in file.items(): fileDict[attr] = value files.append(fileDict) self.setSubRequestFiles(index,type,files) if requestDict.has_key('Datasets'): datasets = [] for dataset in requestDict['Datasets']: datasetDict = {'Status':'Waiting'} for attr,value in file.items(): fileDict[attr] = value datasets.append(datasetDict) self.setSubRequestDatasets(index,type,datasets) return S_OK(index)
def __writeSub(self, executable, nJobs): """ Create the Sub File for submission """ self.log.debug("Working directory: %s " % self.workingDirectory) # We randomize the location of the pilotoutput and log, because there are just too many of them pre1 = makeGuid()[:3] pre2 = makeGuid()[:3] mkDir(os.path.join(self.workingDirectory, pre1, pre2)) initialDirPrefix = "%s/%s" % (pre1, pre2) self.log.debug("InitialDir: %s" % os.path.join(self.workingDirectory, initialDirPrefix)) self.log.debug("ExtraSubmitString:\n### \n %s \n###" % self.extraSubmitString) fd, name = tempfile.mkstemp(suffix='.sub', prefix='HTCondorCE_', dir=self.workingDirectory) subFile = os.fdopen(fd, 'w') executable = os.path.join(self.workingDirectory, executable) localScheddOptions = """ ShouldTransferFiles = YES WhenToTransferOutput = ON_EXIT_OR_EVICT """ if self.useLocalSchedd else "" targetUniverse = "grid" if self.useLocalSchedd else "vanilla" sub = """ executable = %(executable)s universe = %(targetUniverse)s use_x509userproxy = true output = $(Cluster).$(Process).out error = $(Cluster).$(Process).err log = $(Cluster).$(Process).log environment = "HTCONDOR_JOBID=$(Cluster).$(Process)" initialdir = %(initialDir)s grid_resource = condor %(ceName)s %(ceName)s:9619 transfer_output_files = "" %(localScheddOptions)s kill_sig=SIGTERM %(extraString)s Queue %(nJobs)s """ % dict(executable=executable, nJobs=nJobs, ceName=self.ceName, extraString=self.extraSubmitString, initialDir=os.path.join(self.workingDirectory, initialDirPrefix), localScheddOptions=localScheddOptions, targetUniverse=targetUniverse, ) subFile.write(sub) subFile.close() return name
def testMakeGuid(self): """ makeGuid tests """ # no filename - fake guid produced self.assertEqual(checkGuid(makeGuid()), True, "fake guid for inexisting file") # using this python file self.assertEqual( checkGuid( makeGuid( os.path.abspath(__file__))), True, "guid for FileTestCase.py file")
def addSubRequest(self,type,requestDict): """ Add a new sub-requests of specified type. Overrides the corresponding method of the base class """ # Initialise the sub-request ind = self.initiateSubRequest(type)['Value'] # Stuff the sub-request with the attributes attributeDict = {} for key in self.requestAttributes: if requestDict['Attributes'].has_key(key): attributeDict[key] = requestDict['Attributes'][key] else: attributeDict[key] = '' if not attributeDict['RequestType']: attributeDict['RequestType'] = type if not attributeDict['Status']: attributeDict['Status'] = 'Waiting' if not attributeDict['SubRequestID']: attributeDict['SubRequestID'] = makeGuid() self.setSubRequestAttributes(ind,type,attributeDict) # Stuff the sub-request with the files fileDict = {} files = requestDict['Files'] ifile = 1 for file in files.values(): for key in self.fileAttributes: if not file.has_key(key): file[key] = '' if not file['Status']: file['Status'] = 'Waiting' if not file['GUID']: file['GUID'] = makeGuid() if not file['Attempt']: file['Attempt'] = 1 fileDict['File%d' % ifile] = file ifile += 1 res = self.setSubRequestFiles(ind,type,files.values()) # Stuff the sub-request with the datasets if requestDict.has_key('Datasets'): datasets = requestDict['Datasets'] else: datasets = {} self.setSubRequestDatasets(ind,type,datasets.values())
def submitJob( self, executableFile, proxy, numberOfJobs = 1, processors = 1 ): """ Method to submit job """ self.log.verbose( "Executable file path: %s" % executableFile ) if not os.access( executableFile, 5 ): os.chmod( executableFile, stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH ) batchIDList = [] stampDict = {} if numberOfJobs == 1: jdlName, diracStamp = self.__writeJDL( executableFile, processors = processors ) cmd = ['glite-ce-job-submit', '-n', '-a', '-N', '-r', '%s/%s' % ( self.ceName, self.queue ), '%s' % jdlName ] result = executeGridCommand( self.proxy, cmd, self.gridEnv ) os.unlink( jdlName ) if result['OK']: if result['Value'][0]: # We have got a non-zero status code errorString = '\n'.join( result['Value'][1:] ).strip() return S_ERROR( 'Pilot submission failed with error: %s ' % errorString ) pilotJobReference = result['Value'][1].strip() if not pilotJobReference: return S_ERROR( 'No pilot reference returned from the glite job submission command' ) if not pilotJobReference.startswith( 'https' ): return S_ERROR( 'Invalid pilot reference %s' % pilotJobReference ) batchIDList.append( pilotJobReference ) stampDict[pilotJobReference] = diracStamp else: delegationID = makeGuid() cmd = [ 'glite-ce-delegate-proxy', '-e', '%s' % self.ceName, '%s' % delegationID ] result = executeGridCommand( self.proxy, cmd, self.gridEnv ) if not result['OK']: self.log.error( 'Failed to delegate proxy', result['Message'] ) return result for _i in range( numberOfJobs ): jdlName, diracStamp = self.__writeJDL( executableFile, processors = processors ) cmd = ['glite-ce-job-submit', '-n', '-N', '-r', '%s/%s' % ( self.ceName, self.queue ), '-D', '%s' % delegationID, '%s' % jdlName ] result = executeGridCommand( self.proxy, cmd, self.gridEnv ) os.unlink( jdlName ) if not result['OK']: break if result['Value'][0] != 0: break pilotJobReference = result['Value'][1].strip() if pilotJobReference and pilotJobReference.startswith( 'https' ): batchIDList.append( pilotJobReference ) stampDict[pilotJobReference] = diracStamp else: break if batchIDList: result = S_OK( batchIDList ) result['PilotStampDict'] = stampDict else: result = S_ERROR( 'No pilot references obtained from the glite job submission' ) return result
def __writeXRSL(self, executableFile, processors=1): """ Create the JDL for submission """ diracStamp = makeGuid()[:8] xrslMPAdditions = '' if processors > 1: xrslMPAdditions = """ (count = %(processors)u) %(xrslMPExtraString)s """ % { 'processors': processors, 'xrslMPExtraString': self.xrslMPExtraString } xrsl = """ &(executable="%(executable)s") (inputFiles=(%(executable)s "%(executableFile)s")) (stdout="%(diracStamp)s.out") (stderr="%(diracStamp)s.err") (outputFiles=("%(diracStamp)s.out" "") ("%(diracStamp)s.err" "")) (queue=%(queue)s) %(xrslMPAdditions)s %(xrslExtraString)s """ % { 'executableFile': executableFile, 'executable': os.path.basename(executableFile), 'diracStamp': diracStamp, 'queue': self.arcQueue, 'xrslMPAdditions': xrslMPAdditions, 'xrslExtraString': self.xrslExtraString } return xrsl, diracStamp
def test_registerFile(self): lfn = '/lhcb/test/unit-test/ReplicaManager/registerFile/testFile.%s' % time.time() physicalFile = 'srm://host:port/srm/managerv2?SFN=/sa/path%s' % lfn fileSize = 10000 storageElementName = 'CERN-RAW' fileGuid = makeGuid() fileTuple = (lfn,physicalFile,fileSize,storageElementName,fileGuid) registerRes = self.dataManager.registerFile(fileTuple) removeCatalogReplicaRes = self.dataManager.removeCatalogReplica(storageElementName,lfn) removeFileRes = self.dataManager.removeFile(lfn) # Check that the file registration was done correctly self.assert_(registerRes['OK']) self.assert_(registerRes['Value'].has_key('Successful')) self.assert_(registerRes['Value']['Successful'].has_key(lfn)) self.assert_(registerRes['Value']['Successful'][lfn]) # Check that the replica removal was successful self.assert_(removeCatalogReplicaRes['OK']) self.assert_(removeCatalogReplicaRes['Value'].has_key('Successful')) self.assert_(removeCatalogReplicaRes['Value']['Successful'].has_key(lfn)) self.assert_(removeCatalogReplicaRes['Value']['Successful'][lfn]) # Check that the removal was successful self.assert_(removeFileRes['OK']) self.assert_(removeFileRes['Value'].has_key('Successful')) self.assert_(removeFileRes['Value']['Successful'].has_key(lfn)) self.assert_(removeFileRes['Value']['Successful'][lfn])
def __writeSub( self, executable, nJobs ): """ Create the Sub File for submission """ workingDirectory = self.ceParameters['WorkingDirectory'] initialDir = os.path.dirname( workingDirectory ) self.log.debug( "Working directory: %s " % workingDirectory ) ##We randomize the location of the pilotoutput and log, because there are just too many of them pre1 = makeGuid()[:3] pre2 = makeGuid()[:3] try: os.makedirs( os.path.join( initialDir, pre1, pre2 ) ) except OSError: pass initialDirPrefix = "%s/%s" %( pre1, pre2 ) self.log.debug( "InitialDir: %s" % os.path.join(initialDir,initialDirPrefix) ) fd, name = tempfile.mkstemp( suffix = '.sub', prefix = 'HTCondorCE_', dir = workingDirectory ) subFile = os.fdopen( fd, 'w' ) executable = os.path.join( workingDirectory, executable ) sub = """ executable = %(executable)s universe = grid use_x509userproxy = true output = $(Cluster).$(Process).out error = $(Cluster).$(Process).err log = $(Cluster).$(Process).log environment = "HTCONDOR_JOBID=$(Cluster).$(Process)" initialdir = %(initialDir)s grid_resource = condor %(ceName)s %(ceName)s:9619 ShouldTransferFiles = YES WhenToTransferOutput = ON_EXIT_OR_EVICT kill_sig=SIGTERM Queue %(nJobs)s """ % dict( executable=executable, nJobs=nJobs, ceName=self.ceName, initialDir=os.path.join(initialDir,initialDirPrefix), ) subFile.write( sub ) subFile.close() return name
def getFileMetadata(self, candidateFiles): """Returns the candidate file dictionary with associated metadata. :param dict candidateFiles: The input candidate files dictionary has the structure: {'lfn':'','path':'','workflowSE':''} This also assumes the files are in the current working directory. :return: S_OK with File Metadata, S_ERROR """ #Retrieve the POOL File GUID(s) for any final output files LOG.info('Will search for POOL GUIDs for: %s' % (', '.join(candidateFiles.keys()))) pfnGUIDs = {} generated = [] for fname in candidateFiles.keys(): guid = makeGuid(fname) pfnGUIDs[fname] = guid generated.append(fname) pfnGUID = S_OK(pfnGUIDs) pfnGUID['generated'] = generated #pfnGUID = getGUID(candidateFiles.keys()) #if not pfnGUID['OK']: # LOG.error('PoolXMLFile failed to determine POOL GUID(s) for output file list, these will be generated by \ # the DataManager',pfnGUID['Message']) # for fileName in candidateFiles.keys(): # candidateFiles[fileName]['guid']='' #if pfnGUID['generated']: LOG.debug('Generated GUID(s) for the following files ', ', '.join(pfnGUID['generated'])) #else: # LOG.info('GUIDs found for all specified POOL files: %s' %(string.join(candidateFiles.keys(),', '))) for pfn, guid in pfnGUID['Value'].items(): candidateFiles[pfn]['GUID'] = guid #Get all additional metadata about the file necessary for requests final = {} for fileName, metadata in candidateFiles.items(): fileDict = {} fileDict['LFN'] = metadata['lfn'] fileDict['Size'] = os.path.getsize(fileName) adlerChecksum = fileAdler(fileName) fileDict['Addler'] = adlerChecksum fileDict['ADLER32'] = adlerChecksum fileDict['Checksum'] = adlerChecksum fileDict['ChecksumType'] = "ADLER32" fileDict['GUID'] = metadata['GUID'] fileDict['Status'] = "Waiting" final[fileName] = metadata final[fileName]['filedict'] = fileDict final[fileName]['localpath'] = '%s/%s' % (os.getcwd(), fileName) #Sanity check all final candidate metadata keys are present (return S_ERROR if not) mandatoryKeys = ['GUID', 'filedict'] #filedict is used for requests (this method adds guid and filedict) for fileName, metadata in final.items(): for key in mandatoryKeys: if key not in metadata: return S_ERROR('File %s has missing %s' % (fileName, key)) return S_OK(final)
def addSubRequest(self, requestDict, rType): """ Add a new sub-requests of specified type The 'Attributes':ExecutionOrder can be set to 'last' to always be the last to be executed. """ # Initialise the sub-request index = self.initiateSubRequest(rType)["Value"] # Stuff the sub-request with the attributes attributeDict = { "Status": "Waiting", "SubRequestID": makeGuid(), "CreationTime": str(datetime.datetime.utcnow()), "ExecutionOrder": 0, } for attr, value in requestDict["Attributes"].items(): if attr == "ExecutionOrder" and str(value).lower() == "last": value = self._getLastOrder() + 1 attributeDict[attr] = value for attr in self.subAttributeNames: if not attr in attributeDict.keys(): attributeDict[attr] = "" if attr == "ExecutionOrder": attributeDict[attr] = 0 if attr == "Status": attributeDict[attr] = "Waiting" self.setSubRequestAttributes(index, rType, attributeDict) if requestDict.has_key("Files"): files = [] for rFile in requestDict["Files"]: fileDict = {"Status": "Waiting", "FileID": makeGuid(), "Attempt": 1} for attr, value in rFile.items(): fileDict[attr] = value files.append(fileDict) self.setSubRequestFiles(index, rType, files) if requestDict.has_key("Datasets"): datasets = [] for dataset in requestDict["Datasets"]: datasetDict = {"Status": "Waiting"} for attr, value in rFile.items(): fileDict[attr] = value datasets.append(datasetDict) self.setSubRequestDatasets(index, rType, datasets) return S_OK(index)
def registerFile(self, lfn): pfn = "protocol://host:port/storage/path%s" % lfn size = 10000000 se = "DIRAC-storage" guid = makeGuid() adler = stringAdler(guid) fileDict = {} fileDict[lfn] = {"PFN": pfn, "Size": size, "SE": se, "GUID": guid, "Checksum": adler} res = self.catalog.addFile(fileDict) return self.parseResult(res, lfn)
def registerFile(self,lfn): pfn = 'protocol://host:port/storage/path%s' % lfn size = 10000000 se = 'DIRAC-storage' guid = makeGuid() adler = stringAdler(guid) fileDict = {} fileDict[lfn] = {'PFN':pfn,'Size':size,'SE':se,'GUID':guid,'Checksum':adler} res = self.catalog.addFile(fileDict) return self.parseResult(res,lfn)
def getFileMetadata(self, candidateFiles): """Returns the candidate file dictionary with associated metadata. @param candidateFiles: The input candidate files dictionary has the structure: {'lfn':'','path':'','workflowSE':''} This also assumes the files are in the current working directory. @return: File Metadata """ # Retrieve the POOL File GUID(s) for any final output files self.log.info("Will search for POOL GUIDs for: %s" % (string.join(candidateFiles.keys(), ", "))) pfnGUIDs = {} generated = [] for fname in candidateFiles.keys(): guid = makeGuid(fname) pfnGUIDs[fname] = guid generated.append(fname) pfnGUID = S_OK(pfnGUIDs) pfnGUID["generated"] = generated # pfnGUID = getGUID(candidateFiles.keys()) # if not pfnGUID['OK']: # self.log.error('PoolXMLFile failed to determine POOL GUID(s) for output file list, these will be generated by \ # the ReplicaManager',pfnGUID['Message']) # for fileName in candidateFiles.keys(): # candidateFiles[fileName]['guid']='' # if pfnGUID['generated']: self.log.debug("Generated GUID(s) for the following files ", string.join(pfnGUID["generated"], ", ")) # else: # self.log.info('GUIDs found for all specified POOL files: %s' %(string.join(candidateFiles.keys(),', '))) for pfn, guid in pfnGUID["Value"].items(): candidateFiles[pfn]["guid"] = guid # Get all additional metadata about the file necessary for requests final = {} for fileName, metadata in candidateFiles.items(): fileDict = {} fileDict["LFN"] = metadata["lfn"] fileDict["Size"] = os.path.getsize(fileName) fileDict["Addler"] = fileAdler(fileName) fileDict["GUID"] = metadata["guid"] fileDict["Status"] = "Waiting" final[fileName] = metadata final[fileName]["filedict"] = fileDict final[fileName]["localpath"] = "%s/%s" % (os.getcwd(), fileName) # Sanity check all final candidate metadata keys are present (return S_ERROR if not) mandatoryKeys = ["guid", "filedict"] # filedict is used for requests (this method adds guid and filedict) for fileName, metadata in final.items(): for key in mandatoryKeys: if not metadata.has_key(key): return S_ERROR("File %s has missing %s" % (fileName, key)) return S_OK(final)
def submitJob( self, executableFile, proxy, numberOfJobs = 1 ): """ Method to submit job """ self.log.verbose( "Executable file path: %s" % executableFile ) if not os.access( executableFile, 5 ): os.chmod( executableFile, 0755 ) batchIDList = [] stampDict = {} if numberOfJobs == 1: jdlName, diracStamp = self.__writeJDL( executableFile ) cmd = ['glite-ce-job-submit', '-n', '-a', '-N', '-r', '%s/%s' % ( self.ceName, self.queue ), '%s' % jdlName ] result = executeGridCommand( self.proxy, cmd, self.gridEnv ) if result['OK']: pilotJobReference = result['Value'][1].strip() batchIDList.append( pilotJobReference ) stampDict[pilotJobReference] = diracStamp os.unlink( jdlName ) else: delegationID = makeGuid() cmd = [ 'glite-ce-delegate-proxy', '-e', '%s' % self.ceName, '%s' % delegationID ] result = executeGridCommand( self.proxy, cmd, self.gridEnv ) if not result['OK']: self.log.error('Failed to delegate proxy: %s' % result['Message']) return result for i in range( numberOfJobs ): jdlName, diracStamp = self.__writeJDL( executableFile ) cmd = ['glite-ce-job-submit', '-n', '-N', '-r', '%s/%s' % ( self.ceName, self.queue ), '-D', '%s' % delegationID, '%s' % jdlName ] result = executeGridCommand( self.proxy, cmd, self.gridEnv ) if not result['OK']: break if result['Value'][0] != 0: break pilotJobReference = result['Value'][1].strip() if pilotJobReference: batchIDList.append( pilotJobReference ) stampDict[pilotJobReference] = diracStamp else: break os.unlink( jdlName ) os.unlink( executableFile ) if batchIDList: result = S_OK( batchIDList ) result['PilotStampDict'] = stampDict else: result = S_ERROR('No pilot references obtained from the glite job submission') return result
def addSubRequest( self, requestDict, rType ): """ Add a new sub-requests of specified type The 'Attributes':ExecutionOrder can be set to 'last' to always be the last to be executed. """ # Initialise the sub-request index = self.initiateSubRequest( rType )['Value'] # Stuff the sub-request with the attributes attributeDict = {'Status':'Waiting', 'SubRequestID':makeGuid(), 'CreationTime': str( datetime.datetime.utcnow() ), 'ExecutionOrder':0} for attr, value in requestDict['Attributes'].items(): if attr == 'ExecutionOrder' and str( value ).lower() == 'last': value = self._getLastOrder() + 1 attributeDict[attr] = value for attr in self.subAttributeNames: if not attr in attributeDict.keys(): attributeDict[attr] = '' if attr == "ExecutionOrder": attributeDict[attr] = 0 if attr == "Status": attributeDict[attr] = "Waiting" self.setSubRequestAttributes( index, rType, attributeDict ) if requestDict.has_key( 'Files' ): files = [] for rFile in requestDict['Files']: fileDict = {'Status':'Waiting', 'FileID':makeGuid(), 'Attempt':1} for attr, value in rFile.items(): fileDict[attr] = value files.append( fileDict ) self.setSubRequestFiles( index, rType, files ) if requestDict.has_key( 'Datasets' ): datasets = [] for dataset in requestDict['Datasets']: datasetDict = {'Status':'Waiting'} for attr, value in rFile.items(): fileDict[attr] = value datasets.append( datasetDict ) self.setSubRequestDatasets( index, rType, datasets ) return S_OK( index )
def getFileMetadata(self, candidateFiles): """Returns the candidate file dictionary with associated metadata. @param candidateFiles: The input candidate files dictionary has the structure: {'lfn':'','path':'','workflowSE':''} This also assumes the files are in the current working directory. @return: File Metadata """ #Retrieve the POOL File GUID(s) for any final output files self.log.info('Will search GUIDs for: %s' %(', '.join(candidateFiles.keys()))) pfnGUIDs = {} generated = [] for fname in candidateFiles.keys(): guid = makeGuid(fname) pfnGUIDs[fname] = guid generated.append(fname) pfnGUID = S_OK(pfnGUIDs) pfnGUID['generated'] = generated self.log.debug('Generated GUID(s) for the following files ', ', '.join(pfnGUID['generated'])) for pfn, guid in pfnGUID['Value'].items(): candidateFiles[pfn]['GUID'] = guid #Get all additional metadata about the file necessary for requests final = {} for fileName, metadata in candidateFiles.items(): fileDict = {} fileDict['LFN'] = metadata['lfn'] fileDict['Size'] = os.path.getsize(fileName) fileDict['Addler'] = fileAdler(fileName) fileDict['GUID'] = metadata['GUID'] fileDict['Status'] = "Waiting" final[fileName] = metadata final[fileName]['filedict'] = fileDict final[fileName]['localpath'] = '%s/%s' % (os.getcwd(), fileName) gLogger.verbose("Full file dict", str(final)) #Sanity check all final candidate metadata keys are present (return S_ERROR if not) mandatoryKeys = ['GUID', 'filedict'] #filedict is used for requests (this method adds guid and filedict) for fileName, metadata in final.items(): for key in mandatoryKeys: if not metadata.has_key(key): return S_ERROR('File %s has missing %s' % (fileName, key)) return S_OK(final)
def _submitJobToHost(self, executableFile, numberOfJobs, host=None): """ Submit prepared executable to the given host """ ssh = SSH(host=host, parameters=self.ceParameters) # Copy the executable submitFile = '%s/%s' % (self.executableArea, os.path.basename(executableFile)) result = ssh.scpCall(30, executableFile, submitFile, postUploadCommand='chmod +x %s' % submitFile) if not result['OK']: return result jobStamps = [] for _i in xrange(numberOfJobs): jobStamps.append(makeGuid()[:8]) numberOfProcessors = self.ceParameters.get('NumberOfProcessors', 1) wholeNode = self.ceParameters.get('WholeNode', False) # Collect command options commandOptions = {'Executable': submitFile, 'NJobs': numberOfJobs, 'SubmitOptions': self.submitOptions, 'JobStamps': jobStamps, 'WholeNode': wholeNode, 'NumberOfProcessors': numberOfProcessors, 'Preamble': self.preamble} resultCommand = self.__executeHostCommand('submitJob', commandOptions, ssh=ssh, host=host) if not resultCommand['OK']: return resultCommand result = resultCommand['Value'] if result['Status'] != 0: return S_ERROR('Failed job submission: %s' % result['Message']) else: batchIDs = result['Jobs'] if batchIDs: ceHost = host if host is None: ceHost = self.ceName jobIDs = ['%s%s://%s/%s' % (self.ceType.lower(), self.batchSystem.lower(), ceHost, _id) for _id in batchIDs] else: return S_ERROR('No jobs IDs returned') result = S_OK(jobIDs) self.submittedJobs += len(batchIDs) return result
def submitJob(self, executableFile, proxy=None, numberOfJobs=1): if not os.access(executableFile, 5): os.chmod(executableFile, stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH) # if no proxy is supplied, the executable can be submitted directly # otherwise a wrapper script is needed to get the proxy to the execution node # The wrapper script makes debugging more complicated and thus it is # recommended to transfer a proxy inside the executable if possible. if proxy: self.log.verbose('Setting up proxy for payload') wrapperContent = bundleProxy(executableFile, proxy) name = writeScript(wrapperContent, os.getcwd()) submitFile = name else: # no proxy submitFile = executableFile jobStamps = [] for _i in range(numberOfJobs): jobStamps.append(makeGuid()[:8]) batchDict = {'Executable': submitFile, 'NJobs': numberOfJobs, 'OutputDir': self.batchOutput, 'ErrorDir': self.batchError, 'SubmitOptions': self.submitOptions, 'ExecutionContext': self.execution, 'JobStamps': jobStamps, 'Queue': self.queue} resultSubmit = self.batch.submitJob(**batchDict) if proxy: os.remove(submitFile) if resultSubmit['Status'] == 0: self.submittedJobs += len(resultSubmit['Jobs']) # jobIDs = [ self.ceType.lower()+'://'+self.ceName+'/'+_id for _id in resultSubmit['Jobs'] ] # FIXME: It would be more proper to fix pilotCommands.__setFlavour where 'ssh' is hardcoded than # making this illogical fix, but there is no good way for pilotCommands to know its origin ceType. # So, the jobIDs here need to start with 'ssh', not ceType, to accomodate # them to those hardcoded in pilotCommands.__setFlavour jobIDs = ['ssh' + self.batchSystem.lower() + '://' + self.ceName + '/' + _id for _id in resultSubmit['Jobs']] result = S_OK(jobIDs) else: result = S_ERROR(resultSubmit['Message']) return result
def __init__(self,rpcStub= None,executionOrder=0): """Instantiates the Workflow object and some default parameters. """ self.subAttributeNames = ['Status','SubRequestID','Operation','ExecutionOrder','CreationTime','LastUpdate','Arguments'] self.subAttributes = {} for attr in self.subAttributeNames: self.subAttributes[attr] = "Unknown" # Some initial values self.subAttributes['Status'] = "Waiting" self.subAttributes['SubRequestID'] = makeGuid() self.subAttributes['CreationTime'] = Time.toString() self.subAttributes['ExecutionOrder'] = executionOrder if rpcStub: self.subAttributes['Arguments'] = DEncode.encode(rpcStub) self.subAttributes['Operation'] = rpcStub[1]
def files( self, userName, userGroup ): """ get list of files in user domain """ files = [] for i in range( 10 ): fname = "/tmp/testUserFile-%s" % i if userGroup == "lhcb_user": lfn = "/lhcb/user/%s/%s/%s" % ( userName[0], userName, fname.split( "/" )[-1] ) else: lfn = "/lhcb/certification/test/rmsdms/%s" % fname.split( "/" )[-1] fh = open( fname, "w+" ) for i in range( 100 ): fh.write( str( random.randint( 0, i ) ) ) fh.close() size = os.stat( fname ).st_size checksum = fileAdler( fname ) guid = makeGuid( fname ) files.append( ( fname, lfn, size, checksum, guid ) ) return files
def submitJob(self, executableFile, proxy, numberOfJobs=1): """ Method to submit job """ self.log.verbose("Executable file path: %s" % executableFile) if not os.access(executableFile, 5): os.chmod(executableFile, 0o755) subName = self.__writeSub(executableFile, numberOfJobs) jobStamps = [] for _i in range(numberOfJobs): jobStamps.append(makeGuid()[:8]) cmd = ['condor_submit', '-terse', subName] # the options for submit to remote are different than the other remoteScheddOptions scheddOptions = [] if self.useLocalSchedd else ['-pool', '%s:9619' % self.ceName, '-remote', self.ceName] for op in scheddOptions: cmd.insert(-1, op) result = executeGridCommand(self.proxy, cmd, self.gridEnv) self.log.verbose(result) os.unlink(subName) if not result['OK']: self.log.error("Failed to submit jobs to htcondor", result['Message']) return result if result['Value'][0]: # We have got a non-zero status code errorString = result['Value'][2] if result['Value'][2] else result['Value'][1] return S_ERROR('Pilot submission failed with error: %s ' % errorString.strip()) pilotJobReferences = self.__getPilotReferences(result['Value'][1].strip()) if not pilotJobReferences['OK']: return pilotJobReferences pilotJobReferences = pilotJobReferences['Value'] self.log.verbose("JobStamps: %s " % jobStamps) self.log.verbose("pilotRefs: %s " % pilotJobReferences) result = S_OK(pilotJobReferences) result['PilotStampDict'] = dict(zip(pilotJobReferences, jobStamps)) self.log.verbose("Result for submission: %s " % result) return result
def __writeXRSL( self, executableFile ): """ Create the JDL for submission """ diracStamp = makeGuid()[:8] xrsl = """ &(executable="%(executable)s") (inputFiles=(%(executable)s "%(executableFile)s")) (stdout="%(diracStamp)s.out") (stderr="%(diracStamp)s.err") (outputFiles=("%(diracStamp)s.out" "") ("%(diracStamp)s.err" "")) %(xrslExtraString)s """ % { 'executableFile':executableFile, 'executable':os.path.basename( executableFile ), 'diracStamp':diracStamp, # 'queue':self.queue, 'xrslExtraString':self.xrslExtraString } return xrsl, diracStamp
def getGUID( fileNames, directory = '' ): """ This function searches the directory for POOL XML catalog files and extracts the GUID. fileNames can be a string or a list, directory defaults to PWD. """ if not directory: directory = os.getcwd() if not os.path.isdir( directory ): return S_ERROR( '%s is not a directory' % directory ) if not type( fileNames ) == type( [] ): fileNames = [fileNames] gLogger.verbose( 'Will look for POOL XML Catalog GUIDs in %s for %s' % ( directory, ', '.join( fileNames ) ) ) finalCatList = _getPoolCatalogs( directory ) #Create POOL catalog with final list of catalog files and extract GUIDs generated = [] pfnGUIDs = {} catalog = PoolXMLCatalog( finalCatList ) for fname in fileNames: guid = str( catalog.getGuidByPfn( fname ) ) if not guid: guid = makeGuid( fname ) generated.append( fname ) pfnGUIDs[fname] = guid if not generated: gLogger.info( 'Found GUIDs from POOL XML Catalogue for all files: %s' % ', '.join( fileNames) ) else: gLogger.info( 'GUIDs not found from POOL XML Catalogue (and were generated) for: %s' % ', '.join( generated) ) result = S_OK( pfnGUIDs ) result['directory'] = directory result['generated'] = generated return result
def submitJob( self, executableFile, proxy = None, numberOfJobs = 1 ): if not os.access( executableFile, 5 ): os.chmod( executableFile, stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH ) # if no proxy is supplied, the executable can be submitted directly # otherwise a wrapper script is needed to get the proxy to the execution node # The wrapper script makes debugging more complicated and thus it is # recommended to transfer a proxy inside the executable if possible. if proxy: self.log.verbose( 'Setting up proxy for payload' ) wrapperContent = bundleProxy( executableFile, proxy ) name = writeScript( wrapperContent, os.getcwd() ) submitFile = name else: # no proxy submitFile = executableFile jobStamps = [] for _i in range( numberOfJobs ): jobStamps.append( makeGuid()[:8] ) batchDict = { 'Executable': submitFile, 'NJobs': numberOfJobs, 'OutputDir': self.batchOutput, 'ErrorDir': self.batchError, 'SubmitOptions': self.submitOptions, 'ExecutionContext': self.execution, 'JobStamps': jobStamps } resultSubmit = self.batch.submitJob( **batchDict ) if proxy: os.remove( submitFile ) if resultSubmit['Status'] == 0: self.submittedJobs += len( resultSubmit['Jobs'] ) jobIDs = [ self.ceType.lower()+'://'+self.ceName+'/'+_id for _id in resultSubmit['Jobs'] ] result = S_OK( jobIDs ) else: result = S_ERROR( resultSubmit['Message'] ) return result
def __writeXRSL(self, executableFile): """ Create the JDL for submission """ diracStamp = makeGuid()[:8] xrsl = """ &(executable="%(executable)s") (inputFiles=(%(executable)s "%(executableFile)s")) (stdout="%(diracStamp)s.out") (stderr="%(diracStamp)s.err") (outputFiles=("%(diracStamp)s.out" "") ("%(diracStamp)s.err" "")) (queue=%(queue)s) %(xrslExtraString)s """ % { "executableFile": executableFile, "executable": os.path.basename(executableFile), "diracStamp": diracStamp, "queue": self.arcQueue, "xrslExtraString": self.xrslExtraString, } return xrsl, diracStamp
def sweeper( cls ): """ move cached request to the central request manager :param cls: class reference """ cacheDir = cls.cacheDir() ## cache dir empty? if not os.listdir( cacheDir ): gLogger.always("sweeper: CacheDir %s is empty, nothing to do" % cacheDir ) return S_OK() else: ## read 10 cache dir files, the oldest first cachedRequests = [ os.path.abspath( requestFile ) for requestFile in sorted( filter( os.path.isfile, [ os.path.join( cacheDir, requestName ) for requestName in os.listdir( cacheDir ) ] ), key = os.path.getctime ) ][:30] ## set cached requests to the central RequestManager for cachedFile in cachedRequests: try: requestString = "".join( open( cachedFile, "r" ).readlines() ) cachedRequest = RequestContainer( requestString ) requestName = cachedRequest.getAttribute("RequestName")["Value"] ## cibak: hack for DISET requests if requestName == "Unknown": cachedRequest.setAttribute( "RequestName", makeGuid() ) requestName = cachedRequest.getAttribute("RequestName")["Value"] setRequest = cls.requestManager().setRequest( requestName, requestString ) if not setRequest["OK"]: gLogger.error("sweeper: unable to set request '%s' @ RequestManager: %s" % ( requestName, setRequest["Message"] ) ) continue gLogger.info("sweeper: successfully set request '%s' @ RequestManager" % requestName ) os.unlink( cachedFile ) except Exception, error: gLogger.exception( "sweeper: hit by exception %s" % str(error) ) return S_ERROR( "sweeper: hit by exception: %s" % str(error) ) return S_OK()
def __writeXRSL(self, executableFile, processors=1): """ Create the JDL for submission """ diracStamp = makeGuid()[:8] # Evaluate the number of processors to allocate nProcessors = processors if processors > 1 else self.ceParameters.get('NumberOfProcessors', processors) xrslMPAdditions = '' if nProcessors > 1: xrslMPAdditions = """ (count = %(processors)u) %(xrslMPExtraString)s """ % { 'processors': nProcessors, 'xrslMPExtraString': self.xrslMPExtraString } xrsl = """ &(executable="%(executable)s") (inputFiles=(%(executable)s "%(executableFile)s")) (stdout="%(diracStamp)s.out") (stderr="%(diracStamp)s.err") (outputFiles=("%(diracStamp)s.out" "") ("%(diracStamp)s.err" "")) (queue=%(queue)s) %(xrslMPAdditions)s %(xrslExtraString)s """ % { 'executableFile': executableFile, 'executable': os.path.basename(executableFile), 'diracStamp': diracStamp, 'queue': self.arcQueue, 'xrslMPAdditions': xrslMPAdditions, 'xrslExtraString': self.xrslExtraString } return xrsl, diracStamp
def submitJob( self, executableFile, proxy, numberOfJobs = 1 ): """ Method to submit job """ self.log.verbose( "Executable file path: %s" % executableFile ) if not os.access( executableFile, 5 ): os.chmod( executableFile, 0755 ) batchIDList = [] stampDict = {} for _i in xrange(numberOfJobs): diracStamp = makeGuid()[:8] queueName = '%s/%s' % ( self.ceName, self.queue ) cmd = ['globus-job-submit', queueName, "-s", executableFile ] #cmd = ['globus-job-submit', '-r %s' % queueName, '-f %s' % jdlName ] result = executeGridCommand( self.proxy, cmd, self.gridEnv ) self.log.verbose(result) if result['OK']: if result['Value'][0]: # We have got a non-zero status code errorString = result['Value'][2] if result['Value'][2] else result['Value'][1] return S_ERROR( 'Pilot submission failed with error: %s ' % errorString.strip() ) pilotJobReference = result['Value'][1].strip() if not pilotJobReference: return S_ERROR( 'No pilot reference returned from the glite job submission command' ) if not pilotJobReference.startswith( 'https' ): return S_ERROR( 'Invalid pilot reference %s' % pilotJobReference ) batchIDList.append( pilotJobReference ) stampDict[pilotJobReference] = diracStamp if batchIDList: result = S_OK( batchIDList ) result['PilotStampDict'] = stampDict else: result = S_ERROR( 'No pilot references obtained from the glite job submission' ) return result
def main(): Script.parseCommandLine(ignoreErrors=False) files = Script.getPositionalArgs() if len(files) == 0: Script.showHelp() exitCode = 0 import DIRAC from DIRAC.Core.Utilities.File import makeGuid for file in files: try: md5 = makeGuid(file) if md5: print(file.rjust(100), md5.ljust(10)) else: print('ERROR %s: Failed to get md5' % file) exitCode = 2 except Exception as x: print('ERROR %s: Failed to get md5' % file, str(x)) exitCode = 2 DIRAC.exit(exitCode)
def testMakeGuid(): """makeGuid tests""" # no filename - fake guid produced assert checkGuid(makeGuid()) is True, "fake guid for inexisting file" # using this python file assert checkGuid(makeGuid(abspath(__file__))) is True, "guid for FileTestCase.py file"
def main(): dm = DataManager() fileTupleBuffer = [] res = getProxyInfo( False, False ) if not res['OK']: gLogger.error( "Failed to get client proxy information.", res['Message'] ) DIRAC.exit( 2 ) proxyInfo = res['Value'] if proxyInfo['secondsLeft'] == 0: gLogger.error( "Proxy expired" ) DIRAC.exit( 2 ) username = proxyInfo['username'] vo = '' if 'group' in proxyInfo: vo = getVOForGroup( proxyInfo['group'] ) counter = 0 for f in files: counter += 1 local_f=f if not f.startswith('/cefs'): # gLogger.error('File must be under "/cefs"') # continue #if the file to reg is not under /cefs, use put and register folder_name=os.path.basename(os.path.dirname(f)) # lfn = '/cepc/user/%s/%s/jsub/'%(username[0],username) + folder_name + '/' + os.path.basename(f) lfn = '/cepc/user/%s/%s/jsub/'%(username[0],username) + folder_name + '/' + os.path.basename(f) # dirname = os.path.dirname(local_f) # os.system('mkdir -p %s'%(dirname)) # os.system('cp %s %s' %(f,local_f)) do_put_and_register=True else: lfn = '/cepc/lustre-ro' + os.path.abspath(f) do_put_and_register=False result = fcc.isFile(lfn) if result['OK'] and lfn in result['Value']['Successful'] and result['Value']['Successful'][lfn]: continue size = os.path.getsize(f) adler32 = fileAdler(f) guid = makeGuid() fileTuple = (lfn, local_f, size, _se, guid, adler32) fileTupleBuffer.append(fileTuple) gLogger.debug('Register to lfn: %s' % lfn) gLogger.debug('fileTuple: %s' % (fileTuple,)) if len(fileTupleBuffer) >= _bufferSize: if do_put_and_register: result = dm.putAndRegister(lfn, local_f, _se, guid, overwrite=overwrite) else: result = dm.registerFile(fileTupleBuffer) print('register result', result) # if not result['OK']: # gLogger.error('Register file failed') # return 1 del fileTupleBuffer[:] gLogger.debug('%s files registered' % counter) if fileTupleBuffer: if do_put_and_register: result = dm.putAndRegister(lfn, local_f, _se, guid, overwrite=overwrite) else: result = dm.registerFile(fileTupleBuffer) print('register result', result) # if not result['OK']: # gLogger.error('Register file failed') # return 1 del fileTupleBuffer[:] gLogger.info('Totally %s files registered' % counter) return 0
__RCSID__ = "$Id$" import DIRAC from DIRAC.Core.Utilities.File import makeGuid from DIRAC.Core.Base import Script Script.setUsageMessage( '\n'.join( [ __doc__.split( '\n' )[1], 'Usage:', ' %s [option|cfgfile] ... File ...' % Script.scriptName, 'Arguments:', ' File: File Name' ] ) ) Script.parseCommandLine( ignoreErrors = False ) files = Script.getPositionalArgs() if len( files ) == 0: Script.showHelp() exitCode = 0 for file in files: try: md5 = makeGuid( file ) if md5: print file.rjust( 100 ), md5.ljust( 10 ) else: print 'ERROR %s: Failed to get md5' % file exitCode = 2 except Exception as x: print 'ERROR %s: Failed to get md5' % file, str( x ) exitCode = 2 DIRAC.exit( exitCode )
def submitJob(self, executableFile, proxy, numberOfJobs=1): """Method to submit job""" self.log.verbose("Executable file path: %s" % executableFile) if not os.access(executableFile, 5): os.chmod(executableFile, 0o755) # The submitted pilots are going to have a common part of the stamp to construct a path to retrieve results # Then they also have an individual part to make them unique jobStamps = [] commonJobStampPart = makeGuid()[:3] for _i in range(numberOfJobs): jobStamp = commonJobStampPart + makeGuid()[:5] jobStamps.append(jobStamp) # We randomize the location of the pilot output and log, because there are just too many of them location = logDir(self.ceName, commonJobStampPart) nProcessors = self.ceParameters.get("NumberOfProcessors", 1) subName = self.__writeSub(executableFile, numberOfJobs, location, nProcessors) cmd = ["condor_submit", "-terse", subName] # the options for submit to remote are different than the other remoteScheddOptions # -remote: submit to a remote condor_schedd and spool all the required inputs scheddOptions = [] if self.useLocalSchedd else [ "-pool", "%s:9619" % self.ceName, "-remote", self.ceName ] for op in scheddOptions: cmd.insert(-1, op) result = executeGridCommand(self.proxy, cmd, self.gridEnv) self.log.verbose(result) os.remove(subName) if not result["OK"]: self.log.error("Failed to submit jobs to htcondor", result["Message"]) return result if result["Value"][0]: # We have got a non-zero status code errorString = result["Value"][2] if result["Value"][2] else result[ "Value"][1] return S_ERROR("Pilot submission failed with error: %s " % errorString.strip()) pilotJobReferences = self.__getPilotReferences( result["Value"][1].strip()) if not pilotJobReferences["OK"]: return pilotJobReferences pilotJobReferences = pilotJobReferences["Value"] self.log.verbose("JobStamps: %s " % jobStamps) self.log.verbose("pilotRefs: %s " % pilotJobReferences) result = S_OK(pilotJobReferences) result["PilotStampDict"] = dict(zip(pilotJobReferences, jobStamps)) if self.useLocalSchedd: # Executable is transferred afterward # Inform the caller that Condor cannot delete it before the end of the execution result["ExecutableToKeep"] = executableFile self.log.verbose("Result for submission: %s " % result) return result
if lfn in lfnQuery: if counter % 1000 == 0: gLogger.notice('Skip file in query counter: %s' % counter) continue if existCheck: result = fcc.isFile(lfn) if result['OK'] and lfn in result['Value'][ 'Successful'] and result['Value']['Successful'][lfn]: if counter % 1000 == 0: gLogger.notice('Skip file existed counter: %s' % counter) continue size = os.path.getsize(fullFn) adler32 = fileAdler(fullFn) guid = makeGuid() fileTuple = (lfn, fullFn, size, toSE, guid, adler32) fileTupleBuffer.append(fileTuple) gLogger.debug('Register to lfn: %s' % lfn) gLogger.debug('fileTuple: %s' % (fileTuple, )) if len(fileTupleBuffer) >= bufferSize: result = dm.registerFile(fileTupleBuffer) if not result['OK']: gLogger.error('Can not register %s' % fullFn) exit(1) del fileTupleBuffer[:] gLogger.notice('%s files registered' % counter) if fileTupleBuffer: result = dm.registerFile(fileTupleBuffer)
def setBenderModule(self, benderVersion, modulePath, inputData=None, numberOfEvents=-1): """Specifies Bender module to be executed. Any additional files should be specified in the job input sandbox. Input data for Bender should be specified here (can be string or list). Example usage: >>> job = LHCbJob() >>> job.setBenderModule('v8r3','BenderExample.PhiMC', inputData=['LFN:/lhcb/production/DC06/phys-v2-lumi2/00001758/DST/00001758_00000001_5.dst'],numberOfEvents=100) :param benderVersion: Bender Project Version :type benderVersion: string :param modulePath: Import path to module e.g. BenderExample.PhiMC :type modulePath: string :param inputData: Input data for application :type inputData: single LFN or list of LFNs :param numberOfEvents: Number of events to process e.g. -1 :type numberOfEvents: integer """ kwargs = { 'benderVersion': benderVersion, 'modulePath': modulePath, 'inputData': inputData, 'numberOfEvents': numberOfEvents } if not isinstance(benderVersion, str): return self._reportError('Bender version should be a string', __name__, **kwargs) if not isinstance(modulePath, str): return self._reportError('Bender module path should be a string', __name__, **kwargs) if not isinstance(numberOfEvents, int): try: numberOfEvents = int(numberOfEvents) except ValueError: return self._reportError( 'Number of events should be an integer or convertible to an integer', __name__, **kwargs) if not inputData: return S_ERROR("Need input data for Bender applications") if isinstance(inputData, str): inputData = [inputData] if not isinstance(inputData, list): return self._reportError( 'Input data should be specified as a list or a string', __name__, **kwargs) poolCatName = 'xmlcatalog_file:pool_xml_catalog.xml' benderScript = ['#!/usr/bin/env python'] benderScript.append('from Gaudi.Configuration import FileCatalog') benderScript.append('FileCatalog (Catalogs = ["%s"] )' % poolCatName) benderScript.append('import %s as USERMODULE' % modulePath) benderScript.append('USERMODULE.configure()') benderScript.append('gaudi = USERMODULE.appMgr()') benderScript.append('evtSel = gaudi.evtSel()') benderScript.append('evtSel.open ( %s ) ' % inputData) benderScript.append('USERMODULE.run( %s )\n' % numberOfEvents) guid = makeGuid() tmpdir = '/tmp/' + guid self.log.verbose('Created temporary directory for submission %s' % (tmpdir)) mkDir(tmpdir) with open('%s/BenderScript.py' % tmpdir, 'w') as fopen: self.log.verbose('Bender script is: %s/BenderScript.py' % tmpdir) fopen.write('\n'.join(benderScript)) # should try all components of the PYTHONPATH before giving up... userModule = '%s.py' % (modulePath.split('.')[-1]) self.log.verbose('Looking for user module with name: %s' % userModule) if os.path.exists(userModule): self.addToInputSandbox.append(userModule) self.setInputData(inputData) self.setApplicationScript('Bender', benderVersion, '%s/BenderScript.py' % tmpdir, logFile='Bender%s.log' % benderVersion) return S_OK(benderScript)
def createInstance(self, instanceID=''): if not instanceID: instanceID = makeGuid()[:8] self.parameters['VMUUID'] = instanceID self.parameters['VMType'] = self.parameters.get('CEType', 'EC2') createNodeDict = {} # Image if "ImageID" in self.parameters and 'ImageName' not in self.parameters: try: images = self.__ec2.images.filter( Filters=[{ 'Name': 'name', 'Values': [self.parameters['ImageName']] }]) imageId = None for image in images: imageId = image.id break except Exception as e: self.log.error( "Exception when get ID from image name %s:" % self.parameters['ImageName'], e) return S_ERROR("Failed to get image for Name %s" % self.parameters['ImageName']) if imageId is None: return S_ERROR("Image name %s not found" % self.parameters['ImageName']) elif "ImageID" in self.parameters: try: self.__ec2.images.filter(ImageIds=[self.parameters['ImageID']]) except Exception as e: return S_ERROR("Failed to get image for ID %s" % self.parameters['ImageID']) imageId = self.parameters['ImageID'] else: return S_ERROR('No image specified') createNodeDict['ImageId'] = imageId # Instance type if 'FlavorName' not in self.parameters: return S_ERROR('No flavor specified') instanceType = self.parameters['FlavorName'] createNodeDict['InstanceType'] = instanceType # User data result = self._createUserDataScript() if not result['OK']: return result createNodeDict['UserData'] = str(result['Value']) # Other params for param in ['KeyName', 'SubnetId', 'EbsOptimized']: if param in self.parameters: createNodeDict[param] = self.parameters[param] self.log.info("Creating node:") for key, value in createNodeDict.items(): self.log.verbose("%s: %s" % (key, value)) # Create the VM instance now try: instances = self.__ec2.create_instances(MinCount=1, MaxCount=1, **createNodeDict) except Exception as e: errmsg = 'Exception in ec2 create_instances: %s' % e self.log.error(errmsg) return S_ERROR(errmsg) if len(instances) < 1: errmsg = 'ec2 create_instances failed to create any VM' self.log.error(errmsg) return S_ERROR(errmsg) # Create the name in tags ec2Id = instances[0].id tags = [{'Key': 'Name', 'Value': 'DIRAC_%s' % instanceID}] try: self.__ec2.create_tags(Resources=[ec2Id], Tags=tags) except Exception as e: errmsg = 'Exception setup name for %s: %s' % (ec2Id, e) self.log.error(errmsg) return S_ERROR(errmsg) # Properties of the instance nodeDict = {} # nodeDict['PublicIP'] = publicIP nodeDict['InstanceID'] = instanceID if instanceType in self.__instanceTypeInfo: nodeDict['NumberOfProcessors'] = self.__instanceTypeInfo[ instanceType]['vCPU'] nodeDict['RAM'] = self.__instanceTypeInfo[instanceType]['Memory'] else: nodeDict['NumberOfProcessors'] = 1 return S_OK((ec2Id, nodeDict))
def createInstance(self, instanceID=""): if not instanceID: instanceID = makeGuid()[:8] self.parameters["VMUUID"] = instanceID self.parameters["VMType"] = self.parameters.get("CEType", "EC2") createNodeDict = {} # Image if "ImageID" in self.parameters and "ImageName" not in self.parameters: try: images = self.__ec2.images.filter( Filters=[{ "Name": "name", "Values": [self.parameters["ImageName"]] }]) imageId = None for image in images: imageId = image.id break except Exception as e: self.log.exception( "Exception when get ID from image name %s:" % self.parameters["ImageName"]) return S_ERROR("Failed to get image for Name %s" % self.parameters["ImageName"]) if imageId is None: return S_ERROR("Image name %s not found" % self.parameters["ImageName"]) elif "ImageID" in self.parameters: try: self.__ec2.images.filter(ImageIds=[self.parameters["ImageID"]]) except Exception as e: self.log.exception("Failed to get EC2 image list") return S_ERROR("Failed to get image for ID %s" % self.parameters["ImageID"]) imageId = self.parameters["ImageID"] else: return S_ERROR("No image specified") createNodeDict["ImageId"] = imageId # Instance type if "FlavorName" not in self.parameters: return S_ERROR("No flavor specified") instanceType = self.parameters["FlavorName"] createNodeDict["InstanceType"] = instanceType # User data result = self._createUserDataScript() if not result["OK"]: return result createNodeDict["UserData"] = str(result["Value"]) # Other params for param in ["KeyName", "SubnetId", "EbsOptimized"]: if param in self.parameters: createNodeDict[param] = self.parameters[param] self.log.info("Creating node:") for key, value in createNodeDict.items(): self.log.verbose("%s: %s" % (key, value)) # Create the VM instance now try: instances = self.__ec2.create_instances(MinCount=1, MaxCount=1, **createNodeDict) except Exception as e: self.log.exception("Failed to create EC2 instance") return S_ERROR("Exception in ec2 create_instances: %s" % e) if len(instances) < 1: errmsg = "ec2 create_instances failed to create any VM" self.log.error(errmsg) return S_ERROR(errmsg) # Create the name in tags ec2Id = instances[0].id tags = [{"Key": "Name", "Value": "DIRAC_%s" % instanceID}] try: self.__ec2.create_tags(Resources=[ec2Id], Tags=tags) except Exception as e: self.log.exception("Failed to tag EC2 instance") return S_ERROR("Exception setup name for %s: %s" % (ec2Id, e)) # Properties of the instance nodeDict = {} # nodeDict['PublicIP'] = publicIP nodeDict["InstanceID"] = instanceID if instanceType in self.__instanceTypeInfo: nodeDict["NumberOfProcessors"] = self.__instanceTypeInfo[ instanceType]["vCPU"] nodeDict["RAM"] = self.__instanceTypeInfo[instanceType]["Memory"] else: nodeDict["NumberOfProcessors"] = 1 return S_OK((ec2Id, nodeDict))
def _submitJobToHost( self, executableFile, numberOfJobs, host = None ): """ Submit prepared executable to the given host """ ssh = SSH( host = host, parameters = self.ceParameters ) # Copy the executable sFile = os.path.basename( executableFile ) result = ssh.scpCall( 10, executableFile, '%s/%s' % ( self.executableArea, os.path.basename( executableFile ) ) ) if not result['OK']: return result jobStamps = [] for i in range( numberOfJobs ): jobStamps.append( makeGuid()[:8] ) jobStamp = '#'.join( jobStamps ) subOptions = urllib.quote( self.submitOptions ) cmd = "bash --login -c '%s/%s submit_job %s/%s %s %s %s %d %s %s %s %s'" % ( self.sharedArea, self.controlScript, self.executableArea, os.path.basename( executableFile ), self.batchOutput, self.batchError, self.workArea, numberOfJobs, self.infoArea, jobStamp, self.execQueue, subOptions ) self.log.verbose( 'CE submission command: %s' % cmd ) result = ssh.sshCall( 120, cmd ) if not result['OK']: self.log.error( '%s CE job submission failed' % self.ceType, result['Message'] ) return result sshStatus = result['Value'][0] sshStdout = result['Value'][1] sshStderr = result['Value'][2] # Examine results of the job submission submitHost = host if host is None: submitHost = self.ceParameters['SSHHost'].split('/')[0] if sshStatus == 0: outputLines = sshStdout.strip().replace('\r','').split('\n') try: index = outputLines.index('============= Start output ===============') outputLines = outputLines[index+1:] except: return S_ERROR( "Invalid output from job submission: %s" % outputLines[0] ) try: status = int( outputLines[0] ) except: return S_ERROR( "Failed local batch job submission: %s" % outputLines[0] ) if status != 0: message = "Unknown reason" if len( outputLines ) > 1: message = outputLines[1] return S_ERROR( 'Failed job submission, reason: %s' % message ) else: batchIDs = outputLines[1:] jobIDs = [ self.ceType.lower()+'://'+self.ceName+'/'+id for id in batchIDs ] else: return S_ERROR( '\n'.join( [sshStdout,sshStderr] ) ) result = S_OK ( jobIDs ) self.submittedJobs += len( batchIDs ) return result
try: cat = PoolXMLCatalog(possibleCat) finalCatList.append(possibleCat) except Exception, x: gLogger.debug('Ignoring non-POOL catalogue file %s' % possibleCat) #Create POOL catalog with final list of catalog files and extract GUIDs generated = [] pfnGUIDs = {} gLogger.debug('Final list of catalog files are: %s' % string.join(finalCatList, ', ')) catalog = PoolXMLCatalog(finalCatList) for fname in fileNames: guid = str(catalog.getGuidByPfn(fname)) if not guid: guid = makeGuid(fname) generated.append(fname) pfnGUIDs[fname] = guid if not generated: gLogger.info('Found GUIDs from POOL XML Catalogue for all files: %s' % string.join(fileNames, ', ')) else: gLogger.info( 'GUIDs not found from POOL XML Catalogue (and were generated) for: %s' % string.join(generated, ', ')) result = S_OK(pfnGUIDs) result['directory'] = directory result['generated'] = generated
def createInstance(self, instanceID="", createPublicIP=True): """ This creates a VM instance for the given boot image and creates a context script, taken the given parameters. Successful creation returns instance VM Boots a new node on the OpenStack server defined by self.endpointConfig. The 'personality' of the node is done by self.imageConfig. Both variables are defined on initialization phase. The node name has the following format: <bootImageName><contextMethod><time> It boots the node. If IPpool is defined on the imageConfiguration, a floating IP is created and assigned to the node. :return: S_OK( ( nodeID, publicIP ) ) | S_ERROR """ if not instanceID: instanceID = makeGuid()[:8] self.parameters["VMUUID"] = instanceID self.parameters["VMType"] = self.parameters.get("CEType", "OpenStack") createNodeDict = {} # Get the image object if "ImageID" in self.parameters: try: image = self.__driver.get_image(self.parameters["ImageID"]) except BaseHTTPError as err: if err.code == 404: # Image not found return S_ERROR("Image with ID %s not found" % self.parameters["ImageID"]) return S_ERROR("Failed to get image for ID %s (%s)" % (self.parameters["ImageID"], str(err))) elif "ImageName" in self.parameters: result = self.__getImageByName(self.parameters["ImageName"]) if not result["OK"]: return result image = result["Value"] else: return S_ERROR("No image specified") createNodeDict["image"] = image # Get the flavor object if "FlavorID" in self.parameters: flavor = self.__driver.ex_get_size(self.parameters["FlavorID"]) elif "FlavorName" in self.parameters: result = self.__getFlavorByName(self.parameters["FlavorName"]) if not result["OK"]: return result flavor = result["Value"] else: return S_ERROR("No flavor specified") self.flavor = flavor createNodeDict["size"] = flavor # Get security groups # if 'ex_security_groups' in self.parameters: # result = self.__getSecurityGroups( self.parameters['ex_security_groups'] ) # if not result[ 'OK' ]: # self.log.error( result[ 'Message' ] ) # return result # self.parameters['ex_security_groups'] = result[ 'Value' ] result = self._createUserDataScript() if not result["OK"]: return result createNodeDict["ex_userdata"] = result["Value"] # Optional node contextualization parameters for param in [ "ex_metadata", "ex_pubkey_path", "ex_keyname", "ex_config_drive" ]: if param in self.parameters: createNodeDict[param] = self.parameters[param] createNodeDict["name"] = "DIRAC_%s" % instanceID # createNodeDict['ex_config_drive'] = True self.log.verbose("Creating node:") for key, value in createNodeDict.items(): self.log.verbose("%s: %s" % (key, value)) if "networks" in self.parameters: result = self.getVMNetwork() if not result["OK"]: return result createNodeDict["networks"] = result["Value"] if "keyname" in self.parameters: createNodeDict["ex_keyname"] = self.parameters["keyname"] if "availability_zone" in self.parameters: createNodeDict["ex_availability_zone"] = self.parameters[ "availability_zone"] # Create the VM instance now try: vmNode = self.__driver.create_node(**createNodeDict) except Exception as errmsg: self.log.error("Exception in driver.create_node", errmsg) return S_ERROR(errmsg) publicIP = None if createPublicIP: # Wait until the node is running, otherwise getting public IP fails try: self.__driver.wait_until_running([vmNode], timeout=600) result = self.assignFloatingIP(vmNode) if result["OK"]: publicIP = result["Value"] else: vmNode.destroy() return result except Exception as exc: self.log.debug("Failed to wait node running %s" % str(exc)) vmNode.destroy() return S_ERROR("Failed to wait until the node is Running") return S_OK((vmNode, publicIP))
def submitJob(self, executableFile, proxy, numberOfJobs=1): """Method to submit job""" self.log.verbose("Executable file path: %s" % executableFile) if not os.access(executableFile, 5): os.chmod(executableFile, stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH) nProcessors = self.ceParameters.get("NumberOfProcessors", 1) batchIDList = [] stampDict = {} if numberOfJobs == 1: jdlName, diracStamp = self.__writeJDL(executableFile, processors=nProcessors) cmd = ["glite-ce-job-submit", "-n", "-a", "-N", "-r", "%s/%s" % (self.ceName, self.queue), "%s" % jdlName] result = executeGridCommand(self.proxy, cmd, self.gridEnv) os.unlink(jdlName) if result["OK"]: if result["Value"][0]: # We have got a non-zero status code errorString = "\n".join(result["Value"][1:]).strip() return S_ERROR("Pilot submission failed with error: %s " % errorString) pilotJobReference = result["Value"][1].strip() if not pilotJobReference: return S_ERROR("No pilot reference returned from the glite job submission command") if not pilotJobReference.startswith("https"): return S_ERROR("Invalid pilot reference %s" % pilotJobReference) batchIDList.append(pilotJobReference) stampDict[pilotJobReference] = diracStamp else: delegationID = makeGuid() cmd = ["glite-ce-delegate-proxy", "-e", "%s" % self.ceName, "%s" % delegationID] result = executeGridCommand(self.proxy, cmd, self.gridEnv) if not result["OK"]: self.log.error("Failed to delegate proxy", result["Message"]) return result for _i in range(numberOfJobs): jdlName, diracStamp = self.__writeJDL(executableFile, processors=nProcessors) cmd = [ "glite-ce-job-submit", "-n", "-N", "-r", "%s/%s" % (self.ceName, self.queue), "-D", "%s" % delegationID, "%s" % jdlName, ] result = executeGridCommand(self.proxy, cmd, self.gridEnv) os.unlink(jdlName) if not result["OK"]: self.log.error("General error in execution of glite-ce-job-submit command") break if result["Value"][0] != 0: self.log.error("Error in glite-ce-job-submit command", result["Value"][1] + result["Value"][2]) break pilotJobReference = result["Value"][1].strip() if pilotJobReference and pilotJobReference.startswith("https"): batchIDList.append(pilotJobReference) stampDict[pilotJobReference] = diracStamp else: break if batchIDList: result = S_OK(batchIDList) result["PilotStampDict"] = stampDict else: result = S_ERROR("No pilot references obtained from the glite job submission") return result
__RCSID__ = "$Id$" import DIRAC from DIRAC.Core.Utilities.File import makeGuid from DIRAC.Core.Base import Script Script.setUsageMessage('\n'.join([ __doc__.split('\n')[1], 'Usage:', ' %s [option|cfgfile] ... File ...' % Script.scriptName, 'Arguments:', ' File: File Name' ])) Script.parseCommandLine(ignoreErrors=False) files = Script.getPositionalArgs() if len(files) == 0: Script.showHelp() exitCode = 0 for file in files: try: md5 = makeGuid(file) if md5: print(file.rjust(100), md5.ljust(10)) else: print('ERROR %s: Failed to get md5' % file) exitCode = 2 except Exception as x: print('ERROR %s: Failed to get md5' % file, str(x)) exitCode = 2 DIRAC.exit(exitCode)
def createInstance(self, instanceID=''): if not instanceID: instanceID = makeGuid()[:8] self.parameters['VMUUID'] = instanceID self.parameters['VMType'] = self.parameters.get('CEType', 'EC2') actionArgs = ['--action', 'create'] actionArgs += ['--resource', 'compute'] # Image if not "ImageID" in self.parameters and 'ImageName' in self.parameters: result = self.__getImageByName(self.parameters['ImageName']) if not result['OK']: return result imageId = result['Value'] elif "ImageID" in self.parameters: result = self.__occiCommand([ '--action', 'describe', '--resource', 'os_tpl#%s' % self.parameters['ImageID'] ]) if not result['OK']: return S_ERROR( "Failed to get image for ID %s" % self.parameters['ImageID'], result['Message']) imageId = self.parameters['ImageID'] else: return S_ERROR('No image specified') actionArgs += ['--mixin', 'os_tpl#%s' % imageId] # Optional flavor name if 'FlavorName' in self.parameters: result = self.__occiCommand([ '--action', 'describe', '--resource', 'resource_tpl#%s' % self.parameters['FlavorName'] ]) if not result['OK']: return S_ERROR( "Failed to get flavor %s" % self.parameters['FlavorName'], result['Message']) actionArgs += [ '--mixin', 'resource_tpl#%s' % self.parameters['FlavorName'] ] # Instance name actionArgs += ['--attribute', 'occi.core.title=DIRAC_%s' % instanceID] # Other params for param in []: if param in self.parameters: actionArgs += ['--%s' % param, '%s' % self.parameters[param]] self.log.info("Creating node:") self.log.verbose(' '.join(actionArgs)) # User data result = self._createUserDataScript() if not result['OK']: return result # actionArgs += ['--context', 'user_data=%s' % str( result['Value'] )] f = tempfile.NamedTemporaryFile(delete=False) f.write(str(result['Value'])) f.close() self.log.debug('Write user_data to temp file:', f.name) actionArgs += ['--context', 'user_data=file://%s' % f.name] # Create the VM instance now result = self.__occiCommand(actionArgs) os.unlink(f.name) if not result['OK']: errmsg = 'Error in rOCCI create instances: %s' % result['Message'] self.log.error(errmsg) return S_ERROR(errmsg) occiId = result['Value'].strip() # Properties of the instance nodeDict = {} nodeDict['InstanceID'] = instanceID result = self.__occiCommand( ['--action', 'describe', '--resource', occiId]) if result['OK']: nodeInfo = json.loads(result['Value']) try: nodeDict['NumberOfCPUs'] = nodeInfo[0]['attributes']['occi'][ 'compute']['cores'] nodeDict['RAM'] = nodeInfo[0]['attributes']['occi']['compute'][ 'memory'] except Exception as e: nodeDict['NumberOfCPUs'] = 1 else: nodeDict['NumberOfCPUs'] = 1 return S_OK((occiId, nodeDict))
) for i in fileUnlinkedList: gLogger.notice(i) gLogger.notice("---------- Total: %d unlinked files. ----------\n" % len(fileUnlinkedList)) if deletionSwitch: for i in fileDarkList: g.unlink(HEAD0 + '/' + i) gLogger.notice("%s deleted!" % (HEAD0 + '/' + i)) gLogger.notice("---------- All dark files deleted! ----------\n") if registerOption != 'none': if registerOption in ['dark', 'both']: gLogger.notice("---------- Register dark files: ----------") for i in fileDarkList: fileStat = g.stat(HEAD0 + '/' + i) fileSize = fileStat.st_size fileGuid = makeGuid() fileSum = g.checksum(HEAD0 + '/' + i, "ADLER32") fileTuple = (i, HEAD0 + '/' + i, fileSize, SITE, fileGuid, fileSum) dm.registerFile(fileTuple) gLogger.notice(i + ' registered successfully!') if registerOption in ['unlinked', 'both']: gLogger.notice("---------- Register unlinked files: ----------") for i in fileUnlinkedList: fileTuple = (i, HEAD0 + '/' + i, SITE) dm.registerReplica(fileTuple) gLogger.notice(i + ' registered successfully!') gLogger.notice("---------- Register finished ----------\n")
def _writeXRSL(self, executableFile, inputs=None, outputs=None, executables=None): """Create the JDL for submission :param str executableFile: executable to wrap in a XRSL file :param str/list inputs: path of the dependencies to include along with the executable :param str/list outputs: path of the outputs that we want to get at the end of the execution :param str/list executables: path to inputs that should have execution mode on the remote worker node """ diracStamp = makeGuid()[:8] # Evaluate the number of processors to allocate nProcessors = self.ceParameters.get("NumberOfProcessors", 1) xrslMPAdditions = "" if nProcessors and nProcessors > 1: xrslMPAdditions = """ (count = %(processors)u) (countpernode = %(processorsPerNode)u) %(xrslMPExtraString)s """ % { "processors": nProcessors, "processorsPerNode": nProcessors, # This basically says that we want all processors on the same node "xrslMPExtraString": self.xrslMPExtraString, } # Files that would need execution rights on the remote worker node xrslExecutables = "" if executables: if not isinstance(executables, list): executables = [executables] xrslExecutables = "(executables=%s)" % " ".join( map(os.path.basename, executables)) # Add them to the inputFiles if not inputs: inputs = [] if not isinstance(inputs, list): inputs = [inputs] inputs += executables # Dependencies that have to be embedded along with the executable xrslInputs = "" if inputs: if not isinstance(inputs, list): inputs = [inputs] for inputFile in inputs: xrslInputs += '(%s "%s")' % (os.path.basename(inputFile), inputFile) # Output files to retrieve once the execution is complete xrslOutputs = '("%s.out" "") ("%s.err" "")' % (diracStamp, diracStamp) if outputs: if not isinstance(outputs, list): outputs = [outputs] for outputFile in outputs: xrslOutputs += '(%s "")' % (outputFile) xrsl = """ &(executable="%(executable)s") (inputFiles=(%(executable)s "%(executableFile)s") %(xrslInputAdditions)s) (stdout="%(diracStamp)s.out") (stderr="%(diracStamp)s.err") (outputFiles=%(xrslOutputFiles)s) (queue=%(queue)s) %(xrslMPAdditions)s %(xrslExecutables)s %(xrslExtraString)s """ % { "executableFile": executableFile, "executable": os.path.basename(executableFile), "xrslInputAdditions": xrslInputs, "diracStamp": diracStamp, "queue": self.arcQueue, "xrslOutputFiles": xrslOutputs, "xrslMPAdditions": xrslMPAdditions, "xrslExecutables": xrslExecutables, "xrslExtraString": self.xrslExtraString, } return xrsl, diracStamp
def createInstance(self, instanceID=""): if not instanceID: instanceID = makeGuid()[:8] self.parameters["VMUUID"] = instanceID self.parameters["VMType"] = self.parameters.get("CEType", "EC2") actionArgs = ["--action", "create"] actionArgs += ["--resource", "compute"] # Image if "ImageID" in self.parameters and "ImageName" not in self.parameters: result = self.__getImageByName(self.parameters["ImageName"]) if not result["OK"]: return result imageId = result["Value"] elif "ImageID" in self.parameters: result = self.__occiCommand([ "--action", "describe", "--resource", "os_tpl#%s" % self.parameters["ImageID"] ]) if not result["OK"]: return S_ERROR( "Failed to get image for ID %s" % self.parameters["ImageID"], result["Message"]) imageId = self.parameters["ImageID"] else: return S_ERROR("No image specified") actionArgs += ["--mixin", "os_tpl#%s" % imageId] # Optional flavor name if "FlavorName" in self.parameters: result = self.__occiCommand([ "--action", "describe", "--resource", "resource_tpl#%s" % self.parameters["FlavorName"] ]) if not result["OK"]: return S_ERROR( "Failed to get flavor %s" % self.parameters["FlavorName"], result["Message"]) actionArgs += [ "--mixin", "resource_tpl#%s" % self.parameters["FlavorName"] ] # Instance name actionArgs += ["--attribute", "occi.core.title=DIRAC_%s" % instanceID] # Other params for param in []: if param in self.parameters: actionArgs += ["--%s" % param, "%s" % self.parameters[param]] self.log.info("Creating node:") self.log.verbose(" ".join(actionArgs)) # User data result = self._createUserDataScript() if not result["OK"]: return result # actionArgs += ['--context', 'user_data=%s' % str( result['Value'] )] f = tempfile.NamedTemporaryFile(delete=False) f.write(str(result["Value"])) f.close() self.log.debug("Write user_data to temp file:", f.name) actionArgs += ["--context", "user_data=file://%s" % f.name] # Create the VM instance now result = self.__occiCommand(actionArgs) os.unlink(f.name) if not result["OK"]: errmsg = "Error in rOCCI create instances: %s" % result["Message"] self.log.error(errmsg) return S_ERROR(errmsg) occiId = result["Value"].strip() # Properties of the instance nodeDict = {} nodeDict["InstanceID"] = instanceID result = self.__occiCommand( ["--action", "describe", "--resource", occiId]) if result["OK"]: nodeInfo = json.loads(result["Value"]) try: nodeDict["NumberOfProcessors"] = nodeInfo[0]["attributes"][ "occi"]["compute"]["cores"] nodeDict["RAM"] = nodeInfo[0]["attributes"]["occi"]["compute"][ "memory"] except Exception as e: nodeDict["NumberOfProcessors"] = 1 else: nodeDict["NumberOfProcessors"] = 1 return S_OK((occiId, nodeDict))
def __writeSub(self, executable, nJobs): """ Create the Sub File for submission """ self.log.debug("Working directory: %s " % self.workingDirectory) ##We randomize the location of the pilotoutput and log, because there are just too many of them pre1 = makeGuid()[:3] pre2 = makeGuid()[:3] mkDir(os.path.join(self.workingDirectory, pre1, pre2)) initialDirPrefix = "%s/%s" % (pre1, pre2) self.log.debug("InitialDir: %s" % os.path.join(self.workingDirectory, initialDirPrefix)) self.log.debug("ExtraSubmitString:\n### \n %s \n###" % self.extraSubmitString) fd, name = tempfile.mkstemp(suffix='.sub', prefix='HTCondorCE_', dir=self.workingDirectory) subFile = os.fdopen(fd, 'w') executable = os.path.join(self.workingDirectory, executable) localScheddOptions = """ ShouldTransferFiles = YES WhenToTransferOutput = ON_EXIT_OR_EVICT """ if self.useLocalSchedd else "" targetUniverse = "grid" if self.useLocalSchedd else "vanilla" sub = """ executable = %(executable)s universe = %(targetUniverse)s use_x509userproxy = true output = $(Cluster).$(Process).out error = $(Cluster).$(Process).err log = $(Cluster).$(Process).log environment = "HTCONDOR_JOBID=$(Cluster).$(Process)" initialdir = %(initialDir)s grid_resource = condor %(ceName)s %(ceName)s:9619 transfer_output_files = "" %(localScheddOptions)s kill_sig=SIGTERM %(extraString)s Queue %(nJobs)s """ % dict( executable=executable, nJobs=nJobs, ceName=self.ceName, extraString=self.extraSubmitString, initialDir=os.path.join(self.workingDirectory, initialDirPrefix), localScheddOptions=localScheddOptions, targetUniverse=targetUniverse, ) subFile.write(sub) subFile.close() return name
def test_files(self): ###################################################### # # First create a file to use for remaining tests # lfn = '/lhcb/test/unit-test/testfile.%s' % time.time() pfn = 'protocol://host:port/storage/path%s' % lfn size = 10000000 se = 'DIRAC-storage' guid = makeGuid() fileTuple = (lfn,pfn,size,se,guid) res = self.lfc.addFile(fileTuple) print res self.assert_(res['OK']) self.assert_(res['Value'].has_key('Successful')) self.assert_(res['Value'].has_key('Failed')) self.assert_(res['Value']['Successful'].has_key(lfn)) self.assert_(res['Value']['Successful'][lfn]) ###################################################### # # Test the creation of links using the test file # targetLfn = lfn linkName = '/lhcb/test/unit-test/testlink.%s' % time.time() linkTuple = (linkName,targetLfn) res = self.lfc.createLink(linkTuple) self.assert_(res['OK']) self.assert_(res['Value'].has_key('Successful')) self.assert_(res['Value'].has_key('Failed')) self.assert_(res['Value']['Successful'].has_key(linkName)) self.assert_(res['Value']['Successful'][linkName]) ###################################################### # # Test the recognition of links works (with file it should fail) # res = self.lfc.isLink(targetLfn) self.assert_(res['OK']) self.assert_(res['Value'].has_key('Successful')) self.assert_(res['Value'].has_key('Failed')) self.assert_(res['Value']['Successful'].has_key(targetLfn)) self.assertFalse(res['Value']['Successful'][targetLfn]) ###################################################### # # Test the recognition of links works (with link it shouldn't fail) # res = self.lfc.isLink(linkName) self.assert_(res['OK']) self.assert_(res['Value'].has_key('Successful')) self.assert_(res['Value'].has_key('Failed')) self.assert_(res['Value']['Successful'].has_key(linkName)) self.assert_(res['Value']['Successful'][linkName]) ###################################################### # # Test the resolution of links # res = self.lfc.readLink(linkName) self.assert_(res['OK']) self.assert_(res['Value'].has_key('Successful')) self.assert_(res['Value'].has_key('Failed')) self.assert_(res['Value']['Successful'].has_key(linkName)) self.assertEqual(res['Value']['Successful'][linkName],targetLfn) ###################################################### # # Test the removal of links # res = self.lfc.removeLink(linkName) self.assert_(res['OK']) self.assert_(res['Value'].has_key('Successful')) self.assert_(res['Value'].has_key('Failed')) self.assert_(res['Value']['Successful'].has_key(linkName)) self.assert_(res['Value']['Successful'][linkName]) ###################################################### # # Test the recognition of non existant links # res = self.lfc.isLink(linkName) self.assert_(res['OK']) self.assert_(res['Value'].has_key('Successful')) self.assert_(res['Value'].has_key('Failed')) self.assert_(res['Value']['Failed'].has_key(linkName)) self.assertEqual(res['Value']['Failed'][linkName],'No such file or directory') ###################################################### # # Add a replica to the test file # replicaPfn = 'protocol://replicaHost:port/storage/path%s' % lfn replicase = 'Replica-storage' replicaTuple = (lfn,replicaPfn,replicase,0) res = self.lfc.addReplica(replicaTuple) self.assert_(res['OK']) self.assert_(res['Value'].has_key('Successful')) self.assert_(res['Value'].has_key('Failed')) self.assert_(res['Value']['Successful'].has_key(lfn)) self.assert_(res['Value']['Successful'][lfn]) ###################################################### # # Ensure the file exists (quite redundant here) # res = self.lfc.exists(lfn) self.assert_(res['OK']) self.assert_(res['Value'].has_key('Successful')) self.assert_(res['Value'].has_key('Failed')) self.assert_(res['Value']['Successful'].has_key(lfn)) self.assert_(res['Value']['Successful'][lfn]) ###################################################### # # Test the recognition of files # res = self.lfc.isFile(lfn) self.assert_(res['OK']) self.assert_(res['Value'].has_key('Successful')) self.assert_(res['Value'].has_key('Failed')) self.assert_(res['Value']['Successful'].has_key(lfn)) self.assert_(res['Value']['Successful'][lfn]) ###################################################### # # Test obtaining the file metadata # res = self.lfc.getFileMetadata(lfn) self.assert_(res['OK']) self.assert_(res['Value'].has_key('Successful')) self.assert_(res['Value'].has_key('Failed')) self.assert_(res['Value']['Successful'].has_key(lfn)) metadataDict = res['Value']['Successful'][lfn] self.assertEqual(metadataDict['Status'],'-') self.assertEqual( metadataDict['ChecksumType'], '' ) self.assertEqual(metadataDict['Checksum'],'') self.assertEqual(metadataDict['Size'],10000000) ###################################################### # # Test obtaining the file replicas # res = self.lfc.getReplicas(lfn) self.assert_(res['OK']) self.assert_(res['Value'].has_key('Successful')) self.assert_(res['Value'].has_key('Failed')) self.assert_(res['Value']['Successful'].has_key(lfn)) self.assert_(res['Value']['Successful'][lfn].has_key('DIRAC-storage')) self.assertEqual(res['Value']['Successful'][lfn]['DIRAC-storage'],pfn) self.assert_(res['Value']['Successful'][lfn].has_key('Replica-storage')) self.assertEqual(res['Value']['Successful'][lfn]['Replica-storage'],replicaPfn) ###################################################### # # Test obtaining the replica status for the master replica # replicaTuple = (lfn,pfn,se) res = self.lfc.getReplicaStatus(replicaTuple) self.assert_(res['OK']) self.assert_(res['Value'].has_key('Successful')) self.assert_(res['Value'].has_key('Failed')) self.assert_(res['Value']['Successful'].has_key(lfn)) self.assertEqual(res['Value']['Successful'][lfn],'U') ###################################################### # # Test setting the replica status for the master replica # replicaTuple = (lfn,pfn,se,'C') res = self.lfc.setReplicaStatus(replicaTuple) self.assert_(res['OK']) self.assert_(res['Value'].has_key('Successful')) self.assert_(res['Value'].has_key('Failed')) self.assert_(res['Value']['Successful'].has_key(lfn)) self.assert_(res['Value']['Successful'][lfn]) ###################################################### # # Ensure the changing of the replica status worked # replicaTuple = (lfn,pfn,se) res = self.lfc.getReplicaStatus(replicaTuple) self.assert_(res['OK']) self.assert_(res['Value'].has_key('Successful')) self.assert_(res['Value'].has_key('Failed')) self.assert_(res['Value']['Successful'].has_key(lfn)) self.assertEqual(res['Value']['Successful'][lfn],'C') ###################################################### # # Test the change of storage element works # newse = 'New-storage' newToken = 'SpaceToken' replicaTuple = (lfn,pfn,newse,newToken) res = self.lfc.setReplicaHost(replicaTuple) self.assert_(res['OK']) self.assert_(res['Value'].has_key('Successful')) self.assert_(res['Value'].has_key('Failed')) self.assert_(res['Value']['Successful'].has_key(lfn)) self.assert_(res['Value']['Successful'][lfn]) ###################################################### # # Check the change of storage element works # res = self.lfc.getReplicas(lfn) self.assert_(res['OK']) self.assert_(res['Value'].has_key('Successful')) self.assert_(res['Value'].has_key('Failed')) self.assert_(res['Value']['Successful'].has_key(lfn)) self.assert_(res['Value']['Successful'][lfn].has_key(newse)) self.assertEqual(res['Value']['Successful'][lfn][newse],pfn) self.assert_(res['Value']['Successful'][lfn].has_key(replicase)) self.assertEqual(res['Value']['Successful'][lfn][replicase],replicaPfn) ###################################################### # # Test getting the file size # res = self.lfc.getFileSize(lfn) self.assert_(res['OK']) self.assert_(res['Value'].has_key('Successful')) self.assert_(res['Value'].has_key('Failed')) self.assert_(res['Value']['Successful'].has_key(lfn)) self.assertEqual(res['Value']['Successful'][lfn],size) ###################################################### # # Test the recognition of directories # dir = os.path.dirname(lfn) res = self.lfc.isDirectory(dir) self.assert_(res['OK']) self.assert_(res['Value'].has_key('Successful')) self.assert_(res['Value'].has_key('Failed')) self.assert_(res['Value']['Successful'].has_key(dir)) self.assert_(res['Value']['Successful'][dir]) ###################################################### # # Test getting replicas for directories # res = self.lfc.getDirectoryReplicas(dir) self.assert_(res['OK']) self.assert_(res['Value'].has_key('Successful')) self.assert_(res['Value'].has_key('Failed')) self.assert_(res['Value']['Successful'].has_key(dir)) self.assert_(res['Value']['Successful'][dir].has_key(lfn)) self.assert_(res['Value']['Successful'][dir][lfn].has_key(newse)) self.assertEqual(res['Value']['Successful'][dir][lfn][newse],pfn) self.assert_(res['Value']['Successful'][dir][lfn].has_key(replicase)) self.assertEqual(res['Value']['Successful'][dir][lfn][replicase],replicaPfn) ###################################################### # # Test listing directories # res = self.lfc.listDirectory(dir) self.assert_(res['OK']) self.assert_(res['Value'].has_key('Successful')) self.assert_(res['Value'].has_key('Failed')) self.assert_(res['Value']['Successful'].has_key(dir)) self.assertEqual(res['Value']['Successful'][dir],[lfn]) ###################################################### # # Test getting directory metadata # res = self.lfc.getDirectoryMetadata(dir) self.assert_(res['OK']) self.assert_(res['Value'].has_key('Successful')) self.assert_(res['Value'].has_key('Failed')) self.assert_(res['Value']['Successful'].has_key(dir)) self.assert_(res['Value']['Successful'][dir].has_key('NumberOfSubPaths')) self.assertEqual(res['Value']['Successful'][dir]['NumberOfSubPaths'],1) self.assert_(res['Value']['Successful'][dir].has_key('CreationDate')) ###################################################### # # Test getting directory size # res = self.lfc.getDirectorySize(dir) self.assert_(res['OK']) self.assert_(res['Value'].has_key('Successful')) self.assert_(res['Value'].has_key('Failed')) self.assert_(res['Value']['Successful'].has_key(dir)) self.assert_(res['Value']['Successful'][dir].has_key('Files')) self.assertEqual(res['Value']['Successful'][dir]['Files'],1) self.assert_(res['Value']['Successful'][dir].has_key('TotalSize')) self.assertEqual(res['Value']['Successful'][dir]['TotalSize'],size) self.assert_(res['Value']['Successful'][dir].has_key('SiteFiles')) self.assert_(res['Value']['Successful'][dir]['SiteFiles'].has_key(newse)) self.assertEqual(res['Value']['Successful'][dir]['SiteFiles'][newse],1) self.assert_(res['Value']['Successful'][dir]['SiteFiles'].has_key(replicase)) self.assertEqual(res['Value']['Successful'][dir]['SiteFiles'][replicase],1) self.assert_(res['Value']['Successful'][dir].has_key('SiteUsage')) self.assert_(res['Value']['Successful'][dir]['SiteUsage'].has_key(newse)) self.assertEqual(res['Value']['Successful'][dir]['SiteUsage'][newse],size) self.assert_(res['Value']['Successful'][dir]['SiteUsage'].has_key(replicase)) self.assertEqual(res['Value']['Successful'][dir]['SiteUsage'][replicase],size) ###################################################### # # Test creation of directories # newDir = '%s/%s' % (dir,'testDir') res = self.lfc.createDirectory(newDir) self.assert_(res['OK']) self.assert_(res['Value'].has_key('Successful')) self.assert_(res['Value'].has_key('Failed')) self.assert_(res['Value']['Successful'].has_key(newDir)) self.assert_(res['Value']['Successful'][newDir]) ###################################################### # # Test removal of directories # res = self.lfc.removeDirectory(newDir) self.assert_(res['OK']) self.assert_(res['Value'].has_key('Successful')) self.assert_(res['Value'].has_key('Failed')) self.assert_(res['Value']['Successful'].has_key(newDir)) self.assert_(res['Value']['Successful'][newDir]) ###################################################### # # Test removal of replicas # res = self.lfc.listDirectory(dir) self.assert_(res['OK']) self.assert_(res['Value'].has_key('Successful')) self.assert_(res['Value'].has_key('Failed')) self.assert_(res['Value']['Successful'].has_key(dir)) lfnsToDelete = res['Value']['Successful'][dir] res = self.lfc.getReplicas(lfnsToDelete) self.assert_(res['OK']) self.assert_(res['Value'].has_key('Successful')) self.assert_(res['Value'].has_key('Failed')) replicas = res['Value']['Successful'] replicaTupleList = [] for lfn in replicas.keys(): for se in replicas[lfn].keys(): replicaTuple = (lfn,replicas[lfn][se],se) replicaTupleList.append(replicaTuple) res = self.lfc.removeReplica(replicaTupleList) self.assert_(res['OK']) self.assert_(res['Value'].has_key('Successful')) self.assert_(res['Value'].has_key('Failed')) self.assertFalse(res['Value']['Failed']) ###################################################### # # Test removal of files # res = self.lfc.removeFile(lfnsToDelete) self.assert_(res['OK']) self.assert_(res['Value'].has_key('Successful')) self.assert_(res['Value'].has_key('Failed')) self.assertFalse(res['Value']['Failed'])
def testMakeGuid( self ): """ makeGuid tests """ # no filename - fake guid produced self.assertEqual( checkGuid( makeGuid() ), True , "fake guid for inexisting file" ) # using this python file self.assertEqual( checkGuid( makeGuid( os.path.abspath(__file__) ) ), True, "guid for FileTestCase.py file" )
def _submitJob(self, executableFile, numberOfJobs): """ Submit prepared executable """ # Copy the executable executable = os.path.basename(executableFile) executable = os.path.join(self.executableArea, executable) try: shutil.copy(executableFile, executable) except Exception, x: self.log.warn('Failed copying executable', x) return S_ERROR(x) jobStamps = [] for i in range(numberOfJobs): jobStamps.append(makeGuid()[:8]) jobStamp = '#'.join(jobStamps) subOptions = urllib.quote(self.submitOptions) cmdTuple = [ self.finalScript, 'submit_job', executable, self.batchOutput, self.batchError, self.workArea, str(numberOfJobs), self.infoArea, jobStamp, self.execQueue, subOptions ] self.log.verbose('CE submission command: %s' % ' '.join(cmdTuple)) result = systemCall(120, cmdTuple)
def submitJob(self, executableFile, proxy, numberOfJobs=1, processors=1): """ Method to submit job """ self.log.verbose("Executable file path: %s" % executableFile) if not os.access(executableFile, 5): os.chmod( executableFile, stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH) batchIDList = [] stampDict = {} if numberOfJobs == 1: jdlName, diracStamp = self.__writeJDL(executableFile, processors=processors) cmd = [ 'glite-ce-job-submit', '-n', '-a', '-N', '-r', '%s/%s' % (self.ceName, self.queue), '%s' % jdlName ] result = executeGridCommand(self.proxy, cmd, self.gridEnv) os.unlink(jdlName) if result['OK']: if result['Value'][0]: # We have got a non-zero status code errorString = '\n'.join(result['Value'][1:]).strip() return S_ERROR('Pilot submission failed with error: %s ' % errorString) pilotJobReference = result['Value'][1].strip() if not pilotJobReference: return S_ERROR( 'No pilot reference returned from the glite job submission command' ) if not pilotJobReference.startswith('https'): return S_ERROR('Invalid pilot reference %s' % pilotJobReference) batchIDList.append(pilotJobReference) stampDict[pilotJobReference] = diracStamp else: delegationID = makeGuid() cmd = [ 'glite-ce-delegate-proxy', '-e', '%s' % self.ceName, '%s' % delegationID ] result = executeGridCommand(self.proxy, cmd, self.gridEnv) if not result['OK']: self.log.error('Failed to delegate proxy', result['Message']) return result for _i in range(numberOfJobs): jdlName, diracStamp = self.__writeJDL(executableFile, processors=processors) cmd = [ 'glite-ce-job-submit', '-n', '-N', '-r', '%s/%s' % (self.ceName, self.queue), '-D', '%s' % delegationID, '%s' % jdlName ] result = executeGridCommand(self.proxy, cmd, self.gridEnv) os.unlink(jdlName) if not result['OK']: self.log.error( "General error in execution of glite-ce-job-submit command" ) break if result['Value'][0] != 0: self.log.error("Error in glite-ce-job-submit command", result['Value'][1] + result['Value'][2]) break pilotJobReference = result['Value'][1].strip() if pilotJobReference and pilotJobReference.startswith('https'): batchIDList.append(pilotJobReference) stampDict[pilotJobReference] = diracStamp else: break if batchIDList: result = S_OK(batchIDList) result['PilotStampDict'] = stampDict else: result = S_ERROR( 'No pilot references obtained from the glite job submission') return result
def submitJob(self, executableFile, proxy, numberOfJobs=1): """ Method to submit job """ self.log.verbose("Executable file path: %s" % executableFile) if not os.access(executableFile, 5): os.chmod(executableFile, 0755) batchIDList = [] stampDict = {} if numberOfJobs == 1: jdlName, diracStamp = self.__writeJDL(executableFile) cmd = [ 'glite-ce-job-submit', '-n', '-a', '-N', '-r', '%s/%s' % (self.ceName, self.queue), '%s' % jdlName ] result = executeGridCommand(self.proxy, cmd, self.gridEnv) if result['OK']: if result['Value'][0]: # We have got a non-zero status code return S_ERROR('Pilot submission failed with error: %s ' % result['Value'][2].strip()) pilotJobReference = result['Value'][1].strip() if not pilotJobReference: return S_ERROR( 'No pilot reference returned from the glite job submission command' ) batchIDList.append(pilotJobReference) stampDict[pilotJobReference] = diracStamp os.unlink(jdlName) else: delegationID = makeGuid() cmd = [ 'glite-ce-delegate-proxy', '-e', '%s' % self.ceName, '%s' % delegationID ] result = executeGridCommand(self.proxy, cmd, self.gridEnv) if not result['OK']: self.log.error('Failed to delegate proxy: %s' % result['Message']) return result for i in range(numberOfJobs): jdlName, diracStamp = self.__writeJDL(executableFile) cmd = [ 'glite-ce-job-submit', '-n', '-N', '-r', '%s/%s' % (self.ceName, self.queue), '-D', '%s' % delegationID, '%s' % jdlName ] result = executeGridCommand(self.proxy, cmd, self.gridEnv) os.unlink(jdlName) if not result['OK']: break if result['Value'][0] != 0: break pilotJobReference = result['Value'][1].strip() if pilotJobReference: batchIDList.append(pilotJobReference) stampDict[pilotJobReference] = diracStamp else: break os.unlink(executableFile) if batchIDList: result = S_OK(batchIDList) result['PilotStampDict'] = stampDict else: result = S_ERROR( 'No pilot references obtained from the glite job submission') return result
def processDir(initPath,recursive=False,host=None,fcInit=None,dfcInit=None): """ Process one directory, possibly recursively """ global dirCount, fileCount, globalStart, dnCache, roleCache, outputFile fc = fcInit if not fc: fc = LcgFileCatalogClient.LcgFileCatalogClient( host=host ) #fc = FileCatalogClient() dfc = dfcInit if not dfc: #dfc = LcgFileCatalogClient.LcgFileCatalogClient( host=host ) dfc = FileCatalogClient() start = time.time() initPath = initPath.rstrip("/") resultList = fc.listDirectory(initPath,True) #print resultList #return S_OK() lfc_time = (time.time() - start) s = time.time() print resultList if resultList['OK']: # Add directories if resultList['Value']['Failed']: return S_ERROR("Path %s failed: %s" % (initPath,resultList['Value']['Failed'][initPath])) dirDict = resultList['Value']['Successful'][initPath]['SubDirs'] paths = {} for path,info in dirDict.items(): print info paths[path] = {} paths[path]['Mode'] = info['Mode'] owner = getUserNameAndGroup( info ) #owner = getDNandRole( info ) if owner: paths[path]['Owner'] = owner #return S_OK() p_dirs = time.time() - s s = time.time() nDir = len(paths) if nDir: print "Adding %d directories in %s" % (nDir,initPath) result = dfc.createDirectory(paths) if not result['OK']: print "Error adding directories:",result['Message'] dirCount += nDir print "Total directories added", dirCount e_dirs = time.time() - s # Add files s = time.time() fileDict = resultList['Value']['Successful'][initPath]['Files'] lfns = {} for lfn,info in fileDict.items(): lfns[lfn] = {} lfns[lfn]['Size'] = info['MetaData']['Size'] lfns[lfn]['Checksum'] = info['MetaData']['Checksum'] if 'GUID' in info['MetaData']: lfns[lfn]['GUID'] = info['MetaData']['GUID'] else: lfns[lfn]['GUID'] = makeGuid() lfns[lfn]['Mode'] = info['MetaData']['Mode'] lfns[lfn]['PFN'] = '' owner = getUserNameAndGroup( info['MetaData'] ) if owner: lfns[lfn]['Owner'] = owner if info['Replicas']: seList = info['Replicas'].keys() lfns[lfn]['SE'] = seList p_files = time.time() - s s = time.time() nFile = len(lfns) nRep = 0 if nFile: for lfn in lfns: if 'SE' in lfns[lfn]: nRep += len(lfns[lfn]['SE']) print "Adding %d files in %s" % (nFile,initPath) #print lfns done = False count = 0 error = False while not done: count += 1 result = dfc.addFile(lfns) #print result if not result['OK']: print "Error adding files %d:" % count,result['Message'] if count > 10: print "Completely failed path", initPath break error = True time.sleep(2) elif error: print "Successfully added files on retry %d" % count done = True else: done = True fileCount += nFile print "Total files added", fileCount e_files = time.time() - s dfc_time = time.time() - start - lfc_time total_time = time.time() - globalStart format = "== %s: time lfc/dfc %.2f/%.2f, files %d/%d, dirs %d/%d, time: %.2f/%.2f/%.2f/%.2f %.2f \n" outputFile = open('lfc_dfc.out','a') outputFile.write( format % (initPath,lfc_time,dfc_time,nFile,fileCount,nDir,dirCount,p_dirs,e_dirs,p_files,e_files,total_time) ) outputFile.close() # print format % (initPath,lfc_time,dfc_time,nFile,fileCount,nDir,dirCount,p_dirs,e_dirs,p_files,e_files,total_time) # Go into directories if recursive: for path in paths: result = processDir(path,True,host=host,fcInit=fc,dfcInit=dfc) if result['OK']: nFile += result['Value'].get('NumberOfFiles',0) nDir += result['Value'].get('NumberOfDirectories',0) nRep += result['Value'].get('NumberOfReplicas',0) resultDict = {} resultDict['NumberOfFiles'] = nFile resultDict['NumberOfDirectories'] = nDir resultDict['NumberOfReplicas'] = nRep resultDict['Path'] = initPath return S_OK(resultDict)
def submitJob(self, executableFile, proxy=None, numberOfJobs=1): if not os.access(executableFile, 5): os.chmod( executableFile, stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH) # if no proxy is supplied, the executable can be submitted directly # otherwise a wrapper script is needed to get the proxy to the execution node # The wrapper script makes debugging more complicated and thus it is # recommended to transfer a proxy inside the executable if possible. if self.proxy and not proxy: proxy = self.proxy if proxy: self.log.verbose("Setting up proxy for payload") wrapperContent = bundleProxy(executableFile, proxy) name = writeScript(wrapperContent, os.getcwd()) submitFile = name else: # no proxy submitFile = executableFile jobStamps = [] for _i in range(numberOfJobs): jobStamps.append(makeGuid()[:8]) batchDict = { "Executable": submitFile, "NJobs": numberOfJobs, "OutputDir": self.batchOutput, "ErrorDir": self.batchError, "SubmitOptions": self.submitOptions, "ExecutionContext": self.execution, "JobStamps": jobStamps, "Queue": self.queue, "WholeNode": self.wholeNode, "NumberOfProcessors": self.numberOfProcessors, "NumberOfNodes": self.numberOfNodes, "NumberOfGPUs": self.numberOfGPUs, } resultSubmit = self.batchSystem.submitJob(**batchDict) if proxy: os.remove(submitFile) if resultSubmit["Status"] == 0: self.submittedJobs += len(resultSubmit["Jobs"]) # jobIDs = [ self.ceType.lower()+'://'+self.ceName+'/'+_id for _id in resultSubmit['Jobs'] ] # FIXME: It would be more proper to fix pilotCommands.__setFlavour where 'ssh' is hardcoded than # making this illogical fix, but there is no good way for pilotCommands to know its origin ceType. # So, the jobIDs here need to start with 'ssh', not ceType, to accomodate # them to those hardcoded in pilotCommands.__setFlavour batchSystemName = self.batchSystem.__class__.__name__.lower() jobIDs = [ "ssh" + batchSystemName + "://" + self.ceName + "/" + _id for _id in resultSubmit["Jobs"] ] result = S_OK(jobIDs) if "ExecutableToKeep" in resultSubmit: result["ExecutableToKeep"] = resultSubmit["ExecutableToKeep"] else: result = S_ERROR(resultSubmit["Message"]) return result
def _submitJobToHost(self, executableFile, numberOfJobs, host=None): """Submit prepared executable to the given host""" ssh = SSH(host=host, parameters=self.ceParameters) # Copy the executable submitFile = os.path.join(self.executableArea, os.path.basename(executableFile)) result = ssh.scpCall(30, executableFile, submitFile, postUploadCommand="chmod +x %s" % submitFile) if not result["OK"]: return result jobStamps = [] for _i in range(numberOfJobs): jobStamps.append(makeGuid()[:8]) numberOfProcessors = self.ceParameters.get("NumberOfProcessors", 1) wholeNode = self.ceParameters.get("WholeNode", False) # numberOfNodes is treated as a string as it can contain values such as "2-4" # where 2 would represent the minimum number of nodes to allocate, and 4 the maximum numberOfNodes = self.ceParameters.get("NumberOfNodes", "1") self.numberOfGPUs = self.ceParameters.get("NumberOfGPUs") # Collect command options commandOptions = { "Executable": submitFile, "NJobs": numberOfJobs, "SubmitOptions": self.submitOptions, "JobStamps": jobStamps, "WholeNode": wholeNode, "NumberOfProcessors": numberOfProcessors, "NumberOfNodes": numberOfNodes, "Preamble": self.preamble, "NumberOfGPUs": self.numberOfGPUs, "Account": self.account, } if host: commandOptions["SSHNodeHost"] = host resultCommand = self.__executeHostCommand("submitJob", commandOptions, ssh=ssh, host=host) if not resultCommand["OK"]: return resultCommand result = resultCommand["Value"] if result["Status"] != 0: return S_ERROR("Failed job submission: %s" % result["Message"]) else: batchIDs = result["Jobs"] if batchIDs: batchSystemName = self.batchSystem.__class__.__name__.lower() if host is None: jobIDs = [ "%s%s://%s/%s" % (self.ceType.lower(), batchSystemName, self.ceName, _id) for _id in batchIDs ] else: jobIDs = [ "%s%s://%s/%s/%s" % (self.ceType.lower(), batchSystemName, self.ceName, host, _id) for _id in batchIDs ] else: return S_ERROR("No jobs IDs returned") result = S_OK(jobIDs) self.submittedJobs += len(batchIDs) return result
def test_files(self): ###################################################### # # First create a file to use for remaining tests # lfn = '/lhcb/test/unit-test/testfile.%s' % time.time() pfn = 'protocol://host:port/storage/path%s' % lfn size = 10000000 se = 'DIRAC-storage' guid = makeGuid() fileTuple = (lfn, pfn, size, se, guid) res = self.lfc.addFile(fileTuple) self.assert_(res['OK']) self.assert_(res['Value'].has_key('Successful')) self.assert_(res['Value'].has_key('Failed')) self.assert_(res['Value']['Successful'].has_key(lfn)) self.assert_(res['Value']['Successful'][lfn]) ###################################################### # # Test the creation of links using the test file # targetLfn = lfn linkName = '/lhcb/test/unit-test/testlink.%s' % time.time() linkTuple = (linkName, targetLfn) res = self.lfc.createLink(linkTuple) self.assert_(res['OK']) self.assert_(res['Value'].has_key('Successful')) self.assert_(res['Value'].has_key('Failed')) self.assert_(res['Value']['Successful'].has_key(linkName)) self.assert_(res['Value']['Successful'][linkName]) ###################################################### # # Test the recognition of links works (with file it should fail) # res = self.lfc.isLink(targetLfn) print res self.assert_(res['OK']) self.assert_(res['Value'].has_key('Successful')) self.assert_(res['Value'].has_key('Failed')) self.assert_(res['Value']['Successful'].has_key(targetLfn)) self.assertFalse(res['Value']['Successful'][targetLfn]) ###################################################### # # Test the recognition of links works (with link it shouldn't fail) # res = self.lfc.isLink(linkName) self.assert_(res['OK']) self.assert_(res['Value'].has_key('Successful')) self.assert_(res['Value'].has_key('Failed')) self.assert_(res['Value']['Successful'].has_key(linkName)) self.assert_(res['Value']['Successful'][linkName]) ###################################################### # # Test the resolution of links # res = self.lfc.readLink(linkName) self.assert_(res['OK']) self.assert_(res['Value'].has_key('Successful')) self.assert_(res['Value'].has_key('Failed')) self.assert_(res['Value']['Successful'].has_key(linkName)) self.assertEqual(res['Value']['Successful'][linkName], targetLfn) ###################################################### # # Test the removal of links # res = self.lfc.removeLink(linkName) self.assert_(res['OK']) self.assert_(res['Value'].has_key('Successful')) self.assert_(res['Value'].has_key('Failed')) self.assert_(res['Value']['Successful'].has_key(linkName)) self.assert_(res['Value']['Successful'][linkName]) ###################################################### # # Test the recognition of non existant links # res = self.lfc.isLink(linkName) self.assert_(res['OK']) self.assert_(res['Value'].has_key('Successful')) self.assert_(res['Value'].has_key('Failed')) self.assert_(res['Value']['Failed'].has_key(linkName)) self.assertEqual(res['Value']['Failed'][linkName], 'No such file or directory') ###################################################### # # Add a replica to the test file # replicaPfn = 'protocol://replicaHost:port/storage/path%s' % lfn replicase = 'Replica-storage' replicaTuple = (lfn, replicaPfn, replicase, 0) res = self.lfc.addReplica(replicaTuple) self.assert_(res['OK']) self.assert_(res['Value'].has_key('Successful')) self.assert_(res['Value'].has_key('Failed')) self.assert_(res['Value']['Successful'].has_key(lfn)) self.assert_(res['Value']['Successful'][lfn]) ###################################################### # # Ensure the file exists (quite redundant here) # res = self.lfc.exists(lfn) self.assert_(res['OK']) self.assert_(res['Value'].has_key('Successful')) self.assert_(res['Value'].has_key('Failed')) self.assert_(res['Value']['Successful'].has_key(lfn)) self.assert_(res['Value']['Successful'][lfn]) ###################################################### # # Test the recognition of files # res = self.lfc.isFile(lfn) self.assert_(res['OK']) self.assert_(res['Value'].has_key('Successful')) self.assert_(res['Value'].has_key('Failed')) self.assert_(res['Value']['Successful'].has_key(lfn)) self.assert_(res['Value']['Successful'][lfn]) ###################################################### # # Test obtaining the file metadata # res = self.lfc.getFileMetadata(lfn) self.assert_(res['OK']) self.assert_(res['Value'].has_key('Successful')) self.assert_(res['Value'].has_key('Failed')) self.assert_(res['Value']['Successful'].has_key(lfn)) metadataDict = res['Value']['Successful'][lfn] self.assertEqual(metadataDict['Status'], '-') self.assertEqual(metadataDict['CheckSumType'], '') self.assertEqual(metadataDict['Checksum'], '') self.assertEqual(metadataDict['Size'], 10000000) ###################################################### # # Test obtaining the file replicas # res = self.lfc.getReplicas(lfn) self.assert_(res['OK']) self.assert_(res['Value'].has_key('Successful')) self.assert_(res['Value'].has_key('Failed')) self.assert_(res['Value']['Successful'].has_key(lfn)) self.assert_(res['Value']['Successful'][lfn].has_key('DIRAC-storage')) self.assertEqual(res['Value']['Successful'][lfn]['DIRAC-storage'], pfn) self.assert_( res['Value']['Successful'][lfn].has_key('Replica-storage')) self.assertEqual(res['Value']['Successful'][lfn]['Replica-storage'], replicaPfn) ###################################################### # # Test obtaining the replica status for the master replica # replicaTuple = (lfn, pfn, se) res = self.lfc.getReplicaStatus(replicaTuple) self.assert_(res['OK']) self.assert_(res['Value'].has_key('Successful')) self.assert_(res['Value'].has_key('Failed')) self.assert_(res['Value']['Successful'].has_key(lfn)) self.assertEqual(res['Value']['Successful'][lfn], 'U') ###################################################### # # Test setting the replica status for the master replica # replicaTuple = (lfn, pfn, se, 'C') res = self.lfc.setReplicaStatus(replicaTuple) self.assert_(res['OK']) self.assert_(res['Value'].has_key('Successful')) self.assert_(res['Value'].has_key('Failed')) self.assert_(res['Value']['Successful'].has_key(lfn)) self.assert_(res['Value']['Successful'][lfn]) ###################################################### # # Ensure the changing of the replica status worked # replicaTuple = (lfn, pfn, se) res = self.lfc.getReplicaStatus(replicaTuple) self.assert_(res['OK']) self.assert_(res['Value'].has_key('Successful')) self.assert_(res['Value'].has_key('Failed')) self.assert_(res['Value']['Successful'].has_key(lfn)) self.assertEqual(res['Value']['Successful'][lfn], 'C') ###################################################### # # Test the change of storage element works # newse = 'New-storage' newToken = 'SpaceToken' replicaTuple = (lfn, pfn, newse, newToken) res = self.lfc.setReplicaHost(replicaTuple) self.assert_(res['OK']) self.assert_(res['Value'].has_key('Successful')) self.assert_(res['Value'].has_key('Failed')) self.assert_(res['Value']['Successful'].has_key(lfn)) self.assert_(res['Value']['Successful'][lfn]) ###################################################### # # Check the change of storage element works # res = self.lfc.getReplicas(lfn) self.assert_(res['OK']) self.assert_(res['Value'].has_key('Successful')) self.assert_(res['Value'].has_key('Failed')) self.assert_(res['Value']['Successful'].has_key(lfn)) self.assert_(res['Value']['Successful'][lfn].has_key(newse)) self.assertEqual(res['Value']['Successful'][lfn][newse], pfn) self.assert_(res['Value']['Successful'][lfn].has_key(replicase)) self.assertEqual(res['Value']['Successful'][lfn][replicase], replicaPfn) ###################################################### # # Test getting the file size # res = self.lfc.getFileSize(lfn) self.assert_(res['OK']) self.assert_(res['Value'].has_key('Successful')) self.assert_(res['Value'].has_key('Failed')) self.assert_(res['Value']['Successful'].has_key(lfn)) self.assertEqual(res['Value']['Successful'][lfn], size) ###################################################### # # Test the recognition of directories # dir = os.path.dirname(lfn) res = self.lfc.isDirectory(dir) self.assert_(res['OK']) self.assert_(res['Value'].has_key('Successful')) self.assert_(res['Value'].has_key('Failed')) self.assert_(res['Value']['Successful'].has_key(dir)) self.assert_(res['Value']['Successful'][dir]) ###################################################### # # Test getting replicas for directories # res = self.lfc.getDirectoryReplicas(dir) self.assert_(res['OK']) self.assert_(res['Value'].has_key('Successful')) self.assert_(res['Value'].has_key('Failed')) self.assert_(res['Value']['Successful'].has_key(dir)) self.assert_(res['Value']['Successful'][dir].has_key(lfn)) self.assert_(res['Value']['Successful'][dir][lfn].has_key(newse)) self.assertEqual(res['Value']['Successful'][dir][lfn][newse], pfn) self.assert_(res['Value']['Successful'][dir][lfn].has_key(replicase)) self.assertEqual(res['Value']['Successful'][dir][lfn][replicase], replicaPfn) ###################################################### # # Test listing directories # res = self.lfc.listDirectory(dir) self.assert_(res['OK']) self.assert_(res['Value'].has_key('Successful')) self.assert_(res['Value'].has_key('Failed')) self.assert_(res['Value']['Successful'].has_key(dir)) self.assertEqual(res['Value']['Successful'][dir], [lfn]) ###################################################### # # Test getting directory metadata # res = self.lfc.getDirectoryMetadata(dir) self.assert_(res['OK']) self.assert_(res['Value'].has_key('Successful')) self.assert_(res['Value'].has_key('Failed')) self.assert_(res['Value']['Successful'].has_key(dir)) self.assert_( res['Value']['Successful'][dir].has_key('NumberOfSubPaths')) self.assertEqual(res['Value']['Successful'][dir]['NumberOfSubPaths'], 1) self.assert_(res['Value']['Successful'][dir].has_key('CreationDate')) ###################################################### # # Test getting directory size # res = self.lfc.getDirectorySize(dir) self.assert_(res['OK']) self.assert_(res['Value'].has_key('Successful')) self.assert_(res['Value'].has_key('Failed')) self.assert_(res['Value']['Successful'].has_key(dir)) self.assert_(res['Value']['Successful'][dir].has_key('Files')) self.assertEqual(res['Value']['Successful'][dir]['Files'], 1) self.assert_(res['Value']['Successful'][dir].has_key('TotalSize')) self.assertEqual(res['Value']['Successful'][dir]['TotalSize'], size) self.assert_(res['Value']['Successful'][dir].has_key('SiteFiles')) self.assert_( res['Value']['Successful'][dir]['SiteFiles'].has_key(newse)) self.assertEqual(res['Value']['Successful'][dir]['SiteFiles'][newse], 1) self.assert_( res['Value']['Successful'][dir]['SiteFiles'].has_key(replicase)) self.assertEqual( res['Value']['Successful'][dir]['SiteFiles'][replicase], 1) self.assert_(res['Value']['Successful'][dir].has_key('SiteUsage')) self.assert_( res['Value']['Successful'][dir]['SiteUsage'].has_key(newse)) self.assertEqual(res['Value']['Successful'][dir]['SiteUsage'][newse], size) self.assert_( res['Value']['Successful'][dir]['SiteUsage'].has_key(replicase)) self.assertEqual( res['Value']['Successful'][dir]['SiteUsage'][replicase], size) ###################################################### # # Test creation of directories # newDir = '%s/%s' % (dir, 'testDir') res = self.lfc.createDirectory(newDir) self.assert_(res['OK']) self.assert_(res['Value'].has_key('Successful')) self.assert_(res['Value'].has_key('Failed')) self.assert_(res['Value']['Successful'].has_key(newDir)) self.assert_(res['Value']['Successful'][newDir]) ###################################################### # # Test removal of directories # res = self.lfc.removeDirectory(newDir) self.assert_(res['OK']) self.assert_(res['Value'].has_key('Successful')) self.assert_(res['Value'].has_key('Failed')) self.assert_(res['Value']['Successful'].has_key(newDir)) self.assert_(res['Value']['Successful'][newDir]) ###################################################### # # Test removal of replicas # res = self.lfc.listDirectory(dir) self.assert_(res['OK']) self.assert_(res['Value'].has_key('Successful')) self.assert_(res['Value'].has_key('Failed')) self.assert_(res['Value']['Successful'].has_key(dir)) lfnsToDelete = res['Value']['Successful'][dir] res = self.lfc.getReplicas(lfnsToDelete) self.assert_(res['OK']) self.assert_(res['Value'].has_key('Successful')) self.assert_(res['Value'].has_key('Failed')) replicas = res['Value']['Successful'] replicaTupleList = [] for lfn in replicas.keys(): for se in replicas[lfn].keys(): replicaTuple = (lfn, replicas[lfn][se], se) replicaTupleList.append(replicaTuple) res = self.lfc.removeReplica(replicaTupleList) self.assert_(res['OK']) self.assert_(res['Value'].has_key('Successful')) self.assert_(res['Value'].has_key('Failed')) self.assertFalse(res['Value']['Failed']) ###################################################### # # Test removal of files # res = self.lfc.removeFile(lfnsToDelete) self.assert_(res['OK']) self.assert_(res['Value'].has_key('Successful')) self.assert_(res['Value'].has_key('Failed')) self.assertFalse(res['Value']['Failed'])