def addSubRequest(self,requestDict,type):
""" Add a new sub-requests of specified type
"""
# Initialise the sub-request
index = self.initiateSubRequest(type)['Value']
# Stuff the sub-request with the attributes
attributeDict = {'Status':'Waiting','SubRequestID':makeGuid(),
'CreationTime': str(datetime.datetime.utcnow()),
'ExecutionOrder':0}
for attr,value in requestDict['Attributes'].items():
attributeDict[attr] = value
self.setSubRequestAttributes(index,type,attributeDict)
if requestDict.has_key('Files'):
files = []
for file in requestDict['Files']:
fileDict = {'Status':'Waiting','FileID':makeGuid(),'Attempt':1}
for attr,value in file.items():
fileDict[attr] = value
files.append(fileDict)
self.setSubRequestFiles(index,type,files)
if requestDict.has_key('Datasets'):
datasets = []
for dataset in requestDict['Datasets']:
datasetDict = {'Status':'Waiting'}
for attr,value in file.items():
fileDict[attr] = value
datasets.append(datasetDict)
self.setSubRequestDatasets(index,type,datasets)
return S_OK(index)
def __writeSub(self, executable, nJobs):
""" Create the Sub File for submission
"""
self.log.debug("Working directory: %s " % self.workingDirectory)
# We randomize the location of the pilotoutput and log, because there are just too many of them
pre1 = makeGuid()[:3]
pre2 = makeGuid()[:3]
mkDir(os.path.join(self.workingDirectory, pre1, pre2))
initialDirPrefix = "%s/%s" % (pre1, pre2)
self.log.debug("InitialDir: %s" % os.path.join(self.workingDirectory, initialDirPrefix))
self.log.debug("ExtraSubmitString:\n### \n %s \n###" % self.extraSubmitString)
fd, name = tempfile.mkstemp(suffix='.sub', prefix='HTCondorCE_', dir=self.workingDirectory)
subFile = os.fdopen(fd, 'w')
executable = os.path.join(self.workingDirectory, executable)
localScheddOptions = """
ShouldTransferFiles = YES
WhenToTransferOutput = ON_EXIT_OR_EVICT
""" if self.useLocalSchedd else ""
targetUniverse = "grid" if self.useLocalSchedd else "vanilla"
sub = """
executable = %(executable)s
universe = %(targetUniverse)s
use_x509userproxy = true
output = $(Cluster).$(Process).out
error = $(Cluster).$(Process).err
log = $(Cluster).$(Process).log
environment = "HTCONDOR_JOBID=$(Cluster).$(Process)"
initialdir = %(initialDir)s
grid_resource = condor %(ceName)s %(ceName)s:9619
transfer_output_files = ""
%(localScheddOptions)s
kill_sig=SIGTERM
%(extraString)s
Queue %(nJobs)s
""" % dict(executable=executable,
nJobs=nJobs,
ceName=self.ceName,
extraString=self.extraSubmitString,
initialDir=os.path.join(self.workingDirectory, initialDirPrefix),
localScheddOptions=localScheddOptions,
targetUniverse=targetUniverse,
)
subFile.write(sub)
subFile.close()
return name
def testMakeGuid(self):
""" makeGuid tests """
# no filename - fake guid produced
self.assertEqual(checkGuid(makeGuid()), True, "fake guid for inexisting file")
# using this python file
self.assertEqual(
checkGuid(
makeGuid(
os.path.abspath(__file__))),
True,
"guid for FileTestCase.py file")
def addSubRequest(self,type,requestDict):
""" Add a new sub-requests of specified type. Overrides the corresponding
method of the base class
"""
# Initialise the sub-request
ind = self.initiateSubRequest(type)['Value']
# Stuff the sub-request with the attributes
attributeDict = {}
for key in self.requestAttributes:
if requestDict['Attributes'].has_key(key):
attributeDict[key] = requestDict['Attributes'][key]
else:
attributeDict[key] = ''
if not attributeDict['RequestType']:
attributeDict['RequestType'] = type
if not attributeDict['Status']:
attributeDict['Status'] = 'Waiting'
if not attributeDict['SubRequestID']:
attributeDict['SubRequestID'] = makeGuid()
self.setSubRequestAttributes(ind,type,attributeDict)
# Stuff the sub-request with the files
fileDict = {}
files = requestDict['Files']
ifile = 1
for file in files.values():
for key in self.fileAttributes:
if not file.has_key(key):
file[key] = ''
if not file['Status']:
file['Status'] = 'Waiting'
if not file['GUID']:
file['GUID'] = makeGuid()
if not file['Attempt']:
file['Attempt'] = 1
fileDict['File%d' % ifile] = file
ifile += 1
res = self.setSubRequestFiles(ind,type,files.values())
# Stuff the sub-request with the datasets
if requestDict.has_key('Datasets'):
datasets = requestDict['Datasets']
else:
datasets = {}
self.setSubRequestDatasets(ind,type,datasets.values())
def submitJob( self, executableFile, proxy, numberOfJobs = 1, processors = 1 ):
""" Method to submit job
"""
self.log.verbose( "Executable file path: %s" % executableFile )
if not os.access( executableFile, 5 ):
os.chmod( executableFile, stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH )
batchIDList = []
stampDict = {}
if numberOfJobs == 1:
jdlName, diracStamp = self.__writeJDL( executableFile, processors = processors )
cmd = ['glite-ce-job-submit', '-n', '-a', '-N', '-r',
'%s/%s' % ( self.ceName, self.queue ),
'%s' % jdlName ]
result = executeGridCommand( self.proxy, cmd, self.gridEnv )
os.unlink( jdlName )
if result['OK']:
if result['Value'][0]:
# We have got a non-zero status code
errorString = '\n'.join( result['Value'][1:] ).strip()
return S_ERROR( 'Pilot submission failed with error: %s ' % errorString )
pilotJobReference = result['Value'][1].strip()
if not pilotJobReference:
return S_ERROR( 'No pilot reference returned from the glite job submission command' )
if not pilotJobReference.startswith( 'https' ):
return S_ERROR( 'Invalid pilot reference %s' % pilotJobReference )
batchIDList.append( pilotJobReference )
stampDict[pilotJobReference] = diracStamp
else:
delegationID = makeGuid()
cmd = [ 'glite-ce-delegate-proxy', '-e', '%s' % self.ceName, '%s' % delegationID ]
result = executeGridCommand( self.proxy, cmd, self.gridEnv )
if not result['OK']:
self.log.error( 'Failed to delegate proxy', result['Message'] )
return result
for _i in range( numberOfJobs ):
jdlName, diracStamp = self.__writeJDL( executableFile, processors = processors )
cmd = ['glite-ce-job-submit', '-n', '-N', '-r',
'%s/%s' % ( self.ceName, self.queue ),
'-D', '%s' % delegationID, '%s' % jdlName ]
result = executeGridCommand( self.proxy, cmd, self.gridEnv )
os.unlink( jdlName )
if not result['OK']:
break
if result['Value'][0] != 0:
break
pilotJobReference = result['Value'][1].strip()
if pilotJobReference and pilotJobReference.startswith( 'https' ):
batchIDList.append( pilotJobReference )
stampDict[pilotJobReference] = diracStamp
else:
break
if batchIDList:
result = S_OK( batchIDList )
result['PilotStampDict'] = stampDict
else:
result = S_ERROR( 'No pilot references obtained from the glite job submission' )
return result
def __writeXRSL(self, executableFile, processors=1):
""" Create the JDL for submission
"""
diracStamp = makeGuid()[:8]
xrslMPAdditions = ''
if processors > 1:
xrslMPAdditions = """
(count = %(processors)u)
%(xrslMPExtraString)s
""" % {
'processors': processors,
'xrslMPExtraString': self.xrslMPExtraString
}
xrsl = """
&(executable="%(executable)s")
(inputFiles=(%(executable)s "%(executableFile)s"))
(stdout="%(diracStamp)s.out")
(stderr="%(diracStamp)s.err")
(outputFiles=("%(diracStamp)s.out" "") ("%(diracStamp)s.err" ""))
(queue=%(queue)s)
%(xrslMPAdditions)s
%(xrslExtraString)s
""" % {
'executableFile': executableFile,
'executable': os.path.basename(executableFile),
'diracStamp': diracStamp,
'queue': self.arcQueue,
'xrslMPAdditions': xrslMPAdditions,
'xrslExtraString': self.xrslExtraString
}
return xrsl, diracStamp
def test_registerFile(self):
lfn = '/lhcb/test/unit-test/ReplicaManager/registerFile/testFile.%s' % time.time()
physicalFile = 'srm://host:port/srm/managerv2?SFN=/sa/path%s' % lfn
fileSize = 10000
storageElementName = 'CERN-RAW'
fileGuid = makeGuid()
fileTuple = (lfn,physicalFile,fileSize,storageElementName,fileGuid)
registerRes = self.dataManager.registerFile(fileTuple)
removeCatalogReplicaRes = self.dataManager.removeCatalogReplica(storageElementName,lfn)
removeFileRes = self.dataManager.removeFile(lfn)
# Check that the file registration was done correctly
self.assert_(registerRes['OK'])
self.assert_(registerRes['Value'].has_key('Successful'))
self.assert_(registerRes['Value']['Successful'].has_key(lfn))
self.assert_(registerRes['Value']['Successful'][lfn])
# Check that the replica removal was successful
self.assert_(removeCatalogReplicaRes['OK'])
self.assert_(removeCatalogReplicaRes['Value'].has_key('Successful'))
self.assert_(removeCatalogReplicaRes['Value']['Successful'].has_key(lfn))
self.assert_(removeCatalogReplicaRes['Value']['Successful'][lfn])
# Check that the removal was successful
self.assert_(removeFileRes['OK'])
self.assert_(removeFileRes['Value'].has_key('Successful'))
self.assert_(removeFileRes['Value']['Successful'].has_key(lfn))
self.assert_(removeFileRes['Value']['Successful'][lfn])
def __writeSub( self, executable, nJobs ):
""" Create the Sub File for submission
"""
workingDirectory = self.ceParameters['WorkingDirectory']
initialDir = os.path.dirname( workingDirectory )
self.log.debug( "Working directory: %s " % workingDirectory )
##We randomize the location of the pilotoutput and log, because there are just too many of them
pre1 = makeGuid()[:3]
pre2 = makeGuid()[:3]
try:
os.makedirs( os.path.join( initialDir, pre1, pre2 ) )
except OSError:
pass
initialDirPrefix = "%s/%s" %( pre1, pre2 )
self.log.debug( "InitialDir: %s" % os.path.join(initialDir,initialDirPrefix) )
fd, name = tempfile.mkstemp( suffix = '.sub', prefix = 'HTCondorCE_', dir = workingDirectory )
subFile = os.fdopen( fd, 'w' )
executable = os.path.join( workingDirectory, executable )
sub = """
executable = %(executable)s
universe = grid
use_x509userproxy = true
output = $(Cluster).$(Process).out
error = $(Cluster).$(Process).err
log = $(Cluster).$(Process).log
environment = "HTCONDOR_JOBID=$(Cluster).$(Process)"
initialdir = %(initialDir)s
grid_resource = condor %(ceName)s %(ceName)s:9619
ShouldTransferFiles = YES
WhenToTransferOutput = ON_EXIT_OR_EVICT
kill_sig=SIGTERM
Queue %(nJobs)s
""" % dict( executable=executable,
nJobs=nJobs,
ceName=self.ceName,
initialDir=os.path.join(initialDir,initialDirPrefix),
)
subFile.write( sub )
subFile.close()
return name
def getFileMetadata(self, candidateFiles):
"""Returns the candidate file dictionary with associated metadata.
:param dict candidateFiles: The input candidate files dictionary has the structure: {'lfn':'','path':'','workflowSE':''}
This also assumes the files are in the current working directory.
:return: S_OK with File Metadata, S_ERROR
"""
#Retrieve the POOL File GUID(s) for any final output files
LOG.info('Will search for POOL GUIDs for: %s' % (', '.join(candidateFiles.keys())))
pfnGUIDs = {}
generated = []
for fname in candidateFiles.keys():
guid = makeGuid(fname)
pfnGUIDs[fname] = guid
generated.append(fname)
pfnGUID = S_OK(pfnGUIDs)
pfnGUID['generated'] = generated
#pfnGUID = getGUID(candidateFiles.keys())
#if not pfnGUID['OK']:
# LOG.error('PoolXMLFile failed to determine POOL GUID(s) for output file list, these will be generated by \
# the DataManager',pfnGUID['Message'])
# for fileName in candidateFiles.keys():
# candidateFiles[fileName]['guid']=''
#if pfnGUID['generated']:
LOG.debug('Generated GUID(s) for the following files ', ', '.join(pfnGUID['generated']))
#else:
# LOG.info('GUIDs found for all specified POOL files: %s' %(string.join(candidateFiles.keys(),', ')))
for pfn, guid in pfnGUID['Value'].items():
candidateFiles[pfn]['GUID'] = guid
#Get all additional metadata about the file necessary for requests
final = {}
for fileName, metadata in candidateFiles.items():
fileDict = {}
fileDict['LFN'] = metadata['lfn']
fileDict['Size'] = os.path.getsize(fileName)
adlerChecksum = fileAdler(fileName)
fileDict['Addler'] = adlerChecksum
fileDict['ADLER32'] = adlerChecksum
fileDict['Checksum'] = adlerChecksum
fileDict['ChecksumType'] = "ADLER32"
fileDict['GUID'] = metadata['GUID']
fileDict['Status'] = "Waiting"
final[fileName] = metadata
final[fileName]['filedict'] = fileDict
final[fileName]['localpath'] = '%s/%s' % (os.getcwd(), fileName)
#Sanity check all final candidate metadata keys are present (return S_ERROR if not)
mandatoryKeys = ['GUID', 'filedict'] #filedict is used for requests (this method adds guid and filedict)
for fileName, metadata in final.items():
for key in mandatoryKeys:
if key not in metadata:
return S_ERROR('File %s has missing %s' % (fileName, key))
return S_OK(final)
def addSubRequest(self, requestDict, rType):
""" Add a new sub-requests of specified type
The 'Attributes':ExecutionOrder can be set to 'last' to always be the last to be executed.
"""
# Initialise the sub-request
index = self.initiateSubRequest(rType)["Value"]
# Stuff the sub-request with the attributes
attributeDict = {
"Status": "Waiting",
"SubRequestID": makeGuid(),
"CreationTime": str(datetime.datetime.utcnow()),
"ExecutionOrder": 0,
}
for attr, value in requestDict["Attributes"].items():
if attr == "ExecutionOrder" and str(value).lower() == "last":
value = self._getLastOrder() + 1
attributeDict[attr] = value
for attr in self.subAttributeNames:
if not attr in attributeDict.keys():
attributeDict[attr] = ""
if attr == "ExecutionOrder":
attributeDict[attr] = 0
if attr == "Status":
attributeDict[attr] = "Waiting"
self.setSubRequestAttributes(index, rType, attributeDict)
if requestDict.has_key("Files"):
files = []
for rFile in requestDict["Files"]:
fileDict = {"Status": "Waiting", "FileID": makeGuid(), "Attempt": 1}
for attr, value in rFile.items():
fileDict[attr] = value
files.append(fileDict)
self.setSubRequestFiles(index, rType, files)
if requestDict.has_key("Datasets"):
datasets = []
for dataset in requestDict["Datasets"]:
datasetDict = {"Status": "Waiting"}
for attr, value in rFile.items():
fileDict[attr] = value
datasets.append(datasetDict)
self.setSubRequestDatasets(index, rType, datasets)
return S_OK(index)
def registerFile(self, lfn):
pfn = "protocol://host:port/storage/path%s" % lfn
size = 10000000
se = "DIRAC-storage"
guid = makeGuid()
adler = stringAdler(guid)
fileDict = {}
fileDict[lfn] = {"PFN": pfn, "Size": size, "SE": se, "GUID": guid, "Checksum": adler}
res = self.catalog.addFile(fileDict)
return self.parseResult(res, lfn)
def registerFile(self,lfn):
pfn = 'protocol://host:port/storage/path%s' % lfn
size = 10000000
se = 'DIRAC-storage'
guid = makeGuid()
adler = stringAdler(guid)
fileDict = {}
fileDict[lfn] = {'PFN':pfn,'Size':size,'SE':se,'GUID':guid,'Checksum':adler}
res = self.catalog.addFile(fileDict)
return self.parseResult(res,lfn)
def getFileMetadata(self, candidateFiles):
"""Returns the candidate file dictionary with associated metadata.
@param candidateFiles: The input candidate files dictionary has the structure:
{'lfn':'','path':'','workflowSE':''}
This also assumes the files are in the current working directory.
@return: File Metadata
"""
# Retrieve the POOL File GUID(s) for any final output files
self.log.info("Will search for POOL GUIDs for: %s" % (string.join(candidateFiles.keys(), ", ")))
pfnGUIDs = {}
generated = []
for fname in candidateFiles.keys():
guid = makeGuid(fname)
pfnGUIDs[fname] = guid
generated.append(fname)
pfnGUID = S_OK(pfnGUIDs)
pfnGUID["generated"] = generated
# pfnGUID = getGUID(candidateFiles.keys())
# if not pfnGUID['OK']:
# self.log.error('PoolXMLFile failed to determine POOL GUID(s) for output file list, these will be generated by \
# the ReplicaManager',pfnGUID['Message'])
# for fileName in candidateFiles.keys():
# candidateFiles[fileName]['guid']=''
# if pfnGUID['generated']:
self.log.debug("Generated GUID(s) for the following files ", string.join(pfnGUID["generated"], ", "))
# else:
# self.log.info('GUIDs found for all specified POOL files: %s' %(string.join(candidateFiles.keys(),', ')))
for pfn, guid in pfnGUID["Value"].items():
candidateFiles[pfn]["guid"] = guid
# Get all additional metadata about the file necessary for requests
final = {}
for fileName, metadata in candidateFiles.items():
fileDict = {}
fileDict["LFN"] = metadata["lfn"]
fileDict["Size"] = os.path.getsize(fileName)
fileDict["Addler"] = fileAdler(fileName)
fileDict["GUID"] = metadata["guid"]
fileDict["Status"] = "Waiting"
final[fileName] = metadata
final[fileName]["filedict"] = fileDict
final[fileName]["localpath"] = "%s/%s" % (os.getcwd(), fileName)
# Sanity check all final candidate metadata keys are present (return S_ERROR if not)
mandatoryKeys = ["guid", "filedict"] # filedict is used for requests (this method adds guid and filedict)
for fileName, metadata in final.items():
for key in mandatoryKeys:
if not metadata.has_key(key):
return S_ERROR("File %s has missing %s" % (fileName, key))
return S_OK(final)
def submitJob( self, executableFile, proxy, numberOfJobs = 1 ):
""" Method to submit job
"""
self.log.verbose( "Executable file path: %s" % executableFile )
if not os.access( executableFile, 5 ):
os.chmod( executableFile, 0755 )
batchIDList = []
stampDict = {}
if numberOfJobs == 1:
jdlName, diracStamp = self.__writeJDL( executableFile )
cmd = ['glite-ce-job-submit', '-n', '-a', '-N', '-r',
'%s/%s' % ( self.ceName, self.queue ),
'%s' % jdlName ]
result = executeGridCommand( self.proxy, cmd, self.gridEnv )
if result['OK']:
pilotJobReference = result['Value'][1].strip()
batchIDList.append( pilotJobReference )
stampDict[pilotJobReference] = diracStamp
os.unlink( jdlName )
else:
delegationID = makeGuid()
cmd = [ 'glite-ce-delegate-proxy', '-e', '%s' % self.ceName, '%s' % delegationID ]
result = executeGridCommand( self.proxy, cmd, self.gridEnv )
if not result['OK']:
self.log.error('Failed to delegate proxy: %s' % result['Message'])
return result
for i in range( numberOfJobs ):
jdlName, diracStamp = self.__writeJDL( executableFile )
cmd = ['glite-ce-job-submit', '-n', '-N', '-r',
'%s/%s' % ( self.ceName, self.queue ),
'-D', '%s' % delegationID, '%s' % jdlName ]
result = executeGridCommand( self.proxy, cmd, self.gridEnv )
if not result['OK']:
break
if result['Value'][0] != 0:
break
pilotJobReference = result['Value'][1].strip()
if pilotJobReference:
batchIDList.append( pilotJobReference )
stampDict[pilotJobReference] = diracStamp
else:
break
os.unlink( jdlName )
os.unlink( executableFile )
if batchIDList:
result = S_OK( batchIDList )
result['PilotStampDict'] = stampDict
else:
result = S_ERROR('No pilot references obtained from the glite job submission')
return result
def addSubRequest( self, requestDict, rType ):
""" Add a new sub-requests of specified type
The 'Attributes':ExecutionOrder can be set to 'last' to always be the last to be executed.
"""
# Initialise the sub-request
index = self.initiateSubRequest( rType )['Value']
# Stuff the sub-request with the attributes
attributeDict = {'Status':'Waiting', 'SubRequestID':makeGuid(),
'CreationTime': str( datetime.datetime.utcnow() ),
'ExecutionOrder':0}
for attr, value in requestDict['Attributes'].items():
if attr == 'ExecutionOrder' and str( value ).lower() == 'last':
value = self._getLastOrder() + 1
attributeDict[attr] = value
for attr in self.subAttributeNames:
if not attr in attributeDict.keys():
attributeDict[attr] = ''
if attr == "ExecutionOrder":
attributeDict[attr] = 0
if attr == "Status":
attributeDict[attr] = "Waiting"
self.setSubRequestAttributes( index, rType, attributeDict )
if requestDict.has_key( 'Files' ):
files = []
for rFile in requestDict['Files']:
fileDict = {'Status':'Waiting', 'FileID':makeGuid(), 'Attempt':1}
for attr, value in rFile.items():
fileDict[attr] = value
files.append( fileDict )
self.setSubRequestFiles( index, rType, files )
if requestDict.has_key( 'Datasets' ):
datasets = []
for dataset in requestDict['Datasets']:
datasetDict = {'Status':'Waiting'}
for attr, value in rFile.items():
fileDict[attr] = value
datasets.append( datasetDict )
self.setSubRequestDatasets( index, rType, datasets )
return S_OK( index )
def getFileMetadata(self, candidateFiles):
"""Returns the candidate file dictionary with associated metadata.
@param candidateFiles: The input candidate files dictionary has the structure:
{'lfn':'','path':'','workflowSE':''}
This also assumes the files are in the current working directory.
@return: File Metadata
"""
#Retrieve the POOL File GUID(s) for any final output files
self.log.info('Will search GUIDs for: %s' %(', '.join(candidateFiles.keys())))
pfnGUIDs = {}
generated = []
for fname in candidateFiles.keys():
guid = makeGuid(fname)
pfnGUIDs[fname] = guid
generated.append(fname)
pfnGUID = S_OK(pfnGUIDs)
pfnGUID['generated'] = generated
self.log.debug('Generated GUID(s) for the following files ', ', '.join(pfnGUID['generated']))
for pfn, guid in pfnGUID['Value'].items():
candidateFiles[pfn]['GUID'] = guid
#Get all additional metadata about the file necessary for requests
final = {}
for fileName, metadata in candidateFiles.items():
fileDict = {}
fileDict['LFN'] = metadata['lfn']
fileDict['Size'] = os.path.getsize(fileName)
fileDict['Addler'] = fileAdler(fileName)
fileDict['GUID'] = metadata['GUID']
fileDict['Status'] = "Waiting"
final[fileName] = metadata
final[fileName]['filedict'] = fileDict
final[fileName]['localpath'] = '%s/%s' % (os.getcwd(), fileName)
gLogger.verbose("Full file dict", str(final))
#Sanity check all final candidate metadata keys are present (return S_ERROR if not)
mandatoryKeys = ['GUID', 'filedict'] #filedict is used for requests (this method adds guid and filedict)
for fileName, metadata in final.items():
for key in mandatoryKeys:
if not metadata.has_key(key):
return S_ERROR('File %s has missing %s' % (fileName, key))
return S_OK(final)
def _submitJobToHost(self, executableFile, numberOfJobs, host=None):
""" Submit prepared executable to the given host
"""
ssh = SSH(host=host, parameters=self.ceParameters)
# Copy the executable
submitFile = '%s/%s' % (self.executableArea, os.path.basename(executableFile))
result = ssh.scpCall(30, executableFile, submitFile, postUploadCommand='chmod +x %s' % submitFile)
if not result['OK']:
return result
jobStamps = []
for _i in xrange(numberOfJobs):
jobStamps.append(makeGuid()[:8])
numberOfProcessors = self.ceParameters.get('NumberOfProcessors', 1)
wholeNode = self.ceParameters.get('WholeNode', False)
# Collect command options
commandOptions = {'Executable': submitFile,
'NJobs': numberOfJobs,
'SubmitOptions': self.submitOptions,
'JobStamps': jobStamps,
'WholeNode': wholeNode,
'NumberOfProcessors': numberOfProcessors,
'Preamble': self.preamble}
resultCommand = self.__executeHostCommand('submitJob', commandOptions, ssh=ssh, host=host)
if not resultCommand['OK']:
return resultCommand
result = resultCommand['Value']
if result['Status'] != 0:
return S_ERROR('Failed job submission: %s' % result['Message'])
else:
batchIDs = result['Jobs']
if batchIDs:
ceHost = host
if host is None:
ceHost = self.ceName
jobIDs = ['%s%s://%s/%s' % (self.ceType.lower(), self.batchSystem.lower(), ceHost, _id) for _id in batchIDs]
else:
return S_ERROR('No jobs IDs returned')
result = S_OK(jobIDs)
self.submittedJobs += len(batchIDs)
return result
def submitJob(self, executableFile, proxy=None, numberOfJobs=1):
if not os.access(executableFile, 5):
os.chmod(executableFile, stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH)
# if no proxy is supplied, the executable can be submitted directly
# otherwise a wrapper script is needed to get the proxy to the execution node
# The wrapper script makes debugging more complicated and thus it is
# recommended to transfer a proxy inside the executable if possible.
if proxy:
self.log.verbose('Setting up proxy for payload')
wrapperContent = bundleProxy(executableFile, proxy)
name = writeScript(wrapperContent, os.getcwd())
submitFile = name
else: # no proxy
submitFile = executableFile
jobStamps = []
for _i in range(numberOfJobs):
jobStamps.append(makeGuid()[:8])
batchDict = {'Executable': submitFile,
'NJobs': numberOfJobs,
'OutputDir': self.batchOutput,
'ErrorDir': self.batchError,
'SubmitOptions': self.submitOptions,
'ExecutionContext': self.execution,
'JobStamps': jobStamps,
'Queue': self.queue}
resultSubmit = self.batch.submitJob(**batchDict)
if proxy:
os.remove(submitFile)
if resultSubmit['Status'] == 0:
self.submittedJobs += len(resultSubmit['Jobs'])
# jobIDs = [ self.ceType.lower()+'://'+self.ceName+'/'+_id for _id in resultSubmit['Jobs'] ]
# FIXME: It would be more proper to fix pilotCommands.__setFlavour where 'ssh' is hardcoded than
# making this illogical fix, but there is no good way for pilotCommands to know its origin ceType.
# So, the jobIDs here need to start with 'ssh', not ceType, to accomodate
# them to those hardcoded in pilotCommands.__setFlavour
jobIDs = ['ssh' + self.batchSystem.lower() + '://' + self.ceName + '/' + _id for _id in resultSubmit['Jobs']]
result = S_OK(jobIDs)
else:
result = S_ERROR(resultSubmit['Message'])
return result
def __init__(self,rpcStub= None,executionOrder=0):
"""Instantiates the Workflow object and some default parameters.
"""
self.subAttributeNames = ['Status','SubRequestID','Operation','ExecutionOrder','CreationTime','LastUpdate','Arguments']
self.subAttributes = {}
for attr in self.subAttributeNames:
self.subAttributes[attr] = "Unknown"
# Some initial values
self.subAttributes['Status'] = "Waiting"
self.subAttributes['SubRequestID'] = makeGuid()
self.subAttributes['CreationTime'] = Time.toString()
self.subAttributes['ExecutionOrder'] = executionOrder
if rpcStub:
self.subAttributes['Arguments'] = DEncode.encode(rpcStub)
self.subAttributes['Operation'] = rpcStub[1]
def files( self, userName, userGroup ):
""" get list of files in user domain """
files = []
for i in range( 10 ):
fname = "/tmp/testUserFile-%s" % i
if userGroup == "lhcb_user":
lfn = "/lhcb/user/%s/%s/%s" % ( userName[0], userName, fname.split( "/" )[-1] )
else:
lfn = "/lhcb/certification/test/rmsdms/%s" % fname.split( "/" )[-1]
fh = open( fname, "w+" )
for i in range( 100 ):
fh.write( str( random.randint( 0, i ) ) )
fh.close()
size = os.stat( fname ).st_size
checksum = fileAdler( fname )
guid = makeGuid( fname )
files.append( ( fname, lfn, size, checksum, guid ) )
return files
def submitJob(self, executableFile, proxy, numberOfJobs=1):
""" Method to submit job
"""
self.log.verbose("Executable file path: %s" % executableFile)
if not os.access(executableFile, 5):
os.chmod(executableFile, 0o755)
subName = self.__writeSub(executableFile, numberOfJobs)
jobStamps = []
for _i in range(numberOfJobs):
jobStamps.append(makeGuid()[:8])
cmd = ['condor_submit', '-terse', subName]
# the options for submit to remote are different than the other remoteScheddOptions
scheddOptions = [] if self.useLocalSchedd else ['-pool', '%s:9619' % self.ceName, '-remote', self.ceName]
for op in scheddOptions:
cmd.insert(-1, op)
result = executeGridCommand(self.proxy, cmd, self.gridEnv)
self.log.verbose(result)
os.unlink(subName)
if not result['OK']:
self.log.error("Failed to submit jobs to htcondor", result['Message'])
return result
if result['Value'][0]:
# We have got a non-zero status code
errorString = result['Value'][2] if result['Value'][2] else result['Value'][1]
return S_ERROR('Pilot submission failed with error: %s ' % errorString.strip())
pilotJobReferences = self.__getPilotReferences(result['Value'][1].strip())
if not pilotJobReferences['OK']:
return pilotJobReferences
pilotJobReferences = pilotJobReferences['Value']
self.log.verbose("JobStamps: %s " % jobStamps)
self.log.verbose("pilotRefs: %s " % pilotJobReferences)
result = S_OK(pilotJobReferences)
result['PilotStampDict'] = dict(zip(pilotJobReferences, jobStamps))
self.log.verbose("Result for submission: %s " % result)
return result
def __writeXRSL( self, executableFile ):
""" Create the JDL for submission
"""
diracStamp = makeGuid()[:8]
xrsl = """
&(executable="%(executable)s")
(inputFiles=(%(executable)s "%(executableFile)s"))
(stdout="%(diracStamp)s.out")
(stderr="%(diracStamp)s.err")
(outputFiles=("%(diracStamp)s.out" "") ("%(diracStamp)s.err" ""))
%(xrslExtraString)s
""" % {
'executableFile':executableFile,
'executable':os.path.basename( executableFile ),
'diracStamp':diracStamp,
# 'queue':self.queue,
'xrslExtraString':self.xrslExtraString
}
return xrsl, diracStamp
def getGUID( fileNames, directory = '' ):
""" This function searches the directory for POOL XML catalog files and extracts the GUID.
fileNames can be a string or a list, directory defaults to PWD.
"""
if not directory:
directory = os.getcwd()
if not os.path.isdir( directory ):
return S_ERROR( '%s is not a directory' % directory )
if not type( fileNames ) == type( [] ):
fileNames = [fileNames]
gLogger.verbose( 'Will look for POOL XML Catalog GUIDs in %s for %s' % ( directory, ', '.join( fileNames ) ) )
finalCatList = _getPoolCatalogs( directory )
#Create POOL catalog with final list of catalog files and extract GUIDs
generated = []
pfnGUIDs = {}
catalog = PoolXMLCatalog( finalCatList )
for fname in fileNames:
guid = str( catalog.getGuidByPfn( fname ) )
if not guid:
guid = makeGuid( fname )
generated.append( fname )
pfnGUIDs[fname] = guid
if not generated:
gLogger.info( 'Found GUIDs from POOL XML Catalogue for all files: %s' % ', '.join( fileNames) )
else:
gLogger.info( 'GUIDs not found from POOL XML Catalogue (and were generated) for: %s' % ', '.join( generated) )
result = S_OK( pfnGUIDs )
result['directory'] = directory
result['generated'] = generated
return result
def submitJob( self, executableFile, proxy = None, numberOfJobs = 1 ):
if not os.access( executableFile, 5 ):
os.chmod( executableFile, stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH )
# if no proxy is supplied, the executable can be submitted directly
# otherwise a wrapper script is needed to get the proxy to the execution node
# The wrapper script makes debugging more complicated and thus it is
# recommended to transfer a proxy inside the executable if possible.
if proxy:
self.log.verbose( 'Setting up proxy for payload' )
wrapperContent = bundleProxy( executableFile, proxy )
name = writeScript( wrapperContent, os.getcwd() )
submitFile = name
else: # no proxy
submitFile = executableFile
jobStamps = []
for _i in range( numberOfJobs ):
jobStamps.append( makeGuid()[:8] )
batchDict = { 'Executable': submitFile,
'NJobs': numberOfJobs,
'OutputDir': self.batchOutput,
'ErrorDir': self.batchError,
'SubmitOptions': self.submitOptions,
'ExecutionContext': self.execution,
'JobStamps': jobStamps }
resultSubmit = self.batch.submitJob( **batchDict )
if proxy:
os.remove( submitFile )
if resultSubmit['Status'] == 0:
self.submittedJobs += len( resultSubmit['Jobs'] )
jobIDs = [ self.ceType.lower()+'://'+self.ceName+'/'+_id for _id in resultSubmit['Jobs'] ]
result = S_OK( jobIDs )
else:
result = S_ERROR( resultSubmit['Message'] )
return result
def __writeXRSL(self, executableFile):
""" Create the JDL for submission
"""
diracStamp = makeGuid()[:8]
xrsl = """
&(executable="%(executable)s")
(inputFiles=(%(executable)s "%(executableFile)s"))
(stdout="%(diracStamp)s.out")
(stderr="%(diracStamp)s.err")
(outputFiles=("%(diracStamp)s.out" "") ("%(diracStamp)s.err" ""))
(queue=%(queue)s)
%(xrslExtraString)s
""" % {
"executableFile": executableFile,
"executable": os.path.basename(executableFile),
"diracStamp": diracStamp,
"queue": self.arcQueue,
"xrslExtraString": self.xrslExtraString,
}
return xrsl, diracStamp
def sweeper( cls ):
""" move cached request to the central request manager
:param cls: class reference
"""
cacheDir = cls.cacheDir()
## cache dir empty?
if not os.listdir( cacheDir ):
gLogger.always("sweeper: CacheDir %s is empty, nothing to do" % cacheDir )
return S_OK()
else:
## read 10 cache dir files, the oldest first
cachedRequests = [ os.path.abspath( requestFile ) for requestFile in
sorted( filter( os.path.isfile,
[ os.path.join( cacheDir, requestName )
for requestName in os.listdir( cacheDir ) ] ),
key = os.path.getctime ) ][:30]
## set cached requests to the central RequestManager
for cachedFile in cachedRequests:
try:
requestString = "".join( open( cachedFile, "r" ).readlines() )
cachedRequest = RequestContainer( requestString )
requestName = cachedRequest.getAttribute("RequestName")["Value"]
## cibak: hack for DISET requests
if requestName == "Unknown":
cachedRequest.setAttribute( "RequestName", makeGuid() )
requestName = cachedRequest.getAttribute("RequestName")["Value"]
setRequest = cls.requestManager().setRequest( requestName, requestString )
if not setRequest["OK"]:
gLogger.error("sweeper: unable to set request '%s' @ RequestManager: %s" % ( requestName,
setRequest["Message"] ) )
continue
gLogger.info("sweeper: successfully set request '%s' @ RequestManager" % requestName )
os.unlink( cachedFile )
except Exception, error:
gLogger.exception( "sweeper: hit by exception %s" % str(error) )
return S_ERROR( "sweeper: hit by exception: %s" % str(error) )
return S_OK()
def __writeXRSL(self, executableFile, processors=1):
""" Create the JDL for submission
"""
diracStamp = makeGuid()[:8]
# Evaluate the number of processors to allocate
nProcessors = processors if processors > 1 else self.ceParameters.get('NumberOfProcessors', processors)
xrslMPAdditions = ''
if nProcessors > 1:
xrslMPAdditions = """
(count = %(processors)u)
%(xrslMPExtraString)s
""" % {
'processors': nProcessors,
'xrslMPExtraString': self.xrslMPExtraString
}
xrsl = """
&(executable="%(executable)s")
(inputFiles=(%(executable)s "%(executableFile)s"))
(stdout="%(diracStamp)s.out")
(stderr="%(diracStamp)s.err")
(outputFiles=("%(diracStamp)s.out" "") ("%(diracStamp)s.err" ""))
(queue=%(queue)s)
%(xrslMPAdditions)s
%(xrslExtraString)s
""" % {
'executableFile': executableFile,
'executable': os.path.basename(executableFile),
'diracStamp': diracStamp,
'queue': self.arcQueue,
'xrslMPAdditions': xrslMPAdditions,
'xrslExtraString': self.xrslExtraString
}
return xrsl, diracStamp
def submitJob( self, executableFile, proxy, numberOfJobs = 1 ):
""" Method to submit job
"""
self.log.verbose( "Executable file path: %s" % executableFile )
if not os.access( executableFile, 5 ):
os.chmod( executableFile, 0755 )
batchIDList = []
stampDict = {}
for _i in xrange(numberOfJobs):
diracStamp = makeGuid()[:8]
queueName = '%s/%s' % ( self.ceName, self.queue )
cmd = ['globus-job-submit', queueName, "-s", executableFile ]
#cmd = ['globus-job-submit', '-r %s' % queueName, '-f %s' % jdlName ]
result = executeGridCommand( self.proxy, cmd, self.gridEnv )
self.log.verbose(result)
if result['OK']:
if result['Value'][0]:
# We have got a non-zero status code
errorString = result['Value'][2] if result['Value'][2] else result['Value'][1]
return S_ERROR( 'Pilot submission failed with error: %s ' % errorString.strip() )
pilotJobReference = result['Value'][1].strip()
if not pilotJobReference:
return S_ERROR( 'No pilot reference returned from the glite job submission command' )
if not pilotJobReference.startswith( 'https' ):
return S_ERROR( 'Invalid pilot reference %s' % pilotJobReference )
batchIDList.append( pilotJobReference )
stampDict[pilotJobReference] = diracStamp
if batchIDList:
result = S_OK( batchIDList )
result['PilotStampDict'] = stampDict
else:
result = S_ERROR( 'No pilot references obtained from the glite job submission' )
return result
def main():
Script.parseCommandLine(ignoreErrors=False)
files = Script.getPositionalArgs()
if len(files) == 0:
Script.showHelp()
exitCode = 0
import DIRAC
from DIRAC.Core.Utilities.File import makeGuid
for file in files:
try:
md5 = makeGuid(file)
if md5:
print(file.rjust(100), md5.ljust(10))
else:
print('ERROR %s: Failed to get md5' % file)
exitCode = 2
except Exception as x:
print('ERROR %s: Failed to get md5' % file, str(x))
exitCode = 2
DIRAC.exit(exitCode)
def testMakeGuid():
"""makeGuid tests"""
# no filename - fake guid produced
assert checkGuid(makeGuid()) is True, "fake guid for inexisting file"
# using this python file
assert checkGuid(makeGuid(abspath(__file__))) is True, "guid for FileTestCase.py file"
def main():
dm = DataManager()
fileTupleBuffer = []
res = getProxyInfo( False, False )
if not res['OK']:
gLogger.error( "Failed to get client proxy information.", res['Message'] )
DIRAC.exit( 2 )
proxyInfo = res['Value']
if proxyInfo['secondsLeft'] == 0:
gLogger.error( "Proxy expired" )
DIRAC.exit( 2 )
username = proxyInfo['username']
vo = ''
if 'group' in proxyInfo:
vo = getVOForGroup( proxyInfo['group'] )
counter = 0
for f in files:
counter += 1
local_f=f
if not f.startswith('/cefs'):
# gLogger.error('File must be under "/cefs"')
# continue
#if the file to reg is not under /cefs, use put and register
folder_name=os.path.basename(os.path.dirname(f))
# lfn = '/cepc/user/%s/%s/jsub/'%(username[0],username) + folder_name + '/' + os.path.basename(f)
lfn = '/cepc/user/%s/%s/jsub/'%(username[0],username) + folder_name + '/' + os.path.basename(f)
# dirname = os.path.dirname(local_f)
# os.system('mkdir -p %s'%(dirname))
# os.system('cp %s %s' %(f,local_f))
do_put_and_register=True
else:
lfn = '/cepc/lustre-ro' + os.path.abspath(f)
do_put_and_register=False
result = fcc.isFile(lfn)
if result['OK'] and lfn in result['Value']['Successful'] and result['Value']['Successful'][lfn]:
continue
size = os.path.getsize(f)
adler32 = fileAdler(f)
guid = makeGuid()
fileTuple = (lfn, local_f, size, _se, guid, adler32)
fileTupleBuffer.append(fileTuple)
gLogger.debug('Register to lfn: %s' % lfn)
gLogger.debug('fileTuple: %s' % (fileTuple,))
if len(fileTupleBuffer) >= _bufferSize:
if do_put_and_register:
result = dm.putAndRegister(lfn, local_f, _se, guid, overwrite=overwrite)
else:
result = dm.registerFile(fileTupleBuffer)
print('register result', result)
# if not result['OK']:
# gLogger.error('Register file failed')
# return 1
del fileTupleBuffer[:]
gLogger.debug('%s files registered' % counter)
if fileTupleBuffer:
if do_put_and_register:
result = dm.putAndRegister(lfn, local_f, _se, guid, overwrite=overwrite)
else:
result = dm.registerFile(fileTupleBuffer)
print('register result', result)
# if not result['OK']:
# gLogger.error('Register file failed')
# return 1
del fileTupleBuffer[:]
gLogger.info('Totally %s files registered' % counter)
return 0
__RCSID__ = "$Id$"
import DIRAC
from DIRAC.Core.Utilities.File import makeGuid
from DIRAC.Core.Base import Script
Script.setUsageMessage( '\n'.join( [ __doc__.split( '\n' )[1],
'Usage:',
' %s [option|cfgfile] ... File ...' % Script.scriptName,
'Arguments:',
' File: File Name' ] ) )
Script.parseCommandLine( ignoreErrors = False )
files = Script.getPositionalArgs()
if len( files ) == 0:
Script.showHelp()
exitCode = 0
for file in files:
try:
md5 = makeGuid( file )
if md5:
print file.rjust( 100 ), md5.ljust( 10 )
else:
print 'ERROR %s: Failed to get md5' % file
exitCode = 2
except Exception as x:
print 'ERROR %s: Failed to get md5' % file, str( x )
exitCode = 2
DIRAC.exit( exitCode )
def submitJob(self, executableFile, proxy, numberOfJobs=1):
"""Method to submit job"""
self.log.verbose("Executable file path: %s" % executableFile)
if not os.access(executableFile, 5):
os.chmod(executableFile, 0o755)
# The submitted pilots are going to have a common part of the stamp to construct a path to retrieve results
# Then they also have an individual part to make them unique
jobStamps = []
commonJobStampPart = makeGuid()[:3]
for _i in range(numberOfJobs):
jobStamp = commonJobStampPart + makeGuid()[:5]
jobStamps.append(jobStamp)
# We randomize the location of the pilot output and log, because there are just too many of them
location = logDir(self.ceName, commonJobStampPart)
nProcessors = self.ceParameters.get("NumberOfProcessors", 1)
subName = self.__writeSub(executableFile, numberOfJobs, location,
nProcessors)
cmd = ["condor_submit", "-terse", subName]
# the options for submit to remote are different than the other remoteScheddOptions
# -remote: submit to a remote condor_schedd and spool all the required inputs
scheddOptions = [] if self.useLocalSchedd else [
"-pool", "%s:9619" % self.ceName, "-remote", self.ceName
]
for op in scheddOptions:
cmd.insert(-1, op)
result = executeGridCommand(self.proxy, cmd, self.gridEnv)
self.log.verbose(result)
os.remove(subName)
if not result["OK"]:
self.log.error("Failed to submit jobs to htcondor",
result["Message"])
return result
if result["Value"][0]:
# We have got a non-zero status code
errorString = result["Value"][2] if result["Value"][2] else result[
"Value"][1]
return S_ERROR("Pilot submission failed with error: %s " %
errorString.strip())
pilotJobReferences = self.__getPilotReferences(
result["Value"][1].strip())
if not pilotJobReferences["OK"]:
return pilotJobReferences
pilotJobReferences = pilotJobReferences["Value"]
self.log.verbose("JobStamps: %s " % jobStamps)
self.log.verbose("pilotRefs: %s " % pilotJobReferences)
result = S_OK(pilotJobReferences)
result["PilotStampDict"] = dict(zip(pilotJobReferences, jobStamps))
if self.useLocalSchedd:
# Executable is transferred afterward
# Inform the caller that Condor cannot delete it before the end of the execution
result["ExecutableToKeep"] = executableFile
self.log.verbose("Result for submission: %s " % result)
return result
if lfn in lfnQuery:
if counter % 1000 == 0:
gLogger.notice('Skip file in query counter: %s' % counter)
continue
if existCheck:
result = fcc.isFile(lfn)
if result['OK'] and lfn in result['Value'][
'Successful'] and result['Value']['Successful'][lfn]:
if counter % 1000 == 0:
gLogger.notice('Skip file existed counter: %s' % counter)
continue
size = os.path.getsize(fullFn)
adler32 = fileAdler(fullFn)
guid = makeGuid()
fileTuple = (lfn, fullFn, size, toSE, guid, adler32)
fileTupleBuffer.append(fileTuple)
gLogger.debug('Register to lfn: %s' % lfn)
gLogger.debug('fileTuple: %s' % (fileTuple, ))
if len(fileTupleBuffer) >= bufferSize:
result = dm.registerFile(fileTupleBuffer)
if not result['OK']:
gLogger.error('Can not register %s' % fullFn)
exit(1)
del fileTupleBuffer[:]
gLogger.notice('%s files registered' % counter)
if fileTupleBuffer:
result = dm.registerFile(fileTupleBuffer)
def setBenderModule(self,
benderVersion,
modulePath,
inputData=None,
numberOfEvents=-1):
"""Specifies Bender module to be executed.
Any additional files should be specified in the job input sandbox. Input data for
Bender should be specified here (can be string or list).
Example usage:
>>> job = LHCbJob()
>>> job.setBenderModule('v8r3','BenderExample.PhiMC',
inputData=['LFN:/lhcb/production/DC06/phys-v2-lumi2/00001758/DST/00001758_00000001_5.dst'],numberOfEvents=100)
:param benderVersion: Bender Project Version
:type benderVersion: string
:param modulePath: Import path to module e.g. BenderExample.PhiMC
:type modulePath: string
:param inputData: Input data for application
:type inputData: single LFN or list of LFNs
:param numberOfEvents: Number of events to process e.g. -1
:type numberOfEvents: integer
"""
kwargs = {
'benderVersion': benderVersion,
'modulePath': modulePath,
'inputData': inputData,
'numberOfEvents': numberOfEvents
}
if not isinstance(benderVersion, str):
return self._reportError('Bender version should be a string',
__name__, **kwargs)
if not isinstance(modulePath, str):
return self._reportError('Bender module path should be a string',
__name__, **kwargs)
if not isinstance(numberOfEvents, int):
try:
numberOfEvents = int(numberOfEvents)
except ValueError:
return self._reportError(
'Number of events should be an integer or convertible to an integer',
__name__, **kwargs)
if not inputData:
return S_ERROR("Need input data for Bender applications")
if isinstance(inputData, str):
inputData = [inputData]
if not isinstance(inputData, list):
return self._reportError(
'Input data should be specified as a list or a string',
__name__, **kwargs)
poolCatName = 'xmlcatalog_file:pool_xml_catalog.xml'
benderScript = ['#!/usr/bin/env python']
benderScript.append('from Gaudi.Configuration import FileCatalog')
benderScript.append('FileCatalog (Catalogs = ["%s"] )' % poolCatName)
benderScript.append('import %s as USERMODULE' % modulePath)
benderScript.append('USERMODULE.configure()')
benderScript.append('gaudi = USERMODULE.appMgr()')
benderScript.append('evtSel = gaudi.evtSel()')
benderScript.append('evtSel.open ( %s ) ' % inputData)
benderScript.append('USERMODULE.run( %s )\n' % numberOfEvents)
guid = makeGuid()
tmpdir = '/tmp/' + guid
self.log.verbose('Created temporary directory for submission %s' %
(tmpdir))
mkDir(tmpdir)
with open('%s/BenderScript.py' % tmpdir, 'w') as fopen:
self.log.verbose('Bender script is: %s/BenderScript.py' % tmpdir)
fopen.write('\n'.join(benderScript))
# should try all components of the PYTHONPATH before giving up...
userModule = '%s.py' % (modulePath.split('.')[-1])
self.log.verbose('Looking for user module with name: %s' % userModule)
if os.path.exists(userModule):
self.addToInputSandbox.append(userModule)
self.setInputData(inputData)
self.setApplicationScript('Bender',
benderVersion,
'%s/BenderScript.py' % tmpdir,
logFile='Bender%s.log' % benderVersion)
return S_OK(benderScript)
def createInstance(self, instanceID=''):
if not instanceID:
instanceID = makeGuid()[:8]
self.parameters['VMUUID'] = instanceID
self.parameters['VMType'] = self.parameters.get('CEType', 'EC2')
createNodeDict = {}
# Image
if "ImageID" in self.parameters and 'ImageName' not in self.parameters:
try:
images = self.__ec2.images.filter(
Filters=[{
'Name': 'name',
'Values': [self.parameters['ImageName']]
}])
imageId = None
for image in images:
imageId = image.id
break
except Exception as e:
self.log.error(
"Exception when get ID from image name %s:" %
self.parameters['ImageName'], e)
return S_ERROR("Failed to get image for Name %s" %
self.parameters['ImageName'])
if imageId is None:
return S_ERROR("Image name %s not found" %
self.parameters['ImageName'])
elif "ImageID" in self.parameters:
try:
self.__ec2.images.filter(ImageIds=[self.parameters['ImageID']])
except Exception as e:
return S_ERROR("Failed to get image for ID %s" %
self.parameters['ImageID'])
imageId = self.parameters['ImageID']
else:
return S_ERROR('No image specified')
createNodeDict['ImageId'] = imageId
# Instance type
if 'FlavorName' not in self.parameters:
return S_ERROR('No flavor specified')
instanceType = self.parameters['FlavorName']
createNodeDict['InstanceType'] = instanceType
# User data
result = self._createUserDataScript()
if not result['OK']:
return result
createNodeDict['UserData'] = str(result['Value'])
# Other params
for param in ['KeyName', 'SubnetId', 'EbsOptimized']:
if param in self.parameters:
createNodeDict[param] = self.parameters[param]
self.log.info("Creating node:")
for key, value in createNodeDict.items():
self.log.verbose("%s: %s" % (key, value))
# Create the VM instance now
try:
instances = self.__ec2.create_instances(MinCount=1,
MaxCount=1,
**createNodeDict)
except Exception as e:
errmsg = 'Exception in ec2 create_instances: %s' % e
self.log.error(errmsg)
return S_ERROR(errmsg)
if len(instances) < 1:
errmsg = 'ec2 create_instances failed to create any VM'
self.log.error(errmsg)
return S_ERROR(errmsg)
# Create the name in tags
ec2Id = instances[0].id
tags = [{'Key': 'Name', 'Value': 'DIRAC_%s' % instanceID}]
try:
self.__ec2.create_tags(Resources=[ec2Id], Tags=tags)
except Exception as e:
errmsg = 'Exception setup name for %s: %s' % (ec2Id, e)
self.log.error(errmsg)
return S_ERROR(errmsg)
# Properties of the instance
nodeDict = {}
# nodeDict['PublicIP'] = publicIP
nodeDict['InstanceID'] = instanceID
if instanceType in self.__instanceTypeInfo:
nodeDict['NumberOfProcessors'] = self.__instanceTypeInfo[
instanceType]['vCPU']
nodeDict['RAM'] = self.__instanceTypeInfo[instanceType]['Memory']
else:
nodeDict['NumberOfProcessors'] = 1
return S_OK((ec2Id, nodeDict))
def createInstance(self, instanceID=""):
if not instanceID:
instanceID = makeGuid()[:8]
self.parameters["VMUUID"] = instanceID
self.parameters["VMType"] = self.parameters.get("CEType", "EC2")
createNodeDict = {}
# Image
if "ImageID" in self.parameters and "ImageName" not in self.parameters:
try:
images = self.__ec2.images.filter(
Filters=[{
"Name": "name",
"Values": [self.parameters["ImageName"]]
}])
imageId = None
for image in images:
imageId = image.id
break
except Exception as e:
self.log.exception(
"Exception when get ID from image name %s:" %
self.parameters["ImageName"])
return S_ERROR("Failed to get image for Name %s" %
self.parameters["ImageName"])
if imageId is None:
return S_ERROR("Image name %s not found" %
self.parameters["ImageName"])
elif "ImageID" in self.parameters:
try:
self.__ec2.images.filter(ImageIds=[self.parameters["ImageID"]])
except Exception as e:
self.log.exception("Failed to get EC2 image list")
return S_ERROR("Failed to get image for ID %s" %
self.parameters["ImageID"])
imageId = self.parameters["ImageID"]
else:
return S_ERROR("No image specified")
createNodeDict["ImageId"] = imageId
# Instance type
if "FlavorName" not in self.parameters:
return S_ERROR("No flavor specified")
instanceType = self.parameters["FlavorName"]
createNodeDict["InstanceType"] = instanceType
# User data
result = self._createUserDataScript()
if not result["OK"]:
return result
createNodeDict["UserData"] = str(result["Value"])
# Other params
for param in ["KeyName", "SubnetId", "EbsOptimized"]:
if param in self.parameters:
createNodeDict[param] = self.parameters[param]
self.log.info("Creating node:")
for key, value in createNodeDict.items():
self.log.verbose("%s: %s" % (key, value))
# Create the VM instance now
try:
instances = self.__ec2.create_instances(MinCount=1,
MaxCount=1,
**createNodeDict)
except Exception as e:
self.log.exception("Failed to create EC2 instance")
return S_ERROR("Exception in ec2 create_instances: %s" % e)
if len(instances) < 1:
errmsg = "ec2 create_instances failed to create any VM"
self.log.error(errmsg)
return S_ERROR(errmsg)
# Create the name in tags
ec2Id = instances[0].id
tags = [{"Key": "Name", "Value": "DIRAC_%s" % instanceID}]
try:
self.__ec2.create_tags(Resources=[ec2Id], Tags=tags)
except Exception as e:
self.log.exception("Failed to tag EC2 instance")
return S_ERROR("Exception setup name for %s: %s" % (ec2Id, e))
# Properties of the instance
nodeDict = {}
# nodeDict['PublicIP'] = publicIP
nodeDict["InstanceID"] = instanceID
if instanceType in self.__instanceTypeInfo:
nodeDict["NumberOfProcessors"] = self.__instanceTypeInfo[
instanceType]["vCPU"]
nodeDict["RAM"] = self.__instanceTypeInfo[instanceType]["Memory"]
else:
nodeDict["NumberOfProcessors"] = 1
return S_OK((ec2Id, nodeDict))
def _submitJobToHost( self, executableFile, numberOfJobs, host = None ):
""" Submit prepared executable to the given host
"""
ssh = SSH( host = host, parameters = self.ceParameters )
# Copy the executable
sFile = os.path.basename( executableFile )
result = ssh.scpCall( 10, executableFile, '%s/%s' % ( self.executableArea, os.path.basename( executableFile ) ) )
if not result['OK']:
return result
jobStamps = []
for i in range( numberOfJobs ):
jobStamps.append( makeGuid()[:8] )
jobStamp = '#'.join( jobStamps )
subOptions = urllib.quote( self.submitOptions )
cmd = "bash --login -c '%s/%s submit_job %s/%s %s %s %s %d %s %s %s %s'" % ( self.sharedArea,
self.controlScript,
self.executableArea,
os.path.basename( executableFile ),
self.batchOutput,
self.batchError,
self.workArea,
numberOfJobs,
self.infoArea,
jobStamp,
self.execQueue,
subOptions )
self.log.verbose( 'CE submission command: %s' % cmd )
result = ssh.sshCall( 120, cmd )
if not result['OK']:
self.log.error( '%s CE job submission failed' % self.ceType, result['Message'] )
return result
sshStatus = result['Value'][0]
sshStdout = result['Value'][1]
sshStderr = result['Value'][2]
# Examine results of the job submission
submitHost = host
if host is None:
submitHost = self.ceParameters['SSHHost'].split('/')[0]
if sshStatus == 0:
outputLines = sshStdout.strip().replace('\r','').split('\n')
try:
index = outputLines.index('============= Start output ===============')
outputLines = outputLines[index+1:]
except:
return S_ERROR( "Invalid output from job submission: %s" % outputLines[0] )
try:
status = int( outputLines[0] )
except:
return S_ERROR( "Failed local batch job submission: %s" % outputLines[0] )
if status != 0:
message = "Unknown reason"
if len( outputLines ) > 1:
message = outputLines[1]
return S_ERROR( 'Failed job submission, reason: %s' % message )
else:
batchIDs = outputLines[1:]
jobIDs = [ self.ceType.lower()+'://'+self.ceName+'/'+id for id in batchIDs ]
else:
return S_ERROR( '\n'.join( [sshStdout,sshStderr] ) )
result = S_OK ( jobIDs )
self.submittedJobs += len( batchIDs )
return result
try:
cat = PoolXMLCatalog(possibleCat)
finalCatList.append(possibleCat)
except Exception, x:
gLogger.debug('Ignoring non-POOL catalogue file %s' % possibleCat)
#Create POOL catalog with final list of catalog files and extract GUIDs
generated = []
pfnGUIDs = {}
gLogger.debug('Final list of catalog files are: %s' %
string.join(finalCatList, ', '))
catalog = PoolXMLCatalog(finalCatList)
for fname in fileNames:
guid = str(catalog.getGuidByPfn(fname))
if not guid:
guid = makeGuid(fname)
generated.append(fname)
pfnGUIDs[fname] = guid
if not generated:
gLogger.info('Found GUIDs from POOL XML Catalogue for all files: %s' %
string.join(fileNames, ', '))
else:
gLogger.info(
'GUIDs not found from POOL XML Catalogue (and were generated) for: %s'
% string.join(generated, ', '))
result = S_OK(pfnGUIDs)
result['directory'] = directory
result['generated'] = generated
def createInstance(self, instanceID="", createPublicIP=True):
"""
This creates a VM instance for the given boot image
and creates a context script, taken the given parameters.
Successful creation returns instance VM
Boots a new node on the OpenStack server defined by self.endpointConfig. The
'personality' of the node is done by self.imageConfig. Both variables are
defined on initialization phase.
The node name has the following format:
<bootImageName><contextMethod><time>
It boots the node. If IPpool is defined on the imageConfiguration, a floating
IP is created and assigned to the node.
:return: S_OK( ( nodeID, publicIP ) ) | S_ERROR
"""
if not instanceID:
instanceID = makeGuid()[:8]
self.parameters["VMUUID"] = instanceID
self.parameters["VMType"] = self.parameters.get("CEType", "OpenStack")
createNodeDict = {}
# Get the image object
if "ImageID" in self.parameters:
try:
image = self.__driver.get_image(self.parameters["ImageID"])
except BaseHTTPError as err:
if err.code == 404:
# Image not found
return S_ERROR("Image with ID %s not found" %
self.parameters["ImageID"])
return S_ERROR("Failed to get image for ID %s (%s)" %
(self.parameters["ImageID"], str(err)))
elif "ImageName" in self.parameters:
result = self.__getImageByName(self.parameters["ImageName"])
if not result["OK"]:
return result
image = result["Value"]
else:
return S_ERROR("No image specified")
createNodeDict["image"] = image
# Get the flavor object
if "FlavorID" in self.parameters:
flavor = self.__driver.ex_get_size(self.parameters["FlavorID"])
elif "FlavorName" in self.parameters:
result = self.__getFlavorByName(self.parameters["FlavorName"])
if not result["OK"]:
return result
flavor = result["Value"]
else:
return S_ERROR("No flavor specified")
self.flavor = flavor
createNodeDict["size"] = flavor
# Get security groups
# if 'ex_security_groups' in self.parameters:
# result = self.__getSecurityGroups( self.parameters['ex_security_groups'] )
# if not result[ 'OK' ]:
# self.log.error( result[ 'Message' ] )
# return result
# self.parameters['ex_security_groups'] = result[ 'Value' ]
result = self._createUserDataScript()
if not result["OK"]:
return result
createNodeDict["ex_userdata"] = result["Value"]
# Optional node contextualization parameters
for param in [
"ex_metadata", "ex_pubkey_path", "ex_keyname",
"ex_config_drive"
]:
if param in self.parameters:
createNodeDict[param] = self.parameters[param]
createNodeDict["name"] = "DIRAC_%s" % instanceID
# createNodeDict['ex_config_drive'] = True
self.log.verbose("Creating node:")
for key, value in createNodeDict.items():
self.log.verbose("%s: %s" % (key, value))
if "networks" in self.parameters:
result = self.getVMNetwork()
if not result["OK"]:
return result
createNodeDict["networks"] = result["Value"]
if "keyname" in self.parameters:
createNodeDict["ex_keyname"] = self.parameters["keyname"]
if "availability_zone" in self.parameters:
createNodeDict["ex_availability_zone"] = self.parameters[
"availability_zone"]
# Create the VM instance now
try:
vmNode = self.__driver.create_node(**createNodeDict)
except Exception as errmsg:
self.log.error("Exception in driver.create_node", errmsg)
return S_ERROR(errmsg)
publicIP = None
if createPublicIP:
# Wait until the node is running, otherwise getting public IP fails
try:
self.__driver.wait_until_running([vmNode], timeout=600)
result = self.assignFloatingIP(vmNode)
if result["OK"]:
publicIP = result["Value"]
else:
vmNode.destroy()
return result
except Exception as exc:
self.log.debug("Failed to wait node running %s" % str(exc))
vmNode.destroy()
return S_ERROR("Failed to wait until the node is Running")
return S_OK((vmNode, publicIP))
def submitJob(self, executableFile, proxy, numberOfJobs=1):
"""Method to submit job"""
self.log.verbose("Executable file path: %s" % executableFile)
if not os.access(executableFile, 5):
os.chmod(executableFile, stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH)
nProcessors = self.ceParameters.get("NumberOfProcessors", 1)
batchIDList = []
stampDict = {}
if numberOfJobs == 1:
jdlName, diracStamp = self.__writeJDL(executableFile, processors=nProcessors)
cmd = ["glite-ce-job-submit", "-n", "-a", "-N", "-r", "%s/%s" % (self.ceName, self.queue), "%s" % jdlName]
result = executeGridCommand(self.proxy, cmd, self.gridEnv)
os.unlink(jdlName)
if result["OK"]:
if result["Value"][0]:
# We have got a non-zero status code
errorString = "\n".join(result["Value"][1:]).strip()
return S_ERROR("Pilot submission failed with error: %s " % errorString)
pilotJobReference = result["Value"][1].strip()
if not pilotJobReference:
return S_ERROR("No pilot reference returned from the glite job submission command")
if not pilotJobReference.startswith("https"):
return S_ERROR("Invalid pilot reference %s" % pilotJobReference)
batchIDList.append(pilotJobReference)
stampDict[pilotJobReference] = diracStamp
else:
delegationID = makeGuid()
cmd = ["glite-ce-delegate-proxy", "-e", "%s" % self.ceName, "%s" % delegationID]
result = executeGridCommand(self.proxy, cmd, self.gridEnv)
if not result["OK"]:
self.log.error("Failed to delegate proxy", result["Message"])
return result
for _i in range(numberOfJobs):
jdlName, diracStamp = self.__writeJDL(executableFile, processors=nProcessors)
cmd = [
"glite-ce-job-submit",
"-n",
"-N",
"-r",
"%s/%s" % (self.ceName, self.queue),
"-D",
"%s" % delegationID,
"%s" % jdlName,
]
result = executeGridCommand(self.proxy, cmd, self.gridEnv)
os.unlink(jdlName)
if not result["OK"]:
self.log.error("General error in execution of glite-ce-job-submit command")
break
if result["Value"][0] != 0:
self.log.error("Error in glite-ce-job-submit command", result["Value"][1] + result["Value"][2])
break
pilotJobReference = result["Value"][1].strip()
if pilotJobReference and pilotJobReference.startswith("https"):
batchIDList.append(pilotJobReference)
stampDict[pilotJobReference] = diracStamp
else:
break
if batchIDList:
result = S_OK(batchIDList)
result["PilotStampDict"] = stampDict
else:
result = S_ERROR("No pilot references obtained from the glite job submission")
return result
__RCSID__ = "$Id$"
import DIRAC
from DIRAC.Core.Utilities.File import makeGuid
from DIRAC.Core.Base import Script
Script.setUsageMessage('\n'.join([
__doc__.split('\n')[1], 'Usage:',
' %s [option|cfgfile] ... File ...' % Script.scriptName, 'Arguments:',
' File: File Name'
]))
Script.parseCommandLine(ignoreErrors=False)
files = Script.getPositionalArgs()
if len(files) == 0:
Script.showHelp()
exitCode = 0
for file in files:
try:
md5 = makeGuid(file)
if md5:
print(file.rjust(100), md5.ljust(10))
else:
print('ERROR %s: Failed to get md5' % file)
exitCode = 2
except Exception as x:
print('ERROR %s: Failed to get md5' % file, str(x))
exitCode = 2
DIRAC.exit(exitCode)
def createInstance(self, instanceID=''):
if not instanceID:
instanceID = makeGuid()[:8]
self.parameters['VMUUID'] = instanceID
self.parameters['VMType'] = self.parameters.get('CEType', 'EC2')
actionArgs = ['--action', 'create']
actionArgs += ['--resource', 'compute']
# Image
if not "ImageID" in self.parameters and 'ImageName' in self.parameters:
result = self.__getImageByName(self.parameters['ImageName'])
if not result['OK']:
return result
imageId = result['Value']
elif "ImageID" in self.parameters:
result = self.__occiCommand([
'--action', 'describe', '--resource',
'os_tpl#%s' % self.parameters['ImageID']
])
if not result['OK']:
return S_ERROR(
"Failed to get image for ID %s" %
self.parameters['ImageID'], result['Message'])
imageId = self.parameters['ImageID']
else:
return S_ERROR('No image specified')
actionArgs += ['--mixin', 'os_tpl#%s' % imageId]
# Optional flavor name
if 'FlavorName' in self.parameters:
result = self.__occiCommand([
'--action', 'describe', '--resource',
'resource_tpl#%s' % self.parameters['FlavorName']
])
if not result['OK']:
return S_ERROR(
"Failed to get flavor %s" % self.parameters['FlavorName'],
result['Message'])
actionArgs += [
'--mixin',
'resource_tpl#%s' % self.parameters['FlavorName']
]
# Instance name
actionArgs += ['--attribute', 'occi.core.title=DIRAC_%s' % instanceID]
# Other params
for param in []:
if param in self.parameters:
actionArgs += ['--%s' % param, '%s' % self.parameters[param]]
self.log.info("Creating node:")
self.log.verbose(' '.join(actionArgs))
# User data
result = self._createUserDataScript()
if not result['OK']:
return result
# actionArgs += ['--context', 'user_data=%s' % str( result['Value'] )]
f = tempfile.NamedTemporaryFile(delete=False)
f.write(str(result['Value']))
f.close()
self.log.debug('Write user_data to temp file:', f.name)
actionArgs += ['--context', 'user_data=file://%s' % f.name]
# Create the VM instance now
result = self.__occiCommand(actionArgs)
os.unlink(f.name)
if not result['OK']:
errmsg = 'Error in rOCCI create instances: %s' % result['Message']
self.log.error(errmsg)
return S_ERROR(errmsg)
occiId = result['Value'].strip()
# Properties of the instance
nodeDict = {}
nodeDict['InstanceID'] = instanceID
result = self.__occiCommand(
['--action', 'describe', '--resource', occiId])
if result['OK']:
nodeInfo = json.loads(result['Value'])
try:
nodeDict['NumberOfCPUs'] = nodeInfo[0]['attributes']['occi'][
'compute']['cores']
nodeDict['RAM'] = nodeInfo[0]['attributes']['occi']['compute'][
'memory']
except Exception as e:
nodeDict['NumberOfCPUs'] = 1
else:
nodeDict['NumberOfCPUs'] = 1
return S_OK((occiId, nodeDict))
)
for i in fileUnlinkedList:
gLogger.notice(i)
gLogger.notice("---------- Total: %d unlinked files. ----------\n" %
len(fileUnlinkedList))
if deletionSwitch:
for i in fileDarkList:
g.unlink(HEAD0 + '/' + i)
gLogger.notice("%s deleted!" % (HEAD0 + '/' + i))
gLogger.notice("---------- All dark files deleted! ----------\n")
if registerOption != 'none':
if registerOption in ['dark', 'both']:
gLogger.notice("---------- Register dark files: ----------")
for i in fileDarkList:
fileStat = g.stat(HEAD0 + '/' + i)
fileSize = fileStat.st_size
fileGuid = makeGuid()
fileSum = g.checksum(HEAD0 + '/' + i, "ADLER32")
fileTuple = (i, HEAD0 + '/' + i, fileSize, SITE, fileGuid, fileSum)
dm.registerFile(fileTuple)
gLogger.notice(i + ' registered successfully!')
if registerOption in ['unlinked', 'both']:
gLogger.notice("---------- Register unlinked files: ----------")
for i in fileUnlinkedList:
fileTuple = (i, HEAD0 + '/' + i, SITE)
dm.registerReplica(fileTuple)
gLogger.notice(i + ' registered successfully!')
gLogger.notice("---------- Register finished ----------\n")
def _writeXRSL(self,
executableFile,
inputs=None,
outputs=None,
executables=None):
"""Create the JDL for submission
:param str executableFile: executable to wrap in a XRSL file
:param str/list inputs: path of the dependencies to include along with the executable
:param str/list outputs: path of the outputs that we want to get at the end of the execution
:param str/list executables: path to inputs that should have execution mode on the remote worker node
"""
diracStamp = makeGuid()[:8]
# Evaluate the number of processors to allocate
nProcessors = self.ceParameters.get("NumberOfProcessors", 1)
xrslMPAdditions = ""
if nProcessors and nProcessors > 1:
xrslMPAdditions = """
(count = %(processors)u)
(countpernode = %(processorsPerNode)u)
%(xrslMPExtraString)s
""" % {
"processors": nProcessors,
"processorsPerNode":
nProcessors, # This basically says that we want all processors on the same node
"xrslMPExtraString": self.xrslMPExtraString,
}
# Files that would need execution rights on the remote worker node
xrslExecutables = ""
if executables:
if not isinstance(executables, list):
executables = [executables]
xrslExecutables = "(executables=%s)" % " ".join(
map(os.path.basename, executables))
# Add them to the inputFiles
if not inputs:
inputs = []
if not isinstance(inputs, list):
inputs = [inputs]
inputs += executables
# Dependencies that have to be embedded along with the executable
xrslInputs = ""
if inputs:
if not isinstance(inputs, list):
inputs = [inputs]
for inputFile in inputs:
xrslInputs += '(%s "%s")' % (os.path.basename(inputFile),
inputFile)
# Output files to retrieve once the execution is complete
xrslOutputs = '("%s.out" "") ("%s.err" "")' % (diracStamp, diracStamp)
if outputs:
if not isinstance(outputs, list):
outputs = [outputs]
for outputFile in outputs:
xrslOutputs += '(%s "")' % (outputFile)
xrsl = """
&(executable="%(executable)s")
(inputFiles=(%(executable)s "%(executableFile)s") %(xrslInputAdditions)s)
(stdout="%(diracStamp)s.out")
(stderr="%(diracStamp)s.err")
(outputFiles=%(xrslOutputFiles)s)
(queue=%(queue)s)
%(xrslMPAdditions)s
%(xrslExecutables)s
%(xrslExtraString)s
""" % {
"executableFile": executableFile,
"executable": os.path.basename(executableFile),
"xrslInputAdditions": xrslInputs,
"diracStamp": diracStamp,
"queue": self.arcQueue,
"xrslOutputFiles": xrslOutputs,
"xrslMPAdditions": xrslMPAdditions,
"xrslExecutables": xrslExecutables,
"xrslExtraString": self.xrslExtraString,
}
return xrsl, diracStamp
def createInstance(self, instanceID=""):
if not instanceID:
instanceID = makeGuid()[:8]
self.parameters["VMUUID"] = instanceID
self.parameters["VMType"] = self.parameters.get("CEType", "EC2")
actionArgs = ["--action", "create"]
actionArgs += ["--resource", "compute"]
# Image
if "ImageID" in self.parameters and "ImageName" not in self.parameters:
result = self.__getImageByName(self.parameters["ImageName"])
if not result["OK"]:
return result
imageId = result["Value"]
elif "ImageID" in self.parameters:
result = self.__occiCommand([
"--action", "describe", "--resource",
"os_tpl#%s" % self.parameters["ImageID"]
])
if not result["OK"]:
return S_ERROR(
"Failed to get image for ID %s" %
self.parameters["ImageID"], result["Message"])
imageId = self.parameters["ImageID"]
else:
return S_ERROR("No image specified")
actionArgs += ["--mixin", "os_tpl#%s" % imageId]
# Optional flavor name
if "FlavorName" in self.parameters:
result = self.__occiCommand([
"--action", "describe", "--resource",
"resource_tpl#%s" % self.parameters["FlavorName"]
])
if not result["OK"]:
return S_ERROR(
"Failed to get flavor %s" % self.parameters["FlavorName"],
result["Message"])
actionArgs += [
"--mixin",
"resource_tpl#%s" % self.parameters["FlavorName"]
]
# Instance name
actionArgs += ["--attribute", "occi.core.title=DIRAC_%s" % instanceID]
# Other params
for param in []:
if param in self.parameters:
actionArgs += ["--%s" % param, "%s" % self.parameters[param]]
self.log.info("Creating node:")
self.log.verbose(" ".join(actionArgs))
# User data
result = self._createUserDataScript()
if not result["OK"]:
return result
# actionArgs += ['--context', 'user_data=%s' % str( result['Value'] )]
f = tempfile.NamedTemporaryFile(delete=False)
f.write(str(result["Value"]))
f.close()
self.log.debug("Write user_data to temp file:", f.name)
actionArgs += ["--context", "user_data=file://%s" % f.name]
# Create the VM instance now
result = self.__occiCommand(actionArgs)
os.unlink(f.name)
if not result["OK"]:
errmsg = "Error in rOCCI create instances: %s" % result["Message"]
self.log.error(errmsg)
return S_ERROR(errmsg)
occiId = result["Value"].strip()
# Properties of the instance
nodeDict = {}
nodeDict["InstanceID"] = instanceID
result = self.__occiCommand(
["--action", "describe", "--resource", occiId])
if result["OK"]:
nodeInfo = json.loads(result["Value"])
try:
nodeDict["NumberOfProcessors"] = nodeInfo[0]["attributes"][
"occi"]["compute"]["cores"]
nodeDict["RAM"] = nodeInfo[0]["attributes"]["occi"]["compute"][
"memory"]
except Exception as e:
nodeDict["NumberOfProcessors"] = 1
else:
nodeDict["NumberOfProcessors"] = 1
return S_OK((occiId, nodeDict))
def __writeSub(self, executable, nJobs):
""" Create the Sub File for submission
"""
self.log.debug("Working directory: %s " % self.workingDirectory)
##We randomize the location of the pilotoutput and log, because there are just too many of them
pre1 = makeGuid()[:3]
pre2 = makeGuid()[:3]
mkDir(os.path.join(self.workingDirectory, pre1, pre2))
initialDirPrefix = "%s/%s" % (pre1, pre2)
self.log.debug("InitialDir: %s" %
os.path.join(self.workingDirectory, initialDirPrefix))
self.log.debug("ExtraSubmitString:\n### \n %s \n###" %
self.extraSubmitString)
fd, name = tempfile.mkstemp(suffix='.sub',
prefix='HTCondorCE_',
dir=self.workingDirectory)
subFile = os.fdopen(fd, 'w')
executable = os.path.join(self.workingDirectory, executable)
localScheddOptions = """
ShouldTransferFiles = YES
WhenToTransferOutput = ON_EXIT_OR_EVICT
""" if self.useLocalSchedd else ""
targetUniverse = "grid" if self.useLocalSchedd else "vanilla"
sub = """
executable = %(executable)s
universe = %(targetUniverse)s
use_x509userproxy = true
output = $(Cluster).$(Process).out
error = $(Cluster).$(Process).err
log = $(Cluster).$(Process).log
environment = "HTCONDOR_JOBID=$(Cluster).$(Process)"
initialdir = %(initialDir)s
grid_resource = condor %(ceName)s %(ceName)s:9619
transfer_output_files = ""
%(localScheddOptions)s
kill_sig=SIGTERM
%(extraString)s
Queue %(nJobs)s
""" % dict(
executable=executable,
nJobs=nJobs,
ceName=self.ceName,
extraString=self.extraSubmitString,
initialDir=os.path.join(self.workingDirectory, initialDirPrefix),
localScheddOptions=localScheddOptions,
targetUniverse=targetUniverse,
)
subFile.write(sub)
subFile.close()
return name
def test_files(self):
######################################################
#
# First create a file to use for remaining tests
#
lfn = '/lhcb/test/unit-test/testfile.%s' % time.time()
pfn = 'protocol://host:port/storage/path%s' % lfn
size = 10000000
se = 'DIRAC-storage'
guid = makeGuid()
fileTuple = (lfn,pfn,size,se,guid)
res = self.lfc.addFile(fileTuple)
print res
self.assert_(res['OK'])
self.assert_(res['Value'].has_key('Successful'))
self.assert_(res['Value'].has_key('Failed'))
self.assert_(res['Value']['Successful'].has_key(lfn))
self.assert_(res['Value']['Successful'][lfn])
######################################################
#
# Test the creation of links using the test file
#
targetLfn = lfn
linkName = '/lhcb/test/unit-test/testlink.%s' % time.time()
linkTuple = (linkName,targetLfn)
res = self.lfc.createLink(linkTuple)
self.assert_(res['OK'])
self.assert_(res['Value'].has_key('Successful'))
self.assert_(res['Value'].has_key('Failed'))
self.assert_(res['Value']['Successful'].has_key(linkName))
self.assert_(res['Value']['Successful'][linkName])
######################################################
#
# Test the recognition of links works (with file it should fail)
#
res = self.lfc.isLink(targetLfn)
self.assert_(res['OK'])
self.assert_(res['Value'].has_key('Successful'))
self.assert_(res['Value'].has_key('Failed'))
self.assert_(res['Value']['Successful'].has_key(targetLfn))
self.assertFalse(res['Value']['Successful'][targetLfn])
######################################################
#
# Test the recognition of links works (with link it shouldn't fail)
#
res = self.lfc.isLink(linkName)
self.assert_(res['OK'])
self.assert_(res['Value'].has_key('Successful'))
self.assert_(res['Value'].has_key('Failed'))
self.assert_(res['Value']['Successful'].has_key(linkName))
self.assert_(res['Value']['Successful'][linkName])
######################################################
#
# Test the resolution of links
#
res = self.lfc.readLink(linkName)
self.assert_(res['OK'])
self.assert_(res['Value'].has_key('Successful'))
self.assert_(res['Value'].has_key('Failed'))
self.assert_(res['Value']['Successful'].has_key(linkName))
self.assertEqual(res['Value']['Successful'][linkName],targetLfn)
######################################################
#
# Test the removal of links
#
res = self.lfc.removeLink(linkName)
self.assert_(res['OK'])
self.assert_(res['Value'].has_key('Successful'))
self.assert_(res['Value'].has_key('Failed'))
self.assert_(res['Value']['Successful'].has_key(linkName))
self.assert_(res['Value']['Successful'][linkName])
######################################################
#
# Test the recognition of non existant links
#
res = self.lfc.isLink(linkName)
self.assert_(res['OK'])
self.assert_(res['Value'].has_key('Successful'))
self.assert_(res['Value'].has_key('Failed'))
self.assert_(res['Value']['Failed'].has_key(linkName))
self.assertEqual(res['Value']['Failed'][linkName],'No such file or directory')
######################################################
#
# Add a replica to the test file
#
replicaPfn = 'protocol://replicaHost:port/storage/path%s' % lfn
replicase = 'Replica-storage'
replicaTuple = (lfn,replicaPfn,replicase,0)
res = self.lfc.addReplica(replicaTuple)
self.assert_(res['OK'])
self.assert_(res['Value'].has_key('Successful'))
self.assert_(res['Value'].has_key('Failed'))
self.assert_(res['Value']['Successful'].has_key(lfn))
self.assert_(res['Value']['Successful'][lfn])
######################################################
#
# Ensure the file exists (quite redundant here)
#
res = self.lfc.exists(lfn)
self.assert_(res['OK'])
self.assert_(res['Value'].has_key('Successful'))
self.assert_(res['Value'].has_key('Failed'))
self.assert_(res['Value']['Successful'].has_key(lfn))
self.assert_(res['Value']['Successful'][lfn])
######################################################
#
# Test the recognition of files
#
res = self.lfc.isFile(lfn)
self.assert_(res['OK'])
self.assert_(res['Value'].has_key('Successful'))
self.assert_(res['Value'].has_key('Failed'))
self.assert_(res['Value']['Successful'].has_key(lfn))
self.assert_(res['Value']['Successful'][lfn])
######################################################
#
# Test obtaining the file metadata
#
res = self.lfc.getFileMetadata(lfn)
self.assert_(res['OK'])
self.assert_(res['Value'].has_key('Successful'))
self.assert_(res['Value'].has_key('Failed'))
self.assert_(res['Value']['Successful'].has_key(lfn))
metadataDict = res['Value']['Successful'][lfn]
self.assertEqual(metadataDict['Status'],'-')
self.assertEqual( metadataDict['ChecksumType'], '' )
self.assertEqual(metadataDict['Checksum'],'')
self.assertEqual(metadataDict['Size'],10000000)
######################################################
#
# Test obtaining the file replicas
#
res = self.lfc.getReplicas(lfn)
self.assert_(res['OK'])
self.assert_(res['Value'].has_key('Successful'))
self.assert_(res['Value'].has_key('Failed'))
self.assert_(res['Value']['Successful'].has_key(lfn))
self.assert_(res['Value']['Successful'][lfn].has_key('DIRAC-storage'))
self.assertEqual(res['Value']['Successful'][lfn]['DIRAC-storage'],pfn)
self.assert_(res['Value']['Successful'][lfn].has_key('Replica-storage'))
self.assertEqual(res['Value']['Successful'][lfn]['Replica-storage'],replicaPfn)
######################################################
#
# Test obtaining the replica status for the master replica
#
replicaTuple = (lfn,pfn,se)
res = self.lfc.getReplicaStatus(replicaTuple)
self.assert_(res['OK'])
self.assert_(res['Value'].has_key('Successful'))
self.assert_(res['Value'].has_key('Failed'))
self.assert_(res['Value']['Successful'].has_key(lfn))
self.assertEqual(res['Value']['Successful'][lfn],'U')
######################################################
#
# Test setting the replica status for the master replica
#
replicaTuple = (lfn,pfn,se,'C')
res = self.lfc.setReplicaStatus(replicaTuple)
self.assert_(res['OK'])
self.assert_(res['Value'].has_key('Successful'))
self.assert_(res['Value'].has_key('Failed'))
self.assert_(res['Value']['Successful'].has_key(lfn))
self.assert_(res['Value']['Successful'][lfn])
######################################################
#
# Ensure the changing of the replica status worked
#
replicaTuple = (lfn,pfn,se)
res = self.lfc.getReplicaStatus(replicaTuple)
self.assert_(res['OK'])
self.assert_(res['Value'].has_key('Successful'))
self.assert_(res['Value'].has_key('Failed'))
self.assert_(res['Value']['Successful'].has_key(lfn))
self.assertEqual(res['Value']['Successful'][lfn],'C')
######################################################
#
# Test the change of storage element works
#
newse = 'New-storage'
newToken = 'SpaceToken'
replicaTuple = (lfn,pfn,newse,newToken)
res = self.lfc.setReplicaHost(replicaTuple)
self.assert_(res['OK'])
self.assert_(res['Value'].has_key('Successful'))
self.assert_(res['Value'].has_key('Failed'))
self.assert_(res['Value']['Successful'].has_key(lfn))
self.assert_(res['Value']['Successful'][lfn])
######################################################
#
# Check the change of storage element works
#
res = self.lfc.getReplicas(lfn)
self.assert_(res['OK'])
self.assert_(res['Value'].has_key('Successful'))
self.assert_(res['Value'].has_key('Failed'))
self.assert_(res['Value']['Successful'].has_key(lfn))
self.assert_(res['Value']['Successful'][lfn].has_key(newse))
self.assertEqual(res['Value']['Successful'][lfn][newse],pfn)
self.assert_(res['Value']['Successful'][lfn].has_key(replicase))
self.assertEqual(res['Value']['Successful'][lfn][replicase],replicaPfn)
######################################################
#
# Test getting the file size
#
res = self.lfc.getFileSize(lfn)
self.assert_(res['OK'])
self.assert_(res['Value'].has_key('Successful'))
self.assert_(res['Value'].has_key('Failed'))
self.assert_(res['Value']['Successful'].has_key(lfn))
self.assertEqual(res['Value']['Successful'][lfn],size)
######################################################
#
# Test the recognition of directories
#
dir = os.path.dirname(lfn)
res = self.lfc.isDirectory(dir)
self.assert_(res['OK'])
self.assert_(res['Value'].has_key('Successful'))
self.assert_(res['Value'].has_key('Failed'))
self.assert_(res['Value']['Successful'].has_key(dir))
self.assert_(res['Value']['Successful'][dir])
######################################################
#
# Test getting replicas for directories
#
res = self.lfc.getDirectoryReplicas(dir)
self.assert_(res['OK'])
self.assert_(res['Value'].has_key('Successful'))
self.assert_(res['Value'].has_key('Failed'))
self.assert_(res['Value']['Successful'].has_key(dir))
self.assert_(res['Value']['Successful'][dir].has_key(lfn))
self.assert_(res['Value']['Successful'][dir][lfn].has_key(newse))
self.assertEqual(res['Value']['Successful'][dir][lfn][newse],pfn)
self.assert_(res['Value']['Successful'][dir][lfn].has_key(replicase))
self.assertEqual(res['Value']['Successful'][dir][lfn][replicase],replicaPfn)
######################################################
#
# Test listing directories
#
res = self.lfc.listDirectory(dir)
self.assert_(res['OK'])
self.assert_(res['Value'].has_key('Successful'))
self.assert_(res['Value'].has_key('Failed'))
self.assert_(res['Value']['Successful'].has_key(dir))
self.assertEqual(res['Value']['Successful'][dir],[lfn])
######################################################
#
# Test getting directory metadata
#
res = self.lfc.getDirectoryMetadata(dir)
self.assert_(res['OK'])
self.assert_(res['Value'].has_key('Successful'))
self.assert_(res['Value'].has_key('Failed'))
self.assert_(res['Value']['Successful'].has_key(dir))
self.assert_(res['Value']['Successful'][dir].has_key('NumberOfSubPaths'))
self.assertEqual(res['Value']['Successful'][dir]['NumberOfSubPaths'],1)
self.assert_(res['Value']['Successful'][dir].has_key('CreationDate'))
######################################################
#
# Test getting directory size
#
res = self.lfc.getDirectorySize(dir)
self.assert_(res['OK'])
self.assert_(res['Value'].has_key('Successful'))
self.assert_(res['Value'].has_key('Failed'))
self.assert_(res['Value']['Successful'].has_key(dir))
self.assert_(res['Value']['Successful'][dir].has_key('Files'))
self.assertEqual(res['Value']['Successful'][dir]['Files'],1)
self.assert_(res['Value']['Successful'][dir].has_key('TotalSize'))
self.assertEqual(res['Value']['Successful'][dir]['TotalSize'],size)
self.assert_(res['Value']['Successful'][dir].has_key('SiteFiles'))
self.assert_(res['Value']['Successful'][dir]['SiteFiles'].has_key(newse))
self.assertEqual(res['Value']['Successful'][dir]['SiteFiles'][newse],1)
self.assert_(res['Value']['Successful'][dir]['SiteFiles'].has_key(replicase))
self.assertEqual(res['Value']['Successful'][dir]['SiteFiles'][replicase],1)
self.assert_(res['Value']['Successful'][dir].has_key('SiteUsage'))
self.assert_(res['Value']['Successful'][dir]['SiteUsage'].has_key(newse))
self.assertEqual(res['Value']['Successful'][dir]['SiteUsage'][newse],size)
self.assert_(res['Value']['Successful'][dir]['SiteUsage'].has_key(replicase))
self.assertEqual(res['Value']['Successful'][dir]['SiteUsage'][replicase],size)
######################################################
#
# Test creation of directories
#
newDir = '%s/%s' % (dir,'testDir')
res = self.lfc.createDirectory(newDir)
self.assert_(res['OK'])
self.assert_(res['Value'].has_key('Successful'))
self.assert_(res['Value'].has_key('Failed'))
self.assert_(res['Value']['Successful'].has_key(newDir))
self.assert_(res['Value']['Successful'][newDir])
######################################################
#
# Test removal of directories
#
res = self.lfc.removeDirectory(newDir)
self.assert_(res['OK'])
self.assert_(res['Value'].has_key('Successful'))
self.assert_(res['Value'].has_key('Failed'))
self.assert_(res['Value']['Successful'].has_key(newDir))
self.assert_(res['Value']['Successful'][newDir])
######################################################
#
# Test removal of replicas
#
res = self.lfc.listDirectory(dir)
self.assert_(res['OK'])
self.assert_(res['Value'].has_key('Successful'))
self.assert_(res['Value'].has_key('Failed'))
self.assert_(res['Value']['Successful'].has_key(dir))
lfnsToDelete = res['Value']['Successful'][dir]
res = self.lfc.getReplicas(lfnsToDelete)
self.assert_(res['OK'])
self.assert_(res['Value'].has_key('Successful'))
self.assert_(res['Value'].has_key('Failed'))
replicas = res['Value']['Successful']
replicaTupleList = []
for lfn in replicas.keys():
for se in replicas[lfn].keys():
replicaTuple = (lfn,replicas[lfn][se],se)
replicaTupleList.append(replicaTuple)
res = self.lfc.removeReplica(replicaTupleList)
self.assert_(res['OK'])
self.assert_(res['Value'].has_key('Successful'))
self.assert_(res['Value'].has_key('Failed'))
self.assertFalse(res['Value']['Failed'])
######################################################
#
# Test removal of files
#
res = self.lfc.removeFile(lfnsToDelete)
self.assert_(res['OK'])
self.assert_(res['Value'].has_key('Successful'))
self.assert_(res['Value'].has_key('Failed'))
self.assertFalse(res['Value']['Failed'])
def testMakeGuid( self ): """ makeGuid tests """ # no filename - fake guid produced self.assertEqual( checkGuid( makeGuid() ), True , "fake guid for inexisting file" ) # using this python file self.assertEqual( checkGuid( makeGuid( os.path.abspath(__file__) ) ), True, "guid for FileTestCase.py file" )
def _submitJob(self, executableFile, numberOfJobs):
""" Submit prepared executable
"""
# Copy the executable
executable = os.path.basename(executableFile)
executable = os.path.join(self.executableArea, executable)
try:
shutil.copy(executableFile, executable)
except Exception, x:
self.log.warn('Failed copying executable', x)
return S_ERROR(x)
jobStamps = []
for i in range(numberOfJobs):
jobStamps.append(makeGuid()[:8])
jobStamp = '#'.join(jobStamps)
subOptions = urllib.quote(self.submitOptions)
cmdTuple = [
self.finalScript, 'submit_job', executable, self.batchOutput,
self.batchError, self.workArea,
str(numberOfJobs), self.infoArea, jobStamp, self.execQueue,
subOptions
]
self.log.verbose('CE submission command: %s' % ' '.join(cmdTuple))
result = systemCall(120, cmdTuple)
def submitJob(self, executableFile, proxy, numberOfJobs=1, processors=1):
""" Method to submit job
"""
self.log.verbose("Executable file path: %s" % executableFile)
if not os.access(executableFile, 5):
os.chmod(
executableFile, stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP
| stat.S_IROTH | stat.S_IXOTH)
batchIDList = []
stampDict = {}
if numberOfJobs == 1:
jdlName, diracStamp = self.__writeJDL(executableFile,
processors=processors)
cmd = [
'glite-ce-job-submit', '-n', '-a', '-N', '-r',
'%s/%s' % (self.ceName, self.queue),
'%s' % jdlName
]
result = executeGridCommand(self.proxy, cmd, self.gridEnv)
os.unlink(jdlName)
if result['OK']:
if result['Value'][0]:
# We have got a non-zero status code
errorString = '\n'.join(result['Value'][1:]).strip()
return S_ERROR('Pilot submission failed with error: %s ' %
errorString)
pilotJobReference = result['Value'][1].strip()
if not pilotJobReference:
return S_ERROR(
'No pilot reference returned from the glite job submission command'
)
if not pilotJobReference.startswith('https'):
return S_ERROR('Invalid pilot reference %s' %
pilotJobReference)
batchIDList.append(pilotJobReference)
stampDict[pilotJobReference] = diracStamp
else:
delegationID = makeGuid()
cmd = [
'glite-ce-delegate-proxy', '-e',
'%s' % self.ceName,
'%s' % delegationID
]
result = executeGridCommand(self.proxy, cmd, self.gridEnv)
if not result['OK']:
self.log.error('Failed to delegate proxy', result['Message'])
return result
for _i in range(numberOfJobs):
jdlName, diracStamp = self.__writeJDL(executableFile,
processors=processors)
cmd = [
'glite-ce-job-submit', '-n', '-N', '-r',
'%s/%s' % (self.ceName, self.queue), '-D',
'%s' % delegationID,
'%s' % jdlName
]
result = executeGridCommand(self.proxy, cmd, self.gridEnv)
os.unlink(jdlName)
if not result['OK']:
self.log.error(
"General error in execution of glite-ce-job-submit command"
)
break
if result['Value'][0] != 0:
self.log.error("Error in glite-ce-job-submit command",
result['Value'][1] + result['Value'][2])
break
pilotJobReference = result['Value'][1].strip()
if pilotJobReference and pilotJobReference.startswith('https'):
batchIDList.append(pilotJobReference)
stampDict[pilotJobReference] = diracStamp
else:
break
if batchIDList:
result = S_OK(batchIDList)
result['PilotStampDict'] = stampDict
else:
result = S_ERROR(
'No pilot references obtained from the glite job submission')
return result
def submitJob(self, executableFile, proxy, numberOfJobs=1):
""" Method to submit job
"""
self.log.verbose("Executable file path: %s" % executableFile)
if not os.access(executableFile, 5):
os.chmod(executableFile, 0755)
batchIDList = []
stampDict = {}
if numberOfJobs == 1:
jdlName, diracStamp = self.__writeJDL(executableFile)
cmd = [
'glite-ce-job-submit', '-n', '-a', '-N', '-r',
'%s/%s' % (self.ceName, self.queue),
'%s' % jdlName
]
result = executeGridCommand(self.proxy, cmd, self.gridEnv)
if result['OK']:
if result['Value'][0]:
# We have got a non-zero status code
return S_ERROR('Pilot submission failed with error: %s ' %
result['Value'][2].strip())
pilotJobReference = result['Value'][1].strip()
if not pilotJobReference:
return S_ERROR(
'No pilot reference returned from the glite job submission command'
)
batchIDList.append(pilotJobReference)
stampDict[pilotJobReference] = diracStamp
os.unlink(jdlName)
else:
delegationID = makeGuid()
cmd = [
'glite-ce-delegate-proxy', '-e',
'%s' % self.ceName,
'%s' % delegationID
]
result = executeGridCommand(self.proxy, cmd, self.gridEnv)
if not result['OK']:
self.log.error('Failed to delegate proxy: %s' %
result['Message'])
return result
for i in range(numberOfJobs):
jdlName, diracStamp = self.__writeJDL(executableFile)
cmd = [
'glite-ce-job-submit', '-n', '-N', '-r',
'%s/%s' % (self.ceName, self.queue), '-D',
'%s' % delegationID,
'%s' % jdlName
]
result = executeGridCommand(self.proxy, cmd, self.gridEnv)
os.unlink(jdlName)
if not result['OK']:
break
if result['Value'][0] != 0:
break
pilotJobReference = result['Value'][1].strip()
if pilotJobReference:
batchIDList.append(pilotJobReference)
stampDict[pilotJobReference] = diracStamp
else:
break
os.unlink(executableFile)
if batchIDList:
result = S_OK(batchIDList)
result['PilotStampDict'] = stampDict
else:
result = S_ERROR(
'No pilot references obtained from the glite job submission')
return result
def processDir(initPath,recursive=False,host=None,fcInit=None,dfcInit=None):
""" Process one directory, possibly recursively
"""
global dirCount, fileCount, globalStart, dnCache, roleCache, outputFile
fc = fcInit
if not fc:
fc = LcgFileCatalogClient.LcgFileCatalogClient( host=host )
#fc = FileCatalogClient()
dfc = dfcInit
if not dfc:
#dfc = LcgFileCatalogClient.LcgFileCatalogClient( host=host )
dfc = FileCatalogClient()
start = time.time()
initPath = initPath.rstrip("/")
resultList = fc.listDirectory(initPath,True)
#print resultList
#return S_OK()
lfc_time = (time.time() - start)
s = time.time()
print resultList
if resultList['OK']:
# Add directories
if resultList['Value']['Failed']:
return S_ERROR("Path %s failed: %s" % (initPath,resultList['Value']['Failed'][initPath]))
dirDict = resultList['Value']['Successful'][initPath]['SubDirs']
paths = {}
for path,info in dirDict.items():
print info
paths[path] = {}
paths[path]['Mode'] = info['Mode']
owner = getUserNameAndGroup( info )
#owner = getDNandRole( info )
if owner:
paths[path]['Owner'] = owner
#return S_OK()
p_dirs = time.time() - s
s = time.time()
nDir = len(paths)
if nDir:
print "Adding %d directories in %s" % (nDir,initPath)
result = dfc.createDirectory(paths)
if not result['OK']:
print "Error adding directories:",result['Message']
dirCount += nDir
print "Total directories added", dirCount
e_dirs = time.time() - s
# Add files
s = time.time()
fileDict = resultList['Value']['Successful'][initPath]['Files']
lfns = {}
for lfn,info in fileDict.items():
lfns[lfn] = {}
lfns[lfn]['Size'] = info['MetaData']['Size']
lfns[lfn]['Checksum'] = info['MetaData']['Checksum']
if 'GUID' in info['MetaData']:
lfns[lfn]['GUID'] = info['MetaData']['GUID']
else:
lfns[lfn]['GUID'] = makeGuid()
lfns[lfn]['Mode'] = info['MetaData']['Mode']
lfns[lfn]['PFN'] = ''
owner = getUserNameAndGroup( info['MetaData'] )
if owner:
lfns[lfn]['Owner'] = owner
if info['Replicas']:
seList = info['Replicas'].keys()
lfns[lfn]['SE'] = seList
p_files = time.time() - s
s = time.time()
nFile = len(lfns)
nRep = 0
if nFile:
for lfn in lfns:
if 'SE' in lfns[lfn]:
nRep += len(lfns[lfn]['SE'])
print "Adding %d files in %s" % (nFile,initPath)
#print lfns
done = False
count = 0
error = False
while not done:
count += 1
result = dfc.addFile(lfns)
#print result
if not result['OK']:
print "Error adding files %d:" % count,result['Message']
if count > 10:
print "Completely failed path", initPath
break
error = True
time.sleep(2)
elif error:
print "Successfully added files on retry %d" % count
done = True
else:
done = True
fileCount += nFile
print "Total files added", fileCount
e_files = time.time() - s
dfc_time = time.time() - start - lfc_time
total_time = time.time() - globalStart
format = "== %s: time lfc/dfc %.2f/%.2f, files %d/%d, dirs %d/%d, time: %.2f/%.2f/%.2f/%.2f %.2f \n"
outputFile = open('lfc_dfc.out','a')
outputFile.write( format % (initPath,lfc_time,dfc_time,nFile,fileCount,nDir,dirCount,p_dirs,e_dirs,p_files,e_files,total_time) )
outputFile.close()
# print format % (initPath,lfc_time,dfc_time,nFile,fileCount,nDir,dirCount,p_dirs,e_dirs,p_files,e_files,total_time)
# Go into directories
if recursive:
for path in paths:
result = processDir(path,True,host=host,fcInit=fc,dfcInit=dfc)
if result['OK']:
nFile += result['Value'].get('NumberOfFiles',0)
nDir += result['Value'].get('NumberOfDirectories',0)
nRep += result['Value'].get('NumberOfReplicas',0)
resultDict = {}
resultDict['NumberOfFiles'] = nFile
resultDict['NumberOfDirectories'] = nDir
resultDict['NumberOfReplicas'] = nRep
resultDict['Path'] = initPath
return S_OK(resultDict)
def submitJob(self, executableFile, proxy=None, numberOfJobs=1):
if not os.access(executableFile, 5):
os.chmod(
executableFile, stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP
| stat.S_IROTH | stat.S_IXOTH)
# if no proxy is supplied, the executable can be submitted directly
# otherwise a wrapper script is needed to get the proxy to the execution node
# The wrapper script makes debugging more complicated and thus it is
# recommended to transfer a proxy inside the executable if possible.
if self.proxy and not proxy:
proxy = self.proxy
if proxy:
self.log.verbose("Setting up proxy for payload")
wrapperContent = bundleProxy(executableFile, proxy)
name = writeScript(wrapperContent, os.getcwd())
submitFile = name
else: # no proxy
submitFile = executableFile
jobStamps = []
for _i in range(numberOfJobs):
jobStamps.append(makeGuid()[:8])
batchDict = {
"Executable": submitFile,
"NJobs": numberOfJobs,
"OutputDir": self.batchOutput,
"ErrorDir": self.batchError,
"SubmitOptions": self.submitOptions,
"ExecutionContext": self.execution,
"JobStamps": jobStamps,
"Queue": self.queue,
"WholeNode": self.wholeNode,
"NumberOfProcessors": self.numberOfProcessors,
"NumberOfNodes": self.numberOfNodes,
"NumberOfGPUs": self.numberOfGPUs,
}
resultSubmit = self.batchSystem.submitJob(**batchDict)
if proxy:
os.remove(submitFile)
if resultSubmit["Status"] == 0:
self.submittedJobs += len(resultSubmit["Jobs"])
# jobIDs = [ self.ceType.lower()+'://'+self.ceName+'/'+_id for _id in resultSubmit['Jobs'] ]
# FIXME: It would be more proper to fix pilotCommands.__setFlavour where 'ssh' is hardcoded than
# making this illogical fix, but there is no good way for pilotCommands to know its origin ceType.
# So, the jobIDs here need to start with 'ssh', not ceType, to accomodate
# them to those hardcoded in pilotCommands.__setFlavour
batchSystemName = self.batchSystem.__class__.__name__.lower()
jobIDs = [
"ssh" + batchSystemName + "://" + self.ceName + "/" + _id
for _id in resultSubmit["Jobs"]
]
result = S_OK(jobIDs)
if "ExecutableToKeep" in resultSubmit:
result["ExecutableToKeep"] = resultSubmit["ExecutableToKeep"]
else:
result = S_ERROR(resultSubmit["Message"])
return result
def _submitJobToHost(self, executableFile, numberOfJobs, host=None):
"""Submit prepared executable to the given host"""
ssh = SSH(host=host, parameters=self.ceParameters)
# Copy the executable
submitFile = os.path.join(self.executableArea,
os.path.basename(executableFile))
result = ssh.scpCall(30,
executableFile,
submitFile,
postUploadCommand="chmod +x %s" % submitFile)
if not result["OK"]:
return result
jobStamps = []
for _i in range(numberOfJobs):
jobStamps.append(makeGuid()[:8])
numberOfProcessors = self.ceParameters.get("NumberOfProcessors", 1)
wholeNode = self.ceParameters.get("WholeNode", False)
# numberOfNodes is treated as a string as it can contain values such as "2-4"
# where 2 would represent the minimum number of nodes to allocate, and 4 the maximum
numberOfNodes = self.ceParameters.get("NumberOfNodes", "1")
self.numberOfGPUs = self.ceParameters.get("NumberOfGPUs")
# Collect command options
commandOptions = {
"Executable": submitFile,
"NJobs": numberOfJobs,
"SubmitOptions": self.submitOptions,
"JobStamps": jobStamps,
"WholeNode": wholeNode,
"NumberOfProcessors": numberOfProcessors,
"NumberOfNodes": numberOfNodes,
"Preamble": self.preamble,
"NumberOfGPUs": self.numberOfGPUs,
"Account": self.account,
}
if host:
commandOptions["SSHNodeHost"] = host
resultCommand = self.__executeHostCommand("submitJob",
commandOptions,
ssh=ssh,
host=host)
if not resultCommand["OK"]:
return resultCommand
result = resultCommand["Value"]
if result["Status"] != 0:
return S_ERROR("Failed job submission: %s" % result["Message"])
else:
batchIDs = result["Jobs"]
if batchIDs:
batchSystemName = self.batchSystem.__class__.__name__.lower()
if host is None:
jobIDs = [
"%s%s://%s/%s" % (self.ceType.lower(), batchSystemName,
self.ceName, _id) for _id in batchIDs
]
else:
jobIDs = [
"%s%s://%s/%s/%s" %
(self.ceType.lower(), batchSystemName, self.ceName,
host, _id) for _id in batchIDs
]
else:
return S_ERROR("No jobs IDs returned")
result = S_OK(jobIDs)
self.submittedJobs += len(batchIDs)
return result
def test_files(self):
######################################################
#
# First create a file to use for remaining tests
#
lfn = '/lhcb/test/unit-test/testfile.%s' % time.time()
pfn = 'protocol://host:port/storage/path%s' % lfn
size = 10000000
se = 'DIRAC-storage'
guid = makeGuid()
fileTuple = (lfn, pfn, size, se, guid)
res = self.lfc.addFile(fileTuple)
self.assert_(res['OK'])
self.assert_(res['Value'].has_key('Successful'))
self.assert_(res['Value'].has_key('Failed'))
self.assert_(res['Value']['Successful'].has_key(lfn))
self.assert_(res['Value']['Successful'][lfn])
######################################################
#
# Test the creation of links using the test file
#
targetLfn = lfn
linkName = '/lhcb/test/unit-test/testlink.%s' % time.time()
linkTuple = (linkName, targetLfn)
res = self.lfc.createLink(linkTuple)
self.assert_(res['OK'])
self.assert_(res['Value'].has_key('Successful'))
self.assert_(res['Value'].has_key('Failed'))
self.assert_(res['Value']['Successful'].has_key(linkName))
self.assert_(res['Value']['Successful'][linkName])
######################################################
#
# Test the recognition of links works (with file it should fail)
#
res = self.lfc.isLink(targetLfn)
print res
self.assert_(res['OK'])
self.assert_(res['Value'].has_key('Successful'))
self.assert_(res['Value'].has_key('Failed'))
self.assert_(res['Value']['Successful'].has_key(targetLfn))
self.assertFalse(res['Value']['Successful'][targetLfn])
######################################################
#
# Test the recognition of links works (with link it shouldn't fail)
#
res = self.lfc.isLink(linkName)
self.assert_(res['OK'])
self.assert_(res['Value'].has_key('Successful'))
self.assert_(res['Value'].has_key('Failed'))
self.assert_(res['Value']['Successful'].has_key(linkName))
self.assert_(res['Value']['Successful'][linkName])
######################################################
#
# Test the resolution of links
#
res = self.lfc.readLink(linkName)
self.assert_(res['OK'])
self.assert_(res['Value'].has_key('Successful'))
self.assert_(res['Value'].has_key('Failed'))
self.assert_(res['Value']['Successful'].has_key(linkName))
self.assertEqual(res['Value']['Successful'][linkName], targetLfn)
######################################################
#
# Test the removal of links
#
res = self.lfc.removeLink(linkName)
self.assert_(res['OK'])
self.assert_(res['Value'].has_key('Successful'))
self.assert_(res['Value'].has_key('Failed'))
self.assert_(res['Value']['Successful'].has_key(linkName))
self.assert_(res['Value']['Successful'][linkName])
######################################################
#
# Test the recognition of non existant links
#
res = self.lfc.isLink(linkName)
self.assert_(res['OK'])
self.assert_(res['Value'].has_key('Successful'))
self.assert_(res['Value'].has_key('Failed'))
self.assert_(res['Value']['Failed'].has_key(linkName))
self.assertEqual(res['Value']['Failed'][linkName],
'No such file or directory')
######################################################
#
# Add a replica to the test file
#
replicaPfn = 'protocol://replicaHost:port/storage/path%s' % lfn
replicase = 'Replica-storage'
replicaTuple = (lfn, replicaPfn, replicase, 0)
res = self.lfc.addReplica(replicaTuple)
self.assert_(res['OK'])
self.assert_(res['Value'].has_key('Successful'))
self.assert_(res['Value'].has_key('Failed'))
self.assert_(res['Value']['Successful'].has_key(lfn))
self.assert_(res['Value']['Successful'][lfn])
######################################################
#
# Ensure the file exists (quite redundant here)
#
res = self.lfc.exists(lfn)
self.assert_(res['OK'])
self.assert_(res['Value'].has_key('Successful'))
self.assert_(res['Value'].has_key('Failed'))
self.assert_(res['Value']['Successful'].has_key(lfn))
self.assert_(res['Value']['Successful'][lfn])
######################################################
#
# Test the recognition of files
#
res = self.lfc.isFile(lfn)
self.assert_(res['OK'])
self.assert_(res['Value'].has_key('Successful'))
self.assert_(res['Value'].has_key('Failed'))
self.assert_(res['Value']['Successful'].has_key(lfn))
self.assert_(res['Value']['Successful'][lfn])
######################################################
#
# Test obtaining the file metadata
#
res = self.lfc.getFileMetadata(lfn)
self.assert_(res['OK'])
self.assert_(res['Value'].has_key('Successful'))
self.assert_(res['Value'].has_key('Failed'))
self.assert_(res['Value']['Successful'].has_key(lfn))
metadataDict = res['Value']['Successful'][lfn]
self.assertEqual(metadataDict['Status'], '-')
self.assertEqual(metadataDict['CheckSumType'], '')
self.assertEqual(metadataDict['Checksum'], '')
self.assertEqual(metadataDict['Size'], 10000000)
######################################################
#
# Test obtaining the file replicas
#
res = self.lfc.getReplicas(lfn)
self.assert_(res['OK'])
self.assert_(res['Value'].has_key('Successful'))
self.assert_(res['Value'].has_key('Failed'))
self.assert_(res['Value']['Successful'].has_key(lfn))
self.assert_(res['Value']['Successful'][lfn].has_key('DIRAC-storage'))
self.assertEqual(res['Value']['Successful'][lfn]['DIRAC-storage'], pfn)
self.assert_(
res['Value']['Successful'][lfn].has_key('Replica-storage'))
self.assertEqual(res['Value']['Successful'][lfn]['Replica-storage'],
replicaPfn)
######################################################
#
# Test obtaining the replica status for the master replica
#
replicaTuple = (lfn, pfn, se)
res = self.lfc.getReplicaStatus(replicaTuple)
self.assert_(res['OK'])
self.assert_(res['Value'].has_key('Successful'))
self.assert_(res['Value'].has_key('Failed'))
self.assert_(res['Value']['Successful'].has_key(lfn))
self.assertEqual(res['Value']['Successful'][lfn], 'U')
######################################################
#
# Test setting the replica status for the master replica
#
replicaTuple = (lfn, pfn, se, 'C')
res = self.lfc.setReplicaStatus(replicaTuple)
self.assert_(res['OK'])
self.assert_(res['Value'].has_key('Successful'))
self.assert_(res['Value'].has_key('Failed'))
self.assert_(res['Value']['Successful'].has_key(lfn))
self.assert_(res['Value']['Successful'][lfn])
######################################################
#
# Ensure the changing of the replica status worked
#
replicaTuple = (lfn, pfn, se)
res = self.lfc.getReplicaStatus(replicaTuple)
self.assert_(res['OK'])
self.assert_(res['Value'].has_key('Successful'))
self.assert_(res['Value'].has_key('Failed'))
self.assert_(res['Value']['Successful'].has_key(lfn))
self.assertEqual(res['Value']['Successful'][lfn], 'C')
######################################################
#
# Test the change of storage element works
#
newse = 'New-storage'
newToken = 'SpaceToken'
replicaTuple = (lfn, pfn, newse, newToken)
res = self.lfc.setReplicaHost(replicaTuple)
self.assert_(res['OK'])
self.assert_(res['Value'].has_key('Successful'))
self.assert_(res['Value'].has_key('Failed'))
self.assert_(res['Value']['Successful'].has_key(lfn))
self.assert_(res['Value']['Successful'][lfn])
######################################################
#
# Check the change of storage element works
#
res = self.lfc.getReplicas(lfn)
self.assert_(res['OK'])
self.assert_(res['Value'].has_key('Successful'))
self.assert_(res['Value'].has_key('Failed'))
self.assert_(res['Value']['Successful'].has_key(lfn))
self.assert_(res['Value']['Successful'][lfn].has_key(newse))
self.assertEqual(res['Value']['Successful'][lfn][newse], pfn)
self.assert_(res['Value']['Successful'][lfn].has_key(replicase))
self.assertEqual(res['Value']['Successful'][lfn][replicase],
replicaPfn)
######################################################
#
# Test getting the file size
#
res = self.lfc.getFileSize(lfn)
self.assert_(res['OK'])
self.assert_(res['Value'].has_key('Successful'))
self.assert_(res['Value'].has_key('Failed'))
self.assert_(res['Value']['Successful'].has_key(lfn))
self.assertEqual(res['Value']['Successful'][lfn], size)
######################################################
#
# Test the recognition of directories
#
dir = os.path.dirname(lfn)
res = self.lfc.isDirectory(dir)
self.assert_(res['OK'])
self.assert_(res['Value'].has_key('Successful'))
self.assert_(res['Value'].has_key('Failed'))
self.assert_(res['Value']['Successful'].has_key(dir))
self.assert_(res['Value']['Successful'][dir])
######################################################
#
# Test getting replicas for directories
#
res = self.lfc.getDirectoryReplicas(dir)
self.assert_(res['OK'])
self.assert_(res['Value'].has_key('Successful'))
self.assert_(res['Value'].has_key('Failed'))
self.assert_(res['Value']['Successful'].has_key(dir))
self.assert_(res['Value']['Successful'][dir].has_key(lfn))
self.assert_(res['Value']['Successful'][dir][lfn].has_key(newse))
self.assertEqual(res['Value']['Successful'][dir][lfn][newse], pfn)
self.assert_(res['Value']['Successful'][dir][lfn].has_key(replicase))
self.assertEqual(res['Value']['Successful'][dir][lfn][replicase],
replicaPfn)
######################################################
#
# Test listing directories
#
res = self.lfc.listDirectory(dir)
self.assert_(res['OK'])
self.assert_(res['Value'].has_key('Successful'))
self.assert_(res['Value'].has_key('Failed'))
self.assert_(res['Value']['Successful'].has_key(dir))
self.assertEqual(res['Value']['Successful'][dir], [lfn])
######################################################
#
# Test getting directory metadata
#
res = self.lfc.getDirectoryMetadata(dir)
self.assert_(res['OK'])
self.assert_(res['Value'].has_key('Successful'))
self.assert_(res['Value'].has_key('Failed'))
self.assert_(res['Value']['Successful'].has_key(dir))
self.assert_(
res['Value']['Successful'][dir].has_key('NumberOfSubPaths'))
self.assertEqual(res['Value']['Successful'][dir]['NumberOfSubPaths'],
1)
self.assert_(res['Value']['Successful'][dir].has_key('CreationDate'))
######################################################
#
# Test getting directory size
#
res = self.lfc.getDirectorySize(dir)
self.assert_(res['OK'])
self.assert_(res['Value'].has_key('Successful'))
self.assert_(res['Value'].has_key('Failed'))
self.assert_(res['Value']['Successful'].has_key(dir))
self.assert_(res['Value']['Successful'][dir].has_key('Files'))
self.assertEqual(res['Value']['Successful'][dir]['Files'], 1)
self.assert_(res['Value']['Successful'][dir].has_key('TotalSize'))
self.assertEqual(res['Value']['Successful'][dir]['TotalSize'], size)
self.assert_(res['Value']['Successful'][dir].has_key('SiteFiles'))
self.assert_(
res['Value']['Successful'][dir]['SiteFiles'].has_key(newse))
self.assertEqual(res['Value']['Successful'][dir]['SiteFiles'][newse],
1)
self.assert_(
res['Value']['Successful'][dir]['SiteFiles'].has_key(replicase))
self.assertEqual(
res['Value']['Successful'][dir]['SiteFiles'][replicase], 1)
self.assert_(res['Value']['Successful'][dir].has_key('SiteUsage'))
self.assert_(
res['Value']['Successful'][dir]['SiteUsage'].has_key(newse))
self.assertEqual(res['Value']['Successful'][dir]['SiteUsage'][newse],
size)
self.assert_(
res['Value']['Successful'][dir]['SiteUsage'].has_key(replicase))
self.assertEqual(
res['Value']['Successful'][dir]['SiteUsage'][replicase], size)
######################################################
#
# Test creation of directories
#
newDir = '%s/%s' % (dir, 'testDir')
res = self.lfc.createDirectory(newDir)
self.assert_(res['OK'])
self.assert_(res['Value'].has_key('Successful'))
self.assert_(res['Value'].has_key('Failed'))
self.assert_(res['Value']['Successful'].has_key(newDir))
self.assert_(res['Value']['Successful'][newDir])
######################################################
#
# Test removal of directories
#
res = self.lfc.removeDirectory(newDir)
self.assert_(res['OK'])
self.assert_(res['Value'].has_key('Successful'))
self.assert_(res['Value'].has_key('Failed'))
self.assert_(res['Value']['Successful'].has_key(newDir))
self.assert_(res['Value']['Successful'][newDir])
######################################################
#
# Test removal of replicas
#
res = self.lfc.listDirectory(dir)
self.assert_(res['OK'])
self.assert_(res['Value'].has_key('Successful'))
self.assert_(res['Value'].has_key('Failed'))
self.assert_(res['Value']['Successful'].has_key(dir))
lfnsToDelete = res['Value']['Successful'][dir]
res = self.lfc.getReplicas(lfnsToDelete)
self.assert_(res['OK'])
self.assert_(res['Value'].has_key('Successful'))
self.assert_(res['Value'].has_key('Failed'))
replicas = res['Value']['Successful']
replicaTupleList = []
for lfn in replicas.keys():
for se in replicas[lfn].keys():
replicaTuple = (lfn, replicas[lfn][se], se)
replicaTupleList.append(replicaTuple)
res = self.lfc.removeReplica(replicaTupleList)
self.assert_(res['OK'])
self.assert_(res['Value'].has_key('Successful'))
self.assert_(res['Value'].has_key('Failed'))
self.assertFalse(res['Value']['Failed'])
######################################################
#
# Test removal of files
#
res = self.lfc.removeFile(lfnsToDelete)
self.assert_(res['OK'])
self.assert_(res['Value'].has_key('Successful'))
self.assert_(res['Value'].has_key('Failed'))
self.assertFalse(res['Value']['Failed'])
