def __call__( self ):
""" call me maybe """
# # counter for failed files
failedFiles = 0
# # catalog(s) to use
catalogs = self.operation.Catalog
if catalogs:
catalogs = [ cat.strip() for cat in catalogs.split( ',' ) ]
dm = DataManager( catalogs = catalogs )
# # get waiting files
waitingFiles = self.getWaitingFilesList()
# # loop over files
for opFile in waitingFiles:
gMonitor.addMark( "RegisterAtt", 1 )
# # get LFN
lfn = opFile.LFN
# # and others
fileTuple = ( lfn , opFile.PFN, opFile.Size, self.operation.targetSEList[0], opFile.GUID, opFile.Checksum )
# # call DataManager
registerFile = dm.registerFile( fileTuple )
# # check results
if not registerFile["OK"] or lfn in registerFile["Value"]["Failed"]:
gMonitor.addMark( "RegisterFail", 1 )
# self.dataLoggingClient().addFileRecord( lfn, "RegisterFail", ','.join( catalogs ) if catalogs else "all catalogs", "", "RegisterFile" )
reason = str( registerFile.get( "Message", registerFile.get( "Value", {} ).get( "Failed", {} ).get( lfn, 'Unknown' ) ) )
errorStr = "failed to register LFN"
opFile.Error = "%s: %s" % ( errorStr, reason )
if 'GUID already registered' in reason:
opFile.Status = 'Failed'
self.log.error( errorStr, "%s: %s" % ( lfn, reason ) )
elif 'File already registered with no replicas' in reason:
self.log.warn( errorStr, "%s: %s, will remove it and retry" % ( lfn, reason ) )
dm.removeFile( lfn )
else:
self.log.warn( errorStr, "%s: %s" % ( lfn, reason ) )
failedFiles += 1
else:
gMonitor.addMark( "RegisterOK", 1 )
# self.dataLoggingClient().addFileRecord( lfn, "Register", ','.join( catalogs ) if catalogs else "all catalogs", "", "RegisterFile" )
self.log.verbose( "file %s has been registered at %s" % ( lfn, ','.join( catalogs ) if catalogs else "all catalogs" ) )
opFile.Status = "Done"
# # final check
if failedFiles:
self.log.warn( "all files processed, %s files failed to register" % failedFiles )
self.operation.Error = "some files failed to register"
return S_ERROR( self.operation.Error )
return S_OK()
def __call__( self ):
""" call me maybe """
# # counter for failed files
failedFiles = 0
# # catalog(s) to use
catalogs = self.operation.Catalog
if catalogs:
catalogs = [ cat.strip() for cat in catalogs.split( ',' ) ]
dm = DataManager( catalogs = catalogs )
# # get waiting files
waitingFiles = self.getWaitingFilesList()
# # loop over files
for opFile in waitingFiles:
gMonitor.addMark( "RegisterAtt", 1 )
# # get LFN
lfn = opFile.LFN
# # and others
fileTuple = ( lfn , opFile.PFN, opFile.Size, self.operation.targetSEList[0], opFile.GUID, opFile.Checksum )
# # call DataManager
registerFile = dm.registerFile( fileTuple )
# # check results
if not registerFile["OK"] or lfn in registerFile["Value"]["Failed"]:
gMonitor.addMark( "RegisterFail", 1 )
# self.dataLoggingClient().addFileRecord( lfn, "RegisterFail", ','.join( catalogs ) if catalogs else "all catalogs", "", "RegisterFile" )
reason = str( registerFile.get( "Message", registerFile.get( "Value", {} ).get( "Failed", {} ).get( lfn, 'Unknown' ) ) )
errorStr = "failed to register LFN"
opFile.Error = "%s: %s" % ( errorStr, reason )
if 'GUID already registered' in reason:
opFile.Status = 'Failed'
self.log.error( errorStr, "%s: %s" % ( lfn, reason ) )
elif 'File already registered with no replicas' in reason:
self.log.warn( errorStr, "%s: %s, will remove it and retry" % ( lfn, reason ) )
dm.removeFile( lfn )
else:
self.log.warn( errorStr, "%s: %s" % ( lfn, reason ) )
failedFiles += 1
else:
gMonitor.addMark( "RegisterOK", 1 )
# self.dataLoggingClient().addFileRecord( lfn, "Register", ','.join( catalogs ) if catalogs else "all catalogs", "", "RegisterFile" )
self.log.verbose( "file %s has been registered at %s" % ( lfn, ','.join( catalogs ) if catalogs else "all catalogs" ) )
opFile.Status = "Done"
# # final check
if failedFiles:
self.log.warn( "all files processed, %s files failed to register" % failedFiles )
self.operation.Error = "some files failed to register"
return S_ERROR( self.operation.Error )
return S_OK()
def uploadProcessListToFileCatalog(self, path_to_process_list, appVersion):
"""Upload the new processList to the FileCatalog
"""
from ILCDIRAC.Core.Utilities.FileUtils import upload
from DIRAC.DataManagementSystem.Client.DataManager import DataManager
from DIRAC import gConfig, exit as dexit
datMan = DataManager()
LOG.notice("Removing process list from file catalog" + path_to_process_list)
res = datMan.removeFile(path_to_process_list)
if not res['OK']:
LOG.error("Could not remove process list from file catalog, do it by hand")
dexit(2)
LOG.notice("Done removing process list from file catalog")
res = upload(os.path.dirname(path_to_process_list) + "/", self.location )
if not res['OK']:
LOG.error("something went wrong in the copy")
dexit(2)
LOG.notice("Putting process list to local processlist directory")
localprocesslistpath = gConfig.getOption("/LocalSite/ProcessListPath", "")
if localprocesslistpath['Value']:
try:
localSvnRepo = "/afs/cern.ch/eng/clic/software/whizard/whizard_195/"
shutil.copy(self.location, localSvnRepo) ## because it does not make a difference if we hardcode it here or in ${DIRAC}/etc/dirac.cfg, yours truly APS, JFS
except OSError, err:
LOG.error("Copy of process list to %s failed with error %s!" % (localSvnRepo, str(err)))
try:
subprocess.call( ["svn","ci", os.path.join( localSvnRepo, os.path.basename(localprocesslistpath['Value'] )), "-m'Process list for whizard version %s'" % appVersion ], shell=False )
except OSError, err:
LOG.error("Commit failed! Error: %s" % str(err))
def uploadProcessListToFileCatalog(self, path_to_process_list, appVersion):
"""Upload the new processList to the FileCatalog
"""
from ILCDIRAC.Core.Utilities.FileUtils import upload
from DIRAC.DataManagementSystem.Client.DataManager import DataManager
from DIRAC import gConfig, exit as dexit
datMan = DataManager()
gLogger.notice("Removing process list from file catalog" + path_to_process_list)
res = datMan.removeFile(path_to_process_list)
if not res['OK']:
gLogger.error("Could not remove process list from file catalog, do it by hand")
dexit(2)
gLogger.notice("Done removing process list from file catalog")
res = upload(os.path.dirname(path_to_process_list) + "/", self.location )
if not res['OK']:
gLogger.error("something went wrong in the copy")
dexit(2)
gLogger.notice("Putting process list to local processlist directory")
localprocesslistpath = gConfig.getOption("/LocalSite/ProcessListPath", "")
if localprocesslistpath['Value']:
try:
localSvnRepo = "/afs/cern.ch/eng/clic/software/whizard/whizard_195/"
shutil.copy(self.location, localSvnRepo) ## because it does not make a difference if we hardcode it here or in ${DIRAC}/etc/dirac.cfg, yours truly APS, JFS
except OSError, err:
gLogger.error("Copy of process list to %s failed with error %s!" % (localSvnRepo, str(err)))
try:
subprocess.call( ["svn","ci", os.path.join( localSvnRepo, os.path.basename(localprocesslistpath['Value'] )), "-m'Process list for whizard version %s'" % appVersion ], shell=False )
except OSError, err:
gLogger.error("Commit failed! Error: %s" % str(err))
def _remove_file(self, lfn):
dm = DataManager()
res = dm.removeFile([lfn])
if not res['OK']:
gLogger.error("Failed to remove data", res['Message'])
return res
if lfn in res['Value']['Successful']:
return S_OK(res['Value']['Successful'])
return S_ERROR(res['Value']['Failed'])
def setUp( self ):
super( IntegrationTest, self ).setUp()
dm = DataManager()
res = dm.removeFile( ['/lhcb/testCfg/testVer/LOG/00012345/0006/00012345_00067890.tar',
'/lhcb/testCfg/testVer/SIM/00012345/0006/00012345_00067890_1.sim'],
force = True )
if not res['OK']:
print "Could not remove files", res['Message']
exit( 1 )
def setUp( self ):
super( FailingUserJobTestCase, self ).setUp()
dm = DataManager()
res = dm.removeFile( ['/lhcb/testCfg/testVer/LOG/00012345/0006/00012345_00067890.tar',
'/lhcb/testCfg/testVer/SIM/00012345/0006/00012345_00067890_1.sim'],
force = True )
if not res['OK']:
print("Could not remove files", res['Message'])
exit( 1 )
def setUp(self):
super(IntegrationTest, self).setUp()
dm = DataManager()
res = dm.removeFile([
'/lhcb/testCfg/testVer/LOG/00012345/0006/00012345_00067890.tar',
'/lhcb/testCfg/testVer/SIM/00012345/0006/00012345_00067890_1.sim'
],
force=True)
if not res['OK']:
print "Could not remove files", res['Message']
exit(1)
def setUp(self):
super(FailingUserJobTestCase, self).setUp()
dm = DataManager()
res = dm.removeFile(
[
"/lhcb/testCfg/testVer/LOG/00012345/0006/00012345_00067890.tar",
"/lhcb/testCfg/testVer/SIM/00012345/0006/00012345_00067890_1.sim",
],
force=True,
)
if not res["OK"]:
print("Could not remove files", res["Message"])
exit(1)
def setUp(self):
super(FailingUserJobTestCase, self).setUp()
dm = DataManager()
res = dm.removeFile(
[
"/lhcb/testCfg/testVer/LOG/00012345/0006/00012345_00067890.tar",
"/lhcb/testCfg/testVer/SIM/00012345/0006/00012345_00067890_1.sim",
],
force=True,
)
if not res["OK"]:
print "Could not remove files", res["Message"]
exit(1)
def main():
# Registering arguments will automatically add their description to the help menu
Script.registerArgument(("LocalFile: Path to local file containing LFNs",
"LFN: Logical File Names"))
Script.registerArgument(["LFN: Logical File Names"], mandatory=False)
Script.parseCommandLine()
import os
import DIRAC
from DIRAC import gLogger
first, lfns = Script.getPositionalArgs(group=True)
if os.path.exists(first):
with open(first, "r") as inputFile:
string = inputFile.read()
lfns.extend([lfn.strip() for lfn in string.splitlines()])
else:
lfns.insert(0, first)
from DIRAC.Core.Utilities.List import breakListIntoChunks
from DIRAC.DataManagementSystem.Client.DataManager import DataManager
dm = DataManager()
errorReasons = {}
successfullyRemoved = 0
for lfnList in breakListIntoChunks(lfns, 100):
res = dm.removeFile(lfnList)
if not res["OK"]:
gLogger.error("Failed to remove data", res["Message"])
DIRAC.exit(-2)
for lfn, r in res["Value"]["Failed"].items():
reason = str(r)
if reason not in errorReasons:
errorReasons[reason] = []
errorReasons[reason].append(lfn)
successfullyRemoved += len(res["Value"]["Successful"])
for reason, lfns in errorReasons.items():
gLogger.notice("Failed to remove %d files with error: %s" %
(len(lfns), reason))
if successfullyRemoved > 0:
gLogger.notice("Successfully removed %d files" % successfullyRemoved)
DIRAC.exit(0)
def main():
Script.parseCommandLine()
import os
import DIRAC
from DIRAC import gLogger
args = Script.getPositionalArgs()
lfns = []
for inputFileName in args:
if os.path.exists(inputFileName):
inputFile = open(inputFileName, 'r')
string = inputFile.read()
inputFile.close()
lfns.extend([lfn.strip() for lfn in string.splitlines()])
else:
lfns.append(inputFileName)
from DIRAC.Core.Utilities.List import breakListIntoChunks
from DIRAC.DataManagementSystem.Client.DataManager import DataManager
dm = DataManager()
errorReasons = {}
successfullyRemoved = 0
for lfnList in breakListIntoChunks(lfns, 100):
res = dm.removeFile(lfnList)
if not res['OK']:
gLogger.error("Failed to remove data", res['Message'])
DIRAC.exit(-2)
for lfn, r in res['Value']['Failed'].items():
reason = str(r)
if reason not in errorReasons:
errorReasons[reason] = []
errorReasons[reason].append(lfn)
successfullyRemoved += len(res['Value']['Successful'])
for reason, lfns in errorReasons.items():
gLogger.notice("Failed to remove %d files with error: %s" %
(len(lfns), reason))
if successfullyRemoved > 0:
gLogger.notice("Successfully removed %d files" % successfullyRemoved)
DIRAC.exit(0)
class RemoveInputData(ModuleBase):
""" Remove the input data: to be used when Merging things
"""
def __init__(self):
super(RemoveInputData, self).__init__()
self.datMan = DataManager()
self.log = gLogger.getSubLogger("RemoveInputData")
self.enable = True
def applicationSpecificInputs(self):
self.enable = self.step_commons.get('Enable', self.enable)
if not isinstance(self.enable, bool):
self.log.warn(
'Enable flag set to non-boolean value %s, setting to False' %
self.enable)
self.enable = False
return S_OK('Parameters resolved')
def execute(self):
""" Remove the input data, and pass by failover in case of failure
"""
self.result = self.resolveInputVariables()
if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
self.log.verbose(
'Workflow status = %s, step status = %s' %
(self.workflowStatus['OK'], self.stepStatus['OK']))
return S_OK('No removal of input data attempted')
if not self.enable:
self.log.info("Would have tried to remove %s" % self.InputData)
return S_OK('Input Data Removed')
try:
#Try to remove the file list with failover if necessary
failover = []
self.log.info('Attempting rm.removeFile("%s")' % (self.InputData))
result = self.datMan.removeFile(self.InputData)
self.log.verbose(result)
if not result['OK']:
self.log.error('Could not remove files with message:\n"%s"\n\
Will set removal requests just in case.' % (result['Message']))
failover = self.InputData
try:
if result['Value']['Failed']:
failureDict = result['Value']['Failed']
if failureDict:
self.log.info(
'Not all files were successfully removed, see "LFN : reason" below\n%s'
% (failureDict))
failover = failureDict.keys()
except KeyError:
self.log.error(
'Setting files for removal request to be the input data: %s'
% self.InputData)
failover = self.InputData
for lfn in failover:
self.__setFileRemovalRequest(lfn)
return S_OK('Input Data Removed')
except OSError as e:
self.log.exception(e)
return S_ERROR(e)
return S_OK()
def __setFileRemovalRequest(self, lfn):
""" Sets a removal request for a file including all replicas.
"""
self.log.info('Setting file removal request for %s' % lfn)
self.addRemovalRequests([lfn])
from DIRAC import S_OK, S_ERROR, gLogger, exit
from DIRAC.DataManagementSystem.Client.DataManager import DataManager
lfn = args[0]
pfn = args[1]
se = args[2]
exit_code = 0
log = ''
dm = DataManager()
start = time.time()
result = dm.removeFile(lfn)
result = dm.putAndRegister(lfn, pfn, se)
uploadTime = time.time() - start
if result['OK']:
log += 'Succeed to upload file to SE %s.\n' % se
log += 'Upload Time : %ss\n' % uploadTime
start = time.time()
result = dm.getReplica(lfn, se, tempfile.gettempdir())
downloadTime = time.time() - start
if result['OK']:
log += 'Succeed to download file from SE %s.\n' % se
log += 'Download Time : %ss\n' % downloadTime
else:
exit_code = 1
log += 'Failed to download file from SE %s : %s\n' % (
class SETest( TestBase ):
"""
SETest is used to test the availability of SE.
"""
def __init__( self, args = None, apis = None ):
super( SETest, self ).__init__( args, apis )
self.__lfnPath = '/bes/user/z/zhaoxh/'
self.__testFile = 'test.dat'
self.__localPath = '/tmp/'
if 'DataManager' in self.apis:
self.dm = self.apis[ 'DataManager' ]
else:
self.dm = DataManager()
def doTest( self, elementDict ):
"""
Test upload and download for specified SE.
"""
elementName = elementDict[ 'ElementName' ]
testFilePath = self.__localPath + self.__testFile
if not os.path.exists( testFilePath ) or not os.path.isfile( testFilePath ):
f = open( testFilePath, 'w' )
f.write( 'hello' )
f.close()
status = 'OK'
log = ''
lfnPath = self.__lfnPath + elementName + '-' + self.__testFile
submissionTime = datetime.utcnow().replace( microsecond = 0 )
LOCK.acquire()
start = time.time()
result = self.dm.putAndRegister( lfnPath, testFilePath, elementName )
uploadTime = time.time() - start
if result[ 'OK' ]:
log += 'Succeed to upload file to SE %s.\n' % elementName
log += 'Upload Time : %ss\n' % uploadTime
start = time.time()
result = self.dm.getReplica( lfnPath, elementName, self.__localPath )
downloadTime = time.time() - start
if result[ 'OK' ]:
log += 'Succeed to download file from SE %s.\n' % elementName
log += 'Download Time : %ss\n' % downloadTime
else:
status = 'Bad'
log += 'Failed to download file from SE %s : %s\n' % ( elementName, result[ 'Message' ] )
result = self.dm.removeFile( lfnPath )
if result[ 'OK' ]:
log += 'Succeed to delete file from SE %s.\n' % elementName
else:
log += 'Faile to delete file from SE %s : %s\n' % ( elementName, result[ 'Message' ] )
else:
status = 'Bad'
log += 'Failed to upload file to SE %s : %s\n' % ( elementName, result[ 'Message' ] )
LOCK.release()
completionTime = datetime.utcnow().replace( microsecond = 0 )
applicationTime = ( completionTime - submissionTime ).total_seconds()
result = { 'Result' : { 'Status' : status,
'Log' : log,
'SubmissionTime' : submissionTime,
'CompletionTime' : completionTime,
'ApplicationTime' : applicationTime },
'Finish' : True }
localFile = self.__localPath + elementName +'-' + self.__testFile
if os.path.exists( localFile ) and os.path.isfile( localFile ):
os.remove( localFile )
return S_OK( result )
class MoveInFC(ModuleBase):
'''
classdocs
'''
def __init__(self):
'''
Constructor
'''
super(MoveInFC, self).__init__()
self.enable = False
self.STEP_NUMBER = ''
self.applicationName = 'MoveInFC'
self.repMan = DataManager()
self.listoutput = {}
self.outputpath = ''
def applicationSpecificInputs(self):
""" Resolve all input variables for the module here.
:return: S_OK()
"""
if not len(self.InputFile) and len(self.InputData):
for files in self.InputData:
self.InputFile.append(files)
if 'listoutput' in self.step_commons:
self.listoutput = self.step_commons['listoutput'][0]
return S_OK()
def execute(self):
""" Run the module
"""
result = self.resolveInputVariables()
if not result['OK']:
return result
self.result = S_OK()
if not self.applicationLog:
self.result = S_ERROR( 'No Log file provided' )
if not self.result['OK']:
return self.result
if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
LOG.verbose('Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK']))
return S_OK('%s should not proceed as previous step did not end properly'% self.applicationName)
### Now remove the files in the FC
lfns = self.InputFile
##Check that all the files are here:
res = resolveIFpaths(lfns)
if not res['OK']:
LOG.error(res['Message'])
return S_ERROR("Failed to find a file locally")
#All files are here and available
paths = res['Value']
localpaths = []
for inputfile in paths:
basename = os.path.basename(inputfile)
locname = os.path.join(os.getcwd(),basename)
if not locname == inputfile:
try:
shutil.copy(inputfile, locname)
except shutil.Error:
LOG.error("Failed to copy file locally, will have to stop")
return S_ERROR("Failed copy to local directory")
localpaths.append(locname)
try:
os.unlink(inputfile)
except OSError:
LOG.warn("Failed to remove initial file, increased \
disk space usage")
#all the files are in the run directory
#get all metadata, ancestor/daughter relations, etc. for all the files
#Update the listoutput
if self.listoutput:
outputlist = []
for localFile in localpaths:
item = {}
item['outputFile'] = localFile
item['outputPath'] = self.listoutput['outputPath']
item['outputDataSE'] = self.listoutput['outputDataSE']
outputlist.append(item)
if self.enable:
self.step_commons['listoutput'] = outputlist
else:
LOG.info("listoutput would have been ", outputlist)
## Make sure the path contains / at the end as we are going to
## concatenate final path and local files
if not self.outputpath[-1]=='/':
self.outputpath += "/"
if 'ProductionOutputData' in self.workflow_commons:
file_list = ";".join([self.outputpath+name for name in [os.path.basename(fin) for fin in localpaths]])
if self.enable:
self.workflow_commons['ProductionOutputData'] = file_list
else:
LOG.info("ProductionOutputData would have been", file_list)
#Now remove them
if self.enable:
res = self.repMan.removeFile(lfns, force=True)
if not res['OK']:
LOG.error("Failed to remove the files")
self.setApplicationStatus("Failed to remove the file")
#return S_ERROR("Failed to remove the files")
else:
LOG.info("Would have removed: ", "%s" % str(lfns))
## Now the files are not on the storage anymore, they exist only locally. We can hope
## that the job will not be killed between now and the time the UploadOutputData module
## is called
return self.finalStatusReport(0)
class Prod3DataManager(object) :
""" Manage data and meta-data
"""
def __init__(self,catalogs=['DIRACFileCatalog']):
""" Constructor
"""
self.setupCatalogClient( catalogs )
self.printCatalogConfig( catalogs )
self.setupDataManagerClient( catalogs )
def setupCatalogClient( self, catalogs ):
""" Init FileCatalog client
Ideally we would like to use only FileCatalog but it doesn't work for setMetadata
because the returned value has a wrong format. So use of FileCatalogClient instead
"""
self.fc = FileCatalog( catalogs )
self.fcc = FileCatalogClient()
def printCatalogConfig( self, catalogs ):
""" Dumps catalog configuration
"""
for catalog in catalogs:
res = self.fc._getCatalogConfigDetails( catalog )
DIRAC.gLogger.info( 'CatalogConfigDetails:', res )
def setupDataManagerClient( self, catalogs ):
""" Init DataManager client
"""
self.dm = DataManager( catalogs )
def _getSEList( self, SEType = 'ProductionOutputs', DataType = 'SimtelProd' ):
""" get from CS the list of available SE for data upload
"""
opsHelper = Operations()
optionName = os.path.join( SEType, DataType )
SEList = opsHelper.getValue( optionName , [] )
SEList = List.randomize( SEList )
DIRAC.gLogger.notice( 'List of %s SE: %s ' % ( SEType, SEList ) )
# # Check if the local SE is in the list. If yes try it first by reversing list order
localSEList = []
res = getSEsForSite( DIRAC.siteName() )
if res['OK']:
localSEList = res['Value']
retainedlocalSEList = []
for localSE in localSEList:
if localSE in SEList:
DIRAC.gLogger.notice( 'The local Storage Element is an available SE: ', localSE )
retainedlocalSEList.append( localSE )
SEList.remove( localSE )
SEList = retainedlocalSEList + SEList
if len( SEList ) == 0:
return DIRAC.S_ERROR( 'Error in building SEList' )
return DIRAC.S_OK( SEList )
def _checkemptydir( self, path ):
""" check that the directory is not empty
"""
if len ( glob.glob( path ) ) == 0:
error = 'Empty directory: %s' % path
return DIRAC.S_ERROR( error )
else:
return DIRAC.S_OK()
def _getRunPath( self, filemetadata ):
""" format path to string with 1 digit precision
run_number is taken from filemetadata
filemetadata can be a dict or the run_number itself
"""
if type( filemetadata ) == type( dict() ):
run_number = int( filemetadata['runNumber'] )
else:
run_number = int( filemetadata )
run_numberMod = run_number % 1000
runpath = '%03dxxx' % ( ( run_number - run_numberMod ) / 1000 )
return runpath
def _formatPath( self, path ):
""" format path to string with 1 digit precision
"""
if type( path ) == float or type( path ) == int:
path = '%.1f' % path
return str( path )
def createTarLogFiles ( self, inputpath, tarname ):
""" create tar of log and histogram files
"""
tarmode = 'w:gz'
tar = tarfile.open( tarname, tarmode )
for subdir in ['Log/*', 'Histograms/*']:
logdir = os.path.join( inputpath, subdir )
res = self._checkemptydir( logdir )
if not res['OK']:
return res
for localfile in glob.glob( logdir ):
tar.add( localfile, arcname = localfile.split( '/' )[-1] )
tar.close()
return DIRAC.S_OK()
def createMDStructure( self, metadata, metadatafield, basepath, program_category ):
""" create meta data structure
"""
# ## Add metadata fields to the DFC
mdfield = json.loads( metadatafield )
for key, value in mdfield.items():
res = self.fc.addMetadataField( key, value )
if not res['OK']:
return res
# ## Create the directory structure
md = json.loads( metadata , object_pairs_hook = collections.OrderedDict )
path = basepath
process_program = program_category + '_prog'
for key, value in collections.OrderedDict( ( k, md[k] ) for k in ( 'site', 'particle', process_program ) if k in md ).items():
path = os.path.join( path, self._formatPath( value ) )
res = self.fc.createDirectory( path )
if not res['OK']:
return res
# Set directory metadata for each subdir: 'site', 'particle', 'process_program'
res = self.fcc.setMetadata( path, {key:value} )
if not res['OK']:
return res
# Create the TransformationID subdir and set MD
# ## Get the TransformationID
if os.environ.has_key( 'JOBID' ):
jobID = os.environ['JOBID']
dirac = Dirac()
res = dirac.getJobJDL( jobID )
TransformationID = '0000'
if res['Value'].has_key( 'TransformationID' ):
TransformationID = res['Value']['TransformationID']
path = os.path.join( path, TransformationID )
res = self.fc.createDirectory( path )
if not res['OK']:
return res
process_program_version = process_program + '_version'
res = self.fcc.setMetadata( path, dict( ( k, md[k] ) for k in ( 'phiP', 'thetaP', 'array_layout', process_program_version ) if k in md ) )
if not res['OK']:
return res
# Create the Data and Log subdirs and set MD
Transformation_path = path
for subdir in ['Data', 'Log']:
path = os.path.join( Transformation_path, subdir )
res = self.fc.createDirectory( path )
if not res['OK']:
return res
# Set metadata if not already defined
res = self.fcc.getDirectoryUserMetadata( path )
if not res['OK']:
return res
if 'outputType' not in res['Value']:
res = self.fcc.setMetadata( path, {'outputType':subdir} )
if not res['OK']:
return res
# MD for the Data directory - data_level and configuration_id
path = os.path.join(Transformation_path, 'Data')
# Set metadata if not already defined
res = self.fcc.getDirectoryUserMetadata( path )
if not res['OK']:
return res
if 'data_level' and 'configuration_id' not in res['Value']:
res = self.fcc.setMetadata(path, {'data_level': md['data_level'],
'configuration_id': md['configuration_id']
})
if not res['OK']:
return res
return DIRAC.S_OK( Transformation_path )
def putAndRegister( self, lfn, localfile, filemetadata, DataType = 'SimtelProd' ):
""" put and register one file and set metadata
"""
# ## Get the list of Production SE
res = self._getSEList( 'ProductionOutputs', DataType )
if res['OK']:
ProductionSEList = res['Value']
else:
return res
# ## Get the list of Failover SE
res = self._getSEList( 'ProductionOutputsFailover', DataType )
if res['OK']:
FailoverSEList = res['Value']
else:
return res
# ## Upload file to a Production SE
res = self._putAndRegisterToSEList( lfn, localfile, ProductionSEList )
if not res['OK']:
DIRAC.gLogger.error( 'Failed to upload file to any Production SE: %s' % ProductionSEList )
# ## Upload file to a Failover SE
res = self._putAndRegisterToSEList( lfn, localfile, FailoverSEList )
if not res['OK']:
return DIRAC.S_ERROR( 'Failed to upload file to any Failover SE: %s' % FailoverSEList )
# ## Set file metadata: jobID, subarray, sct
if res['OK']:
fmd = json.loads( filemetadata )
if os.environ.has_key( 'JOBID' ):
fmd.update( {'jobID':os.environ['JOBID']} )
filename = os.path.basename( localfile )
# set subarray and sct md from filename
p = re.compile( 'subarray-\d+' )
if p.search( filename ) != None:
subarray = p.search( filename ).group()
fmd.update( {'subarray':subarray} )
sct = 'False'
p = re.compile( 'nosct' )
psct = re.compile( 'sct' )
if p.search( filename ) == None and psct.search( filename ) != None:
sct = 'True'
# ## Set sct flag only for production data
res = self.fcc.getFileUserMetadata( lfn )
if DataType == 'SimtelProd' and res['Value']['outputType'] == 'Data':
fmd.update( {'sct':sct} )
res = self.fc.setMetadata( lfn, fmd )
if not res['OK']:
return res
return DIRAC.S_OK()
def _putAndRegisterToSEList( self, lfn, localfile, SEList ):
""" put and register one file to one SE in the SEList
"""
# ## Try to upload file to a SE in the list
for SE in SEList:
msg = 'Try to upload local file: %s \nwith LFN: %s \nto %s' % ( localfile, lfn, SE )
DIRAC.gLogger.notice( msg )
res = self.dm.putAndRegister( lfn, localfile, SE )
DIRAC.gLogger.notice(res)
# ## check if failed
if not res['OK']:
DIRAC.gLogger.error( 'Failed to putAndRegister %s \nto %s \nwith message: %s' % ( lfn, SE, res['Message'] ) )
DIRAC.gLogger.notice('Trying to clean up %s' % lfn)
res = self.dm.removeFile(lfn)
if res['OK']:
DIRAC.gLogger.notice('Successfully removed %s \n that was not supposed to have been uploaded successfully' % lfn)
continue
elif res['Value']['Failed'].has_key( lfn ):
DIRAC.gLogger.error( 'Failed to putAndRegister %s to %s' % ( lfn, SE ) )
DIRAC.gLogger.notice('Trying to clean up %s' % lfn)
res = self.dm.removeFile(lfn)
if res['OK']:
DIRAC.gLogger.notice('Successfully removed %s \n that was not supposed to have been uploaded successfully' % lfn)
continue
else:
return DIRAC.S_OK()
return DIRAC.S_ERROR()
def cleanLocalFiles ( self, datadir, pattern = '*' ):
""" remove files matching pattern in datadir
"""
for localfile in glob.glob( os.path.join( datadir, pattern ) ):
DIRAC.gLogger.notice( 'Removing local file: ', localfile )
os.remove( localfile )
return DIRAC.S_OK()
class MoveInFC(ModuleBase):
'''
classdocs
'''
def __init__(self):
'''
Constructor
'''
super(MoveInFC, self).__init__()
self.enable = False
self.STEP_NUMBER = ''
self.log = gLogger.getSubLogger("MoveInFC")
self.applicationName = 'MoveInFC'
self.repMan = DataManager()
self.listoutput = {}
self.outputpath = ''
def applicationSpecificInputs(self):
""" Resolve all input variables for the module here.
:return: S_OK()
"""
if not len(self.InputFile) and len(self.InputData):
for files in self.InputData:
self.InputFile.append(files)
if 'listoutput' in self.step_commons:
self.listoutput = self.step_commons['listoutput'][0]
return S_OK()
def execute(self):
""" Run the module
"""
result = self.resolveInputVariables()
if not result['OK']:
return result
self.result = S_OK()
if not self.applicationLog:
self.result = S_ERROR('No Log file provided')
if not self.result['OK']:
return self.result
if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
self.log.verbose(
'Workflow status = %s, step status = %s' %
(self.workflowStatus['OK'], self.stepStatus['OK']))
return S_OK(
'%s should not proceed as previous step did not end properly' %
self.applicationName)
### Now remove the files in the FC
lfns = self.InputFile
##Check that all the files are here:
res = resolveIFpaths(lfns)
if not res['OK']:
self.log.error(res['Message'])
return S_ERROR("Failed to find a file locally")
#All files are here and available
paths = res['Value']
localpaths = []
for inputfile in paths:
basename = os.path.basename(inputfile)
locname = os.path.join(os.getcwd(), basename)
if not locname == inputfile:
try:
shutil.copy(inputfile, locname)
except shutil.Error:
self.log.error(
"Failed to copy file locally, will have to stop")
return S_ERROR("Failed copy to local directory")
localpaths.append(locname)
try:
os.unlink(inputfile)
except OSError:
self.log.warn("Failed to remove initial file, increased \
disk space usage")
#all the files are in the run directory
#get all metadata, ancestor/daughter relations, etc. for all the files
#Update the listoutput
if self.listoutput:
outputlist = []
for localFile in localpaths:
item = {}
item['outputFile'] = localFile
item['outputPath'] = self.listoutput['outputPath']
item['outputDataSE'] = self.listoutput['outputDataSE']
outputlist.append(item)
if self.enable:
self.step_commons['listoutput'] = outputlist
else:
self.log.info("listoutput would have been ", outputlist)
## Make sure the path contains / at the end as we are going to
## concatenate final path and local files
if not self.outputpath[-1] == '/':
self.outputpath += "/"
if 'ProductionOutputData' in self.workflow_commons:
file_list = ";".join([
self.outputpath + name
for name in [os.path.basename(fin) for fin in localpaths]
])
if self.enable:
self.workflow_commons['ProductionOutputData'] = file_list
else:
self.log.info("ProductionOutputData would have been",
file_list)
#Now remove them
if self.enable:
res = self.repMan.removeFile(lfns, force=True)
if not res['OK']:
self.log.error("Failed to remove the files")
self.setApplicationStatus("Failed to remove the file")
#return S_ERROR("Failed to remove the files")
else:
self.log.info("Would have removed: ", "%s" % str(lfns))
## Now the files are not on the storage anymore, they exist only locally. We can hope
## that the job will not be killed between now and the time the UploadOutputData module
## is called
return self.finalStatusReport(0)
if os.path.exists( inputFileName ):
inputFile = open( inputFileName, 'r' )
string = inputFile.read()
inputFile.close()
lfns.extend( [ lfn.strip() for lfn in string.splitlines() ] )
else:
lfns.append( inputFileName )
from DIRAC.Core.Utilities.List import breakListIntoChunks
from DIRAC.DataManagementSystem.Client.DataManager import DataManager
dm = DataManager()
errorReasons = {}
successfullyRemoved = 0
for lfnList in breakListIntoChunks( lfns, 100 ):
res = dm.removeFile( lfnList )
if not res['OK']:
gLogger.error( "Failed to remove data", res['Message'] )
DIRAC.exit( -2 )
for lfn, r in res['Value']['Failed'].items():
reason = str( r )
if not reason in errorReasons.keys():
errorReasons[reason] = []
errorReasons[reason].append( lfn )
successfullyRemoved += len( res['Value']['Successful'].keys() )
for reason, lfns in errorReasons.items():
gLogger.notice( "Failed to remove %d files with error: %s" % ( len( lfns ), reason ) )
if successfullyRemoved > 0:
gLogger.notice( "Successfully removed %d files" % successfullyRemoved )
DIRAC.exit( 0 )
def __call__(self):
""" call me maybe """
# The flag 'rmsMonitoring' is set by the RequestTask and is False by default.
# Here we use 'createRMSRecord' to create the ES record which is defined inside OperationHandlerBase.
if self.rmsMonitoring:
self.rmsMonitoringReporter = MonitoringReporter(
monitoringType="RMSMonitoring")
else:
# # RegisterFile specific monitor info
gMonitor.registerActivity("RegisterAtt",
"Attempted file registrations",
"RequestExecutingAgent", "Files/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("RegisterOK",
"Successful file registrations",
"RequestExecutingAgent", "Files/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("RegisterFail",
"Failed file registrations",
"RequestExecutingAgent", "Files/min",
gMonitor.OP_SUM)
# # counter for failed files
failedFiles = 0
# # catalog(s) to use
catalogs = self.operation.Catalog
if catalogs:
catalogs = [cat.strip() for cat in catalogs.split(',')]
dm = DataManager(catalogs=catalogs)
# # get waiting files
waitingFiles = self.getWaitingFilesList()
if self.rmsMonitoring:
self.rmsMonitoringReporter.addRecord(
self.createRMSRecord("Attempted", len(waitingFiles)))
# # loop over files
for opFile in waitingFiles:
if not self.rmsMonitoring:
gMonitor.addMark("RegisterAtt", 1)
# # get LFN
lfn = opFile.LFN
# # and others
fileTuple = (lfn, opFile.PFN, opFile.Size,
self.operation.targetSEList[0], opFile.GUID,
opFile.Checksum)
# # call DataManager
registerFile = dm.registerFile(fileTuple)
# # check results
if not registerFile["OK"] or lfn in registerFile["Value"]["Failed"]:
if self.rmsMonitoring:
self.rmsMonitoringReporter.addRecord(
self.createRMSRecord("Failed", 1))
else:
gMonitor.addMark("RegisterFail", 1)
# self.dataLoggingClient().addFileRecord(
# lfn, "RegisterFail", ','.join(catalogs) if catalogs else "all catalogs", "", "RegisterFile")
reason = str(
registerFile.get(
"Message",
registerFile.get("Value",
{}).get("Failed",
{}).get(lfn, 'Unknown')))
errorStr = "failed to register LFN"
opFile.Error = "%s: %s" % (errorStr, reason)
if 'GUID already registered' in reason:
opFile.Status = 'Failed'
self.log.error(errorStr, "%s: %s" % (lfn, reason))
elif 'File already registered with no replicas' in reason:
self.log.warn(
errorStr,
"%s: %s, will remove it and retry" % (lfn, reason))
dm.removeFile(lfn)
else:
self.log.warn(errorStr, "%s: %s" % (lfn, reason))
failedFiles += 1
else:
if self.rmsMonitoring:
self.rmsMonitoringReporter.addRecord(
self.createRMSRecord("Successful", 1))
else:
gMonitor.addMark("RegisterOK", 1)
# self.dataLoggingClient().addFileRecord(
# lfn, "Register", ','.join(catalogs) if catalogs else "all catalogs", "", "RegisterFile")
self.log.verbose(
"file %s has been registered at %s" %
(lfn, ','.join(catalogs) if catalogs else "all catalogs"))
opFile.Status = "Done"
if self.rmsMonitoring:
self.rmsMonitoringReporter.commit()
# # final check
if failedFiles:
self.log.warn("all files processed, %s files failed to register" %
failedFiles)
self.operation.Error = "some files failed to register"
return S_ERROR(self.operation.Error)
return S_OK()
class ReplicaManagerTestCase(unittest.TestCase):
""" Base class for the Replica Manager test cases
"""
def setUp(self):
self.dataManager = DataManager()
self.fileName = '/tmp/temporaryLocalFile'
file = open(self.fileName, 'w')
file.write("%s" % time.time())
file.close()
def test_putAndRegister(self):
print '\n\n#########################################################################\n\n\t\t\tPut and register test\n'
lfn = '/lhcb/test/unit-test/ReplicaManager/putAndRegister/testFile.%s' % time.time(
)
diracSE = 'GRIDKA-RAW'
putRes = self.dataManager.putAndRegister(lfn, self.fileName, diracSE)
removeRes = self.dataManager.removeFile(lfn)
# Check that the put was successful
self.assert_(putRes['OK'])
self.assert_(putRes['Value'].has_key('Successful'))
self.assert_(putRes['Value']['Successful'].has_key(lfn))
self.assert_(putRes['Value']['Successful'][lfn])
# Check that the removal was successful
self.assert_(removeRes['OK'])
self.assert_(removeRes['Value'].has_key('Successful'))
self.assert_(removeRes['Value']['Successful'].has_key(lfn))
self.assert_(removeRes['Value']['Successful'][lfn])
def test_putAndRegisterReplicate(self):
print '\n\n#########################################################################\n\n\t\t\tReplication test\n'
lfn = '/lhcb/test/unit-test/ReplicaManager/putAndRegisterReplicate/testFile.%s' % time.time(
)
diracSE = 'GRIDKA-RAW'
putRes = self.dataManager.putAndRegister(lfn, self.fileName, diracSE)
replicateRes = self.dataManager.replicateAndRegister(
lfn, 'CNAF-DST') #,sourceSE='',destPath='',localCache='')
removeRes = self.dataManager.removeFile(lfn)
# Check that the put was successful
self.assert_(putRes['OK'])
self.assert_(putRes['Value'].has_key('Successful'))
self.assert_(putRes['Value']['Successful'].has_key(lfn))
self.assert_(putRes['Value']['Successful'][lfn])
# Check that the replicate was successful
self.assert_(replicateRes['OK'])
self.assert_(replicateRes['Value'].has_key('Successful'))
self.assert_(replicateRes['Value']['Successful'].has_key(lfn))
self.assert_(replicateRes['Value']['Successful'][lfn])
# Check that the removal was successful
self.assert_(removeRes['OK'])
self.assert_(removeRes['Value'].has_key('Successful'))
self.assert_(removeRes['Value']['Successful'].has_key(lfn))
self.assert_(removeRes['Value']['Successful'][lfn])
def test_putAndRegisterGetReplicaMetadata(self):
print '\n\n#########################################################################\n\n\t\t\tGet metadata test\n'
lfn = '/lhcb/test/unit-test/ReplicaManager/putAndRegisterGetReplicaMetadata/testFile.%s' % time.time(
)
diracSE = 'GRIDKA-RAW'
putRes = self.dataManager.putAndRegister(lfn, self.fileName, diracSE)
metadataRes = self.dataManager.getReplicaMetadata(lfn, diracSE)
removeRes = self.dataManager.removeFile(lfn)
# Check that the put was successful
self.assert_(putRes['OK'])
self.assert_(putRes['Value'].has_key('Successful'))
self.assert_(putRes['Value']['Successful'].has_key(lfn))
self.assert_(putRes['Value']['Successful'][lfn])
# Check that the metadata query was successful
self.assert_(metadataRes['OK'])
self.assert_(metadataRes['Value'].has_key('Successful'))
self.assert_(metadataRes['Value']['Successful'].has_key(lfn))
self.assert_(metadataRes['Value']['Successful'][lfn])
metadataDict = metadataRes['Value']['Successful'][lfn]
self.assert_(metadataDict.has_key('Cached'))
self.assert_(metadataDict.has_key('Migrated'))
self.assert_(metadataDict.has_key('Size'))
# Check that the removal was successful
self.assert_(removeRes['OK'])
self.assert_(removeRes['Value'].has_key('Successful'))
self.assert_(removeRes['Value']['Successful'].has_key(lfn))
self.assert_(removeRes['Value']['Successful'][lfn])
def test_putAndRegsiterGetAccessUrl(self):
print '\n\n#########################################################################\n\n\t\t\tGet Access Url test\n'
lfn = '/lhcb/test/unit-test/ReplicaManager/putAndRegisterGetAccessUrl/testFile.%s' % time.time(
)
diracSE = 'GRIDKA-RAW'
putRes = self.dataManager.putAndRegister(lfn, self.fileName, diracSE)
getAccessUrlRes = self.dataManager.getReplicaAccessUrl(lfn, diracSE)
print getAccessUrlRes
removeRes = self.dataManager.removeFile(lfn)
# Check that the put was successful
self.assert_(putRes['OK'])
self.assert_(putRes['Value'].has_key('Successful'))
self.assert_(putRes['Value']['Successful'].has_key(lfn))
self.assert_(putRes['Value']['Successful'][lfn])
# Check that the access url was successful
self.assert_(getAccessUrlRes['OK'])
self.assert_(getAccessUrlRes['Value'].has_key('Successful'))
self.assert_(getAccessUrlRes['Value']['Successful'].has_key(lfn))
self.assert_(getAccessUrlRes['Value']['Successful'][lfn])
# Check that the removal was successful
self.assert_(removeRes['OK'])
self.assert_(removeRes['Value'].has_key('Successful'))
self.assert_(removeRes['Value']['Successful'].has_key(lfn))
self.assert_(removeRes['Value']['Successful'][lfn])
def test_putAndRegisterRemoveReplica(self):
print '\n\n#########################################################################\n\n\t\t\tRemove replica test\n'
lfn = '/lhcb/test/unit-test/ReplicaManager/putAndRegisterRemoveReplica/testFile.%s' % time.time(
)
diracSE = 'GRIDKA-RAW'
putRes = self.dataManager.putAndRegister(lfn, self.fileName, diracSE)
removeReplicaRes = self.dataManager.removeReplica(diracSE, lfn)
removeRes = self.dataManager.removeFile(lfn)
# Check that the put was successful
self.assert_(putRes['OK'])
self.assert_(putRes['Value'].has_key('Successful'))
self.assert_(putRes['Value']['Successful'].has_key(lfn))
self.assert_(putRes['Value']['Successful'][lfn])
# Check that the replica removal was successful
self.assert_(removeReplicaRes['OK'])
self.assert_(removeReplicaRes['Value'].has_key('Successful'))
self.assert_(removeReplicaRes['Value']['Successful'].has_key(lfn))
self.assert_(removeReplicaRes['Value']['Successful'][lfn])
# Check that the removal was successful
self.assert_(removeRes['OK'])
self.assert_(removeRes['Value'].has_key('Successful'))
self.assert_(removeRes['Value']['Successful'].has_key(lfn))
self.assert_(removeRes['Value']['Successful'][lfn])
def test_registerFile(self):
lfn = '/lhcb/test/unit-test/ReplicaManager/registerFile/testFile.%s' % time.time(
)
physicalFile = 'srm://host:port/srm/managerv2?SFN=/sa/path%s' % lfn
fileSize = 10000
storageElementName = 'CERN-RAW'
fileGuid = makeGuid()
fileTuple = (lfn, physicalFile, fileSize, storageElementName, fileGuid)
registerRes = self.dataManager.registerFile(fileTuple)
removeCatalogReplicaRes = self.dataManager.removeCatalogReplica(
storageElementName, lfn)
removeFileRes = self.dataManager.removeFile(lfn)
# Check that the file registration was done correctly
self.assert_(registerRes['OK'])
self.assert_(registerRes['Value'].has_key('Successful'))
self.assert_(registerRes['Value']['Successful'].has_key(lfn))
self.assert_(registerRes['Value']['Successful'][lfn])
# Check that the replica removal was successful
self.assert_(removeCatalogReplicaRes['OK'])
self.assert_(removeCatalogReplicaRes['Value'].has_key('Successful'))
self.assert_(
removeCatalogReplicaRes['Value']['Successful'].has_key(lfn))
self.assert_(removeCatalogReplicaRes['Value']['Successful'][lfn])
# Check that the removal was successful
self.assert_(removeFileRes['OK'])
self.assert_(removeFileRes['Value'].has_key('Successful'))
self.assert_(removeFileRes['Value']['Successful'].has_key(lfn))
self.assert_(removeFileRes['Value']['Successful'][lfn])
def test_registerReplica(self):
print '\n\n#########################################################################\n\n\t\t\tRegister replica test\n'
lfn = '/lhcb/test/unit-test/ReplicaManager/registerReplica/testFile.%s' % time.time(
)
physicalFile = 'srm://host:port/srm/managerv2?SFN=/sa/path%s' % lfn
fileSize = 10000
storageElementName = 'CERN-RAW'
fileGuid = makeGuid()
fileTuple = (lfn, physicalFile, fileSize, storageElementName, fileGuid)
registerRes = self.dataManager.registerFile(fileTuple)
seName = 'GRIDKA-RAW'
replicaTuple = (lfn, physicalFile, seName)
registerReplicaRes = self.dataManager.registerReplica(replicaTuple)
removeCatalogReplicaRes1 = self.dataManager.removeCatalogReplica(
storageElementName, lfn)
removeCatalogReplicaRes2 = self.dataManager.removeCatalogReplica(
seName, lfn)
removeFileRes = self.dataManager.removeFile(lfn)
# Check that the file registration was done correctly
self.assert_(registerRes['OK'])
self.assert_(registerRes['Value'].has_key('Successful'))
self.assert_(registerRes['Value']['Successful'].has_key(lfn))
self.assert_(registerRes['Value']['Successful'][lfn])
# Check that the replica registration was successful
self.assert_(registerReplicaRes['OK'])
self.assert_(registerReplicaRes['Value'].has_key('Successful'))
self.assert_(registerReplicaRes['Value']['Successful'].has_key(lfn))
self.assert_(registerReplicaRes['Value']['Successful'][lfn])
# Check that the replica removal was successful
self.assert_(removeCatalogReplicaRes1['OK'])
self.assert_(removeCatalogReplicaRes1['Value'].has_key('Successful'))
self.assert_(
removeCatalogReplicaRes1['Value']['Successful'].has_key(lfn))
self.assert_(removeCatalogReplicaRes1['Value']['Successful'][lfn])
# Check that the replica removal was successful
self.assert_(removeCatalogReplicaRes2['OK'])
self.assert_(removeCatalogReplicaRes2['Value'].has_key('Successful'))
self.assert_(
removeCatalogReplicaRes2['Value']['Successful'].has_key(lfn))
self.assert_(removeCatalogReplicaRes2['Value']['Successful'][lfn])
# Check that the removal was successful
self.assert_(removeFileRes['OK'])
self.assert_(removeFileRes['Value'].has_key('Successful'))
self.assert_(removeFileRes['Value']['Successful'].has_key(lfn))
self.assert_(removeFileRes['Value']['Successful'][lfn])
def test_putAndRegisterGet(self):
print '\n\n#########################################################################\n\n\t\t\tGet file test\n'
lfn = '/lhcb/test/unit-test/ReplicaManager/putAndRegisterGet/testFile.%s' % time.time(
)
diracSE = 'GRIDKA-RAW'
putRes = self.dataManager.putAndRegister(lfn, self.fileName, diracSE)
getRes = self.dataManager.getFile(lfn)
removeRes = self.dataManager.removeFile(lfn)
localFilePath = "%s/%s" % (os.getcwd(), os.path.basename(lfn))
if os.path.exists(localFilePath):
os.remove(localFilePath)
# Check that the put was successful
self.assert_(putRes['OK'])
self.assert_(putRes['Value'].has_key('Successful'))
self.assert_(putRes['Value']['Successful'].has_key(lfn))
self.assert_(putRes['Value']['Successful'][lfn])
# Check that the replica removal was successful
self.assert_(getRes['OK'])
self.assert_(getRes['Value'].has_key('Successful'))
self.assert_(getRes['Value']['Successful'].has_key(lfn))
self.assertEqual(getRes['Value']['Successful'][lfn], localFilePath)
# Check that the removal was successful
self.assert_(removeRes['OK'])
self.assert_(removeRes['Value'].has_key('Successful'))
self.assert_(removeRes['Value']['Successful'].has_key(lfn))
self.assert_(removeRes['Value']['Successful'][lfn])
class ReplicaManagerTestCase(unittest.TestCase):
""" Base class for the Replica Manager test cases
"""
def setUp(self):
self.dataManager = DataManager()
self.fileName = '/tmp/temporaryLocalFile'
file = open(self.fileName,'w')
file.write("%s" % time.time())
file.close()
def test_putAndRegister(self):
print '\n\n#########################################################################\n\n\t\t\tPut and register test\n'
lfn = '/lhcb/test/unit-test/ReplicaManager/putAndRegister/testFile.%s' % time.time()
diracSE = 'GRIDKA-RAW'
putRes = self.dataManager.putAndRegister(lfn, self.fileName, diracSE)
removeRes = self.dataManager.removeFile(lfn)
# Check that the put was successful
self.assert_(putRes['OK'])
self.assert_(putRes['Value'].has_key('Successful'))
self.assert_(putRes['Value']['Successful'].has_key(lfn))
self.assert_(putRes['Value']['Successful'][lfn])
# Check that the removal was successful
self.assert_(removeRes['OK'])
self.assert_(removeRes['Value'].has_key('Successful'))
self.assert_(removeRes['Value']['Successful'].has_key(lfn))
self.assert_(removeRes['Value']['Successful'][lfn])
def test_putAndRegisterReplicate(self):
print '\n\n#########################################################################\n\n\t\t\tReplication test\n'
lfn = '/lhcb/test/unit-test/ReplicaManager/putAndRegisterReplicate/testFile.%s' % time.time()
diracSE = 'GRIDKA-RAW'
putRes = self.dataManager.putAndRegister(lfn, self.fileName, diracSE)
replicateRes = self.dataManager.replicateAndRegister(lfn,'CNAF-DST') #,sourceSE='',destPath='',localCache='')
removeRes = self.dataManager.removeFile(lfn)
# Check that the put was successful
self.assert_(putRes['OK'])
self.assert_(putRes['Value'].has_key('Successful'))
self.assert_(putRes['Value']['Successful'].has_key(lfn))
self.assert_(putRes['Value']['Successful'][lfn])
# Check that the replicate was successful
self.assert_(replicateRes['OK'])
self.assert_(replicateRes['Value'].has_key('Successful'))
self.assert_(replicateRes['Value']['Successful'].has_key(lfn))
self.assert_(replicateRes['Value']['Successful'][lfn])
# Check that the removal was successful
self.assert_(removeRes['OK'])
self.assert_(removeRes['Value'].has_key('Successful'))
self.assert_(removeRes['Value']['Successful'].has_key(lfn))
self.assert_(removeRes['Value']['Successful'][lfn])
def test_putAndRegisterGetReplicaMetadata(self):
print '\n\n#########################################################################\n\n\t\t\tGet metadata test\n'
lfn = '/lhcb/test/unit-test/ReplicaManager/putAndRegisterGetReplicaMetadata/testFile.%s' % time.time()
diracSE = 'GRIDKA-RAW'
putRes = self.dataManager.putAndRegister(lfn, self.fileName, diracSE)
metadataRes = self.dataManager.getReplicaMetadata(lfn,diracSE)
removeRes = self.dataManager.removeFile(lfn)
# Check that the put was successful
self.assert_(putRes['OK'])
self.assert_(putRes['Value'].has_key('Successful'))
self.assert_(putRes['Value']['Successful'].has_key(lfn))
self.assert_(putRes['Value']['Successful'][lfn])
# Check that the metadata query was successful
self.assert_(metadataRes['OK'])
self.assert_(metadataRes['Value'].has_key('Successful'))
self.assert_(metadataRes['Value']['Successful'].has_key(lfn))
self.assert_(metadataRes['Value']['Successful'][lfn])
metadataDict = metadataRes['Value']['Successful'][lfn]
self.assert_(metadataDict.has_key('Cached'))
self.assert_(metadataDict.has_key('Migrated'))
self.assert_(metadataDict.has_key('Size'))
# Check that the removal was successful
self.assert_(removeRes['OK'])
self.assert_(removeRes['Value'].has_key('Successful'))
self.assert_(removeRes['Value']['Successful'].has_key(lfn))
self.assert_(removeRes['Value']['Successful'][lfn])
def test_putAndRegsiterGetAccessUrl(self):
print '\n\n#########################################################################\n\n\t\t\tGet Access Url test\n'
lfn = '/lhcb/test/unit-test/ReplicaManager/putAndRegisterGetAccessUrl/testFile.%s' % time.time()
diracSE = 'GRIDKA-RAW'
putRes = self.dataManager.putAndRegister(lfn, self.fileName, diracSE)
getAccessUrlRes = self.dataManager.getReplicaAccessUrl(lfn,diracSE)
print getAccessUrlRes
removeRes = self.dataManager.removeFile(lfn)
# Check that the put was successful
self.assert_(putRes['OK'])
self.assert_(putRes['Value'].has_key('Successful'))
self.assert_(putRes['Value']['Successful'].has_key(lfn))
self.assert_(putRes['Value']['Successful'][lfn])
# Check that the access url was successful
self.assert_(getAccessUrlRes['OK'])
self.assert_(getAccessUrlRes['Value'].has_key('Successful'))
self.assert_(getAccessUrlRes['Value']['Successful'].has_key(lfn))
self.assert_(getAccessUrlRes['Value']['Successful'][lfn])
# Check that the removal was successful
self.assert_(removeRes['OK'])
self.assert_(removeRes['Value'].has_key('Successful'))
self.assert_(removeRes['Value']['Successful'].has_key(lfn))
self.assert_(removeRes['Value']['Successful'][lfn])
def test_putAndRegisterRemoveReplica(self):
print '\n\n#########################################################################\n\n\t\t\tRemove replica test\n'
lfn = '/lhcb/test/unit-test/ReplicaManager/putAndRegisterRemoveReplica/testFile.%s' % time.time()
diracSE = 'GRIDKA-RAW'
putRes = self.dataManager.putAndRegister(lfn, self.fileName, diracSE)
removeReplicaRes = self.dataManager.removeReplica(diracSE,lfn)
removeRes = self.dataManager.removeFile(lfn)
# Check that the put was successful
self.assert_(putRes['OK'])
self.assert_(putRes['Value'].has_key('Successful'))
self.assert_(putRes['Value']['Successful'].has_key(lfn))
self.assert_(putRes['Value']['Successful'][lfn])
# Check that the replica removal was successful
self.assert_(removeReplicaRes['OK'])
self.assert_(removeReplicaRes['Value'].has_key('Successful'))
self.assert_(removeReplicaRes['Value']['Successful'].has_key(lfn))
self.assert_(removeReplicaRes['Value']['Successful'][lfn])
# Check that the removal was successful
self.assert_(removeRes['OK'])
self.assert_(removeRes['Value'].has_key('Successful'))
self.assert_(removeRes['Value']['Successful'].has_key(lfn))
self.assert_(removeRes['Value']['Successful'][lfn])
def test_registerFile(self):
lfn = '/lhcb/test/unit-test/ReplicaManager/registerFile/testFile.%s' % time.time()
physicalFile = 'srm://host:port/srm/managerv2?SFN=/sa/path%s' % lfn
fileSize = 10000
storageElementName = 'CERN-RAW'
fileGuid = makeGuid()
fileTuple = (lfn,physicalFile,fileSize,storageElementName,fileGuid)
registerRes = self.dataManager.registerFile(fileTuple)
removeCatalogReplicaRes = self.dataManager.removeCatalogReplica(storageElementName,lfn)
removeFileRes = self.dataManager.removeFile(lfn)
# Check that the file registration was done correctly
self.assert_(registerRes['OK'])
self.assert_(registerRes['Value'].has_key('Successful'))
self.assert_(registerRes['Value']['Successful'].has_key(lfn))
self.assert_(registerRes['Value']['Successful'][lfn])
# Check that the replica removal was successful
self.assert_(removeCatalogReplicaRes['OK'])
self.assert_(removeCatalogReplicaRes['Value'].has_key('Successful'))
self.assert_(removeCatalogReplicaRes['Value']['Successful'].has_key(lfn))
self.assert_(removeCatalogReplicaRes['Value']['Successful'][lfn])
# Check that the removal was successful
self.assert_(removeFileRes['OK'])
self.assert_(removeFileRes['Value'].has_key('Successful'))
self.assert_(removeFileRes['Value']['Successful'].has_key(lfn))
self.assert_(removeFileRes['Value']['Successful'][lfn])
def test_registerReplica(self):
print '\n\n#########################################################################\n\n\t\t\tRegister replica test\n'
lfn = '/lhcb/test/unit-test/ReplicaManager/registerReplica/testFile.%s' % time.time()
physicalFile = 'srm://host:port/srm/managerv2?SFN=/sa/path%s' % lfn
fileSize = 10000
storageElementName = 'CERN-RAW'
fileGuid = makeGuid()
fileTuple = (lfn,physicalFile,fileSize,storageElementName,fileGuid)
registerRes = self.dataManager.registerFile(fileTuple)
seName = 'GRIDKA-RAW'
replicaTuple = (lfn,physicalFile,seName)
registerReplicaRes = self.dataManager.registerReplica(replicaTuple)
removeCatalogReplicaRes1 = self.dataManager.removeCatalogReplica(storageElementName,lfn)
removeCatalogReplicaRes2 = self.dataManager.removeCatalogReplica(seName,lfn)
removeFileRes = self.dataManager.removeFile(lfn)
# Check that the file registration was done correctly
self.assert_(registerRes['OK'])
self.assert_(registerRes['Value'].has_key('Successful'))
self.assert_(registerRes['Value']['Successful'].has_key(lfn))
self.assert_(registerRes['Value']['Successful'][lfn])
# Check that the replica registration was successful
self.assert_(registerReplicaRes['OK'])
self.assert_(registerReplicaRes['Value'].has_key('Successful'))
self.assert_(registerReplicaRes['Value']['Successful'].has_key(lfn))
self.assert_(registerReplicaRes['Value']['Successful'][lfn])
# Check that the replica removal was successful
self.assert_(removeCatalogReplicaRes1['OK'])
self.assert_(removeCatalogReplicaRes1['Value'].has_key('Successful'))
self.assert_(removeCatalogReplicaRes1['Value']['Successful'].has_key(lfn))
self.assert_(removeCatalogReplicaRes1['Value']['Successful'][lfn])
# Check that the replica removal was successful
self.assert_(removeCatalogReplicaRes2['OK'])
self.assert_(removeCatalogReplicaRes2['Value'].has_key('Successful'))
self.assert_(removeCatalogReplicaRes2['Value']['Successful'].has_key(lfn))
self.assert_(removeCatalogReplicaRes2['Value']['Successful'][lfn])
# Check that the removal was successful
self.assert_(removeFileRes['OK'])
self.assert_(removeFileRes['Value'].has_key('Successful'))
self.assert_(removeFileRes['Value']['Successful'].has_key(lfn))
self.assert_(removeFileRes['Value']['Successful'][lfn])
def test_putAndRegisterGet(self):
print '\n\n#########################################################################\n\n\t\t\tGet file test\n'
lfn = '/lhcb/test/unit-test/ReplicaManager/putAndRegisterGet/testFile.%s' % time.time()
diracSE = 'GRIDKA-RAW'
putRes = self.dataManager.putAndRegister(lfn, self.fileName, diracSE)
getRes = self.dataManager.getFile(lfn)
removeRes = self.dataManager.removeFile(lfn)
localFilePath = "%s/%s" % (os.getcwd(),os.path.basename(lfn))
if os.path.exists(localFilePath):
os.remove(localFilePath)
# Check that the put was successful
self.assert_(putRes['OK'])
self.assert_(putRes['Value'].has_key('Successful'))
self.assert_(putRes['Value']['Successful'].has_key(lfn))
self.assert_(putRes['Value']['Successful'][lfn])
# Check that the replica removal was successful
self.assert_(getRes['OK'])
self.assert_(getRes['Value'].has_key('Successful'))
self.assert_(getRes['Value']['Successful'].has_key(lfn))
self.assertEqual(getRes['Value']['Successful'][lfn],localFilePath)
# Check that the removal was successful
self.assert_(removeRes['OK'])
self.assert_(removeRes['Value'].has_key('Successful'))
self.assert_(removeRes['Value']['Successful'].has_key(lfn))
self.assert_(removeRes['Value']['Successful'][lfn])
class RecursiveRm(object):
def __init__(self):
self.__rpcclient = RPCClient("DataManagement/FileCatalog")
self.__dm = DataManager()
self.__n_dirs = 0
self.__n_files = 0
self.__space_freed = 0L
def clear_directory(self, directory_path, se_name, dry_run=True):
print "Clearing directory: %s" % directory_path
dir_content = self.__rpcclient.listDirectory(directory_path, False)
if not dir_content["OK"]:
print "Failed to contact DIRAC server for %s" % directory_path
return
if directory_path in dir_content['Value']['Failed']:
print "Could not access %s, maybe it doesn't exist?" % directory_path
return
subdirs = dir_content['Value']['Successful'][directory_path]['SubDirs']
for subdir in subdirs.keys():
self.clear_directory(subdir, se_name, dry_run)
# Now do files...
files = dir_content['Value']['Successful'][directory_path]['Files']
for filename in files.keys():
fullpath = os.path.join(directory_path, filename)
if self.clear_file(fullpath, se_name, dry_run):
self.__n_files += 1
self.__space_freed += files[filename]['MetaData']['Size']
if self.remove_empty_dir(directory_path, dry_run):
self.__n_dirs += 1
def remove_empty_dir(self, directory_path, dry_run=True):
# check if directory is now empty and the remove the directory
dir_content = self.__rpcclient.listDirectory(directory_path, False)
if not dir_content["OK"]:
print "Could not access %s" % directory_path
return False
subdirs = dir_content['Value']['Successful'][directory_path]['SubDirs']
files = dir_content['Value']['Successful'][directory_path]['Files']
if not subdirs and not files:
if not dry_run:
self.__dm.fileCatalog.removeDirectory(directory_path,
recursive=False)
return True
return False
def clear_file(self, filename, se_name, dry_run=True):
res = self.__rpcclient.getReplicas(filename, False)
if not res["OK"]:
print "Could not get replica status for %s" % filename
return False
ses = res['Value']['Successful'][filename].keys()
# remove file regardless of number of replicas
if se_name == "Any":
print "%s" % filename
if not dry_run:
deleted = self.__dm.removeFile(filename)
if not deleted["OK"]:
print "Function call to removeFile failed, file not deleted: %s" \
% str(deleted)
return False
elif not deleted["Value"]["Successful"]:
print "Failed to delete file: %s" % str(deleted)
return False
return True
# file exists only at the chosen SE
# -> delete file and remove from file catalogue
if len(ses) == 1 and se_name in ses:
print "%s" % filename
if not dry_run:
deleted = self.__dm.removeFile(filename)
if not deleted["OK"]:
print "Function call to removeFile failed, file not deleted: %s" \
% str(deleted)
return False
elif not deleted["Value"]["Successful"]:
print "Failed to delete file: %s" % str(deleted)
return False
return True
# file exists at the chosen SE and elswhere -> delete replica at chosen SE
if len(ses) > 1 and se_name in ses:
print "%s" % filename
if not dry_run:
deleted = self.__dm.removeReplica(se_name, filename)
if not deleted["OK"]:
print "Function call to removeReplica failed, replica not deleted: %s" \
% str(deleted)
return False
elif not deleted["Value"]["Successful"]:
print "Failed to delete replica: %s" % str(deleted)
return False
return True
return False
def print_stats(self):
print ""
print "Number of files deleted: %s" % self.__n_files
print "NUmber of (sub)directories deleted: %s" % self.__n_dirs
space = self.__space_freed / (1024.0 * 1024.0 * 1024.0)
print "Space freed: %0.3f GB" % space
class DIRACBackend(GridBackend):
"""Grid backend using the GFAL command line tools `gfal-*`."""
def __init__(self, **kwargs):
GridBackend.__init__(self, catalogue_prefix='', **kwargs)
from DIRAC.Core.Base import Script
Script.initialize()
from DIRAC.FrameworkSystem.Client.ProxyManagerClient import ProxyManagerClient
self.pm = ProxyManagerClient()
proxy = self.pm.getUserProxiesInfo()
if not proxy['OK']:
raise BackendException("Proxy error.")
from DIRAC.Interfaces.API.Dirac import Dirac
self.dirac = Dirac()
from DIRAC.Resources.Catalog.FileCatalog import FileCatalog
self.fc = FileCatalog()
from DIRAC.DataManagementSystem.Client.DataManager import DataManager
self.dm = DataManager()
self._xattr_cmd = sh.Command('gfal-xattr').bake(_tty_out=False)
self._replica_checksum_cmd = sh.Command('gfal-sum').bake(_tty_out=False)
self._bringonline_cmd = sh.Command('gfal-legacy-bringonline').bake(_tty_out=False)
self._cp_cmd = sh.Command('gfal-copy').bake(_tty_out=False)
self._ls_se_cmd = sh.Command('gfal-ls').bake(color='never', _tty_out=False)
self._move_cmd = sh.Command('gfal-rename').bake(_tty_out=False)
self._mkdir_cmd = sh.Command('gfal-mkdir').bake(_tty_out=False)
self._replicate_cmd = sh.Command('dirac-dms-replicate-lfn').bake(_tty_out=False)
self._add_cmd = sh.Command('dirac-dms-add-file').bake(_tty_out=False)
@staticmethod
def _check_return_value(ret):
if not ret['OK']:
raise BackendException("Failed: %s", ret['Message'])
for path, error in ret['Value']['Failed'].items():
if ('No such' in error) or ('Directory does not' in error):
raise DoesNotExistException("No such file or directory.")
else:
raise BackendException(error)
def _is_dir(self, lurl):
isdir = self.fc.isDirectory(lurl)
self._check_return_value(isdir)
return isdir['Value']['Successful'][lurl]
def _is_file(self, lurl):
isfile = self.fc.isFile(lurl)
self._check_return_value(isfile)
return isfile['Value']['Successful'][lurl]
def _get_dir_entry(self, lurl, infodict=None):
"""Take a lurl and return a DirEntry."""
# If no dctionary with the information is specified, get it from the catalogue
try:
md = infodict['MetaData']
except TypeError:
md = self.fc.getFileMetadata(lurl)
if not md['OK']:
raise BackendException("Failed to list path '%s': %s", lurl, md['Message'])
for path, error in md['Value']['Failed'].items():
if 'No such file' in error:
# File does not exist, maybe a directory?
md = self.fc.getDirectoryMetadata(lurl)
for path, error in md['Value']['Failed'].items():
raise DoesNotExistException("No such file or directory.")
else:
raise BackendException(md['Value']['Failed'][lurl])
md = md['Value']['Successful'][lurl]
return DirEntry(posixpath.basename(lurl), mode=oct(md.get('Mode', -1)), links=md.get('links', -1), gid=md['OwnerGroup'], uid=md['Owner'], size=md.get('Size', -1), modified=str(md.get('ModificationDate', '?')))
def _iter_directory(self, lurl):
"""Iterate over entries in a directory."""
ret = self.fc.listDirectory(lurl)
if not ret['OK']:
raise BackendException("Failed to list path '%s': %s", lurl, ret['Message'])
for path, error in ret['Value']['Failed'].items():
if 'Directory does not' in error:
# Dir does not exist, maybe a File?
if self.fc.isFile(lurl):
lst = [(lurl, None)]
break
else:
raise DoesNotExistException("No such file or Directory.")
else:
raise BackendException(ret['Value']['Failed'][lurl])
else:
# Sort items by keys, i.e. paths
lst = sorted(ret['Value']['Successful'][lurl]['Files'].items() + ret['Value']['Successful'][lurl]['SubDirs'].items())
for item in lst:
yield item # = path, dict
def _ls(self, lurl, **kwargs):
# Translate keyword arguments
d = kwargs.pop('directory', False)
if d:
# Just the requested entry itself
yield self._get_dir_entry(lurl)
return
for path, info in self._iter_directory(lurl):
yield self._get_dir_entry(path, info)
def _ls_se(self, surl, **kwargs):
# Translate keyword arguments
d = kwargs.pop('directory', False)
args = []
if -d:
args.append('-d')
args.append('-l')
args.append(surl)
try:
output = self._ls_se_cmd(*args, **kwargs)
except sh.ErrorReturnCode as e:
if 'No such file' in e.stderr:
raise DoesNotExistException("No such file or Directory.")
else:
raise BackendException(e.stderr)
for line in output:
fields = line.split()
mode, links, gid, uid, size = fields[:5]
name = fields[-1]
modified = ' '.join(fields[5:-1])
yield DirEntry(name, mode=mode, links=int(links), gid=gid, uid=uid, size=int(size), modified=modified)
def _replicas(self, lurl, **kwargs):
# Check the lurl actually exists
self._ls(lurl, directory=True)
rep = self.dirac.getReplicas(lurl)
self._check_return_value(rep)
rep = rep['Value']['Successful'][lurl]
return rep.values()
def _exists(self, surl, **kwargs):
try:
ret = self._ls_se_cmd(surl, '-d', '-l', **kwargs).strip()
except sh.ErrorReturnCode as e:
if 'No such file' in e.stderr:
return False
else:
if len(e.stderr) == 0:
raise BackendException(e.stdout)
else:
raise BackendException(e.stderr)
else:
return ret[0] != 'd' # Return `False` for directories
def _register(self, surl, lurl, verbose=False, **kwargs):
# Register an existing physical copy in the file catalogue
se = storage.get_SE(surl).name
# See if file already exists in DFC
ret = self.fc.getFileMetadata(lurl)
try:
self._check_return_value(ret)
except DoesNotExistException:
# Add new file
size = next(self._ls_se(surl, directory=True)).size
checksum = self.checksum(surl)
guid = str(uuid.uuid4()) # The guid does not seem to be important. Make it unique if possible.
ret = self.dm.registerFile((lurl, surl, size, se, guid, checksum))
else:
# Add new replica
ret = self.dm.registerReplica((lurl, surl, se))
self._check_return_value(ret)
if verbose:
print_("Successfully registered replica %s of %s from %s."%(surl, lurl, se))
return True
def _deregister(self, surl, lurl, verbose=False, **kwargs):
# DIRAC only needs to know the SE name to deregister a replica
se = storage.get_SE(surl).name
ret = self.dm.removeReplicaFromCatalog(se, [lurl])
self._check_return_value(ret)
if verbose:
print_("Successfully deregistered replica of %s from %s."%(lurl, se))
return True
def _state(self, surl, **kwargs):
try:
state = self._xattr_cmd(surl, 'user.status', **kwargs).strip()
except sh.ErrorReturnCode as e:
if "No such file" in e.stderr:
raise DoesNotExistException("No such file or Directory.")
state = '?'
except sh.SignalException_SIGSEGV:
state = '?'
return state
def _checksum(self, surl, **kwargs):
try:
checksum = self._replica_checksum_cmd(surl, 'ADLER32', **kwargs).split()[1]
except sh.ErrorReturnCode:
checksum = '?'
except sh.SignalException_SIGSEGV:
checksum = '?'
except IndexError:
checksum = '?'
return checksum
def _bringonline(self, surl, timeout, verbose=False, **kwargs):
if verbose:
out = sys.stdout
else:
out = None
# gfal does not notice when files come online, it seems
# Just send a single short request, then check regularly
if verbose:
out = sys.stdout
else:
out = None
end = time.time() + timeout
try:
self._bringonline_cmd('-t', 10, surl, _out=out, **kwargs)
except sh.ErrorReturnCode as e:
# The command fails if the file is not online
# To be expected after 10 seconds
if "No such file" in e.stderr:
# Except when the file does not actually exist on the tape storage
raise DoesNotExistException("No such file or Directory.")
wait = 5
while(True):
if verbose:
print_("Checking replica state...")
if self.is_online(surl):
if verbose:
print_("Replica brought online.")
return True
time_left = end - time.time()
if time_left <= 0:
if verbose:
print_("Could not bring replica online.")
return False
wait *= 2
if time_left < wait:
wait = time_left
if verbose:
print_("Timeout remaining: %d s"%(time_left))
print_("Checking again in: %d s"%(wait))
time.sleep(wait)
def _replicate(self, source_surl, destination_surl, lurl, verbose=False, **kwargs):
if verbose:
out = sys.stdout
else:
out = None
source = storage.get_SE(source_surl).name
destination = storage.get_SE(destination_surl).name
try:
self._replicate_cmd(lurl, destination, source, _out=out, **kwargs)
except sh.ErrorReturnCode as e:
if 'No such file' in e.stderr:
raise DoesNotExistException("No such file or directory.")
else:
if len(e.stderr) == 0:
raise BackendException(e.stdout)
else:
raise BackendException(e.stderr)
return True
def _get(self, surl, localpath, verbose=False, **kwargs):
if verbose:
out = sys.stdout
else:
out = None
try:
self._cp_cmd('-f', '--checksum', 'ADLER32', surl, localpath, _out=out, **kwargs)
except sh.ErrorReturnCode as e:
if 'No such file' in e.stderr:
raise DoesNotExistException("No such file or directory.")
else:
if len(e.stderr) == 0:
raise BackendException(e.stdout)
else:
raise BackendException(e.stderr)
return os.path.isfile(localpath)
def _put(self, localpath, surl, lurl, verbose=False, **kwargs):
if verbose:
out = sys.stdout
else:
out = None
se = storage.get_SE(surl).name
try:
self._add_cmd(lurl, localpath, se, _out=out, **kwargs)
except sh.ErrorReturnCode as e:
if 'No such file' in e.stderr:
raise DoesNotExistException("No such file or directory.")
else:
if len(e.stderr) == 0:
raise BackendException(e.stdout)
else:
raise BackendException(e.stderr)
return True
def _remove(self, surl, lurl, last=False, verbose=False, **kwargs):
se = storage.get_SE(surl).name
if last:
# Delete lfn
if verbose:
print_("Removing all replicas of %s."%(lurl,))
ret = self.dm.removeFile([lurl])
else:
if verbose:
print_("Removing replica of %s from %s."%(lurl, se))
ret = self.dm.removeReplica(se, [lurl])
if not ret['OK']:
raise BackendException('Failed: %s'%(ret['Message']))
for lurl, error in ret['Value']['Failed'].items():
if 'No such file' in error:
raise DoesNotExistException("No such file or directory.")
else:
raise BackendException(error)
return True
def _rmdir(self, lurl, verbose=False):
"""Remove the an empty directory from the catalogue."""
rep = self.fc.removeDirectory(lurl)
self._check_return_value(rep)
return True
def _move_replica(self, surl, new_surl, verbose=False, **kwargs):
if verbose:
out = sys.stdout
else:
out = None
try:
folder = posixpath.dirname(new_surl)
self._mkdir_cmd(folder, '-p', _out=out, **kwargs)
self._move_cmd(surl, new_surl, _out=out, **kwargs)
except sh.ErrorReturnCode as e:
if 'No such file' in e.stderr:
raise DoesNotExistException("No such file or directory.")
else:
if len(e.stderr) == 0:
raise BackendException(e.stdout)
else:
raise BackendException(e.stderr)
return True
class RecursiveRm(object):
def __init__(self):
self.__rpcclient = RPCClient( "DataManagement/FileCatalog" )
self.__dm = DataManager()
self.__n_dirs = 0
self.__n_files = 0
self.__space_freed = 0L
def clear_directory(self, directory_path, se_name, dry_run=True):
print "Clearing directory: %s" % directory_path
dir_content = self.__rpcclient.listDirectory(directory_path, False)
if not dir_content["OK"]:
print "Failed to contact DIRAC server for %s" % directory_path
return
if directory_path in dir_content['Value']['Failed']:
print "Could not access %s, maybe it doesn't exist?" % directory_path
return
subdirs = dir_content['Value']['Successful'][directory_path]['SubDirs']
for subdir in subdirs.keys():
self.clear_directory(subdir, se_name, dry_run)
# Now do files...
files = dir_content['Value']['Successful'][directory_path]['Files']
for filename in files.keys():
fullpath = os.path.join(directory_path, filename)
if self.clear_file(fullpath, se_name, dry_run):
self.__n_files += 1
self.__space_freed += files[filename]['MetaData']['Size']
if self.remove_empty_dir(directory_path, dry_run):
self.__n_dirs += 1
def remove_empty_dir(self, directory_path, dry_run=True):
# check if directory is now empty and the remove the directory
dir_content = self.__rpcclient.listDirectory(directory_path, False)
if not dir_content["OK"]:
print "Could not access %s" % directory_path
return False
subdirs = dir_content['Value']['Successful'][directory_path]['SubDirs']
files = dir_content['Value']['Successful'][directory_path]['Files']
if not subdirs and not files:
if not dry_run:
self.__dm.fileCatalog.removeDirectory(directory_path, recursive=False)
return True
return False
def clear_file(self, filename, se_name, dry_run=True):
res = self.__rpcclient.getReplicas(filename, False)
if not res["OK"]:
print "Could not get replica status for %s" % filename
return False
ses = res['Value']['Successful'][filename].keys()
# remove file regardless of number of replicas
if se_name == "Any":
print "%s" % filename
if not dry_run:
deleted = self.__dm.removeFile(filename)
if not deleted["OK"]:
print "Function call to removeFile failed, file not deleted: %s" \
% str(deleted)
return False
elif not deleted["Value"]["Successful"]:
print "Failed to delete file: %s" % str(deleted)
return False
return True
# file exists only at the chosen SE
# -> delete file and remove from file catalogue
if len(ses) == 1 and se_name in ses:
print "%s" % filename
if not dry_run:
deleted = self.__dm.removeFile(filename)
if not deleted["OK"]:
print "Function call to removeFile failed, file not deleted: %s" \
% str(deleted)
return False
elif not deleted["Value"]["Successful"]:
print "Failed to delete file: %s" % str(deleted)
return False
return True
# file exists at the chosen SE and elswhere -> delete replica at chosen SE
if len(ses) > 1 and se_name in ses:
print "%s" % filename
if not dry_run:
deleted = self.__dm.removeReplica(se_name, filename)
if not deleted["OK"]:
print "Function call to removeReplica failed, replica not deleted: %s" \
% str(deleted)
return False
elif not deleted["Value"]["Successful"]:
print "Failed to delete replica: %s" % str(deleted)
return False
return True
return False
def print_stats(self):
print ""
print "Number of files deleted: %s" % self.__n_files
print "NUmber of (sub)directories deleted: %s" % self.__n_dirs
space = self.__space_freed/(1024.0 * 1024.0 * 1024.0)
print "Space freed: %0.3f GB" % space
class TransformationCleaningAgent( AgentModule ):
"""
.. class:: TransformationCleaningAgent
:param DataManger dm: DataManager instance
:param TransfromationClient transClient: TransfromationClient instance
:param FileCatalogClient metadataClient: FileCatalogClient instance
"""
def __init__( self, *args, **kwargs ):
""" c'tor
"""
AgentModule.__init__( self, *args, **kwargs )
# # data manager
self.dm = None
# # transformation client
self.transClient = None
# # wms client
self.wmsClient = None
# # request client
self.reqClient = None
# # file catalog client
self.metadataClient = None
# # transformations types
self.transformationTypes = None
# # directory locations
self.directoryLocations = None
# # transformation metadata
self.transfidmeta = None
# # archive periof in days
self.archiveAfter = None
# # active SEs
self.activeStorages = None
# # transformation log SEs
self.logSE = None
# # enable/disable execution
self.enableFlag = None
def initialize( self ):
""" agent initialisation
reading and setting confing opts
:param self: self reference
"""
# # shifter proxy
self.am_setOption( 'shifterProxy', 'DataManager' )
# # transformations types
self.dataProcTTypes = Operations().getValue( 'Transformations/DataProcessing', ['MCSimulation', 'Merge'] )
self.dataManipTTypes = Operations().getValue( 'Transformations/DataManipulation', ['Replication', 'Removal'] )
agentTSTypes = self.am_getOption( 'TransformationTypes', [] )
if agentTSTypes:
self.transformationTypes = sorted( agentTSTypes )
else:
self.transformationTypes = sorted( self.dataProcTTypes + self.dataManipTTypes )
self.log.info( "Will consider the following transformation types: %s" % str( self.transformationTypes ) )
# # directory locations
self.directoryLocations = sorted( self.am_getOption( 'DirectoryLocations', [ 'TransformationDB',
'MetadataCatalog' ] ) )
self.log.info( "Will search for directories in the following locations: %s" % str( self.directoryLocations ) )
# # transformation metadata
self.transfidmeta = self.am_getOption( 'TransfIDMeta', "TransformationID" )
self.log.info( "Will use %s as metadata tag name for TransformationID" % self.transfidmeta )
# # archive periof in days
self.archiveAfter = self.am_getOption( 'ArchiveAfter', 7 ) # days
self.log.info( "Will archive Completed transformations after %d days" % self.archiveAfter )
# # active SEs
self.activeStorages = sorted( self.am_getOption( 'ActiveSEs', [] ) )
self.log.info( "Will check the following storage elements: %s" % str( self.activeStorages ) )
# # transformation log SEs
self.logSE = self.am_getOption( 'TransformationLogSE', 'LogSE' )
self.log.info( "Will remove logs found on storage element: %s" % self.logSE )
# # enable/disable execution, should be using CS option Status?? with default value as 'Active'??
self.enableFlag = self.am_getOption( 'EnableFlag', 'True' )
# # data manager
self.dm = DataManager()
# # transformation client
self.transClient = TransformationClient()
# # wms client
self.wmsClient = WMSClient()
# # request client
self.reqClient = ReqClient()
# # file catalog client
self.metadataClient = FileCatalogClient()
return S_OK()
#############################################################################
def execute( self ):
""" execution in one agent's cycle
:param self: self reference
"""
self.enableFlag = self.am_getOption( 'EnableFlag', 'True' )
if not self.enableFlag == 'True':
self.log.info( 'TransformationCleaningAgent is disabled by configuration option EnableFlag' )
return S_OK( 'Disabled via CS flag' )
# # Obtain the transformations in Cleaning status and remove any mention of the jobs/files
res = self.transClient.getTransformations( { 'Status' : 'Cleaning',
'Type' : self.transformationTypes } )
if res['OK']:
for transDict in res['Value']:
# # if transformation is of type `Replication` or `Removal`, there is nothing to clean.
# # We just archive
if transDict[ 'Type' ] in self.dataManipTTypes:
res = self.archiveTransformation( transDict['TransformationID'] )
if not res['OK']:
self.log.error( "Problems archiving transformation %s: %s" % ( transDict['TransformationID'],
res['Message'] ) )
else:
res = self.cleanTransformation( transDict['TransformationID'] )
if not res['OK']:
self.log.error( "Problems cleaning transformation %s: %s" % ( transDict['TransformationID'],
res['Message'] ) )
# # Obtain the transformations in RemovingFiles status and (wait for it) removes the output files
res = self.transClient.getTransformations( { 'Status' : 'RemovingFiles',
'Type' : self.transformationTypes} )
if res['OK']:
for transDict in res['Value']:
res = self.removeTransformationOutput( transDict['TransformationID'] )
if not res['OK']:
self.log.error( "Problems removing transformation %s: %s" % ( transDict['TransformationID'],
res['Message'] ) )
# # Obtain the transformations in Completed status and archive if inactive for X days
olderThanTime = datetime.utcnow() - timedelta( days = self.archiveAfter )
res = self.transClient.getTransformations( { 'Status' : 'Completed',
'Type' : self.transformationTypes },
older = olderThanTime,
timeStamp = 'LastUpdate' )
if res['OK']:
for transDict in res['Value']:
res = self.archiveTransformation( transDict['TransformationID'] )
if not res['OK']:
self.log.error( "Problems archiving transformation %s: %s" % ( transDict['TransformationID'],
res['Message'] ) )
else:
self.log.error( "Could not get the transformations" )
return S_OK()
#############################################################################
#
# Get the transformation directories for checking
#
def getTransformationDirectories( self, transID ):
""" get the directories for the supplied transformation from the transformation system
:param self: self reference
:param int transID: transformation ID
"""
directories = []
if 'TransformationDB' in self.directoryLocations:
res = self.transClient.getTransformationParameters( transID, ['OutputDirectories'] )
if not res['OK']:
self.log.error( "Failed to obtain transformation directories", res['Message'] )
return res
transDirectories = res['Value'].splitlines()
directories = self._addDirs( transID, transDirectories, directories )
if 'MetadataCatalog' in self.directoryLocations:
res = self.metadataClient.findDirectoriesByMetadata( {self.transfidmeta:transID} )
if not res['OK']:
self.log.error( "Failed to obtain metadata catalog directories", res['Message'] )
return res
transDirectories = res['Value']
directories = self._addDirs( transID, transDirectories, directories )
if not directories:
self.log.info( "No output directories found" )
directories = sorted( directories )
return S_OK( directories )
# FIXME If a classmethod, should it not have cls instead of self?
@classmethod
def _addDirs( self, transID, newDirs, existingDirs ):
""" append uniqe :newDirs: list to :existingDirs: list
:param self: self reference
:param int transID: transformationID
:param list newDirs: src list of paths
:param list existingDirs: dest list of paths
"""
for folder in newDirs:
transStr = str( transID ).zfill( 8 )
if re.search( transStr, str( folder ) ):
if not folder in existingDirs:
existingDirs.append( folder )
return existingDirs
#############################################################################
#
# These are the methods for performing the cleaning of catalogs and storage
#
def cleanStorageContents( self, directory ):
""" delete lfn dir from all active SE
:param self: self reference
:param sre directory: folder name
"""
for storageElement in self.activeStorages:
res = self.__removeStorageDirectory( directory, storageElement )
if not res['OK']:
return res
return S_OK()
def __removeStorageDirectory( self, directory, storageElement ):
""" wipe out all contents from :directory: at :storageElement:
:param self: self reference
:param str directory: path
:param str storageElement: SE name
"""
self.log.info( 'Removing the contents of %s at %s' % ( directory, storageElement ) )
se = StorageElement( storageElement )
res = se.getPfnForLfn( [directory] )
if not res['OK']:
self.log.error( "Failed to get PFN for directory", res['Message'] )
return res
if directory in res['Value']['Failed']:
self.log.verbose( 'Failed to obtain directory PFN from LFN', '%s %s' % ( directory, res['Value']['Failed'][directory] ) )
return S_ERROR( 'Failed to obtain directory PFN from LFNs' )
storageDirectory = res['Value']['Successful'][directory]
res = returnSingleResult( se.exists( storageDirectory ) )
if not res['OK']:
self.log.error( "Failed to obtain existance of directory", res['Message'] )
return res
exists = res['Value']
if not exists:
self.log.info( "The directory %s does not exist at %s " % ( directory, storageElement ) )
return S_OK()
res = returnSingleResult( se.removeDirectory( storageDirectory, recursive = True ) )
if not res['OK']:
self.log.error( "Failed to remove storage directory", res['Message'] )
return res
self.log.info( "Successfully removed %d files from %s at %s" % ( res['Value']['FilesRemoved'],
directory,
storageElement ) )
return S_OK()
def cleanCatalogContents( self, directory ):
""" wipe out everything from catalog under folder :directory:
:param self: self reference
:params str directory: folder name
"""
res = self.__getCatalogDirectoryContents( [directory] )
if not res['OK']:
return res
filesFound = res['Value']
if not filesFound:
self.log.info( "No files are registered in the catalog directory %s" % directory )
return S_OK()
self.log.info( "Attempting to remove %d possible remnants from the catalog and storage" % len( filesFound ) )
res = self.dm.removeFile( filesFound, force = True )
if not res['OK']:
return res
realFailure = False
for lfn, reason in res['Value']['Failed'].items():
if "File does not exist" in str( reason ):
self.log.warn( "File %s not found in some catalog: " % ( lfn ) )
else:
self.log.error( "Failed to remove file found in the catalog", "%s %s" % ( lfn, reason ) )
realFailure = True
if realFailure:
return S_ERROR( "Failed to remove all files found in the catalog" )
return S_OK()
def __getCatalogDirectoryContents( self, directories ):
""" get catalog contents under paths :directories:
:param self: self reference
:param list directories: list of paths in catalog
"""
self.log.info( 'Obtaining the catalog contents for %d directories:' % len( directories ) )
for directory in directories:
self.log.info( directory )
activeDirs = directories
allFiles = {}
fc = FileCatalog()
while len( activeDirs ) > 0:
currentDir = activeDirs[0]
res = returnSingleResult( fc.listDirectory( currentDir ) )
activeDirs.remove( currentDir )
if not res['OK'] and res['Message'].endswith( 'The supplied path does not exist' ):
self.log.info( "The supplied directory %s does not exist" % currentDir )
elif not res['OK']:
if "No such file or directory" in res['Message']:
self.log.info( "%s: %s" % ( currentDir, res['Message'] ) )
else:
self.log.error( "Failed to get directory %s content: %s" % ( currentDir, res['Message'] ) )
else:
dirContents = res['Value']
activeDirs.extend( dirContents['SubDirs'] )
allFiles.update( dirContents['Files'] )
self.log.info( "Found %d files" % len( allFiles ) )
return S_OK( allFiles.keys() )
def cleanTransformationLogFiles( self, directory ):
""" clean up transformation logs from directory :directory:
:param self: self reference
:param str directory: folder name
"""
self.log.info( "Removing log files found in the directory %s" % directory )
res = returnSingleResult( StorageElement( self.logSE ).removeDirectory( directory ) )
if not res['OK']:
self.log.error( "Failed to remove log files", res['Message'] )
return res
self.log.info( "Successfully removed transformation log directory" )
return S_OK()
#############################################################################
#
# These are the functional methods for archiving and cleaning transformations
#
def removeTransformationOutput( self, transID ):
""" This just removes any mention of the output data from the catalog and storage """
self.log.info( "Removing output data for transformation %s" % transID )
res = self.getTransformationDirectories( transID )
if not res['OK']:
self.log.error( 'Problem obtaining directories for transformation %s with result "%s"' % ( transID, res ) )
return S_OK()
directories = res['Value']
for directory in directories:
if not re.search( '/LOG/', directory ):
res = self.cleanCatalogContents( directory )
if not res['OK']:
return res
res = self.cleanStorageContents( directory )
if not res['OK']:
return res
self.log.info( "Removed directories in the catalog and storage for transformation" )
# Clean ALL the possible remnants found in the metadata catalog
res = self.cleanMetadataCatalogFiles( transID )
if not res['OK']:
return res
self.log.info( "Successfully removed output of transformation %d" % transID )
# Change the status of the transformation to RemovedFiles
res = self.transClient.setTransformationParameter( transID, 'Status', 'RemovedFiles' )
if not res['OK']:
self.log.error( "Failed to update status of transformation %s to RemovedFiles" % ( transID ), res['Message'] )
return res
self.log.info( "Updated status of transformation %s to RemovedFiles" % ( transID ) )
return S_OK()
def archiveTransformation( self, transID ):
""" This just removes job from the jobDB and the transformation DB
:param self: self reference
:param int transID: transformation ID
"""
self.log.info( "Archiving transformation %s" % transID )
# Clean the jobs in the WMS and any failover requests found
res = self.cleanTransformationTasks( transID )
if not res['OK']:
return res
# Clean the transformation DB of the files and job information
res = self.transClient.cleanTransformation( transID )
if not res['OK']:
return res
self.log.info( "Successfully archived transformation %d" % transID )
# Change the status of the transformation to archived
res = self.transClient.setTransformationParameter( transID, 'Status', 'Archived' )
if not res['OK']:
self.log.error( "Failed to update status of transformation %s to Archived" % ( transID ), res['Message'] )
return res
self.log.info( "Updated status of transformation %s to Archived" % ( transID ) )
return S_OK()
def cleanTransformation( self, transID ):
""" This removes what was produced by the supplied transformation,
leaving only some info and log in the transformation DB.
"""
self.log.info( "Cleaning transformation %s" % transID )
res = self.getTransformationDirectories( transID )
if not res['OK']:
self.log.error( 'Problem obtaining directories for transformation %s with result "%s"' % ( transID, res ) )
return S_OK()
directories = res['Value']
# Clean the jobs in the WMS and any failover requests found
res = self.cleanTransformationTasks( transID )
if not res['OK']:
return res
# Clean the log files for the jobs
for directory in directories:
if re.search( '/LOG/', directory ):
res = self.cleanTransformationLogFiles( directory )
if not res['OK']:
return res
res = self.cleanCatalogContents( directory )
if not res['OK']:
return res
res = self.cleanStorageContents( directory )
if not res['OK']:
return res
# Clean ALL the possible remnants found in the BK
res = self.cleanMetadataCatalogFiles( transID )
if not res['OK']:
return res
# Clean the transformation DB of the files and job information
res = self.transClient.cleanTransformation( transID )
if not res['OK']:
return res
self.log.info( "Successfully cleaned transformation %d" % transID )
res = self.transClient.setTransformationParameter( transID, 'Status', 'Cleaned' )
if not res['OK']:
self.log.error( "Failed to update status of transformation %s to Cleaned" % ( transID ), res['Message'] )
return res
self.log.info( "Updated status of transformation %s to Cleaned" % ( transID ) )
return S_OK()
def cleanMetadataCatalogFiles( self, transID ):
""" wipe out files from catalog """
res = self.metadataClient.findFilesByMetadata( { self.transfidmeta : transID } )
if not res['OK']:
return res
fileToRemove = res['Value']
if not fileToRemove:
self.log.info( 'No files found for transID %s' % transID )
return S_OK()
res = self.dm.removeFile( fileToRemove, force = True )
if not res['OK']:
return res
for lfn, reason in res['Value']['Failed'].items():
self.log.error( "Failed to remove file found in metadata catalog", "%s %s" % ( lfn, reason ) )
if res['Value']['Failed']:
return S_ERROR( "Failed to remove all files found in the metadata catalog" )
self.log.info( "Successfully removed all files found in the BK" )
return S_OK()
#############################################################################
#
# These are the methods for removing the jobs from the WMS and transformation DB
#
def cleanTransformationTasks( self, transID ):
""" clean tasks from WMS, or from the RMS if it is a DataManipulation transformation
"""
res = self.__getTransformationExternalIDs( transID )
if not res['OK']:
return res
externalIDs = res['Value']
if externalIDs:
res = self.transClient.getTransformationParameters( transID, ['Type'] )
if not res['OK']:
self.log.error( "Failed to determine transformation type" )
return res
transType = res['Value']
if transType in self.dataProcTTypes:
res = self.__removeWMSTasks( externalIDs )
else:
res = self.__removeRequests( externalIDs )
if not res['OK']:
return res
return S_OK()
def __getTransformationExternalIDs( self, transID ):
""" collect all ExternalIDs for transformation :transID:
:param self: self reference
:param int transID: transforamtion ID
"""
res = self.transClient.getTransformationTasks( condDict = { 'TransformationID' : transID } )
if not res['OK']:
self.log.error( "Failed to get externalIDs for transformation %d" % transID, res['Message'] )
return res
externalIDs = [ taskDict['ExternalID'] for taskDict in res["Value"] ]
self.log.info( "Found %d tasks for transformation" % len( externalIDs ) )
return S_OK( externalIDs )
def __removeRequests( self, requestIDs ):
""" This will remove requests from the (new) RMS system -
#FIXME: if the old system is still installed, it won't remove anything!!!
(we don't want to risk removing from the new RMS what is instead in the old)
"""
# FIXME: checking if the old system is still installed!
from DIRAC.ConfigurationSystem.Client import PathFinder
if PathFinder.getServiceURL( "RequestManagement/RequestManager" ):
self.log.warn( "NOT removing requests!!" )
return S_OK()
rIDs = [ int( long( j ) ) for j in requestIDs if long( j ) ]
for requestName in rIDs:
self.reqClient.deleteRequest( requestName )
return S_OK()
def __removeWMSTasks( self, transJobIDs ):
""" wipe out jobs and their requests from the system
TODO: should check request status, maybe FTS files as well ???
:param self: self reference
:param list trasnJobIDs: job IDs
"""
# Prevent 0 job IDs
jobIDs = [ int( j ) for j in transJobIDs if int( j ) ]
allRemove = True
for jobList in breakListIntoChunks( jobIDs, 500 ):
res = self.wmsClient.killJob( jobList )
if res['OK']:
self.log.info( "Successfully killed %d jobs from WMS" % len( jobList ) )
elif ( "InvalidJobIDs" in res ) and ( "NonauthorizedJobIDs" not in res ) and ( "FailedJobIDs" not in res ):
self.log.info( "Found %s jobs which did not exist in the WMS" % len( res['InvalidJobIDs'] ) )
elif "NonauthorizedJobIDs" in res:
self.log.error( "Failed to kill %s jobs because not authorized" % len( res['NonauthorizedJobIDs'] ) )
allRemove = False
elif "FailedJobIDs" in res:
self.log.error( "Failed to kill %s jobs" % len( res['FailedJobIDs'] ) )
allRemove = False
res = self.wmsClient.deleteJob( jobList )
if res['OK']:
self.log.info( "Successfully removed %d jobs from WMS" % len( jobList ) )
elif ( "InvalidJobIDs" in res ) and ( "NonauthorizedJobIDs" not in res ) and ( "FailedJobIDs" not in res ):
self.log.info( "Found %s jobs which did not exist in the WMS" % len( res['InvalidJobIDs'] ) )
elif "NonauthorizedJobIDs" in res:
self.log.error( "Failed to remove %s jobs because not authorized" % len( res['NonauthorizedJobIDs'] ) )
allRemove = False
elif "FailedJobIDs" in res:
self.log.error( "Failed to remove %s jobs" % len( res['FailedJobIDs'] ) )
allRemove = False
if not allRemove:
return S_ERROR( "Failed to remove all remnants from WMS" )
self.log.info( "Successfully removed all tasks from the WMS" )
if not jobIDs:
self.log.info( "JobIDs not present, unable to remove asociated requests." )
return S_OK()
failed = 0
# FIXME: double request client: old/new -> only the new will survive sooner or later
# this is the old
try:
res = RequestClient().getRequestForJobs( jobIDs )
if not res['OK']:
self.log.error( "Failed to get requestID for jobs.", res['Message'] )
return res
failoverRequests = res['Value']
self.log.info( "Found %d jobs with associated failover requests (in the old RMS)" % len( failoverRequests ) )
if not failoverRequests:
return S_OK()
for jobID, requestName in failoverRequests.items():
# Put this check just in case, tasks must have associated jobs
if jobID == 0 or jobID == '0':
continue
res = RequestClient().deleteRequest( requestName )
if not res['OK']:
self.log.error( "Failed to remove request from RequestDB", res['Message'] )
failed += 1
else:
self.log.verbose( "Removed request %s associated to job %d." % ( requestName, jobID ) )
except RuntimeError:
failoverRequests = {}
pass
# FIXME: and this is the new
res = self.reqClient.getRequestNamesForJobs( jobIDs )
if not res['OK']:
self.log.error( "Failed to get requestID for jobs.", res['Message'] )
return res
failoverRequests.update( res['Value']['Successful'] )
if not failoverRequests:
return S_OK()
for jobID, requestName in res['Value']['Successful'].items():
# Put this check just in case, tasks must have associated jobs
if jobID == 0 or jobID == '0':
continue
res = self.reqClient.deleteRequest( requestName )
if not res['OK']:
self.log.error( "Failed to remove request from RequestDB", res['Message'] )
failed += 1
else:
self.log.verbose( "Removed request %s associated to job %d." % ( requestName, jobID ) )
if failed:
self.log.info( "Successfully removed %s requests" % ( len( failoverRequests ) - failed ) )
self.log.info( "Failed to remove %s requests" % failed )
return S_ERROR( "Failed to remove all the request from RequestDB" )
self.log.info( "Successfully removed all the associated failover requests" )
return S_OK()
from DIRAC import S_OK, S_ERROR, gLogger, exit
from DIRAC.DataManagementSystem.Client.DataManager import DataManager
lfn = args[0]
pfn = args[1]
se = args[2]
exit_code = 0
log = ''
dm = DataManager()
start = time.time()
result = dm.removeFile( lfn )
result = dm.putAndRegister( lfn, pfn, se )
uploadTime = time.time() - start
if result[ 'OK' ]:
log += 'Succeed to upload file to SE %s.\n' % se
log += 'Upload Time : %ss\n' % uploadTime
start = time.time()
result = dm.getReplica( lfn, se, tempfile.gettempdir() )
downloadTime = time.time() - start
if result[ 'OK' ]:
log += 'Succeed to download file from SE %s.\n' % se
log += 'Download Time : %ss\n' % downloadTime
else:
exit_code = 1
log += 'Failed to download file from SE %s : %s\n' % ( se, result[ 'Message' ] )
