def do_getOutputFiles( self, args ):
"""Get output files for the transformation
usage: getOutputFiles <transName|ID>
"""
argss = args.split()
if not len( argss ) > 0:
print "no transformation supplied"
return
transName = argss[0]
res = self.server.getTransformation( transName )
if not res['OK']:
print "Failed to get transformation information: %s" % res['Message']
else:
fc = FileCatalog()
meta = {}
meta ['ProdID'] = transName
res = fc.findFilesByMetadata( meta )
if not res['OK']:
print res['Message']
return
if not len( res['Value'] ) > 0:
print 'No output files yet for transformation %d' %int(transName)
return
else:
for lfn in res['Value']:
print lfn
def __init__( self, operation = None, csPath = None ):
"""c'tor
:param self: self reference
:param Operation operation: Operation instance
:param str csPath: CS path for this handler
"""
super( ReplicateAndRegister, self ).__init__( operation, csPath )
# # own gMonitor stuff for files
gMonitor.registerActivity( "ReplicateAndRegisterAtt", "Replicate and register attempted",
"RequestExecutingAgent", "Files/min", gMonitor.OP_SUM )
gMonitor.registerActivity( "ReplicateOK", "Replications successful",
"RequestExecutingAgent", "Files/min", gMonitor.OP_SUM )
gMonitor.registerActivity( "ReplicateFail", "Replications failed",
"RequestExecutingAgent", "Files/min", gMonitor.OP_SUM )
gMonitor.registerActivity( "RegisterOK", "Registrations successful",
"RequestExecutingAgent", "Files/min", gMonitor.OP_SUM )
gMonitor.registerActivity( "RegisterFail", "Registrations failed",
"RequestExecutingAgent", "Files/min", gMonitor.OP_SUM )
# # for FTS
gMonitor.registerActivity( "FTSScheduleAtt", "Files schedule attempted",
"RequestExecutingAgent", "Files/min", gMonitor.OP_SUM )
gMonitor.registerActivity( "FTSScheduleOK", "File schedule successful",
"RequestExecutingAgent", "Files/min", gMonitor.OP_SUM )
gMonitor.registerActivity( "FTSScheduleFail", "File schedule failed",
"RequestExecutingAgent", "Files/min", gMonitor.OP_SUM )
# # SE cache
# Clients
self.fc = FileCatalog()
if hasattr( self, "FTSMode" ) and getattr( self, "FTSMode" ):
from DIRAC.DataManagementSystem.Client.FTSClient import FTSClient
self.ftsClient = FTSClient()
def getOutgoingFiles( self, transferDict ):
"""
Get list of files to be processed from InputPath
"""
inputFCName = transferDict['InputFC']
inputPath = transferDict['InputPath']
if inputFCName == 'LocalDisk':
files = []
try:
for fileName in os.listdir( inputPath ):
if os.path.isfile( os.path.join( inputPath, fileName ) ):
files.append( fileName )
except:
pass
return files
inputFC = FileCatalog( [inputFCName] )
result = inputFC.listDirectory( inputPath, True )
if not result['OK']:
self.log.error( result['Message'] )
return []
if not inputPath in result['Value']['Successful']:
self.log.error( result['Value']['Failed'][inputPath] )
return []
subDirs = result['Value']['Successful'][inputPath]['SubDirs']
files = result['Value']['Successful'][inputPath]['Files']
for subDir in subDirs:
self.log.info( 'Ignoring subdirectory:', subDir )
return files.keys()
def __transferIfNotRegistered( self, file, transferDict ):
result = self.isRegisteredInOutputCatalog( file, transferDict )
if not result[ 'OK' ]:
self.log.error( result[ 'Message' ] )
return result
#Already registered. Need to delete
if result[ 'Value' ]:
self.log.info( "Transfer file %s is already registered in the output catalog" % file )
#Delete
filePath = os.path.join( transferDict[ 'InputPath' ], file )
if transferDict[ 'InputFC' ] == 'LocalDisk':
os.unlink( filePath )
#FIXME: what is inFile supposed to be ??
else:
inputFC = FileCatalog( [ transferDict['InputFC'] ] )
replicaDict = inputFC.getReplicas( filePath )
if not replicaDict['OK']:
self.log.error( "Error deleting file", replicaDict['Message'] )
elif not inFile in replicaDict['Value']['Successful']:
self.log.error( "Error deleting file", replicaDict['Value']['Failed'][inFile] )
else:
seList = replicaDict['Value']['Successful'][inFile].keys()
for se in seList:
se = StorageElement( se )
self.log.info( 'Removing from %s:' % se.name, inFile )
se.removeFile( inFile )
inputFC.removeFile( file )
self.log.info( "File %s deleted from %s" % ( file, transferDict[ 'InputFC' ] ) )
self.__processingFiles.discard( file )
return S_OK( file )
#Do the transfer
return self.__retrieveAndUploadFile( file, transferDict )
def __getCatalogDirectoryContents(self, directories):
""" get catalog contents under paths :directories:
:param self: self reference
:param list directories: list of paths in catalog
"""
self.log.info('Obtaining the catalog contents for %d directories:' %
len(directories))
for directory in directories:
self.log.info(directory)
activeDirs = directories
allFiles = {}
fc = FileCatalog()
while activeDirs:
currentDir = activeDirs[0]
res = returnSingleResult(fc.listDirectory(currentDir))
activeDirs.remove(currentDir)
if not res['OK'] and 'Directory does not exist' in res[
'Message']: # FIXME: DFC should return errno
self.log.info("The supplied directory %s does not exist" %
currentDir)
elif not res['OK']:
if "No such file or directory" in res['Message']:
self.log.info("%s: %s" % (currentDir, res['Message']))
else:
self.log.error("Failed to get directory %s content: %s" %
(currentDir, res['Message']))
else:
dirContents = res['Value']
activeDirs.extend(dirContents['SubDirs'])
allFiles.update(dirContents['Files'])
self.log.info("Found %d files" % len(allFiles))
return S_OK(allFiles.keys())
def main():
Script.parseCommandLine()
import DIRAC
from DIRAC import gLogger
args = Script.getPositionalArgs()
if len(args) != 1:
Script.showHelp()
guids = args[0]
try:
guids = guids.split(',')
except BaseException:
pass
from DIRAC.Resources.Catalog.FileCatalog import FileCatalog
fc = FileCatalog()
res = fc.getLFNForGUID(guids)
if not res['OK']:
gLogger.error("Failed to get the LFNs", res['Message'])
DIRAC.exit(-2)
errorGuid = {}
for guid, reason in res['Value']['Failed'].items():
errorGuid.setdefault(reason, []).append(guid)
for error, guidList in errorGuid.items():
gLogger.notice("Error '%s' for guids %s" % (error, guidList))
for guid, lfn in res['Value']['Successful'].items():
gLogger.notice("%s -> %s" % (guid, lfn))
DIRAC.exit(0)
class TestUserMetadataBasicTestCase(unittest.TestCase):
def setUp(self):
self.dirac = Dirac()
csAPI = CSAPI()
self.lfn5 = os.path.join(DESTINATION_PATH, 'test_file_10MB_v5.bin')
self.dir5 = os.path.dirname(self.lfn5)
# local file, for now:
self.fname = os.path.basename(self.lfn5)
random_dd(self.fname, 10)
self.diracSE = 'SE-1'
try:
self.fc = FileCatalog(['MultiVOFileCatalog'])
except Exception:
self.fail(
" FileCatalog(['MultiVOFileCatalog']) raised Exception unexpectedly!\n"
+ traceback.format_exc())
return
# add a replica
self.fileadded = self.dirac.addFile(self.lfn5, self.fname,
self.diracSE)
self.assertTrue(self.fileadded['OK'])
def tearDown(self):
# meta index -r
result = self.fc.deleteMetadataField('MetaInt6')
self.assertTrue(result['OK'])
result = self.fc.deleteMetadataField('TestDirectory6')
self.assertTrue(result['OK'])
# remove the MultiVOFileCatalog
self.fc.removeCatalog('MultiVOFileCatalog')
# delete a sole replica: dirac-dms-remove-files
result = self.dirac.removeFile(self.lfn5)
self.assertTrue(result['OK'])
os.remove(self.fname)
def __init__(self, **kwargs):
GridBackend.__init__(self, catalogue_prefix='', **kwargs)
from DIRAC.Core.Base import Script
Script.initialize()
from DIRAC.FrameworkSystem.Client.ProxyManagerClient import ProxyManagerClient
self.pm = ProxyManagerClient()
proxy = self.pm.getUserProxiesInfo()
if not proxy['OK']:
raise BackendException("Proxy error.")
from DIRAC.Interfaces.API.Dirac import Dirac
self.dirac = Dirac()
from DIRAC.Resources.Catalog.FileCatalog import FileCatalog
self.fc = FileCatalog()
from DIRAC.DataManagementSystem.Client.DataManager import DataManager
self.dm = DataManager()
self._xattr_cmd = sh.Command('gfal-xattr').bake(_tty_out=False)
self._replica_checksum_cmd = sh.Command('gfal-sum').bake(_tty_out=False)
self._bringonline_cmd = sh.Command('gfal-legacy-bringonline').bake(_tty_out=False)
self._cp_cmd = sh.Command('gfal-copy').bake(_tty_out=False)
self._ls_se_cmd = sh.Command('gfal-ls').bake(color='never', _tty_out=False)
self._move_cmd = sh.Command('gfal-rename').bake(_tty_out=False)
self._mkdir_cmd = sh.Command('gfal-mkdir').bake(_tty_out=False)
self._replicate_cmd = sh.Command('dirac-dms-replicate-lfn').bake(_tty_out=False)
self._add_cmd = sh.Command('dirac-dms-add-file').bake(_tty_out=False)
def main():
# Registering arguments will automatically add their description to the help menu
Script.registerArgument(
("LFN: LFN",
"File: File name containing a list of affected LFNs"))
Script.registerArgument(" SE: Name of Storage Element")
Script.registerArgument(" Status: New Status for the replica")
Script.parseCommandLine(ignoreErrors=False)
import DIRAC
from DIRAC import gLogger
from DIRAC.Resources.Catalog.FileCatalog import FileCatalog
import os
# parseCommandLine show help when mandatory arguments are not specified or incorrect argument
inputFileName, storageElement, status = Script.getPositionalArgs(
group=True)
if os.path.exists(inputFileName):
inputFile = open(inputFileName, "r")
string = inputFile.read()
inputFile.close()
lfns = sorted(string.splitlines())
else:
lfns = [inputFileName]
fc = FileCatalog()
res = fc.getReplicas(lfns, allStatus=True)
if not res["OK"]:
gLogger.error("Failed to get catalog replicas.", res["Message"])
DIRAC.exit(-1)
lfnDict = {}
for lfn, error in res["Value"]["Failed"].items():
gLogger.error("Failed to get replicas for file.",
"%s:%s" % (lfn, error))
for lfn, replicas in res["Value"]["Successful"].items():
if storageElement not in replicas.keys():
gLogger.error("LFN not registered at provided storage element.",
"%s %s" % (lfn, storageElement))
else:
lfnDict[lfn] = {
"SE": storageElement,
"PFN": replicas[storageElement],
"Status": status
}
if not lfnDict:
gLogger.error("No files found at the supplied storage element.")
DIRAC.exit(2)
res = fc.setReplicaStatus(lfnDict)
if not res["OK"]:
gLogger.error("Failed to set catalog replica status.", res["Message"])
DIRAC.exit(-1)
for lfn, error in res["Value"]["Failed"].items():
gLogger.error("Failed to set replica status for file.",
"%s:%s" % (lfn, error))
gLogger.notice("Successfully updated the status of %d files at %s." %
(len(res["Value"]["Successful"].keys()), storageElement))
DIRAC.exit(0)
def do_getOutputFiles(self, args):
"""Get output files for the transformation
usage: getOutputFiles <transName|ID>
"""
argss = args.split()
if not len(argss) > 0:
print("no transformation supplied")
return
transName = argss[0]
res = self.transClient.getTransformation(transName)
if not res["OK"]:
print("Failed to get transformation information: %s" %
res["Message"])
else:
fc = FileCatalog()
meta = {}
meta["ProdID"] = transName
res = fc.findFilesByMetadata(meta)
if not res["OK"]:
print(res["Message"])
return
if not len(res["Value"]) > 0:
print("No output files yet for transformation %d" %
int(transName))
return
else:
for lfn in res["Value"]:
print(lfn)
class TestClientProductionTestCase(unittest.TestCase):
def setUp(self):
self.prodClient = ProductionClient()
self.transClient = TransformationClient()
self.fc = FileCatalog()
# ## Add metadata fields to the DFC
self.MDFieldDict = {
'particle': 'VARCHAR(128)',
'analysis_prog': 'VARCHAR(128)',
'tel_sim_prog': 'VARCHAR(128)',
'outputType': 'VARCHAR(128)',
'zenith': 'int',
'data_level': 'int'
}
for MDField in self.MDFieldDict:
MDFieldType = self.MDFieldDict[MDField]
res = self.fc.addMetadataField(MDField, MDFieldType)
self.assert_(res['OK'])
def tearDown(self):
# Delete meta data fields
for MDField in self.MDFieldDict:
res = self.fc.deleteMetadataField(MDField)
self.assert_(res['OK'])
def getOutgoingFiles(self, transferDict):
"""
Get list of files to be processed from InputPath
"""
inputFCName = transferDict['InputFC']
inputPath = transferDict['InputPath']
if inputFCName == 'LocalDisk':
files = []
try:
for file in os.listdir(inputPath):
if os.path.isfile(os.path.join(inputPath, file)):
files.append(file)
except:
pass
return files
inputFC = FileCatalog([inputFCName])
result = inputFC.listDirectory(inputPath, True)
if not result['OK']:
self.log.error(result['Message'])
return []
if not inputPath in result['Value']['Successful']:
self.log.error(result['Value']['Failed'][inputPath])
return []
subDirs = result['Value']['Successful'][inputPath]['SubDirs']
files = result['Value']['Successful'][inputPath]['Files']
for dir in subDirs:
self.log.info('Ignoring subdirectory:', dir)
return files.keys()
def web_getLaunchpadSetupWithLFNs(self):
""" Method obtain launchpad setup with pre-selected LFNs as input data parameter,
the caller js client will use setup to open an new Launchpad
"""
# On the fly file catalog for advanced launchpad
if not hasattr(self, 'fc'):
userData = self.getSessionData()
group = str(userData["user"]["group"])
vo = getVOForGroup(group)
self.fc = FileCatalog(vo=vo)
self.set_header('Content-type', 'text/plain')
arguments = self.request.arguments
gLogger.always("submit: incoming arguments %s to getLaunchpadSetupWithLFNs" % arguments)
lfnList = str(arguments['path'][0]).split(',')
# Modified for Eiscat
# Checks if the experiments folder in lfn list has a rtg_def.m file at some subfolder
gLogger.always("submit: checking if some rtg_def.m", arguments)
processed = []
metaDict = {'type': 'info'}
for lfn in lfnList:
pos_relative = lfn.find("/")
pos_relative = lfn.find("/", pos_relative + 1)
pos_relative = lfn.find("/", pos_relative + 1)
pos_relative = lfn.find("/", pos_relative + 1)
pos_relative = lfn.find("/", pos_relative + 1)
experiment_lfn = lfn[0:pos_relative]
if experiment_lfn in processed:
continue
processed.append(experiment_lfn)
gLogger.always( "checking rtg_def.m in %s" % experiment_lfn )
result = self.fc.findFilesByMetadata( metaDict, path=str(experiment_lfn) )
if not result['OK'] or not result['Value']:
gLogger.error( "Failed to get type info from $s, %s" % (experiment_lfn,result[ "Message" ]) )
continue
for candidate_lfn in result['Value']:
if candidate_lfn.find('rtg_def.m') > 0:
lfnList.append(candidate_lfn)
# End modified
ptlfn = ''
for lfn in lfnList:
ptlfn += (', ' + lfn) if ptlfn else lfn
params = self.defaultParams.copy()
params["InputData"] = [1, ptlfn]
obj = Operations(vo=vo)
predefinedSets = {}
launchpadSections = obj.getSections("Launchpad")
if launchpadSections['OK']:
for section in launchpadSections["Value"]:
predefinedSets[section] = {}
sectionOptions = obj.getOptionsDict("Launchpad/" + section)
pprint.pprint(sectionOptions)
if sectionOptions['OK']:
predefinedSets[section] = sectionOptions["Value"]
self.write({"success": "true", "result": params, "predefinedSets": predefinedSets})
def test_02_condParser(self, mk_getSelectedCatalogs, mk_getEligibleCatalogs):
"""Test behavior of write methode when using FCConditionParser"""
fc = FileCatalog(
catalogs=["c1_True_True_True_2_0_2_0", "c2_False_True_True_3_0_1_0", "c3_False_True_True_3_0_1_0"]
)
# No condition for c3, so it should always pass
fcConditions = {"c1": "Filename=find('c1_pass')", "c2": "Filename=find('c2_pass')"}
# Everything pass everywhere
lfn1 = "/lhcb/c1_pass/c2_pass/lfn1"
lfn2 = "/lhcb/c1_pass/c2_pass/lfn2"
res = fc.write1([lfn1, lfn2], fcConditions=fcConditions)
self.assertTrue(res["OK"])
self.assertEqual(sorted(res["Value"]["Successful"]), sorted([lfn1, lfn2]))
self.assertEqual(sorted(res["Value"]["Successful"][lfn1]), sorted(["c1", "c2", "c3"]))
self.assertEqual(sorted(res["Value"]["Successful"][lfn2]), sorted(["c1", "c2", "c3"]))
self.assertTrue(not res["Value"]["Failed"])
# Everything pass for the master, only lfn2 for c2
lfn1 = "/lhcb/c1_pass/lfn1"
lfn2 = "/lhcb/c1_pass/c2_pass/lfn2"
res = fc.write1([lfn1, lfn2], fcConditions=fcConditions)
self.assertTrue(res["OK"])
self.assertEqual(sorted(res["Value"]["Successful"]), sorted([lfn1, lfn2]))
self.assertEqual(sorted(res["Value"]["Successful"][lfn1]), ["c1", "c3"])
self.assertEqual(sorted(res["Value"]["Successful"][lfn2]), sorted(["c1", "c2", "c3"]))
self.assertTrue(not res["Value"]["Failed"])
# One is not valid for the master, so we do nothing
lfn1 = "/lhcb/c2_pass/lfn1"
lfn2 = "/lhcb/c1_pass/c2_pass/lfn2"
res = fc.write1([lfn1, lfn2], fcConditions=fcConditions)
self.assertTrue(not res["OK"])
def __getCatalogDirectoryContents(self, directories):
""" get catalog contents under paths :directories:
:param self: self reference
:param list directories: list of paths in catalog
"""
self.log.info('Obtaining the catalog contents for %d directories:' % len(directories))
for directory in directories:
self.log.info(directory)
activeDirs = directories
allFiles = {}
fc = FileCatalog()
while activeDirs:
currentDir = activeDirs[0]
res = returnSingleResult(fc.listDirectory(currentDir))
activeDirs.remove(currentDir)
if not res['OK'] and 'Directory does not exist' in res['Message']: # FIXME: DFC should return errno
self.log.info("The supplied directory %s does not exist" % currentDir)
elif not res['OK']:
if "No such file or directory" in res['Message']:
self.log.info("%s: %s" % (currentDir, res['Message']))
else:
self.log.error("Failed to get directory %s content: %s" % (currentDir, res['Message']))
else:
dirContents = res['Value']
activeDirs.extend(dirContents['SubDirs'])
allFiles.update(dirContents['Files'])
self.log.info("Found %d files" % len(allFiles))
return S_OK(allFiles.keys())
def __addHomeDirectory( self, vo, newUsers ):
fc = FileCatalog( vo = vo )
defaultVOGroup = getVOOption( vo, "DefaultGroup", "%s_user" % vo )
failed = {}
successful = {}
for user in newUsers:
result = fc.addUser( user )
if not result['OK']:
failed[user] = "addUser"
continue
dirName = '/%s/user/%s/%s' % ( vo, user[0], user )
result = fc.createDirectory( dirName )
if not result['OK']:
failed[user] = "createDirectory"
continue
result = fc.changePathOwner( { dirName: user }, recursive = True )
if not result['OK']:
failed[user] = "changePathOwner"
continue
result = fc.changePathGroup( { dirName: defaultVOGroup }, recursive = True )
if not result['OK']:
failed[user] = "changePathGroup"
continue
successful[user] = True
return S_OK( { "Successful": successful, "Failed": failed } )
class TestClientProductionTestCase(unittest.TestCase):
def setUp(self):
self.prodClient = ProductionClient()
self.transClient = TransformationClient()
self.fc = FileCatalog()
# ## Add metadata fields to the DFC
self.MDFieldDict = {
"particle": "VARCHAR(128)",
"analysis_prog": "VARCHAR(128)",
"tel_sim_prog": "VARCHAR(128)",
"outputType": "VARCHAR(128)",
"zenith": "int",
"data_level": "int",
}
for MDField in self.MDFieldDict:
MDFieldType = self.MDFieldDict[MDField]
res = self.fc.addMetadataField(MDField, MDFieldType)
self.assert_(res["OK"])
def tearDown(self):
# Delete meta data fields
for MDField in self.MDFieldDict:
res = self.fc.deleteMetadataField(MDField)
self.assert_(res["OK"])
def do_getOutputFiles(self, args):
"""Get output files for the transformation
usage: getOutputFiles <transName|ID>
"""
argss = args.split()
if not len(argss) > 0:
print "no transformation supplied"
return
transName = argss[0]
res = self.server.getTransformation(transName)
if not res['OK']:
print "Failed to get transformation information: %s" % res[
'Message']
else:
fc = FileCatalog()
meta = {}
meta['ProdID'] = transName
res = fc.findFilesByMetadata(meta)
if not res['OK']:
print res['Message']
return
if not len(res['Value']) > 0:
print 'No output files yet for transformation %d' % int(
transName)
return
else:
for lfn in res['Value']:
print lfn
def setUp(self):
self.fullMetadata = [
'Status', 'ChecksumType', 'OwnerRole', 'CreationDate', 'Checksum',
'ModificationDate', 'OwnerDN', 'Mode', 'GUID', 'Size'
]
self.dirMetadata = self.fullMetadata + ['NumberOfSubPaths']
self.fileMetadata = self.fullMetadata + ['NumberOfLinks']
self.catalog = FileCatalog(catalogs=[catalogClientToTest])
valid = self.catalog.isOK()
self.assert_(valid)
self.destDir = '/lhcb/test/unit-test/TestCatalogPlugin'
self.link = "%s/link" % self.destDir
# Clean the existing directory
self.cleanDirectory()
res = self.catalog.createDirectory(self.destDir)
returnValue = self.parseResult(res, self.destDir)
# Register some files to work with
self.numberOfFiles = 2
self.files = []
for i in range(self.numberOfFiles):
lfn = "%s/testFile_%d" % (self.destDir, i)
res = self.registerFile(lfn)
self.assert_(res)
self.files.append(lfn)
def setupCatalogClient( self, catalogs ):
""" Init FileCatalog client
Ideally we would like to use only FileCatalog but it doesn't work for setMetadata
because the returned value has a wrong format. So use of FileCatalogClient instead
"""
self.fc = FileCatalog( catalogs )
self.fcc = FileCatalogClient()
def setUp(self):
self.fullMetadata = [
"Status",
"ChecksumType",
"OwnerRole",
"CreationDate",
"Checksum",
"ModificationDate",
"OwnerDN",
"Mode",
"GUID",
"Size",
]
self.dirMetadata = self.fullMetadata + ["NumberOfSubPaths"]
self.fileMetadata = self.fullMetadata + ["NumberOfLinks"]
self.catalog = FileCatalog(catalogs=[catalogClientToTest])
valid = self.catalog.isOK()
self.assertTrue(valid)
self.destDir = "/lhcb/test/unit-test/TestCatalogPlugin"
self.link = "%s/link" % self.destDir
# Clean the existing directory
self.cleanDirectory()
res = self.catalog.createDirectory(self.destDir)
returnValue = self.parseResult(res, self.destDir)
# Register some files to work with
self.numberOfFiles = 2
self.files = []
for i in range(self.numberOfFiles):
lfn = "%s/testFile_%d" % (self.destDir, i)
res = self.registerFile(lfn)
self.assertTrue(res)
self.files.append(lfn)
def test_01_oneMasterNormal(self, mk_getSelectedCatalogs,
mk_getEligibleCatalogs):
"""Test behavior with one master and only standard read methods"""
fc = FileCatalog(catalogs=[
'c1_True_True_True_2_0_2_0', 'c2_False_True_True_3_0_1_0'
])
# Test a write method which is not in the master catalog
with self.assertRaises(AttributeError):
fc.write4('/lhcb/toto')
# Test a write method which works for everybody
lfn = '/lhcb/toto'
res = fc.write1(lfn)
self.assertTrue(res['OK'])
self.assertTrue(lfn in res['Value']['Successful'])
self.assertEqual(sorted(['c1', 'c2']),
sorted(res['Value']['Successful'][lfn].keys()))
self.assertTrue(not res['Value']['Failed'])
# Test a write method that only the master has
lfn = '/lhcb/toto'
res = fc.write2(lfn)
self.assertTrue(res['OK'])
self.assertTrue(lfn in res['Value']['Successful'])
self.assertEqual(['c1'], res['Value']['Successful'][lfn].keys())
self.assertTrue(not res['Value']['Failed'])
# Test a write method that makes an error for master
# We should get an error
lfn = '/lhcb/c1/Error'
res = fc.write1(lfn)
self.assertTrue(not res['OK'])
# Test a write method that fails for master
# The lfn should be in failed and only attempted for the master
lfn = '/lhcb/c1/Failed'
res = fc.write1(lfn)
self.assertTrue(res['OK'])
self.assertTrue(not res['Value']['Successful'])
self.assertEqual(['c1'], res['Value']['Failed'][lfn].keys())
# Test a write method that makes an error for non master
# The lfn should be in failed for non master and successful for the master
lfn = '/lhcb/c2/Error'
res = fc.write1(lfn)
self.assertTrue(res['OK'])
self.assertEqual(['c1'], res['Value']['Successful'][lfn].keys())
self.assertEqual(['c2'], res['Value']['Failed'][lfn].keys())
# Test a write method that fails for non master
# The lfn should be in failed for non master and successful for the master
lfn = '/lhcb/c2/Failed'
res = fc.write1(lfn)
self.assertTrue(res['OK'])
self.assertEqual(['c1'], res['Value']['Successful'][lfn].keys())
self.assertEqual(['c2'], res['Value']['Failed'][lfn].keys())
def main():
Script.parseCommandLine(ignoreErrors=False)
import DIRAC
from DIRAC import gLogger
from DIRAC.Resources.Catalog.FileCatalog import FileCatalog
import os
args = Script.getPositionalArgs()
if not len(args) == 3:
Script.showHelp()
inputFileName = args[0]
storageElement = args[1]
status = args[2]
if os.path.exists(inputFileName):
inputFile = open(inputFileName, 'r')
string = inputFile.read()
inputFile.close()
lfns = sorted(string.splitlines())
else:
lfns = [inputFileName]
fc = FileCatalog()
res = fc.getReplicas(lfns, allStatus=True)
if not res['OK']:
gLogger.error("Failed to get catalog replicas.", res['Message'])
DIRAC.exit(-1)
lfnDict = {}
for lfn, error in res['Value']['Failed'].items():
gLogger.error("Failed to get replicas for file.",
"%s:%s" % (lfn, error))
for lfn, replicas in res['Value']['Successful'].items():
if storageElement not in replicas.keys():
gLogger.error("LFN not registered at provided storage element.",
"%s %s" % (lfn, storageElement))
else:
lfnDict[lfn] = {
'SE': storageElement,
'PFN': replicas[storageElement],
'Status': status
}
if not lfnDict:
gLogger.error("No files found at the supplied storage element.")
DIRAC.exit(2)
res = fc.setReplicaStatus(lfnDict)
if not res['OK']:
gLogger.error("Failed to set catalog replica status.", res['Message'])
DIRAC.exit(-1)
for lfn, error in res['Value']['Failed'].items():
gLogger.error("Failed to set replica status for file.",
"%s:%s" % (lfn, error))
gLogger.notice("Successfully updated the status of %d files at %s." %
(len(res['Value']['Successful'].keys()), storageElement))
DIRAC.exit(0)
def __call__( self ):
""" action for 'removeFile' operation """
# # get waiting files
waitingFiles = self.getWaitingFilesList()
fc = FileCatalog( self.operation.catalogList )
res = fc.getReplicas( [wf.LFN for wf in waitingFiles] )
if not res['OK']:
gMonitor.addMark( "RemoveFileAtt" )
gMonitor.addMark( "RemoveFileFail" )
return res
# We check the status of the SE from the LFN that are successful
# No idea what to do with the others...
succ = res['Value']['Successful']
targetSEs = set( [se for lfn in succ for se in succ[lfn] ] )
if targetSEs:
bannedTargets = self.checkSEsRSS( targetSEs, access = 'RemoveAccess' )
if not bannedTargets['OK']:
gMonitor.addMark( "RemoveFileAtt" )
gMonitor.addMark( "RemoveFileFail" )
return bannedTargets
if bannedTargets['Value']:
return S_OK( "%s targets are banned for removal" % ",".join( bannedTargets['Value'] ) )
# # prepare waiting file dict
toRemoveDict = dict( [ ( opFile.LFN, opFile ) for opFile in waitingFiles ] )
gMonitor.addMark( "RemoveFileAtt", len( toRemoveDict ) )
# # 1st step - bulk removal
self.log.debug( "bulk removal of %s files" % len( toRemoveDict ) )
bulkRemoval = self.bulkRemoval( toRemoveDict )
if not bulkRemoval["OK"]:
self.log.error( "Bulk file removal failed", bulkRemoval["Message"] )
else:
gMonitor.addMark( "RemoveFileOK", len( toRemoveDict ) - len( bulkRemoval["Value"] ) )
toRemoveDict = bulkRemoval["Value"]
# # 2nd step - single file removal
for lfn, opFile in toRemoveDict.items():
self.log.info( "removing single file %s" % lfn )
singleRemoval = self.singleRemoval( opFile )
if not singleRemoval["OK"]:
self.log.error( 'Error removing single file', singleRemoval["Message"] )
gMonitor.addMark( "RemoveFileFail", 1 )
else:
self.log.info( "file %s has been removed" % lfn )
gMonitor.addMark( "RemoveFileOK", 1 )
# # set
failedFiles = [ ( lfn, opFile ) for ( lfn, opFile ) in toRemoveDict.items()
if opFile.Status in ( "Failed", "Waiting" ) ]
if failedFiles:
self.operation.Error = "failed to remove %d files" % len( failedFiles )
return S_OK()
def main():
from DIRAC.Core.Base import Script
Script.registerSwitch('', 'Path=', ' Path to search for')
Script.registerSwitch(
'', 'SE=',
' (comma-separated list of) SEs/SE-groups to be searched')
Script.parseCommandLine(ignoreErrors=True)
args = Script.getPositionalArgs()
import DIRAC
from DIRAC import gLogger
from DIRAC.Resources.Catalog.FileCatalog import FileCatalog
from DIRAC.DataManagementSystem.Client.MetaQuery import MetaQuery, FILE_STANDARD_METAKEYS
from DIRAC.DataManagementSystem.Utilities.DMSHelpers import resolveSEGroup
path = '/'
seList = None
for opt, val in Script.getUnprocessedSwitches():
if opt == 'Path':
path = val
elif opt == 'SE':
seList = resolveSEGroup(val.split(','))
if seList:
args.append("SE=%s" % ','.join(seList))
fc = FileCatalog()
result = fc.getMetadataFields()
if not result['OK']:
gLogger.error('Can not access File Catalog:', result['Message'])
DIRAC.exit(-1)
typeDict = result['Value']['FileMetaFields']
typeDict.update(result['Value']['DirectoryMetaFields'])
# Special meta tags
typeDict.update(FILE_STANDARD_METAKEYS)
if len(args) < 1:
print("Error: No argument provided\n%s:" % Script.scriptName)
gLogger.notice("MetaDataDictionary: \n%s" % str(typeDict))
Script.showHelp(exitCode=1)
mq = MetaQuery(typeDict=typeDict)
result = mq.setMetaQuery(args)
if not result['OK']:
gLogger.error("Illegal metaQuery:", result['Message'])
DIRAC.exit(-1)
metaDict = result['Value']
path = metaDict.pop('Path', path)
result = fc.findFilesByMetadata(metaDict, path)
if not result['OK']:
gLogger.error('Can not access File Catalog:', result['Message'])
DIRAC.exit(-1)
lfnList = sorted(result['Value'])
gLogger.notice('\n'.join(lfn for lfn in lfnList))
def initialize(self):
""" agent initialisation """
self.rawIntegrityDB = RAWIntegrityDB()
# The file catalog is used to register file once it has been transfered
# But we want to register it in all the catalogs except the RAWIntegrityDB
# otherwise it is register twice
# We also remove the BK catalog because some files are not registered there
# (detector calibration files for example). The real data are registered in
# the bookeeping by the DataMover
self.fileCatalog = FileCatalog(catalogs = 'FileCatalog')
gMonitor.registerActivity("Iteration", "Agent Loops/min", "RAWIntegriryAgent", "Loops",
gMonitor.OP_SUM)
gMonitor.registerActivity("WaitingFiles", "Files waiting for migration", "RAWIntegriryAgent",
"Files", gMonitor.OP_MEAN)
gMonitor.registerActivity("WaitSize", "Size of migration buffer", "RAWIntegrityAgent", "GB",
gMonitor.OP_MEAN)
gMonitor.registerActivity("NewlyMigrated", "Newly migrated files", "RAWIntegriryAgent", "Files",
gMonitor.OP_SUM)
gMonitor.registerActivity("TotMigrated", "Total migrated files", "RAWIntegriryAgent", "Files",
gMonitor.OP_ACUM)
gMonitor.registerActivity("TotMigratedSize", "Total migrated file size", "RAWIntegriryAgent",
"GB", gMonitor.OP_ACUM)
gMonitor.registerActivity("BadChecksum", "Checksums mismatch", "RAWIntegriryAgent", "Files",
gMonitor.OP_SUM)
gMonitor.registerActivity("ErrorMetadata", "Error when getting files metadata",
"RAWIntegriryAgent", "Files", gMonitor.OP_SUM)
gMonitor.registerActivity("ErrorRegister", "Error when registering files", "RAWIntegriryAgent",
"Files", gMonitor.OP_SUM)
gMonitor.registerActivity("ErrorRemove", "Error when removing files", "RAWIntegriryAgent",
"Files", gMonitor.OP_SUM)
gMonitor.registerActivity("FailedMigrated",
"Number of files encountering error during migration",
"RAWIntegriryAgent", "Files", gMonitor.OP_SUM)
gMonitor.registerActivity("TotFailMigrated",
"Total number of files encountering error during migration",
"RAWIntegriryAgent", "Files", gMonitor.OP_ACUM)
gMonitor.registerActivity("MigrationTime", "Average migration time", "RAWIntegriryAgent",
"Seconds", gMonitor.OP_MEAN)
# gMonitor.registerActivity("TimeInQueue", "Average current wait for migration",
# "RAWIntegriryAgent", "Minutes", gMonitor.OP_MEAN)
gMonitor.registerActivity("MigrationRate", "Observed migration rate", "RAWIntegriryAgent",
"MB/s", gMonitor.OP_MEAN)
# This sets the Default Proxy to used as that defined under
# /Operations/Shifter/DataManager
# the shifterProxy option in the Configuration can be used to change this default.
self.am_setOption('shifterProxy', 'DataProcessing')
return S_OK()
def initialize(self):
self.fileCatalog = FileCatalog()
self.dm = DataManager()
self.stagerClient = StorageManagerClient()
self.dataIntegrityClient = DataIntegrityClient()
# This sets the Default Proxy to used as that defined under
# /Operations/Shifter/DataManager
# the shifterProxy option in the Configuration can be used to change this default.
self.am_setOption("shifterProxy", "DataManager")
return S_OK()
def test_01_oneMasterNormal(self, mk_getSelectedCatalogs, mk_getEligibleCatalogs):
"""Test behavior with one master and only standard read methods"""
fc = FileCatalog(catalogs=["c1_True_True_True_2_0_2_0", "c2_False_True_True_3_0_1_0"])
# Test a write method which is not in the master catalog
with self.assertRaises(AttributeError):
fc.write4("/lhcb/toto")
# Test a write method which works for everybody
lfn = "/lhcb/toto"
res = fc.write1(lfn)
self.assertTrue(res["OK"])
self.assertTrue(lfn in res["Value"]["Successful"])
self.assertEqual(sorted(["c1", "c2"]), sorted(res["Value"]["Successful"][lfn]))
self.assertTrue(not res["Value"]["Failed"])
# Test a write method that only the master has
lfn = "/lhcb/toto"
res = fc.write2(lfn)
self.assertTrue(res["OK"])
self.assertTrue(lfn in res["Value"]["Successful"])
self.assertEqual(["c1"], sorted(res["Value"]["Successful"][lfn]))
self.assertTrue(not res["Value"]["Failed"])
# Test a write method that makes an error for master
# We should get an error
lfn = "/lhcb/c1/Error"
res = fc.write1(lfn)
self.assertTrue(not res["OK"])
# Test a write method that fails for master
# The lfn should be in failed and only attempted for the master
lfn = "/lhcb/c1/Failed"
res = fc.write1(lfn)
self.assertTrue(res["OK"])
self.assertTrue(not res["Value"]["Successful"])
self.assertEqual(["c1"], sorted(res["Value"]["Failed"][lfn]))
# Test a write method that makes an error for non master
# The lfn should be in failed for non master and successful for the master
lfn = "/lhcb/c2/Error"
res = fc.write1(lfn)
self.assertTrue(res["OK"])
self.assertEqual(["c1"], sorted(res["Value"]["Successful"][lfn]))
self.assertEqual(["c2"], sorted(res["Value"]["Failed"][lfn]))
# Test a write method that fails for non master
# The lfn should be in failed for non master and successful for the master
lfn = "/lhcb/c2/Failed"
res = fc.write1(lfn)
self.assertTrue(res["OK"])
self.assertEqual(["c1"], sorted(res["Value"]["Successful"][lfn]))
self.assertEqual(["c2"], sorted(res["Value"]["Failed"][lfn]))
def __getCatalogObject( self ):
""" CatalogInterface instance facade
:param self: self reference
"""
try:
if not self.oCatalog:
self.oCatalog = FileCatalog()
return S_OK()
except:
return S_ERROR()
def __init__(self, *args, **kwargs):
super(TransferAppHandler, self).__init__(*args, **kwargs)
sessionData = self.getSessionData()
self.user = sessionData['user'].get('username', '')
self.group = sessionData['user'].get('group', '')
self.vo = getVOForGroup(self.group)
self.fc = FileCatalog(vo=self.vo)
self.log.always(self.user)
self.log.always(self.group)
self.log.always(self.vo)
self.log.always(self.fc)
def main():
# Registering arguments will automatically add their description to the help menu
Script.registerArgument(("LocalFile: Path to local file containing LFNs",
"LFN: Logical File Names"))
Script.parseCommandLine()
from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
allowUsers = Operations().getValue(
"DataManagement/AllowUserReplicaManagement", False)
from DIRAC.Core.Security.ProxyInfo import getProxyInfo
res = getProxyInfo()
if not res["OK"]:
gLogger.fatal("Can't get proxy info", res["Message"])
dexit(1)
properties = res["Value"].get("groupProperties", [])
if not allowUsers:
if "FileCatalogManagement" not in properties:
gLogger.error(
"You need to use a proxy from a group with FileCatalogManagement"
)
dexit(5)
from DIRAC.Resources.Catalog.FileCatalog import FileCatalog
fc = FileCatalog()
import os
# parseCommandLine show help when mandatory arguments are not specified or incorrect argument
args = Script.getPositionalArgs()
inputFileName = args[0]
if os.path.exists(inputFileName):
inputFile = open(inputFileName, "r")
string = inputFile.read()
lfns = [lfn.strip() for lfn in string.splitlines()]
inputFile.close()
else:
lfns = [inputFileName]
res = fc.removeFile(lfns)
if not res["OK"]:
print("Error:", res["Message"])
dexit(1)
for lfn in sorted(res["Value"]["Failed"].keys()):
message = res["Value"]["Failed"][lfn]
print("Error: failed to remove %s: %s" % (lfn, message))
print("Successfully removed %d catalog files." %
(len(res["Value"]["Successful"])))
def test_01_oneMasterNormal( self, mk_getSelectedCatalogs, mk_getEligibleCatalogs ):
"""Test behavior with one master and only standard read methods"""
fc = FileCatalog( catalogs = ['c1_True_True_True_2_0_2_0', 'c2_False_True_True_3_0_1_0'] )
# Test a write method which is not in the master catalog
with self.assertRaises( AttributeError ):
fc.write4( '/lhcb/toto' )
# Test a write method which works for everybody
lfn = '/lhcb/toto'
res = fc.write1( lfn )
self.assert_( res['OK'] )
self.assert_( lfn in res['Value']['Successful'] )
self.assertEqual( sorted( ['c1', 'c2'] ), sorted( res['Value']['Successful'][lfn].keys() ) )
self.assert_( not res['Value']['Failed'] )
# Test a write method that only the master has
lfn = '/lhcb/toto'
res = fc.write2( lfn )
self.assert_( res['OK'] )
self.assert_( lfn in res['Value']['Successful'] )
self.assertEqual( ['c1'], res['Value']['Successful'][lfn].keys() )
self.assert_( not res['Value']['Failed'] )
# Test a write method that makes an error for master
# We should get an error
lfn = '/lhcb/c1/Error'
res = fc.write1( lfn )
self.assert_( not res['OK'] )
# Test a write method that fails for master
# The lfn should be in failed and only attempted for the master
lfn = '/lhcb/c1/Failed'
res = fc.write1( lfn )
self.assert_( res['OK'] )
self.assert_( not res['Value']['Successful'] )
self.assertEqual( ['c1'], res['Value']['Failed'][lfn].keys() )
# Test a write method that makes an error for non master
# The lfn should be in failed for non master and successful for the master
lfn = '/lhcb/c2/Error'
res = fc.write1( lfn )
self.assert_( res['OK'] )
self.assertEqual( ['c1'], res['Value']['Successful'][lfn].keys() )
self.assertEqual( ['c2'], res['Value']['Failed'][lfn].keys() )
# Test a write method that fails for non master
# The lfn should be in failed for non master and successful for the master
lfn = '/lhcb/c2/Failed'
res = fc.write1( lfn )
self.assert_( res['OK'] )
self.assertEqual( ['c1'], res['Value']['Successful'][lfn].keys() )
self.assertEqual( ['c2'], res['Value']['Failed'][lfn].keys() )
def __init__(self, operation=None, csPath=None):
"""c'tor
:param self: self reference
:param Operation operation: Operation instance
:param str csPath: CS path for this handler
"""
super(ReplicateAndRegister, self).__init__(operation, csPath)
# # SE cache
# Clients
self.fc = FileCatalog()
def initialize(self):
""" standard initialize method for DIRAC agents
"""
res = DIRACValidateOutputDataAgent.initialize(self)
if not res['OK']:
return res
self.integrityClient = DataIntegrityClient()
self.fileCatalog = FileCatalog()
self.transClient = TransformationClient()
self.storageUsageClient = StorageUsageClient()
return S_OK()
def isRegisteredInOutputCatalog( self, file, transferDict ):
fc = FileCatalog( [ transferDict[ 'OutputFC' ] ] )
lfn = os.path.join( transferDict['OutputPath'], os.path.basename( file ) )
result = fc.getReplicas( lfn )
if not result[ 'OK' ]:
return result
if lfn not in result[ 'Value' ][ 'Successful' ]:
return S_OK( False )
replicas = result[ 'Value' ][ 'Successful' ][ lfn ]
for seName in List.fromChar( transferDict[ 'OutputSE' ], "," ):
if seName in replicas:
self.log.verbose( "Transfer file %s is already registered in %s SE" % ( file, seName ) )
return S_OK( True )
return S_OK( False )
def test_01_init( self, mk_getSelectedCatalogs, mk_getEligibleCatalogs ):
""" Check logic of init"""
# We should not be able to have 2 masters
twoMastersFc = FileCatalog( catalogs = ['c1_True_True_True_5_2_2_0', 'c2_True_True_True_5_2_2_0'] )
self.assert_( not twoMastersFc.isOK() )
# One master should be ok
oneMasterFc = FileCatalog( catalogs = ['c1_True_True_True_2_0_2_2', 'c2_False_True_True_3_1_4_2'] )
self.assert_( oneMasterFc.isOK() )
# With a master, the write method should be the method of the master
self.assertEqual( sorted( oneMasterFc.write_methods ), writeList( 2 ) )
# The read methods and no_lfn should be from all catalogs
self.assertEqual( sorted( oneMasterFc.ro_methods ), readList( 3 ) )
# The no_lfns methods are from everywhere
# write1 and write2 from c1
# write3, write4, read3 from c2
self.assertEqual( sorted( oneMasterFc.no_lfn_methods ), sorted( readList( 1, reverse = 3 ) + writeList( 4 ) ) )
# No master should be ok
noMasterFc = FileCatalog( catalogs = ['c1_False_True_True_2_0_2_0', 'c2_False_True_True_3_0_4_0'] )
self.assert_( oneMasterFc.isOK() )
# With no master, the write method should be from all catalogs
self.assertEqual( sorted( noMasterFc.write_methods ), writeList( 4 ) )
# The read methods and no_lfn should be from all catalogs
self.assertEqual( sorted( noMasterFc.ro_methods ), readList( 3 ) )
def checkMatchQuery(self, mq, mqParent):
""" Check the logical intersection between the two metaqueries
:param dict mq: a dictionary of the MetaQuery to be checked against the mqParent
:param dict mqParent: a dictionary of the parent MetaQuery to be checked against the mq
"""
# Get the metadata types defined in the catalog
catalog = FileCatalog()
res = catalog.getMetadataFields()
if not res['OK']:
gLogger.error("Error in getMetadataFields: %s" % res['Message'])
return res
if not res['Value']:
gLogger.error("Error: no metadata fields defined")
return res
MetaTypeDict = res['Value']['FileMetaFields']
MetaTypeDict.update(res['Value']['DirectoryMetaFields'])
res = self.checkformatQuery(mq)
if not res['OK']:
return res
MetaQueryDict = res['Value']
res = self.checkformatQuery(mqParent)
if not res['OK']:
return res
ParentMetaQueryDict = res['Value']
for meta, value in MetaQueryDict.items():
if meta not in MetaTypeDict:
msg = 'Metadata %s is not defined in the Catalog' % meta
return S_ERROR(msg)
mtype = MetaTypeDict[meta]
if mtype.lower() not in ['varchar(128)', 'int', 'float']:
msg = 'Metatype %s is not supported' % mtype.lower()
return S_ERROR(msg)
if meta not in ParentMetaQueryDict:
msg = 'Metadata %s is not in parent transformation query' % meta
return S_ERROR(msg)
if self.compareValues(value, ParentMetaQueryDict[meta]):
continue
else:
msg = "Metadata values %s do not match with %s" % (
value, ParentMetaQueryDict[meta])
gLogger.error(msg)
return S_OK(False)
return S_OK(True)
def main():
Script.parseCommandLine()
from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
allowUsers = Operations().getValue(
"DataManagement/AllowUserReplicaManagement", False)
from DIRAC.Core.Security.ProxyInfo import getProxyInfo
res = getProxyInfo()
if not res['OK']:
gLogger.fatal("Can't get proxy info", res['Message'])
dexit(1)
properties = res['Value'].get('groupProperties', [])
if not allowUsers:
if 'FileCatalogManagement' not in properties:
gLogger.error(
"You need to use a proxy from a group with FileCatalogManagement"
)
dexit(5)
from DIRAC.Resources.Catalog.FileCatalog import FileCatalog
fc = FileCatalog()
import os
args = Script.getPositionalArgs()
if len(args) < 1:
Script.showHelp(exitCode=1)
else:
inputFileName = args[0]
if os.path.exists(inputFileName):
inputFile = open(inputFileName, 'r')
string = inputFile.read()
lfns = [lfn.strip() for lfn in string.splitlines()]
inputFile.close()
else:
lfns = [inputFileName]
res = fc.removeFile(lfns)
if not res['OK']:
print("Error:", res['Message'])
dexit(1)
for lfn in sorted(res['Value']['Failed'].keys()):
message = res['Value']['Failed'][lfn]
print('Error: failed to remove %s: %s' % (lfn, message))
print('Successfully removed %d catalog files.' %
(len(res['Value']['Successful'])))
def setUp(self):
self.fullMetadata = [
"Status",
"ChecksumType",
"OwnerRole",
"CreationDate",
"Checksum",
"ModificationDate",
"OwnerDN",
"Mode",
"GUID",
"Size",
]
self.dirMetadata = self.fullMetadata + ["NumberOfSubPaths"]
self.fileMetadata = self.fullMetadata + ["NumberOfLinks"]
self.catalog = FileCatalog(catalogs=[catalogClientToTest])
valid = self.catalog.isOK()
self.assert_(valid)
self.destDir = "/lhcb/test/unit-test/TestCatalogPlugin"
self.link = "%s/link" % self.destDir
# Clean the existing directory
self.cleanDirectory()
res = self.catalog.createDirectory(self.destDir)
returnValue = self.parseResult(res, self.destDir)
# Register some files to work with
self.numberOfFiles = 2
self.files = []
for i in range(self.numberOfFiles):
lfn = "%s/testFile_%d" % (self.destDir, i)
res = self.registerFile(lfn)
self.assert_(res)
self.files.append(lfn)
def __init__( self, fromDict = None ):
"""c'tor
:param self: self reference
:param dict fromDict: data dict
"""
Record.__init__( self )
now = datetime.datetime.utcnow().replace( microsecond = 0 )
self.__data__["CreationTime"] = now
self.__data__["SubmitTime"] = now
self.__data__["LastUpdate"] = now
self.__data__["Status"] = "Submitted"
self.__data__["Completeness"] = 0
self.__data__["FTSJobID"] = 0
self._regTime = 0.
self._regSuccess = 0
self._regTotal = 0
self.__files__ = TypedList( allowedTypes = FTSFile )
self._fc = FileCatalog()
self._log = gLogger.getSubLogger( "FTSJob-%s" % self.FTSJobID , True )
self._states = tuple( set( self.INITSTATES + self.TRANSSTATES + self.FAILEDSTATES + self.FINALSTATES ) )
fromDict = fromDict if fromDict else {}
for ftsFileDict in fromDict.get( "FTSFiles", [] ):
self +=FTSFile( ftsFileDict )
if "FTSFiles" in fromDict: del fromDict["FTSFiles"]
for key, value in fromDict.items():
if key not in self.__data__:
raise AttributeError( "Unknown FTSJob attribute '%s'" % key )
if value:
setattr( self, key, value )
def __init__( self, operation = None, csPath = None ):
"""c'tor
:param self: self reference
:param Operation operation: Operation instance
:param str csPath: CS path for this handler
"""
super( ReplicateAndRegister, self ).__init__( operation, csPath )
# # own gMonitor stuff for files
gMonitor.registerActivity( "ReplicateAndRegisterAtt", "Replicate and register attempted",
"RequestExecutingAgent", "Files/min", gMonitor.OP_SUM )
gMonitor.registerActivity( "ReplicateOK", "Replications successful",
"RequestExecutingAgent", "Files/min", gMonitor.OP_SUM )
gMonitor.registerActivity( "ReplicateFail", "Replications failed",
"RequestExecutingAgent", "Files/min", gMonitor.OP_SUM )
gMonitor.registerActivity( "RegisterOK", "Registrations successful",
"RequestExecutingAgent", "Files/min", gMonitor.OP_SUM )
gMonitor.registerActivity( "RegisterFail", "Registrations failed",
"RequestExecutingAgent", "Files/min", gMonitor.OP_SUM )
# # for FTS
gMonitor.registerActivity( "FTSScheduleAtt", "Files schedule attempted",
"RequestExecutingAgent", "Files/min", gMonitor.OP_SUM )
gMonitor.registerActivity( "FTSScheduleOK", "File schedule successful",
"RequestExecutingAgent", "Files/min", gMonitor.OP_SUM )
gMonitor.registerActivity( "FTSScheduleFail", "File schedule failed",
"RequestExecutingAgent", "Files/min", gMonitor.OP_SUM )
# # SE cache
self.seCache = {}
# Clients
self.fc = FileCatalog()
self.ftsClient = FTSClient()
def resolveCatalogPFNSizeMismatch( self, problematicDict ):
""" This takes the problematic dictionary returned by the integrity DB and resolved the CatalogPFNSizeMismatch prognosis
"""
lfn = problematicDict['LFN']
pfn = problematicDict['PFN']
se = problematicDict['SE']
fileID = problematicDict['FileID']
res = Utils.executeSingleFileOrDirWrapper( self.fc.getFileSize( lfn ) )
if not res['OK']:
return self.__returnProblematicError( fileID, res )
catalogSize = res['Value']
res = Utils.executeSingleFileOrDirWrapper( StorageElement( se ).getFileSize( pfn ) )
if not res['OK']:
return self.__returnProblematicError( fileID, res )
storageSize = res['Value']
bkKCatalog = FileCatalog( ['BookkeepingDB'] )
res = Utils.executeSingleFileOrDirWrapper( bkKCatalog.getFileSize( lfn ) )
if not res['OK']:
return self.__returnProblematicError( fileID, res )
bookkeepingSize = res['Value']
if bookkeepingSize == catalogSize == storageSize:
gLogger.info( "CatalogPFNSizeMismatch replica (%d) matched all registered sizes." % fileID )
return self.__updateReplicaToChecked( problematicDict )
if ( catalogSize == bookkeepingSize ):
gLogger.info( "CatalogPFNSizeMismatch replica (%d) found to mismatch the bookkeeping also" % fileID )
res = Utils.executeSingleFileOrDirWrapper( self.fc.getReplicas( lfn ) )
if not res['OK']:
return self.__returnProblematicError( fileID, res )
if len( res['Value'] ) <= 1:
gLogger.info( "CatalogPFNSizeMismatch replica (%d) has no other replicas." % fileID )
return S_ERROR( "Not removing catalog file mismatch since the only replica" )
else:
gLogger.info( "CatalogPFNSizeMismatch replica (%d) has other replicas. Removing..." % fileID )
res = self.dm.removeReplica( se, lfn )
if not res['OK']:
return self.__returnProblematicError( fileID, res )
return self.__updateCompletedFiles( 'CatalogPFNSizeMismatch', fileID )
if ( catalogSize != bookkeepingSize ) and ( bookkeepingSize == storageSize ):
gLogger.info( "CatalogPFNSizeMismatch replica (%d) found to match the bookkeeping size" % fileID )
res = self.__updateReplicaToChecked( problematicDict )
if not res['OK']:
return self.__returnProblematicError( fileID, res )
return self.changeProblematicPrognosis( fileID, 'BKCatalogSizeMismatch' )
gLogger.info( "CatalogPFNSizeMismatch replica (%d) all sizes found mismatch. Updating retry count" % fileID )
return self.incrementProblematicRetry( fileID )
def initialize( self ):
self.fileCatalog = FileCatalog()
self.dm = DataManager()
self.stagerClient = StorageManagerClient()
self.dataIntegrityClient = DataIntegrityClient()
# This sets the Default Proxy to used as that defined under
# /Operations/Shifter/DataManager
# the shifterProxy option in the Configuration can be used to change this default.
self.am_setOption( 'shifterProxy', 'DataManager' )
return S_OK()
def __getCatalogObject( self ):
""" CatalogInterface instance facade
:param self: self reference
"""
try:
if not self.oCatalog:
self.oCatalog = FileCatalog()
return S_OK()
except:
return S_ERROR()
def __init__(self, *args, **kwargs ):
super( TransferAppHandler, self ).__init__( *args, **kwargs )
sessionData = self.getSessionData()
self.user = sessionData['user'].get( 'username', '' )
self.group = sessionData['user'].get( 'group', '' )
self.vo = getVOForGroup( self.group )
self.fc = FileCatalog( vo = self.vo )
self.log.always(self.user)
self.log.always(self.group)
self.log.always(self.vo)
self.log.always(self.fc)
def test_03_noLFN( self, mk_getSelectedCatalogs, mk_getEligibleCatalogs ):
""" Test the no_lfn methods """
fc = FileCatalog( catalogs = ['c1_True_True_True_2_0_2_1', 'c2_False_True_True_3_0_2_1'] )
# all good
res = fc.write2( "/lhcb/toto" )
self.assert_( res['OK'] )
self.assertEqual( res['Value'], 'yeah' )
# Fail in the master
res = fc.write2( "/lhcb/c1" )
self.assert_( not res['OK'] )
self.assert_( not 'Value' in res )
# Fail in the non master
res = fc.write2( "/lhcb/c2" )
self.assert_( res['OK'] )
self.assert_( 'Value' in res )
self.assertEqual( res['Value'], 'yeah' )
def test_02_condParser( self, mk_getSelectedCatalogs, mk_getEligibleCatalogs ):
"""Test behavior of write methode when using FCConditionParser"""
fc = FileCatalog( catalogs = ['c1_True_True_True_2_0_2_0', 'c2_False_True_True_3_0_1_0', 'c3_False_True_True_3_0_1_0'] )
# No condition for c3, so it should always pass
fcConditions = { 'c1' : "Filename=find('c1_pass')",
'c2' : "Filename=find('c2_pass')"}
# Everything pass everywhere
lfn1 = '/lhcb/c1_pass/c2_pass/lfn1'
lfn2 = '/lhcb/c1_pass/c2_pass/lfn2'
res = fc.write1( [lfn1, lfn2],
fcConditions = fcConditions )
self.assert_( res['OK'] )
self.assertEqual( sorted( res['Value']['Successful'] ), sorted( [lfn1, lfn2] ) )
self.assertEqual( sorted( res['Value']['Successful'][lfn1] ), sorted( ['c1', 'c2', 'c3'] ) )
self.assertEqual( sorted( res['Value']['Successful'][lfn2] ), sorted( ['c1', 'c2', 'c3'] ) )
self.assert_( not res['Value']['Failed'] )
# Everything pass for the master, only lfn2 for c2
lfn1 = '/lhcb/c1_pass/lfn1'
lfn2 = '/lhcb/c1_pass/c2_pass/lfn2'
res = fc.write1( [lfn1, lfn2],
fcConditions = fcConditions )
self.assert_( res['OK'] )
self.assertEqual( sorted( res['Value']['Successful'] ), sorted( [lfn1, lfn2] ) )
self.assertEqual( sorted( res['Value']['Successful'][lfn1] ) , ['c1', 'c3'] )
self.assertEqual( sorted( res['Value']['Successful'][lfn2] ), sorted( ['c1', 'c2','c3'] ) )
self.assert_( not res['Value']['Failed'] )
# One is not valid for the master, so we do nothing
lfn1 = '/lhcb/c2_pass/lfn1'
lfn2 = '/lhcb/c1_pass/c2_pass/lfn2'
res = fc.write1( [lfn1, lfn2],
fcConditions = fcConditions )
self.assert_( not res['OK'] )
def __getSEListFromReplicas(self, lfnDict):
""" Get the SEs which have a replica of the lfn
@param: self - self reference
@param: string lfn - lfn for which the replicas are retrieved
@returns S_ERROR when retrieving replicas failed
S_OK(SEList) otherwise
"""
fc = FileCatalog()
# lfnDict = {lfn : True}
res = fc.getReplicas(lfnDict)
if not res['OK']:
self.log.debug("readFederation.__compareFileListWithCatalog: Completely failed to get Replicas")
return S_ERROR("getReplicas: %s" % res['Message'])
res = res['Value']
# if not lfn in res['Successful']:
# self.log.debug("readFederation.__compareFileListWithCatalog: Failed to get Replicas")
# return S_ERROR("getReplicas: %s" % res['Failed'][lfn])
# we have a list of replicas for a given LFN. SEList contains all the SE
# that store that file according to the catalog
return res.get('Successful', None)
def initializeOptimizer( self ):
"""Initialize specific parameters for JobSanityAgent.
"""
self.failedMinorStatus = self.am_getOption( '/FailedJobStatus', 'Input Data Not Available' )
#this will ignore failover SE files
self.checkFileMetadata = self.am_getOption( 'CheckFileMetadata', True )
self.dataManager = DataManager()
self.resourceStatus = ResourceStatus()
self.fc = FileCatalog()
self.seToSiteMapping = {}
self.lastCScheck = 0
self.cacheLength = 600
return S_OK()
def __init__( self, operation = None, csPath = None ):
"""c'tor
:param Operation operation: Operation instance
:param str csPath: config path in CS for this operation
"""
# # placeholders for operation and request
self.operation = None
self.request = None
self.dm = DataManager()
self.fc = FileCatalog()
self.csPath = csPath if csPath else ""
# # get name
name = self.__class__.__name__
# # all options are r/o properties now
csOptionsDict = gConfig.getOptionsDict( self.csPath )
csOptionsDict = csOptionsDict.get( "Value", {} )
for option, value in csOptionsDict.iteritems():
# # hack to set proper types
try:
value = eval( value )
except NameError:
pass
self.makeProperty( option, value, True ) #pylint: disable=no-member
# # pre setup logger
self.log = gLogger.getSubLogger( name, True )
# # set log level
logLevel = getattr( self, "LogLevel" ) if hasattr( self, "LogLevel" ) else "INFO"
self.log.setLevel( logLevel )
# # list properties
for option in csOptionsDict:
self.log.debug( "%s = %s" % ( option, getattr( self, option ) ) )
# # setup operation
if operation:
self.setOperation( operation )
# # initialize at least
if hasattr( self, "initialize" ) and callable( getattr( self, "initialize" ) ):
getattr( self, "initialize" )()
def __init__( self, plugin, transClient = None, dataManager = None ):
""" plugin name has to be passed in: it will then be executed as one of the functions below, e.g.
plugin = 'BySize' will execute TransformationPlugin('BySize')._BySize()
"""
super( TransformationPlugin, self ).__init__( plugin )
self.data = {}
self.files = False
if transClient is None:
self.transClient = TransformationClient()
else:
self.transClient = transClient
if dataManager is None:
self.dm = DataManager()
else:
self.dm = dataManager
self.fc = FileCatalog()
def __init__(self, lfn='', status='', size=0, guid='', checksum=''):
# These are the possible attributes for a file
if not type(lfn) in types.StringTypes:
raise AttributeError, "lfn should be string type"
self.lfn = str(lfn)
if not type(status) in types.StringTypes:
raise AttributeError, "status should be string type"
self.status = str(status)
try:
self.size = int(size)
except:
raise AttributeError, "size should be integer type"
if not type(guid) in types.StringTypes:
raise AttributeError, "guid should be string type"
self.guid = str(guid)
if not type(checksum) in types.StringTypes:
raise AttributeError, "checksum should be string type"
self.checksum = str(checksum)
self.catalogReplicas = []
self.fc = FileCatalog()
def __init__( self, *args, **kwargs ):
""" c'tor
"""
AgentModule.__init__( self, *args, **kwargs )
self.integrityClient = DataIntegrityClient()
self.fc = FileCatalog()
self.transClient = TransformationClient()
self.fileCatalogClient = FileCatalogClient()
agentTSTypes = self.am_getOption( 'TransformationTypes', [] )
if agentTSTypes:
self.transformationTypes = agentTSTypes
else:
self.transformationTypes = Operations().getValue( 'Transformations/DataProcessing', ['MCSimulation', 'Merge'] )
self.directoryLocations = sorted( self.am_getOption( 'DirectoryLocations', ['TransformationDB',
'MetadataCatalog'] ) )
self.activeStorages = sorted( self.am_getOption( 'ActiveSEs', [] ) )
self.transfidmeta = self.am_getOption( 'TransfIDMeta', "TransformationID" )
self.enableFlag = True
def __init__( self, plugin = 'Standard', transClient = None, dataManager = None, fc = None,
debug = False, transInThread = None, transID = None ):
"""
c'tor
Setting defaults
"""
# clients
if transClient is None:
self.transClient = TransformationClient()
else:
self.transClient = transClient
if dataManager is None:
self.dm = DataManager()
else:
self.dm = dataManager
if fc is None:
self.fc = FileCatalog()
else:
self.fc = fc
self.dmsHelper = DMSHelpers()
self.plugin = plugin
self.transID = transID
self.params = {}
self.groupSize = 0
self.maxFiles = 0
self.cachedLFNSize = {}
self.transString = ''
self.debug = debug
self.seConfig = {}
if transInThread is None:
self.transInThread = {}
else:
self.transInThread = transInThread
self.log = gLogger.getSubLogger( "%s/PluginUtilities" % plugin )
def setUp(self):
self.fullMetadata = ['Status', 'ChecksumType', 'OwnerRole', 'CreationDate', 'Checksum', 'ModificationDate', 'OwnerDN', 'Mode', 'GUID', 'Size']
self.dirMetadata = self.fullMetadata + ['NumberOfSubPaths']
self.fileMetadata = self.fullMetadata + ['NumberOfLinks']
self.catalog = FileCatalog(catalogs=[catalogClientToTest])
valid = self.catalog.isOK()
self.assertTrue(valid)
self.destDir = '/lhcb/test/unit-test/TestCatalogPlugin'
self.link = "%s/link" % self.destDir
# Clean the existing directory
self.cleanDirectory()
res = self.catalog.createDirectory(self.destDir)
returnValue = self.parseResult(res,self.destDir)
# Register some files to work with
self.numberOfFiles = 2
self.files = []
for i in xrange(self.numberOfFiles):
lfn = "%s/testFile_%d" % (self.destDir,i)
res = self.registerFile(lfn)
self.assertTrue(res)
self.files.append(lfn)
class OperationHandlerBase( object ):
"""
.. class:: OperationHandlerBase
request operation handler base class
"""
__metaclass__ = DynamicProps
# # private data logging client
# __dataLoggingClient = None
# # private ResourceStatusClient
__rssClient = None
# # shifter list
__shifterList = []
def __init__( self, operation = None, csPath = None ):
"""c'tor
:param Operation operation: Operation instance
:param str csPath: config path in CS for this operation
"""
# # placeholders for operation and request
self.operation = None
self.request = None
self.dm = DataManager()
self.fc = FileCatalog()
self.csPath = csPath if csPath else ""
# # get name
name = self.__class__.__name__
# # all options are r/o properties now
csOptionsDict = gConfig.getOptionsDict( self.csPath )
csOptionsDict = csOptionsDict.get( "Value", {} )
for option, value in csOptionsDict.iteritems():
# # hack to set proper types
try:
value = eval( value )
except NameError:
pass
self.makeProperty( option, value, True ) #pylint: disable=no-member
# # pre setup logger
self.log = gLogger.getSubLogger( name, True )
# # set log level
logLevel = getattr( self, "LogLevel" ) if hasattr( self, "LogLevel" ) else "INFO"
self.log.setLevel( logLevel )
# # list properties
for option in csOptionsDict:
self.log.debug( "%s = %s" % ( option, getattr( self, option ) ) )
# # setup operation
if operation:
self.setOperation( operation )
# # initialize at least
if hasattr( self, "initialize" ) and callable( getattr( self, "initialize" ) ):
getattr( self, "initialize" )()
def setOperation( self, operation ):
""" operation and request setter
:param ~DIRAC.RequestManagementSystem.Client.Operation.Operation operation: operation instance
:raises TypeError: if `operation` in not an instance of :class:`~DIRAC.RequestManagementSystem.Client.Operation.Operation`
"""
if not isinstance( operation, Operation ):
raise TypeError( "expecting Operation instance" )
self.operation = operation
self.request = operation._parent
self.log = gLogger.getSubLogger( "pid_%s/%s/%s/%s" % ( os.getpid(), self.request.RequestName,
self.request.Order,
self.operation.Type ) )
# @classmethod
# def dataLoggingClient( cls ):
# """ DataLoggingClient getter """
# if not cls.__dataLoggingClient:
# from DIRAC.DataManagementSystem.Client.DataLoggingClient import DataLoggingClient
# cls.__dataLoggingClient = DataLoggingClient()
# return cls.__dataLoggingClient
@classmethod
def rssClient( cls ):
""" ResourceStatusClient getter """
if not cls.__rssClient:
from DIRAC.ResourceStatusSystem.Client.ResourceStatus import ResourceStatus
cls.__rssClient = ResourceStatus()
return cls.__rssClient
def getProxyForLFN( self, lfn ):
""" get proxy for lfn
:param str lfn: LFN
:return: S_ERROR or S_OK( "/path/to/proxy/file" )
"""
dirMeta = returnSingleResult( self.fc.getDirectoryMetadata( os.path.dirname( lfn ) ) )
if not dirMeta["OK"]:
return dirMeta
dirMeta = dirMeta["Value"]
ownerRole = "/%s" % dirMeta["OwnerRole"] if not dirMeta["OwnerRole"].startswith( "/" ) else dirMeta["OwnerRole"]
ownerDN = dirMeta["OwnerDN"]
ownerProxy = None
for ownerGroup in getGroupsWithVOMSAttribute( ownerRole ):
vomsProxy = gProxyManager.downloadVOMSProxy( ownerDN, ownerGroup, limited = True,
requiredVOMSAttribute = ownerRole )
if not vomsProxy["OK"]:
self.log.debug( "getProxyForLFN: failed to get VOMS proxy for %s role=%s: %s" % ( ownerDN,
ownerRole,
vomsProxy["Message"] ) )
continue
ownerProxy = vomsProxy["Value"]
self.log.debug( "getProxyForLFN: got proxy for %s@%s [%s]" % ( ownerDN, ownerGroup, ownerRole ) )
break
if not ownerProxy:
return S_ERROR( "Unable to get owner proxy" )
dumpToFile = ownerProxy.dumpAllToFile()
if not dumpToFile["OK"]:
self.log.error( "getProxyForLFN: error dumping proxy to file: %s" % dumpToFile["Message"] )
else:
os.environ["X509_USER_PROXY"] = dumpToFile["Value"]
return dumpToFile
def getWaitingFilesList( self ):
""" prepare waiting files list, update Attempt, filter out MaxAttempt """
if not self.operation:
self.log.warning( "getWaitingFilesList: operation not set, returning empty list" )
return []
waitingFiles = [ opFile for opFile in self.operation if opFile.Status == "Waiting" ]
for opFile in waitingFiles:
opFile.Attempt += 1
maxAttempts = getattr( self, "MaxAttempts" ) if hasattr( self, "MaxAttempts" ) else 1024
if opFile.Attempt > maxAttempts:
opFile.Status = "Failed"
if opFile.Error is None:
opFile.Error = ''
opFile.Error += " (Max attempts limit reached)"
return [ opFile for opFile in self.operation if opFile.Status == "Waiting" ]
def rssSEStatus( self, se, status, retries = 2 ):
""" check SE :se: for status :status:
:param str se: SE name
:param str status: RSS status
"""
# Allow a transient failure
for _i in range( retries ):
rssStatus = self.rssClient().getElementStatus( se, "StorageElement", status )
# gLogger.always( rssStatus )
if rssStatus["OK"]:
return S_OK( rssStatus["Value"][se][status] != "Banned" )
return S_ERROR( "%s status not found in RSS for SE %s" % ( status, se ) )
@property
def shifter( self ):
return self.__shifterList
@shifter.setter
def shifter( self, shifterList ):
self.__shifterList = shifterList
def __call__( self ):
""" this one should be implemented in the inherited class
should return S_OK/S_ERROR
"""
raise NotImplementedError( "Implement me please!" )
class FTSJob( object ):
""" Class describing one FTS job
"""
# # initial states
INITSTATES = ( "Submitted", "Ready", "Staging" )
# # ongoing transfer states
TRANSSTATES = ( "Active", "Hold" )
# # failed states
FAILEDSTATES = ( "Canceled", "Failed" )
# # finished (careful, must be capitalized)
FINALSTATES = ( "Finished", "Finisheddirty", "FinishedDirty", "Failed", "Canceled" )
# # missing source regexp patterns
missingSourceErrors = [
re.compile( r".*INVALID_PATH\] Failed" ),
re.compile( r".*INVALID_PATH\] No such file or directory" ),
re.compile( r".*INVALID_PATH\] The requested file either does not exist" ),
re.compile( r".*INVALID_PATH\] the server sent an error response: 500 500"\
" Command failed. : open error: No such file or directory" ),
re.compile( r"SOURCE error during TRANSFER_PREPARATION phase: \[USER_ERROR\] source file doesnt exist" ) ]
def __init__( self, fromDict = None ):
"""c'tor
:param self: self reference
:param dict fromDict: data dict
"""
self.__data__ = dict.fromkeys( self.tableDesc()["Fields"].keys(), None )
now = datetime.datetime.utcnow().replace( microsecond = 0 )
self.__data__["CreationTime"] = now
self.__data__["SubmitTime"] = now
self.__data__["LastUpdate"] = now
self.__data__["Status"] = "Submitted"
self.__data__["Completeness"] = 0
self.__data__["FTSJobID"] = 0
self._regTime = 0.
self._regSuccess = 0
self._regTotal = 0
self.__files__ = TypedList( allowedTypes = FTSFile )
self._fc = FileCatalog()
self._fts3context = None
self._states = tuple( set( self.INITSTATES + self.TRANSSTATES + self.FAILEDSTATES + self.FINALSTATES ) )
fromDict = fromDict if fromDict else {}
for ftsFileDict in fromDict.get( "FTSFiles", [] ):
self +=FTSFile( ftsFileDict )
if "FTSFiles" in fromDict:
del fromDict["FTSFiles"]
for key, value in fromDict.items():
if key not in self.__data__:
raise AttributeError( "Unknown FTSJob attribute '%s'" % key )
if value:
setattr( self, key, value )
self._log = gLogger.getSubLogger( "req_%s/FTSJob-%s" % ( self.RequestID, self.FTSGUID ) , True )
@staticmethod
def tableDesc():
""" get table desc """
return { "Fields" :
{ "FTSJobID" : "INTEGER NOT NULL AUTO_INCREMENT",
"FTSGUID" : "VARCHAR(64) NOT NULL",
"OperationID": "INTEGER NOT NULL",
"RequestID": "INTEGER NOT NULL",
"SourceSE" : "VARCHAR(128) NOT NULL",
"TargetSE" : "VARCHAR(128) NOT NULL",
"FTSServer" : "VARCHAR(255) NOT NULL",
"TargetToken": "VARCHAR(255)",
"SourceToken": "VARCHAR(255)",
"Size": "BIGINT NOT NULL",
"Files": "INTEGER NOT NULL",
"Completeness": "INTEGER NOT NULL DEFAULT 0",
"FailedFiles": "INTEGER DEFAULT 0",
"FailedSize": "INTEGER DEFAULT 0",
"Status" : "ENUM( 'Submitted', 'Ready', 'Staging', 'Canceled', 'Active', 'Hold', "\
"'Failed', 'Finished', 'FinishedDirty', 'Assigned' ) DEFAULT 'Submitted'",
"Error" : "VARCHAR(255)",
"CreationTime" : "DATETIME",
"SubmitTime" : "DATETIME",
"LastUpdate" : "DATETIME" },
"PrimaryKey" : [ "FTSJobID" ],
"Indexes" : { "FTSJobID" : [ "FTSJobID" ], "FTSGUID": [ "FTSGUID" ] } }
@property
def FTSJobID( self ):
""" FTSJobID getter """
return self.__data__["FTSJobID"]
@FTSJobID.setter
def FTSJobID( self, value ):
""" FTSJobID setter """
self.__data__["FTSJobID"] = long( value ) if value else 0
@property
def RequestID( self ):
""" RequestID getter """
return self.__data__["RequestID"]
@RequestID.setter
def RequestID( self, value ):
""" RequestID setter """
self.__data__["RequestID"] = long( value ) if value else 0
@property
def OperationID( self ):
""" OperationID getter """
return self.__data__["OperationID"]
@OperationID.setter
def OperationID( self, value ):
""" OperationID setter """
self.__data__["OperationID"] = long( value ) if value else 0
@property
def FTSGUID( self ):
""" FTSGUID prop """
return self.__data__["FTSGUID"]
@FTSGUID.setter
def FTSGUID( self, value ):
""" FTSGUID setter """
if value:
if type( value ) not in ( str, unicode ):
raise TypeError( "FTSGUID should be a string!" )
if not checkGuid( value ):
raise ValueError( "'%s' is not a valid GUID!" % str( value ) )
self.__data__["FTSGUID"] = value
@property
def FTSServer( self ):
""" FTSServer getter """
return self.__data__["FTSServer"]
@FTSServer.setter
def FTSServer( self, url ):
""" FTSServer getter """
self.__data__["FTSServer"] = url
# I REALLY don't see that happening
# but in case we change the server after the
# context was created, I reset it
# (I don't initialize because maybe we are in FTS2 mode...)
self._fts3context = None
@property
def Completeness( self ):
""" completeness getter """
return self.__data__["Completeness"]
@Completeness.setter
def Completeness( self, value ):
""" completeness setter """
self.__data__["Completeness"] = int( value ) if value else 0
@property
def Error( self ):
""" error getter """
return self.__data__["Error"]
@Error.setter
def Error( self, error ):
""" error setter """
self.__data__["Error"] = str( error )[255:]
@property
def Files( self ):
""" nb files getter """
self.__data__["Files"] = len( self )
return self.__data__["Files"]
@Files.setter
def Files( self, value ):
""" nb files setter """
self.__data__["Files"] = len( self )
@property
def Status( self ):
""" status prop """
if not self.__data__["Status"]:
self.__data__["Status"] = "Waiting"
return self.__data__["Status"]
@Status.setter
def Status( self, value ):
""" status setter """
value = self._normalizedStatus( value.strip() )
if value not in self._states:
raise ValueError( "Unknown FTSJob Status: '%s'" % str( value ) )
self.__data__["Status"] = value
@property
def FailedFiles( self ):
""" nb failed files getter """
self.__data__["FailedFiles"] = len( [ ftsFile for ftsFile in self
if ftsFile.Status in FTSFile.FAILED_STATES ] )
return self.__data__["FailedFiles"]
@FailedFiles.setter
def FailedFiles( self, value ):
""" nb failed files setter """
if value:
self.__data__["FailedFiles"] = value
else:
self.__data__["FailedFiles"] = len( [ftsFile for ftsFile in self if ftsFile.Status in FTSFile.FAILED_STATES] )
@property
def Size( self ):
""" size getter """
# if not self.__data__["Size"]:
self.__data__["Size"] = sum( ftsFile.Size for ftsFile in self )
return self.__data__["Size"]
@Size.setter
def Size( self, value ):
""" size setter """
if value:
self.__data__["Size"] = value
else:
self.__data__["Size"] = sum( ftsFile.Size for ftsFile in self )
@property
def FailedSize( self ):
""" size getter """
if not self.__data__["FailedSize"]:
self.__data__["FailedSize"] = sum( ftsFile.Size for ftsFile in self if ftsFile.Status in FTSFile.FAILED_STATES )
return self.__data__["FailedSize"]
@FailedSize.setter
def FailedSize( self, value ):
""" size setter """
if value:
self.__data__["FailedSize"] = value
else:
self.__data__["FailedSize"] = sum( ftsFile.Size for ftsFile in self if ftsFile.Status in FTSFile.FAILED_STATES )
@property
def CreationTime( self ):
""" creation time getter """
return self.__data__["CreationTime"]
@CreationTime.setter
def CreationTime( self, value = None ):
""" creation time setter """
if type( value ) not in ( datetime.datetime, str ) :
raise TypeError( "CreationTime should be a datetime.datetime!" )
if type( value ) == str:
value = datetime.datetime.strptime( value.split( "." )[0], '%Y-%m-%d %H:%M:%S' )
self.__data__["CreationTime"] = value
@property
def SubmitTime( self ):
""" request's submission time getter """
return self.__data__["SubmitTime"]
@SubmitTime.setter
def SubmitTime( self, value = None ):
""" submission time setter """
if type( value ) not in ( datetime.datetime, str ):
raise TypeError( "SubmitTime should be a datetime.datetime!" )
if type( value ) == str:
value = datetime.datetime.strptime( value.split( "." )[0], '%Y-%m-%d %H:%M:%S' )
self.__data__["SubmitTime"] = value
@property
def LastUpdate( self ):
""" last update getter """
return self.__data__["LastUpdate"]
@LastUpdate.setter
def LastUpdate( self, value = None ):
""" last update setter """
if type( value ) not in ( datetime.datetime, str ):
raise TypeError( "LastUpdate should be a datetime.datetime!" )
if type( value ) == str:
value = datetime.datetime.strptime( value.split( "." )[0], '%Y-%m-%d %H:%M:%S' )
self.__data__["LastUpdate"] = value
@property
def TargetSE( self ):
""" target SE getter """
return self.__data__["TargetSE"]
@TargetSE.setter
def TargetSE( self, targetSE ):
""" target SE setter """
self.__data__["TargetSE"] = targetSE
@property
def SourceSE( self ):
""" source SE getter """
return self.__data__["SourceSE"]
@SourceSE.setter
def SourceSE( self, sourceSE ):
""" source SE setter """
self.__data__["SourceSE"] = sourceSE
@property
def SourceToken( self ):
""" source token getter """
return self.__data__["SourceToken"]
@SourceToken.setter
def SourceToken( self, sourceToken ):
""" source SE setter """
self.__data__["SourceToken"] = sourceToken
@property
def TargetToken( self ):
""" target token getter """
return self.__data__["TargetToken"]
@TargetToken.setter
def TargetToken( self, targetToken ):
""" target SE setter """
self.__data__["TargetToken"] = targetToken
# # FTSJobFiles arithmetics
def __contains__( self, subFile ):
""" in operator """
return subFile in self.__files__
def __iadd__( self, ftsFile ):
""" += operator """
if ftsFile not in self:
self.__files__.append( ftsFile )
ftsFile._parent = self
self.Files
self.Size
return self
def __add__( self, ftsFile ):
""" + operator """
self +=ftsFile
def addFile( self, ftsFile ):
""" add :ftsFile: to FTS job """
self +=ftsFile
def subFile( self, ftsFile ):
""" remove ftsFile from this job """
if ftsFile in self:
ftsFile._parent = None
self.__files__.remove( ftsFile )
# # helpers for looping
def __iter__( self ):
""" files iterator """
return self.__files__.__iter__()
def __getitem__( self, i ):
""" [] op for files """
return self.__files__.__getitem__( i )
def __delitem__( self, i ):
""" del ftsJob[i] """
self.__files__.__delitem__( i )
def __setitem__( self, i, ftsFile ):
""" ftsJob[i] = ftsFile """
self.__files__.__setitem__( i, ftsFile )
def fileStatusList( self ):
""" get list of files statuses """
return [ ftsFile.Status for ftsFile in self ]
def __nonzero__( self ):
""" for comparisons
"""
return True
def __len__( self ):
""" nb of subFiles """
return len( self.__files__ )
def _surlPairs( self ):
""" create and return SURL pair file """
surls = []
for ftsFile in self:
checksum = "%s:%s" % ( ftsFile.ChecksumType, ftsFile.Checksum ) if ftsFile.ChecksumType and ftsFile.Checksum else ""
surls.append( "%s %s %s" % ( ftsFile.SourceSURL, ftsFile.TargetSURL, checksum ) )
return "\n".join( surls )
def submitFTS2( self, command = 'glite-transfer-submit', pinTime = False ):
""" submit fts job using FTS2 client """
if self.FTSGUID:
return S_ERROR( "FTSJob has already been submitted" )
surls = self._surlPairs()
if not surls:
return S_ERROR( "No files to submit" )
fd, fileName = tempfile.mkstemp()
surlFile = os.fdopen( fd, 'w' )
surlFile.write( surls )
surlFile.close()
submitCommand = command.split() + \
[ "-s",
self.FTSServer,
"-f",
fileName,
"-o",
"-K" ]
if self.TargetToken:
submitCommand += [ "-t", self.TargetToken]
if self.SourceToken:
submitCommand += [ "-S", self.SourceToken ]
if pinTime:
submitCommand += [ "--copy-pin-lifetime", "%d" % pinTime, "--bring-online", '86400' ]
submit = executeGridCommand( "", submitCommand )
os.remove( fileName )
if not submit["OK"]:
return submit
returnCode, output, errStr = submit["Value"]
if returnCode != 0:
return S_ERROR( errStr if errStr else output )
self.FTSGUID = output.replace( "\n", "" )
self.Status = "Submitted"
for ftsFile in self:
ftsFile.FTSGUID = self.FTSGUID
ftsFile.Status = "Submitted"
return S_OK()
def _normalizedStatus( self, status ):
for st in self._states:
if status.lower() == st.lower():
return st
return status
def monitorFTS2( self, command = "glite-transfer-status", full = False ):
""" monitor fts job """
if not self.FTSGUID:
return S_ERROR( "FTSGUID not set, FTS job not submitted?" )
monitorCommand = command.split() + \
["--verbose",
"-s",
self.FTSServer,
self.FTSGUID ]
if full:
monitorCommand.append( "-l" )
monitor = executeGridCommand( "", monitorCommand )
if not monitor["OK"]:
return monitor
returnCode, outputStr, errStr = monitor["Value"]
# Returns a non zero status if error
if returnCode != 0:
if 'was not found' in outputStr and not errStr:
errStr = 'Job was not found'
return S_ERROR( errStr )
outputStr = outputStr.replace( "'" , "" ).replace( "<", "" ).replace( ">", "" )
# # set FTS job status
regExp = re.compile( "Status:\\s+(\\S+)" )
# with FTS3 this can be uppercase
self.Status = re.search( regExp, outputStr ).group( 1 )
statusSummary = {}
# This is capitalized, even in FTS3!
for state in FTSFile.ALL_STATES:
regExp = re.compile( "\\s+%s:\\s+(\\d+)" % state )
if regExp.search( outputStr ):
statusSummary[state] = int( re.search( regExp, outputStr ).group( 1 ) )
total = sum( statusSummary.values() )
completed = sum( statusSummary.get( state, 0 ) for state in FTSFile.FINAL_STATES )
self.Completeness = 100 * completed / total if total else 0
if not full:
return S_OK( statusSummary )
# The order of informations is not the same for glite- and fts- !!!
# In order: new fts-, old fts-, glite-
realJob = len( self ) != 0
iExptr = None
for iExptr, exptr in enumerate( (
'[ ]+Source:[ ]+(\\S+)\n[ ]+Destination:[ ]+(\\S+)\n[ ]+State:[ ]+(\\S+)\n[ ]+Reason:[ ]+([\\S ]+).+?[ ]+Duration:[ ]+(\\d+)\n[ ]+Staging:[ ]+(\\d+)\n[ ]+Retries:[ ]+(\\d+)',
'[ ]+Source:[ ]+(\\S+)\n[ ]+Destination:[ ]+(\\S+)\n[ ]+State:[ ]+(\\S+)\n[ ]+Reason:[ ]+([\\S ]+).+?[ ]+Duration:[ ]+(\\d+)\n[ ]+Retries:[ ]+(\\d+)',
'[ ]+Source:[ ]+(\\S+)\n[ ]+Destination:[ ]+(\\S+)\n[ ]+State:[ ]+(\\S+)\n[ ]+Retries:[ ]+(\\d+)\n[ ]+Reason:[ ]+([\\S ]+).+?[ ]+Duration:[ ]+(\\d+)'
) ):
regExp = re.compile( exptr, re.S )
fileInfo = re.findall( regExp, outputStr )
if fileInfo:
break
if not fileInfo:
return S_ERROR( "Error monitoring job (no regexp match)" )
for info in fileInfo:
if iExptr == 0:
# version >= 3.2.30
sourceURL, targetURL, fileStatus, reason, duration, _retries, _staging = info
elif iExptr == 1:
# version FTS3 < 3.2.30
sourceURL, targetURL, fileStatus, reason, duration, _retries = info
elif iExptr == 2:
# version FTS2
sourceURL, targetURL, fileStatus, _retries, reason, duration = info
else:
return S_ERROR( 'Error monitoring job (implement match %d)' % iExptr )
candidateFile = None
if not realJob:
# This is used by the CLI monitoring of jobs in case no file was specified
candidateFile = FTSFile()
candidateFile.LFN = overlap( sourceURL, targetURL )
candidateFile.SourceSURL = sourceURL
candidateFile.Size = 0
self +=candidateFile
else:
for ftsFile in self:
if ftsFile.SourceSURL == sourceURL:
candidateFile = ftsFile
break
if not candidateFile:
continue
# Can be uppercase for FTS3
if not candidateFile.TargetSURL:
candidateFile.TargetSURL = targetURL
candidateFile.Status = fileStatus
candidateFile.Error = reason
candidateFile._duration = duration
if candidateFile.Status == "Failed":
for missingSource in self.missingSourceErrors:
if missingSource.match( reason ):
candidateFile.Error = "MissingSource"
# If the staging info was present, record it
if len( info ) > 6:
candidateFile._staging = info[6]
# # register successful files
if self.Status in FTSJob.FINALSTATES:
return self.finalize()
return S_OK()
def submitFTS3( self, pinTime = False ):
""" submit fts job using FTS3 rest API """
if self.FTSGUID:
return S_ERROR( "FTSJob already has been submitted" )
transfers = []
for ftsFile in self:
trans = fts3.new_transfer( ftsFile.SourceSURL,
ftsFile.TargetSURL,
checksum = 'ADLER32:%s'%ftsFile.Checksum,
filesize = ftsFile.Size )
transfers.append( trans )
source_spacetoken = self.SourceToken if self.SourceToken else None
dest_spacetoken = self.TargetToken if self.TargetToken else None
copy_pin_lifetime = pinTime if pinTime else None
bring_online = 86400 if pinTime else None
job = fts3.new_job( transfers = transfers, overwrite = True,
source_spacetoken = source_spacetoken, spacetoken = dest_spacetoken,
bring_online = bring_online, copy_pin_lifetime = copy_pin_lifetime, retry = 3 )
try:
if not self._fts3context:
self._fts3context = fts3.Context( endpoint = self.FTSServer, request_class = ftsSSLRequest, verify = False )
context = self._fts3context
self.FTSGUID = fts3.submit( context, job )
except Exception as e:
return S_ERROR( "Error at submission: %s" % e )
self.Status = "Submitted"
self._log = gLogger.getSubLogger( "req_%s/FTSJob-%s" % ( self.RequestID, self.FTSGUID ) , True )
for ftsFile in self:
ftsFile.FTSGUID = self.FTSGUID
ftsFile.Status = "Submitted"
return S_OK()
def monitorFTS3( self, full = False ):
if not self.FTSGUID:
return S_ERROR( "FTSGUID not set, FTS job not submitted?" )
jobStatusDict = None
try:
if not self._fts3context:
self._fts3context = fts3.Context( endpoint = self.FTSServer, request_class = ftsSSLRequest, verify = False )
context = self._fts3context
jobStatusDict = fts3.get_job_status( context, self.FTSGUID, list_files = True )
except Exception as e:
return S_ERROR( "Error getting the job status %s" % e )
self.Status = jobStatusDict['job_state'].capitalize()
filesInfoList = jobStatusDict['files']
statusSummary = {}
for fileDict in filesInfoList:
file_state = fileDict['file_state'].capitalize()
statusSummary[file_state] = statusSummary.get( file_state, 0 ) + 1
total = len( filesInfoList )
completed = sum( [ statusSummary.get( state, 0 ) for state in FTSFile.FINAL_STATES ] )
self.Completeness = 100 * completed / total
if not full:
return S_OK( statusSummary )
ftsFilesPrinted = False
for fileDict in filesInfoList:
sourceURL = fileDict['source_surl']
targetURL = fileDict['dest_surl']
fileStatus = fileDict['file_state'].capitalize()
reason = fileDict['reason']
duration = fileDict['tx_duration']
candidateFile = None
for ftsFile in self:
if ftsFile.SourceSURL == sourceURL and ftsFile.TargetSURL == targetURL :
candidateFile = ftsFile
break
if candidateFile is None:
self._log.warn( 'FTSFile not found', 'Source: %s, Target: %s' % ( sourceURL, targetURL ) )
if not ftsFilesPrinted:
ftsFilesPrinted = True
if not len( self ):
self._log.warn( 'Monitored FTS job is empty!' )
else:
self._log.warn( 'All FTS files are:', '\n' + '\n'.join( ['Source: %s, Target: %s' % ( ftsFile.SourceSURL, ftsFile.TargetSURL ) for ftsFile in self] ) )
else:
candidateFile.Status = fileStatus
candidateFile.Error = reason
candidateFile._duration = duration
if candidateFile.Status == "Failed":
for missingSource in self.missingSourceErrors:
if missingSource.match( reason ):
candidateFile.Error = "MissingSource"
# # register successful files
if self.Status in FTSJob.FINALSTATES:
return self.finalize()
return S_OK()
def monitorFTS( self, ftsVersion, command = "glite-transfer-status", full = False ):
""" Wrapper calling the proper method for a given version of FTS"""
if ftsVersion == "FTS2":
return self.monitorFTS2( command = command, full = full )
elif ftsVersion == "FTS3":
return self.monitorFTS3( full = full )
else:
return S_ERROR( "monitorFTS: unknown FTS version %s" % ftsVersion )
def submitFTS( self, ftsVersion, command = 'glite-transfer-submit', pinTime = False ):
""" Wrapper calling the proper method for a given version of FTS"""
if ftsVersion == "FTS2":
return self.submitFTS2( command = command, pinTime = pinTime )
elif ftsVersion == "FTS3":
return self.submitFTS3( pinTime = pinTime )
else:
return S_ERROR( "submitFTS: unknown FTS version %s" % ftsVersion )
def finalize( self ):
""" register successfully transferred files """
if self.Status not in FTSJob.FINALSTATES:
return S_OK()
if not len( self ):
return S_ERROR( "Empty job in finalize" )
startTime = time.time()
targetSE = StorageElement( self.TargetSE )
toRegister = [ ftsFile for ftsFile in self if ftsFile.Status == "Finished" ]
toRegisterDict = {}
for ftsFile in toRegister:
pfn = returnSingleResult( targetSE.getURL( ftsFile.LFN, protocol = 'srm' ) )
if pfn["OK"]:
pfn = pfn["Value"]
toRegisterDict[ ftsFile.LFN ] = { "PFN": pfn, "SE": self.TargetSE }
else:
self._log.error( "Error getting SRM URL", pfn['Message'] )
if toRegisterDict:
self._regTotal += len( toRegisterDict )
register = self._fc.addReplica( toRegisterDict )
self._regTime += time.time() - startTime
if not register["OK"]:
self._log.error( 'Error registering replica', register['Message'] )
for ftsFile in toRegister:
ftsFile.Error = "AddCatalogReplicaFailed"
return register
register = register["Value"]
self._regSuccess += len( register.get( 'Successful', {} ) )
if self._regSuccess:
self._log.info( 'Successfully registered %d replicas' % self._regSuccess )
failedFiles = register.get( "Failed", {} )
errorReason = {}
for lfn, reason in failedFiles.items():
errorReason.setdefault( str( reason ), [] ).append( lfn )
for reason in errorReason:
self._log.error( 'Error registering %d replicas' % len( errorReason[reason] ), reason )
for ftsFile in toRegister:
if ftsFile.LFN in failedFiles:
ftsFile.Error = "AddCatalogReplicaFailed"
else:
statuses = set( [ftsFile.Status for ftsFile in self] )
self._log.warn( "No replicas to register for FTSJob (%s) - Files status: '%s'" % \
( self.Status, ','.join( sorted( statuses ) ) ) )
return S_OK()
def toSQL( self ):
""" prepare SQL INSERT or UPDATE statement
:return: str with SQL fragment
"""
colVals = []
for column, value in self.__data__.items():
if value and column not in ( "FTSJobID", "LastUpdate" ):
colStr = "`%s`" % column
if isinstance( value, datetime.datetime ) or isinstance( value, basestring ):
valStr = "'%s'" % value
else:
valStr = str( value )
colVals.append( ( colStr, valStr ) )
colVals.append( ( "`LastUpdate`", "UTC_TIMESTAMP()" ) )
query = []
if self.FTSJobID:
query.append( "UPDATE `FTSJob` SET " )
query.append( ",".join( [ "%s=%s" % item for item in colVals ] ) )
query.append( " WHERE `FTSJobID`=%d;\n" % self.FTSJobID )
else:
query.append( "INSERT INTO `FTSJob` " )
columns = "(%s)" % ",".join( [ column for column, value in colVals ] )
values = "(%s)" % ",".join( [ value for column, value in colVals ] )
query.append( columns )
query.append( " VALUES %s;" % values )
return S_OK( "".join( query ) )
def toJSON( self ):
""" dump to JSON format """
digest = dict( zip( self.__data__.keys(),
[ str( val ) if val else "" for val in self.__data__.values() ] ) )
digest["FTSFiles"] = []
for ftsFile in self:
fileJSON = ftsFile.toJSON()
if not fileJSON["OK"]:
return fileJSON
digest["FTSFiles"].append( fileJSON["Value"] )
return S_OK( digest )
# Check is provided SE is OK
se = StorageElement( targetSE )
if not se.valid:
print se.errorReason
print
Script.showHelp()
from DIRAC.RequestManagementSystem.Client.ReqClient import ReqClient
from DIRAC.RequestManagementSystem.Client.Request import Request
from DIRAC.RequestManagementSystem.Client.Operation import Operation
from DIRAC.RequestManagementSystem.Client.File import File
from DIRAC.RequestManagementSystem.private.RequestValidator import RequestValidator
from DIRAC.Resources.Catalog.FileCatalog import FileCatalog
reqClient = ReqClient()
fc = FileCatalog()
for lfnList in breakListIntoChunks( lfns, 100 ):
oRequest = Request()
oRequest.RequestName = "%s_%s" % ( md5( repr( time.time() ) ).hexdigest()[:16], md5( repr( time.time() ) ).hexdigest()[:16] )
replicateAndRegister = Operation()
replicateAndRegister.Type = 'ReplicateAndRegister'
replicateAndRegister.TargetSE = targetSE
res = fc.getFileMetadata( lfnList )
if not res['OK']:
print "Can't get file metadata: %s" % res['Message']
DIRAC.exit( 1 )
if res['Value']['Failed']:
class FTSRequest( object ):
"""
.. class:: FTSRequest
Helper class for FTS job submission and monitoring.
"""
# # default checksum type
__defaultCksmType = "ADLER32"
# # flag to disablr/enable checksum test, default: disabled
__cksmTest = False
def __init__( self ):
"""c'tor
:param self: self reference
"""
self.log = gLogger.getSubLogger( self.__class__.__name__, True )
# # final states tuple
self.finalStates = ( 'Canceled', 'Failed', 'Hold',
'Finished', 'FinishedDirty' )
# # failed states tuple
self.failedStates = ( 'Canceled', 'Failed',
'Hold', 'FinishedDirty' )
# # successful states tuple
self.successfulStates = ( 'Finished', 'Done' )
# # all file states tuple
self.fileStates = ( 'Done', 'Active', 'Pending', 'Ready', 'Canceled', 'Failed',
'Finishing', 'Finished', 'Submitted', 'Hold', 'Waiting' )
self.statusSummary = {}
# # request status
self.requestStatus = 'Unknown'
# # dict for FTS job files
self.fileDict = {}
# # dict for replicas information
self.catalogReplicas = {}
# # dict for metadata information
self.catalogMetadata = {}
# # dict for files that failed to register
self.failedRegistrations = {}
# # placehoder for FileCatalog reference
self.oCatalog = None
# # submit timestamp
self.submitTime = ''
# # placeholder FTS job GUID
self.ftsGUID = ''
# # placeholder for FTS server URL
self.ftsServer = ''
# # flag marking FTS job completness
self.isTerminal = False
# # completness percentage
self.percentageComplete = 0.0
# # source SE name
self.sourceSE = ''
# # flag marking source SE validity
self.sourceValid = False
# # source space token
self.sourceToken = ''
# # target SE name
self.targetSE = ''
# # flag marking target SE validity
self.targetValid = False
# # target space token
self.targetToken = ''
# # placeholder for target StorageElement
self.oTargetSE = None
# # placeholder for source StorageElement
self.oSourceSE = None
# # checksum type, set it to default
self.__cksmType = self.__defaultCksmType
# # disable checksum test by default
self.__cksmTest = False
# # statuses that prevent submitting to FTS
self.noSubmitStatus = ( 'Failed', 'Done', 'Staging' )
# # were sources resolved?
self.sourceResolved = False
# # Number of file transfers actually submitted
self.submittedFiles = 0
self.transferTime = 0
self.submitCommand = Operations().getValue( 'DataManagement/FTSPlacement/FTS2/SubmitCommand', 'glite-transfer-submit' )
self.monitorCommand = Operations().getValue( 'DataManagement/FTSPlacement/FTS2/MonitorCommand', 'glite-transfer-status' )
self.ftsJob = None
self.ftsFiles = []
####################################################################
#
# Methods for setting/getting/checking the SEs
#
def setSourceSE( self, se ):
""" set SE for source
:param self: self reference
:param str se: source SE name
"""
if se == self.targetSE:
return S_ERROR( "SourceSE is TargetSE" )
self.sourceSE = se
self.oSourceSE = StorageElement( self.sourceSE )
return self.__checkSourceSE()
def __checkSourceSE( self ):
""" check source SE availability
:param self: self reference
"""
if not self.sourceSE:
return S_ERROR( "SourceSE not set" )
res = self.oSourceSE.isValid( 'Read' )
if not res['OK']:
return S_ERROR( "SourceSE not available for reading" )
res = self.__getSESpaceToken( self.oSourceSE )
if not res['OK']:
self.log.error( "FTSRequest failed to get SRM Space Token for SourceSE", res['Message'] )
return S_ERROR( "SourceSE does not support FTS transfers" )
if self.__cksmTest:
res = self.oSourceSE.getChecksumType()
if not res["OK"]:
self.log.error( "Unable to get checksum type for SourceSE %s: %s" % ( self.sourceSE,
res["Message"] ) )
cksmType = res["Value"]
if cksmType in ( "NONE", "NULL" ):
self.log.warn( "Checksum type set to %s at SourceSE %s, disabling checksum test" % ( cksmType,
self.sourceSE ) )
self.__cksmTest = False
elif cksmType != self.__cksmType:
self.log.warn( "Checksum type mismatch, disabling checksum test" )
self.__cksmTest = False
self.sourceToken = res['Value']
self.sourceValid = True
return S_OK()
def setTargetSE( self, se ):
""" set target SE
:param self: self reference
:param str se: target SE name
"""
if se == self.sourceSE:
return S_ERROR( "TargetSE is SourceSE" )
self.targetSE = se
self.oTargetSE = StorageElement( self.targetSE )
return self.__checkTargetSE()
def setTargetToken( self, token ):
""" target space token setter
:param self: self reference
:param str token: target space token
"""
self.targetToken = token
return S_OK()
def __checkTargetSE( self ):
""" check target SE availability
:param self: self reference
"""
if not self.targetSE:
return S_ERROR( "TargetSE not set" )
res = self.oTargetSE.isValid( 'Write' )
if not res['OK']:
return S_ERROR( "TargetSE not available for writing" )
res = self.__getSESpaceToken( self.oTargetSE )
if not res['OK']:
self.log.error( "FTSRequest failed to get SRM Space Token for TargetSE", res['Message'] )
return S_ERROR( "TargetSE does not support FTS transfers" )
# # check checksum types
if self.__cksmTest:
res = self.oTargetSE.getChecksumType()
if not res["OK"]:
self.log.error( "Unable to get checksum type for TargetSE %s: %s" % ( self.targetSE,
res["Message"] ) )
cksmType = res["Value"]
if cksmType in ( "NONE", "NULL" ):
self.log.warn( "Checksum type set to %s at TargetSE %s, disabling checksum test" % ( cksmType,
self.targetSE ) )
self.__cksmTest = False
elif cksmType != self.__cksmType:
self.log.warn( "Checksum type mismatch, disabling checksum test" )
self.__cksmTest = False
self.targetToken = res['Value']
self.targetValid = True
return S_OK()
@staticmethod
def __getSESpaceToken( oSE ):
""" get space token from StorageElement instance
:param self: self reference
:param StorageElement oSE: StorageElement instance
"""
res = oSE.getStorageParameters( "SRM2" )
if not res['OK']:
return res
return S_OK( res['Value'].get( 'SpaceToken' ) )
####################################################################
#
# Methods for setting/getting FTS request parameters
#
def setFTSGUID( self, guid ):
""" FTS job GUID setter
:param self: self reference
:param str guid: string containg GUID
"""
if not checkGuid( guid ):
return S_ERROR( "Incorrect GUID format" )
self.ftsGUID = guid
return S_OK()
def setFTSServer( self, server ):
""" FTS server setter
:param self: self reference
:param str server: FTS server URL
"""
self.ftsServer = server
return S_OK()
def isRequestTerminal( self ):
""" check if FTS job has terminated
:param self: self reference
"""
if self.requestStatus in self.finalStates:
self.isTerminal = True
return S_OK( self.isTerminal )
def setCksmTest( self, cksmTest = False ):
""" set cksm test
:param self: self reference
:param bool cksmTest: flag to enable/disable checksum test
"""
self.__cksmTest = bool( cksmTest )
return S_OK( self.__cksmTest )
####################################################################
#
# Methods for setting/getting/checking files and their metadata
#
def setLFN( self, lfn ):
""" add LFN :lfn: to :fileDict:
:param self: self reference
:param str lfn: LFN to add to
"""
self.fileDict.setdefault( lfn, {'Status':'Waiting'} )
return S_OK()
def setSourceSURL( self, lfn, surl ):
""" source SURL setter
:param self: self reference
:param str lfn: LFN
:param str surl: source SURL
"""
target = self.fileDict[lfn].get( 'Target' )
if target == surl:
return S_ERROR( "Source and target the same" )
return self.__setFileParameter( lfn, 'Source', surl )
def getSourceSURL( self, lfn ):
""" get source SURL for LFN :lfn:
:param self: self reference
:param str lfn: LFN
"""
return self.__getFileParameter( lfn, 'Source' )
def setTargetSURL( self, lfn, surl ):
""" set target SURL for LFN :lfn:
:param self: self reference
:param str lfn: LFN
:param str surl: target SURL
"""
source = self.fileDict[lfn].get( 'Source' )
if source == surl:
return S_ERROR( "Source and target the same" )
return self.__setFileParameter( lfn, 'Target', surl )
def getFailReason( self, lfn ):
""" get fail reason for file :lfn:
:param self: self reference
:param str lfn: LFN
"""
return self.__getFileParameter( lfn, 'Reason' )
def getRetries( self, lfn ):
""" get number of attepmts made to transfer file :lfn:
:param self: self reference
:param str lfn: LFN
"""
return self.__getFileParameter( lfn, 'Retries' )
def getTransferTime( self, lfn ):
""" get duration of transfer for file :lfn:
:param self: self reference
:param str lfn: LFN
"""
return self.__getFileParameter( lfn, 'Duration' )
def getFailed( self ):
""" get list of wrongly transferred LFNs
:param self: self reference
"""
return S_OK( [ lfn for lfn in self.fileDict
if self.fileDict[lfn].get( 'Status', '' ) in self.failedStates ] )
def getStaging( self ):
""" get files set for prestaging """
return S_OK( [lfn for lfn in self.fileDict
if self.fileDict[lfn].get( 'Status', '' ) == 'Staging'] )
def getDone( self ):
""" get list of succesfully transferred LFNs
:param self: self reference
"""
return S_OK( [ lfn for lfn in self.fileDict
if self.fileDict[lfn].get( 'Status', '' ) in self.successfulStates ] )
def __setFileParameter( self, lfn, paramName, paramValue ):
""" set :paramName: to :paramValue: for :lfn: file
:param self: self reference
:param str lfn: LFN
:param str paramName: parameter name
:param mixed paramValue: a new parameter value
"""
self.setLFN( lfn )
self.fileDict[lfn][paramName] = paramValue
return S_OK()
def __getFileParameter( self, lfn, paramName ):
""" get value of :paramName: for file :lfn:
:param self: self reference
:param str lfn: LFN
:param str paramName: parameter name
"""
if lfn not in self.fileDict:
return S_ERROR( "Supplied file not set" )
if paramName not in self.fileDict[lfn]:
return S_ERROR( "%s not set for file" % paramName )
return S_OK( self.fileDict[lfn][paramName] )
####################################################################
#
# Methods for submission
#
def submit( self, monitor = False, printOutput = True ):
""" submit FTS job
:param self: self reference
:param bool monitor: flag to monitor progress of FTS job
:param bool printOutput: flag to print output of execution to stdout
"""
res = self.__prepareForSubmission()
if not res['OK']:
return res
res = self.__submitFTSTransfer()
if not res['OK']:
return res
resDict = { 'ftsGUID' : self.ftsGUID, 'ftsServer' : self.ftsServer, 'submittedFiles' : self.submittedFiles }
if monitor or printOutput:
gLogger.always( "Submitted %s@%s" % ( self.ftsGUID, self.ftsServer ) )
if monitor:
self.monitor( untilTerminal = True, printOutput = printOutput, full = False )
return S_OK( resDict )
def __prepareForSubmission( self ):
""" check validity of job before submission
:param self: self reference
"""
if not self.fileDict:
return S_ERROR( "No files set" )
if not self.sourceValid:
return S_ERROR( "SourceSE not valid" )
if not self.targetValid:
return S_ERROR( "TargetSE not valid" )
if not self.ftsServer:
res = self.__resolveFTSServer()
if not res['OK']:
return S_ERROR( "FTSServer not valid" )
self.resolveSource()
self.resolveTarget()
res = self.__filesToSubmit()
if not res['OK']:
return S_ERROR( "No files to submit" )
return S_OK()
def __getCatalogObject( self ):
""" CatalogInterface instance facade
:param self: self reference
"""
try:
if not self.oCatalog:
self.oCatalog = FileCatalog()
return S_OK()
except:
return S_ERROR()
def __updateReplicaCache( self, lfns = None, overwrite = False ):
""" update replica cache for list of :lfns:
:param self: self reference
:param mixed lfns: list of LFNs
:param bool overwrite: flag to trigger cache clearing and updating
"""
if not lfns:
lfns = self.fileDict.keys()
toUpdate = [ lfn for lfn in lfns if ( lfn not in self.catalogReplicas ) or overwrite ]
if not toUpdate:
return S_OK()
res = self.__getCatalogObject()
if not res['OK']:
return res
res = self.oCatalog.getReplicas( toUpdate )
if not res['OK']:
return S_ERROR( "Failed to update replica cache: %s" % res['Message'] )
for lfn, error in res['Value']['Failed'].items():
self.__setFileParameter( lfn, 'Reason', error )
self.__setFileParameter( lfn, 'Status', 'Failed' )
for lfn, replicas in res['Value']['Successful'].items():
self.catalogReplicas[lfn] = replicas
return S_OK()
def __updateMetadataCache( self, lfns = None ):
""" update metadata cache for list of LFNs
:param self: self reference
:param list lnfs: list of LFNs
"""
if not lfns:
lfns = self.fileDict.keys()
toUpdate = [ lfn for lfn in lfns if lfn not in self.catalogMetadata ]
if not toUpdate:
return S_OK()
res = self.__getCatalogObject()
if not res['OK']:
return res
res = self.oCatalog.getFileMetadata( toUpdate )
if not res['OK']:
return S_ERROR( "Failed to get source catalog metadata: %s" % res['Message'] )
for lfn, error in res['Value']['Failed'].items():
self.__setFileParameter( lfn, 'Reason', error )
self.__setFileParameter( lfn, 'Status', 'Failed' )
for lfn, metadata in res['Value']['Successful'].items():
self.catalogMetadata[lfn] = metadata
return S_OK()
def resolveSource( self ):
""" resolve source SE eligible for submission
:param self: self reference
"""
# Avoid resolving sources twice
if self.sourceResolved:
return S_OK()
# Only resolve files that need a transfer
toResolve = [ lfn for lfn in self.fileDict if self.fileDict[lfn].get( "Status", "" ) != "Failed" ]
if not toResolve:
return S_OK()
res = self.__updateMetadataCache( toResolve )
if not res['OK']:
return res
res = self.__updateReplicaCache( toResolve )
if not res['OK']:
return res
# Define the source URLs
for lfn in toResolve:
replicas = self.catalogReplicas.get( lfn, {} )
if self.sourceSE not in replicas:
gLogger.warn( "resolveSource: skipping %s - not replicas at SourceSE %s" % ( lfn, self.sourceSE ) )
self.__setFileParameter( lfn, 'Reason', "No replica at SourceSE" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
continue
# Fix first the PFN
pfn = self.oSourceSE.getPfnForLfn( lfn ).get( 'Value', {} ).get( 'Successful', {} ).get( lfn, replicas[self.sourceSE] )
res = returnSingleResult( self.oSourceSE.getPfnForProtocol( pfn, protocol = 'SRM2', withPort = True ) )
if not res['OK']:
gLogger.warn( "resolveSource: skipping %s - %s" % ( lfn, res["Message"] ) )
self.__setFileParameter( lfn, 'Reason', res['Message'] )
self.__setFileParameter( lfn, 'Status', 'Failed' )
continue
res = self.setSourceSURL( lfn, res['Value'] )
if not res['OK']:
gLogger.warn( "resolveSource: skipping %s - %s" % ( lfn, res["Message"] ) )
self.__setFileParameter( lfn, 'Reason', res['Message'] )
self.__setFileParameter( lfn, 'Status', 'Failed' )
continue
toResolve = {}
for lfn in self.fileDict:
if "Source" in self.fileDict[lfn]:
toResolve[self.fileDict[lfn]['Source']] = lfn
if not toResolve:
return S_ERROR( "No eligible Source files" )
# Get metadata of the sources, to check for existance, availability and caching
res = self.oSourceSE.getFileMetadata( toResolve.keys() )
if not res['OK']:
return S_ERROR( "Failed to check source file metadata" )
for pfn, error in res['Value']['Failed'].items():
lfn = toResolve[pfn]
if re.search( 'File does not exist', error ):
gLogger.warn( "resolveSource: skipping %s - source file does not exists" % lfn )
self.__setFileParameter( lfn, 'Reason', "Source file does not exist" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
else:
gLogger.warn( "resolveSource: skipping %s - failed to get source metadata" % lfn )
self.__setFileParameter( lfn, 'Reason', "Failed to get Source metadata" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
toStage = []
nbStagedFiles = 0
for pfn, metadata in res['Value']['Successful'].items():
lfn = toResolve[pfn]
lfnStatus = self.fileDict.get( lfn, {} ).get( 'Status' )
if metadata['Unavailable']:
gLogger.warn( "resolveSource: skipping %s - source file unavailable" % lfn )
self.__setFileParameter( lfn, 'Reason', "Source file Unavailable" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
elif metadata['Lost']:
gLogger.warn( "resolveSource: skipping %s - source file lost" % lfn )
self.__setFileParameter( lfn, 'Reason', "Source file Lost" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
elif not metadata['Cached']:
if lfnStatus != 'Staging':
toStage.append( pfn )
elif metadata['Size'] != self.catalogMetadata[lfn]['Size']:
gLogger.warn( "resolveSource: skipping %s - source file size mismatch" % lfn )
self.__setFileParameter( lfn, 'Reason', "Source size mismatch" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
elif self.catalogMetadata[lfn]['Checksum'] and metadata['Checksum'] and \
not compareAdler( metadata['Checksum'], self.catalogMetadata[lfn]['Checksum'] ):
gLogger.warn( "resolveSource: skipping %s - source file checksum mismatch" % lfn )
self.__setFileParameter( lfn, 'Reason', "Source checksum mismatch" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
elif lfnStatus == 'Staging':
# file that was staging is now cached
self.__setFileParameter( lfn, 'Status', 'Waiting' )
nbStagedFiles += 1
# Some files were being staged
if nbStagedFiles:
self.log.info( 'resolveSource: %d files have been staged' % nbStagedFiles )
# Launching staging of files not in cache
if toStage:
gLogger.warn( "resolveSource: %s source files not cached, prestaging..." % len( toStage ) )
stage = self.oSourceSE.prestageFile( toStage )
if not stage["OK"]:
gLogger.error( "resolveSource: error is prestaging - %s" % stage["Message"] )
for pfn in toStage:
lfn = toResolve[pfn]
self.__setFileParameter( lfn, 'Reason', stage["Message"] )
self.__setFileParameter( lfn, 'Status', 'Failed' )
else:
for pfn in toStage:
lfn = toResolve[pfn]
if pfn in stage['Value']['Successful']:
self.__setFileParameter( lfn, 'Status', 'Staging' )
elif pfn in stage['Value']['Failed']:
self.__setFileParameter( lfn, 'Reason', stage['Value']['Failed'][pfn] )
self.__setFileParameter( lfn, 'Status', 'Failed' )
self.sourceResolved = True
return S_OK()
def resolveTarget( self ):
""" find target SE eligible for submission
:param self: self reference
"""
toResolve = [ lfn for lfn in self.fileDict
if self.fileDict[lfn].get( 'Status' ) not in self.noSubmitStatus ]
if not toResolve:
return S_OK()
res = self.__updateReplicaCache( toResolve )
if not res['OK']:
return res
for lfn in toResolve:
res = self.oTargetSE.getPfnForLfn( lfn )
if not res['OK'] or lfn not in res['Value']['Successful']:
gLogger.warn( "resolveTarget: skipping %s - failed to create target pfn" % lfn )
self.__setFileParameter( lfn, 'Reason', "Failed to create Target" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
continue
pfn = res['Value']['Successful'][lfn]
res = self.oTargetSE.getPfnForProtocol( pfn, protocol = 'SRM2', withPort = True )
if not res['OK'] or pfn not in res['Value']['Successful']:
reason = res.get( 'Message', res.get( 'Value', {} ).get( 'Failed', {} ).get( pfn ) )
gLogger.warn( "resolveTarget: skipping %s - %s" % ( lfn, reason ) )
self.__setFileParameter( lfn, 'Reason', reason )
self.__setFileParameter( lfn, 'Status', 'Failed' )
continue
pfn = res['Value']['Successful'][pfn]
res = self.setTargetSURL( lfn, pfn )
if not res['OK']:
gLogger.warn( "resolveTarget: skipping %s - %s" % ( lfn, res["Message"] ) )
self.__setFileParameter( lfn, 'Reason', res['Message'] )
self.__setFileParameter( lfn, 'Status', 'Failed' )
continue
toResolve = {}
for lfn in self.fileDict:
if "Target" in self.fileDict[lfn]:
toResolve[self.fileDict[lfn]['Target']] = lfn
if not toResolve:
return S_ERROR( "No eligible Target files" )
res = self.oTargetSE.exists( toResolve.keys() )
if not res['OK']:
return S_ERROR( "Failed to check target existence" )
for pfn, error in res['Value']['Failed'].items():
lfn = toResolve[pfn]
self.__setFileParameter( lfn, 'Reason', error )
self.__setFileParameter( lfn, 'Status', 'Failed' )
toRemove = []
for pfn, exists in res['Value']['Successful'].items():
if exists:
lfn = toResolve[pfn]
res = self.getSourceSURL( lfn )
if not res['OK']:
gLogger.warn( "resolveTarget: skipping %s - target exists" % lfn )
self.__setFileParameter( lfn, 'Reason', "Target exists" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
elif res['Value'] == pfn:
gLogger.warn( "resolveTarget: skipping %s - source and target pfns are the same" % lfn )
self.__setFileParameter( lfn, 'Reason', "Source and Target the same" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
else:
toRemove.append( pfn )
if toRemove:
self.oTargetSE.removeFile( toRemove )
return S_OK()
def __filesToSubmit( self ):
"""
check if there is at least one file to submit
:return: S_OK if at least one file is present, S_ERROR otherwise
"""
for lfn in self.fileDict:
lfnStatus = self.fileDict[lfn].get( 'Status' )
source = self.fileDict[lfn].get( 'Source' )
target = self.fileDict[lfn].get( 'Target' )
if lfnStatus not in self.noSubmitStatus and source and target:
return S_OK()
return S_ERROR()
def __createFTSFiles( self ):
""" create LFNs file for glite-transfer-submit command
This file consists one line for each fiel to be transferred:
sourceSURL targetSURL [CHECKSUMTYPE:CHECKSUM]
:param self: self reference
"""
self.__updateMetadataCache()
for lfn in self.fileDict:
lfnStatus = self.fileDict[lfn].get( 'Status' )
if lfnStatus not in self.noSubmitStatus:
cksmStr = ""
# # add chsmType:cksm only if cksmType is specified, else let FTS decide by itself
if self.__cksmTest and self.__cksmType:
checkSum = self.catalogMetadata.get( lfn, {} ).get( 'Checksum' )
if checkSum:
cksmStr = " %s:%s" % ( self.__cksmType, intAdlerToHex( hexAdlerToInt( checkSum ) ) )
ftsFile = FTSFile()
ftsFile.LFN = lfn
ftsFile.SourceSURL = self.fileDict[lfn].get( 'Source' )
ftsFile.TargetSURL = self.fileDict[lfn].get( 'Target' )
ftsFile.SourceSE = self.sourceSE
ftsFile.TargetSE = self.targetSE
ftsFile.Status = self.fileDict[lfn].get( 'Status' )
ftsFile.Checksum = cksmStr
ftsFile.Size = self.catalogMetadata.get( lfn, {} ).get( 'Size' )
self.ftsFiles.append( ftsFile )
self.submittedFiles += 1
return S_OK()
def __createFTSJob( self, guid = None ):
self.__createFTSFiles()
ftsJob = FTSJob()
ftsJob.RequestID = 0
ftsJob.OperationID = 0
ftsJob.SourceSE = self.sourceSE
ftsJob.TargetSE = self.targetSE
ftsJob.SourceToken = self.sourceToken
ftsJob.TargetToken = self.targetToken
ftsJob.FTSServer = self.ftsServer
if guid:
ftsJob.FTSGUID = guid
for ftsFile in self.ftsFiles:
ftsFile.Attempt += 1
ftsFile.Error = ""
ftsJob.addFile( ftsFile )
self.ftsJob = ftsJob
def __submitFTSTransfer( self ):
""" create and execute glite-transfer-submit CLI command
:param self: self reference
"""
log = gLogger.getSubLogger( 'Submit' )
self.__createFTSJob()
submit = self.ftsJob.submitFTS2( command = self.submitCommand )
if not submit["OK"]:
log.error( "unable to submit FTSJob: %s" % submit["Message"] )
return submit
log.info( "FTSJob '%s'@'%s' has been submitted" % ( self.ftsJob.FTSGUID, self.ftsJob.FTSServer ) )
# # update statuses for job files
for ftsFile in self.ftsJob:
ftsFile.FTSGUID = self.ftsJob.FTSGUID
ftsFile.Status = "Submitted"
ftsFile.Attempt += 1
log.info( "FTSJob '%s'@'%s' has been submitted" % ( self.ftsJob.FTSGUID, self.ftsJob.FTSServer ) )
self.ftsGUID = self.ftsJob.FTSGUID
return S_OK()
def __resolveFTSServer( self ):
"""
resolve FTS server to use, it should be the closest one from target SE
:param self: self reference
"""
from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getFTSServersForSites
if not self.targetSE:
return S_ERROR( "Target SE not set" )
res = getSitesForSE( self.targetSE )
if not res['OK'] or not res['Value']:
return S_ERROR( "Could not determine target site" )
targetSites = res['Value']
targetSite = ''
for targetSite in targetSites:
targetFTS = getFTSServersForSites( [targetSite] )
if targetFTS['OK']:
ftsTarget = targetFTS['Value'][targetSite]
if ftsTarget:
self.ftsServer = ftsTarget
return S_OK( self.ftsServer )
else:
return targetFTS
return S_ERROR( 'No FTS server found for %s' % targetSite )
####################################################################
#
# Methods for monitoring
#
def summary( self, untilTerminal = False, printOutput = False ):
""" summary of FTS job
:param self: self reference
:param bool untilTerminal: flag to monitor FTS job to its final state
:param bool printOutput: flag to print out monitoring information to the stdout
"""
res = self.__isSummaryValid()
if not res['OK']:
return res
while not self.isTerminal:
res = self.__parseOutput( full = True )
if not res['OK']:
return res
if untilTerminal:
self.__print()
self.isRequestTerminal()
if res['Value'] or ( not untilTerminal ):
break
time.sleep( 1 )
if untilTerminal:
print ""
if printOutput and ( not untilTerminal ):
return self.dumpSummary( printOutput = printOutput )
return S_OK()
def monitor( self, untilTerminal = False, printOutput = False, full = True ):
""" monitor FTS job
:param self: self reference
:param bool untilTerminal: flag to monitor FTS job to its final state
:param bool printOutput: flag to print out monitoring information to the stdout
"""
if not self.ftsJob:
self.resolveSource()
self.__createFTSJob( self.ftsGUID )
res = self.__isSummaryValid()
if not res['OK']:
return res
if untilTerminal:
res = self.summary( untilTerminal = untilTerminal, printOutput = printOutput )
if not res['OK']:
return res
res = self.__parseOutput( full = full )
if not res['OK']:
return res
if untilTerminal:
self.finalize()
if printOutput:
self.dump()
return res
def dumpSummary( self, printOutput = False ):
""" get FTS job summary as str
:param self: self reference
:param bool printOutput: print summary to stdout
"""
outStr = ''
for status in sorted( self.statusSummary ):
if self.statusSummary[status]:
outStr = '%s\t%-10s : %-10s\n' % ( outStr, status, str( self.statusSummary[status] ) )
outStr = outStr.rstrip( '\n' )
if printOutput:
print outStr
return S_OK( outStr )
def __print( self ):
""" print progress bar of FTS job completeness to stdout
:param self: self reference
"""
width = 100
bits = int( ( width * self.percentageComplete ) / 100 )
outStr = "|%s>%s| %.1f%s %s %s" % ( "="*bits, " "*( width - bits ),
self.percentageComplete, "%",
self.requestStatus, " "*10 )
sys.stdout.write( "%s\r" % ( outStr ) )
sys.stdout.flush()
def dump( self ):
""" print FTS job parameters and files to stdout
:param self: self reference
"""
print "%-10s : %-10s" % ( "Status", self.requestStatus )
print "%-10s : %-10s" % ( "Source", self.sourceSE )
print "%-10s : %-10s" % ( "Target", self.targetSE )
print "%-10s : %-128s" % ( "Server", self.ftsServer )
print "%-10s : %-128s" % ( "GUID", self.ftsGUID )
for lfn in sorted( self.fileDict ):
print "\n %-15s : %-128s" % ( 'LFN', lfn )
for key in ['Source', 'Target', 'Status', 'Reason', 'Duration']:
print " %-15s : %-128s" % ( key, str( self.fileDict[lfn].get( key ) ) )
return S_OK()
def __isSummaryValid( self ):
""" check validity of FTS job summary report
:param self: self reference
"""
if not self.ftsServer:
return S_ERROR( "FTSServer not set" )
if not self.ftsGUID:
return S_ERROR( "FTSGUID not set" )
return S_OK()
def __parseOutput( self, full = False ):
""" execute glite-transfer-status command and parse its output
:param self: self reference
:param bool full: glite-transfer-status verbosity level, when set, collect information of files as well
"""
monitor = self.ftsJob.monitorFTS2( command = self.monitorCommand, full = full )
if not monitor['OK']:
return monitor
self.percentageComplete = self.ftsJob.Completeness
self.requestStatus = self.ftsJob.Status
self.submitTime = self.ftsJob.SubmitTime
statusSummary = monitor['Value']
if statusSummary:
for state in statusSummary:
self.statusSummary[state] = statusSummary[state]
self.transferTime = 0
for ftsFile in self.ftsJob:
lfn = ftsFile.LFN
self.__setFileParameter( lfn, 'Status', ftsFile.Status )
self.__setFileParameter( lfn, 'Reason', ftsFile.Error )
self.__setFileParameter( lfn, 'Duration', ftsFile._duration )
targetURL = self.__getFileParameter( lfn, 'Target' )
if not targetURL['OK']:
self.__setFileParameter( lfn, 'Target', ftsFile.TargetSURL )
self.transferTime += int( ftsFile._duration )
return S_OK()
####################################################################
#
# Methods for finalization
#
def finalize( self ):
""" finalize FTS job
:param self: self reference
"""
self.__updateMetadataCache()
transEndTime = dateTime()
regStartTime = time.time()
res = self.getTransferStatistics()
transDict = res['Value']
res = self.__registerSuccessful( transDict['transLFNs'] )
regSuc, regTotal = res['Value']
regTime = time.time() - regStartTime
if self.sourceSE and self.targetSE:
self.__sendAccounting( regSuc, regTotal, regTime, transEndTime, transDict )
return S_OK()
def getTransferStatistics( self ):
""" collect information of Transfers that can be used by Accounting
:param self: self reference
"""
transDict = { 'transTotal': len( self.fileDict ),
'transLFNs': [],
'transOK': 0,
'transSize': 0 }
for lfn in self.fileDict:
if self.fileDict[lfn].get( 'Status' ) in self.successfulStates:
if self.fileDict[lfn].get( 'Duration', 0 ):
transDict['transLFNs'].append( lfn )
transDict['transOK'] += 1
if lfn in self.catalogMetadata:
transDict['transSize'] += self.catalogMetadata[lfn].get( 'Size', 0 )
return S_OK( transDict )
def getFailedRegistrations( self ):
""" get failed registrations dict
:param self: self reference
"""
return S_OK( self.failedRegistrations )
def __registerSuccessful( self, transLFNs ):
""" register successfully transferred files to the catalogs,
fill failedRegistrations dict for files that failed to register
:param self: self reference
:param list transLFNs: LFNs in FTS job
"""
self.failedRegistrations = {}
toRegister = {}
for lfn in transLFNs:
res = returnSingleResult( self.oTargetSE.getPfnForProtocol( self.fileDict[lfn].get( 'Target' ), protocol = 'SRM2', withPort = False ) )
if not res['OK']:
self.__setFileParameter( lfn, 'Reason', res['Message'] )
self.__setFileParameter( lfn, 'Status', 'Failed' )
else:
toRegister[lfn] = { 'PFN' : res['Value'], 'SE' : self.targetSE }
if not toRegister:
return S_OK( ( 0, 0 ) )
res = self.__getCatalogObject()
if not res['OK']:
for lfn in toRegister:
self.failedRegistrations = toRegister
self.log.error( 'Failed to get Catalog Object', res['Message'] )
return S_OK( ( 0, len( toRegister ) ) )
res = self.oCatalog.addReplica( toRegister )
if not res['OK']:
self.failedRegistrations = toRegister
self.log.error( 'Failed to get Catalog Object', res['Message'] )
return S_OK( ( 0, len( toRegister ) ) )
for lfn, error in res['Value']['Failed'].items():
self.failedRegistrations[lfn] = toRegister[lfn]
self.log.error( 'Registration of Replica failed', '%s : %s' % ( lfn, str( error ) ) )
return S_OK( ( len( res['Value']['Successful'] ), len( toRegister ) ) )
def __sendAccounting( self, regSuc, regTotal, regTime, transEndTime, transDict ):
""" send accounting record
:param self: self reference
:param regSuc: number of files successfully registered
:param regTotal: number of files attepted to register
:param regTime: time stamp at the end of registration
:param transEndTime: time stamp at the end of FTS job
:param dict transDict: dict holding couters for files being transerred, their sizes and successfull transfers
"""
oAccounting = DataOperation()
oAccounting.setEndTime( transEndTime )
oAccounting.setStartTime( self.submitTime )
accountingDict = {}
accountingDict['OperationType'] = 'replicateAndRegister'
result = getProxyInfo()
if not result['OK']:
userName = '******'
else:
userName = result['Value'].get( 'username', 'unknown' )
accountingDict['User'] = userName
accountingDict['Protocol'] = 'FTS' if 'fts3' not in self.ftsServer else 'FTS3'
accountingDict['RegistrationTime'] = regTime
accountingDict['RegistrationOK'] = regSuc
accountingDict['RegistrationTotal'] = regTotal
accountingDict['TransferOK'] = transDict['transOK']
accountingDict['TransferTotal'] = transDict['transTotal']
accountingDict['TransferSize'] = transDict['transSize']
accountingDict['FinalStatus'] = self.requestStatus
accountingDict['Source'] = self.sourceSE
accountingDict['Destination'] = self.targetSE
accountingDict['TransferTime'] = self.transferTime
oAccounting.setValuesFromDict( accountingDict )
self.log.verbose( "Attempting to commit accounting message..." )
oAccounting.commit()
self.log.verbose( "...committed." )
return S_OK()
