def test_pure_inheritance(self, _sf_generateStorageObject, _rss_getSEStatus): """ In this test, we load a storage element CERN-NO-DEF that inherits from CERN-BASE, but does not redefine ANYTHING. We expect it to be just like the parent """ sf = StorageFactory(vo = 'lhcb') storages = sf.getStorages('CERN-NO-DEF') self.assertTrue(storages['OK'], storages) storages = storages['Value'] self.assertListEqual(storages['RemotePlugins'], ['GFAL2_SRM2']) # There should be a single protocol self.assertEqual(len(storages['ProtocolOptions']), 1) # There should be one storage object self.assertEqual(len(storages['StorageObjects']), 1) protocolDetail = storages['ProtocolOptions'][0] # These are the values we expect self.assertEqual(protocolDetail['Access'], 'remote') self.assertEqual(protocolDetail['Host'], 'srm-eoslhcb.cern.ch') self.assertEqual(protocolDetail['Path'], '/eos/lhcb/grid/prod') self.assertEqual(protocolDetail['PluginName'], 'GFAL2_SRM2') self.assertEqual(protocolDetail['Port'], 8443) self.assertEqual(protocolDetail['Protocol'], 'srm') self.assertEqual(protocolDetail['SpaceToken'], 'LHCb-EOS') self.assertEqual(protocolDetail['WSUrl'], '/srm/v2/server?SFN:') self.assertDictEqual(storages['StorageOptions'], {'BackendType': 'Eos', 'SEType': 'T0D1', 'BaseSE': 'CERN-BASE'})
def setUp( self ): factory = StorageFactory() res = factory.getStorages( 'CERN-RAW', ['RFIO'] ) self.assert_( res['OK'] ) storageDetails = res['Value'] self.storage = storageDetails['StorageObjects'][0] self.storage.changeDirectory( 'lhcb/test/unit-test/Storage/RFIOStorage' )
def test_simple_inheritance_overwrite(self, _sf_generateStorageObject, _rss_getSEStatus): """ In this test, we load a storage element CERN-USER that inherits from CERN-BASE, add a storage option, redefine the path and the space token """ sf = StorageFactory(vo = 'lhcb') storages = sf.getStorages('CERN-USER') self.assertTrue(storages['OK'], storages) storages = storages['Value'] self.assertListEqual(storages['RemotePlugins'], ['GFAL2_SRM2']) # There should be a single protocol self.assertEqual(len(storages['ProtocolOptions']), 1) # There should be one storage object self.assertEqual(len(storages['StorageObjects']), 1) protocolDetail = storages['ProtocolOptions'][0] # These are the values we expect self.assertEqual(protocolDetail['Access'], 'remote') self.assertEqual(protocolDetail['Host'], 'srm-eoslhcb.cern.ch') self.assertEqual(protocolDetail['Path'], '/eos/lhcb/grid/user') self.assertEqual(protocolDetail['PluginName'], 'GFAL2_SRM2') self.assertEqual(protocolDetail['Port'], 8443) self.assertEqual(protocolDetail['Protocol'], 'srm') self.assertEqual(protocolDetail['SpaceToken'], 'LHCb_USER') self.assertEqual(protocolDetail['WSUrl'], '/srm/v2/server?SFN:') self.assertDictEqual(storages['StorageOptions'], { 'BackendType': 'Eos', 'SEType': 'T0D1', 'PledgedSpace': 205, 'BaseSE': 'CERN-BASE' })
def test_more_protocol(self, _sf_generateStorageObject, _rss_getSEStatus): """ In this test, we load a storage element CERN-MORE that inherits from CERN-BASE, and adds an extra protocol """ sf = StorageFactory(vo = 'lhcb') storages = sf.getStorages('CERN-MORE') self.assertTrue(storages['OK'], storages) storages = storages['Value'] self.assertListEqual(storages['RemotePlugins'], ['Extra', 'GFAL2_SRM2']) expectedProtocols = [{ 'Access': 'remote', 'Host': 'srm-eoslhcb.cern.ch', 'Path': '/eos/lhcb/grid/prod', 'PluginName': 'Extra', 'Port': 8443, 'Protocol': 'srm', 'SpaceToken': 'LHCb-EOS', 'WSUrl': '' }, { 'Access': 'remote', 'Host': 'srm-eoslhcb.cern.ch', 'Path': '/eos/lhcb/grid/user', 'PluginName': 'GFAL2_SRM2', 'Port': 8443, 'Protocol': 'srm', 'SpaceToken': 'LHCb-EOS', 'WSUrl': '/srm/v2/server?SFN:' }] self.assertListEqual(storages['ProtocolOptions'], expectedProtocols)
def test_standalone(self, _sf_generateStorageObject, _rss_getSEStatus): """ Test loading a storage element with everything defined in itself. It should have two storage plugins """ sf = StorageFactory(vo = 'lhcb') storages = sf.getStorages('CERN-SIMPLE') self.assertTrue(storages['OK'], storages) storages = storages['Value'] self.assertListEqual(storages['LocalPlugins'], ['File']) self.assertListEqual(storages['RemotePlugins'], ['GFAL2_SRM2']) allProtocols = [] for protocol in ['RemoteAccessProtocol', 'LocalAccessProtocol']: protocolDef = copy.copy(mandatoryProtocolOptions) protocolDef.update( fake_gConfig.crawlCS('/Resources/StorageElements/CERN-SIMPLE/%s' % protocol)) allProtocols.append(protocolDef) self.assertEqual(len(storages['ProtocolOptions']), len(allProtocols)) self.assertEqual(len(storages['StorageObjects']), len(allProtocols)) self.assertListEqual(sorted(allProtocols), sorted(storages['ProtocolOptions'])) self.assertDictEqual(storages['StorageOptions'], {'BackendType': 'Eos', 'SEType': 'T0D1'})
def test_child_inherit_from_base_with_two_same_plugins(self, _sf_generateStorageObject, _rss_getSEStatus): """ In this test, we load a storage element CERN-CHILD-INHERIT-FROM-BASE-WITH-TWO-SAME-PLUGINS that inherits from CERN-BASE-WITH-TWO-SAME-PLUGINS, using two identical plugin names in two sections. """ sf = StorageFactory(vo = 'lhcb') storages = sf.getStorages('CERN-CHILD-INHERIT-FROM-BASE-WITH-TWO-SAME-PLUGINS') self.assertTrue(storages['OK'], storages) storages = storages['Value'] self.assertListEqual(storages['RemotePlugins'], ['GFAL2_SRM2', 'GFAL2_SRM2']) expectedProtocols = [{ 'Access': 'remote', 'Host': 'srm-eoslhcb.cern.ch', 'Path': '/eos/lhcb/grid/user', 'PluginName': 'GFAL2_SRM2', 'Port': 8443, 'Protocol': 'srm', 'SpaceToken': '', 'WSUrl': '/srm/v2/server?SFN:' }, { 'Access': 'remote', 'Host': 'eoslhcb.cern.ch', 'Path': '', 'PluginName': 'GFAL2_SRM2', 'Port': 8443, 'Protocol': 'root', 'SpaceToken': '', 'WSUrl': '/srm/v2/server?SFN:' }] self.assertListEqual(storages['ProtocolOptions'], expectedProtocols)
def setUp(self): factory = StorageFactory() res = factory.getStorages("CERN-RAW", ["RFIO"]) self.assert_(res["OK"]) storageDetails = res["Value"] self.storage = storageDetails["StorageObjects"][0] self.storage.changeDirectory("lhcb/test/unit-test/Storage/RFIOStorage")
def test_pure_abstract(self, _sf_generateStorageObject, _rss_getSEStatus): """ In this test, we load a storage element CERN-CHILD that inherits from CERN-ABSTRACT. CERN-ABSTRACT has two uncomplete protocols, and CERN-CHILD defines them """ sf = StorageFactory(vo = 'lhcb') storages = sf.getStorages('CERN-CHILD') self.assertTrue(storages['OK'], storages) storages = storages['Value'] self.assertListEqual(storages['RemotePlugins'], ['GFAL2_SRM2', 'GFAL2_XROOT']) expectedProtocols = [{ 'Access': 'remote', 'Host': 'srm-eoslhcb.cern.ch', 'Path': '/eos/lhcb/grid/user', 'PluginName': 'GFAL2_SRM2', 'Port': 8443, 'Protocol': 'srm', 'SpaceToken': 'LHCb_USER', 'WSUrl': '/srm/v2/server?SFN:' }, { 'Access': 'remote', 'Host': 'eoslhcb.cern.ch', 'Path': '/eos/lhcb/grid/xrootuser', 'PluginName': 'GFAL2_XROOT', 'Port': '', 'Protocol': 'root', 'SpaceToken': '', 'WSUrl': '' }] self.assertListEqual(storages['ProtocolOptions'], expectedProtocols)
def test_use_plugin_as_protocol_name_with_plugin_name(self, _sf_generateStorageObject, _rss_getSEStatus): """ In this test, we load a storage element CERN-USE-PLUGIN-AS-PROTOCOL-NAME that inherits from CERN-BASE, and uses a protocol named as a plugin name, the plugin name is also present. """ sf = StorageFactory(vo = 'lhcb') storages = sf.getStorages('CERN-USE-PLUGIN-AS-PROTOCOL-NAME-WITH-PLUGIN-NAME') self.assertTrue(storages['OK'], storages) storages = storages['Value'] self.assertListEqual(storages['RemotePlugins'], ['GFAL2_SRM2', 'GFAL2_XROOT']) expectedProtocols = [{ 'Access': 'remote', 'Host': 'srm-eoslhcb.cern.ch', 'Path': '/eos/lhcb/grid/prod', 'PluginName': 'GFAL2_SRM2', 'Port': 8443, 'Protocol': 'srm', 'SpaceToken': 'LHCb-EOS', 'WSUrl': '/srm/v2/server?SFN:' }, { 'Access': 'remote', 'Host': 'srm-eoslhcb.cern.ch', 'Path': '/eos/lhcb/grid/user', 'PluginName': 'GFAL2_XROOT', 'Port': 8443, 'Protocol': 'srm', 'SpaceToken': 'LHCb-EOS', 'WSUrl': '/srm/v2/server?SFN:' }] self.assertListEqual(storages['ProtocolOptions'], expectedProtocols)
def test_redefine_plugin_name(self, _sf_generateStorageObject, _rss_getSEStatus): """ In this test, we load a storage element CERN-REDEFINE-PLUGIN-NAME that inherits from CERN-BASE, and uses the same Plugin with a different section. """ sf = StorageFactory(vo = 'lhcb') storages = sf.getStorages('CERN-REDEFINE-PLUGIN-NAME') self.assertTrue(storages['OK'], storages) storages = storages['Value'] self.assertListEqual(storages['RemotePlugins'], ['GFAL2_SRM2','GFAL2_SRM2']) expectedProtocols = [{ 'Access': 'remote', 'Host': 'srm-eoslhcb.cern.ch', 'Path': '/eos/lhcb/grid/prod', 'PluginName': 'GFAL2_SRM2', 'Port': 8443, 'Protocol': 'srm', 'SpaceToken': 'LHCb-EOS', 'WSUrl': '/srm/v2/server?SFN:' }, { 'Access': 'remote', 'Host': '', 'Path': '/eos/lhcb/grid/other', 'PluginName': 'GFAL2_SRM2', 'Port': '', 'Protocol': '', 'SpaceToken': '', 'WSUrl': '' }] self.assertListEqual(storages['ProtocolOptions'], expectedProtocols)
def setUp( self ): factory = StorageFactory( 'lhcb' ) res = factory.getStorages( storageElementToTest, [plugin] ) self.assert_( res['OK'] ) storageDetails = res['Value'] self.storage = storageDetails['StorageObjects'][0] self.storage.changeDirectory( 'lhcb/test/unit-test/TestStoragePlugIn' ) destDir = self.storage.getCurrentURL( '' )['Value'] res = self.storage.createDirectory( destDir ) self.assert_( res['OK'] ) self.assert_( res['Value']['Successful'].has_key( destDir ) ) self.assert_( res['Value']['Successful'][destDir] ) self.numberOfFiles = 1
def get_base_pfn(base_lfn, target_se): """constructs a pfn from an lfn and a target se""" storage_factory = StorageFactory() result = storage_factory.getStorages(target_se) if not result['OK']: raise Exception("Failed to look up storage element details: %s" % result) # look for srm (don't use anything else) storage_info = result['Value'] for proto_info in storage_info['ProtocolOptions']: if proto_info['Protocol'] != 'srm': continue se_host = proto_info['Host'] se_port = proto_info['Port'] se_wsurl = proto_info['WSUrl'] se_vopath = proto_info['Path'] base_pfn = "srm://%s:%s%s%s%s" %(se_host, se_port, se_wsurl, se_vopath, base_lfn) return base_pfn raise Exception("No srm protocol found for storage element: %s" %target_se)
def test_getStorage( self ): storageDict = {} storageDict['StorageName'] = 'IN2P3-disk' storageDict['PluginName'] = 'SRM2' storageDict['Protocol'] = 'srm' storageDict['Host'] = 'ccsrmtestv2.in2p3.fr' storageDict['Port'] = '8443' storageDict['WSUrl'] = '/srm/managerv2?SFN=' storageDict['Path'] = '/pnfs/in2p3.fr/data' storageDict['SpaceToken'] = 'LHCb_FAKE' factory = StorageFactory( vo = 'lhcb' ) res = factory.getStorage( storageDict ) self.assert_( res['OK'] ) storageStub = res['Value'] parameters = storageStub.getParameters() self.assertEqual( parameters, storageDict ) res = storageStub.getTransportURL( '/lhcb/user' ) self.assert_( res['OK'] ) self.assertEqual( res['Value']['Successful']['/lhcb/user'], 'srm://ccsrmtestv2.in2p3.fr:8443/srm/managerv2?SFN=/pnfs/in2p3.fr/data/lhcb/user' )
def test_getStorages( self ): factory = StorageFactory( vo = 'lhcb' ) storageName = 'IN2P3-disk' protocolList = ['SRM2'] res = factory.getStorages( storageName, protocolList ) self.assert_( res['OK'] ) storageStubs = res['Value']['StorageObjects'] storageStub = storageStubs[0] storageDict = {} storageDict['StorageName'] = 'IN2P3-disk' storageDict['PluginName'] = 'SRM2' storageDict['Protocol'] = 'srm' storageDict['Host'] = 'ccsrm02.in2p3.fr' storageDict['Port'] = '8443' storageDict['WSUrl'] = '/srm/managerv2?SFN=' storageDict['Path'] = '/pnfs/in2p3.fr/data/lhcb' storageDict['SpaceToken'] = '' parameterDict = storageStub.getParameters() self.assertEqual( parameterDict, storageDict ) res = storageStub.getTransportURL( '/lhcb/production/DC06/test.file' ) self.assert_( res['OK'] ) self.assertEqual( res['Value'], 'srm://ccsrm02.in2p3.fr:8443/srm/managerv2?SFN=/pnfs/in2p3.fr/data/lhcb/production/DC06/test.file' ) res = storageStub.removeFile( 'srm://ccsrm02.in2p3.fr:8443/srm/managerv2?SFN=/pnfs/in2p3.fr/data/lhcb/production/DC06/test.file' ) listOfDirs = ['srm://ccsrm02.in2p3.fr:8443/srm/managerv2?SFN=/pnfs/in2p3.fr/data/lhcb/production/DC06/v1-lumi2/00001368/DIGI'] res = storageStub.listDirectory( listOfDirs ) #directoryPath = 'srm://ccsrmtestv2.in2p3.fr:8443/srm/managerv2?SFN=/pnfs/in2p3.fr/data/lhcb/production/DC06/v1-lumi2/1368' #res = storageStub.removeDir(directoryPath) destFile = 'srm://ccsrmtestv2.in2p3.fr:8443/srm/managerv2?SFN=/pnfs/in2p3.fr/data/lhcb/production/DC06/v1-lumi2/1368/dirac_directory.7' res = storageStub.putFile( destFile ) print res res = storageStub.getFile( destFile )
def initialize( self ): self.section = PathFinder.getAgentSection( AGENT_NAME ) self.RequestDB = RequestDBMySQL() self.TransferDB = TransferDB() self.DataLog = DataLoggingClient() self.factory = StorageFactory() self.rm = ReplicaManager() # This sets the Default Proxy to used as that defined under # /Operations/Shifter/DataManager # the shifterProxy option in the Configuration can be used to change this default. self.am_setOption( 'shifterProxy', 'DataManager' ) return S_OK()
def __init__( self, useCertificates = False ): """c'tor :param self: self reference :param bool useCertificates: flag to enable/disable certificates """ Client.__init__( self ) self.log = gLogger.getSubLogger( "DataManagement/FTSClient" ) self.setServer( "DataManagement/FTSManager" ) # getting other clients self.ftsValidator = FTSValidator() self.dataManager = DataManager() self.storageFactory = StorageFactory() url = PathFinder.getServiceURL( "DataManagement/FTSManager" ) if not url: raise RuntimeError( "CS option DataManagement/FTSManager URL is not set!" ) self.ftsManager = RPCClient( url )
class FTSClient(Client): """ .. class:: FTSClient """ def __init__(self, useCertificates=False): """c'tor :param self: self reference :param bool useCertificates: flag to enable/disable certificates """ Client.__init__(self) self.log = gLogger.getSubLogger("DataManagement/FTSClient") self.setServer("DataManagement/FTSManager") # getting other clients self.ftsValidator = FTSValidator() self.replicaManager = ReplicaManager() self.storageFactory = StorageFactory() url = PathFinder.getServiceURL("DataManagement/FTSManager") if not url: raise RuntimeError( "CS option DataManagement/FTSManager URL is not set!") self.ftsManager = RPCClient(url) def getFTSFileList(self, statusList=None, limit=None): """ get list of FTSFiles with status in statusList """ statusList = statusList if statusList else ["Waiting"] limit = limit if limit else 1000 getFTSFileList = self.ftsManager.getFTSFileList(statusList, limit) if not getFTSFileList['OK']: self.log.error("getFTSFileList: %s" % getFTSFileList['Message']) return getFTSFileList getFTSFileList = getFTSFileList['Value'] return S_OK([FTSFile(ftsFile) for ftsFile in getFTSFileList]) def getFTSJobList(self, statusList=None, limit=None): """ get FTSJobs wit statues in :statusList: """ statusList = statusList if statusList else list(FTSJob.INITSTATES + FTSJob.TRANSSTATES) limit = limit if limit else 500 getFTSJobList = self.ftsManager.getFTSJobList(statusList, limit) if not getFTSJobList['OK']: self.log.error("getFTSJobList: %s" % getFTSJobList['Message']) return getFTSJobList getFTSJobList = getFTSJobList['Value'] return S_OK([FTSJob(ftsJobDict) for ftsJobDict in getFTSJobList]) def getFTSFilesForRequest(self, requestID, operationID=None): """ read FTSFiles for a given :requestID: :param int requestID: ReqDB.Request.RequestID :param int operationID: ReqDB.Operation.OperationID """ ftsFiles = self.ftsManager.getFTSFilesForRequest( requestID, operationID) if not ftsFiles['OK']: self.log.error("getFTSFilesForRequest: %s" % ftsFiles['Message']) return ftsFiles return S_OK( [FTSFile(ftsFileDict) for ftsFileDict in ftsFiles['Value']]) def getFTSJobsForRequest(self, requestID, statusList=None): """ get list of FTSJobs with statues in :statusList: given requestID :param int requestID: ReqDB.Request.RequestID :param list statusList: list with FTSJob statuses :return: [ FTSJob, FTSJob, ... ] """ statusList = statusList if statusList else list(FTSJob.INITSTATES + FTSJob.TRANSSTATES) getJobs = self.ftsManager.getFTSJobsForRequest(requestID, statusList) if not getJobs['OK']: self.log.error("getFTSJobsForRequest: %s" % getJobs['Message']) return getJobs return S_OK([FTSJob(ftsJobDict) for ftsJobDict in getJobs['Value']]) def getFTSFile(self, ftsFileID=None): """ get FTSFile :param int ftsFileID: FTSFileID """ getFile = self.ftsManager.getFTSFile(ftsFileID) if not getFile['OK']: self.log.error(getFile['Message']) # # de-serialize if getFile['Value']: ftsFile = FTSFile(getFile['Value']) return S_OK(ftsFile) def putFTSJob(self, ftsJob): """ put FTSJob into FTSDB :param FTSJob ftsJob: FTSJob instance """ isValid = self.ftsValidator.validate(ftsJob) if not isValid['OK']: self.log.error(isValid['Message']) return isValid ftsJobJSON = ftsJob.toJSON() if not ftsJobJSON['OK']: self.log.error(ftsJobJSON['Message']) return ftsJobJSON return self.ftsManager.putFTSJob(ftsJobJSON['Value']) def getFTSJob(self, ftsJobID): """ get FTS job, change its status to 'Assigned' :param int ftsJobID: FTSJobID """ getJob = self.ftsManager.getFTSJob(ftsJobID) if not getJob['OK']: self.log.error(getJob['Message']) return getJob setStatus = self.ftsManager.setFTSJobStatus(ftsJobID, 'Assigned') if not setStatus['OK']: self.log.error(setStatus['Message']) # # de-serialize # if getJob['Value']: # getJob = FTSJob( getJob['Value'] ) return getJob def peekFTSJob(self, ftsJobID): """ just peek FTSJob :param int ftsJobID: FTSJobID """ getJob = self.ftsManager.getFTSJob(ftsJobID) if not getJob['OK']: self.log.error(getJob['Message']) return getJob return getJob def deleteFTSJob(self, ftsJobID): """ delete FTSJob into FTSDB :param int ftsJob: FTSJobID """ deleteJob = self.ftsManager.deleteFTSJob(ftsJobID) if not deleteJob['OK']: self.log.error(deleteJob['Message']) return deleteJob def getFTSJobIDs(self, statusList=None): """ get list of FTSJobIDs for a given status list """ statusList = statusList if statusList else [ "Submitted", "Ready", "Active" ] ftsJobIDs = self.ftsManager.getFTSJobIDs(statusList) if not ftsJobIDs['OK']: self.log.error(ftsJobIDs['Message']) return ftsJobIDs def getFTSFileIDs(self, statusList=None): """ get list of FTSFileIDs for a given status list """ statusList = statusList if statusList else ["Waiting"] ftsFileIDs = self.ftsManager.getFTSFileIDs(statusList) if not ftsFileIDs['OK']: self.log.error(ftsFileIDs['Message']) return ftsFileIDs def getFTSHistory(self): """ get FTS history snapshot """ getFTSHistory = self.ftsManager.getFTSHistory() if not getFTSHistory['OK']: self.log.error(getFTSHistory['Message']) return getFTSHistory getFTSHistory = getFTSHistory['Value'] return S_OK( [FTSHistoryView(ftsHistory) for ftsHistory in getFTSHistory]) def getDBSummary(self): """ get FTDB summary """ dbSummary = self.ftsManager.getDBSummary() if not dbSummary['OK']: self.log.error("getDBSummary: %s" % dbSummary['Message']) return dbSummary def setFTSFilesWaiting(self, operationID, sourceSE, opFileIDList=None): """ update status for waiting FTSFiles from 'Waiting#SourceSE' to 'Waiting' :param int operationID: ReqDB.Operation.OperationID :param str sourceSE: source SE name :param opFileIDList: [ ReqDB.File.FileID, ... ] """ return self.ftsManager.setFTSFilesWaiting(operationID, sourceSE, opFileIDList) def deleteFTSFiles(self, operationID, opFileIDList=None): """ delete FTSFiles for rescheduling :param int operationID: ReqDB.Operation.OperationID :param list opFileIDList: [ ReqDB.File.FileID, ... ] """ return self.ftsManager.deleteFTSFiles(operationID, opFileIDList) def ftsSchedule(self, requestID, operationID, opFileList): """ schedule lfn for FTS job :param int requestID: RequestDB.Request.RequestID :param int operationID: RequestDB.Operation.OperationID :param list opFileList: list of tuples ( File.toJSON()['Value'], sourcesList, targetList ) """ fileIDs = [] for fileJSON, _sourceSEs, _targetSEs in opFileList: fileID = int(fileJSON.get('FileID')) fileIDs.append(fileID) res = self.ftsManager.cleanUpFTSFiles(requestID, fileIDs) if not res['OK']: self.log.error("ftsSchedule: %s" % res['Message']) return S_ERROR("ftsSchedule: %s" % res['Message']) ftsFiles = [] # # this will be returned on success ret = {"Successful": [], "Failed": {}} for fileJSON, sourceSEs, targetSEs in opFileList: lfn = fileJSON.get("LFN", "") size = int(fileJSON.get("Size", 0)) fileID = int(fileJSON.get("FileID", 0)) opID = int(fileJSON.get("OperationID", 0)) gLogger.info( "ftsSchedule: LFN=%s FileID=%s OperationID=%s sources=%s targets=%s" % (lfn, fileID, opID, sourceSEs, targetSEs)) res = self.replicaManager.getActiveReplicas(lfn) if not res['OK']: gLogger.error("ftsSchedule: %s" % res['Message']) ret["Failed"][fileID] = res['Message'] continue replicaDict = res['Value'] if lfn in replicaDict["Failed"] and lfn not in replicaDict[ "Successful"]: ret["Failed"][fileID] = "no active replicas found" continue replicaDict = replicaDict["Successful"][lfn] if lfn in replicaDict[ "Successful"] else {} # # use valid replicas only validReplicasDict = dict([(se, pfn) for se, pfn in replicaDict.items() if se in sourceSEs]) if not validReplicasDict: ret["Failed"][fileID] = "no active replicas found in sources" continue tree = self.ftsManager.getReplicationTree(sourceSEs, targetSEs, size) if not tree['OK']: gLogger.error("ftsSchedule: %s cannot be scheduled: %s" % (lfn, tree['Message'])) ret["Failed"][fileID] = tree['Message'] continue tree = tree['Value'] gLogger.info("LFN=%s tree=%s" % (lfn, tree)) for repDict in tree.values(): gLogger.info( "Strategy=%s Ancestor=%s SourceSE=%s TargetSE=%s" % (repDict["Strategy"], repDict["Ancestor"], repDict["SourceSE"], repDict["TargetSE"])) transferSURLs = self._getTransferURLs(lfn, repDict, sourceSEs, validReplicasDict) if not transferSURLs['OK']: ret["Failed"][fileID] = transferSURLs['Message'] continue sourceSURL, targetSURL, fileStatus = transferSURLs['Value'] if sourceSURL == targetSURL: ret["Failed"][ fileID] = "sourceSURL equals to targetSURL for %s" % lfn continue gLogger.info("sourceURL=%s targetURL=%s FTSFile.Status=%s" % (sourceSURL, targetSURL, fileStatus)) ftsFile = FTSFile() for key in ("LFN", "FileID", "OperationID", "Checksum", "ChecksumType", "Size"): setattr(ftsFile, key, fileJSON.get(key)) ftsFile.RequestID = requestID ftsFile.OperationID = operationID ftsFile.SourceSURL = sourceSURL ftsFile.TargetSURL = targetSURL ftsFile.SourceSE = repDict["SourceSE"] ftsFile.TargetSE = repDict["TargetSE"] ftsFile.Status = fileStatus ftsFiles.append(ftsFile) if not ftsFiles: self.log.error("ftsSchedule: no FTSFiles to put") return S_ERROR("ftsSchedule: no FTSFiles to put") res = self.ftsManager.putFTSFileList(ftsFiles) if not res['OK']: self.log.error("ftsSchedule: %s" % res['Message']) return S_ERROR("ftsSchedule: %s" % res['Message']) for fileJSON, _sources, _targets in opFileList: lfn = fileJSON.get("LFN", "") fileID = fileJSON.get("FileID", 0) if fileID not in ret["Failed"]: ret["Successful"].append(int(fileID)) # # if we land here some files have been properly scheduled return S_OK(ret) ################################################################################################################ # Some utilities function def _getSurlForLFN(self, targetSE, lfn): """ Get the targetSURL for the storage and LFN supplied. :param self: self reference :param str targetSE: target SE :param str lfn: LFN """ res = self.storageFactory.getStorages(targetSE, protocolList=["SRM2"]) if not res['OK']: errStr = "_getSurlForLFN: Failed to create SRM2 storage for %s: %s" % ( targetSE, res['Message']) gLogger.error(errStr) return S_ERROR(errStr) storageObjects = res['Value']["StorageObjects"] for storageObject in storageObjects: res = storageObject.getCurrentURL(lfn) if res['OK']: return res gLogger.error("_getSurlForLFN: Failed to get SRM compliant storage.", targetSE) return S_ERROR("_getSurlForLFN: Failed to get SRM compliant storage.") def _getSurlForPFN(self, sourceSE, pfn): """Creates the targetSURL for the storage and PFN supplied. :param self: self reference :param str sourceSE: source storage element :param str pfn: physical file name """ res = self.replicaManager.getPfnForProtocol([pfn], sourceSE) if not res['OK']: return res if pfn in res['Value']["Failed"]: return S_ERROR(res['Value']["Failed"][pfn]) return S_OK(res['Value']["Successful"][pfn]) def _getTransferURLs(self, lfn, repDict, replicas, replicaDict): """ prepare TURLs for given LFN and replication tree :param self: self reference :param str lfn: LFN :param dict repDict: replication dictionary :param dict replicas: LFN replicas """ hopSourceSE = repDict["SourceSE"] hopTargetSE = repDict["TargetSE"] hopAncestor = repDict["Ancestor"] # # get targetSURL res = self._getSurlForLFN(hopTargetSE, lfn) if not res['OK']: self.log.error("_getTransferURLs: %s" % res['Message']) return res targetSURL = res['Value'] status = "Waiting" # # get the sourceSURL if hopAncestor: status = "Waiting#%s" % (hopAncestor) res = self._getSurlForLFN(hopSourceSE, lfn) if not res['OK']: self.log.error("_getTransferURLs: %s" % res['Message']) return res sourceSURL = res['Value'] else: res = self._getSurlForPFN(hopSourceSE, replicaDict[hopSourceSE]) sourceSURL = res['Value'] if res['OK'] else replicaDict[hopSourceSE] return S_OK((sourceSURL, targetSURL, status))
class FTSClient( Client ): """ .. class:: FTSClient """ def __init__( self, useCertificates = False ): """c'tor :param self: self reference :param bool useCertificates: flag to enable/disable certificates """ Client.__init__( self ) self.log = gLogger.getSubLogger( "DataManagement/FTSClient" ) self.setServer( "DataManagement/FTSManager" ) # getting other clients self.ftsValidator = FTSValidator() self.dataManager = DataManager() self.storageFactory = StorageFactory() url = PathFinder.getServiceURL( "DataManagement/FTSManager" ) if not url: raise RuntimeError( "CS option DataManagement/FTSManager URL is not set!" ) self.ftsManager = RPCClient( url ) def getFTSFileList( self, statusList = None, limit = None ): """ get list of FTSFiles with status in statusList """ statusList = statusList if statusList else [ "Waiting" ] limit = limit if limit else 1000 getFTSFileList = self.ftsManager.getFTSFileList( statusList, limit ) if not getFTSFileList['OK']: self.log.error( "getFTSFileList: %s" % getFTSFileList['Message'] ) return getFTSFileList getFTSFileList = getFTSFileList['Value'] return S_OK( [ FTSFile( ftsFile ) for ftsFile in getFTSFileList ] ) def getFTSJobList( self, statusList = None, limit = None ): """ get FTSJobs wit statues in :statusList: """ statusList = statusList if statusList else list( FTSJob.INITSTATES + FTSJob.TRANSSTATES ) limit = limit if limit else 500 getFTSJobList = self.ftsManager.getFTSJobList( statusList, limit ) if not getFTSJobList['OK']: self.log.error( "getFTSJobList: %s" % getFTSJobList['Message'] ) return getFTSJobList getFTSJobList = getFTSJobList['Value'] return S_OK( [ FTSJob( ftsJobDict ) for ftsJobDict in getFTSJobList ] ) def getFTSFilesForRequest( self, requestID, statusList = None ): """ read FTSFiles for a given :requestID: :param int requestID: ReqDB.Request.RequestID :param list statusList: List of statuses (default: Waiting) """ ftsFiles = self.ftsManager.getFTSFilesForRequest( requestID, statusList ) if not ftsFiles['OK']: self.log.error( "getFTSFilesForRequest: %s" % ftsFiles['Message'] ) return ftsFiles return S_OK( [ FTSFile( ftsFileDict ) for ftsFileDict in ftsFiles['Value'] ] ) def getAllFTSFilesForRequest( self, requestID ): """ read FTSFiles for a given :requestID: :param int requestID: ReqDB.Request.RequestID """ ftsFiles = self.ftsManager.getAllFTSFilesForRequest( requestID ) if not ftsFiles['OK']: self.log.error( "getFTSFilesForRequest: %s" % ftsFiles['Message'] ) return ftsFiles return S_OK( [ FTSFile( ftsFileDict ) for ftsFileDict in ftsFiles['Value'] ] ) def getFTSJobsForRequest( self, requestID, statusList = None ): """ get list of FTSJobs with statues in :statusList: given requestID :param int requestID: ReqDB.Request.RequestID :param list statusList: list with FTSJob statuses :return: [ FTSJob, FTSJob, ... ] """ statusList = statusList if statusList else list( FTSJob.INITSTATES + FTSJob.TRANSSTATES ) getJobs = self.ftsManager.getFTSJobsForRequest( requestID, statusList ) if not getJobs['OK']: self.log.error( "getFTSJobsForRequest: %s" % getJobs['Message'] ) return getJobs return S_OK( [ FTSJob( ftsJobDict ) for ftsJobDict in getJobs['Value'] ] ) def getFTSFile( self, ftsFileID = None ): """ get FTSFile :param int ftsFileID: FTSFileID """ getFile = self.ftsManager.getFTSFile( ftsFileID ) if not getFile['OK']: self.log.error( getFile['Message'] ) # # de-serialize if getFile['Value']: ftsFile = FTSFile( getFile['Value'] ) return S_OK( ftsFile ) def putFTSJob( self, ftsJob ): """ put FTSJob into FTSDB :param FTSJob ftsJob: FTSJob instance """ isValid = self.ftsValidator.validate( ftsJob ) if not isValid['OK']: self.log.error( isValid['Message'] ) return isValid ftsJobJSON = ftsJob.toJSON() if not ftsJobJSON['OK']: self.log.error( ftsJobJSON['Message'] ) return ftsJobJSON return self.ftsManager.putFTSJob( ftsJobJSON['Value'] ) def getFTSJob( self, ftsJobID ): """ get FTS job, change its status to 'Assigned' :param int ftsJobID: FTSJobID """ getJob = self.ftsManager.getFTSJob( ftsJobID ) if not getJob['OK']: self.log.error( getJob['Message'] ) return getJob setStatus = self.ftsManager.setFTSJobStatus( ftsJobID, 'Assigned' ) if not setStatus['OK']: self.log.error( setStatus['Message'] ) # # de-serialize # if getJob['Value']: # getJob = FTSJob( getJob['Value'] ) return getJob def peekFTSJob( self, ftsJobID ): """ just peek FTSJob :param int ftsJobID: FTSJobID """ getJob = self.ftsManager.getFTSJob( ftsJobID ) if not getJob['OK']: self.log.error( getJob['Message'] ) return getJob return getJob def deleteFTSJob( self, ftsJobID ): """ delete FTSJob into FTSDB :param int ftsJob: FTSJobID """ deleteJob = self.ftsManager.deleteFTSJob( ftsJobID ) if not deleteJob['OK']: self.log.error( deleteJob['Message'] ) return deleteJob def getFTSJobIDs( self, statusList = None ): """ get list of FTSJobIDs for a given status list """ statusList = statusList if statusList else [ "Submitted", "Ready", "Active" ] ftsJobIDs = self.ftsManager.getFTSJobIDs( statusList ) if not ftsJobIDs['OK']: self.log.error( ftsJobIDs['Message'] ) return ftsJobIDs def getFTSFileIDs( self, statusList = None ): """ get list of FTSFileIDs for a given status list """ statusList = statusList if statusList else [ "Waiting" ] ftsFileIDs = self.ftsManager.getFTSFileIDs( statusList ) if not ftsFileIDs['OK']: self.log.error( ftsFileIDs['Message'] ) return ftsFileIDs def getFTSHistory( self ): """ get FTS history snapshot """ getFTSHistory = self.ftsManager.getFTSHistory() if not getFTSHistory['OK']: self.log.error( getFTSHistory['Message'] ) return getFTSHistory getFTSHistory = getFTSHistory['Value'] return S_OK( [ FTSHistoryView( ftsHistory ) for ftsHistory in getFTSHistory ] ) def getDBSummary( self ): """ get FTDB summary """ dbSummary = self.ftsManager.getDBSummary() if not dbSummary['OK']: self.log.error( "getDBSummary: %s" % dbSummary['Message'] ) return dbSummary def setFTSFilesWaiting( self, operationID, sourceSE, opFileIDList = None ): """ update status for waiting FTSFiles from 'Waiting#SourceSE' to 'Waiting' :param int operationID: ReqDB.Operation.OperationID :param str sourceSE: source SE name :param opFileIDList: [ ReqDB.File.FileID, ... ] """ return self.ftsManager.setFTSFilesWaiting( operationID, sourceSE, opFileIDList ) def deleteFTSFiles( self, operationID, opFileIDList = None ): """ delete FTSFiles for rescheduling :param int operationID: ReqDB.Operation.OperationID :param list opFileIDList: [ ReqDB.File.FileID, ... ] """ return self.ftsManager.deleteFTSFiles( operationID, opFileIDList ) def ftsSchedule( self, requestID, operationID, opFileList ): """ schedule lfn for FTS job :param int requestID: RequestDB.Request.RequestID :param int operationID: RequestDB.Operation.OperationID :param list opFileList: list of tuples ( File.toJSON()['Value'], sourcesList, targetList ) """ fileIDs = [int( fileJSON.get( 'FileID', 0 ) ) for fileJSON, _sourceSEs, _targetSEs in opFileList ] res = self.ftsManager.cleanUpFTSFiles( requestID, fileIDs ) if not res['OK']: self.log.error( "ftsSchedule: %s" % res['Message'] ) return S_ERROR( "ftsSchedule: %s" % res['Message'] ) ftsFiles = [] # # this will be returned on success result = { "Successful": [], "Failed": {} } for fileJSON, sourceSEs, targetSEs in opFileList: lfn = fileJSON.get( "LFN", "" ) size = int( fileJSON.get( "Size", 0 ) ) fileID = int( fileJSON.get( "FileID", 0 ) ) opID = int( fileJSON.get( "OperationID", 0 ) ) self.log.verbose( "ftsSchedule: LFN=%s FileID=%s OperationID=%s sources=%s targets=%s" % ( lfn, fileID, opID, sourceSEs, targetSEs ) ) res = self.dataManager.getActiveReplicas( lfn ) if not res['OK']: self.log.error( "ftsSchedule: %s" % res['Message'] ) result["Failed"][fileID] = res['Message'] continue replicaDict = res['Value'] if lfn in replicaDict["Failed"] and lfn not in replicaDict["Successful"]: result["Failed"][fileID] = "no active replicas found" continue replicaDict = replicaDict["Successful"].get( lfn, {} ) # # use valid replicas only validReplicasDict = dict( [ ( se, pfn ) for se, pfn in replicaDict.items() if se in sourceSEs ] ) if not validReplicasDict: self.log.warn( "No active replicas found in sources" ) result["Failed"][fileID] = "no active replicas found in sources" continue tree = self.ftsManager.getReplicationTree( sourceSEs, targetSEs, size ) if not tree['OK']: self.log.error( "ftsSchedule: %s cannot be scheduled: %s" % ( lfn, tree['Message'] ) ) result["Failed"][fileID] = tree['Message'] continue tree = tree['Value'] self.log.verbose( "LFN=%s tree=%s" % ( lfn, tree ) ) for repDict in tree.values(): self.log.verbose( "Strategy=%s Ancestor=%s SourceSE=%s TargetSE=%s" % ( repDict["Strategy"], repDict["Ancestor"], repDict["SourceSE"], repDict["TargetSE"] ) ) transferSURLs = self._getTransferURLs( lfn, repDict, sourceSEs, validReplicasDict ) if not transferSURLs['OK']: result["Failed"][fileID] = transferSURLs['Message'] continue sourceSURL, targetSURL, fileStatus = transferSURLs['Value'] if sourceSURL == targetSURL: result["Failed"][fileID] = "sourceSURL equals to targetSURL for %s" % lfn continue self.log.verbose( "sourceURL=%s targetURL=%s FTSFile.Status=%s" % ( sourceSURL, targetSURL, fileStatus ) ) ftsFile = FTSFile() for key in ( "LFN", "FileID", "OperationID", "Checksum", "ChecksumType", "Size" ): if fileJSON.get( key ): setattr( ftsFile, key, fileJSON.get( key ) ) ftsFile.RequestID = requestID ftsFile.OperationID = operationID ftsFile.SourceSURL = sourceSURL ftsFile.TargetSURL = targetSURL ftsFile.SourceSE = repDict["SourceSE"] ftsFile.TargetSE = repDict["TargetSE"] ftsFile.Status = fileStatus ftsFiles.append( ftsFile ) if not ftsFiles: self.log.info( "ftsSchedule: no FTSFiles to put for request %d" % requestID ) return S_OK( result ) ftsFilesJSONList = [ftsFile.toJSON()['Value'] for ftsFile in ftsFiles] res = self.ftsManager.putFTSFileList( ftsFilesJSONList ) if not res['OK']: self.log.error( "ftsSchedule: %s" % res['Message'] ) return S_ERROR( "ftsSchedule: %s" % res['Message'] ) result['Successful'] += [ fileID for fileID in fileIDs if fileID not in result['Failed']] # # if we land here some files have been properly scheduled return S_OK( result ) ################################################################################################################ # Some utilities function def _getSurlForLFN( self, targetSE, lfn ): """ Get the targetSURL for the storage and LFN supplied. :param self: self reference :param str targetSE: target SE :param str lfn: LFN """ res = self.storageFactory.getStorages( targetSE, protocolList = ["SRM2"] ) if not res['OK']: errStr = "_getSurlForLFN: Failed to create SRM2 storage for %s: %s" % ( targetSE, res['Message'] ) self.log.error( errStr ) return S_ERROR( errStr ) storageObjects = res['Value']["StorageObjects"] for storageObject in storageObjects: res = storageObject.getCurrentURL( lfn ) if res['OK']: return res self.log.error( "_getSurlForLFN: Failed to get SRM compliant storage.", targetSE ) return S_ERROR( "_getSurlForLFN: Failed to get SRM compliant storage." ) def _getTransferURLs( self, lfn, repDict, replicas, replicaDict ): """ prepare TURLs for given LFN and replication tree :param self: self reference :param str lfn: LFN :param dict repDict: replication dictionary :param dict replicas: LFN replicas """ hopSourceSE = repDict["SourceSE"] hopTargetSE = repDict["TargetSE"] hopAncestor = repDict["Ancestor"] # # get targetSURL res = self._getSurlForLFN( hopTargetSE, lfn ) if not res['OK']: self.log.error( "_getTransferURLs: %s" % res['Message'] ) return res targetSURL = res['Value'] status = "Waiting" # # get the sourceSURL if hopAncestor: status = "Waiting#%s" % ( hopAncestor ) res = self._getSurlForLFN( hopSourceSE, lfn ) sourceSURL = res.get( 'Value', replicaDict.get( hopSourceSE, None ) ) if not sourceSURL: self.log.error( "_getTransferURLs: %s" % res['Message'] ) return res return S_OK( ( sourceSURL, targetSURL, status ) )
def __init__(self, name, protocols=None, vo=None): """ c'tor :param str name: SE name :param list protocols: requested protocols """ self.vo = vo if not vo: result = getVOfromProxyGroup() if not result['OK']: return result self.vo = result['Value'] self.opHelper = Operations(vo=self.vo) self.resources = Resources(vo=self.vo) proxiedProtocols = gConfig.getValue( '/LocalSite/StorageElements/ProxyProtocols', "").split(',') result = self.resources.getAccessProtocols(name) if result['OK']: ap = result['Value'][0] useProxy = (self.resources.getAccessProtocolValue( ap, "Protocol", "UnknownProtocol") in proxiedProtocols) #print "Proxy", name, proxiedProtocols, \ #gConfig.getValue( "/Resources/StorageElements/%s/AccessProtocol.1/Protocol" % name, "xxx" ) if not useProxy: useProxy = gConfig.getValue( '/LocalSite/StorageElements/%s/UseProxy' % name, False) if not useProxy: useProxy = self.opHelper.getValue( '/Services/StorageElements/%s/UseProxy' % name, False) self.valid = True if protocols == None: res = StorageFactory(useProxy).getStorages(name, protocolList=[]) else: res = StorageFactory(useProxy).getStorages(name, protocolList=protocols) if not res['OK']: self.valid = False self.name = name self.errorReason = res['Message'] else: factoryDict = res['Value'] self.name = factoryDict['StorageName'] self.options = factoryDict['StorageOptions'] self.localProtocols = factoryDict['LocalProtocols'] self.remoteProtocols = factoryDict['RemoteProtocols'] self.storages = factoryDict['StorageObjects'] self.protocolOptions = factoryDict['ProtocolOptions'] self.turlProtocols = factoryDict['TurlProtocols'] self.log = gLogger.getSubLogger("SE[%s]" % self.name) self.readMethods = [ 'getFile', 'getAccessUrl', 'getTransportURL', 'prestageFile', 'prestageFileStatus', 'getDirectory' ] self.writeMethods = [ 'retransferOnlineFile', 'putFile', 'replicateFile', 'pinFile', 'releaseFile', 'createDirectory', 'putDirectory' ] self.removeMethods = ['removeFile', 'removeDirectory'] self.checkMethods = [ 'exists', 'getDirectoryMetadata', 'getDirectorySize', 'getFileSize', 'getFileMetadata', 'listDirectory', 'isDirectory', 'isFile', ] self.okMethods = [ 'getLocalProtocols', 'getPfnForProtocol', 'getPfnForLfn', 'getPfnPath', 'getProtocols', 'getRemoteProtocols', 'getStorageElementName', 'getStorageElementOption', 'getStorageParameters', 'isLocalSE' ] self.__resourceStatus = ResourceStatus()
class ReplicationScheduler( AgentModule ): def initialize( self ): self.section = PathFinder.getAgentSection( AGENT_NAME ) self.RequestDB = RequestDBMySQL() self.TransferDB = TransferDB() self.DataLog = DataLoggingClient() self.factory = StorageFactory() self.rm = ReplicaManager() # This sets the Default Proxy to used as that defined under # /Operations/Shifter/DataManager # the shifterProxy option in the Configuration can be used to change this default. self.am_setOption( 'shifterProxy', 'DataManager' ) return S_OK() def execute( self ): """ The main agent execution method """ # This allows dynamic changing of the throughput timescale self.throughputTimescale = self.am_getOption( 'ThroughputTimescale', 3600 ) self.throughputTimescale = 60 * 60 * 1 #print 'ThroughputTimescale:',self.throughputTimescale ###################################################################################### # # Obtain information on the current state of the channel queues # res = self.TransferDB.getChannelQueues() if not res['OK']: errStr = "ReplicationScheduler._execute: Failed to get channel queues from TransferDB." gLogger.error( errStr, res['Message'] ) return S_OK() if not res['Value']: gLogger.info( "ReplicationScheduler._execute: No active channels found for replication." ) return S_OK() channels = res['Value'] res = self.TransferDB.getChannelObservedThroughput( self.throughputTimescale ) if not res['OK']: errStr = "ReplicationScheduler._execute: Failed to get observed throughput from TransferDB." gLogger.error( errStr, res['Message'] ) return S_OK() if not res['Value']: gLogger.info( "ReplicationScheduler._execute: No active channels found for replication." ) return S_OK() bandwidths = res['Value'] self.strategyHandler = StrategyHandler( bandwidths, channels, self.section ) processedRequests = [] requestsPresent = True while requestsPresent: ###################################################################################### # # The first step is to obtain a transfer request from the RequestDB which should be scheduled. # gLogger.info( "ReplicationScheduler._execute: Contacting RequestDB for suitable requests." ) res = self.RequestDB.getRequest( 'transfer' ) if not res['OK']: gLogger.error( "ReplicationScheduler._execute: Failed to get a request list from RequestDB.", res['Message'] ) continue if not res['Value']: gLogger.info( "ReplicationScheduler._execute: No requests found in RequestDB." ) requestsPresent = False return S_OK() requestString = res['Value']['RequestString'] requestName = res['Value']['RequestName'] gLogger.info( "ReplicationScheduler._execute: Obtained Request %s from RequestDB." % ( requestName ) ) ###################################################################################### # # The request must then be parsed to obtain the sub-requests, their attributes and files. # logStr = 'ReplicationScheduler._execute: Parsing Request %s.' % ( requestName ) gLogger.info( logStr ) oRequest = RequestContainer( requestString ) res = oRequest.getAttribute( 'RequestID' ) if not res['OK']: gLogger.error( 'ReplicationScheduler._execute: Failed to get requestID.', res['Message'] ) return S_ERROR( 'ReplicationScheduler._execute: Failed to get number of sub-requests.' ) requestID = res['Value'] if requestID in processedRequests: # Break the loop once we have iterated once over all requests res = self.RequestDB.updateRequest( requestName, requestString ) if not res['OK']: gLogger.error( "Failed to update request", "%s %s" % ( requestName, res['Message'] ) ) return S_OK() processedRequests.append( requestID ) res = oRequest.getNumSubRequests( 'transfer' ) if not res['OK']: gLogger.error( 'ReplicationScheduler._execute: Failed to get number of sub-requests.', res['Message'] ) return S_ERROR( 'ReplicationScheduler._execute: Failed to get number of sub-requests.' ) numberRequests = res['Value'] gLogger.info( "ReplicationScheduler._execute: '%s' found with %s sub-requests." % ( requestName, numberRequests ) ) ###################################################################################### # # The important request attributes are the source and target SEs. # for ind in range( numberRequests ): gLogger.info( "ReplicationScheduler._execute: Treating sub-request %s from '%s'." % ( ind, requestName ) ) attributes = oRequest.getSubRequestAttributes( ind, 'transfer' )['Value'] if attributes['Status'] != 'Waiting': # If the sub-request is already in terminal state gLogger.info( "ReplicationScheduler._execute: Sub-request %s is status '%s' and not to be executed." % ( ind, attributes['Status'] ) ) continue sourceSE = attributes['SourceSE'] targetSE = attributes['TargetSE'] """ This section should go in the transfer request class """ if type( targetSE ) in types.StringTypes: if re.search( ',', targetSE ): targetSEs = targetSE.split( ',' ) else: targetSEs = [targetSE] """----------------------------------------------------- """ operation = attributes['Operation'] reqRepStrategy = None if operation in self.strategyHandler.getSupportedStrategies(): reqRepStrategy = operation ###################################################################################### # # Then obtain the file attribute of interest are the LFN and FileID # res = oRequest.getSubRequestFiles( ind, 'transfer' ) if not res['OK']: gLogger.error( 'ReplicationScheduler._execute: Failed to obtain sub-request files.' , res['Message'] ) continue files = res['Value'] gLogger.info( "ReplicationScheduler._execute: Sub-request %s found with %s files." % ( ind, len( files ) ) ) filesDict = {} for file in files: lfn = file['LFN'] if file['Status'] != 'Waiting': gLogger.debug( "ReplicationScheduler._execute: %s will not be scheduled because it is %s." % ( lfn, file['Status'] ) ) else: fileID = file['FileID'] filesDict[lfn] = fileID if not filesDict: gLogger.info( "ReplicationScheduler._execute: No Waiting files found for request" ) continue notSched = len( files ) - len( filesDict ) if notSched: gLogger.info( "ReplicationScheduler._execute: %d files found not Waiting" % notSched ) ###################################################################################### # # Now obtain replica information for the files associated to the sub-request. # lfns = filesDict.keys() gLogger.info( "ReplicationScheduler._execute: Obtaining replica information for %d sub-request files." % len( lfns ) ) res = self.rm.getCatalogReplicas( lfns ) if not res['OK']: gLogger.error( "ReplicationScheduler._execute: Failed to get replica information.", res['Message'] ) continue for lfn, failure in res['Value']['Failed'].items(): gLogger.error( "ReplicationScheduler._execute: Failed to get replicas.", '%s: %s' % ( lfn, failure ) ) replicas = res['Value']['Successful'] if not replicas.keys(): gLogger.error( "ReplicationScheduler._execute: Failed to get replica information for all files." ) continue ###################################################################################### # # Now obtain the file sizes for the files associated to the sub-request. # lfns = replicas.keys() gLogger.info( "ReplicationScheduler._execute: Obtaining file sizes for %d sub-request files." % len( lfns ) ) res = self.rm.getCatalogFileMetadata( lfns ) if not res['OK']: gLogger.error( "ReplicationScheduler._execute: Failed to get file size information.", res['Message'] ) continue for lfn, failure in res['Value']['Failed'].items(): gLogger.error( 'ReplicationScheduler._execute: Failed to get file size.', '%s: %s' % ( lfn, failure ) ) metadata = res['Value']['Successful'] if not metadata.keys(): gLogger.error( "ReplicationScheduler._execute: Failed to get metadata for all files." ) continue ###################################################################################### # # For each LFN determine the replication tree # for lfn in sortList( metadata.keys() ): fileSize = metadata[lfn]['Size'] lfnReps = replicas[lfn] fileID = filesDict[lfn] targets = [] for targetSE in targetSEs: if targetSE in lfnReps.keys(): gLogger.debug( "ReplicationScheduler.execute: %s already present at %s." % ( lfn, targetSE ) ) else: targets.append( targetSE ) if not targets: gLogger.info( "ReplicationScheduler.execute: %s present at all targets." % lfn ) oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn, 'Status', 'Done' ) continue if not lfnReps: gLogger.error( "ReplicationScheduler.execute: The file has no replicas.", lfn ) continue res = self.strategyHandler.determineReplicationTree( sourceSE, targets, lfnReps, fileSize, strategy = reqRepStrategy ) if not res['OK']: gLogger.error( "ReplicationScheduler.execute: Failed to determine replication tree.", res['Message'] ) continue tree = res['Value'] ###################################################################################### # # For each item in the replication tree obtain the source and target SURLS # for channelID, dict in tree.items(): gLogger.info( "ReplicationScheduler.execute: processing for channel %d %s" % ( channelID, str( dict ) ) ) hopSourceSE = dict['SourceSE'] hopDestSE = dict['DestSE'] hopAncestor = dict['Ancestor'] # Get the sourceSURL if hopAncestor: status = 'Waiting%s' % ( hopAncestor ) res = self.obtainLFNSURL( hopSourceSE, lfn ) if not res['OK']: errStr = res['Message'] gLogger.error( errStr ) return S_ERROR( errStr ) sourceSURL = res['Value'] else: status = 'Waiting' res = self.resolvePFNSURL( hopSourceSE, lfnReps[hopSourceSE] ) if not res['OK']: sourceSURL = lfnReps[hopSourceSE] else: sourceSURL = res['Value'] # Get the targetSURL res = self.obtainLFNSURL( hopDestSE, lfn ) if not res['OK']: errStr = res['Message'] gLogger.error( errStr ) return S_ERROR( errStr ) targetSURL = res['Value'] ###################################################################################### # # For each item in the replication tree add the file to the channel # res = self.TransferDB.addFileToChannel( channelID, fileID, hopSourceSE, sourceSURL, hopDestSE, targetSURL, fileSize, fileStatus = status ) if not res['OK']: errStr = res['Message'] gLogger.error( "ReplicationScheduler._execute: Failed to add File to Channel." , "%s %s" % ( fileID, channelID ) ) return S_ERROR( errStr ) res = self.TransferDB.addFileRegistration( channelID, fileID, lfn, targetSURL, hopDestSE ) if not res['OK']: errStr = res['Message'] gLogger.error( "ReplicationScheduler._execute: Failed to add File registration." , "%s %s" % ( fileID, channelID ) ) result = self.TransferDB.removeFileFromChannel( channelID, fileID ) if not result['OK']: errStr += result['Message'] gLogger.error( "ReplicationScheduler._execute: Failed to remove File." , "%s %s" % ( fileID, channelID ) ) return S_ERROR( errStr ) oRequest.setSubRequestFileAttributeValue( ind, 'transfer', lfn, 'Status', 'Scheduled' ) res = self.TransferDB.addReplicationTree( fileID, tree ) if oRequest.isSubRequestEmpty( ind, 'transfer' )['Value']: oRequest.setSubRequestStatus( ind, 'transfer', 'Scheduled' ) ################################################ # Generate the new request string after operation requestString = oRequest.toXML()['Value'] res = self.RequestDB.updateRequest( requestName, requestString ) if not res['OK']: gLogger.error( "ReplicationScheduler._execute: Failed to update request", "%s %s" % ( requestName, res['Message'] ) ) def obtainLFNSURL( self, targetSE, lfn ): """ Creates the targetSURL for the storage and LFN supplied """ res = self.factory.getStorages( targetSE, protocolList = ['SRM2'] ) if not res['OK']: errStr = 'ReplicationScheduler._execute: Failed to create SRM2 storage for %s: %s. ' % ( targetSE, res['Message'] ) gLogger.error( errStr ) return S_ERROR( errStr ) storageObjects = res['Value']['StorageObjects'] for storageObject in storageObjects: res = storageObject.getCurrentURL( lfn ) if res['OK']: return res gLogger.error( 'ReplicationScheduler._execute: Failed to get SRM compliant storage.' , targetSE ) return S_ERROR( 'ReplicationScheduler._execute: Failed to get SRM compliant storage.' ) def resolvePFNSURL( self, sourceSE, pfn ): """ Creates the targetSURL for the storage and LFN supplied """ res = self.rm.getPfnForProtocol( [pfn], sourceSE ) if not res['OK']: return res if pfn in res['Value']['Failed'].keys(): return S_ERROR( res['Value']['Failed'][pfn] ) return S_OK( res['Value']['Successful'][pfn] )