def getFTSJob(self, ftsJobID=None, readOnly=False): """ get FTSJob given FTSJobID """ getJob = ["SELECT * FROM `FTSJob` WHERE `FTSJobID` = %s;" % ftsJobID] getJob = self._transaction(getJob) if not getJob["OK"]: self.log.error(getJob["Message"]) return getJob getJob = getJob["Value"] if not getJob: return S_OK() ftsJob = FTSJob(getJob.values()[0][0]) selectFiles = self._transaction([ "SELECT * FROM `FTSFile` WHERE `FTSGUID` = '%s';" % ftsJob.FTSGUID ]) if not selectFiles["OK"]: self.log.error(selectFiles["Message"]) return selectFiles selectFiles = selectFiles["Value"] ftsFiles = [FTSFile(item) for item in selectFiles.values()[0]] for ftsFile in ftsFiles: ftsJob.addFile(ftsFile) if not readOnly: setAssigned = "UPDATE `FTSJob` SET `Status`='Assigned' WHERE `FTSJobID` = %s;" % ftsJobID setAssigned = self._query(setAssigned) if not setAssigned["OK"]: self.log.error(setAssigned["Message"]) return setAssigned return S_OK(ftsJob)
def getFTSJob( self, ftsJobID = None, readOnly = False ): """ get FTSJob given FTSJobID """ getJob = [ "SELECT * FROM `FTSJob` WHERE `FTSJobID` = %s;" % ftsJobID ] getJob = self._transaction( getJob ) if not getJob["OK"]: self.log.error( getJob["Message"] ) return getJob getJob = getJob["Value"] if not getJob: return S_OK() ftsJob = FTSJob( getJob.values()[0][0] ) selectFiles = self._transaction( [ "SELECT * FROM `FTSFile` WHERE `FTSGUID` = '%s';" % ftsJob.FTSGUID ] ) if not selectFiles["OK"]: self.log.error( selectFiles["Message"] ) return selectFiles selectFiles = selectFiles["Value"] ftsFiles = [ FTSFile( item ) for item in selectFiles.values()[0] ] for ftsFile in ftsFiles: ftsJob.addFile( ftsFile ) if not readOnly: setAssigned = "UPDATE `FTSJob` SET `Status`='Assigned' WHERE `FTSJobID` = %s;" % ftsJobID setAssigned = self._query( setAssigned ) if not setAssigned["OK"]: self.log.error( setAssigned["Message"] ) return setAssigned return S_OK( ftsJob )
def test03Submit(self): """ submit """ ftsJob = FTSJob() ftsJob.addFile(self.fileA) ftsJob.addFile(self.fileB) submit = ftsJob.submitFTS2() print submit
def test03Submit( self ): """ submit """ ftsJob = FTSJob() ftsJob.addFile( self.fileA ) ftsJob.addFile( self.fileB ) submit = ftsJob.submitFTS2() print submit
def test01Ctor( self ): """ test ctor and (de-)serilisation """ ftsJob = FTSJob() self.assertEqual( isinstance( ftsJob, FTSJob ), True ) json = ftsJob.toJSON() self.assertEqual( json["OK"], True, "JSON serialization error" ) self.assertEqual( type( json["Value"] ), dict, "JSON serialization value error" ) ftsJobJSON = FTSJob( json["Value"] ) self.assertEqual( isinstance( ftsJobJSON, FTSJob ), True, "JSON de-serialization error" ) ftsJob.addFile( self.fileA ) ftsJob.addFile( self.fileB ) self.assertEqual( len( ftsJob ), 2 ) self.assertEqual( ftsJob.Files, 2 ) self.assertEqual( ftsJob.Size, 20 ) json = ftsJob.toJSON() ftsJobJSON = FTSJob( json["Value"] ) self.assertEqual( isinstance( ftsJobJSON, FTSJob ), True, "JSON de-serilization error" ) SQL = ftsJob.toSQL() self.assertEqual( SQL["OK"], True, "SQL serialization error" ) self.assertEqual( SQL["Value"].startswith( "INSERT" ), True, "SQL serialization INSERT error" ) ftsJob.FTSJobID = 123456 SQL = ftsJob.toSQL() self.assertEqual( SQL["OK"], True, "SQL serialization error" ) self.assertEqual( SQL["Value"].startswith( "UPDATE" ), True, "SQL serialization UPDATE error" )
def export_putFTSJob(self, ftsJobJSON): """ put FTSJob (serialized in JSON into FTSDB """ ftsFiles = [] if "FTSFiles" in ftsJobJSON: ftsFiles = ftsJobJSON.get("FTSFiles", []) del ftsJobJSON["FTSFiles"] try: ftsJob = FTSJob(ftsJobJSON) for ftsFile in ftsFiles: ftsJob.addFile(FTSFile(ftsFile)) except Exception, error: gLogger.exception(error) return S_ERROR(error)
def export_putFTSJob( self, ftsJobJSON ): """ put FTSJob (serialized in JSON into FTSDB """ ftsFiles = [] if "FTSFiles" in ftsJobJSON: ftsFiles = ftsJobJSON.get( "FTSFiles", [] ) del ftsJobJSON["FTSFiles"] try: ftsJob = FTSJob( ftsJobJSON ) for ftsFile in ftsFiles: ftsJob.addFile( FTSFile( ftsFile ) ) except Exception, error: gLogger.exception( error ) return S_ERROR( error )
def test02Files( self ): """ FTSFiles arithmetic """ ftsJob = FTSJob() ftsJob.FTSGUID = str( uuid.uuid4() ) self.assertEqual( len( ftsJob ), 0, "1. len(ftsJob) error" ) self.assertEqual( ftsJob.Files, 0 , "1. Files prop error" ) self.assertEqual( ftsJob.Size, 0, "1. Size prop error" ) ftsJob.addFile( self.fileA ) ftsJob.addFile( self.fileB ) self.assertEqual( self.fileA.FTSGUID, ftsJob.FTSGUID, "FTSGUID mismatch for fileA" ) self.assertEqual( self.fileB.FTSGUID, ftsJob.FTSGUID, "FTSGUID mismatch for fileB" ) self.assertEqual( len( ftsJob ), 2, "2. len(ftsJob) error" ) self.assertEqual( ftsJob.Files, 2, "2. Files prop error" ) self.assertEqual( ftsJob.Size, 20, "2. Size prop error" )
def getFTSJobList( self, statusList = None, limit = None ): """ get FTSJobs wit statues in :statusList: """ statusList = statusList if statusList else list( FTSJob.INITSTATES + FTSJob.TRANSSTATES ) limit = limit if limit else 500 getFTSJobList = self.ftsManager.getFTSJobList( statusList, limit ) if not getFTSJobList['OK']: self.log.error( "Failed getFTSJobList", "%s" % getFTSJobList['Message'] ) return getFTSJobList getFTSJobList = getFTSJobList['Value'] return S_OK( [ FTSJob( ftsJobDict ) for ftsJobDict in getFTSJobList ] )
def test02Files(self): """ FTSFiles arithmetic """ ftsJob = FTSJob() ftsJob.FTSGUID = str(uuid.uuid4()) self.assertEqual(len(ftsJob), 0, "1. len(ftsJob) error") self.assertEqual(ftsJob.Files, 0, "1. Files prop error") self.assertEqual(ftsJob.Size, 0, "1. Size prop error") ftsJob.addFile(self.fileA) ftsJob.addFile(self.fileB) self.assertEqual(self.fileA.FTSGUID, ftsJob.FTSGUID, "FTSGUID mismatch for fileA") self.assertEqual(self.fileB.FTSGUID, ftsJob.FTSGUID, "FTSGUID mismatch for fileB") self.assertEqual(len(ftsJob), 2, "2. len(ftsJob) error") self.assertEqual(ftsJob.Files, 2, "2. Files prop error") self.assertEqual(ftsJob.Size, 20, "2. Size prop error")
def getFTSJob( self, ftsJobID = None ): """ get FTSJob given FTSJobID """ getJob = [ "SELECT * FROM `FTSJob` WHERE `FTSJobID` = %s;" % ftsJobID ] getJob = self._transaction( getJob ) if not getJob['OK']: self.log.error( 'Failed ftsFileSQL', getJob['Message'] ) return getJob getJob = getJob['Value'] if not getJob: return S_OK() ftsJob = FTSJob( getJob.values()[0][0] ) selectFiles = self._transaction( [ "SELECT * FROM `FTSFile` WHERE `FTSGUID` = '%s';" % ftsJob.FTSGUID ] ) if not selectFiles['OK']: self.log.error( 'Failed ftsFileSQL', selectFiles['Message'] ) return selectFiles selectFiles = selectFiles['Value'] ftsFiles = [ FTSFile( item ) for item in selectFiles.values()[0] ] for ftsFile in ftsFiles: ftsJob.addFile( ftsFile ) return S_OK( ftsJob )
def getFTSJobsForRequest( self, requestID, statusList = None ): """ get list of FTSJobs with statues in :statusList: given requestID :param int requestID: ReqDB.Request.RequestID :param statusList: list with FTSJob statuses :type statusList: python:list :return: [ FTSJob, FTSJob, ... ] """ statusList = statusList if statusList else list( FTSJob.INITSTATES + FTSJob.TRANSSTATES ) getJobs = self.ftsManager.getFTSJobsForRequest( requestID, statusList ) if not getJobs['OK']: self.log.error( "Failed getFTSJobsForRequest", "%s" % getJobs['Message'] ) return getJobs return S_OK( [ FTSJob( ftsJobDict ) for ftsJobDict in getJobs['Value'] ] )
def setUp( self ): """ test case set up """ gLogger.setLevel( 'NOTICE' ) self.ftsSites = [ FTSSite( ftsServer = 'https://fts22-t0-export.cern.ch:8443/glite-data-transfer-fts/services/FileTransfer', name = 'CERN.ch' ), FTSSite( ftsServer = 'https://fts.pic.es:8443/glite-data-transfer-fts/services/FileTransfer', name = 'PIC.es' ), FTSSite( ftsServer = 'https://lcgfts.gridpp.rl.ac.uk:8443/glite-data-transfer-fts/services/FileTransfer', name = 'RAL.uk' ), ] self.ses = [ 'CERN-USER', 'RAL-USER' ] self.statuses = [ 'Submitted', 'Finished', 'FinishedDirty', 'Active', 'Ready' ] self.submitted = 0 self.numberOfJobs = 10 self.opIDs = [] self.ftsJobs = [] for i in xrange( self.numberOfJobs ): opID = i % 3 if opID not in self.opIDs: self.opIDs.append( opID ) ftsJob = FTSJob() ftsJob.FTSGUID = str( uuid.uuid4() ) ftsJob.FTSServer = self.ftsSites[0].FTSServer ftsJob.Status = self.statuses[ i % len( self.statuses ) ] ftsJob.OperationID = opID if ftsJob.Status in FTSJob.FINALSTATES: ftsJob.Completeness = 100 if ftsJob.Status == 'Active': ftsJob.Completeness = 90 ftsJob.SourceSE = self.ses[ i % len( self.ses ) ] ftsJob.TargetSE = 'PIC-USER' ftsJob.RequestID = 12345 ftsFile = FTSFile() ftsFile.FileID = i + 1 ftsFile.OperationID = i + 1 ftsFile.LFN = '/a/b/c/%d' % i ftsFile.Size = 1000000 ftsFile.OperationID = opID ftsFile.SourceSE = ftsJob.SourceSE ftsFile.TargetSE = ftsJob.TargetSE ftsFile.SourceSURL = 'foo://source.bar.baz/%s' % ftsFile.LFN ftsFile.TargetSURL = 'foo://target.bar.baz/%s' % ftsFile.LFN ftsFile.Status = 'Waiting' if ftsJob.Status != 'FinishedDirty' else 'Failed' ftsFile.RequestID = 12345 ftsFile.Checksum = 'addler' ftsFile.ChecksumType = 'adler32' ftsFile.FTSGUID = ftsJob.FTSGUID if ftsJob.Status == 'FinishedDirty': ftsJob.FailedFiles = 1 ftsJob.FailedSize = ftsFile.Size ftsJob.addFile( ftsFile ) self.ftsJobs.append( ftsJob ) self.submitted = len( [ i for i in self.ftsJobs if i.Status == 'Submitted' ] ) self.ftsClient = FTSClient() self.ftsClient.replicaManager = mock.Mock() self.ftsClient.replicaManager.getActiveReplicas.return_value = {'OK': True, 'Value': {'Successful': {'/a/b/c/1':{'CERN-USER':'******', 'RAL-USER':'******'}, '/a/b/c/2':{'CERN-USER':'******', 'RAL-USER':'******'}, '/a/b/c/3':{'CERN-USER':'******', 'RAL-USER':'******'} }, 'Failed': {'/a/b/c/4':'/aa/a/b/c/4d', '/a/b/c/5':'/aa/a/b/c/5d'} } }
def __createFTSJob( self, guid = None ): self.__createFTSFiles() ftsJob = FTSJob() ftsJob.RequestID = 0 ftsJob.OperationID = 0 ftsJob.SourceSE = self.sourceSE ftsJob.TargetSE = self.targetSE ftsJob.SourceToken = self.sourceToken ftsJob.TargetToken = self.targetToken ftsJob.FTSServer = self.ftsServer if guid: ftsJob.FTSGUID = guid for ftsFile in self.ftsFiles: ftsFile.Attempt += 1 ftsFile.Error = "" ftsJob.addFile( ftsFile ) self.ftsJob = ftsJob
def setUp( self ): """ test case set up """ # ## set some defaults gConfig.setOptionValue( 'DIRAC/Setup', 'Test' ) gConfig.setOptionValue( '/DIRAC/Setups/Test/DataManagement', 'Test' ) gConfig.setOptionValue( '/Systems/DataManagement/Test/Databases/FTSDB/Host', 'localhost' ) gConfig.setOptionValue( '/Systems/DataManagement/Test/Databases/FTSDB/DBName', 'FTSDB' ) gConfig.setOptionValue( '/Systems/DataManagement/Test/Databases/FTSDB/User', 'Dirac' ) self.ftsSites = [ FTSSite( { "FTSServer": "https://fts22-t0-export.cern.ch:8443/glite-data-transfer-fts/services/FileTransfer", "Name": "CERN.ch" } ), FTSSite( { "FTSServer": "https://fts.pic.es:8443/glite-data-transfer-fts/services/FileTransfer", "Name": "PIC.es" } ), FTSSite( { "FTSServer": "https://lcgfts.gridpp.rl.ac.uk:8443/glite-data-transfer-fts/services/FileTransfer", "Name": "RAL.uk" } ), FTSSite( { "FTSServer": "https://fts.grid.sara.nl:8443/glite-data-transfer-fts/services/FileTransfer", "Name": "SARA.nl" } ), FTSSite( { "FTSServer": "https://fts.cr.cnaf.infn.it:8443/glite-data-transfer-fts/services/FileTransfer", "Name": "CNAF.it" } ), FTSSite( { "FTSServer": "https://fts.grid.sara.nl:8443/glite-data-transfer-fts/services/FileTransfer", "Name": "NIKHEF.nl" } ), FTSSite( { "FTSServer": "https://fts-fzk.gridka.de:8443/glite-data-transfer-fts/services/FileTransfer", "Name": "GRIDKA.de" } ), FTSSite( { "FTSServer": "https://cclcgftsprod.in2p3.fr:8443/glite-data-transfer-fts/services/FileTransfer", "Name": "IN2P3.fr" } ) ] self.ftsFiles = [] for i in range ( 100 ): ftsFile = FTSFile() ftsFile.FileID = i + 1 ftsFile.OperationID = 9999 ftsFile.LFN = "/a/b/c/%d" % i ftsFile.Size = 10 ftsFile.SourceSE = "CERN-USER" ftsFile.TargetSE = "PIC-USER" ftsFile.SourceSURL = "foo://source.bar.baz/%s" % ftsFile.LFN ftsFile.TargetSURL = "foo://target.bar.baz/%s" % ftsFile.LFN ftsFile.Status = "Waiting" self.ftsFiles.append( ftsFile ) ses = [ "CERN-USER", "RAL-USER" ] statuses = [ "Submitted", "Finished", "FinishedDirty", "Active", "Ready" ] self.submitted = 0 self.ftsJobs = [] for i in range( 7200 ): ftsJob = FTSJob() ftsJob.FTSGUID = str( uuid.uuid4() ) ftsJob.FTSServer = self.ftsSites[0].FTSServer ftsJob.Status = statuses[ i % len( statuses ) ] if ftsJob.Status in FTSJob.FINALSTATES: ftsJob.Completeness = 100 if ftsJob.Status == "Active": ftsJob.Completeness = 90 ftsJob.SourceSE = ses[ i % len( ses ) ] ftsJob.TargetSE = "PIC-USER" ftsFile = FTSFile() ftsFile.FileID = i + 1 ftsFile.OperationID = i + 1 ftsFile.LFN = "/a/b/c/%d" % i ftsFile.Size = 1000000 ftsFile.SourceSE = ftsJob.SourceSE ftsFile.TargetSE = ftsJob.TargetSE ftsFile.SourceSURL = "foo://source.bar.baz/%s" % ftsFile.LFN ftsFile.TargetSURL = "foo://target.bar.baz/%s" % ftsFile.LFN ftsFile.Status = "Waiting" if ftsJob.Status != "FinishedDirty" else "Failed" ftsFile.FTSGUID = ftsJob.FTSGUID if ftsJob.Status == "FinishedDirty": ftsJob.FailedFiles = 1 ftsJob.FailedSize = ftsFile.Size ftsJob.addFile( ftsFile ) self.ftsJobs.append( ftsJob ) self.submitted = len( [ i for i in self.ftsJobs if i.Status == "Submitted" ] )
def setUp(self): """ test case set up """ gLogger.setLevel('NOTICE') self.ftsSites = [ FTSSite( ftsServer= 'https://fts22-t0-export.cern.ch:8443/glite-data-transfer-fts/services/FileTransfer', name='CERN.ch'), FTSSite( ftsServer= 'https://fts.pic.es:8443/glite-data-transfer-fts/services/FileTransfer', name='PIC.es'), FTSSite( ftsServer= 'https://lcgfts.gridpp.rl.ac.uk:8443/glite-data-transfer-fts/services/FileTransfer', name='RAL.uk'), ] self.ses = ['CERN-USER', 'RAL-USER'] self.statuses = [ 'Submitted', 'Finished', 'FinishedDirty', 'Active', 'Ready' ] self.submitted = 0 self.numberOfJobs = 10 self.opIDs = [] self.ftsJobs = [] for i in xrange(self.numberOfJobs): opID = i % 3 if opID not in self.opIDs: self.opIDs.append(opID) ftsJob = FTSJob() ftsJob.FTSGUID = str(uuid.uuid4()) ftsJob.FTSServer = self.ftsSites[0].FTSServer ftsJob.Status = self.statuses[i % len(self.statuses)] ftsJob.OperationID = opID if ftsJob.Status in FTSJob.FINALSTATES: ftsJob.Completeness = 100 if ftsJob.Status == 'Active': ftsJob.Completeness = 90 ftsJob.SourceSE = self.ses[i % len(self.ses)] ftsJob.TargetSE = 'PIC-USER' ftsJob.RequestID = 12345 ftsFile = FTSFile() ftsFile.FileID = i + 1 ftsFile.OperationID = i + 1 ftsFile.LFN = '/a/b/c/%d' % i ftsFile.Size = 1000000 ftsFile.OperationID = opID ftsFile.SourceSE = ftsJob.SourceSE ftsFile.TargetSE = ftsJob.TargetSE ftsFile.SourceSURL = 'foo://source.bar.baz/%s' % ftsFile.LFN ftsFile.TargetSURL = 'foo://target.bar.baz/%s' % ftsFile.LFN ftsFile.Status = 'Waiting' if ftsJob.Status != 'FinishedDirty' else 'Failed' ftsFile.RequestID = 12345 ftsFile.Checksum = 'addler' ftsFile.ChecksumType = 'adler32' ftsFile.FTSGUID = ftsJob.FTSGUID if ftsJob.Status == 'FinishedDirty': ftsJob.FailedFiles = 1 ftsJob.FailedSize = ftsFile.Size ftsJob.addFile(ftsFile) self.ftsJobs.append(ftsJob) self.submitted = len( [i for i in self.ftsJobs if i.Status == 'Submitted']) self.ftsClient = FTSClient()
def test01Ctor(self): """ test ctor and (de-)serilisation """ ftsJob = FTSJob() self.assertEqual(isinstance(ftsJob, FTSJob), True) json = ftsJob.toJSON() self.assertEqual(json["OK"], True, "JSON serialization error") self.assertEqual(type(json["Value"]), dict, "JSON serialization value error") ftsJobJSON = FTSJob(json["Value"]) self.assertEqual(isinstance(ftsJobJSON, FTSJob), True, "JSON de-serialization error") ftsJob.addFile(self.fileA) ftsJob.addFile(self.fileB) self.assertEqual(len(ftsJob), 2) self.assertEqual(ftsJob.Files, 2) self.assertEqual(ftsJob.Size, 20) json = ftsJob.toJSON() ftsJobJSON = FTSJob(json["Value"]) self.assertEqual(isinstance(ftsJobJSON, FTSJob), True, "JSON de-serilization error") SQL = ftsJob.toSQL() self.assertEqual(SQL["OK"], True, "SQL serialization error") self.assertEqual(SQL["Value"].startswith("INSERT"), True, "SQL serialization INSERT error") ftsJob.FTSJobID = 123456 SQL = ftsJob.toSQL() self.assertEqual(SQL["OK"], True, "SQL serialization error") self.assertEqual(SQL["Value"].startswith("UPDATE"), True, "SQL serialization UPDATE error")
def setUp(self): """ test case set up """ # ## set some defaults gConfig.setOptionValue('DIRAC/Setup', 'Test') gConfig.setOptionValue('/DIRAC/Setups/Test/DataManagement', 'Test') gConfig.setOptionValue( '/Systems/DataManagement/Test/Databases/FTSDB/Host', 'localhost') gConfig.setOptionValue( '/Systems/DataManagement/Test/Databases/FTSDB/DBName', 'FTSDB') gConfig.setOptionValue( '/Systems/DataManagement/Test/Databases/FTSDB/User', 'Dirac') self.ftsSites = [ FTSSite({ "FTSServer": "https://fts22-t0-export.cern.ch:8443/glite-data-transfer-fts/services/FileTransfer", "Name": "CERN.ch" }), FTSSite({ "FTSServer": "https://fts.pic.es:8443/glite-data-transfer-fts/services/FileTransfer", "Name": "PIC.es" }), FTSSite({ "FTSServer": "https://lcgfts.gridpp.rl.ac.uk:8443/glite-data-transfer-fts/services/FileTransfer", "Name": "RAL.uk" }), FTSSite({ "FTSServer": "https://fts.grid.sara.nl:8443/glite-data-transfer-fts/services/FileTransfer", "Name": "SARA.nl" }), FTSSite({ "FTSServer": "https://fts.cr.cnaf.infn.it:8443/glite-data-transfer-fts/services/FileTransfer", "Name": "CNAF.it" }), FTSSite({ "FTSServer": "https://fts.grid.sara.nl:8443/glite-data-transfer-fts/services/FileTransfer", "Name": "NIKHEF.nl" }), FTSSite({ "FTSServer": "https://fts-fzk.gridka.de:8443/glite-data-transfer-fts/services/FileTransfer", "Name": "GRIDKA.de" }), FTSSite({ "FTSServer": "https://cclcgftsprod.in2p3.fr:8443/glite-data-transfer-fts/services/FileTransfer", "Name": "IN2P3.fr" }) ] self.ftsFiles = [] for i in range(100): ftsFile = FTSFile() ftsFile.FileID = i + 1 ftsFile.OperationID = 9999 ftsFile.LFN = "/a/b/c/%d" % i ftsFile.Size = 10 ftsFile.SourceSE = "CERN-USER" ftsFile.TargetSE = "PIC-USER" ftsFile.SourceSURL = "foo://source.bar.baz/%s" % ftsFile.LFN ftsFile.TargetSURL = "foo://target.bar.baz/%s" % ftsFile.LFN ftsFile.Status = "Waiting" self.ftsFiles.append(ftsFile) ses = ["CERN-USER", "RAL-USER"] statuses = [ "Submitted", "Finished", "FinishedDirty", "Active", "Ready" ] self.submitted = 0 self.ftsJobs = [] for i in range(7200): ftsJob = FTSJob() ftsJob.FTSGUID = str(uuid.uuid4()) ftsJob.FTSServer = self.ftsSites[0].FTSServer ftsJob.Status = statuses[i % len(statuses)] if ftsJob.Status in FTSJob.FINALSTATES: ftsJob.Completeness = 100 if ftsJob.Status == "Active": ftsJob.Completeness = 90 ftsJob.SourceSE = ses[i % len(ses)] ftsJob.TargetSE = "PIC-USER" ftsFile = FTSFile() ftsFile.FileID = i + 1 ftsFile.OperationID = i + 1 ftsFile.LFN = "/a/b/c/%d" % i ftsFile.Size = 1000000 ftsFile.SourceSE = ftsJob.SourceSE ftsFile.TargetSE = ftsJob.TargetSE ftsFile.SourceSURL = "foo://source.bar.baz/%s" % ftsFile.LFN ftsFile.TargetSURL = "foo://target.bar.baz/%s" % ftsFile.LFN ftsFile.Status = "Waiting" if ftsJob.Status != "FinishedDirty" else "Failed" ftsFile.FTSGUID = ftsJob.FTSGUID if ftsJob.Status == "FinishedDirty": ftsJob.FailedFiles = 1 ftsJob.FailedSize = ftsFile.Size ftsJob.addFile(ftsFile) self.ftsJobs.append(ftsJob) self.submitted = len( [i for i in self.ftsJobs if i.Status == "Submitted"])
def __submit( self, request, operation, toSubmit ): """ create and submit new FTSJobs using list of FTSFiles :param Request request: ReqDB.Request instance :param list ftsFiles: list of FTSFile instances :return: [ FTSJob, FTSJob, ...] """ log = self.log.getSubLogger( "req_%s/%s/submit" % ( request.RequestID, request.RequestName ) ) bySourceAndTarget = {} for ftsFile in toSubmit: if ftsFile.SourceSE not in bySourceAndTarget: bySourceAndTarget.setdefault( ftsFile.SourceSE, {} ) if ftsFile.TargetSE not in bySourceAndTarget[ftsFile.SourceSE]: bySourceAndTarget[ftsFile.SourceSE].setdefault( ftsFile.TargetSE, [] ) bySourceAndTarget[ftsFile.SourceSE][ftsFile.TargetSE].append( ftsFile ) ftsJobs = [] for source, targetDict in bySourceAndTarget.iteritems(): for target, ftsFileList in targetDict.iteritems(): log.info( "found %s files to submit from %s to %s" % ( len( ftsFileList ), source, target ) ) route = self.__ftsPlacement.findRoute( source, target ) if not route["OK"]: log.error( route["Message"] ) continue route = route["Value"] routeValid = self.__ftsPlacement.isRouteValid( route ) if not routeValid['OK']: log.error( "Route invalid : %s" % routeValid['Message'] ) continue sourceSE = StorageElement( source ) sourceToken = sourceSE.getStorageParameters( "SRM2" ) if not sourceToken["OK"]: log.error( "unable to get sourceSE parameters:", "(%s) %s" % ( source, sourceToken["Message"] ) ) continue seStatus = sourceSE.getStatus()['Value'] targetSE = StorageElement( target ) targetToken = targetSE.getStorageParameters( "SRM2" ) if not targetToken["OK"]: log.error( "unable to get targetSE parameters:", "(%s) %s" % ( target, targetToken["Message"] ) ) continue # # create FTSJob for fileList in breakListIntoChunks( ftsFileList, self.MAX_FILES_PER_JOB ): ftsJob = FTSJob() ftsJob.RequestID = request.RequestID ftsJob.OperationID = operation.OperationID ftsJob.SourceSE = source ftsJob.TargetSE = target ftsJob.SourceToken = sourceToken["Value"].get( "SpaceToken", "" ) ftsJob.TargetToken = targetToken["Value"].get( "SpaceToken", "" ) ftsJob.FTSServer = route.ftsServer for ftsFile in fileList: ftsFile.Attempt += 1 ftsFile.Error = "" ftsJob.addFile( ftsFile ) submit = ftsJob.submitFTS( self.__ftsVersion, command = self.SUBMIT_COMMAND, pinTime = self.PIN_TIME if seStatus['TapeSE'] else 0 ) if not submit["OK"]: log.error( "unable to submit FTSJob:", submit["Message"] ) continue log.info( "FTSJob '%s'@'%s' has been submitted" % ( ftsJob.FTSGUID, ftsJob.FTSServer ) ) # # update statuses for job files for ftsFile in ftsJob: ftsFile.FTSGUID = ftsJob.FTSGUID ftsFile.Status = "Submitted" ftsFile.Attempt += 1 # # update placement route try: self.updateLock().acquire() self.__ftsPlacement.startTransferOnRoute( route ) finally: self.updateLock().release() ftsJobs.append( ftsJob ) log.info( "%s new FTSJobs have been submitted" % len( ftsJobs ) ) return S_OK( ftsJobs )
def __submit( self, request, operation, toSubmit ): """ create and submit new FTSJobs using list of FTSFiles :param Request request: ReqDB.Request instance :param list ftsFiles: list of FTSFile instances :return: [ FTSJob, FTSJob, ...] """ log = self.log.getSubLogger( "req_%s/%s/submit" % ( request.RequestID, request.RequestName ) ) bySourceAndTarget = {} for ftsFile in toSubmit: if ftsFile.SourceSE not in bySourceAndTarget: bySourceAndTarget.setdefault( ftsFile.SourceSE, {} ) if ftsFile.TargetSE not in bySourceAndTarget[ftsFile.SourceSE]: bySourceAndTarget[ftsFile.SourceSE].setdefault( ftsFile.TargetSE, [] ) bySourceAndTarget[ftsFile.SourceSE][ftsFile.TargetSE].append( ftsFile ) ftsJobs = [] for source, targetDict in bySourceAndTarget.iteritems(): for target, ftsFileList in targetDict.iteritems(): log.info( "found %s files to submit from %s to %s" % ( len( ftsFileList ), source, target ) ) route = self.__ftsPlacement.findRoute( source, target ) if not route["OK"]: log.error( route["Message"] ) continue route = route["Value"] routeValid = self.__ftsPlacement.isRouteValid( route ) if not routeValid['OK']: log.error( "Route invalid : %s" % routeValid['Message'] ) continue sourceSE = StorageElement( source ) sourceToken = sourceSE.getStorageParameters( protocol = 'srm' ) if not sourceToken["OK"]: log.error( "unable to get sourceSE parameters:", "(%s) %s" % ( source, sourceToken["Message"] ) ) continue seStatus = sourceSE.getStatus()['Value'] targetSE = StorageElement( target ) targetToken = targetSE.getStorageParameters( protocol = 'srm' ) if not targetToken["OK"]: log.error( "unable to get targetSE parameters:", "(%s) %s" % ( target, targetToken["Message"] ) ) continue # # create FTSJob for fileList in breakListIntoChunks( ftsFileList, self.MAX_FILES_PER_JOB ): ftsJob = FTSJob() ftsJob.RequestID = request.RequestID ftsJob.OperationID = operation.OperationID ftsJob.SourceSE = source ftsJob.TargetSE = target ftsJob.SourceToken = sourceToken["Value"].get( "SpaceToken", "" ) ftsJob.TargetToken = targetToken["Value"].get( "SpaceToken", "" ) ftsJob.FTSServer = route.ftsServer for ftsFile in fileList: ftsFile.Attempt += 1 ftsFile.Error = "" ftsJob.addFile( ftsFile ) submit = ftsJob.submitFTS( self.__ftsVersion, command = self.SUBMIT_COMMAND, pinTime = self.PIN_TIME if seStatus['TapeSE'] else 0 ) if not submit["OK"]: log.error( "unable to submit FTSJob:", submit["Message"] ) continue log.info( "FTSJob '%s'@'%s' has been submitted" % ( ftsJob.FTSGUID, ftsJob.FTSServer ) ) # # update statuses for job files for ftsFile in ftsJob: ftsFile.FTSGUID = ftsJob.FTSGUID ftsFile.Status = "Submitted" ftsFile.Attempt += 1 # # update placement route try: self.updateLock().acquire() self.__ftsPlacement.startTransferOnRoute( route ) finally: self.updateLock().release() ftsJobs.append( ftsJob ) log.info( "%s new FTSJobs have been submitted" % len( ftsJobs ) ) return S_OK( ftsJobs )
def _getFTSJobProperties( self, ftsJobID, columnNames = None ): """ select :columnNames: from FTSJob table """ columnNames = columnNames if columnNames else FTSJob.tableDesc()["Fields"].keys() columnNames = ",".join( [ '`%s`' % str( columnName ) for columnName in columnNames ] ) return "SELECT %s FROM `FTSJob` WHERE `FTSJobID` = %s;" % ( columnNames, int( ftsJobID ) )
def __submit( self, request, operation, toSubmit ): """ create and submit new FTSJobs using list of FTSFiles :param Request request: ReqDB.Request instance :param list ftsFiles: list of FTSFile instances :return: [ FTSJob, FTSJob, ...] """ log = self.log.getSubLogger( "%s/submit" % request.RequestName ) bySourceAndTarget = {} for ftsFile in toSubmit: if ftsFile.SourceSE not in bySourceAndTarget: bySourceAndTarget.setdefault( ftsFile.SourceSE, {} ) if ftsFile.TargetSE not in bySourceAndTarget[ftsFile.SourceSE]: bySourceAndTarget[ftsFile.SourceSE].setdefault( ftsFile.TargetSE, [] ) bySourceAndTarget[ftsFile.SourceSE][ftsFile.TargetSE].append( ftsFile ) ftsJobs = [] for source, targetDict in bySourceAndTarget.items(): for target, ftsFileList in targetDict.items(): log.info( "found %s files to submit from %s to %s" % ( len( ftsFileList ), source, target ) ) route = self.__ftsGraph.findRoute( source, target ) if not route["OK"]: log.error( route["Message"] ) continue route = route["Value"] sourceRead = route.fromNode.SEs[source]["read"] if not sourceRead: log.error( "SourceSE %s is banned for reading right now" % source ) continue targetWrite = route.toNode.SEs[target]["write"] if not targetWrite: log.error( "TargetSE %s is banned for writing right now" % target ) continue if route.ActiveJobs > route.toNode.MaxActiveJobs: log.warn( "unable to submit new FTS job, max active jobs reached" ) continue sourceSE = self.getSE( source ) sourceToken = sourceSE.getStorageParameters( "SRM2" ) if not sourceToken["OK"]: log.error( "unable to get sourceSE '%s' parameters: %s" % ( source, sourceToken["Message"] ) ) continue seStatus = sourceSE.getStatus()['Value'] targetSE = self.getSE( target ) targetToken = targetSE.getStorageParameters( "SRM2" ) if not targetToken["OK"]: log.error( "unable to get targetSE '%s' parameters: %s" % ( target, targetToken["Message"] ) ) continue # # create FTSJob for fileList in breakListIntoChunks( ftsFileList, self.MAX_FILES_PER_JOB ): ftsJob = FTSJob() ftsJob.RequestID = request.RequestID ftsJob.OperationID = operation.OperationID ftsJob.SourceSE = source ftsJob.TargetSE = target ftsJob.SourceToken = sourceToken["Value"].get( "SpaceToken", "" ) ftsJob.TargetToken = targetToken["Value"].get( "SpaceToken", "" ) ftsJob.FTSServer = route.toNode.FTSServer for ftsFile in fileList: ftsFile.Attempt += 1 ftsFile.Error = "" ftsJob.addFile( ftsFile ) submit = ftsJob.submitFTS2( command = self.SUBMIT_COMMAND, pinTime = self.PIN_TIME if seStatus['TapeSE'] else 0 ) if not submit["OK"]: log.error( "unable to submit FTSJob: %s" % submit["Message"] ) continue log.info( "FTSJob '%s'@'%s' has been submitted" % ( ftsJob.FTSGUID, ftsJob.FTSServer ) ) # # update statuses for job files for ftsFile in ftsJob: ftsFile.FTSGUID = ftsJob.FTSGUID ftsFile.Status = "Submitted" ftsFile.Attempt += 1 # # update graph route try: self.updateLock().acquire() route.ActiveJobs += 1 finally: self.updateLock().release() ftsJobs.append( ftsJob ) log.info( "%s new FTSJobs have been submitted" % len( ftsJobs ) ) return S_OK( ftsJobs )