def main(): global overwrite Script.registerSwitch("f", "force", "Force overwrite of existing file", setOverwrite) Script.parseCommandLine(ignoreErrors=True) args = Script.getPositionalArgs() if len(args) < 1 or len(args) > 4: Script.showHelp(exitCode=1) from DIRAC.DataManagementSystem.Client.DataManager import DataManager from DIRAC import gLogger import DIRAC exitCode = 0 lfns = [] if len(args) == 1: inputFileName = args[0] if os.path.exists(inputFileName): inputFile = open(inputFileName, "r") for line in inputFile: line = line.rstrip() items = line.split() items[0] = items[0].replace("LFN:", "").replace("lfn:", "") lfns.append(getDict(items)) inputFile.close() else: gLogger.error("Error: LFN list '%s' missing." % inputFileName) exitCode = 4 else: lfns.append(getDict(args)) dm = DataManager() for lfn in lfns: if not os.path.exists(lfn["localfile"]): gLogger.error("File %s must exist locally" % lfn["localfile"]) exitCode = 1 continue if not os.path.isfile(lfn["localfile"]): gLogger.error("%s is not a file" % lfn["localfile"]) exitCode = 2 continue gLogger.notice("\nUploading %s" % lfn["lfn"]) res = dm.putAndRegister(lfn["lfn"], lfn["localfile"], lfn["SE"], lfn["guid"], overwrite=overwrite) if not res["OK"]: exitCode = 3 gLogger.error("Error: failed to upload %s to %s: %s" % (lfn["lfn"], lfn["SE"], res)) continue else: gLogger.notice("Successfully uploaded file to %s" % lfn["SE"]) DIRAC.exit(exitCode)
def finish(self): """ after having set all the files, this one does all the job @return: """ rc = 0 rm = DataManager() for item in self.listFileStaged: # print("SE '"+self.SE+"' == : '"+str(self.SE == "False")+"'") if not self.SE: self.log.info("No SE available for '" + item[0] + "'") rc += 1 continue else: self.log.info("Trying to store '" + item[0] + "' in SE : '" + self.SE + "' ...") result = rm.putAndRegister(item[1], item[0], self.SE) if not result["OK"]: self.log.info("ERROR %s" % (result["Message"])) self.log.info("Wait 5sec before trying again...") time.sleep(5) result = rm.putAndRegister(item[1], item[0], self.SE) if not result["OK"]: self.log.info("ERROR %s" % (result["Message"])) while not result["OK"]: self.listSEs.remove(self.SE) # make sure not to pick the same SE again. self.__pickRandomSE() if not self.SE: rc += 1 break self.log.info("Trying with another SE : '" + self.SE + "' . In 5sec...") time.sleep(5) result = rm.putAndRegister(item[1], item[0], self.SE) if result["OK"]: self.log.info("file stored : '" + item[1] + "' in '" + self.SE + "'") else: self.log.error("ERROR : failed to store the file '" + item[1] + "' ...") rc += 1 return rc
def _add_file(self, lfn, localfile, SE, guid=None): dm = DataManager() self._create_test_file() if not os.path.exists(self.options['test_file']): gLogger.error("File %s must exist locally" % localfile) if not os.path.isfile(self.options['test_file']): gLogger.error("%s is not a file" % localfile) res = dm.putAndRegister(lfn, localfile, SE, guid) if not res['OK']: gLogger.error('Error: failed to upload %s to %s' % (lfn, SE)) return S_ERROR(res['Message']) return S_OK(res['Value']['Successful'][lfn])
def addDataFiles(args): fcc = FileCatalogClient() dm = DataManager(['FileCatalog', 'TSCatalog']) outputPath = args[0] outputPattern = args[1] outputSE = args[2] metadata = args[3] metadata = json.loads(metadata) ## Create path res = fcc.createDirectory(outputPath) if not res['OK']: return res ## Set metadata for key, value in metadata.items(): res = fcc.setMetadata(outputPath, {key: value}) # terrible hack # if not res['OK']: # return res ## Upload data files all_files = glob.glob(outputPattern) ## Check that data files exist if len(all_files) == 0: return DIRAC.S_ERROR('No data files found') for one_file in all_files: lfn = os.path.join(outputPath, one_file) msg = 'Try to upload local file: %s \nwith LFN: %s \nto %s' % ( one_file, lfn, outputSE) DIRAC.gLogger.notice(msg) res = dm.putAndRegister(lfn, one_file, outputSE) ## Check if failed if not res['OK']: DIRAC.gLogger.error( 'Failed to putAndRegister %s \nto %s \nwith message: %s' % (lfn, outputSE, res['Message'])) return res elif res['Value']['Failed'].has_key(lfn): DIRAC.gLogger.error('Failed to putAndRegister %s to %s' % (lfn, outputSE)) return res return DIRAC.S_OK()
def addDataFiles(args): fcc = FileCatalogClient() dm = DataManager(['FileCatalog']) outputPath = args[0] outputPattern = args[1] outputSE = args[2] mode = args[3] if mode == 'closest': outputSE = getSiteSE(outputSE) DIRAC.gLogger.error('OutputSE: %s' % (outputSE)) print "outputSE", outputSE # Create path res = fcc.createDirectory(outputPath) if not res['OK']: return res # Upload data files all_files = glob.glob(outputPattern) # Check that data files exist if len(all_files) == 0: return DIRAC.S_ERROR('No data files found') for one_file in all_files: lfn = os.path.join(outputPath, one_file) msg = 'Try to upload local file: %s \nwith LFN: %s \nto %s' % ( one_file, lfn, outputSE) DIRAC.gLogger.notice(msg) res = dm.putAndRegister(lfn, one_file, outputSE) # Check if failed if not res['OK']: DIRAC.gLogger.error( 'Failed to putAndRegister %s \nto %s \nwith message: %s' % (lfn, outputSE, res['Message'])) return res elif res['Value']['Failed'].has_key(lfn): DIRAC.gLogger.error('Failed to putAndRegister %s to %s' % (lfn, outputSE)) return res return DIRAC.S_OK()
class ReplicaManagerTestCase(unittest.TestCase): """ Base class for the Replica Manager test cases """ def setUp(self): self.dataManager = DataManager() self.fileName = '/tmp/temporaryLocalFile' file = open(self.fileName,'w') file.write("%s" % time.time()) file.close() def test_putAndRegister(self): print '\n\n#########################################################################\n\n\t\t\tPut and register test\n' lfn = '/lhcb/test/unit-test/ReplicaManager/putAndRegister/testFile.%s' % time.time() diracSE = 'GRIDKA-RAW' putRes = self.dataManager.putAndRegister(lfn, self.fileName, diracSE) removeRes = self.dataManager.removeFile(lfn) # Check that the put was successful self.assert_(putRes['OK']) self.assert_(putRes['Value'].has_key('Successful')) self.assert_(putRes['Value']['Successful'].has_key(lfn)) self.assert_(putRes['Value']['Successful'][lfn]) # Check that the removal was successful self.assert_(removeRes['OK']) self.assert_(removeRes['Value'].has_key('Successful')) self.assert_(removeRes['Value']['Successful'].has_key(lfn)) self.assert_(removeRes['Value']['Successful'][lfn]) def test_putAndRegisterReplicate(self): print '\n\n#########################################################################\n\n\t\t\tReplication test\n' lfn = '/lhcb/test/unit-test/ReplicaManager/putAndRegisterReplicate/testFile.%s' % time.time() diracSE = 'GRIDKA-RAW' putRes = self.dataManager.putAndRegister(lfn, self.fileName, diracSE) replicateRes = self.dataManager.replicateAndRegister(lfn,'CNAF-DST') #,sourceSE='',destPath='',localCache='') removeRes = self.dataManager.removeFile(lfn) # Check that the put was successful self.assert_(putRes['OK']) self.assert_(putRes['Value'].has_key('Successful')) self.assert_(putRes['Value']['Successful'].has_key(lfn)) self.assert_(putRes['Value']['Successful'][lfn]) # Check that the replicate was successful self.assert_(replicateRes['OK']) self.assert_(replicateRes['Value'].has_key('Successful')) self.assert_(replicateRes['Value']['Successful'].has_key(lfn)) self.assert_(replicateRes['Value']['Successful'][lfn]) # Check that the removal was successful self.assert_(removeRes['OK']) self.assert_(removeRes['Value'].has_key('Successful')) self.assert_(removeRes['Value']['Successful'].has_key(lfn)) self.assert_(removeRes['Value']['Successful'][lfn]) def test_putAndRegisterGetReplicaMetadata(self): print '\n\n#########################################################################\n\n\t\t\tGet metadata test\n' lfn = '/lhcb/test/unit-test/ReplicaManager/putAndRegisterGetReplicaMetadata/testFile.%s' % time.time() diracSE = 'GRIDKA-RAW' putRes = self.dataManager.putAndRegister(lfn, self.fileName, diracSE) metadataRes = self.dataManager.getReplicaMetadata(lfn,diracSE) removeRes = self.dataManager.removeFile(lfn) # Check that the put was successful self.assert_(putRes['OK']) self.assert_(putRes['Value'].has_key('Successful')) self.assert_(putRes['Value']['Successful'].has_key(lfn)) self.assert_(putRes['Value']['Successful'][lfn]) # Check that the metadata query was successful self.assert_(metadataRes['OK']) self.assert_(metadataRes['Value'].has_key('Successful')) self.assert_(metadataRes['Value']['Successful'].has_key(lfn)) self.assert_(metadataRes['Value']['Successful'][lfn]) metadataDict = metadataRes['Value']['Successful'][lfn] self.assert_(metadataDict.has_key('Cached')) self.assert_(metadataDict.has_key('Migrated')) self.assert_(metadataDict.has_key('Size')) # Check that the removal was successful self.assert_(removeRes['OK']) self.assert_(removeRes['Value'].has_key('Successful')) self.assert_(removeRes['Value']['Successful'].has_key(lfn)) self.assert_(removeRes['Value']['Successful'][lfn]) def test_putAndRegsiterGetAccessUrl(self): print '\n\n#########################################################################\n\n\t\t\tGet Access Url test\n' lfn = '/lhcb/test/unit-test/ReplicaManager/putAndRegisterGetAccessUrl/testFile.%s' % time.time() diracSE = 'GRIDKA-RAW' putRes = self.dataManager.putAndRegister(lfn, self.fileName, diracSE) getAccessUrlRes = self.dataManager.getReplicaAccessUrl(lfn,diracSE) print getAccessUrlRes removeRes = self.dataManager.removeFile(lfn) # Check that the put was successful self.assert_(putRes['OK']) self.assert_(putRes['Value'].has_key('Successful')) self.assert_(putRes['Value']['Successful'].has_key(lfn)) self.assert_(putRes['Value']['Successful'][lfn]) # Check that the access url was successful self.assert_(getAccessUrlRes['OK']) self.assert_(getAccessUrlRes['Value'].has_key('Successful')) self.assert_(getAccessUrlRes['Value']['Successful'].has_key(lfn)) self.assert_(getAccessUrlRes['Value']['Successful'][lfn]) # Check that the removal was successful self.assert_(removeRes['OK']) self.assert_(removeRes['Value'].has_key('Successful')) self.assert_(removeRes['Value']['Successful'].has_key(lfn)) self.assert_(removeRes['Value']['Successful'][lfn]) def test_putAndRegisterRemoveReplica(self): print '\n\n#########################################################################\n\n\t\t\tRemove replica test\n' lfn = '/lhcb/test/unit-test/ReplicaManager/putAndRegisterRemoveReplica/testFile.%s' % time.time() diracSE = 'GRIDKA-RAW' putRes = self.dataManager.putAndRegister(lfn, self.fileName, diracSE) removeReplicaRes = self.dataManager.removeReplica(diracSE,lfn) removeRes = self.dataManager.removeFile(lfn) # Check that the put was successful self.assert_(putRes['OK']) self.assert_(putRes['Value'].has_key('Successful')) self.assert_(putRes['Value']['Successful'].has_key(lfn)) self.assert_(putRes['Value']['Successful'][lfn]) # Check that the replica removal was successful self.assert_(removeReplicaRes['OK']) self.assert_(removeReplicaRes['Value'].has_key('Successful')) self.assert_(removeReplicaRes['Value']['Successful'].has_key(lfn)) self.assert_(removeReplicaRes['Value']['Successful'][lfn]) # Check that the removal was successful self.assert_(removeRes['OK']) self.assert_(removeRes['Value'].has_key('Successful')) self.assert_(removeRes['Value']['Successful'].has_key(lfn)) self.assert_(removeRes['Value']['Successful'][lfn]) def test_registerFile(self): lfn = '/lhcb/test/unit-test/ReplicaManager/registerFile/testFile.%s' % time.time() physicalFile = 'srm://host:port/srm/managerv2?SFN=/sa/path%s' % lfn fileSize = 10000 storageElementName = 'CERN-RAW' fileGuid = makeGuid() fileTuple = (lfn,physicalFile,fileSize,storageElementName,fileGuid) registerRes = self.dataManager.registerFile(fileTuple) removeCatalogReplicaRes = self.dataManager.removeCatalogReplica(storageElementName,lfn) removeFileRes = self.dataManager.removeFile(lfn) # Check that the file registration was done correctly self.assert_(registerRes['OK']) self.assert_(registerRes['Value'].has_key('Successful')) self.assert_(registerRes['Value']['Successful'].has_key(lfn)) self.assert_(registerRes['Value']['Successful'][lfn]) # Check that the replica removal was successful self.assert_(removeCatalogReplicaRes['OK']) self.assert_(removeCatalogReplicaRes['Value'].has_key('Successful')) self.assert_(removeCatalogReplicaRes['Value']['Successful'].has_key(lfn)) self.assert_(removeCatalogReplicaRes['Value']['Successful'][lfn]) # Check that the removal was successful self.assert_(removeFileRes['OK']) self.assert_(removeFileRes['Value'].has_key('Successful')) self.assert_(removeFileRes['Value']['Successful'].has_key(lfn)) self.assert_(removeFileRes['Value']['Successful'][lfn]) def test_registerReplica(self): print '\n\n#########################################################################\n\n\t\t\tRegister replica test\n' lfn = '/lhcb/test/unit-test/ReplicaManager/registerReplica/testFile.%s' % time.time() physicalFile = 'srm://host:port/srm/managerv2?SFN=/sa/path%s' % lfn fileSize = 10000 storageElementName = 'CERN-RAW' fileGuid = makeGuid() fileTuple = (lfn,physicalFile,fileSize,storageElementName,fileGuid) registerRes = self.dataManager.registerFile(fileTuple) seName = 'GRIDKA-RAW' replicaTuple = (lfn,physicalFile,seName) registerReplicaRes = self.dataManager.registerReplica(replicaTuple) removeCatalogReplicaRes1 = self.dataManager.removeCatalogReplica(storageElementName,lfn) removeCatalogReplicaRes2 = self.dataManager.removeCatalogReplica(seName,lfn) removeFileRes = self.dataManager.removeFile(lfn) # Check that the file registration was done correctly self.assert_(registerRes['OK']) self.assert_(registerRes['Value'].has_key('Successful')) self.assert_(registerRes['Value']['Successful'].has_key(lfn)) self.assert_(registerRes['Value']['Successful'][lfn]) # Check that the replica registration was successful self.assert_(registerReplicaRes['OK']) self.assert_(registerReplicaRes['Value'].has_key('Successful')) self.assert_(registerReplicaRes['Value']['Successful'].has_key(lfn)) self.assert_(registerReplicaRes['Value']['Successful'][lfn]) # Check that the replica removal was successful self.assert_(removeCatalogReplicaRes1['OK']) self.assert_(removeCatalogReplicaRes1['Value'].has_key('Successful')) self.assert_(removeCatalogReplicaRes1['Value']['Successful'].has_key(lfn)) self.assert_(removeCatalogReplicaRes1['Value']['Successful'][lfn]) # Check that the replica removal was successful self.assert_(removeCatalogReplicaRes2['OK']) self.assert_(removeCatalogReplicaRes2['Value'].has_key('Successful')) self.assert_(removeCatalogReplicaRes2['Value']['Successful'].has_key(lfn)) self.assert_(removeCatalogReplicaRes2['Value']['Successful'][lfn]) # Check that the removal was successful self.assert_(removeFileRes['OK']) self.assert_(removeFileRes['Value'].has_key('Successful')) self.assert_(removeFileRes['Value']['Successful'].has_key(lfn)) self.assert_(removeFileRes['Value']['Successful'][lfn]) def test_putAndRegisterGet(self): print '\n\n#########################################################################\n\n\t\t\tGet file test\n' lfn = '/lhcb/test/unit-test/ReplicaManager/putAndRegisterGet/testFile.%s' % time.time() diracSE = 'GRIDKA-RAW' putRes = self.dataManager.putAndRegister(lfn, self.fileName, diracSE) getRes = self.dataManager.getFile(lfn) removeRes = self.dataManager.removeFile(lfn) localFilePath = "%s/%s" % (os.getcwd(),os.path.basename(lfn)) if os.path.exists(localFilePath): os.remove(localFilePath) # Check that the put was successful self.assert_(putRes['OK']) self.assert_(putRes['Value'].has_key('Successful')) self.assert_(putRes['Value']['Successful'].has_key(lfn)) self.assert_(putRes['Value']['Successful'][lfn]) # Check that the replica removal was successful self.assert_(getRes['OK']) self.assert_(getRes['Value'].has_key('Successful')) self.assert_(getRes['Value']['Successful'].has_key(lfn)) self.assertEqual(getRes['Value']['Successful'][lfn],localFilePath) # Check that the removal was successful self.assert_(removeRes['OK']) self.assert_(removeRes['Value'].has_key('Successful')) self.assert_(removeRes['Value']['Successful'].has_key(lfn)) self.assert_(removeRes['Value']['Successful'][lfn])
from DIRAC import S_OK, S_ERROR, gLogger, exit from DIRAC.DataManagementSystem.Client.DataManager import DataManager lfn = args[0] pfn = args[1] se = args[2] exit_code = 0 log = '' dm = DataManager() start = time.time() result = dm.removeFile(lfn) result = dm.putAndRegister(lfn, pfn, se) uploadTime = time.time() - start if result['OK']: log += 'Succeed to upload file to SE %s.\n' % se log += 'Upload Time : %ss\n' % uploadTime start = time.time() result = dm.getReplica(lfn, se, tempfile.gettempdir()) downloadTime = time.time() - start if result['OK']: log += 'Succeed to download file from SE %s.\n' % se log += 'Download Time : %ss\n' % downloadTime else: exit_code = 1 log += 'Failed to download file from SE %s : %s\n' % ( se, result['Message'])
class SETest( TestBase ): """ SETest is used to test the availability of SE. """ def __init__( self, args = None, apis = None ): super( SETest, self ).__init__( args, apis ) self.__lfnPath = '/bes/user/z/zhaoxh/' self.__testFile = 'test.dat' self.__localPath = '/tmp/' if 'DataManager' in self.apis: self.dm = self.apis[ 'DataManager' ] else: self.dm = DataManager() def doTest( self, elementDict ): """ Test upload and download for specified SE. """ elementName = elementDict[ 'ElementName' ] testFilePath = self.__localPath + self.__testFile if not os.path.exists( testFilePath ) or not os.path.isfile( testFilePath ): f = open( testFilePath, 'w' ) f.write( 'hello' ) f.close() status = 'OK' log = '' lfnPath = self.__lfnPath + elementName + '-' + self.__testFile submissionTime = datetime.utcnow().replace( microsecond = 0 ) LOCK.acquire() start = time.time() result = self.dm.putAndRegister( lfnPath, testFilePath, elementName ) uploadTime = time.time() - start if result[ 'OK' ]: log += 'Succeed to upload file to SE %s.\n' % elementName log += 'Upload Time : %ss\n' % uploadTime start = time.time() result = self.dm.getReplica( lfnPath, elementName, self.__localPath ) downloadTime = time.time() - start if result[ 'OK' ]: log += 'Succeed to download file from SE %s.\n' % elementName log += 'Download Time : %ss\n' % downloadTime else: status = 'Bad' log += 'Failed to download file from SE %s : %s\n' % ( elementName, result[ 'Message' ] ) result = self.dm.removeFile( lfnPath ) if result[ 'OK' ]: log += 'Succeed to delete file from SE %s.\n' % elementName else: log += 'Faile to delete file from SE %s : %s\n' % ( elementName, result[ 'Message' ] ) else: status = 'Bad' log += 'Failed to upload file to SE %s : %s\n' % ( elementName, result[ 'Message' ] ) LOCK.release() completionTime = datetime.utcnow().replace( microsecond = 0 ) applicationTime = ( completionTime - submissionTime ).total_seconds() result = { 'Result' : { 'Status' : status, 'Log' : log, 'SubmissionTime' : submissionTime, 'CompletionTime' : completionTime, 'ApplicationTime' : applicationTime }, 'Finish' : True } localFile = self.__localPath + elementName +'-' + self.__testFile if os.path.exists( localFile ) and os.path.isfile( localFile ): os.remove( localFile ) return S_OK( result )
class Prod3DataManager(object) : """ Manage data and meta-data """ def __init__(self,catalogs=['DIRACFileCatalog']): """ Constructor """ self.setupCatalogClient( catalogs ) self.printCatalogConfig( catalogs ) self.setupDataManagerClient( catalogs ) def setupCatalogClient( self, catalogs ): """ Init FileCatalog client Ideally we would like to use only FileCatalog but it doesn't work for setMetadata because the returned value has a wrong format. So use of FileCatalogClient instead """ self.fc = FileCatalog( catalogs ) self.fcc = FileCatalogClient() def printCatalogConfig( self, catalogs ): """ Dumps catalog configuration """ for catalog in catalogs: res = self.fc._getCatalogConfigDetails( catalog ) DIRAC.gLogger.info( 'CatalogConfigDetails:', res ) def setupDataManagerClient( self, catalogs ): """ Init DataManager client """ self.dm = DataManager( catalogs ) def _getSEList( self, SEType = 'ProductionOutputs', DataType = 'SimtelProd' ): """ get from CS the list of available SE for data upload """ opsHelper = Operations() optionName = os.path.join( SEType, DataType ) SEList = opsHelper.getValue( optionName , [] ) SEList = List.randomize( SEList ) DIRAC.gLogger.notice( 'List of %s SE: %s ' % ( SEType, SEList ) ) # # Check if the local SE is in the list. If yes try it first by reversing list order localSEList = [] res = getSEsForSite( DIRAC.siteName() ) if res['OK']: localSEList = res['Value'] retainedlocalSEList = [] for localSE in localSEList: if localSE in SEList: DIRAC.gLogger.notice( 'The local Storage Element is an available SE: ', localSE ) retainedlocalSEList.append( localSE ) SEList.remove( localSE ) SEList = retainedlocalSEList + SEList if len( SEList ) == 0: return DIRAC.S_ERROR( 'Error in building SEList' ) return DIRAC.S_OK( SEList ) def _checkemptydir( self, path ): """ check that the directory is not empty """ if len ( glob.glob( path ) ) == 0: error = 'Empty directory: %s' % path return DIRAC.S_ERROR( error ) else: return DIRAC.S_OK() def _getRunPath( self, filemetadata ): """ format path to string with 1 digit precision run_number is taken from filemetadata filemetadata can be a dict or the run_number itself """ if type( filemetadata ) == type( dict() ): run_number = int( filemetadata['runNumber'] ) else: run_number = int( filemetadata ) run_numberMod = run_number % 1000 runpath = '%03dxxx' % ( ( run_number - run_numberMod ) / 1000 ) return runpath def _formatPath( self, path ): """ format path to string with 1 digit precision """ if type( path ) == float or type( path ) == int: path = '%.1f' % path return str( path ) def createTarLogFiles ( self, inputpath, tarname ): """ create tar of log and histogram files """ tarmode = 'w:gz' tar = tarfile.open( tarname, tarmode ) for subdir in ['Log/*', 'Histograms/*']: logdir = os.path.join( inputpath, subdir ) res = self._checkemptydir( logdir ) if not res['OK']: return res for localfile in glob.glob( logdir ): tar.add( localfile, arcname = localfile.split( '/' )[-1] ) tar.close() return DIRAC.S_OK() def createMDStructure( self, metadata, metadatafield, basepath, program_category ): """ create meta data structure """ # ## Add metadata fields to the DFC mdfield = json.loads( metadatafield ) for key, value in mdfield.items(): res = self.fc.addMetadataField( key, value ) if not res['OK']: return res # ## Create the directory structure md = json.loads( metadata , object_pairs_hook = collections.OrderedDict ) path = basepath process_program = program_category + '_prog' for key, value in collections.OrderedDict( ( k, md[k] ) for k in ( 'site', 'particle', process_program ) if k in md ).items(): path = os.path.join( path, self._formatPath( value ) ) res = self.fc.createDirectory( path ) if not res['OK']: return res # Set directory metadata for each subdir: 'site', 'particle', 'process_program' res = self.fcc.setMetadata( path, {key:value} ) if not res['OK']: return res # Create the TransformationID subdir and set MD # ## Get the TransformationID if os.environ.has_key( 'JOBID' ): jobID = os.environ['JOBID'] dirac = Dirac() res = dirac.getJobJDL( jobID ) TransformationID = '0000' if res['Value'].has_key( 'TransformationID' ): TransformationID = res['Value']['TransformationID'] path = os.path.join( path, TransformationID ) res = self.fc.createDirectory( path ) if not res['OK']: return res process_program_version = process_program + '_version' res = self.fcc.setMetadata( path, dict( ( k, md[k] ) for k in ( 'phiP', 'thetaP', 'array_layout', process_program_version ) if k in md ) ) if not res['OK']: return res # Create the Data and Log subdirs and set MD Transformation_path = path for subdir in ['Data', 'Log']: path = os.path.join( Transformation_path, subdir ) res = self.fc.createDirectory( path ) if not res['OK']: return res # Set metadata if not already defined res = self.fcc.getDirectoryUserMetadata( path ) if not res['OK']: return res if 'outputType' not in res['Value']: res = self.fcc.setMetadata( path, {'outputType':subdir} ) if not res['OK']: return res # MD for the Data directory - data_level and configuration_id path = os.path.join(Transformation_path, 'Data') # Set metadata if not already defined res = self.fcc.getDirectoryUserMetadata( path ) if not res['OK']: return res if 'data_level' and 'configuration_id' not in res['Value']: res = self.fcc.setMetadata(path, {'data_level': md['data_level'], 'configuration_id': md['configuration_id'] }) if not res['OK']: return res return DIRAC.S_OK( Transformation_path ) def putAndRegister( self, lfn, localfile, filemetadata, DataType = 'SimtelProd' ): """ put and register one file and set metadata """ # ## Get the list of Production SE res = self._getSEList( 'ProductionOutputs', DataType ) if res['OK']: ProductionSEList = res['Value'] else: return res # ## Get the list of Failover SE res = self._getSEList( 'ProductionOutputsFailover', DataType ) if res['OK']: FailoverSEList = res['Value'] else: return res # ## Upload file to a Production SE res = self._putAndRegisterToSEList( lfn, localfile, ProductionSEList ) if not res['OK']: DIRAC.gLogger.error( 'Failed to upload file to any Production SE: %s' % ProductionSEList ) # ## Upload file to a Failover SE res = self._putAndRegisterToSEList( lfn, localfile, FailoverSEList ) if not res['OK']: return DIRAC.S_ERROR( 'Failed to upload file to any Failover SE: %s' % FailoverSEList ) # ## Set file metadata: jobID, subarray, sct if res['OK']: fmd = json.loads( filemetadata ) if os.environ.has_key( 'JOBID' ): fmd.update( {'jobID':os.environ['JOBID']} ) filename = os.path.basename( localfile ) # set subarray and sct md from filename p = re.compile( 'subarray-\d+' ) if p.search( filename ) != None: subarray = p.search( filename ).group() fmd.update( {'subarray':subarray} ) sct = 'False' p = re.compile( 'nosct' ) psct = re.compile( 'sct' ) if p.search( filename ) == None and psct.search( filename ) != None: sct = 'True' # ## Set sct flag only for production data res = self.fcc.getFileUserMetadata( lfn ) if DataType == 'SimtelProd' and res['Value']['outputType'] == 'Data': fmd.update( {'sct':sct} ) res = self.fc.setMetadata( lfn, fmd ) if not res['OK']: return res return DIRAC.S_OK() def _putAndRegisterToSEList( self, lfn, localfile, SEList ): """ put and register one file to one SE in the SEList """ # ## Try to upload file to a SE in the list for SE in SEList: msg = 'Try to upload local file: %s \nwith LFN: %s \nto %s' % ( localfile, lfn, SE ) DIRAC.gLogger.notice( msg ) res = self.dm.putAndRegister( lfn, localfile, SE ) DIRAC.gLogger.notice(res) # ## check if failed if not res['OK']: DIRAC.gLogger.error( 'Failed to putAndRegister %s \nto %s \nwith message: %s' % ( lfn, SE, res['Message'] ) ) DIRAC.gLogger.notice('Trying to clean up %s' % lfn) res = self.dm.removeFile(lfn) if res['OK']: DIRAC.gLogger.notice('Successfully removed %s \n that was not supposed to have been uploaded successfully' % lfn) continue elif res['Value']['Failed'].has_key( lfn ): DIRAC.gLogger.error( 'Failed to putAndRegister %s to %s' % ( lfn, SE ) ) DIRAC.gLogger.notice('Trying to clean up %s' % lfn) res = self.dm.removeFile(lfn) if res['OK']: DIRAC.gLogger.notice('Successfully removed %s \n that was not supposed to have been uploaded successfully' % lfn) continue else: return DIRAC.S_OK() return DIRAC.S_ERROR() def cleanLocalFiles ( self, datadir, pattern = '*' ): """ remove files matching pattern in datadir """ for localfile in glob.glob( os.path.join( datadir, pattern ) ): DIRAC.gLogger.notice( 'Removing local file: ', localfile ) os.remove( localfile ) return DIRAC.S_OK()
from DIRAC import S_OK, S_ERROR, gLogger, exit from DIRAC.DataManagementSystem.Client.DataManager import DataManager lfn = args[0] pfn = args[1] se = args[2] exit_code = 0 log = '' dm = DataManager() start = time.time() result = dm.removeFile( lfn ) result = dm.putAndRegister( lfn, pfn, se ) uploadTime = time.time() - start if result[ 'OK' ]: log += 'Succeed to upload file to SE %s.\n' % se log += 'Upload Time : %ss\n' % uploadTime start = time.time() result = dm.getReplica( lfn, se, tempfile.gettempdir() ) downloadTime = time.time() - start if result[ 'OK' ]: log += 'Succeed to download file from SE %s.\n' % se log += 'Download Time : %ss\n' % downloadTime else: exit_code = 1 log += 'Failed to download file from SE %s : %s\n' % ( se, result[ 'Message' ] )
def main(): dm = DataManager() fileTupleBuffer = [] res = getProxyInfo( False, False ) if not res['OK']: gLogger.error( "Failed to get client proxy information.", res['Message'] ) DIRAC.exit( 2 ) proxyInfo = res['Value'] if proxyInfo['secondsLeft'] == 0: gLogger.error( "Proxy expired" ) DIRAC.exit( 2 ) username = proxyInfo['username'] vo = '' if 'group' in proxyInfo: vo = getVOForGroup( proxyInfo['group'] ) counter = 0 for f in files: counter += 1 local_f=f if not f.startswith('/cefs'): # gLogger.error('File must be under "/cefs"') # continue #if the file to reg is not under /cefs, use put and register folder_name=os.path.basename(os.path.dirname(f)) # lfn = '/cepc/user/%s/%s/jsub/'%(username[0],username) + folder_name + '/' + os.path.basename(f) lfn = '/cepc/user/%s/%s/jsub/'%(username[0],username) + folder_name + '/' + os.path.basename(f) # dirname = os.path.dirname(local_f) # os.system('mkdir -p %s'%(dirname)) # os.system('cp %s %s' %(f,local_f)) do_put_and_register=True else: lfn = '/cepc/lustre-ro' + os.path.abspath(f) do_put_and_register=False result = fcc.isFile(lfn) if result['OK'] and lfn in result['Value']['Successful'] and result['Value']['Successful'][lfn]: continue size = os.path.getsize(f) adler32 = fileAdler(f) guid = makeGuid() fileTuple = (lfn, local_f, size, _se, guid, adler32) fileTupleBuffer.append(fileTuple) gLogger.debug('Register to lfn: %s' % lfn) gLogger.debug('fileTuple: %s' % (fileTuple,)) if len(fileTupleBuffer) >= _bufferSize: if do_put_and_register: result = dm.putAndRegister(lfn, local_f, _se, guid, overwrite=overwrite) else: result = dm.registerFile(fileTupleBuffer) print('register result', result) # if not result['OK']: # gLogger.error('Register file failed') # return 1 del fileTupleBuffer[:] gLogger.debug('%s files registered' % counter) if fileTupleBuffer: if do_put_and_register: result = dm.putAndRegister(lfn, local_f, _se, guid, overwrite=overwrite) else: result = dm.registerFile(fileTupleBuffer) print('register result', result) # if not result['OK']: # gLogger.error('Register file failed') # return 1 del fileTupleBuffer[:] gLogger.info('Totally %s files registered' % counter) return 0
def _uploadGenFiles(): """uploads the generator files""" clip = _Params() clip.registerSwitches() Script.parseCommandLine() from DIRAC import gLogger, exit as dexit if not clip.dir: gLogger.error('You need to set the path') Script.showHelp() dexit(1) if not clip.storageElement: gLogger.error('You need a storage element') Script.showHelp() dexit(1) for key in MANDATORY_KEYS: if key not in clip.fmeta: gLogger.error("Not all mandatory meta data defined, please check and add key: ", key) Script.showHelp() dexit(1) #resolve the inout files flist = [] if os.path.isdir(clip.dir): flistd = os.listdir(clip.dir) for filename in flistd: if filename.count(".stdhep"): flist.append( os.path.join(clip.dir, filename) ) elif os.path.isfile(clip.dir): flist.append(clip.dir) else: gLogger.error("%s is not a file nor a directory" % clip.dir) dexit(1) gLogger.notice("Will eventually upload %s file(s)" % len(flist)) from DIRAC.Core.Utilities.PromptUser import promptUser from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations basepath = Operations().getValue('Production/ILC_ILD/BasePath','') if not basepath: gLogger.error('Failed to contact CS, please try again') dexit(1) basepath = "/".join(basepath.split("/")[:-2])+"/" #need to get rid of the ild/ part at the end finalpath = os.path.join(basepath, 'generated', clip.energy+"-"+clip.machineParams, clip.evtclass, str(clip.fmeta['GenProcessID'])) gLogger.notice("Will upload the file(s) under %s" % finalpath) if not clip.force: res = promptUser('Continue?', ['y','n'], 'n') if not res['OK']: gLogger.error(res['Message']) dexit(1) if not res['Value'].lower()=='y': dexit(0) dirmeta = [] dirmeta.append({'path':os.path.join(basepath, 'generated'), 'meta':{'Datatype':'gen'}}) dirmeta.append({'path':os.path.join(basepath, 'generated', clip.energy+"-"+clip.machineParams), 'meta':{'Energy':clip.energy, 'MachineParams':clip.machineParams}}) dirmeta.append({'path':os.path.join(basepath, 'generated', clip.energy+"-"+clip.machineParams, clip.evtclass), 'meta':{'EvtClass':clip.evtclass }}) dirmeta.append({'path':finalpath, 'meta': {'EvtType':clip.evttype ,'Luminosity':clip.lumi, 'ProcessID': clip.fmeta['GenProcessID']} }) final_fname_base = 'E'+clip.energy+"-"+clip.machineParams+".P"+clip.fmeta['GenProcessName']+".G"+clip.fmeta['ProgramNameVersion'] + "."+clip.particle1+clip.pol1+"."+clip.particle2+clip.pol2+".I"+str(clip.fmeta['GenProcessID']) gLogger.notice("Final file name(s) will be %s where XX will be replaced by file number, and ext by the input file extension" % (final_fname_base+".XX.ext") ) if not clip.force: res = promptUser('Continue?', ['y','n'], 'n') if not res['OK']: gLogger.error(res['Message']) dexit(1) if not res['Value'].lower()=='y': dexit(0) from DIRAC.DataManagementSystem.Client.DataManager import DataManager from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient fc = FileCatalogClient() for pathdict in dirmeta: res = fc.createDirectory(pathdict['path']) if not res['OK']: gLogger.error("Could not create this directory in FileCatalog, abort:", pathdict['path'] ) dexit(0) res = fc.setMetadata(pathdict['path'], pathdict['meta']) if not res['OK']: gLogger.error( "Failed to set meta data %s to %s\n" %(pathdict['meta'], pathdict['path']), res['Message'] ) datMan = DataManager() for filename in flist: fnum = filename.split(".")[-2] fext = filename.split(".")[-1] final_fname = final_fname_base + '.' + fnum + "." + fext gLogger.notice("Uploading %s to" % filename, finalpath+"/"+final_fname) if not clip.force: res = promptUser('Continue?', ['y','n'], 'n') if not res['OK']: gLogger.error(res['Message']) break if not res['Value'].lower()=='y': break res = datMan.putAndRegister(finalpath+"/"+final_fname, filename, clip.storageElement) if not res['OK']: gLogger.error("Failed to upload %s:" % filename, res['Message']) continue res = fc.setMetadata(finalpath+"/"+final_fname, clip.fmeta) if not res['OK']: gLogger.error("Failed setting the metadata to %s:" % filename, res['Message']) dexit(0)
exitCode = 4 else: lfns.append(getDict(args)) dm = DataManager() for lfn in lfns: if not os.path.exists(lfn['localfile']): gLogger.error("File %s must exist locally" % lfn['localfile']) exitCode = 1 continue if not os.path.isfile(lfn['localfile']): gLogger.error("%s is not a file" % lfn['localfile']) exitCode = 2 continue gLogger.notice("\nUploading %s" % lfn['lfn']) res = dm.putAndRegister(lfn['lfn'], lfn['localfile'], lfn['SE'], lfn['guid'], overwrite=overwrite) if not res['OK']: exitCode = 3 gLogger.error('Error: failed to upload %s to %s' % (lfn['lfn'], lfn['SE'])) continue else: gLogger.notice('Successfully uploaded file to %s' % lfn['SE']) DIRAC.exit(exitCode)
class ReplicaManagerTestCase(unittest.TestCase): """ Base class for the Replica Manager test cases """ def setUp(self): self.dataManager = DataManager() self.fileName = '/tmp/temporaryLocalFile' file = open(self.fileName, 'w') file.write("%s" % time.time()) file.close() def test_putAndRegister(self): print '\n\n#########################################################################\n\n\t\t\tPut and register test\n' lfn = '/lhcb/test/unit-test/ReplicaManager/putAndRegister/testFile.%s' % time.time( ) diracSE = 'GRIDKA-RAW' putRes = self.dataManager.putAndRegister(lfn, self.fileName, diracSE) removeRes = self.dataManager.removeFile(lfn) # Check that the put was successful self.assert_(putRes['OK']) self.assert_(putRes['Value'].has_key('Successful')) self.assert_(putRes['Value']['Successful'].has_key(lfn)) self.assert_(putRes['Value']['Successful'][lfn]) # Check that the removal was successful self.assert_(removeRes['OK']) self.assert_(removeRes['Value'].has_key('Successful')) self.assert_(removeRes['Value']['Successful'].has_key(lfn)) self.assert_(removeRes['Value']['Successful'][lfn]) def test_putAndRegisterReplicate(self): print '\n\n#########################################################################\n\n\t\t\tReplication test\n' lfn = '/lhcb/test/unit-test/ReplicaManager/putAndRegisterReplicate/testFile.%s' % time.time( ) diracSE = 'GRIDKA-RAW' putRes = self.dataManager.putAndRegister(lfn, self.fileName, diracSE) replicateRes = self.dataManager.replicateAndRegister( lfn, 'CNAF-DST') #,sourceSE='',destPath='',localCache='') removeRes = self.dataManager.removeFile(lfn) # Check that the put was successful self.assert_(putRes['OK']) self.assert_(putRes['Value'].has_key('Successful')) self.assert_(putRes['Value']['Successful'].has_key(lfn)) self.assert_(putRes['Value']['Successful'][lfn]) # Check that the replicate was successful self.assert_(replicateRes['OK']) self.assert_(replicateRes['Value'].has_key('Successful')) self.assert_(replicateRes['Value']['Successful'].has_key(lfn)) self.assert_(replicateRes['Value']['Successful'][lfn]) # Check that the removal was successful self.assert_(removeRes['OK']) self.assert_(removeRes['Value'].has_key('Successful')) self.assert_(removeRes['Value']['Successful'].has_key(lfn)) self.assert_(removeRes['Value']['Successful'][lfn]) def test_putAndRegisterGetReplicaMetadata(self): print '\n\n#########################################################################\n\n\t\t\tGet metadata test\n' lfn = '/lhcb/test/unit-test/ReplicaManager/putAndRegisterGetReplicaMetadata/testFile.%s' % time.time( ) diracSE = 'GRIDKA-RAW' putRes = self.dataManager.putAndRegister(lfn, self.fileName, diracSE) metadataRes = self.dataManager.getReplicaMetadata(lfn, diracSE) removeRes = self.dataManager.removeFile(lfn) # Check that the put was successful self.assert_(putRes['OK']) self.assert_(putRes['Value'].has_key('Successful')) self.assert_(putRes['Value']['Successful'].has_key(lfn)) self.assert_(putRes['Value']['Successful'][lfn]) # Check that the metadata query was successful self.assert_(metadataRes['OK']) self.assert_(metadataRes['Value'].has_key('Successful')) self.assert_(metadataRes['Value']['Successful'].has_key(lfn)) self.assert_(metadataRes['Value']['Successful'][lfn]) metadataDict = metadataRes['Value']['Successful'][lfn] self.assert_(metadataDict.has_key('Cached')) self.assert_(metadataDict.has_key('Migrated')) self.assert_(metadataDict.has_key('Size')) # Check that the removal was successful self.assert_(removeRes['OK']) self.assert_(removeRes['Value'].has_key('Successful')) self.assert_(removeRes['Value']['Successful'].has_key(lfn)) self.assert_(removeRes['Value']['Successful'][lfn]) def test_putAndRegsiterGetAccessUrl(self): print '\n\n#########################################################################\n\n\t\t\tGet Access Url test\n' lfn = '/lhcb/test/unit-test/ReplicaManager/putAndRegisterGetAccessUrl/testFile.%s' % time.time( ) diracSE = 'GRIDKA-RAW' putRes = self.dataManager.putAndRegister(lfn, self.fileName, diracSE) getAccessUrlRes = self.dataManager.getReplicaAccessUrl(lfn, diracSE) print getAccessUrlRes removeRes = self.dataManager.removeFile(lfn) # Check that the put was successful self.assert_(putRes['OK']) self.assert_(putRes['Value'].has_key('Successful')) self.assert_(putRes['Value']['Successful'].has_key(lfn)) self.assert_(putRes['Value']['Successful'][lfn]) # Check that the access url was successful self.assert_(getAccessUrlRes['OK']) self.assert_(getAccessUrlRes['Value'].has_key('Successful')) self.assert_(getAccessUrlRes['Value']['Successful'].has_key(lfn)) self.assert_(getAccessUrlRes['Value']['Successful'][lfn]) # Check that the removal was successful self.assert_(removeRes['OK']) self.assert_(removeRes['Value'].has_key('Successful')) self.assert_(removeRes['Value']['Successful'].has_key(lfn)) self.assert_(removeRes['Value']['Successful'][lfn]) def test_putAndRegisterRemoveReplica(self): print '\n\n#########################################################################\n\n\t\t\tRemove replica test\n' lfn = '/lhcb/test/unit-test/ReplicaManager/putAndRegisterRemoveReplica/testFile.%s' % time.time( ) diracSE = 'GRIDKA-RAW' putRes = self.dataManager.putAndRegister(lfn, self.fileName, diracSE) removeReplicaRes = self.dataManager.removeReplica(diracSE, lfn) removeRes = self.dataManager.removeFile(lfn) # Check that the put was successful self.assert_(putRes['OK']) self.assert_(putRes['Value'].has_key('Successful')) self.assert_(putRes['Value']['Successful'].has_key(lfn)) self.assert_(putRes['Value']['Successful'][lfn]) # Check that the replica removal was successful self.assert_(removeReplicaRes['OK']) self.assert_(removeReplicaRes['Value'].has_key('Successful')) self.assert_(removeReplicaRes['Value']['Successful'].has_key(lfn)) self.assert_(removeReplicaRes['Value']['Successful'][lfn]) # Check that the removal was successful self.assert_(removeRes['OK']) self.assert_(removeRes['Value'].has_key('Successful')) self.assert_(removeRes['Value']['Successful'].has_key(lfn)) self.assert_(removeRes['Value']['Successful'][lfn]) def test_registerFile(self): lfn = '/lhcb/test/unit-test/ReplicaManager/registerFile/testFile.%s' % time.time( ) physicalFile = 'srm://host:port/srm/managerv2?SFN=/sa/path%s' % lfn fileSize = 10000 storageElementName = 'CERN-RAW' fileGuid = makeGuid() fileTuple = (lfn, physicalFile, fileSize, storageElementName, fileGuid) registerRes = self.dataManager.registerFile(fileTuple) removeCatalogReplicaRes = self.dataManager.removeCatalogReplica( storageElementName, lfn) removeFileRes = self.dataManager.removeFile(lfn) # Check that the file registration was done correctly self.assert_(registerRes['OK']) self.assert_(registerRes['Value'].has_key('Successful')) self.assert_(registerRes['Value']['Successful'].has_key(lfn)) self.assert_(registerRes['Value']['Successful'][lfn]) # Check that the replica removal was successful self.assert_(removeCatalogReplicaRes['OK']) self.assert_(removeCatalogReplicaRes['Value'].has_key('Successful')) self.assert_( removeCatalogReplicaRes['Value']['Successful'].has_key(lfn)) self.assert_(removeCatalogReplicaRes['Value']['Successful'][lfn]) # Check that the removal was successful self.assert_(removeFileRes['OK']) self.assert_(removeFileRes['Value'].has_key('Successful')) self.assert_(removeFileRes['Value']['Successful'].has_key(lfn)) self.assert_(removeFileRes['Value']['Successful'][lfn]) def test_registerReplica(self): print '\n\n#########################################################################\n\n\t\t\tRegister replica test\n' lfn = '/lhcb/test/unit-test/ReplicaManager/registerReplica/testFile.%s' % time.time( ) physicalFile = 'srm://host:port/srm/managerv2?SFN=/sa/path%s' % lfn fileSize = 10000 storageElementName = 'CERN-RAW' fileGuid = makeGuid() fileTuple = (lfn, physicalFile, fileSize, storageElementName, fileGuid) registerRes = self.dataManager.registerFile(fileTuple) seName = 'GRIDKA-RAW' replicaTuple = (lfn, physicalFile, seName) registerReplicaRes = self.dataManager.registerReplica(replicaTuple) removeCatalogReplicaRes1 = self.dataManager.removeCatalogReplica( storageElementName, lfn) removeCatalogReplicaRes2 = self.dataManager.removeCatalogReplica( seName, lfn) removeFileRes = self.dataManager.removeFile(lfn) # Check that the file registration was done correctly self.assert_(registerRes['OK']) self.assert_(registerRes['Value'].has_key('Successful')) self.assert_(registerRes['Value']['Successful'].has_key(lfn)) self.assert_(registerRes['Value']['Successful'][lfn]) # Check that the replica registration was successful self.assert_(registerReplicaRes['OK']) self.assert_(registerReplicaRes['Value'].has_key('Successful')) self.assert_(registerReplicaRes['Value']['Successful'].has_key(lfn)) self.assert_(registerReplicaRes['Value']['Successful'][lfn]) # Check that the replica removal was successful self.assert_(removeCatalogReplicaRes1['OK']) self.assert_(removeCatalogReplicaRes1['Value'].has_key('Successful')) self.assert_( removeCatalogReplicaRes1['Value']['Successful'].has_key(lfn)) self.assert_(removeCatalogReplicaRes1['Value']['Successful'][lfn]) # Check that the replica removal was successful self.assert_(removeCatalogReplicaRes2['OK']) self.assert_(removeCatalogReplicaRes2['Value'].has_key('Successful')) self.assert_( removeCatalogReplicaRes2['Value']['Successful'].has_key(lfn)) self.assert_(removeCatalogReplicaRes2['Value']['Successful'][lfn]) # Check that the removal was successful self.assert_(removeFileRes['OK']) self.assert_(removeFileRes['Value'].has_key('Successful')) self.assert_(removeFileRes['Value']['Successful'].has_key(lfn)) self.assert_(removeFileRes['Value']['Successful'][lfn]) def test_putAndRegisterGet(self): print '\n\n#########################################################################\n\n\t\t\tGet file test\n' lfn = '/lhcb/test/unit-test/ReplicaManager/putAndRegisterGet/testFile.%s' % time.time( ) diracSE = 'GRIDKA-RAW' putRes = self.dataManager.putAndRegister(lfn, self.fileName, diracSE) getRes = self.dataManager.getFile(lfn) removeRes = self.dataManager.removeFile(lfn) localFilePath = "%s/%s" % (os.getcwd(), os.path.basename(lfn)) if os.path.exists(localFilePath): os.remove(localFilePath) # Check that the put was successful self.assert_(putRes['OK']) self.assert_(putRes['Value'].has_key('Successful')) self.assert_(putRes['Value']['Successful'].has_key(lfn)) self.assert_(putRes['Value']['Successful'][lfn]) # Check that the replica removal was successful self.assert_(getRes['OK']) self.assert_(getRes['Value'].has_key('Successful')) self.assert_(getRes['Value']['Successful'].has_key(lfn)) self.assertEqual(getRes['Value']['Successful'][lfn], localFilePath) # Check that the removal was successful self.assert_(removeRes['OK']) self.assert_(removeRes['Value'].has_key('Successful')) self.assert_(removeRes['Value']['Successful'].has_key(lfn)) self.assert_(removeRes['Value']['Successful'][lfn])
inputFile.close() else: lfns.append( getDict( args ) ) from DIRAC.DataManagementSystem.Client.DataManager import DataManager from DIRAC import gLogger import DIRAC exitCode = 0 dm = DataManager() for lfn in lfns: if not os.path.exists( lfn['localfile'] ): gLogger.error( "File %s must exist locally" % lfn['localfile'] ) exitCode = 1 continue if not os.path.isfile( lfn['localfile'] ): gLogger.error( "%s is not a file" % lfn['localfile'] ) exitCode = 2 continue gLogger.notice( "\nUploading %s" % lfn['lfn'] ) res = dm.putAndRegister( lfn['lfn'], lfn['localfile'], lfn['SE'], lfn['guid'] ) if not res['OK']: exitCode = 3 gLogger.error( 'Error: failed to upload %s to %s' % ( lfn['lfn'], lfn['SE'] ) ) continue else: gLogger.notice( 'Successfully uploaded file to %s' % lfn['SE'] ) DIRAC.exit( exitCode )
inputFile.close() else: lfns.append( getDict( args ) ) from DIRAC.DataManagementSystem.Client.DataManager import DataManager from DIRAC import gLogger import DIRAC exitCode = 0 dm = DataManager() for lfn in lfns: if not os.path.exists( lfn['localfile'] ): gLogger.error( "File %s must exist locally" % lfn['localfile'] ) exitCode = 1 continue if not os.path.isfile( lfn['localfile'] ): gLogger.error( "%s is not a file" % lfn['localfile'] ) exitCode = 2 continue gLogger.notice( "\nUploading %s" % lfn['lfn'] ) res = dm.putAndRegister( lfn['lfn'], lfn['localfile'], lfn['SE'], lfn['guid'], overwrite = overwrite ) if not res['OK']: exitCode = 3 gLogger.error( 'Error: failed to upload %s to %s' % ( lfn['lfn'], lfn['SE'] ) ) continue else: gLogger.notice( 'Successfully uploaded file to %s' % lfn['SE'] ) DIRAC.exit( exitCode )
def main(): source_location = os.environ.get('JSUB_source_location', './') destination_dir = os.environ.get('JSUB_destination_dir', '') destination_dir_jobvar = os.environ.get( 'JSUB_dirac_upload_destination_dir_jobvar' ) # allow using $(jobvar) when defining path if destination_dir_jobvar is not None: destination_dir = destination_dir_jobvar destination_in_user_home = ( os.environ.get('JSUB_user_home', "True").upper() == "TRUE" ) #if true: relative path to cefs user home for upload destination; if false: absolute path files_to_upload = os.environ.get('JSUB_files_to_upload', '*') overwrite = (os.environ.get('JSUB_overwrite', 'False').upper() == 'TRUE') upload_file_jobvar = os.environ.get('JSUB_upload_file_jobvar') relocate_to_cwd = os.environ.get( 'JSUB_relocate_to_cwd' ) #if true, value of jobvar doesn't reflect dir name; file is under cwd instead se = os.environ.get('JSUB_SE', 'IHEP-STORM') upload_status = 0 # if upload_file_jobvar exists, need to reshape output setting to a standard one if upload_file_jobvar is not None: if relocate_to_cwd: #value of jobvar doesn't reflect dir name; file is under cwd instead upfile = os.environ.get('JSUB_' + upload_file_jobvar) source_location = './' files_to_upload = os.path.basename(upfile) if destination_in_user_home: destination_dir = os.path.dirname(upfile) else: files_to_upload = os.environ.get('JSUB_' + upload_file_jobvar) # destination_dir=os.path.dirname(upfile) # files_to_upload should be wildcards splitted by ',' flist_raw = files_to_upload.split(',') flist = [] for f in flist_raw: l = glob.glob(os.path.join(source_location, f)) l = [os.path.relpath(x, source_location) for x in l] flist += l flist = list(set(flist)) # remove repeated items userHome = getUserHome() if not userHome: gLogger.error('Failed to get user home') return 1 for f in flist: if destination_in_user_home: lfn = os.path.join(getUserHome(), destination_dir, f) else: lfn = os.path.join(destination_dir, f) fcc = FileCatalogClient() dm = DataManager(['FileCatalog']) print('registering file: LFN={0}, source={1}, SE={2}'.format( lfn, os.path.join(source_location, f), se)) res = dm.putAndRegister(lfn, os.path.join(source_location, f), se, overwrite=overwrite) if not res['OK']: gLogger.error( 'Failed to putAndRegister %s \nto %s \nwith message: %s' % (lfn, se, res['Message'])) upload_status = 1 elif res['Value']['Failed'].has_key(lfn): gLogger.error('Failed to putAndRegister %s to %s' % (lfn, se)) upload_status = 1 return upload_status
lfns.append(getDict(args)) from DIRAC.DataManagementSystem.Client.DataManager import DataManager from DIRAC import gLogger import DIRAC exitCode = 0 dm = DataManager() for lfn in lfns: if not os.path.exists(lfn['localfile']): gLogger.error("File %s must exist locally" % lfn['localfile']) exitCode = 1 continue if not os.path.isfile(lfn['localfile']): gLogger.error("%s is not a file" % lfn['localfile']) exitCode = 2 continue gLogger.notice("\nUploading %s" % lfn['lfn']) res = dm.putAndRegister(lfn['lfn'], lfn['localfile'], lfn['SE'], lfn['guid']) if not res['OK']: exitCode = 3 gLogger.error('Error: failed to upload %s to %s' % (lfn['lfn'], lfn['SE'])) continue else: gLogger.notice('Successfully uploaded file to %s' % lfn['SE']) DIRAC.exit(exitCode)