def main():
global overwrite
Script.registerSwitch("f", "force", "Force overwrite of existing file",
setOverwrite)
Script.parseCommandLine(ignoreErrors=True)
args = Script.getPositionalArgs()
if len(args) < 1 or len(args) > 4:
Script.showHelp(exitCode=1)
from DIRAC.DataManagementSystem.Client.DataManager import DataManager
from DIRAC import gLogger
import DIRAC
exitCode = 0
lfns = []
if len(args) == 1:
inputFileName = args[0]
if os.path.exists(inputFileName):
inputFile = open(inputFileName, "r")
for line in inputFile:
line = line.rstrip()
items = line.split()
items[0] = items[0].replace("LFN:", "").replace("lfn:", "")
lfns.append(getDict(items))
inputFile.close()
else:
gLogger.error("Error: LFN list '%s' missing." % inputFileName)
exitCode = 4
else:
lfns.append(getDict(args))
dm = DataManager()
for lfn in lfns:
if not os.path.exists(lfn["localfile"]):
gLogger.error("File %s must exist locally" % lfn["localfile"])
exitCode = 1
continue
if not os.path.isfile(lfn["localfile"]):
gLogger.error("%s is not a file" % lfn["localfile"])
exitCode = 2
continue
gLogger.notice("\nUploading %s" % lfn["lfn"])
res = dm.putAndRegister(lfn["lfn"],
lfn["localfile"],
lfn["SE"],
lfn["guid"],
overwrite=overwrite)
if not res["OK"]:
exitCode = 3
gLogger.error("Error: failed to upload %s to %s: %s" %
(lfn["lfn"], lfn["SE"], res))
continue
else:
gLogger.notice("Successfully uploaded file to %s" % lfn["SE"])
DIRAC.exit(exitCode)
def finish(self):
"""
after having set all the files, this one does all the job
@return:
"""
rc = 0
rm = DataManager()
for item in self.listFileStaged:
# print("SE '"+self.SE+"' == : '"+str(self.SE == "False")+"'")
if not self.SE:
self.log.info("No SE available for '" + item[0] + "'")
rc += 1
continue
else:
self.log.info("Trying to store '" + item[0] + "' in SE : '" + self.SE + "' ...")
result = rm.putAndRegister(item[1], item[0], self.SE)
if not result["OK"]:
self.log.info("ERROR %s" % (result["Message"]))
self.log.info("Wait 5sec before trying again...")
time.sleep(5)
result = rm.putAndRegister(item[1], item[0], self.SE)
if not result["OK"]:
self.log.info("ERROR %s" % (result["Message"]))
while not result["OK"]:
self.listSEs.remove(self.SE) # make sure not to pick the same SE again.
self.__pickRandomSE()
if not self.SE:
rc += 1
break
self.log.info("Trying with another SE : '" + self.SE + "' . In 5sec...")
time.sleep(5)
result = rm.putAndRegister(item[1], item[0], self.SE)
if result["OK"]:
self.log.info("file stored : '" + item[1] + "' in '" + self.SE + "'")
else:
self.log.error("ERROR : failed to store the file '" + item[1] + "' ...")
rc += 1
return rc
def _add_file(self, lfn, localfile, SE, guid=None):
dm = DataManager()
self._create_test_file()
if not os.path.exists(self.options['test_file']):
gLogger.error("File %s must exist locally" % localfile)
if not os.path.isfile(self.options['test_file']):
gLogger.error("%s is not a file" % localfile)
res = dm.putAndRegister(lfn, localfile, SE, guid)
if not res['OK']:
gLogger.error('Error: failed to upload %s to %s' % (lfn, SE))
return S_ERROR(res['Message'])
return S_OK(res['Value']['Successful'][lfn])
def addDataFiles(args):
fcc = FileCatalogClient()
dm = DataManager(['FileCatalog', 'TSCatalog'])
outputPath = args[0]
outputPattern = args[1]
outputSE = args[2]
metadata = args[3]
metadata = json.loads(metadata)
## Create path
res = fcc.createDirectory(outputPath)
if not res['OK']:
return res
## Set metadata
for key, value in metadata.items():
res = fcc.setMetadata(outputPath, {key: value})
# terrible hack
# if not res['OK']:
# return res
## Upload data files
all_files = glob.glob(outputPattern)
## Check that data files exist
if len(all_files) == 0:
return DIRAC.S_ERROR('No data files found')
for one_file in all_files:
lfn = os.path.join(outputPath, one_file)
msg = 'Try to upload local file: %s \nwith LFN: %s \nto %s' % (
one_file, lfn, outputSE)
DIRAC.gLogger.notice(msg)
res = dm.putAndRegister(lfn, one_file, outputSE)
## Check if failed
if not res['OK']:
DIRAC.gLogger.error(
'Failed to putAndRegister %s \nto %s \nwith message: %s' %
(lfn, outputSE, res['Message']))
return res
elif res['Value']['Failed'].has_key(lfn):
DIRAC.gLogger.error('Failed to putAndRegister %s to %s' %
(lfn, outputSE))
return res
return DIRAC.S_OK()
def addDataFiles(args):
fcc = FileCatalogClient()
dm = DataManager(['FileCatalog'])
outputPath = args[0]
outputPattern = args[1]
outputSE = args[2]
mode = args[3]
if mode == 'closest':
outputSE = getSiteSE(outputSE)
DIRAC.gLogger.error('OutputSE: %s' % (outputSE))
print "outputSE", outputSE
# Create path
res = fcc.createDirectory(outputPath)
if not res['OK']:
return res
# Upload data files
all_files = glob.glob(outputPattern)
# Check that data files exist
if len(all_files) == 0:
return DIRAC.S_ERROR('No data files found')
for one_file in all_files:
lfn = os.path.join(outputPath, one_file)
msg = 'Try to upload local file: %s \nwith LFN: %s \nto %s' % (
one_file, lfn, outputSE)
DIRAC.gLogger.notice(msg)
res = dm.putAndRegister(lfn, one_file, outputSE)
# Check if failed
if not res['OK']:
DIRAC.gLogger.error(
'Failed to putAndRegister %s \nto %s \nwith message: %s' %
(lfn, outputSE, res['Message']))
return res
elif res['Value']['Failed'].has_key(lfn):
DIRAC.gLogger.error('Failed to putAndRegister %s to %s' %
(lfn, outputSE))
return res
return DIRAC.S_OK()
class ReplicaManagerTestCase(unittest.TestCase):
""" Base class for the Replica Manager test cases
"""
def setUp(self):
self.dataManager = DataManager()
self.fileName = '/tmp/temporaryLocalFile'
file = open(self.fileName,'w')
file.write("%s" % time.time())
file.close()
def test_putAndRegister(self):
print '\n\n#########################################################################\n\n\t\t\tPut and register test\n'
lfn = '/lhcb/test/unit-test/ReplicaManager/putAndRegister/testFile.%s' % time.time()
diracSE = 'GRIDKA-RAW'
putRes = self.dataManager.putAndRegister(lfn, self.fileName, diracSE)
removeRes = self.dataManager.removeFile(lfn)
# Check that the put was successful
self.assert_(putRes['OK'])
self.assert_(putRes['Value'].has_key('Successful'))
self.assert_(putRes['Value']['Successful'].has_key(lfn))
self.assert_(putRes['Value']['Successful'][lfn])
# Check that the removal was successful
self.assert_(removeRes['OK'])
self.assert_(removeRes['Value'].has_key('Successful'))
self.assert_(removeRes['Value']['Successful'].has_key(lfn))
self.assert_(removeRes['Value']['Successful'][lfn])
def test_putAndRegisterReplicate(self):
print '\n\n#########################################################################\n\n\t\t\tReplication test\n'
lfn = '/lhcb/test/unit-test/ReplicaManager/putAndRegisterReplicate/testFile.%s' % time.time()
diracSE = 'GRIDKA-RAW'
putRes = self.dataManager.putAndRegister(lfn, self.fileName, diracSE)
replicateRes = self.dataManager.replicateAndRegister(lfn,'CNAF-DST') #,sourceSE='',destPath='',localCache='')
removeRes = self.dataManager.removeFile(lfn)
# Check that the put was successful
self.assert_(putRes['OK'])
self.assert_(putRes['Value'].has_key('Successful'))
self.assert_(putRes['Value']['Successful'].has_key(lfn))
self.assert_(putRes['Value']['Successful'][lfn])
# Check that the replicate was successful
self.assert_(replicateRes['OK'])
self.assert_(replicateRes['Value'].has_key('Successful'))
self.assert_(replicateRes['Value']['Successful'].has_key(lfn))
self.assert_(replicateRes['Value']['Successful'][lfn])
# Check that the removal was successful
self.assert_(removeRes['OK'])
self.assert_(removeRes['Value'].has_key('Successful'))
self.assert_(removeRes['Value']['Successful'].has_key(lfn))
self.assert_(removeRes['Value']['Successful'][lfn])
def test_putAndRegisterGetReplicaMetadata(self):
print '\n\n#########################################################################\n\n\t\t\tGet metadata test\n'
lfn = '/lhcb/test/unit-test/ReplicaManager/putAndRegisterGetReplicaMetadata/testFile.%s' % time.time()
diracSE = 'GRIDKA-RAW'
putRes = self.dataManager.putAndRegister(lfn, self.fileName, diracSE)
metadataRes = self.dataManager.getReplicaMetadata(lfn,diracSE)
removeRes = self.dataManager.removeFile(lfn)
# Check that the put was successful
self.assert_(putRes['OK'])
self.assert_(putRes['Value'].has_key('Successful'))
self.assert_(putRes['Value']['Successful'].has_key(lfn))
self.assert_(putRes['Value']['Successful'][lfn])
# Check that the metadata query was successful
self.assert_(metadataRes['OK'])
self.assert_(metadataRes['Value'].has_key('Successful'))
self.assert_(metadataRes['Value']['Successful'].has_key(lfn))
self.assert_(metadataRes['Value']['Successful'][lfn])
metadataDict = metadataRes['Value']['Successful'][lfn]
self.assert_(metadataDict.has_key('Cached'))
self.assert_(metadataDict.has_key('Migrated'))
self.assert_(metadataDict.has_key('Size'))
# Check that the removal was successful
self.assert_(removeRes['OK'])
self.assert_(removeRes['Value'].has_key('Successful'))
self.assert_(removeRes['Value']['Successful'].has_key(lfn))
self.assert_(removeRes['Value']['Successful'][lfn])
def test_putAndRegsiterGetAccessUrl(self):
print '\n\n#########################################################################\n\n\t\t\tGet Access Url test\n'
lfn = '/lhcb/test/unit-test/ReplicaManager/putAndRegisterGetAccessUrl/testFile.%s' % time.time()
diracSE = 'GRIDKA-RAW'
putRes = self.dataManager.putAndRegister(lfn, self.fileName, diracSE)
getAccessUrlRes = self.dataManager.getReplicaAccessUrl(lfn,diracSE)
print getAccessUrlRes
removeRes = self.dataManager.removeFile(lfn)
# Check that the put was successful
self.assert_(putRes['OK'])
self.assert_(putRes['Value'].has_key('Successful'))
self.assert_(putRes['Value']['Successful'].has_key(lfn))
self.assert_(putRes['Value']['Successful'][lfn])
# Check that the access url was successful
self.assert_(getAccessUrlRes['OK'])
self.assert_(getAccessUrlRes['Value'].has_key('Successful'))
self.assert_(getAccessUrlRes['Value']['Successful'].has_key(lfn))
self.assert_(getAccessUrlRes['Value']['Successful'][lfn])
# Check that the removal was successful
self.assert_(removeRes['OK'])
self.assert_(removeRes['Value'].has_key('Successful'))
self.assert_(removeRes['Value']['Successful'].has_key(lfn))
self.assert_(removeRes['Value']['Successful'][lfn])
def test_putAndRegisterRemoveReplica(self):
print '\n\n#########################################################################\n\n\t\t\tRemove replica test\n'
lfn = '/lhcb/test/unit-test/ReplicaManager/putAndRegisterRemoveReplica/testFile.%s' % time.time()
diracSE = 'GRIDKA-RAW'
putRes = self.dataManager.putAndRegister(lfn, self.fileName, diracSE)
removeReplicaRes = self.dataManager.removeReplica(diracSE,lfn)
removeRes = self.dataManager.removeFile(lfn)
# Check that the put was successful
self.assert_(putRes['OK'])
self.assert_(putRes['Value'].has_key('Successful'))
self.assert_(putRes['Value']['Successful'].has_key(lfn))
self.assert_(putRes['Value']['Successful'][lfn])
# Check that the replica removal was successful
self.assert_(removeReplicaRes['OK'])
self.assert_(removeReplicaRes['Value'].has_key('Successful'))
self.assert_(removeReplicaRes['Value']['Successful'].has_key(lfn))
self.assert_(removeReplicaRes['Value']['Successful'][lfn])
# Check that the removal was successful
self.assert_(removeRes['OK'])
self.assert_(removeRes['Value'].has_key('Successful'))
self.assert_(removeRes['Value']['Successful'].has_key(lfn))
self.assert_(removeRes['Value']['Successful'][lfn])
def test_registerFile(self):
lfn = '/lhcb/test/unit-test/ReplicaManager/registerFile/testFile.%s' % time.time()
physicalFile = 'srm://host:port/srm/managerv2?SFN=/sa/path%s' % lfn
fileSize = 10000
storageElementName = 'CERN-RAW'
fileGuid = makeGuid()
fileTuple = (lfn,physicalFile,fileSize,storageElementName,fileGuid)
registerRes = self.dataManager.registerFile(fileTuple)
removeCatalogReplicaRes = self.dataManager.removeCatalogReplica(storageElementName,lfn)
removeFileRes = self.dataManager.removeFile(lfn)
# Check that the file registration was done correctly
self.assert_(registerRes['OK'])
self.assert_(registerRes['Value'].has_key('Successful'))
self.assert_(registerRes['Value']['Successful'].has_key(lfn))
self.assert_(registerRes['Value']['Successful'][lfn])
# Check that the replica removal was successful
self.assert_(removeCatalogReplicaRes['OK'])
self.assert_(removeCatalogReplicaRes['Value'].has_key('Successful'))
self.assert_(removeCatalogReplicaRes['Value']['Successful'].has_key(lfn))
self.assert_(removeCatalogReplicaRes['Value']['Successful'][lfn])
# Check that the removal was successful
self.assert_(removeFileRes['OK'])
self.assert_(removeFileRes['Value'].has_key('Successful'))
self.assert_(removeFileRes['Value']['Successful'].has_key(lfn))
self.assert_(removeFileRes['Value']['Successful'][lfn])
def test_registerReplica(self):
print '\n\n#########################################################################\n\n\t\t\tRegister replica test\n'
lfn = '/lhcb/test/unit-test/ReplicaManager/registerReplica/testFile.%s' % time.time()
physicalFile = 'srm://host:port/srm/managerv2?SFN=/sa/path%s' % lfn
fileSize = 10000
storageElementName = 'CERN-RAW'
fileGuid = makeGuid()
fileTuple = (lfn,physicalFile,fileSize,storageElementName,fileGuid)
registerRes = self.dataManager.registerFile(fileTuple)
seName = 'GRIDKA-RAW'
replicaTuple = (lfn,physicalFile,seName)
registerReplicaRes = self.dataManager.registerReplica(replicaTuple)
removeCatalogReplicaRes1 = self.dataManager.removeCatalogReplica(storageElementName,lfn)
removeCatalogReplicaRes2 = self.dataManager.removeCatalogReplica(seName,lfn)
removeFileRes = self.dataManager.removeFile(lfn)
# Check that the file registration was done correctly
self.assert_(registerRes['OK'])
self.assert_(registerRes['Value'].has_key('Successful'))
self.assert_(registerRes['Value']['Successful'].has_key(lfn))
self.assert_(registerRes['Value']['Successful'][lfn])
# Check that the replica registration was successful
self.assert_(registerReplicaRes['OK'])
self.assert_(registerReplicaRes['Value'].has_key('Successful'))
self.assert_(registerReplicaRes['Value']['Successful'].has_key(lfn))
self.assert_(registerReplicaRes['Value']['Successful'][lfn])
# Check that the replica removal was successful
self.assert_(removeCatalogReplicaRes1['OK'])
self.assert_(removeCatalogReplicaRes1['Value'].has_key('Successful'))
self.assert_(removeCatalogReplicaRes1['Value']['Successful'].has_key(lfn))
self.assert_(removeCatalogReplicaRes1['Value']['Successful'][lfn])
# Check that the replica removal was successful
self.assert_(removeCatalogReplicaRes2['OK'])
self.assert_(removeCatalogReplicaRes2['Value'].has_key('Successful'))
self.assert_(removeCatalogReplicaRes2['Value']['Successful'].has_key(lfn))
self.assert_(removeCatalogReplicaRes2['Value']['Successful'][lfn])
# Check that the removal was successful
self.assert_(removeFileRes['OK'])
self.assert_(removeFileRes['Value'].has_key('Successful'))
self.assert_(removeFileRes['Value']['Successful'].has_key(lfn))
self.assert_(removeFileRes['Value']['Successful'][lfn])
def test_putAndRegisterGet(self):
print '\n\n#########################################################################\n\n\t\t\tGet file test\n'
lfn = '/lhcb/test/unit-test/ReplicaManager/putAndRegisterGet/testFile.%s' % time.time()
diracSE = 'GRIDKA-RAW'
putRes = self.dataManager.putAndRegister(lfn, self.fileName, diracSE)
getRes = self.dataManager.getFile(lfn)
removeRes = self.dataManager.removeFile(lfn)
localFilePath = "%s/%s" % (os.getcwd(),os.path.basename(lfn))
if os.path.exists(localFilePath):
os.remove(localFilePath)
# Check that the put was successful
self.assert_(putRes['OK'])
self.assert_(putRes['Value'].has_key('Successful'))
self.assert_(putRes['Value']['Successful'].has_key(lfn))
self.assert_(putRes['Value']['Successful'][lfn])
# Check that the replica removal was successful
self.assert_(getRes['OK'])
self.assert_(getRes['Value'].has_key('Successful'))
self.assert_(getRes['Value']['Successful'].has_key(lfn))
self.assertEqual(getRes['Value']['Successful'][lfn],localFilePath)
# Check that the removal was successful
self.assert_(removeRes['OK'])
self.assert_(removeRes['Value'].has_key('Successful'))
self.assert_(removeRes['Value']['Successful'].has_key(lfn))
self.assert_(removeRes['Value']['Successful'][lfn])
from DIRAC import S_OK, S_ERROR, gLogger, exit
from DIRAC.DataManagementSystem.Client.DataManager import DataManager
lfn = args[0]
pfn = args[1]
se = args[2]
exit_code = 0
log = ''
dm = DataManager()
start = time.time()
result = dm.removeFile(lfn)
result = dm.putAndRegister(lfn, pfn, se)
uploadTime = time.time() - start
if result['OK']:
log += 'Succeed to upload file to SE %s.\n' % se
log += 'Upload Time : %ss\n' % uploadTime
start = time.time()
result = dm.getReplica(lfn, se, tempfile.gettempdir())
downloadTime = time.time() - start
if result['OK']:
log += 'Succeed to download file from SE %s.\n' % se
log += 'Download Time : %ss\n' % downloadTime
else:
exit_code = 1
log += 'Failed to download file from SE %s : %s\n' % (
se, result['Message'])
class SETest( TestBase ):
"""
SETest is used to test the availability of SE.
"""
def __init__( self, args = None, apis = None ):
super( SETest, self ).__init__( args, apis )
self.__lfnPath = '/bes/user/z/zhaoxh/'
self.__testFile = 'test.dat'
self.__localPath = '/tmp/'
if 'DataManager' in self.apis:
self.dm = self.apis[ 'DataManager' ]
else:
self.dm = DataManager()
def doTest( self, elementDict ):
"""
Test upload and download for specified SE.
"""
elementName = elementDict[ 'ElementName' ]
testFilePath = self.__localPath + self.__testFile
if not os.path.exists( testFilePath ) or not os.path.isfile( testFilePath ):
f = open( testFilePath, 'w' )
f.write( 'hello' )
f.close()
status = 'OK'
log = ''
lfnPath = self.__lfnPath + elementName + '-' + self.__testFile
submissionTime = datetime.utcnow().replace( microsecond = 0 )
LOCK.acquire()
start = time.time()
result = self.dm.putAndRegister( lfnPath, testFilePath, elementName )
uploadTime = time.time() - start
if result[ 'OK' ]:
log += 'Succeed to upload file to SE %s.\n' % elementName
log += 'Upload Time : %ss\n' % uploadTime
start = time.time()
result = self.dm.getReplica( lfnPath, elementName, self.__localPath )
downloadTime = time.time() - start
if result[ 'OK' ]:
log += 'Succeed to download file from SE %s.\n' % elementName
log += 'Download Time : %ss\n' % downloadTime
else:
status = 'Bad'
log += 'Failed to download file from SE %s : %s\n' % ( elementName, result[ 'Message' ] )
result = self.dm.removeFile( lfnPath )
if result[ 'OK' ]:
log += 'Succeed to delete file from SE %s.\n' % elementName
else:
log += 'Faile to delete file from SE %s : %s\n' % ( elementName, result[ 'Message' ] )
else:
status = 'Bad'
log += 'Failed to upload file to SE %s : %s\n' % ( elementName, result[ 'Message' ] )
LOCK.release()
completionTime = datetime.utcnow().replace( microsecond = 0 )
applicationTime = ( completionTime - submissionTime ).total_seconds()
result = { 'Result' : { 'Status' : status,
'Log' : log,
'SubmissionTime' : submissionTime,
'CompletionTime' : completionTime,
'ApplicationTime' : applicationTime },
'Finish' : True }
localFile = self.__localPath + elementName +'-' + self.__testFile
if os.path.exists( localFile ) and os.path.isfile( localFile ):
os.remove( localFile )
return S_OK( result )
class Prod3DataManager(object) :
""" Manage data and meta-data
"""
def __init__(self,catalogs=['DIRACFileCatalog']):
""" Constructor
"""
self.setupCatalogClient( catalogs )
self.printCatalogConfig( catalogs )
self.setupDataManagerClient( catalogs )
def setupCatalogClient( self, catalogs ):
""" Init FileCatalog client
Ideally we would like to use only FileCatalog but it doesn't work for setMetadata
because the returned value has a wrong format. So use of FileCatalogClient instead
"""
self.fc = FileCatalog( catalogs )
self.fcc = FileCatalogClient()
def printCatalogConfig( self, catalogs ):
""" Dumps catalog configuration
"""
for catalog in catalogs:
res = self.fc._getCatalogConfigDetails( catalog )
DIRAC.gLogger.info( 'CatalogConfigDetails:', res )
def setupDataManagerClient( self, catalogs ):
""" Init DataManager client
"""
self.dm = DataManager( catalogs )
def _getSEList( self, SEType = 'ProductionOutputs', DataType = 'SimtelProd' ):
""" get from CS the list of available SE for data upload
"""
opsHelper = Operations()
optionName = os.path.join( SEType, DataType )
SEList = opsHelper.getValue( optionName , [] )
SEList = List.randomize( SEList )
DIRAC.gLogger.notice( 'List of %s SE: %s ' % ( SEType, SEList ) )
# # Check if the local SE is in the list. If yes try it first by reversing list order
localSEList = []
res = getSEsForSite( DIRAC.siteName() )
if res['OK']:
localSEList = res['Value']
retainedlocalSEList = []
for localSE in localSEList:
if localSE in SEList:
DIRAC.gLogger.notice( 'The local Storage Element is an available SE: ', localSE )
retainedlocalSEList.append( localSE )
SEList.remove( localSE )
SEList = retainedlocalSEList + SEList
if len( SEList ) == 0:
return DIRAC.S_ERROR( 'Error in building SEList' )
return DIRAC.S_OK( SEList )
def _checkemptydir( self, path ):
""" check that the directory is not empty
"""
if len ( glob.glob( path ) ) == 0:
error = 'Empty directory: %s' % path
return DIRAC.S_ERROR( error )
else:
return DIRAC.S_OK()
def _getRunPath( self, filemetadata ):
""" format path to string with 1 digit precision
run_number is taken from filemetadata
filemetadata can be a dict or the run_number itself
"""
if type( filemetadata ) == type( dict() ):
run_number = int( filemetadata['runNumber'] )
else:
run_number = int( filemetadata )
run_numberMod = run_number % 1000
runpath = '%03dxxx' % ( ( run_number - run_numberMod ) / 1000 )
return runpath
def _formatPath( self, path ):
""" format path to string with 1 digit precision
"""
if type( path ) == float or type( path ) == int:
path = '%.1f' % path
return str( path )
def createTarLogFiles ( self, inputpath, tarname ):
""" create tar of log and histogram files
"""
tarmode = 'w:gz'
tar = tarfile.open( tarname, tarmode )
for subdir in ['Log/*', 'Histograms/*']:
logdir = os.path.join( inputpath, subdir )
res = self._checkemptydir( logdir )
if not res['OK']:
return res
for localfile in glob.glob( logdir ):
tar.add( localfile, arcname = localfile.split( '/' )[-1] )
tar.close()
return DIRAC.S_OK()
def createMDStructure( self, metadata, metadatafield, basepath, program_category ):
""" create meta data structure
"""
# ## Add metadata fields to the DFC
mdfield = json.loads( metadatafield )
for key, value in mdfield.items():
res = self.fc.addMetadataField( key, value )
if not res['OK']:
return res
# ## Create the directory structure
md = json.loads( metadata , object_pairs_hook = collections.OrderedDict )
path = basepath
process_program = program_category + '_prog'
for key, value in collections.OrderedDict( ( k, md[k] ) for k in ( 'site', 'particle', process_program ) if k in md ).items():
path = os.path.join( path, self._formatPath( value ) )
res = self.fc.createDirectory( path )
if not res['OK']:
return res
# Set directory metadata for each subdir: 'site', 'particle', 'process_program'
res = self.fcc.setMetadata( path, {key:value} )
if not res['OK']:
return res
# Create the TransformationID subdir and set MD
# ## Get the TransformationID
if os.environ.has_key( 'JOBID' ):
jobID = os.environ['JOBID']
dirac = Dirac()
res = dirac.getJobJDL( jobID )
TransformationID = '0000'
if res['Value'].has_key( 'TransformationID' ):
TransformationID = res['Value']['TransformationID']
path = os.path.join( path, TransformationID )
res = self.fc.createDirectory( path )
if not res['OK']:
return res
process_program_version = process_program + '_version'
res = self.fcc.setMetadata( path, dict( ( k, md[k] ) for k in ( 'phiP', 'thetaP', 'array_layout', process_program_version ) if k in md ) )
if not res['OK']:
return res
# Create the Data and Log subdirs and set MD
Transformation_path = path
for subdir in ['Data', 'Log']:
path = os.path.join( Transformation_path, subdir )
res = self.fc.createDirectory( path )
if not res['OK']:
return res
# Set metadata if not already defined
res = self.fcc.getDirectoryUserMetadata( path )
if not res['OK']:
return res
if 'outputType' not in res['Value']:
res = self.fcc.setMetadata( path, {'outputType':subdir} )
if not res['OK']:
return res
# MD for the Data directory - data_level and configuration_id
path = os.path.join(Transformation_path, 'Data')
# Set metadata if not already defined
res = self.fcc.getDirectoryUserMetadata( path )
if not res['OK']:
return res
if 'data_level' and 'configuration_id' not in res['Value']:
res = self.fcc.setMetadata(path, {'data_level': md['data_level'],
'configuration_id': md['configuration_id']
})
if not res['OK']:
return res
return DIRAC.S_OK( Transformation_path )
def putAndRegister( self, lfn, localfile, filemetadata, DataType = 'SimtelProd' ):
""" put and register one file and set metadata
"""
# ## Get the list of Production SE
res = self._getSEList( 'ProductionOutputs', DataType )
if res['OK']:
ProductionSEList = res['Value']
else:
return res
# ## Get the list of Failover SE
res = self._getSEList( 'ProductionOutputsFailover', DataType )
if res['OK']:
FailoverSEList = res['Value']
else:
return res
# ## Upload file to a Production SE
res = self._putAndRegisterToSEList( lfn, localfile, ProductionSEList )
if not res['OK']:
DIRAC.gLogger.error( 'Failed to upload file to any Production SE: %s' % ProductionSEList )
# ## Upload file to a Failover SE
res = self._putAndRegisterToSEList( lfn, localfile, FailoverSEList )
if not res['OK']:
return DIRAC.S_ERROR( 'Failed to upload file to any Failover SE: %s' % FailoverSEList )
# ## Set file metadata: jobID, subarray, sct
if res['OK']:
fmd = json.loads( filemetadata )
if os.environ.has_key( 'JOBID' ):
fmd.update( {'jobID':os.environ['JOBID']} )
filename = os.path.basename( localfile )
# set subarray and sct md from filename
p = re.compile( 'subarray-\d+' )
if p.search( filename ) != None:
subarray = p.search( filename ).group()
fmd.update( {'subarray':subarray} )
sct = 'False'
p = re.compile( 'nosct' )
psct = re.compile( 'sct' )
if p.search( filename ) == None and psct.search( filename ) != None:
sct = 'True'
# ## Set sct flag only for production data
res = self.fcc.getFileUserMetadata( lfn )
if DataType == 'SimtelProd' and res['Value']['outputType'] == 'Data':
fmd.update( {'sct':sct} )
res = self.fc.setMetadata( lfn, fmd )
if not res['OK']:
return res
return DIRAC.S_OK()
def _putAndRegisterToSEList( self, lfn, localfile, SEList ):
""" put and register one file to one SE in the SEList
"""
# ## Try to upload file to a SE in the list
for SE in SEList:
msg = 'Try to upload local file: %s \nwith LFN: %s \nto %s' % ( localfile, lfn, SE )
DIRAC.gLogger.notice( msg )
res = self.dm.putAndRegister( lfn, localfile, SE )
DIRAC.gLogger.notice(res)
# ## check if failed
if not res['OK']:
DIRAC.gLogger.error( 'Failed to putAndRegister %s \nto %s \nwith message: %s' % ( lfn, SE, res['Message'] ) )
DIRAC.gLogger.notice('Trying to clean up %s' % lfn)
res = self.dm.removeFile(lfn)
if res['OK']:
DIRAC.gLogger.notice('Successfully removed %s \n that was not supposed to have been uploaded successfully' % lfn)
continue
elif res['Value']['Failed'].has_key( lfn ):
DIRAC.gLogger.error( 'Failed to putAndRegister %s to %s' % ( lfn, SE ) )
DIRAC.gLogger.notice('Trying to clean up %s' % lfn)
res = self.dm.removeFile(lfn)
if res['OK']:
DIRAC.gLogger.notice('Successfully removed %s \n that was not supposed to have been uploaded successfully' % lfn)
continue
else:
return DIRAC.S_OK()
return DIRAC.S_ERROR()
def cleanLocalFiles ( self, datadir, pattern = '*' ):
""" remove files matching pattern in datadir
"""
for localfile in glob.glob( os.path.join( datadir, pattern ) ):
DIRAC.gLogger.notice( 'Removing local file: ', localfile )
os.remove( localfile )
return DIRAC.S_OK()
from DIRAC import S_OK, S_ERROR, gLogger, exit
from DIRAC.DataManagementSystem.Client.DataManager import DataManager
lfn = args[0]
pfn = args[1]
se = args[2]
exit_code = 0
log = ''
dm = DataManager()
start = time.time()
result = dm.removeFile( lfn )
result = dm.putAndRegister( lfn, pfn, se )
uploadTime = time.time() - start
if result[ 'OK' ]:
log += 'Succeed to upload file to SE %s.\n' % se
log += 'Upload Time : %ss\n' % uploadTime
start = time.time()
result = dm.getReplica( lfn, se, tempfile.gettempdir() )
downloadTime = time.time() - start
if result[ 'OK' ]:
log += 'Succeed to download file from SE %s.\n' % se
log += 'Download Time : %ss\n' % downloadTime
else:
exit_code = 1
log += 'Failed to download file from SE %s : %s\n' % ( se, result[ 'Message' ] )
def main():
dm = DataManager()
fileTupleBuffer = []
res = getProxyInfo( False, False )
if not res['OK']:
gLogger.error( "Failed to get client proxy information.", res['Message'] )
DIRAC.exit( 2 )
proxyInfo = res['Value']
if proxyInfo['secondsLeft'] == 0:
gLogger.error( "Proxy expired" )
DIRAC.exit( 2 )
username = proxyInfo['username']
vo = ''
if 'group' in proxyInfo:
vo = getVOForGroup( proxyInfo['group'] )
counter = 0
for f in files:
counter += 1
local_f=f
if not f.startswith('/cefs'):
# gLogger.error('File must be under "/cefs"')
# continue
#if the file to reg is not under /cefs, use put and register
folder_name=os.path.basename(os.path.dirname(f))
# lfn = '/cepc/user/%s/%s/jsub/'%(username[0],username) + folder_name + '/' + os.path.basename(f)
lfn = '/cepc/user/%s/%s/jsub/'%(username[0],username) + folder_name + '/' + os.path.basename(f)
# dirname = os.path.dirname(local_f)
# os.system('mkdir -p %s'%(dirname))
# os.system('cp %s %s' %(f,local_f))
do_put_and_register=True
else:
lfn = '/cepc/lustre-ro' + os.path.abspath(f)
do_put_and_register=False
result = fcc.isFile(lfn)
if result['OK'] and lfn in result['Value']['Successful'] and result['Value']['Successful'][lfn]:
continue
size = os.path.getsize(f)
adler32 = fileAdler(f)
guid = makeGuid()
fileTuple = (lfn, local_f, size, _se, guid, adler32)
fileTupleBuffer.append(fileTuple)
gLogger.debug('Register to lfn: %s' % lfn)
gLogger.debug('fileTuple: %s' % (fileTuple,))
if len(fileTupleBuffer) >= _bufferSize:
if do_put_and_register:
result = dm.putAndRegister(lfn, local_f, _se, guid, overwrite=overwrite)
else:
result = dm.registerFile(fileTupleBuffer)
print('register result', result)
# if not result['OK']:
# gLogger.error('Register file failed')
# return 1
del fileTupleBuffer[:]
gLogger.debug('%s files registered' % counter)
if fileTupleBuffer:
if do_put_and_register:
result = dm.putAndRegister(lfn, local_f, _se, guid, overwrite=overwrite)
else:
result = dm.registerFile(fileTupleBuffer)
print('register result', result)
# if not result['OK']:
# gLogger.error('Register file failed')
# return 1
del fileTupleBuffer[:]
gLogger.info('Totally %s files registered' % counter)
return 0
def _uploadGenFiles():
"""uploads the generator files"""
clip = _Params()
clip.registerSwitches()
Script.parseCommandLine()
from DIRAC import gLogger, exit as dexit
if not clip.dir:
gLogger.error('You need to set the path')
Script.showHelp()
dexit(1)
if not clip.storageElement:
gLogger.error('You need a storage element')
Script.showHelp()
dexit(1)
for key in MANDATORY_KEYS:
if key not in clip.fmeta:
gLogger.error("Not all mandatory meta data defined, please check and add key: ", key)
Script.showHelp()
dexit(1)
#resolve the inout files
flist = []
if os.path.isdir(clip.dir):
flistd = os.listdir(clip.dir)
for filename in flistd:
if filename.count(".stdhep"):
flist.append( os.path.join(clip.dir, filename) )
elif os.path.isfile(clip.dir):
flist.append(clip.dir)
else:
gLogger.error("%s is not a file nor a directory" % clip.dir)
dexit(1)
gLogger.notice("Will eventually upload %s file(s)" % len(flist))
from DIRAC.Core.Utilities.PromptUser import promptUser
from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
basepath = Operations().getValue('Production/ILC_ILD/BasePath','')
if not basepath:
gLogger.error('Failed to contact CS, please try again')
dexit(1)
basepath = "/".join(basepath.split("/")[:-2])+"/" #need to get rid of the ild/ part at the end
finalpath = os.path.join(basepath, 'generated', clip.energy+"-"+clip.machineParams, clip.evtclass, str(clip.fmeta['GenProcessID']))
gLogger.notice("Will upload the file(s) under %s" % finalpath)
if not clip.force:
res = promptUser('Continue?', ['y','n'], 'n')
if not res['OK']:
gLogger.error(res['Message'])
dexit(1)
if not res['Value'].lower()=='y':
dexit(0)
dirmeta = []
dirmeta.append({'path':os.path.join(basepath, 'generated'), 'meta':{'Datatype':'gen'}})
dirmeta.append({'path':os.path.join(basepath, 'generated', clip.energy+"-"+clip.machineParams), 'meta':{'Energy':clip.energy, 'MachineParams':clip.machineParams}})
dirmeta.append({'path':os.path.join(basepath, 'generated', clip.energy+"-"+clip.machineParams, clip.evtclass), 'meta':{'EvtClass':clip.evtclass }})
dirmeta.append({'path':finalpath, 'meta': {'EvtType':clip.evttype ,'Luminosity':clip.lumi, 'ProcessID': clip.fmeta['GenProcessID']} })
final_fname_base = 'E'+clip.energy+"-"+clip.machineParams+".P"+clip.fmeta['GenProcessName']+".G"+clip.fmeta['ProgramNameVersion'] + "."+clip.particle1+clip.pol1+"."+clip.particle2+clip.pol2+".I"+str(clip.fmeta['GenProcessID'])
gLogger.notice("Final file name(s) will be %s where XX will be replaced by file number, and ext by the input file extension" % (final_fname_base+".XX.ext") )
if not clip.force:
res = promptUser('Continue?', ['y','n'], 'n')
if not res['OK']:
gLogger.error(res['Message'])
dexit(1)
if not res['Value'].lower()=='y':
dexit(0)
from DIRAC.DataManagementSystem.Client.DataManager import DataManager
from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient
fc = FileCatalogClient()
for pathdict in dirmeta:
res = fc.createDirectory(pathdict['path'])
if not res['OK']:
gLogger.error("Could not create this directory in FileCatalog, abort:", pathdict['path'] )
dexit(0)
res = fc.setMetadata(pathdict['path'], pathdict['meta'])
if not res['OK']:
gLogger.error( "Failed to set meta data %s to %s\n" %(pathdict['meta'], pathdict['path']), res['Message'] )
datMan = DataManager()
for filename in flist:
fnum = filename.split(".")[-2]
fext = filename.split(".")[-1]
final_fname = final_fname_base + '.' + fnum + "." + fext
gLogger.notice("Uploading %s to" % filename, finalpath+"/"+final_fname)
if not clip.force:
res = promptUser('Continue?', ['y','n'], 'n')
if not res['OK']:
gLogger.error(res['Message'])
break
if not res['Value'].lower()=='y':
break
res = datMan.putAndRegister(finalpath+"/"+final_fname, filename, clip.storageElement)
if not res['OK']:
gLogger.error("Failed to upload %s:" % filename, res['Message'])
continue
res = fc.setMetadata(finalpath+"/"+final_fname, clip.fmeta)
if not res['OK']:
gLogger.error("Failed setting the metadata to %s:" % filename, res['Message'])
dexit(0)
exitCode = 4
else:
lfns.append(getDict(args))
dm = DataManager()
for lfn in lfns:
if not os.path.exists(lfn['localfile']):
gLogger.error("File %s must exist locally" % lfn['localfile'])
exitCode = 1
continue
if not os.path.isfile(lfn['localfile']):
gLogger.error("%s is not a file" % lfn['localfile'])
exitCode = 2
continue
gLogger.notice("\nUploading %s" % lfn['lfn'])
res = dm.putAndRegister(lfn['lfn'],
lfn['localfile'],
lfn['SE'],
lfn['guid'],
overwrite=overwrite)
if not res['OK']:
exitCode = 3
gLogger.error('Error: failed to upload %s to %s' %
(lfn['lfn'], lfn['SE']))
continue
else:
gLogger.notice('Successfully uploaded file to %s' % lfn['SE'])
DIRAC.exit(exitCode)
class ReplicaManagerTestCase(unittest.TestCase):
""" Base class for the Replica Manager test cases
"""
def setUp(self):
self.dataManager = DataManager()
self.fileName = '/tmp/temporaryLocalFile'
file = open(self.fileName, 'w')
file.write("%s" % time.time())
file.close()
def test_putAndRegister(self):
print '\n\n#########################################################################\n\n\t\t\tPut and register test\n'
lfn = '/lhcb/test/unit-test/ReplicaManager/putAndRegister/testFile.%s' % time.time(
)
diracSE = 'GRIDKA-RAW'
putRes = self.dataManager.putAndRegister(lfn, self.fileName, diracSE)
removeRes = self.dataManager.removeFile(lfn)
# Check that the put was successful
self.assert_(putRes['OK'])
self.assert_(putRes['Value'].has_key('Successful'))
self.assert_(putRes['Value']['Successful'].has_key(lfn))
self.assert_(putRes['Value']['Successful'][lfn])
# Check that the removal was successful
self.assert_(removeRes['OK'])
self.assert_(removeRes['Value'].has_key('Successful'))
self.assert_(removeRes['Value']['Successful'].has_key(lfn))
self.assert_(removeRes['Value']['Successful'][lfn])
def test_putAndRegisterReplicate(self):
print '\n\n#########################################################################\n\n\t\t\tReplication test\n'
lfn = '/lhcb/test/unit-test/ReplicaManager/putAndRegisterReplicate/testFile.%s' % time.time(
)
diracSE = 'GRIDKA-RAW'
putRes = self.dataManager.putAndRegister(lfn, self.fileName, diracSE)
replicateRes = self.dataManager.replicateAndRegister(
lfn, 'CNAF-DST') #,sourceSE='',destPath='',localCache='')
removeRes = self.dataManager.removeFile(lfn)
# Check that the put was successful
self.assert_(putRes['OK'])
self.assert_(putRes['Value'].has_key('Successful'))
self.assert_(putRes['Value']['Successful'].has_key(lfn))
self.assert_(putRes['Value']['Successful'][lfn])
# Check that the replicate was successful
self.assert_(replicateRes['OK'])
self.assert_(replicateRes['Value'].has_key('Successful'))
self.assert_(replicateRes['Value']['Successful'].has_key(lfn))
self.assert_(replicateRes['Value']['Successful'][lfn])
# Check that the removal was successful
self.assert_(removeRes['OK'])
self.assert_(removeRes['Value'].has_key('Successful'))
self.assert_(removeRes['Value']['Successful'].has_key(lfn))
self.assert_(removeRes['Value']['Successful'][lfn])
def test_putAndRegisterGetReplicaMetadata(self):
print '\n\n#########################################################################\n\n\t\t\tGet metadata test\n'
lfn = '/lhcb/test/unit-test/ReplicaManager/putAndRegisterGetReplicaMetadata/testFile.%s' % time.time(
)
diracSE = 'GRIDKA-RAW'
putRes = self.dataManager.putAndRegister(lfn, self.fileName, diracSE)
metadataRes = self.dataManager.getReplicaMetadata(lfn, diracSE)
removeRes = self.dataManager.removeFile(lfn)
# Check that the put was successful
self.assert_(putRes['OK'])
self.assert_(putRes['Value'].has_key('Successful'))
self.assert_(putRes['Value']['Successful'].has_key(lfn))
self.assert_(putRes['Value']['Successful'][lfn])
# Check that the metadata query was successful
self.assert_(metadataRes['OK'])
self.assert_(metadataRes['Value'].has_key('Successful'))
self.assert_(metadataRes['Value']['Successful'].has_key(lfn))
self.assert_(metadataRes['Value']['Successful'][lfn])
metadataDict = metadataRes['Value']['Successful'][lfn]
self.assert_(metadataDict.has_key('Cached'))
self.assert_(metadataDict.has_key('Migrated'))
self.assert_(metadataDict.has_key('Size'))
# Check that the removal was successful
self.assert_(removeRes['OK'])
self.assert_(removeRes['Value'].has_key('Successful'))
self.assert_(removeRes['Value']['Successful'].has_key(lfn))
self.assert_(removeRes['Value']['Successful'][lfn])
def test_putAndRegsiterGetAccessUrl(self):
print '\n\n#########################################################################\n\n\t\t\tGet Access Url test\n'
lfn = '/lhcb/test/unit-test/ReplicaManager/putAndRegisterGetAccessUrl/testFile.%s' % time.time(
)
diracSE = 'GRIDKA-RAW'
putRes = self.dataManager.putAndRegister(lfn, self.fileName, diracSE)
getAccessUrlRes = self.dataManager.getReplicaAccessUrl(lfn, diracSE)
print getAccessUrlRes
removeRes = self.dataManager.removeFile(lfn)
# Check that the put was successful
self.assert_(putRes['OK'])
self.assert_(putRes['Value'].has_key('Successful'))
self.assert_(putRes['Value']['Successful'].has_key(lfn))
self.assert_(putRes['Value']['Successful'][lfn])
# Check that the access url was successful
self.assert_(getAccessUrlRes['OK'])
self.assert_(getAccessUrlRes['Value'].has_key('Successful'))
self.assert_(getAccessUrlRes['Value']['Successful'].has_key(lfn))
self.assert_(getAccessUrlRes['Value']['Successful'][lfn])
# Check that the removal was successful
self.assert_(removeRes['OK'])
self.assert_(removeRes['Value'].has_key('Successful'))
self.assert_(removeRes['Value']['Successful'].has_key(lfn))
self.assert_(removeRes['Value']['Successful'][lfn])
def test_putAndRegisterRemoveReplica(self):
print '\n\n#########################################################################\n\n\t\t\tRemove replica test\n'
lfn = '/lhcb/test/unit-test/ReplicaManager/putAndRegisterRemoveReplica/testFile.%s' % time.time(
)
diracSE = 'GRIDKA-RAW'
putRes = self.dataManager.putAndRegister(lfn, self.fileName, diracSE)
removeReplicaRes = self.dataManager.removeReplica(diracSE, lfn)
removeRes = self.dataManager.removeFile(lfn)
# Check that the put was successful
self.assert_(putRes['OK'])
self.assert_(putRes['Value'].has_key('Successful'))
self.assert_(putRes['Value']['Successful'].has_key(lfn))
self.assert_(putRes['Value']['Successful'][lfn])
# Check that the replica removal was successful
self.assert_(removeReplicaRes['OK'])
self.assert_(removeReplicaRes['Value'].has_key('Successful'))
self.assert_(removeReplicaRes['Value']['Successful'].has_key(lfn))
self.assert_(removeReplicaRes['Value']['Successful'][lfn])
# Check that the removal was successful
self.assert_(removeRes['OK'])
self.assert_(removeRes['Value'].has_key('Successful'))
self.assert_(removeRes['Value']['Successful'].has_key(lfn))
self.assert_(removeRes['Value']['Successful'][lfn])
def test_registerFile(self):
lfn = '/lhcb/test/unit-test/ReplicaManager/registerFile/testFile.%s' % time.time(
)
physicalFile = 'srm://host:port/srm/managerv2?SFN=/sa/path%s' % lfn
fileSize = 10000
storageElementName = 'CERN-RAW'
fileGuid = makeGuid()
fileTuple = (lfn, physicalFile, fileSize, storageElementName, fileGuid)
registerRes = self.dataManager.registerFile(fileTuple)
removeCatalogReplicaRes = self.dataManager.removeCatalogReplica(
storageElementName, lfn)
removeFileRes = self.dataManager.removeFile(lfn)
# Check that the file registration was done correctly
self.assert_(registerRes['OK'])
self.assert_(registerRes['Value'].has_key('Successful'))
self.assert_(registerRes['Value']['Successful'].has_key(lfn))
self.assert_(registerRes['Value']['Successful'][lfn])
# Check that the replica removal was successful
self.assert_(removeCatalogReplicaRes['OK'])
self.assert_(removeCatalogReplicaRes['Value'].has_key('Successful'))
self.assert_(
removeCatalogReplicaRes['Value']['Successful'].has_key(lfn))
self.assert_(removeCatalogReplicaRes['Value']['Successful'][lfn])
# Check that the removal was successful
self.assert_(removeFileRes['OK'])
self.assert_(removeFileRes['Value'].has_key('Successful'))
self.assert_(removeFileRes['Value']['Successful'].has_key(lfn))
self.assert_(removeFileRes['Value']['Successful'][lfn])
def test_registerReplica(self):
print '\n\n#########################################################################\n\n\t\t\tRegister replica test\n'
lfn = '/lhcb/test/unit-test/ReplicaManager/registerReplica/testFile.%s' % time.time(
)
physicalFile = 'srm://host:port/srm/managerv2?SFN=/sa/path%s' % lfn
fileSize = 10000
storageElementName = 'CERN-RAW'
fileGuid = makeGuid()
fileTuple = (lfn, physicalFile, fileSize, storageElementName, fileGuid)
registerRes = self.dataManager.registerFile(fileTuple)
seName = 'GRIDKA-RAW'
replicaTuple = (lfn, physicalFile, seName)
registerReplicaRes = self.dataManager.registerReplica(replicaTuple)
removeCatalogReplicaRes1 = self.dataManager.removeCatalogReplica(
storageElementName, lfn)
removeCatalogReplicaRes2 = self.dataManager.removeCatalogReplica(
seName, lfn)
removeFileRes = self.dataManager.removeFile(lfn)
# Check that the file registration was done correctly
self.assert_(registerRes['OK'])
self.assert_(registerRes['Value'].has_key('Successful'))
self.assert_(registerRes['Value']['Successful'].has_key(lfn))
self.assert_(registerRes['Value']['Successful'][lfn])
# Check that the replica registration was successful
self.assert_(registerReplicaRes['OK'])
self.assert_(registerReplicaRes['Value'].has_key('Successful'))
self.assert_(registerReplicaRes['Value']['Successful'].has_key(lfn))
self.assert_(registerReplicaRes['Value']['Successful'][lfn])
# Check that the replica removal was successful
self.assert_(removeCatalogReplicaRes1['OK'])
self.assert_(removeCatalogReplicaRes1['Value'].has_key('Successful'))
self.assert_(
removeCatalogReplicaRes1['Value']['Successful'].has_key(lfn))
self.assert_(removeCatalogReplicaRes1['Value']['Successful'][lfn])
# Check that the replica removal was successful
self.assert_(removeCatalogReplicaRes2['OK'])
self.assert_(removeCatalogReplicaRes2['Value'].has_key('Successful'))
self.assert_(
removeCatalogReplicaRes2['Value']['Successful'].has_key(lfn))
self.assert_(removeCatalogReplicaRes2['Value']['Successful'][lfn])
# Check that the removal was successful
self.assert_(removeFileRes['OK'])
self.assert_(removeFileRes['Value'].has_key('Successful'))
self.assert_(removeFileRes['Value']['Successful'].has_key(lfn))
self.assert_(removeFileRes['Value']['Successful'][lfn])
def test_putAndRegisterGet(self):
print '\n\n#########################################################################\n\n\t\t\tGet file test\n'
lfn = '/lhcb/test/unit-test/ReplicaManager/putAndRegisterGet/testFile.%s' % time.time(
)
diracSE = 'GRIDKA-RAW'
putRes = self.dataManager.putAndRegister(lfn, self.fileName, diracSE)
getRes = self.dataManager.getFile(lfn)
removeRes = self.dataManager.removeFile(lfn)
localFilePath = "%s/%s" % (os.getcwd(), os.path.basename(lfn))
if os.path.exists(localFilePath):
os.remove(localFilePath)
# Check that the put was successful
self.assert_(putRes['OK'])
self.assert_(putRes['Value'].has_key('Successful'))
self.assert_(putRes['Value']['Successful'].has_key(lfn))
self.assert_(putRes['Value']['Successful'][lfn])
# Check that the replica removal was successful
self.assert_(getRes['OK'])
self.assert_(getRes['Value'].has_key('Successful'))
self.assert_(getRes['Value']['Successful'].has_key(lfn))
self.assertEqual(getRes['Value']['Successful'][lfn], localFilePath)
# Check that the removal was successful
self.assert_(removeRes['OK'])
self.assert_(removeRes['Value'].has_key('Successful'))
self.assert_(removeRes['Value']['Successful'].has_key(lfn))
self.assert_(removeRes['Value']['Successful'][lfn])
inputFile.close()
else:
lfns.append( getDict( args ) )
from DIRAC.DataManagementSystem.Client.DataManager import DataManager
from DIRAC import gLogger
import DIRAC
exitCode = 0
dm = DataManager()
for lfn in lfns:
if not os.path.exists( lfn['localfile'] ):
gLogger.error( "File %s must exist locally" % lfn['localfile'] )
exitCode = 1
continue
if not os.path.isfile( lfn['localfile'] ):
gLogger.error( "%s is not a file" % lfn['localfile'] )
exitCode = 2
continue
gLogger.notice( "\nUploading %s" % lfn['lfn'] )
res = dm.putAndRegister( lfn['lfn'], lfn['localfile'], lfn['SE'], lfn['guid'] )
if not res['OK']:
exitCode = 3
gLogger.error( 'Error: failed to upload %s to %s' % ( lfn['lfn'], lfn['SE'] ) )
continue
else:
gLogger.notice( 'Successfully uploaded file to %s' % lfn['SE'] )
DIRAC.exit( exitCode )
inputFile.close()
else:
lfns.append( getDict( args ) )
from DIRAC.DataManagementSystem.Client.DataManager import DataManager
from DIRAC import gLogger
import DIRAC
exitCode = 0
dm = DataManager()
for lfn in lfns:
if not os.path.exists( lfn['localfile'] ):
gLogger.error( "File %s must exist locally" % lfn['localfile'] )
exitCode = 1
continue
if not os.path.isfile( lfn['localfile'] ):
gLogger.error( "%s is not a file" % lfn['localfile'] )
exitCode = 2
continue
gLogger.notice( "\nUploading %s" % lfn['lfn'] )
res = dm.putAndRegister( lfn['lfn'], lfn['localfile'], lfn['SE'], lfn['guid'], overwrite = overwrite )
if not res['OK']:
exitCode = 3
gLogger.error( 'Error: failed to upload %s to %s' % ( lfn['lfn'], lfn['SE'] ) )
continue
else:
gLogger.notice( 'Successfully uploaded file to %s' % lfn['SE'] )
DIRAC.exit( exitCode )
def main():
source_location = os.environ.get('JSUB_source_location', './')
destination_dir = os.environ.get('JSUB_destination_dir', '')
destination_dir_jobvar = os.environ.get(
'JSUB_dirac_upload_destination_dir_jobvar'
) # allow using $(jobvar) when defining path
if destination_dir_jobvar is not None:
destination_dir = destination_dir_jobvar
destination_in_user_home = (
os.environ.get('JSUB_user_home', "True").upper() == "TRUE"
) #if true: relative path to cefs user home for upload destination; if false: absolute path
files_to_upload = os.environ.get('JSUB_files_to_upload', '*')
overwrite = (os.environ.get('JSUB_overwrite', 'False').upper() == 'TRUE')
upload_file_jobvar = os.environ.get('JSUB_upload_file_jobvar')
relocate_to_cwd = os.environ.get(
'JSUB_relocate_to_cwd'
) #if true, value of jobvar doesn't reflect dir name; file is under cwd instead
se = os.environ.get('JSUB_SE', 'IHEP-STORM')
upload_status = 0
# if upload_file_jobvar exists, need to reshape output setting to a standard one
if upload_file_jobvar is not None:
if relocate_to_cwd: #value of jobvar doesn't reflect dir name; file is under cwd instead
upfile = os.environ.get('JSUB_' + upload_file_jobvar)
source_location = './'
files_to_upload = os.path.basename(upfile)
if destination_in_user_home:
destination_dir = os.path.dirname(upfile)
else:
files_to_upload = os.environ.get('JSUB_' + upload_file_jobvar)
# destination_dir=os.path.dirname(upfile)
# files_to_upload should be wildcards splitted by ','
flist_raw = files_to_upload.split(',')
flist = []
for f in flist_raw:
l = glob.glob(os.path.join(source_location, f))
l = [os.path.relpath(x, source_location) for x in l]
flist += l
flist = list(set(flist)) # remove repeated items
userHome = getUserHome()
if not userHome:
gLogger.error('Failed to get user home')
return 1
for f in flist:
if destination_in_user_home:
lfn = os.path.join(getUserHome(), destination_dir, f)
else:
lfn = os.path.join(destination_dir, f)
fcc = FileCatalogClient()
dm = DataManager(['FileCatalog'])
print('registering file: LFN={0}, source={1}, SE={2}'.format(
lfn, os.path.join(source_location, f), se))
res = dm.putAndRegister(lfn,
os.path.join(source_location, f),
se,
overwrite=overwrite)
if not res['OK']:
gLogger.error(
'Failed to putAndRegister %s \nto %s \nwith message: %s' %
(lfn, se, res['Message']))
upload_status = 1
elif res['Value']['Failed'].has_key(lfn):
gLogger.error('Failed to putAndRegister %s to %s' % (lfn, se))
upload_status = 1
return upload_status
lfns.append(getDict(args))
from DIRAC.DataManagementSystem.Client.DataManager import DataManager
from DIRAC import gLogger
import DIRAC
exitCode = 0
dm = DataManager()
for lfn in lfns:
if not os.path.exists(lfn['localfile']):
gLogger.error("File %s must exist locally" % lfn['localfile'])
exitCode = 1
continue
if not os.path.isfile(lfn['localfile']):
gLogger.error("%s is not a file" % lfn['localfile'])
exitCode = 2
continue
gLogger.notice("\nUploading %s" % lfn['lfn'])
res = dm.putAndRegister(lfn['lfn'], lfn['localfile'], lfn['SE'],
lfn['guid'])
if not res['OK']:
exitCode = 3
gLogger.error('Error: failed to upload %s to %s' %
(lfn['lfn'], lfn['SE']))
continue
else:
gLogger.notice('Successfully uploaded file to %s' % lfn['SE'])
DIRAC.exit(exitCode)
