class DiracArchiver(object):
def __init__(self):
""" Constructor for the archiver which initializes the necessary
dependencies """
self.dirac = Dirac()
self.log = logging.getLogger()
def lfnExists(self, lfn):
""" Checks whether a given LFN already exists in DIRAC """
ret = False
result = self.dirac.getMetadata(lfn)
# OK key tells us that the call went fine
if result["OK"]:
# The result has a key called Value.
# It contains maps called Successful and Failed
# with the results for each lfn passed
if lfn in result["Value"]["Successful"].keys():
ret = True
else:
raise Exception("DIRAC getMetadata error:" + result)
return ret
def archiveFile(self, filename, lfn, storageElement=SOFTWARE_SE):
""" Archive a file to the storage element """
ret = self.dirac.addFile(lfn, filename, storageElement)
return ret
Script.setUsageMessage( '\n'.join( [ __doc__.split( '\n' )[1],
'Usage:',
' %s [option|cfgfile] ... LFN ...' % Script.scriptName,
'Arguments:',
' LFN: Logical File Name or file containing LFNs' ] ) )
Script.parseCommandLine( ignoreErrors = True )
lfns = Script.getPositionalArgs()
if len( lfns ) < 1:
Script.showHelp()
from DIRAC.Interfaces.API.Dirac import Dirac
dirac = Dirac()
exitCode = 0
errorList = []
if len( lfns ) == 1:
try:
f = open( lfns[0], 'r' )
lfns = f.read().splitlines()
f.close()
except:
pass
result = dirac.getMetadata( lfns, printOutput = True )
if not result['OK']:
print 'ERROR: ', result['Message']
exitCode = 2
DIRAC.exit( exitCode )
Script.setUsageMessage('\n'.join([
__doc__.split('\n')[1], 'Usage:',
' %s [option|cfgfile] ... LFN ...' % Script.scriptName, 'Arguments:',
' LFN: Logical File Name or file containing LFNs'
]))
Script.parseCommandLine(ignoreErrors=True)
lfns = Script.getPositionalArgs()
if len(lfns) < 1:
Script.showHelp()
from DIRAC.Interfaces.API.Dirac import Dirac
dirac = Dirac()
exitCode = 0
errorList = []
if len(lfns) == 1:
try:
f = open(lfns[0], 'r')
lfns = f.read().splitlines()
f.close()
except:
pass
result = dirac.getMetadata(lfns, printOutput=True)
if not result['OK']:
print 'ERROR: ', result['Message']
exitCode = 2
DIRAC.exit(exitCode)
f_out.writelines(f_in)
f_out.close()
f_in.close()
lfn= os.path.join('###LFN_BASE###', file_name)
wildcard='###WILDCARD###'
storage_elements=###SEs###
from contextlib import closing
with closing(open('###LOCATIONSFILE_NAME###','ab')) as locationsfile:
for se in storage_elements:
try:
result = dirac.addFile(lfn, file_name, se)
except Exception as x:
import sys
sys.stdout.write('Exception running dirac.addFile command: %s' % str(x))
break
if result.get('OK',False) and lfn in result.get('Value',{'Successful':{}})['Successful']:
import datetime
guid = dirac.getMetadata(lfn)['Value']['Successful'][lfn]['GUID']
locationsfile.write("DiracFile:::%s&&%s->%s:::['%s']:::%s\\n" % (wildcard, file_label, lfn, se, guid))
locationsfile.flush()
locationsfile.close()
break
errmsg+="(%s, %s), " % (se, result)
else:
locationsfile.write("DiracFile:::%s&&%s->###FAILED###:::File '%s' could not be uploaded to any SE (%s):::NotAvailable\\n" % (wildcard, file_label, file_name, errmsg))
import sys
sys.stdout.write('Could not upload file %s' % file_name)
#continue
f1l = open(full_file,'r')
fread = f1l.read()
f1l.close()
hasbeensent2 = fread.split("\n")
if not hasbeensent2[0] == "done":
#need to be modified to have a different GUID for each file
print "not done for GUID"
f1l = open(full_file,'w')
f1l.write('done'+'\n'+'on grid/castor: done'+'\n'+'number of event processed of type '+ typea[len(typea)-2]+' stripping '+ typea[len(typea)-1]+ ': '+hasbeensent2[0]+'\n'+str( datetime.datetime.now() ))
f1l.close()
mypfn = full_file
dirac = Dirac()
lfn = full_file.replace('/panasas/radiative/', '/lhcb/user/p/potterat/')
print lfn
metaData = dirac.getMetadata(lfn)
# metaData = metaData['Value']['Successful'][lfn]
# metaData2 = metaData2['Value']['Successful'][lfn2]
print metaData
if len(metaData['Value']['Successful']) < 1:
test = dirac.addFile( lfn, mypfn, 'CERN-USER', DIRAC.makeGuid(mypfn))
print test
if(test.has_key('Message')):
#GUID Exists !
test2 = test['Message'].split(" try again. ")
print "remove the wrong path"
rm.removeFile(test2[1])
dirac.addFile( lfn, mypfn, 'CERN-USER', DIRAC.makeGuid(mypfn))
# mypfn.upload(lfn,'CERN-USER')
print full_file,"copied to",lfn
f1l = open('onGrid.txt','w')
file_name += '.gz'
f_in = open(file_label, 'rb')
f_out = gzip.open(file_name, 'wb')
f_out.writelines(f_in)
f_out.close()
f_in.close()
lfn=os.path.join('###LFN_BASE###', file_name)
wildcard='###WILDCARD###'
storage_elements=###SEs###
with closing(open('###LOCATIONSFILE_NAME###','ab')) as locationsfile:
for se in storage_elements:
sys.stdout.write('\nUploading file: \"%s\" as \"%s\" at \"%s\"\n' % (file_name, lfn, se))
try:
result = dirac.addFile(lfn, file_name, se)
except Exception as x:
sys.stdout.write('Exception running dirac.addFile command: %s' % str(x))
break
if result.get('OK',False) and lfn in result.get('Value',{'Successful':{}})['Successful']:
guid = dirac.getMetadata(lfn)['Value']['Successful'][lfn]['GUID']
locationsfile.write("DiracFile:::%s&&%s->%s:::['%s']:::%s\\n" % (wildcard, file_label, lfn, se, guid))
locationsfile.flush()
locationsfile.close()
break
errmsg+="(%s, %s), " % (se, result)
else:
locationsfile.write("DiracFile:::%s&&%s->###FAILED###:::File '%s' could not be uploaded to any SE (%s):::NotAvailable\\n" % (wildcard, file_label, file_name, errmsg))
sys.stdout.write('Could not upload file %s' % file_name)