class DiracArchiver(object): def __init__(self): """ Constructor for the archiver which initializes the necessary dependencies """ self.dirac = Dirac() self.log = logging.getLogger() def lfnExists(self, lfn): """ Checks whether a given LFN already exists in DIRAC """ ret = False result = self.dirac.getMetadata(lfn) # OK key tells us that the call went fine if result["OK"]: # The result has a key called Value. # It contains maps called Successful and Failed # with the results for each lfn passed if lfn in result["Value"]["Successful"].keys(): ret = True else: raise Exception("DIRAC getMetadata error:" + result) return ret def archiveFile(self, filename, lfn, storageElement=SOFTWARE_SE): """ Archive a file to the storage element """ ret = self.dirac.addFile(lfn, filename, storageElement) return ret
Script.setUsageMessage( '\n'.join( [ __doc__.split( '\n' )[1], 'Usage:', ' %s [option|cfgfile] ... LFN ...' % Script.scriptName, 'Arguments:', ' LFN: Logical File Name or file containing LFNs' ] ) ) Script.parseCommandLine( ignoreErrors = True ) lfns = Script.getPositionalArgs() if len( lfns ) < 1: Script.showHelp() from DIRAC.Interfaces.API.Dirac import Dirac dirac = Dirac() exitCode = 0 errorList = [] if len( lfns ) == 1: try: f = open( lfns[0], 'r' ) lfns = f.read().splitlines() f.close() except: pass result = dirac.getMetadata( lfns, printOutput = True ) if not result['OK']: print 'ERROR: ', result['Message'] exitCode = 2 DIRAC.exit( exitCode )
Script.setUsageMessage('\n'.join([ __doc__.split('\n')[1], 'Usage:', ' %s [option|cfgfile] ... LFN ...' % Script.scriptName, 'Arguments:', ' LFN: Logical File Name or file containing LFNs' ])) Script.parseCommandLine(ignoreErrors=True) lfns = Script.getPositionalArgs() if len(lfns) < 1: Script.showHelp() from DIRAC.Interfaces.API.Dirac import Dirac dirac = Dirac() exitCode = 0 errorList = [] if len(lfns) == 1: try: f = open(lfns[0], 'r') lfns = f.read().splitlines() f.close() except: pass result = dirac.getMetadata(lfns, printOutput=True) if not result['OK']: print 'ERROR: ', result['Message'] exitCode = 2 DIRAC.exit(exitCode)
f_out.writelines(f_in) f_out.close() f_in.close() lfn= os.path.join('###LFN_BASE###', file_name) wildcard='###WILDCARD###' storage_elements=###SEs### from contextlib import closing with closing(open('###LOCATIONSFILE_NAME###','ab')) as locationsfile: for se in storage_elements: try: result = dirac.addFile(lfn, file_name, se) except Exception as x: import sys sys.stdout.write('Exception running dirac.addFile command: %s' % str(x)) break if result.get('OK',False) and lfn in result.get('Value',{'Successful':{}})['Successful']: import datetime guid = dirac.getMetadata(lfn)['Value']['Successful'][lfn]['GUID'] locationsfile.write("DiracFile:::%s&&%s->%s:::['%s']:::%s\\n" % (wildcard, file_label, lfn, se, guid)) locationsfile.flush() locationsfile.close() break errmsg+="(%s, %s), " % (se, result) else: locationsfile.write("DiracFile:::%s&&%s->###FAILED###:::File '%s' could not be uploaded to any SE (%s):::NotAvailable\\n" % (wildcard, file_label, file_name, errmsg)) import sys sys.stdout.write('Could not upload file %s' % file_name)
#continue f1l = open(full_file,'r') fread = f1l.read() f1l.close() hasbeensent2 = fread.split("\n") if not hasbeensent2[0] == "done": #need to be modified to have a different GUID for each file print "not done for GUID" f1l = open(full_file,'w') f1l.write('done'+'\n'+'on grid/castor: done'+'\n'+'number of event processed of type '+ typea[len(typea)-2]+' stripping '+ typea[len(typea)-1]+ ': '+hasbeensent2[0]+'\n'+str( datetime.datetime.now() )) f1l.close() mypfn = full_file dirac = Dirac() lfn = full_file.replace('/panasas/radiative/', '/lhcb/user/p/potterat/') print lfn metaData = dirac.getMetadata(lfn) # metaData = metaData['Value']['Successful'][lfn] # metaData2 = metaData2['Value']['Successful'][lfn2] print metaData if len(metaData['Value']['Successful']) < 1: test = dirac.addFile( lfn, mypfn, 'CERN-USER', DIRAC.makeGuid(mypfn)) print test if(test.has_key('Message')): #GUID Exists ! test2 = test['Message'].split(" try again. ") print "remove the wrong path" rm.removeFile(test2[1]) dirac.addFile( lfn, mypfn, 'CERN-USER', DIRAC.makeGuid(mypfn)) # mypfn.upload(lfn,'CERN-USER') print full_file,"copied to",lfn f1l = open('onGrid.txt','w')
file_name += '.gz' f_in = open(file_label, 'rb') f_out = gzip.open(file_name, 'wb') f_out.writelines(f_in) f_out.close() f_in.close() lfn=os.path.join('###LFN_BASE###', file_name) wildcard='###WILDCARD###' storage_elements=###SEs### with closing(open('###LOCATIONSFILE_NAME###','ab')) as locationsfile: for se in storage_elements: sys.stdout.write('\nUploading file: \"%s\" as \"%s\" at \"%s\"\n' % (file_name, lfn, se)) try: result = dirac.addFile(lfn, file_name, se) except Exception as x: sys.stdout.write('Exception running dirac.addFile command: %s' % str(x)) break if result.get('OK',False) and lfn in result.get('Value',{'Successful':{}})['Successful']: guid = dirac.getMetadata(lfn)['Value']['Successful'][lfn]['GUID'] locationsfile.write("DiracFile:::%s&&%s->%s:::['%s']:::%s\\n" % (wildcard, file_label, lfn, se, guid)) locationsfile.flush() locationsfile.close() break errmsg+="(%s, %s), " % (se, result) else: locationsfile.write("DiracFile:::%s&&%s->###FAILED###:::File '%s' could not be uploaded to any SE (%s):::NotAvailable\\n" % (wildcard, file_label, file_name, errmsg)) sys.stdout.write('Could not upload file %s' % file_name)