def upload_to_seList(FileLFN, FileName): DIRAC.gLogger.notice('Put and register in LFC and DFC:', FileLFN) from DIRAC.Interfaces.API.Dirac import Dirac from DIRAC.Core.Utilities.SiteSEMapping import getSEsForSite result = getSEsForSite(DIRAC.siteName()) if result['OK']: localSEs = result['Value'] dirac = Dirac() upload_result = 'NOTOK' failing_se = [] for se in localSEs: if se in seList: DIRAC.gLogger.notice('Local SE is in the list:', se) ret = dirac.addFile(FileLFN, FileName, se) res = CheckCatalogCoherence(FileLFN) if res != DIRAC.S_OK: DIRAC.gLogger.error( 'Job failed: Catalog Coherence problem found') DIRAC.gLogger.notice('Failing SE:', se) failing_se.append(se) continue upload_result = 'OK' if upload_result != 'OK': for se in seList: DIRAC.gLogger.notice('Try upload to:', se) ret = dirac.addFile(FileLFN, FileName, se) res = CheckCatalogCoherence(FileLFN) if res != DIRAC.S_OK: DIRAC.gLogger.error( 'Job failed: Catalog Coherence problem found') failing_se.append(se) DIRAC.gLogger.notice('Failing SE:', se) continue upload_result = 'OK' break DIRAC.gLogger.notice('Failing SE list:', failing_se) #for se in failing_se: # seList.remove(se) # DIRAC.gLogger.notice('Failing SE list:',failing_se) if upload_result != 'OK': return DIRAC.S_ERROR return DIRAC.S_OK
def do_add(self,args): """ Upload a new file to a SE and register in the File Catalog usage: add <lfn> <pfn> <SE> [<guid>] """ # ToDo - adding directories argss = args.split() if len(argss) < 3: print "Error: unsufficient number of arguments" lfn = argss[0] lfn = self.getPath(lfn) pfn = argss[1] se = argss[2] guid = None if len(argss)>3: guid = argss[3] dirac = Dirac() result = dirac.addFile(lfn,pfn,se,guid,printOutput=False) if not result['OK']: print 'Error: %s' %(result['Message']) else: print "File %s successfully uploaded to the %s SE" % (lfn,se)
def addfile(lfn): dirac = Dirac() res = dirac.addFile(lfn, os.path.basename(lfn), SE) if not res['OK']: print 'Error uploading lfn: ' + lfn return res['Message']
class DiracArchiver(object): def __init__(self): """ Constructor for the archiver which initializes the necessary dependencies """ self.dirac = Dirac() self.log = logging.getLogger() def lfnExists(self, lfn): """ Checks whether a given LFN already exists in DIRAC """ ret = False result = self.dirac.getMetadata(lfn) # OK key tells us that the call went fine if result["OK"]: # The result has a key called Value. # It contains maps called Successful and Failed # with the results for each lfn passed if lfn in result["Value"]["Successful"].keys(): ret = True else: raise Exception("DIRAC getMetadata error:" + result) return ret def archiveFile(self, filename, lfn, storageElement=SOFTWARE_SE): """ Archive a file to the storage element """ ret = self.dirac.addFile(lfn, filename, storageElement) return ret
class TestUserMetadataBasicTestCase(unittest.TestCase): def setUp(self): self.dirac = Dirac() csAPI = CSAPI() self.lfn5 = os.path.join(DESTINATION_PATH, 'test_file_10MB_v5.bin') self.dir5 = os.path.dirname(self.lfn5) # local file, for now: self.fname = os.path.basename(self.lfn5) random_dd(self.fname, 10) self.diracSE = 'SE-1' try: self.fc = FileCatalogClient("DataManagement/MultiVOFileCatalog") except Exception: self.fail( " FileCatalog(['MultiVOFileCatalog']) raised Exception unexpectedly!\n" + traceback.format_exc()) return # add a replica self.fileadded = self.dirac.addFile(self.lfn5, self.fname, self.diracSE) self.assertTrue(self.fileadded['OK']) def tearDown(self): # meta index -r result = self.fc.deleteMetadataField('MetaInt6') self.assertTrue(result['OK']) result = self.fc.deleteMetadataField('TestDirectory6') self.assertTrue(result['OK']) # remove the MultiVOFileCatalog self.fc.removeCatalog('MultiVOFileCatalog') # delete a sole replica: dirac-dms-remove-files result = self.dirac.removeFile(self.lfn5) self.assertTrue(result['OK']) os.remove(self.fname)
def uploadAndRegisterFiles(self, fileList, SE='IHEPD-USER', guid=None, ePoint=''): """upload a set of files to SE and register it in DFC. user input the directory of localfile. argument: ePoint is the energy point,for scan data we can treat localDir as a kind of datasetName. """ result_OK = 1 errorList = [] #fileList = self.getFilenamesByLocaldir(localDir) for fullpath in fileList: #get the attributes of the file fileAttr = self.__getFileAttributes(fullpath) if len(fileAttr) == 0: print "failed to get file %s attributes" % fullpath return S_ERROR("failed to get file attributes") #create dir and set dirMetadata to associated dir lastDir = self.registerHierarchicalDir(fileAttr, rootDir='/bes') dirMeta = self.getDirMetaVal(lastDir) if not (dirMeta.has_key("jobOptions") or dirMeta.has_key("description")): lastDirMetaDict = {} lastDirMetaDict['jobOptions'] = fileAttr['jobOptions'] lastDirMetaDict['description'] = fileAttr['description'] try: self.__registerDirMetadata(lastDir, lastDirMetaDict) except: pass if len(ePoint): lastDir = lastDir + os.sep + ePoint lfn = lastDir + os.sep + fileAttr['LFN'] #upload and register file. dirac = Dirac() result = dirac.addFile(lfn, fullpath, SE, guid, printOutput=True) #register file metadata if not result['OK']: print 'ERROR %s' % (result['Message']) #return S_ERROR(result['Message']) errorList.append(fullpath) result_OK = 0 else: result = self.__registerFileMetadata(lfn, fileAttr) if not result['OK']: result_OK = 0 print "failed to register file metadata" if result_OK: return S_OK() else: return S_ERROR(errorList)
def uploadAndRegisterFiles(self, fileList, SE="IHEPD-USER", guid=None, ePoint=""): """upload a set of files to SE and register it in DFC. user input the directory of localfile. argument: ePoint is the energy point,for scan data we can treat localDir as a kind of datasetName. """ result_OK = 1 errorList = [] # fileList = self.getFilenamesByLocaldir(localDir) for fullpath in fileList: # get the attributes of the file fileAttr = self.__getFileAttributes(fullpath) if len(fileAttr) == 0: print "failed to get file %s attributes" % fullpath return S_ERROR("failed to get file attributes") # create dir and set dirMetadata to associated dir lastDir = self.registerHierarchicalDir(fileAttr, rootDir="/bes") dirMeta = self.getDirMetaVal(lastDir) if not (dirMeta.has_key("jobOptions") or dirMeta.has_key("description")): lastDirMetaDict = {} lastDirMetaDict["jobOptions"] = fileAttr["jobOptions"] lastDirMetaDict["description"] = fileAttr["description"] try: self.__registerDirMetadata(lastDir, lastDirMetaDict) except: pass if len(ePoint): lastDir = lastDir + os.sep + ePoint lfn = lastDir + os.sep + fileAttr["LFN"] # upload and register file. dirac = Dirac() result = dirac.addFile(lfn, fullpath, SE, guid, printOutput=True) # register file metadata if not result["OK"]: print "ERROR %s" % (result["Message"]) # return S_ERROR(result['Message']) errorList.append(fullpath) result_OK = 0 else: result = self.__registerFileMetadata(lfn, fileAttr) if not result["OK"]: result_OK = 0 print "failed to register file metadata" if result_OK: return S_OK() else: return S_ERROR(errorList)
def uploadAndRegisterFiles(self, localDir, SE='IHEP-USER', guid=None): """upload a set of files to SE and register it in DFC. user input the directory of localfile. we can treat localDir as a kind of datasetName. """ result_OK = 1 errorList = [] fileList = self.__getFilenamesByLocaldir(localDir) for fullpath in fileList[:50]: #get the attributes of the file print fullpath fileAttr = self.__getFileAttributes(fullpath) #create dir and set dirMetadata to associated dir metaDict = {} metaDict['dataType'] = fileAttr['dataType'] metaDict['eventType'] = fileAttr['eventType'] metaDict['streamId'] = fileAttr['streamId'] metaDict['resonance'] = fileAttr['resonance'] metaDict['round'] = fileAttr['round'] metaDict['bossVer'] = fileAttr['bossVer'] lastDir = self.registerHierarchicalDir(metaDict, rootDir='/zhanggang_test') lfn = lastDir + os.sep + fileAttr['LFN'] fileAttr['LFN'] = lfn #upload and register file. dirac = Dirac() result = dirac.addFile(lfn, fullpath, SE, guid, printOutput=True) #register file metadata if not result['OK']: print 'ERROR %s' % (result['Message']) #return S_ERROR(result['Message']) errorList.append(fullpath) result_OK = 0 else: #get the truely PFN storageElement = StorageElement(SE) res = storageElement.getPfnForLfn(lfn) destPfn = res['Value'] fileAttr['PFN'] = destPfn result = self.__registerFileMetadata(lfn, fileAttr) if not result['OK']: result_OK = 0 print "failed to register file metadata" if result_OK: return S_OK() else: return S_ERROR(errorList)
Script.setUsageMessage( '\n'.join( [ __doc__.split( '\n' )[1], 'Usage:', ' %s [option|cfgfile] ... LFN Path SE [GUID]' % Script.scriptName, 'Arguments:', ' LFN: Logical File Name', ' Path: Local path to the file', ' SE: DIRAC Storage Element', ' GUID: GUID to use in the registration (optional)' ] ) ) Script.parseCommandLine( ignoreErrors = True ) args = Script.getPositionalArgs() import DIRAC if len( args ) < 3 or len( args ) > 4: Script.showHelp() guid = None if len( args ) > 3: guid = args[3] from DIRAC.Interfaces.API.Dirac import Dirac dirac = Dirac() exitCode = 0 result = dirac.addFile( args[0], args[1], args[2], guid, printOutput = True ) if not result['OK']: print 'ERROR %s' % ( result['Message'] ) exitCode = 2 DIRAC.exit( exitCode )
localDir = args[1] toSE = args[2] from DIRAC.Interfaces.API.Dirac import Dirac dirac = Dirac() lfnList = [] pfnList = [] for root, dirs, files in os.walk(localDir): if not root.startswith(localDir): gLogger.error('Can not find corrent lfn') exit(1) relRoot = root[len(localDir):].lstrip('/') for f in files: fullFn = os.path.join(root, f) lfn = os.path.join(dfcDir, relRoot, f) lfnList.append(lfn) pfnList.append(fullFn) gLogger.notice('%s files will be added to DFC "%s"' % (len(lfnList), dfcDir)) for lfn, pfn in zip(lfnList, pfnList): gLogger.debug('Add file to DFC: %s' % lfn) result = dirac.addFile(lfn, pfn, toSE) if not result['OK']: gLogger.error('Can not add file to DFC "%s": %s' % (lfn, result['Message'])) exit(1) gLogger.debug('File upload successfully: %s' % f) gLogger.notice('%s files added to DFC' % len(lfnList))
'Old and new GUIDs have the same value (%s), exiting without changes' % oldGUID) if safe: leave( 'Safe mode - found file GUID = %s and existing GUID = %s, exiting without changes' % (newGUID, oldGUID)) gLogger.verbose('Will set old GUID to %s from %s' % (newGUID, oldGUID)) if newLFN == oldLFN: gLogger.always('Removing old LFN from storages before adding new LFN') result = dirac.removeFile(oldLFN) if not result['OK']: leave('Could not remove existing LFN from Grid storage', result, exitCode=2) gLogger.always('Uploading %s as LFN:%s with replica at %s and GUID = %s' % (localFile, newLFN, storageElements[0], newGUID)) result = dirac.addFile(newLFN, localFile, storageElements[0], fileGuid=newGUID, printOutput=False) if not result['OK']: leave('Failed to copy and register new LFN:%s' % newLFN, result, exitCode=2) leave('')
def main(): from DIRAC.Core.Base import Script Script.registerSwitch("p:", "run_number=", "Run Number", setRunNumber) Script.registerSwitch("T:", "template=", "Template", setCorsikaTemplate) Script.registerSwitch("E:", "executable=", "Executable", setExecutable) Script.registerSwitch("S:", "simtelConfig=", "SimtelConfig", setConfig) Script.registerSwitch("V:", "version=", "Version", setVersion) Script.registerSwitch("M:", "mode=", "Mode", setMode) Script.registerSwitch("C:", "savecorsika=", "Save Corsika", setSaveCorsika) from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient from DIRAC.Resources.Catalog.FileCatalog import FileCatalog Script.parseCommandLine() global fcc, fcL, storage_element from CTADIRAC.Core.Utilities.SoftwareInstallation import getSoftwareEnviron from CTADIRAC.Core.Utilities.SoftwareInstallation import installSoftwareEnviron from CTADIRAC.Core.Utilities.SoftwareInstallation import workingArea from CTADIRAC.Core.Workflow.Modules.CorsikaApp import CorsikaApp from CTADIRAC.Core.Workflow.Modules.Read_CtaApp import Read_CtaApp from DIRAC.Core.Utilities.Subprocess import systemCall jobID = os.environ['JOBID'] jobID = int(jobID) global jobReport jobReport = JobReport(jobID) ########### ## Checking MD coherence fc = FileCatalog('LcgFileCatalog') res = fc._getCatalogConfigDetails('DIRACFileCatalog') print 'DFC CatalogConfigDetails:', res res = fc._getCatalogConfigDetails('LcgFileCatalog') print 'LCG CatalogConfigDetails:', res fcc = FileCatalogClient() fcL = FileCatalog('LcgFileCatalog') from DIRAC.Interfaces.API.Dirac import Dirac dirac = Dirac() ############# simtelConfigFilesPath = 'sim_telarray/multi' simtelConfigFile = simtelConfigFilesPath + '/multi_cta-ultra5.cfg' #simtelConfigFile = simtelConfigFilesPath + '/multi_cta-prod1s.cfg' createGlobalsFromConfigFiles('prodConfigFile', corsikaTemplate, version) ######################Building prod Directory Metadata ####################### resultCreateProdDirMD = createProdFileSystAndMD() if not resultCreateProdDirMD['OK']: DIRAC.gLogger.error('Failed to create prod Directory MD') jobReport.setApplicationStatus('Failed to create prod Directory MD') DIRAC.gLogger.error('Metadata coherence problem, no file produced') DIRAC.exit(-1) else: print 'prod Directory MD successfully created' ######################Building corsika Directory Metadata ####################### resultCreateCorsikaDirMD = createCorsikaFileSystAndMD() if not resultCreateCorsikaDirMD['OK']: DIRAC.gLogger.error('Failed to create corsika Directory MD') jobReport.setApplicationStatus('Failed to create corsika Directory MD') DIRAC.gLogger.error( 'Metadata coherence problem, no corsikaFile produced') DIRAC.exit(-1) else: print 'corsika Directory MD successfully created' ############ Producing Corsika File global CorsikaSimtelPack CorsikaSimtelPack = os.path.join('corsika_simhessarray', version, 'corsika_simhessarray') install_CorsikaSimtelPack(version, 'sim') cs = CorsikaApp() cs.setSoftwarePackage(CorsikaSimtelPack) cs.csExe = executable cs.csArguments = [ '--run-number', run_number, '--run', 'corsika', corsikaTemplate ] corsikaReturnCode = cs.execute() if corsikaReturnCode != 0: DIRAC.gLogger.error('Corsika Application: Failed') jobReport.setApplicationStatus('Corsika Application: Failed') DIRAC.exit(-1) ###################### rename of corsika output file ####################### rundir = 'run' + run_number filein = rundir + '/' + corsikaOutputFileName corsikaFileName = particle + '_' + thetaP + '_' + phiP + '_alt' + obslev + '_' + 'run' + run_number + '.corsika.gz' mv_cmd = 'mv ' + filein + ' ' + corsikaFileName if (os.system(mv_cmd)): DIRAC.exit(-1) ######################## ######################## ## files spread in 1000-runs subDirectories runNum = int(run_number) subRunNumber = '%03d' % runNum runNumModMille = runNum % 1000 runNumTrunc = (runNum - runNumModMille) / 1000 runNumSeriesDir = '%03dxxx' % runNumTrunc print 'runNumSeriesDir=', runNumSeriesDir ### create corsika tar luisa #################### corsikaTarName = particle + '_' + thetaP + '_' + phiP + '_alt' + obslev + '_' + 'run' + run_number + '.corsika.tar.gz' filetar1 = rundir + '/' + 'input' filetar2 = rundir + '/' + 'DAT' + run_number + '.dbase' filetar3 = rundir + '/run' + str(int(run_number)) + '.log' cmdTuple = [ '/bin/tar', 'zcf', corsikaTarName, filetar1, filetar2, filetar3 ] DIRAC.gLogger.notice('Executing command tuple:', cmdTuple) ret = systemCall(0, cmdTuple, sendOutput) if not ret['OK']: DIRAC.gLogger.error('Failed to execute tar') DIRAC.exit(-1) ###################################################### corsikaOutFileDir = os.path.join(corsikaDirPath, particle, 'Data', runNumSeriesDir) corsikaOutFileLFN = os.path.join(corsikaOutFileDir, corsikaFileName) corsikaRunNumberSeriesDirExist = fcc.isDirectory( corsikaOutFileDir)['Value']['Successful'][corsikaOutFileDir] newCorsikaRunNumberSeriesDir = ( corsikaRunNumberSeriesDirExist != True ) # if new runFileSeries, will need to add new MD #### create a file to DISABLE_WATCHDOG_CPU_WALLCLOCK_CHECK ################ f = open('DISABLE_WATCHDOG_CPU_WALLCLOCK_CHECK', 'w') f.close() if savecorsika == 'True': DIRAC.gLogger.notice('Put and register corsika File in LFC and DFC:', corsikaOutFileLFN) ret = dirac.addFile(corsikaOutFileLFN, corsikaFileName, storage_element) res = CheckCatalogCoherence(corsikaOutFileLFN) if res != DIRAC.S_OK: DIRAC.gLogger.error('Job failed: Catalog Coherence problem found') jobReport.setApplicationStatus('OutputData Upload Error') DIRAC.exit(-1) if not ret['OK']: DIRAC.gLogger.error('Error during addFile call:', ret['Message']) jobReport.setApplicationStatus('OutputData Upload Error') DIRAC.exit(-1) # put and register corsikaTarFile: corsikaTarFileDir = os.path.join(corsikaDirPath, particle, 'Log', runNumSeriesDir) corsikaTarFileLFN = os.path.join(corsikaTarFileDir, corsikaTarName) ##### If storage element is IN2P3-tape save simtel file on disk ############### if storage_element == 'CC-IN2P3-Tape': storage_element = 'CC-IN2P3-Disk' DIRAC.gLogger.notice( 'Put and register corsikaTar File in LFC and DFC:', corsikaTarFileLFN) ret = dirac.addFile(corsikaTarFileLFN, corsikaTarName, storage_element) ####Checking and restablishing catalog coherence ##################### res = CheckCatalogCoherence(corsikaTarFileLFN) if res != DIRAC.S_OK: DIRAC.gLogger.error('Job failed: Catalog Coherence problem found') jobReport.setApplicationStatus('OutputData Upload Error') DIRAC.exit(-1) if not ret['OK']: DIRAC.gLogger.error('Error during addFile call:', ret['Message']) jobReport.setApplicationStatus('OutputData Upload Error') DIRAC.exit(-1) ###################################################################### if newCorsikaRunNumberSeriesDir: insertRunFileSeriesMD(corsikaOutFileDir, runNumTrunc) insertRunFileSeriesMD(corsikaTarFileDir, runNumTrunc) ###### insert corsika File Level metadata ############################################ corsikaFileMD = {} corsikaFileMD['runNumber'] = int(run_number) corsikaFileMD['jobID'] = jobID corsikaFileMD['corsikaReturnCode'] = corsikaReturnCode corsikaFileMD['nbShowers'] = nbShowers result = fcc.setMetadata(corsikaOutFileLFN, corsikaFileMD) print "result setMetadata=", result if not result['OK']: print 'ResultSetMetadata:', result['Message'] result = fcc.setMetadata(corsikaTarFileLFN, corsikaFileMD) print "result setMetadata=", result if not result['OK']: print 'ResultSetMetadata:', result['Message'] ##### Exit now if only corsika simulation required if (mode == 'corsika_standalone'): DIRAC.exit() ############ Producing SimTel File ######################Building simtel Directory Metadata ####################### cfg_dict = { "4MSST": 'cta-prod2-4m-dc', "SCSST": 'cta-prod2-sc-sst', "STD": 'cta-prod2', "NSBX3": 'cta-prod2', "ASTRI": 'cta-prod2-astri', "SCMST": 'cta-prod2-sc3', "NORTH": 'cta-prod2n' } if simtelConfig == "6INROW": all_configs = ["4MSST", "SCSST", "ASTRI", "NSBX3", "STD", "SCMST"] elif simtelConfig == "5INROW": all_configs = ["4MSST", "SCSST", "ASTRI", "NSBX3", "STD"] elif simtelConfig == "3INROW": all_configs = ["SCSST", "STD", "SCMST"] else: all_configs = [simtelConfig] ############################################ #for current_conf in all_configs: #DIRAC.gLogger.notice('current conf is',current_conf) #if current_conf == "SCMST": #current_version = version + '_sc3' #DIRAC.gLogger.notice('current version is', current_version) #if os.path.isdir('sim_telarray'): #DIRAC.gLogger.notice('Package found in the local area. Removing package...') #cmd = 'rm -R sim_telarray corsika-6990 hessioxxx corsika-run' #if(os.system(cmd)): #DIRAC.exit( -1 ) #install_CorsikaSimtelPack(current_version) #else: #current_version = version #DIRAC.gLogger.notice('current version is', current_version) ############################################################# for current_conf in all_configs: DIRAC.gLogger.notice('current conf is', current_conf) if current_conf == "SCMST": current_version = version + '_sc3' DIRAC.gLogger.notice('current version is', current_version) installSoftwareEnviron(CorsikaSimtelPack, workingArea(), 'sim-sc3') else: current_version = version DIRAC.gLogger.notice('current version is', current_version) ######################################################## global simtelDirPath global simtelProdVersion simtelProdVersion = current_version + '_simtel' simtelDirPath = os.path.join(corsikaParticleDirPath, simtelProdVersion) resultCreateSimtelDirMD = createSimtelFileSystAndMD(current_conf) if not resultCreateSimtelDirMD['OK']: DIRAC.gLogger.error('Failed to create simtelArray Directory MD') jobReport.setApplicationStatus( 'Failed to create simtelArray Directory MD') DIRAC.gLogger.error( 'Metadata coherence problem, no simtelArray File produced') DIRAC.exit(-1) else: DIRAC.gLogger.notice('simtel Directory MD successfully created') ############## check simtel data file LFN exists ######################## simtelFileName = particle + '_' + str(thetaP) + '_' + str( phiP) + '_alt' + str( obslev) + '_' + 'run' + run_number + '.simtel.gz' simtelDirPath_conf = simtelDirPath + '_' + current_conf simtelOutFileDir = os.path.join(simtelDirPath_conf, 'Data', runNumSeriesDir) simtelOutFileLFN = os.path.join(simtelOutFileDir, simtelFileName) res = CheckCatalogCoherence(simtelOutFileLFN) if res == DIRAC.S_OK: DIRAC.gLogger.notice('Current conf already done', current_conf) continue #### execute simtelarray ################ fd = open('run_sim.sh', 'w') fd.write("""#! /bin/sh source ./Corsika_simhessarrayEnv.sh export SVNPROD2=$PWD export SVNTAG=SVN-PROD2_rev10503 export CORSIKA_IO_BUFFER=800MB cp ../grid_prod2-repro.sh . ln -s ../%s ln -s ../$SVNTAG ./grid_prod2-repro.sh %s %s""" % (corsikaFileName, corsikaFileName, current_conf)) fd.close() #################################### os.system('chmod u+x run_sim.sh') cmdTuple = ['./run_sim.sh'] ret = systemCall(0, cmdTuple, sendOutputSimTel) simtelReturnCode, stdout, stderr = ret['Value'] if (os.system('grep Broken simtel.log') == 0): DIRAC.gLogger.error('Broken string found in simtel.log') jobReport.setApplicationStatus('Broken pipe') DIRAC.exit(-1) if not ret['OK']: DIRAC.gLogger.error('Failed to execute run_sim.sh') DIRAC.gLogger.error('run_sim.sh status is:', simtelReturnCode) DIRAC.exit(-1) ## check simtel data/log/histo Output File exist cfg = cfg_dict[current_conf] #cmd = 'mv Data/sim_telarray/' + cfg + '/0.0deg/Data/*.simtel.gz ' + simtelFileName if current_conf == "SCMST": cmdprefix = 'mv sim-sc3/Data/sim_telarray/' + cfg + '/0.0deg/' else: cmdprefix = 'mv sim/Data/sim_telarray/' + cfg + '/0.0deg/' cmd = cmdprefix + 'Data/*' + cfg + '_*.simtel.gz ' + simtelFileName if (os.system(cmd)): DIRAC.exit(-1) ############################################ simtelRunNumberSeriesDirExist = fcc.isDirectory( simtelOutFileDir)['Value']['Successful'][simtelOutFileDir] newSimtelRunFileSeriesDir = ( simtelRunNumberSeriesDirExist != True ) # if new runFileSeries, will need to add new MD simtelLogFileName = particle + '_' + str(thetaP) + '_' + str( phiP) + '_alt' + str(obslev) + '_' + 'run' + run_number + '.log.gz' #cmd = 'mv Data/sim_telarray/' + cfg + '/0.0deg/Log/*.log.gz ' + simtelLogFileName cmd = cmdprefix + 'Log/*' + cfg + '_*.log.gz ' + simtelLogFileName if (os.system(cmd)): DIRAC.exit(-1) simtelOutLogFileDir = os.path.join(simtelDirPath_conf, 'Log', runNumSeriesDir) simtelOutLogFileLFN = os.path.join(simtelOutLogFileDir, simtelLogFileName) simtelHistFileName = particle + '_' + str(thetaP) + '_' + str( phiP) + '_alt' + str( obslev) + '_' + 'run' + run_number + '.hdata.gz' #cmd = 'mv Data/sim_telarray/' + cfg + '/0.0deg/Histograms/*.hdata.gz ' + simtelHistFileName cmd = cmdprefix + 'Histograms/*' + cfg + '_*.hdata.gz ' + simtelHistFileName if (os.system(cmd)): DIRAC.exit(-1) simtelOutHistFileDir = os.path.join(simtelDirPath_conf, 'Histograms', runNumSeriesDir) simtelOutHistFileLFN = os.path.join(simtelOutHistFileDir, simtelHistFileName) ########### quality check on Histo ############################################# fd = open('check_histo.sh', 'w') fd.write("""#! /bin/sh nsim=$(list_histograms %s|fgrep 'Histogram 6 '|sed 's/^.*contents: //'| sed 's:/.*$::') nevents=%d if [ $nsim -lt $(( $nevents - 20 )) ]; then echo 'nsim found:' $nsim echo 'nsim expected:' $nevents exit 1 else echo 'nsim found:' $nsim echo 'nsim expected:' $nevents fi """ % (simtelHistFileName, int(nbShowers) * int(cscat))) fd.close() ret = getSoftwareEnviron(CorsikaSimtelPack) if not ret['OK']: error = ret['Message'] DIRAC.gLogger.error(error, CorsikaSimtelPack) DIRAC.exit(-1) corsikaEnviron = ret['Value'] os.system('chmod u+x check_histo.sh') cmdTuple = ['./check_histo.sh'] DIRAC.gLogger.notice('Executing command tuple:', cmdTuple) ret = systemCall(0, cmdTuple, sendOutput, env=corsikaEnviron) checkHistoReturnCode, stdout, stderr = ret['Value'] if not ret['OK']: DIRAC.gLogger.error('Failed to execute check_histo.sh') DIRAC.gLogger.error('check_histo.sh status is:', checkHistoReturnCode) DIRAC.exit(-1) if (checkHistoReturnCode != 0): DIRAC.gLogger.error('Failure during check_histo.sh') DIRAC.gLogger.error('check_histo.sh status is:', checkHistoReturnCode) jobReport.setApplicationStatus('Histo check Failed') DIRAC.exit(-1) ########## quality check on Log ############################# cmd = 'zcat %s | grep Finished.' % simtelLogFileName DIRAC.gLogger.notice('Executing system call:', cmd) if (os.system(cmd)): jobReport.setApplicationStatus('Log check Failed') DIRAC.exit(-1) ################################################ from DIRAC.Core.Utilities import List from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations opsHelper = Operations() global seList seList = opsHelper.getValue('ProductionOutputs/SimtelProd', []) seList = List.randomize(seList) DIRAC.gLogger.notice('SeList is:', seList) ######### Upload simtel data/log/histo ############################################## res = upload_to_seList(simtelOutFileLFN, simtelFileName) if res != DIRAC.S_OK: DIRAC.gLogger.error('OutputData Upload Error', simtelOutFileLFN) jobReport.setApplicationStatus('OutputData Upload Error') DIRAC.exit(-1) res = CheckCatalogCoherence(simtelOutLogFileLFN) if res == DIRAC.S_OK: DIRAC.gLogger.notice('Log file already exists. Removing:', simtelOutLogFileLFN) ret = dirac.removeFile(simtelOutLogFileLFN) res = upload_to_seList(simtelOutLogFileLFN, simtelLogFileName) if res != DIRAC.S_OK: DIRAC.gLogger.error('Upload simtel Log Error', simtelOutLogFileLFN) DIRAC.gLogger.notice('Removing simtel data file:', simtelOutFileLFN) ret = dirac.removeFile(simtelOutFileLFN) jobReport.setApplicationStatus('OutputData Upload Error') DIRAC.exit(-1) res = CheckCatalogCoherence(simtelOutHistFileLFN) if res == DIRAC.S_OK: DIRAC.gLogger.notice('Histo file already exists. Removing:', simtelOutHistFileLFN) ret = dirac.removeFile(simtelOutHistFileLFN) res = upload_to_seList(simtelOutHistFileLFN, simtelHistFileName) if res != DIRAC.S_OK: DIRAC.gLogger.error('Upload simtel Histo Error', simtelOutHistFileLFN) DIRAC.gLogger.notice('Removing simtel data file:', simtelOutFileLFN) ret = dirac.removeFile(simtelOutFileLFN) DIRAC.gLogger.notice('Removing simtel log file:', simtelOutLogFileLFN) ret = dirac.removeFile(simtelOutLogFileLFN) jobReport.setApplicationStatus('OutputData Upload Error') DIRAC.exit(-1) # simtelRunNumberSeriesDirExist = fcc.isDirectory(simtelOutFileDir)['Value']['Successful'][simtelOutFileDir] # newSimtelRunFileSeriesDir = (simtelRunNumberSeriesDirExist != True) # if new runFileSeries, will need to add new MD if newSimtelRunFileSeriesDir: print 'insertRunFileSeriesMD' insertRunFileSeriesMD(simtelOutFileDir, runNumTrunc) insertRunFileSeriesMD(simtelOutLogFileDir, runNumTrunc) insertRunFileSeriesMD(simtelOutHistFileDir, runNumTrunc) else: print 'NotinsertRunFileSeriesMD' ###### simtel File level metadata ############################################ simtelFileMD = {} simtelFileMD['runNumber'] = int(run_number) simtelFileMD['jobID'] = jobID simtelFileMD['simtelReturnCode'] = simtelReturnCode result = fcc.setMetadata(simtelOutFileLFN, simtelFileMD) print "result setMetadata=", result if not result['OK']: print 'ResultSetMetadata:', result['Message'] result = fcc.setMetadata(simtelOutLogFileLFN, simtelFileMD) print "result setMetadata=", result if not result['OK']: print 'ResultSetMetadata:', result['Message'] result = fcc.setMetadata(simtelOutHistFileLFN, simtelFileMD) print "result setMetadata=", result if not result['OK']: print 'ResultSetMetadata:', result['Message'] if savecorsika == 'True': result = fcc.addFileAncestors( {simtelOutFileLFN: { 'Ancestors': [corsikaOutFileLFN] }}) print 'result addFileAncestor:', result result = fcc.addFileAncestors( {simtelOutLogFileLFN: { 'Ancestors': [corsikaOutFileLFN] }}) print 'result addFileAncestor:', result result = fcc.addFileAncestors( {simtelOutHistFileLFN: { 'Ancestors': [corsikaOutFileLFN] }}) print 'result addFileAncestor:', result ##### Exit now if only corsika simulation required if (mode == 'corsika_simtel'): continue ######### run read_cta ####################################### rcta = Read_CtaApp() rcta.setSoftwarePackage(CorsikaSimtelPack) rcta.rctaExe = 'read_cta' powerlaw_dict = { 'gamma': '-2.57', 'gamma_ptsrc': '-2.57', 'proton': '-2.70', 'electron': '-3.21' } dstFileName = particle + '_' + str(thetaP) + '_' + str( phiP) + '_alt' + str( obslev) + '_' + 'run' + run_number + '.simtel-dst0.gz' dstHistoFileName = particle + '_' + str(thetaP) + '_' + str( phiP) + '_alt' + str( obslev) + '_' + 'run' + run_number + '.hdata-dst0.gz' ## added some options starting from Armazones_2K prod. rcta.rctaArguments = [ '-r', '4', '-u', '--integration-scheme', '4', '--integration-window', '7,3', '--tail-cuts', '6,8', '--min-pix', '2', '--min-amp', '20', '--type', '1,0,0,400', '--tail-cuts', '9,12', '--min-amp', '20', '--type', '2,0,0,100', '--tail-cuts', '8,11', '--min-amp', '19', '--type', '3,0,0,40', '--tail-cuts', '6,9', '--min-amp', '15', '--type', '4,0,0,15', '--tail-cuts', '3.7,5.5', '--min-amp', '8', '--type', '5,0,0,70,5.6', '--tail-cuts', '2.4,3.2', '--min-amp', '5.6', '--dst-level', '0', '--dst-file', dstFileName, '--histogram-file', dstHistoFileName, '--powerlaw', powerlaw_dict[particle], simtelFileName ] rctaReturnCode = rcta.execute() if rctaReturnCode != 0: DIRAC.gLogger.error('read_cta Application: Failed') jobReport.setApplicationStatus('read_cta Application: Failed') DIRAC.exit(-1) ######## run dst quality checks ###################################### fd = open('check_dst_histo.sh', 'w') fd.write("""#! /bin/sh dsthistfilename=%s dstfile=%s n6="$(list_histograms -h 6 ${dsthistfilename} | grep 'Histogram of type' | sed 's/.*bins, //' | sed 's/ entries.//')" n12001="$(list_histograms -h 12001 ${dsthistfilename} | grep 'Histogram of type' | sed 's/.*bins, //' | sed 's/ entries.//')" if [ $n6 -ne $n12001 ]; then echo 'n6 found:' $n6 echo 'n12001 found:' $n12001 exit 1 else echo 'n6 found:' $n6 echo 'n12001 found:' $n12001 fi n12002="$(list_histograms -h 12002 ${dsthistfilename} | grep 'Histogram of type' | sed 's/.*bins, //' | sed 's/ entries.//')" nev="$(statio ${dstfile} | egrep '^2010' | cut -f2)" if [ -z "$nev" ]; then nev="0"; fi if [ $nev -ne $n12002 ]; then echo 'nev found:' $nev echo 'n12002 found:' $n12002 exit 1 else echo 'nev found:' $nev echo 'n12002 found:' $n12002 fi """ % (dstHistoFileName, dstFileName)) fd.close() os.system('chmod u+x check_dst_histo.sh') cmdTuple = ['./check_dst_histo.sh'] DIRAC.gLogger.notice('Executing command tuple:', cmdTuple) ret = systemCall(0, cmdTuple, sendOutput, env=corsikaEnviron) checkHistoReturnCode, stdout, stderr = ret['Value'] if not ret['OK']: DIRAC.gLogger.error('Failed to execute check_dst_histo.sh') DIRAC.gLogger.error('check_dst_histo.sh status is:', checkHistoReturnCode) DIRAC.exit(-1) if (checkHistoReturnCode != 0): DIRAC.gLogger.error('Failure during check_dst_histo.sh') DIRAC.gLogger.error('check_dst_histo.sh status is:', checkHistoReturnCode) jobReport.setApplicationStatus('Histo check Failed') DIRAC.exit(-1) ############create MD and upload dst data/histo ########################################################## global dstDirPath global dstProdVersion dstProdVersion = current_version + '_dst' dstDirPath = os.path.join(simtelDirPath_conf, dstProdVersion) dstOutFileDir = os.path.join(dstDirPath, 'Data', runNumSeriesDir) dstOutFileLFN = os.path.join(dstOutFileDir, dstFileName) resultCreateDstDirMD = createDstFileSystAndMD() if not resultCreateDstDirMD['OK']: DIRAC.gLogger.error('Failed to create Dst Directory MD') jobReport.setApplicationStatus('Failed to create Dst Directory MD') DIRAC.gLogger.error( 'Metadata coherence problem, no Dst File produced') DIRAC.exit(-1) else: DIRAC.gLogger.notice('Dst Directory MD successfully created') ############################################################ res = CheckCatalogCoherence(dstOutFileLFN) if res == DIRAC.S_OK: DIRAC.gLogger.notice('dst file already exists. Removing:', dstOutFileLFN) ret = dirac.removeFile(dstOutFileLFN) res = upload_to_seList(dstOutFileLFN, dstFileName) if res != DIRAC.S_OK: DIRAC.gLogger.error('Upload dst Error', dstOutFileLFN) jobReport.setApplicationStatus('OutputData Upload Error') DIRAC.exit(-1) ############################################################## dstHistoFileDir = os.path.join(dstDirPath, 'Histograms', runNumSeriesDir) dstHistoFileLFN = os.path.join(dstHistoFileDir, dstHistoFileName) res = CheckCatalogCoherence(dstHistoFileLFN) if res == DIRAC.S_OK: DIRAC.gLogger.notice('dst histo file already exists. Removing:', dstHistoFileLFN) ret = dirac.removeFile(dstHistoFileLFN) res = upload_to_seList(dstHistoFileLFN, dstHistoFileName) if res != DIRAC.S_OK: DIRAC.gLogger.error('Upload dst Error', dstHistoFileName) jobReport.setApplicationStatus('OutputData Upload Error') DIRAC.exit(-1) ########### Insert RunNumSeries MD ########################## dstRunNumberSeriesDirExist = fcc.isDirectory( dstOutFileDir)['Value']['Successful'][dstOutFileDir] newDstRunFileSeriesDir = ( dstRunNumberSeriesDirExist != True ) # if new runFileSeries, will need to add new MD if newDstRunFileSeriesDir: insertRunFileSeriesMD(dstOutFileDir, runNumTrunc) insertRunFileSeriesMD(dstHistoFileDir, runNumTrunc) ####### dst File level metadata ############################################### dstFileMD = {} dstFileMD['runNumber'] = int(run_number) dstFileMD['jobID'] = jobID dstFileMD['rctaReturnCode'] = rctaReturnCode result = fcc.setMetadata(dstOutFileLFN, dstFileMD) print "result setMetadata=", result if not result['OK']: print 'ResultSetMetadata:', result['Message'] result = fcc.setMetadata(dstHistoFileLFN, dstFileMD) print "result setMetadata=", result if not result['OK']: print 'ResultSetMetadata:', result['Message'] ########## set the ancestors for dst ##################################### result = fcc.addFileAncestors( {dstOutFileLFN: { 'Ancestors': [simtelOutFileLFN] }}) print 'result addFileAncestor:', result result = fcc.addFileAncestors( {dstHistoFileLFN: { 'Ancestors': [simtelOutFileLFN] }}) print 'result addFileAncestor:', result ###################################################### DIRAC.exit()
file_name += '.gz' f_in = open(file_label, 'rb') f_out = gzip.open(file_name, 'wb') f_out.writelines(f_in) f_out.close() f_in.close() lfn= os.path.join('###LFN_BASE###', file_name) wildcard='###WILDCARD###' storage_elements=###SEs### from contextlib import closing with closing(open('###LOCATIONSFILE_NAME###','ab')) as locationsfile: for se in storage_elements: try: result = dirac.addFile(lfn, file_name, se) except Exception as x: import sys sys.stdout.write('Exception running dirac.addFile command: %s' % str(x)) break if result.get('OK',False) and lfn in result.get('Value',{'Successful':{}})['Successful']: import datetime guid = dirac.getMetadata(lfn)['Value']['Successful'][lfn]['GUID'] locationsfile.write("DiracFile:::%s&&%s->%s:::['%s']:::%s\\n" % (wildcard, file_label, lfn, se, guid)) locationsfile.flush() locationsfile.close() break errmsg+="(%s, %s), " % (se, result) else: locationsfile.write("DiracFile:::%s&&%s->###FAILED###:::File '%s' could not be uploaded to any SE (%s):::NotAvailable\\n" % (wildcard, file_label, file_name, errmsg)) import sys
if not hasbeensent2[0] == "done": #need to be modified to have a different GUID for each file print "not done for GUID" f1l = open(full_file,'w') f1l.write('done'+'\n'+'on grid/castor: done'+'\n'+'number of event processed of type '+ typea[len(typea)-2]+' stripping '+ typea[len(typea)-1]+ ': '+hasbeensent2[0]+'\n'+str( datetime.datetime.now() )) f1l.close() mypfn = full_file dirac = Dirac() lfn = full_file.replace('/panasas/radiative/', '/lhcb/user/p/potterat/') print lfn metaData = dirac.getMetadata(lfn) # metaData = metaData['Value']['Successful'][lfn] # metaData2 = metaData2['Value']['Successful'][lfn2] print metaData if len(metaData['Value']['Successful']) < 1: test = dirac.addFile( lfn, mypfn, 'CERN-USER', DIRAC.makeGuid(mypfn)) print test if(test.has_key('Message')): #GUID Exists ! test2 = test['Message'].split(" try again. ") print "remove the wrong path" rm.removeFile(test2[1]) dirac.addFile( lfn, mypfn, 'CERN-USER', DIRAC.makeGuid(mypfn)) # mypfn.upload(lfn,'CERN-USER') print full_file,"copied to",lfn f1l = open('onGrid.txt','w') f1l.write(sent+full_file+'\n') f1l.close() else : if (metaData['Value']['Successful'][lfn]['Size'] == os.path.getsize(full_file)): print " "
from DIRAC.Core.Base import Script Script.setUsageMessage('\n'.join([ __doc__.split('\n')[1], 'Usage:', ' %s [option|cfgfile] ... LFN Path SE [GUID]' % Script.scriptName, 'Arguments:', ' LFN: Logical File Name', ' Path: Local path to the file', ' SE: DIRAC Storage Element', ' GUID: GUID to use in the registration (optional)' ])) Script.parseCommandLine(ignoreErrors=True) args = Script.getPositionalArgs() import DIRAC if len(args) < 3 or len(args) > 4: Script.showHelp() guid = None if len(args) > 3: guid = args[3] from DIRAC.Interfaces.API.Dirac import Dirac dirac = Dirac() exitCode = 0 result = dirac.addFile(args[0], args[1], args[2], guid, printOutput=True) if not result['OK']: print 'ERROR %s' % (result['Message']) exitCode = 2 DIRAC.exit(exitCode)
def main(): from DIRAC.Core.Base import Script Script.registerSwitch("p:", "inputfile=", "Input File", setInputFile) Script.registerSwitch("E:", "simtelExecName=", "SimtelExecName", setExecutable) Script.registerSwitch("S:", "simtelConfig=", "SimtelConfig", setConfig) Script.registerSwitch("V:", "version=", "Version", setVersion) Script.registerSwitch("D:", "storage_element=", "Storage Element", setStorageElement) from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient from DIRAC.Resources.Catalog.FileCatalog import FileCatalog Script.parseCommandLine() DIRAC.gLogger.setLevel('INFO') global fcc, fcL, storage_element from CTADIRAC.Core.Utilities.SoftwareInstallation import checkSoftwarePackage from CTADIRAC.Core.Utilities.SoftwareInstallation import installSoftwarePackage from CTADIRAC.Core.Utilities.SoftwareInstallation import installSoftwareEnviron from CTADIRAC.Core.Utilities.SoftwareInstallation import localArea from CTADIRAC.Core.Utilities.SoftwareInstallation import sharedArea from CTADIRAC.Core.Utilities.SoftwareInstallation import workingArea from DIRAC.Core.Utilities.Subprocess import systemCall from DIRAC.WorkloadManagementSystem.Client.JobReport import JobReport jobID = os.environ['JOBID'] jobID = int(jobID) jobReport = JobReport(jobID) CorsikaSimtelPack = 'corsika_simhessarray/' + version + '/corsika_simhessarray' packs = [CorsikaSimtelPack] for package in packs: DIRAC.gLogger.notice('Checking:', package) if sharedArea: if checkSoftwarePackage(package, sharedArea())['OK']: DIRAC.gLogger.notice('Package found in Shared Area:', package) installSoftwareEnviron(package, workingArea()) packageTuple = package.split('/') corsika_subdir = sharedArea( ) + '/' + packageTuple[0] + '/' + version cmd = 'cp -u -r ' + corsika_subdir + '/* .' os.system(cmd) continue DIRAC.gLogger.error('Check Failed for software package:', package) DIRAC.gLogger.error('Software package not available') DIRAC.exit(-1) ########### ## Checking MD coherence fc = FileCatalog('LcgFileCatalog') res = fc._getCatalogConfigDetails('DIRACFileCatalog') print 'DFC CatalogConfigDetails:', res res = fc._getCatalogConfigDetails('LcgFileCatalog') print 'LCG CatalogConfigDetails:', res fcc = FileCatalogClient() fcL = FileCatalog('LcgFileCatalog') from DIRAC.Interfaces.API.Dirac import Dirac dirac = Dirac() ############################ ############# # CLAUDIA: simtelConfigFile should be built from ??? #simtelConfigFilesPath = 'sim_telarray/multi' #simtelConfigFile = simtelConfigFilesPath + '/multi_cta-ultra5.cfg' #createGlobalsFromConfigFiles(simtelConfigFile) createGlobalsFromConfigFiles() ####################### ## files spread in 1000-runs subDirectories corsikaFileName = os.path.basename(corsikaFileLFN) run_number = corsikaFileName.split('run')[1].split('.corsika.gz')[ 0] # run001412.corsika.gz runNum = int(run_number) subRunNumber = '%03d' % runNum runNumModMille = runNum % 1000 runNumTrunc = (runNum - runNumModMille) / 1000 runNumSeriesDir = '%03dxxx' % runNumTrunc print 'runNumSeriesDir=', runNumSeriesDir f = open('DISABLE_WATCHDOG_CPU_WALLCLOCK_CHECK', 'w') f.close() ##### If storage element is IN2P3-tape save simtel file on disk ############### if storage_element == 'CC-IN2P3-Tape': storage_element = 'CC-IN2P3-Disk' ############ Producing SimTel File ######################Building simtel Directory Metadata ####################### resultCreateSimtelDirMD = createSimtelFileSystAndMD() if not resultCreateSimtelDirMD['OK']: DIRAC.gLogger.error('Failed to create simtelArray Directory MD') jobReport.setApplicationStatus( 'Failed to create simtelArray Directory MD') DIRAC.gLogger.error( 'Metadata coherence problem, no simtelArray File produced') DIRAC.exit(-1) else: print 'simtel Directory MD successfully created' #### execute simtelarray ################ # fd = open('run_sim.sh', 'w' ) # fd.write( """#! /bin/sh #echo "go for sim_telarray" #. ./examples_common.sh #export CORSIKA_IO_BUFFER=800MB #zcat %s | $SIM_TELARRAY_PATH/run_sim_%s""" % (corsikaFileName, simtelExecName)) # fd.close() #### execute simtelarray ################ fd = open('run_sim.sh', 'w') fd.write("""#! /bin/sh export SVNPROD2=$PWD export SVNTAG=SVN-PROD2 export CORSIKA_IO_BUFFER=800MB ./grid_prod2-repro.sh %s %s""" % (corsikaFileName, simtelConfig)) fd.close() os.system('chmod u+x run_sim.sh') cmdTuple = ['./run_sim.sh'] ret = systemCall(0, cmdTuple, sendOutputSimTel) simtelReturnCode, stdout, stderr = ret['Value'] if (os.system('grep Broken simtel.log')): print 'not broken' else: print 'broken' # Tag corsika File if Broken Pipe corsikaTagMD = {} corsikaTagMD['CorsikaToReprocess'] = 'CorsikaToReprocess' result = fcc.setMetadata(corsikaFileLFN, corsikaTagMD) print "result setMetadata=", result if not result['OK']: print 'ResultSetMetadata:', result['Message'] jobReport.setApplicationStatus('Broken pipe') DIRAC.exit(-1) if not ret['OK']: DIRAC.gLogger.error('Failed to execute run_sim.sh') DIRAC.gLogger.error('run_sim.sh status is:', simtelReturnCode) DIRAC.exit(-1) ## putAndRegister simtel data/log/histo Output File: simtelFileName = particle + '_' + str(thetaP) + '_' + str( phiP) + '_alt' + str(obslev) + '_' + 'run' + run_number + '.simtel.gz' cfg = simtelExecName if simtelExecName == "cta-prod2-nsbx3": cfg = "cta-prod2" cmd = 'mv Data/sim_telarray/' + cfg + '/0.0deg/Data/*.simtel.gz ' + simtelFileName if (os.system(cmd)): DIRAC.exit(-1) simtelOutFileDir = os.path.join(simtelDirPath, 'Data', runNumSeriesDir) simtelOutFileLFN = os.path.join(simtelOutFileDir, simtelFileName) simtelRunNumberSeriesDirExist = fcc.isDirectory( simtelOutFileDir)['Value']['Successful'][simtelOutFileDir] newSimtelRunFileSeriesDir = ( simtelRunNumberSeriesDirExist != True ) # if new runFileSeries, will need to add new MD simtelLogFileName = particle + '_' + str(thetaP) + '_' + str( phiP) + '_alt' + str(obslev) + '_' + 'run' + run_number + '.log.gz' cmd = 'mv Data/sim_telarray/' + cfg + '/0.0deg/Log/*.log.gz ' + simtelLogFileName if (os.system(cmd)): DIRAC.exit(-1) simtelOutLogFileDir = os.path.join(simtelDirPath, 'Log', runNumSeriesDir) simtelOutLogFileLFN = os.path.join(simtelOutLogFileDir, simtelLogFileName) simtelHistFileName = particle + '_' + str(thetaP) + '_' + str( phiP) + '_alt' + str(obslev) + '_' + 'run' + run_number + '.hdata.gz' cmd = 'mv Data/sim_telarray/' + cfg + '/0.0deg/Histograms/*.hdata.gz ' + simtelHistFileName if (os.system(cmd)): DIRAC.exit(-1) simtelOutHistFileDir = os.path.join(simtelDirPath, 'Histograms', runNumSeriesDir) simtelOutHistFileLFN = os.path.join(simtelOutHistFileDir, simtelHistFileName) ################################################ DIRAC.gLogger.notice('Put and register simtel File in LFC and DFC:', simtelOutFileLFN) ret = dirac.addFile(simtelOutFileLFN, simtelFileName, storage_element) res = CheckCatalogCoherence(simtelOutFileLFN) if res != DIRAC.S_OK: DIRAC.gLogger.error('Job failed: Catalog Coherence problem found') jobReport.setApplicationStatus('OutputData Upload Error') DIRAC.exit(-1) if not ret['OK']: DIRAC.gLogger.error('Error during addFile call:', ret['Message']) jobReport.setApplicationStatus('OutputData Upload Error') DIRAC.exit(-1) ###################################################################### DIRAC.gLogger.notice('Put and register simtel Log File in LFC and DFC:', simtelOutLogFileLFN) ret = dirac.addFile(simtelOutLogFileLFN, simtelLogFileName, storage_element) res = CheckCatalogCoherence(simtelOutLogFileLFN) if res != DIRAC.S_OK: DIRAC.gLogger.error('Job failed: Catalog Coherence problem found') jobReport.setApplicationStatus('OutputData Upload Error') DIRAC.exit(-1) if not ret['OK']: DIRAC.gLogger.error('Error during addFile call:', ret['Message']) jobReport.setApplicationStatus('OutputData Upload Error') DIRAC.exit(-1) ###################################################################### DIRAC.gLogger.notice('Put and register simtel Histo File in LFC and DFC:', simtelOutHistFileLFN) ret = dirac.addFile(simtelOutHistFileLFN, simtelHistFileName, storage_element) res = CheckCatalogCoherence(simtelOutHistFileLFN) if res != DIRAC.S_OK: DIRAC.gLogger.error('Job failed: Catalog Coherence problem found') jobReport.setApplicationStatus('OutputData Upload Error') DIRAC.exit(-1) if not ret['OK']: DIRAC.gLogger.error('Error during addFile call:', ret['Message']) jobReport.setApplicationStatus('OutputData Upload Error') DIRAC.exit(-1) ###################################################################### if newSimtelRunFileSeriesDir: insertRunFileSeriesMD(simtelOutFileDir, runNumTrunc) insertRunFileSeriesMD(simtelOutLogFileDir, runNumTrunc) insertRunFileSeriesMD(simtelOutHistFileDir, runNumTrunc) ###### simtel File level metadata ############################################ simtelFileMD = {} simtelFileMD['runNumber'] = int(run_number) simtelFileMD['jobID'] = jobID simtelFileMD['simtelReturnCode'] = simtelReturnCode result = fcc.setMetadata(simtelOutFileLFN, simtelFileMD) print "result setMetadata=", result if not result['OK']: print 'ResultSetMetadata:', result['Message'] result = fcc.setMetadata(simtelOutLogFileLFN, simtelFileMD) print "result setMetadata=", result if not result['OK']: print 'ResultSetMetadata:', result['Message'] result = fcc.setMetadata(simtelOutHistFileLFN, simtelFileMD) print "result setMetadata=", result if not result['OK']: print 'ResultSetMetadata:', result['Message'] result = fcc.addFileAncestors( {simtelOutFileLFN: { 'Ancestors': [corsikaFileLFN] }}) print 'result addFileAncestor:', result result = fcc.addFileAncestors( {simtelOutLogFileLFN: { 'Ancestors': [corsikaFileLFN] }}) print 'result addFileAncestor:', result result = fcc.addFileAncestors( {simtelOutHistFileLFN: { 'Ancestors': [corsikaFileLFN] }}) print 'result addFileAncestor:', result result = fcc.setMetadata(simtelOutFileLFN, simtelFileMD) if not result['OK']: print 'ResultSetMetadata:', result['Message'] DIRAC.exit()
file_name += '.gz' f_in = open(file_label, 'rb') f_out = gzip.open(file_name, 'wb') f_out.writelines(f_in) f_out.close() f_in.close() lfn=os.path.join('###LFN_BASE###', file_name) wildcard='###WILDCARD###' storage_elements=###SEs### with closing(open('###LOCATIONSFILE_NAME###','ab')) as locationsfile: for se in storage_elements: sys.stdout.write('\nUploading file: \"%s\" as \"%s\" at \"%s\"\n' % (file_name, lfn, se)) try: result = dirac.addFile(lfn, file_name, se) except Exception as x: sys.stdout.write('Exception running dirac.addFile command: %s' % str(x)) break if result.get('OK',False) and lfn in result.get('Value',{'Successful':{}})['Successful']: guid = dirac.getMetadata(lfn)['Value']['Successful'][lfn]['GUID'] locationsfile.write("DiracFile:::%s&&%s->%s:::['%s']:::%s\\n" % (wildcard, file_label, lfn, se, guid)) locationsfile.flush() locationsfile.close() break errmsg+="(%s, %s), " % (se, result) else: locationsfile.write("DiracFile:::%s&&%s->###FAILED###:::File '%s' could not be uploaded to any SE (%s):::NotAvailable\\n" % (wildcard, file_label, file_name, errmsg)) sys.stdout.write('Could not upload file %s' % file_name)