def main(): from DIRAC.Core.Base import Script Script.registerSwitch('', 'Path=', ' Path to search for') Script.registerSwitch( '', 'SE=', ' (comma-separated list of) SEs/SE-groups to be searched') Script.parseCommandLine(ignoreErrors=True) args = Script.getPositionalArgs() import DIRAC from DIRAC import gLogger from DIRAC.Resources.Catalog.FileCatalog import FileCatalog from DIRAC.DataManagementSystem.Client.MetaQuery import MetaQuery, FILE_STANDARD_METAKEYS from DIRAC.DataManagementSystem.Utilities.DMSHelpers import resolveSEGroup path = '/' seList = None for opt, val in Script.getUnprocessedSwitches(): if opt == 'Path': path = val elif opt == 'SE': seList = resolveSEGroup(val.split(',')) if seList: args.append("SE=%s" % ','.join(seList)) fc = FileCatalog() result = fc.getMetadataFields() if not result['OK']: gLogger.error('Can not access File Catalog:', result['Message']) DIRAC.exit(-1) typeDict = result['Value']['FileMetaFields'] typeDict.update(result['Value']['DirectoryMetaFields']) # Special meta tags typeDict.update(FILE_STANDARD_METAKEYS) if len(args) < 1: print("Error: No argument provided\n%s:" % Script.scriptName) gLogger.notice("MetaDataDictionary: \n%s" % str(typeDict)) Script.showHelp(exitCode=1) mq = MetaQuery(typeDict=typeDict) result = mq.setMetaQuery(args) if not result['OK']: gLogger.error("Illegal metaQuery:", result['Message']) DIRAC.exit(-1) metaDict = result['Value'] path = metaDict.pop('Path', path) result = fc.findFilesByMetadata(metaDict, path) if not result['OK']: gLogger.error('Can not access File Catalog:', result['Message']) DIRAC.exit(-1) lfnList = sorted(result['Value']) gLogger.notice('\n'.join(lfn for lfn in lfnList))
def runEvnDisp3(args=None): """ Simple wrapper to create a EvnDisp3RefJob and setup parameters from positional arguments given on the command line. Parameters: args -- infile mode """ DIRAC.gLogger.notice('runEvnDisp3') # get arguments transName = args[0] ################################ job = EvnDisp3RefTS2Job(cpuTime=432000) # to be adjusted!! ### Main Script ### # override for testing job.setName('EvnDisp3') ## add for testing job.setType('EvnDisp3') # defaults # job.setLayout( "Baseline") ## package and version # job.setPackage( 'evndisplay' ) # job.setVersion( 'prod3b_d20170602' ) # job.setReconstructionParameter( 'EVNDISP.prod3.reconstruction.runparameter.NN' ) # job.setNNcleaninginputcard( 'EVNDISP.NNcleaning.dat' ) # change here for Paranal or La Palma job.setPrefix("CTA.prod3Nb") # set calibration file and parameters file job.setCalibrationFile( 'gamma_20deg_180deg_run3___cta-prod3-lapalma3-2147m-LaPalma.ped.root' ) # for La Palma ### set meta-data to the product of the transformation # set query to add files to the transformation MDdict = { 'MCCampaign': 'PROD3', 'particle': 'gamma', 'array_layout': 'Baseline', 'site': 'LaPalma', 'outputType': 'Data', 'data_level': { "=": 0 }, 'configuration_id': { "=": 0 }, 'tel_sim_prog': 'simtel', 'tel_sim_prog_version': '2017-04-19', 'thetaP': { "=": 20 }, 'phiP': { "=": 0.0 } } job.setEvnDispMD(MDdict) # add the sequence of executables job.setTSTaskId('@{JOB_ID}') # dynamic job.setupWorkflow(debug=False) # output job.setOutputSandbox(['*Log.txt']) inputquery = MetaQuery(MDdict).getMetaQueryAsJson() ### submit the workflow to the TS res = submitTS(job, transName, inputquery) return res
from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient fcc = FileCatalogClient('DataManagement/FileCatalog') result = fcc.getMetadataFields() if not result['OK']: gLogger.error("Error: %s" % result['Message']) exit(0) if not result['Value']: gLogger.error("Error: no metadata fields defined") exit(0) typeDict = result['Value']['FileMetaFields'] typeDict.update(result['Value']['DirectoryMetaFields']) # Special meta tags typeDict.update(FILE_STANDARD_METAKEYS) mq = MetaQuery(typeDict=typeDict) mq.setMetaQuery([metaTransfer]) query = mq.getMetaQuery() gLogger.notice('Query: {0}'.format(query)) t = Transformation() tc = TransformationClient() t.setTransformationName(transformationName) # Must be unique t.setTransformationGroup("Transfer") t.setType(transformationType) t.setPlugin(plugin) t.setDescription("Data Transfer") t.setLongDescription("Data Transfer") # Mandatory t.setGroupSize( groupSize ) # Here you specify how many files should be grouped within he same request, e.g. 100
if not result['OK']: gLogger.error( 'Can not access File Catalog:', result['Message'] ) DIRAC.exit( -1 ) typeDict = result['Value']['FileMetaFields'] typeDict.update( result['Value']['DirectoryMetaFields'] ) # Special meta tags typeDict.update( FILE_STANDARD_METAKEYS ) if len( args ) < 1: print "Error: No argument provided\n%s:" % Script.scriptName Script.showHelp() gLogger.notice( "MetaDataDictionary: \n%s" % str( typeDict ) ) DIRAC.exit( -1 ) mq = MetaQuery( typeDict = typeDict ) result = mq.setMetaQuery( args ) if not result['OK']: gLogger.error( "Illegal metaQuery:", result['Message'] ) DIRAC.exit( -1 ) metaDict = result['Value'] path = metaDict.pop( 'Path', path ) result = fc.findFilesByMetadata( metaDict, path ) if not result['OK']: gLogger.error( 'Can not access File Catalog:', result['Message'] ) DIRAC.exit( -1 ) lfnList = sorted( result['Value'] ) gLogger.notice( '\n'.join( lfn for lfn in lfnList ) )
if len(args) < 1: print "Error: No argument provided\n%s:" % Script.scriptName Script.showHelp() DIRAC.exit(-1) fc = FileCatalog() result = fc.getMetadataFields() if not result['OK']: gLogger.error('Can not access File Catalog:', result['Message']) DIRAC.exit(-1) typeDict = result['Value']['FileMetaFields'] typeDict.update(result['Value']['DirectoryMetaFields']) # Special meta tags typeDict.update(FILE_STANDARD_METAKEYS) mq = MetaQuery(typeDict=typeDict) result = mq.setMetaQuery(args) if not result['OK']: gLogger.error("Illegal metaQuery:", result['Message']) DIRAC.exit(-1) metaDict = result['Value'] path = metaDict.get('Path', '/') metaDict.pop('Path') print metaDict result = fc.findFilesByMetadata(metaDict, path) if not result['OK']: gLogger.error('Can not access File Catalog:', result['Message']) DIRAC.exit(-1) lfnList = result['Value']
def runProd3( args = None ): """ Simple wrapper to create a Prod3MCPipeBaselineJob and setup parameters from positional arguments given on the command line. Parameters: args -- a list of 6 strings corresponding to job arguments array_layout site particle pointing_dir zenith_angle nShower demo LaPalma gamma South 20 1000 """ # get arguments transName = args[0] layout = args[1] site = args[2] particle = args[3] pointing = args[4] zenith = args[5] nShower= args[6] ### Main Script ### job = Prod3MCPipeBaselineTS2Job() # override for testing job.setName('BL_LaPalma_20deg_%s'%particle) # package and version job.setPackage('corsika_simhessarray') job.setVersion( '2017-04-19' ) # final with fix for gamma-diffuse # layout, site, particle, pointing direction, zenith angle # demo, LaPalma, gamma, South, 20 job.setArrayLayout(layout) job.setSite(site) job.setParticle(particle) job.setPointingDir(pointing) job.setZenithAngle( zenith ) #### job.setOutputDataLevel( 0 ) job.setConfigurationId( 0 ) # 5 is enough for testing job.setNShower(nShower) ### Set the startrunNb here (it will be added to the Task_ID) startrunNb = '0' job.setStartRunNumber( startrunNb ) # set run number for TS submission: JOB_ID variable left for dynamic resolution during the Job. It corresponds to the Task_ID job.setRunNumber( '@{JOB_ID}' ) # get dirac log files job.setOutputSandbox( ['*Log.txt'] ) # add the sequence of executables job.setupWorkflow(debug=False) # set the OutputMetaQuery outputquery = MetaQuery( job.outputquery ).getMetaQueryAsJson() # submit to the Transformation System res = submitTS( transName, job, outputquery ) # debug Script.gLogger.info( job.workflow ) return res
def main(): Script.registerSwitch("", "Path=", " Path to search for") Script.registerSwitch( "", "SE=", " (comma-separated list of) SEs/SE-groups to be searched") # Registering arguments will automatically add their description to the help menu Script.registerArgument( [ "metaspec: metadata index specification (of the form: " '"meta=value" or "meta<value", "meta!=value", etc.)' ], mandatory=False, ) Script.parseCommandLine(ignoreErrors=True) args = Script.getPositionalArgs() import DIRAC from DIRAC import gLogger from DIRAC.Resources.Catalog.FileCatalog import FileCatalog from DIRAC.DataManagementSystem.Client.MetaQuery import MetaQuery, FILE_STANDARD_METAKEYS from DIRAC.DataManagementSystem.Utilities.DMSHelpers import resolveSEGroup path = "/" seList = None for opt, val in Script.getUnprocessedSwitches(): if opt == "Path": path = val elif opt == "SE": seList = resolveSEGroup(val.split(",")) if seList: args.append("SE=%s" % ",".join(seList)) fc = FileCatalog() result = fc.getMetadataFields() if not result["OK"]: gLogger.error("Can not access File Catalog:", result["Message"]) DIRAC.exit(-1) typeDict = result["Value"]["FileMetaFields"] typeDict.update(result["Value"]["DirectoryMetaFields"]) # Special meta tags typeDict.update(FILE_STANDARD_METAKEYS) if len(args) < 1: print("Error: No argument provided\n%s:" % Script.scriptName) gLogger.notice("MetaDataDictionary: \n%s" % str(typeDict)) Script.showHelp(exitCode=1) mq = MetaQuery(typeDict=typeDict) result = mq.setMetaQuery(args) if not result["OK"]: gLogger.error("Illegal metaQuery:", result["Message"]) DIRAC.exit(-1) metaDict = result["Value"] path = metaDict.pop("Path", path) result = fc.findFilesByMetadata(metaDict, path) if not result["OK"]: gLogger.error("Can not access File Catalog:", result["Message"]) DIRAC.exit(-1) lfnList = sorted(result["Value"]) gLogger.notice("\n".join(lfn for lfn in lfnList))