def main():
from DIRAC.Core.Base import Script
Script.registerSwitch('', 'Path=', ' Path to search for')
Script.registerSwitch(
'', 'SE=',
' (comma-separated list of) SEs/SE-groups to be searched')
Script.parseCommandLine(ignoreErrors=True)
args = Script.getPositionalArgs()
import DIRAC
from DIRAC import gLogger
from DIRAC.Resources.Catalog.FileCatalog import FileCatalog
from DIRAC.DataManagementSystem.Client.MetaQuery import MetaQuery, FILE_STANDARD_METAKEYS
from DIRAC.DataManagementSystem.Utilities.DMSHelpers import resolveSEGroup
path = '/'
seList = None
for opt, val in Script.getUnprocessedSwitches():
if opt == 'Path':
path = val
elif opt == 'SE':
seList = resolveSEGroup(val.split(','))
if seList:
args.append("SE=%s" % ','.join(seList))
fc = FileCatalog()
result = fc.getMetadataFields()
if not result['OK']:
gLogger.error('Can not access File Catalog:', result['Message'])
DIRAC.exit(-1)
typeDict = result['Value']['FileMetaFields']
typeDict.update(result['Value']['DirectoryMetaFields'])
# Special meta tags
typeDict.update(FILE_STANDARD_METAKEYS)
if len(args) < 1:
print("Error: No argument provided\n%s:" % Script.scriptName)
gLogger.notice("MetaDataDictionary: \n%s" % str(typeDict))
Script.showHelp(exitCode=1)
mq = MetaQuery(typeDict=typeDict)
result = mq.setMetaQuery(args)
if not result['OK']:
gLogger.error("Illegal metaQuery:", result['Message'])
DIRAC.exit(-1)
metaDict = result['Value']
path = metaDict.pop('Path', path)
result = fc.findFilesByMetadata(metaDict, path)
if not result['OK']:
gLogger.error('Can not access File Catalog:', result['Message'])
DIRAC.exit(-1)
lfnList = sorted(result['Value'])
gLogger.notice('\n'.join(lfn for lfn in lfnList))
def runEvnDisp3(args=None):
""" Simple wrapper to create a EvnDisp3RefJob and setup parameters
from positional arguments given on the command line.
Parameters:
args -- infile mode
"""
DIRAC.gLogger.notice('runEvnDisp3')
# get arguments
transName = args[0]
################################
job = EvnDisp3RefTS2Job(cpuTime=432000) # to be adjusted!!
### Main Script ###
# override for testing
job.setName('EvnDisp3')
## add for testing
job.setType('EvnDisp3')
# defaults
# job.setLayout( "Baseline")
## package and version
# job.setPackage( 'evndisplay' )
# job.setVersion( 'prod3b_d20170602' )
# job.setReconstructionParameter( 'EVNDISP.prod3.reconstruction.runparameter.NN' )
# job.setNNcleaninginputcard( 'EVNDISP.NNcleaning.dat' )
# change here for Paranal or La Palma
job.setPrefix("CTA.prod3Nb")
# set calibration file and parameters file
job.setCalibrationFile(
'gamma_20deg_180deg_run3___cta-prod3-lapalma3-2147m-LaPalma.ped.root'
) # for La Palma
### set meta-data to the product of the transformation
# set query to add files to the transformation
MDdict = {
'MCCampaign': 'PROD3',
'particle': 'gamma',
'array_layout': 'Baseline',
'site': 'LaPalma',
'outputType': 'Data',
'data_level': {
"=": 0
},
'configuration_id': {
"=": 0
},
'tel_sim_prog': 'simtel',
'tel_sim_prog_version': '2017-04-19',
'thetaP': {
"=": 20
},
'phiP': {
"=": 0.0
}
}
job.setEvnDispMD(MDdict)
# add the sequence of executables
job.setTSTaskId('@{JOB_ID}') # dynamic
job.setupWorkflow(debug=False)
# output
job.setOutputSandbox(['*Log.txt'])
inputquery = MetaQuery(MDdict).getMetaQueryAsJson()
### submit the workflow to the TS
res = submitTS(job, transName, inputquery)
return res
from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient
fcc = FileCatalogClient('DataManagement/FileCatalog')
result = fcc.getMetadataFields()
if not result['OK']:
gLogger.error("Error: %s" % result['Message'])
exit(0)
if not result['Value']:
gLogger.error("Error: no metadata fields defined")
exit(0)
typeDict = result['Value']['FileMetaFields']
typeDict.update(result['Value']['DirectoryMetaFields'])
# Special meta tags
typeDict.update(FILE_STANDARD_METAKEYS)
mq = MetaQuery(typeDict=typeDict)
mq.setMetaQuery([metaTransfer])
query = mq.getMetaQuery()
gLogger.notice('Query: {0}'.format(query))
t = Transformation()
tc = TransformationClient()
t.setTransformationName(transformationName) # Must be unique
t.setTransformationGroup("Transfer")
t.setType(transformationType)
t.setPlugin(plugin)
t.setDescription("Data Transfer")
t.setLongDescription("Data Transfer") # Mandatory
t.setGroupSize(
groupSize
) # Here you specify how many files should be grouped within he same request, e.g. 100
if not result['OK']:
gLogger.error( 'Can not access File Catalog:', result['Message'] )
DIRAC.exit( -1 )
typeDict = result['Value']['FileMetaFields']
typeDict.update( result['Value']['DirectoryMetaFields'] )
# Special meta tags
typeDict.update( FILE_STANDARD_METAKEYS )
if len( args ) < 1:
print "Error: No argument provided\n%s:" % Script.scriptName
Script.showHelp()
gLogger.notice( "MetaDataDictionary: \n%s" % str( typeDict ) )
DIRAC.exit( -1 )
mq = MetaQuery( typeDict = typeDict )
result = mq.setMetaQuery( args )
if not result['OK']:
gLogger.error( "Illegal metaQuery:", result['Message'] )
DIRAC.exit( -1 )
metaDict = result['Value']
path = metaDict.pop( 'Path', path )
result = fc.findFilesByMetadata( metaDict, path )
if not result['OK']:
gLogger.error( 'Can not access File Catalog:', result['Message'] )
DIRAC.exit( -1 )
lfnList = sorted( result['Value'] )
gLogger.notice( '\n'.join( lfn for lfn in lfnList ) )
if len(args) < 1:
print "Error: No argument provided\n%s:" % Script.scriptName
Script.showHelp()
DIRAC.exit(-1)
fc = FileCatalog()
result = fc.getMetadataFields()
if not result['OK']:
gLogger.error('Can not access File Catalog:', result['Message'])
DIRAC.exit(-1)
typeDict = result['Value']['FileMetaFields']
typeDict.update(result['Value']['DirectoryMetaFields'])
# Special meta tags
typeDict.update(FILE_STANDARD_METAKEYS)
mq = MetaQuery(typeDict=typeDict)
result = mq.setMetaQuery(args)
if not result['OK']:
gLogger.error("Illegal metaQuery:", result['Message'])
DIRAC.exit(-1)
metaDict = result['Value']
path = metaDict.get('Path', '/')
metaDict.pop('Path')
print metaDict
result = fc.findFilesByMetadata(metaDict, path)
if not result['OK']:
gLogger.error('Can not access File Catalog:', result['Message'])
DIRAC.exit(-1)
lfnList = result['Value']
def runProd3( args = None ):
""" Simple wrapper to create a Prod3MCPipeBaselineJob and setup parameters
from positional arguments given on the command line.
Parameters:
args -- a list of 6 strings corresponding to job arguments
array_layout site particle pointing_dir zenith_angle nShower
demo LaPalma gamma South 20 1000
"""
# get arguments
transName = args[0]
layout = args[1]
site = args[2]
particle = args[3]
pointing = args[4]
zenith = args[5]
nShower= args[6]
### Main Script ###
job = Prod3MCPipeBaselineTS2Job()
# override for testing
job.setName('BL_LaPalma_20deg_%s'%particle)
# package and version
job.setPackage('corsika_simhessarray')
job.setVersion( '2017-04-19' ) # final with fix for gamma-diffuse
# layout, site, particle, pointing direction, zenith angle
# demo, LaPalma, gamma, South, 20
job.setArrayLayout(layout)
job.setSite(site)
job.setParticle(particle)
job.setPointingDir(pointing)
job.setZenithAngle( zenith )
####
job.setOutputDataLevel( 0 )
job.setConfigurationId( 0 )
# 5 is enough for testing
job.setNShower(nShower)
### Set the startrunNb here (it will be added to the Task_ID)
startrunNb = '0'
job.setStartRunNumber( startrunNb )
# set run number for TS submission: JOB_ID variable left for dynamic resolution during the Job. It corresponds to the Task_ID
job.setRunNumber( '@{JOB_ID}' )
# get dirac log files
job.setOutputSandbox( ['*Log.txt'] )
# add the sequence of executables
job.setupWorkflow(debug=False)
# set the OutputMetaQuery
outputquery = MetaQuery( job.outputquery ).getMetaQueryAsJson()
# submit to the Transformation System
res = submitTS( transName, job, outputquery )
# debug
Script.gLogger.info( job.workflow )
return res
def main():
Script.registerSwitch("", "Path=", " Path to search for")
Script.registerSwitch(
"", "SE=",
" (comma-separated list of) SEs/SE-groups to be searched")
# Registering arguments will automatically add their description to the help menu
Script.registerArgument(
[
"metaspec: metadata index specification (of the form: "
'"meta=value" or "meta<value", "meta!=value", etc.)'
],
mandatory=False,
)
Script.parseCommandLine(ignoreErrors=True)
args = Script.getPositionalArgs()
import DIRAC
from DIRAC import gLogger
from DIRAC.Resources.Catalog.FileCatalog import FileCatalog
from DIRAC.DataManagementSystem.Client.MetaQuery import MetaQuery, FILE_STANDARD_METAKEYS
from DIRAC.DataManagementSystem.Utilities.DMSHelpers import resolveSEGroup
path = "/"
seList = None
for opt, val in Script.getUnprocessedSwitches():
if opt == "Path":
path = val
elif opt == "SE":
seList = resolveSEGroup(val.split(","))
if seList:
args.append("SE=%s" % ",".join(seList))
fc = FileCatalog()
result = fc.getMetadataFields()
if not result["OK"]:
gLogger.error("Can not access File Catalog:", result["Message"])
DIRAC.exit(-1)
typeDict = result["Value"]["FileMetaFields"]
typeDict.update(result["Value"]["DirectoryMetaFields"])
# Special meta tags
typeDict.update(FILE_STANDARD_METAKEYS)
if len(args) < 1:
print("Error: No argument provided\n%s:" % Script.scriptName)
gLogger.notice("MetaDataDictionary: \n%s" % str(typeDict))
Script.showHelp(exitCode=1)
mq = MetaQuery(typeDict=typeDict)
result = mq.setMetaQuery(args)
if not result["OK"]:
gLogger.error("Illegal metaQuery:", result["Message"])
DIRAC.exit(-1)
metaDict = result["Value"]
path = metaDict.pop("Path", path)
result = fc.findFilesByMetadata(metaDict, path)
if not result["OK"]:
gLogger.error("Can not access File Catalog:", result["Message"])
DIRAC.exit(-1)
lfnList = sorted(result["Value"])
gLogger.notice("\n".join(lfn for lfn in lfnList))