def test_addAndRemove(self):
# add a production step
prodStep = ProductionStep()
res = self.prodClient.addProductionStep(prodStep)
self.assertTrue(res["OK"])
# get the updated production description
prodDesc = self.prodClient.prodDescription
# create the production starting from the production description
res = self.prodClient.addProduction("prodName", json.dumps(prodDesc))
self.assertTrue(res["OK"])
prodID = res["Value"]
# try to add another production with the same Name (this should fail)
res = self.prodClient.addProduction("prodName", json.dumps(prodDesc))
self.assertFalse(res["OK"])
# delete the production
res = self.prodClient.deleteProduction(prodID)
print(type(prodID))
self.assertTrue(res["OK"])
# delete non existing one (fails)
res = self.prodClient.deleteProduction(prodID)
self.assertFalse(res["OK"])
def createProductionStep(name, type, inputQuery=None, outputQuery=None):
# create a production step
prodStep = ProductionStep()
prodStep.Name = name
prodStep.Type = type
prodStep.Inputquery = inputQuery
prodStep.Outputquery = outputQuery
return prodStep
def build_simulation_step(DL0_data_set, name_tag=''):
''' Setup Corsika + sim_telarray step
Note that there is no InputQuery,
since jobs created by this steps don't require any InputData
@return ProductionStep object
'''
# Note that there is no InputQuery,
# since jobs created by this steps don't require any InputData
DIRAC.gLogger.notice('MC Production step')
prod_step_1 = ProductionStep()
prod_step_1.Name = 'Simulation_%s' % DL0_data_set.replace('AdvancedBaseline_NSB1x_','')
prod_step_1.Name += '%s' % name_tag
prod_step_1.Type = 'MCSimulation'
prod_step_1.Outputquery = get_dataset_MQ(DL0_data_set)
prod_step_1.Outputquery['nsb'] = {'in': [1, 5]}
# get meta data to be passed to simulation job
site = prod_step_1.Outputquery['site']
particle = prod_step_1.Outputquery['particle']
if prod_step_1.Outputquery['phiP']['='] == 180:
pointing_dir = 'North'
elif prod_step_1.Outputquery['phiP']['='] == 0:
pointing_dir = 'South'
zenith_angle = prod_step_1.Outputquery['thetaP']['=']
# Here define the job description (i.e. Name, Executable, etc.)
# to be associated to the first ProductionStep, as done when using the TS
job1 = Prod5bMCPipeNSBJob()
job1.version = '2020-06-29b'
job1.compiler = 'gcc83_matchcpu'
# Initialize JOB_ID
job1.workflow.addParameter(Parameter("JOB_ID", "000000", "string", "", "",
True, False, "Temporary fix"))
# configuration
job1.setName('Prod5b_MC_Pipeline_NSB')
job1.set_site(site)
job1.set_particle(particle)
job1.set_pointing_dir(pointing_dir)
job1.zenith_angle = zenith_angle
job1.n_shower = 50000
if particle == 'gamma':
job1.n_shower = 20000
job1.setOutputSandbox(['*Log.txt'])
job1.start_run_number = '0'
job1.run_number = '@{JOB_ID}' # dynamic
job1.setupWorkflow(debug=False)
# Add the job description to the first ProductionStep
prod_step_1.Body = job1.workflow.toXML()
# return ProductionStep object
return prod_step_1
def build_evndisp_step(DL0_data_set, nsb=1):
''' Define a new EventDisplay analysis production step
@return ProductionStep object
'''
if nsb == 1:
DIRAC.gLogger.notice('NSB1x Analysis')
DL0_data_set_NSB = DL0_data_set
elif nsb == 5:
DIRAC.gLogger.notice('NSB5x Analysis')
DL0_data_set_NSB = DL0_data_set.replace('NSB1x', 'NSB5x')
prod_step_2 = ProductionStep()
prod_step_2.Name = 'Analysis_'+DL0_data_set_NSB.replace('AdvancedBaseline_', '').replace('DL0', 'DL1')
prod_step_2.Type = 'DataReprocessing' # This corresponds to the Transformation Type
prod_step_2.Inputquery = get_dataset_MQ(DL0_data_set_NSB)
prod_step_2.Outputquery = get_dataset_MQ(DL0_data_set_NSB.replace('DL0', 'DL1'))
# Here define the job description to be associated to the second ProductionStep
job2 = EvnDispProd5Job()
job2.setName('Prod5_EvnDisp')
# output
job2.setOutputSandbox(['*Log.txt'])
# refine output meta data if needed
output_meta_data = copy(prod_step_2.Outputquery)
job2.set_meta_data(output_meta_data)
job2.set_file_meta_data(nsb=output_meta_data['nsb']['='])
# check if La Palma else use default that is Paranal
if output_meta_data['site'] == 'LaPalma':
job2.prefix = "CTA.prod5N"
job2.layout_list = 'BL-0LSTs05MSTs-MSTF BL-0LSTs05MSTs-MSTN \
BL-4LSTs00MSTs-MSTN BL-4LSTs05MSTs-MSTF \
BL-4LSTs05MSTs-MSTN BL-4LSTs09MSTs-MSTF \
BL-4LSTs09MSTs-MSTN BL-4LSTs15MSTs-MSTF \
BL-4LSTs15MSTs-MSTN'
DIRAC.gLogger.notice('LaPalma layouts:\n',job2.layout_list.split())
elif output_meta_data['site'] == 'Paranal':
DIRAC.gLogger.notice('Paranal layouts:\n',job2.layout_list.split())
job2.ts_task_id = '@{JOB_ID}' # dynamic
job2.setupWorkflow(debug=False)
job2.setType('EvnDisp3') # mandatory *here*
prod_step_2.Body = job2.workflow.toXML()
prod_step_2.GroupSize = 5
return prod_step_2
def build_evndisp_step(DL0_data_set, nsb=1, name_tag=''):
''' Define a new EventDisplay analysis production step
@return ProductionStep object
'''
if nsb == 1:
DIRAC.gLogger.notice('NSB1x Analysis')
DL0_data_set_NSB = DL0_data_set
elif nsb == 5:
DIRAC.gLogger.notice('NSB5x Analysis')
DL0_data_set_NSB = DL0_data_set.replace('NSB1x', 'NSB5x')
prod_step_2 = ProductionStep()
prod_step_2.Name = 'Analysis_'+DL0_data_set_NSB.replace('AdvancedBaseline_', '').replace('DL0', 'DL1')
prod_step_2.Name += '%s' % name_tag
prod_step_2.Type = 'DataReprocessing' # This corresponds to the Transformation Type
prod_step_2.Inputquery = get_dataset_MQ(DL0_data_set_NSB)
prod_step_2.Outputquery = get_dataset_MQ(DL0_data_set_NSB.replace('DL0', 'DL1'))
# Here define the job description to be associated to the second ProductionStep
job2 = EvnDispProd5SingJob(cpuTime=259200.)
job2.version = 'eventdisplay-cta-dl1-prod5.v03'
job2.setName('Prod5b_EvnDisp_Singularity')
# output
job2.setOutputSandbox(['*Log.txt'])
# refine output meta data if needed
output_meta_data = copy(prod_step_2.Outputquery)
job2.set_meta_data(output_meta_data)
job2.set_file_meta_data(nsb=output_meta_data['nsb']['='])
job2.ts_task_id = '@{JOB_ID}' # dynamic
job2.group_size = 5 # for the input files verification
job2.setupWorkflow(debug=False)
job2.setType('EvnDisp3') # mandatory *here*
prod_step_2.Body = job2.workflow.toXML()
prod_step_2.GroupSize = 5 # have to match the above group size?
# return ProductionStep object
return prod_step_2
def test_SeqProduction(self):
# Define the first step of the production
prodStep1 = ProductionStep()
prodStep1.Name = 'Sim_prog'
prodStep1.Type = 'MCSimulation'
outputquery = {
'zenith': {
'in': [20, 40]
},
'particle': 'gamma',
'tel_sim_prog': 'simtel',
'outputType': {
'in': ['Data', 'Log']
}
}
prodStep1.Outputquery = outputquery
# Add the step to the production
res = self.prodClient.addProductionStep(prodStep1)
self.assertTrue(res['OK'])
# Define the second step of the production
prodStep2 = ProductionStep()
prodStep2.Name = 'Reco_prog'
prodStep2.Type = 'DataProcessing'
prodStep2.ParentStep = prodStep1
inputquery = {
'zenith': 20,
'particle': 'gamma',
'tel_sim_prog': 'simtel',
'outputType': 'Data'
}
outputquery = {
'zenith': 20,
'particle': 'gamma',
'analysis_prog': 'evndisp',
'data_level': 1,
'outputType': {
'in': ['Data', 'Log']
}
}
prodStep2.Inputquery = inputquery
prodStep2.Outputquery = outputquery
# Add the step to the production
res = self.prodClient.addProductionStep(prodStep2)
self.assertTrue(res['OK'])
# Define the third step of the production
prodStep3 = ProductionStep()
prodStep3.Name = 'Analyis_prog'
prodStep3.Type = 'DataProcessing'
prodStep3.ParentStep = prodStep2
inputquery = {
'zenith': 20,
'particle': 'gamma',
'analysis_prog': 'evndisp',
'data_level': 1,
'outputType': 'Data'
}
outputquery = {
'zenith': 20,
'particle': 'gamma',
'analysis_prog': 'evndisp',
'data_level': 2,
'outputType': {
'in': ['Data', 'Log']
}
}
prodStep3.Inputquery = inputquery
prodStep3.Outputquery = outputquery
# Add the step to the production
res = self.prodClient.addProductionStep(prodStep3)
self.assertTrue(res['OK'])
# Get the production description
prodDesc = self.prodClient.prodDescription
# Create the production
prodName = 'SeqProd'
res = self.prodClient.addProduction(prodName, json.dumps(prodDesc))
self.assertTrue(res['OK'])
# Start the production, i.e. instatiate the transformation steps
res = self.prodClient.startProduction(prodName)
self.assertTrue(res['OK'])
# Get the transformations of the production
res = self.prodClient.getProduction(prodName)
self.assertTrue(res['OK'])
prodID = res['Value']['ProductionID']
res = self.prodClient.getProductionTransformations(prodID)
self.assertTrue(res['OK'])
self.assertEqual(len(res['Value']), 3)
# Delete the production
res = self.prodClient.deleteProduction(prodName)
self.assertTrue(res['OK'])
def test_SeqProduction(self):
# Define the first step of the production
prodStep1 = ProductionStep()
prodStep1.Name = "Sim_prog"
prodStep1.Type = "MCSimulation"
outputquery = {
"zenith": {
"in": [20, 40]
},
"particle": "gamma",
"tel_sim_prog": "simtel",
"outputType": {
"in": ["Data", "Log"]
},
}
prodStep1.Outputquery = outputquery
# Add the step to the production
res = self.prodClient.addProductionStep(prodStep1)
self.assertTrue(res["OK"])
# Define the second step of the production
prodStep2 = ProductionStep()
prodStep2.Name = "Reco_prog"
prodStep2.Type = "DataProcessing"
prodStep2.ParentStep = prodStep1
inputquery = {
"zenith": 20,
"particle": "gamma",
"tel_sim_prog": "simtel",
"outputType": "Data"
}
outputquery = {
"zenith": 20,
"particle": "gamma",
"analysis_prog": "evndisp",
"data_level": 1,
"outputType": {
"in": ["Data", "Log"]
},
}
prodStep2.Inputquery = inputquery
prodStep2.Outputquery = outputquery
# Add the step to the production
res = self.prodClient.addProductionStep(prodStep2)
self.assertTrue(res["OK"])
# Define the third step of the production
prodStep3 = ProductionStep()
prodStep3.Name = "Analyis_prog"
prodStep3.Type = "DataProcessing"
prodStep3.ParentStep = prodStep2
inputquery = {
"zenith": 20,
"particle": "gamma",
"analysis_prog": "evndisp",
"data_level": 1,
"outputType": "Data",
}
outputquery = {
"zenith": 20,
"particle": "gamma",
"analysis_prog": "evndisp",
"data_level": 2,
"outputType": {
"in": ["Data", "Log"]
},
}
prodStep3.Inputquery = inputquery
prodStep3.Outputquery = outputquery
# Add the step to the production
res = self.prodClient.addProductionStep(prodStep3)
self.assertTrue(res["OK"])
# Get the production description
prodDesc = self.prodClient.prodDescription
# Create the production
prodName = f"SeqProd{random.randint(0, 100_000)}"
res = self.prodClient.addProduction(prodName, json.dumps(prodDesc))
self.assertTrue(res["OK"])
# Start the production, i.e. instatiate the transformation steps
res = self.prodClient.startProduction(prodName)
self.assertTrue(res["OK"])
# Get the transformations of the production
res = self.prodClient.getProduction(prodName)
self.assertTrue(res["OK"])
prodID = res["Value"]["ProductionID"]
res = self.prodClient.getProductionTransformations(prodID)
self.assertTrue(res["OK"])
self.assertEqual(len(res["Value"]), 3)
# Delete the production
res = self.prodClient.deleteProduction(prodName)
self.assertTrue(res["OK"])