def getWorkloadArguments():
baseArgs = AnalysisWorkloadFactory.getWorkloadArguments()
specArgs = {"RequestType" : {"default" : "PrivateMC", "optional" : True,
"attr" : "requestType"},
"PrimaryDataset" : {"default" : "MonteCarloData", "type" : str,
"optional" : False, "validate" : None,
"attr" : "inputPrimaryDataset", "null" : False},
"Seeding" : {"default" : "AutomaticSeeding", "type" : str,
"optional" : True, "validate" : lambda x : x in ["ReproducibleSeeding",
"AutomaticSeeding"],
"attr" : "seeding", "null" : False},
"FirstEvent" : {"default" : 1, "type" : int,
"optional" : True, "validate" : lambda x : x > 0,
"attr" : "firstEvent", "null" : False},
"FirstLumi" : {"default" : 1, "type" : int,
"optional" : True, "validate" : lambda x : x > 0,
"attr" : "firstLumi", "null" : False},
"MCPileup" : {"default" : None, "type" : str,
"optional" : True, "validate" : dataset,
"attr" : "mcPileup", "null" : False},
"DataPileup" : {"default" : None, "type" : str,
"optional" : True, "validate" : dataset,
"attr" : "dataPileup", "null" : False},
"TotalUnits" : {"default" : None, "type" : int,
"optional" : True, "validate" : lambda x : x > 0,
"attr" : "totalEvents", "null" : False}}
baseArgs["InputDataset"]["optional"] = True
baseArgs.update(specArgs)
AnalysisWorkloadFactory.setDefaultArgumentsProperty(baseArgs)
return baseArgs
if not hasattr(wmAgentConfig, "CoreDatabase"):
print "Your config is missing the CoreDatabase section."
socketLoc = getattr(wmAgentConfig.CoreDatabase, "socket", None)
connectUrl = getattr(wmAgentConfig.CoreDatabase, "connectUrl", None)
(dialect, junk) = connectUrl.split(":", 1)
myWMInit = WMInit()
myWMInit.setDatabaseConnection(dbConfig = connectUrl, dialect = dialect,
socketLoc = socketLoc)
workloadName = "CmsRunAnalysis-%s" % arguments["ProcessingVersion"]
workloadFile = "CmsRunAnalysis-%s.pkl" % arguments["ProcessingVersion"]
os.mkdir(workloadName)
cmsRunAna = AnalysisWorkloadFactory()
workload = cmsRunAna(workloadName, arguments)
taskMaker = TaskMaker(workload, os.path.join(os.getcwd(), workloadName))
taskMaker.skipSubscription = True
taskMaker.processWorkload()
workload.save(os.path.join(workloadName, workloadFile))
def doIndent(level):
myStr = ""
while level > 0:
myStr = myStr + " "
level -= 1
def makeWorkload(self, schema):
factory = AnalysisWorkloadFactory()
return factory(schema['RequestName'], schema).data
def testAnalysis(self):
"""
_testAnalysis_
"""
defaultArguments = getTestArguments()
defaultArguments["CouchUrl"] = os.environ["COUCHURL"]
defaultArguments["CouchDBName"] = "analysis_t"
defaultArguments["AnalysisConfigCacheDoc"] = self.injectAnalysisConfig(
)
defaultArguments["ProcessingVersion"] = 1
analysisProcessingFactory = AnalysisWorkloadFactory()
testWorkload = analysisProcessingFactory("TestWorkload",
defaultArguments)
testWorkload.setSpecUrl("somespec")
testWorkload.setOwnerDetails("*****@*****.**", "DMWM")
testWMBSHelper = WMBSHelper(testWorkload, "Analysis", "SomeBlock")
testWMBSHelper.createTopLevelFileset()
testWMBSHelper.createSubscription(testWMBSHelper.topLevelTask,
testWMBSHelper.topLevelFileset)
procWorkflow = Workflow(name="TestWorkload",
task="/TestWorkload/Analysis")
procWorkflow.load()
self.assertEqual(len(procWorkflow.outputMap.keys()), 2,
"Error: Wrong number of WF outputs.")
logArchOutput = procWorkflow.outputMap["logArchive"][0][
"merged_output_fileset"] #Actually Analysis does not have a merge task
unmergedLogArchOutput = procWorkflow.outputMap["logArchive"][0][
"output_fileset"]
logArchOutput.loadData()
unmergedLogArchOutput.loadData()
self.assertEqual(logArchOutput.name,
"/TestWorkload/Analysis/unmerged-logArchive",
"Error: LogArchive output fileset is wrong.")
self.assertEqual(unmergedLogArchOutput.name,
"/TestWorkload/Analysis/unmerged-logArchive",
"Error: LogArchive output fileset is wrong.")
output = procWorkflow.outputMap["output"][0]["output_fileset"]
mergedOutput = procWorkflow.outputMap["output"][0][
"merged_output_fileset"]
output.loadData()
mergedOutput.loadData()
self.assertEqual(
output.name, "/TestWorkload/Analysis/unmerged-output",
"Error: Unmerged output fileset is wrong: " + output.name)
self.assertEqual(
mergedOutput.name, "/TestWorkload/Analysis/unmerged-output",
"Error: Unmerged output fileset is wrong: " + mergedOutput.name)
topLevelFileset = Fileset(name="TestWorkload-Analysis-SomeBlock")
topLevelFileset.loadData()
procSubscription = Subscription(fileset=topLevelFileset,
workflow=procWorkflow)
procSubscription.loadData()
self.assertEqual(procSubscription["type"], "Analysis",
"Error: Wrong subscription type.")
self.assertEqual(procSubscription["split_algo"], "EventBased",
"Error: Wrong split algo.")
procLogCollect = Fileset(
name="/TestWorkload/Analysis/unmerged-logArchive")
procLogCollect.loadData()
procLogCollectWorkflow = Workflow(
name="TestWorkload", task="/TestWorkload/Analysis/LogCollect")
procLogCollectWorkflow.load()
logCollectSub = Subscription(fileset=procLogCollect,
workflow=procLogCollectWorkflow)
logCollectSub.loadData()
self.assertEqual(logCollectSub["type"], "LogCollect",
"Error: Wrong subscription type.")
self.assertEqual(logCollectSub["split_algo"], "MinFileBased",
"Error: Wrong split algo.")
def testAnalysis(self):
"""
_testAnalysis_
"""
defaultArguments = AnalysisWorkloadFactory.getTestArguments()
defaultArguments["CouchUrl"] = os.environ["COUCHURL"]
defaultArguments["CouchDBName"] = "analysis_t"
defaultArguments["AnalysisConfigCacheDoc"] = self.injectAnalysisConfig()
defaultArguments["ProcessingVersion"] = 1
analysisProcessingFactory = AnalysisWorkloadFactory()
testWorkload = analysisProcessingFactory.factoryWorkloadConstruction("TestWorkload", defaultArguments)
testWMBSHelper = WMBSHelper(testWorkload, "Analysis", "SomeBlock", cachepath = self.testDir)
testWMBSHelper.createTopLevelFileset()
testWMBSHelper._createSubscriptionsInWMBS(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset)
procWorkflow = Workflow(name = "TestWorkload",
task = "/TestWorkload/Analysis")
procWorkflow.load()
self.assertEqual(len(procWorkflow.outputMap.keys()), 2,
"Error: Wrong number of WF outputs.")
logArchOutput = procWorkflow.outputMap["logArchive"][0]["merged_output_fileset"]#Actually Analysis does not have a merge task
unmergedLogArchOutput = procWorkflow.outputMap["logArchive"][0]["output_fileset"]
logArchOutput.loadData()
unmergedLogArchOutput.loadData()
self.assertEqual(logArchOutput.name, "/TestWorkload/Analysis/unmerged-logArchive",
"Error: LogArchive output fileset is wrong.")
self.assertEqual(unmergedLogArchOutput.name, "/TestWorkload/Analysis/unmerged-logArchive",
"Error: LogArchive output fileset is wrong.")
output = procWorkflow.outputMap["output"][0]["output_fileset"]
mergedOutput = procWorkflow.outputMap["output"][0]["merged_output_fileset"]
output.loadData()
mergedOutput.loadData()
self.assertEqual(output.name, "/TestWorkload/Analysis/unmerged-output",
"Error: Unmerged output fileset is wrong: " + output.name)
self.assertEqual(mergedOutput.name, "/TestWorkload/Analysis/unmerged-output",
"Error: Unmerged output fileset is wrong: " + mergedOutput.name)
topLevelFileset = Fileset(name = "TestWorkload-Analysis-SomeBlock")
topLevelFileset.loadData()
procSubscription = Subscription(fileset = topLevelFileset, workflow = procWorkflow)
procSubscription.loadData()
self.assertEqual(procSubscription["type"], "Analysis",
"Error: Wrong subscription type.")
self.assertEqual(procSubscription["split_algo"], "EventBased",
"Error: Wrong split algo.")
procLogCollect = Fileset(name = "/TestWorkload/Analysis/unmerged-logArchive")
procLogCollect.loadData()
procLogCollectWorkflow = Workflow(name = "TestWorkload",
task = "/TestWorkload/Analysis/LogCollect")
procLogCollectWorkflow.load()
logCollectSub = Subscription(fileset = procLogCollect, workflow = procLogCollectWorkflow)
logCollectSub.loadData()
self.assertEqual(logCollectSub["type"], "LogCollect",
"Error: Wrong subscription type.")
self.assertEqual(logCollectSub["split_algo"], "MinFileBased",
"Error: Wrong split algo.")
