def help(self, short=False):
"""Print a short introduction and 'cheat sheet' for the Ganga Tasks package"""
print('')
print(markup(" *** Ganga Tasks: Short Introduction and 'Cheat Sheet' ***", fgcol("blue")))
print('')
print(markup("Definitions: ", fgcol("red")) + "'Partition' - A unit of processing, for example processing a file or processing some events from a file")
print(" 'Transform' - A group of partitions that have a common Ganga Application and Backend.")
print(" 'Task' - A group of one or more 'Transforms' that can have dependencies on each other")
print('')
print(markup("Possible status values for partitions:", fgcol("red")))
print(' * "' + markup("ready", overview_colours["ready"]) + '" - ready to be executed ')
print(' * "' + markup("hold", overview_colours["hold"]) + '" - dependencies not completed')
print(' * "' + markup("running", overview_colours["running"]) + '" - at least one job tries to process this partition')
print(' * "' + markup("attempted", overview_colours["attempted"]) + '"- tasks tried to process this partition, but has not yet succeeded')
print(' * "' + markup("failed", overview_colours["failed"]) + '" - tasks failed to process this partition several times')
print(' * "' + markup("bad", overview_colours["bad"]) + '" - this partition is excluded from further processing and will not be used as input to subsequent transforms')
print(' * "' + markup("completed", overview_colours["completed"]) + '" ')
print('')
def c(s):
return markup(s, fgcol("blue"))
print(markup("Important commands:", fgcol("red")))
print(" Get a quick overview : " + c("tasks") + " Get a detailed view : " + c("tasks.table()"))
print(" Access an existing task : " + c("t = tasks(id)") + " Remove a Task : " + c("tasks(id).remove()"))
print(" Create a new (MC) Task : " + c("t = MCTask()") + " Copy a Task : " + c("nt = t.copy()"))
print(" Show task configuration : " + c("t.info()") + " Show processing status : " + c("t.overview()"))
print(" Set the float of a Task : " + c("t.float = 100") + " Set the name of a task : " + c("t.name = 'My Own Task v1'"))
print(" Start processing : " + c("t.run()") + " Pause processing : " + c("t.pause()"))
print(" Access Transform id N : " + c("tf = t.transforms[N]") + " Pause processing of tf : " + c( "tf.pause()") + " # This command is reverted by using t.run()")
print(" Transform Application : " + c("tf.application") + " Transform Backend : " + c("tf.backend"))
print('')
print(" Set parameter in all applications : " + c("t.setParameter(my_software_version='1.42.0')"))
print(" Set backend for all transforms : " + c("t.setBackend(backend) , p.e. t.setBackend(LCG())"))
print(" Limit on how often jobs are resubmitted : " + c("tf.run_limit = 4"))
print(" Manually change the status of partitions: " + c("tf.setPartitionStatus(partition, 'status')"))
print('')
print(" For an ATLAS Monte Carlo Production Example and specific help type: " + c("MCTask?"))
print(" For an ATLAS Analysis Example and help type: " + c("AnaTask?"))
print('')
if not True:
# if not short:
print("ADVANCED COMMANDS:")
print("Add Transform at position N : t.insertTransform(N, transform)")
print("Remove Transform at position N : t.removeTransform(N)")
print("Set Transform Application : tf.application = TaskApp() #This Application must be a 'Task Version' of the usual application")
print(" Adding Task Versions of Applications is easy, contact the developers to request an inclusion")
def __str__(self, short=True, id=None):
"""Prints an overview over the currently running tasks"""
if Ganga.GPIDev.Lib.Registry.RegistrySlice.config["tasks_show_help"]:
self.help(short=True)
Ganga.GPIDev.Lib.Registry.RegistrySlice.config.setUserValue("tasks_show_help", False)
print("To show this help message again, type 'tasks.help()'.")
print('')
print(" The following is the output of " + markup("tasks.table()", fgcol("blue")))
lenfstring = 0
flist = []
for thing in Ganga.GPIDev.Lib.Registry.RegistrySlice.config["tasks_columns"]:
width = Ganga.GPIDev.Lib.Registry.RegistrySlice.config["tasks_columns_width"][thing]
lenfstring += width
flist.append("%" + str(width) + "s ")
fstring = "|".join(flist)
fstring += '\n'
lenfstring += 27
ds = fstring % ("#", "Type", "Name", "State", "Comment", "%4s: %4s/ %4s/ %4s/ %4s/ %4s/ %4s/ %4s" % ("Jobs", markup("done", overview_colours["completed"]), " " + markup("run", overview_colours["running"]), " " + markup("subd", overview_colours["submitted"]), " " + markup("attd", overview_colours["attempted"]), markup("fail", overview_colours["failed"]), markup("hold", overview_colours["hold"]), " " + markup("bad", overview_colours["bad"])), "Float")
ds += "-" * lenfstring + "\n"
from Ganga.GPIDev.Lib.Tasks import ITask
if id is not None and type(id) is int:
iterable_list = [self[id]]
else:
iterable_list = stripProxy(self).objects.values()
for p in iterable_list:
if isType(p, ITask):
stat = "%4i: %4i/ %4i/ %4i/ --/ %4i/ %4i/ %4i" % (p.n_all(), p.n_status("completed"), p.n_status("running"), p.n_status("submitted"), p.n_status("failed"), p.n_status("hold"), p.n_status("bad"))
ds += markup(fstring % (p.id, getName(p), p.name[0:Ganga.GPIDev.Lib.Registry.RegistrySlice.config['tasks_columns_width']['Name']], p.status, p.comment, stat, p.float), status_colours[p.status])
else:
stat = "%4i: %4i/ %4i/ --/ %4i/ %4i/ %4i/ %4i" % (p.n_all(), p.n_status("completed"), p.n_status("running"), p.n_status("attempted"), p.n_status("failed"), p.n_status("hold"), p.n_status("bad"))
ds += markup(fstring % (p.id, getName(p), p.name[0:Ganga.GPIDev.Lib.Registry.RegistrySlice.config['tasks_columns_width']['Name']], p.status, p.comment, stat, p.float), status_colours[p.status])
if short is True:
continue
for ti in range(0, len(p.transforms)):
t = p.transforms[ti]
if isType(p, ITask):
stat = "%4i: %4i/ %4i/ %4i/ --/ %4i/ %4i/ %4s" % (t.n_all(), t.n_status("completed"), t.n_status("running"), t.n_status("submitted"), t.n_status("failed"), t.n_status("hold"), t.n_status("bad"))
ds += "\n" + markup(fstring % ("%i.%i" % (p.id, ti), getName(t), t.name[0:Ganga.GPIDev.Lib.Registry.RegistrySlice.config['tasks_columns_width']['Name']], t.status, "", stat, ""), status_colours[t.status])
else:
stat = "%4i: %4i/ %4i/ --/ %4i/ %4i/ %4i/ %4s" % (t.n_all(), t.n_status("completed"), t.n_status("running"), t.n_status("attempted"), t.n_status("failed"), t.n_status("hold"), t.n_status("bad"))
ds += "zn" + markup(fstring % ("%i.%i" % (p.id, ti), getName(t), t.name[0:Ganga.GPIDev.Lib.Registry.RegistrySlice.config['tasks_columns_width']['Name']], t.status, "", stat, ""), status_colours[t.status])
ds += "-" * lenfstring + "\n"
return ds + "\n"
def c(s):
return markup(s, fgcol("blue"))
from GangaAtlas.Lib.Credentials.ProxyHelper import getNickname
from dq2.clientapi.DQ2 import DQ2, DQUnknownDatasetException, DQDatasetExistsException, DQFileExistsInDatasetException, DQInvalidRequestException
from dq2.container.exceptions import DQContainerAlreadyHasDataset, DQContainerDoesNotHaveDataset
from dq2.common.DQException import DQException
from Ganga.GPIDev.Schema import *
import copy
o = [""]
def c(s):
return markup(s, fgcol("blue"))
o.append(markup(" *** Task for Multiple Athena Analyses ***", fgcol("blue")))
#o.append(" Analysis Application : "+c("t.analysis.application"))
#o.append(" Set Dataset : "+c("t.setDataset('my.dataset')"))
#o.append(" Input Dataset Object : "+c("t.analysis.inputdata"))
#o.append(" Output Dataset Object : "+c("t.analysis.outputdata"))
#o.append(" Files processed per job : "+c("t.analysis.files_per_job = 10"))
#o.append("")
#o.append(markup("Procedure to do a usual analysis:", fgcol("red")))
##o.append("config.Tasks.merged_files_per_job = 1 # default files per job for merged datasets")
##o.append("config.Tasks.recon_files_per_job = 10 # default files per job for recon (non-merged) datasets")
#o.append("t = AnaTask()")
#o.append('t.name = "MyAnalysisR1"')
#o.append("t.analysis.outputdata.outputdata = ['nTuple.root' ]")
#o.append("t.analysis.application.option_file = ['./myTopOptions.py' ]")
#o.append("t.analysis.application.prepare()")
#o.append("t.float = 10")
def c(s):
return markup(s, fgcol("blue"))
from GangaLHCb.Lib.LHCbDataset.OutputData import OutputData
from LHCbTaskDummySplitter import LHCbTaskDummySplitter
from Ganga.Core import GangaException, GangaAttributeError
from GangaLHCb.Lib.LHCbDataset import LHCbDataset
import Ganga.Utility.Config
from copy import deepcopy
import os
from Ganga.Utility.logging import getLogger
markup = ANSIMarkup()
logger = getLogger(modulename=True)
config = Ganga.Utility.Config.getConfig('Configuration')
partition_colours = {
'ignored': "",
'hold': fgcol("lgray"),
'ready': fgcol("lgreen"),
'running': fgcol("green"),
'completed': fgcol("blue"),
'attempted': fgcol("yellow"),
'failed': fgcol("lred"),
'bad': fgcol("red"),
'unknown': fgcol("white"),
}
job_colours = {
'new': col("black", "white"),
'submitting': col("black", "orange"),
'submitted': col("white", "orange"),
'running': col("black", "green"),
'completing': col("white", "green"),
def __str__(self, short=True, id=None):
"""Prints an overview over the currently running tasks"""
if Ganga.GPIDev.Lib.Registry.RegistrySlice.config["tasks_show_help"]:
self.help(short=True)
Ganga.GPIDev.Lib.Registry.RegistrySlice.config.setUserValue(
"tasks_show_help", False)
print("To show this help message again, type 'tasks.help()'.")
print('')
print(" The following is the output of " +
markup("tasks.table()", fgcol("blue")))
lenfstring = 0
flist = []
for thing in Ganga.GPIDev.Lib.Registry.RegistrySlice.config[
"tasks_columns"]:
width = Ganga.GPIDev.Lib.Registry.RegistrySlice.config[
"tasks_columns_width"][thing]
lenfstring += width
flist.append("%" + str(width) + "s ")
fstring = "|".join(flist)
fstring += '\n'
lenfstring += 27
ds = fstring % ("#", "Type", "Name", "State", "Comment",
"%4s: %4s/ %4s/ %4s/ %4s/ %4s/ %4s/ %4s" %
("Jobs", markup("done", overview_colours["completed"]),
" " + markup("run", overview_colours["running"]),
" " + markup("subd", overview_colours["submitted"]),
" " + markup("attd", overview_colours["attempted"]),
markup("fail", overview_colours["failed"]),
markup("hold", overview_colours["hold"]), " " +
markup("bad", overview_colours["bad"])), "Float")
ds += "-" * lenfstring + "\n"
from Ganga.GPIDev.Lib.Tasks import ITask
if id is not None and type(id) is int:
iterable_list = [self[id]]
else:
iterable_list = stripProxy(self).objects.values()
for p in iterable_list:
if isType(p, ITask):
stat = "%4i: %4i/ %4i/ %4i/ --/ %4i/ %4i/ %4i" % (
p.n_all(), p.n_status("completed"), p.n_status("running"),
p.n_status("submitted"), p.n_status("failed"),
p.n_status("hold"), p.n_status("bad"))
ds += markup(
fstring %
(p.id, getName(p),
p.name[0:Ganga.GPIDev.Lib.Registry.RegistrySlice.
config['tasks_columns_width']['Name']], p.status,
p.comment, stat, p.float), status_colours[p.status])
else:
stat = "%4i: %4i/ %4i/ --/ %4i/ %4i/ %4i/ %4i" % (
p.n_all(), p.n_status("completed"), p.n_status("running"),
p.n_status("attempted"), p.n_status("failed"),
p.n_status("hold"), p.n_status("bad"))
ds += markup(
fstring %
(p.id, getName(p),
p.name[0:Ganga.GPIDev.Lib.Registry.RegistrySlice.
config['tasks_columns_width']['Name']], p.status,
p.comment, stat, p.float), status_colours[p.status])
if short is True:
continue
for ti in range(0, len(p.transforms)):
t = p.transforms[ti]
if isType(p, ITask):
stat = "%4i: %4i/ %4i/ %4i/ --/ %4i/ %4i/ %4s" % (
t.n_all(), t.n_status("completed"),
t.n_status("running"), t.n_status("submitted"),
t.n_status("failed"), t.n_status("hold"),
t.n_status("bad"))
ds += "\n" + markup(
fstring %
("%i.%i" % (p.id, ti), getName(t),
t.name[0:Ganga.GPIDev.Lib.Registry.RegistrySlice.
config['tasks_columns_width']['Name']],
t.status, "", stat, ""), status_colours[t.status])
else:
stat = "%4i: %4i/ %4i/ --/ %4i/ %4i/ %4i/ %4s" % (
t.n_all(), t.n_status("completed"),
t.n_status("running"), t.n_status("attempted"),
t.n_status("failed"), t.n_status("hold"),
t.n_status("bad"))
ds += "zn" + markup(
fstring %
("%i.%i" % (p.id, ti), getName(t),
t.name[0:Ganga.GPIDev.Lib.Registry.RegistrySlice.
config['tasks_columns_width']['Name']],
t.status, "", stat, ""), status_colours[t.status])
ds += "-" * lenfstring + "\n"
return ds + "\n"
def help(self, short=False):
"""Print a short introduction and 'cheat sheet' for the Ganga Tasks package"""
print('')
print(
markup(
" *** Ganga Tasks: Short Introduction and 'Cheat Sheet' ***",
fgcol("blue")))
print('')
print(
markup("Definitions: ", fgcol("red")) +
"'Partition' - A unit of processing, for example processing a file or processing some events from a file"
)
print(
" 'Transform' - A group of partitions that have a common Ganga Application and Backend."
)
print(
" 'Task' - A group of one or more 'Transforms' that can have dependencies on each other"
)
print('')
print(markup("Possible status values for partitions:", fgcol("red")))
print(' * "' + markup("ready", overview_colours["ready"]) +
'" - ready to be executed ')
print(' * "' + markup("hold", overview_colours["hold"]) +
'" - dependencies not completed')
print(' * "' + markup("running", overview_colours["running"]) +
'" - at least one job tries to process this partition')
print(
' * "' + markup("attempted", overview_colours["attempted"]) +
'"- tasks tried to process this partition, but has not yet succeeded'
)
print(' * "' + markup("failed", overview_colours["failed"]) +
'" - tasks failed to process this partition several times')
print(
' * "' + markup("bad", overview_colours["bad"]) +
'" - this partition is excluded from further processing and will not be used as input to subsequent transforms'
)
print(' * "' + markup("completed", overview_colours["completed"]) +
'" ')
print('')
def c(s):
return markup(s, fgcol("blue"))
print(markup("Important commands:", fgcol("red")))
print(" Get a quick overview : " + c("tasks") +
" Get a detailed view : " +
c("tasks.table()"))
print(" Access an existing task : " + c("t = tasks(id)") +
" Remove a Task : " + c("tasks(id).remove()"))
print(" Create a new (MC) Task : " + c("t = MCTask()") +
" Copy a Task : " + c("nt = t.copy()"))
print(" Show task configuration : " + c("t.info()") +
" Show processing status : " + c("t.overview()"))
print(" Set the float of a Task : " + c("t.float = 100") +
" Set the name of a task : " +
c("t.name = 'My Own Task v1'"))
print(" Start processing : " + c("t.run()") +
" Pause processing : " + c("t.pause()"))
print(" Access Transform id N : " + c("tf = t.transforms[N]") +
" Pause processing of tf : " + c("tf.pause()") +
" # This command is reverted by using t.run()")
print(" Transform Application : " + c("tf.application") +
" Transform Backend : " + c("tf.backend"))
print('')
print(" Set parameter in all applications : " +
c("t.setParameter(my_software_version='1.42.0')"))
print(" Set backend for all transforms : " +
c("t.setBackend(backend) , p.e. t.setBackend(LCG())"))
print(" Limit on how often jobs are resubmitted : " +
c("tf.run_limit = 4"))
print(" Manually change the status of partitions: " +
c("tf.setPartitionStatus(partition, 'status')"))
print('')
print(
" For an ATLAS Monte Carlo Production Example and specific help type: "
+ c("MCTask?"))
print(" For an ATLAS Analysis Example and help type: " + c("AnaTask?"))
print('')
if not True:
# if not short:
print("ADVANCED COMMANDS:")
print(
"Add Transform at position N : t.insertTransform(N, transform)"
)
print("Remove Transform at position N : t.removeTransform(N)")
print(
"Set Transform Application : tf.application = TaskApp() #This Application must be a 'Task Version' of the usual application"
)
print(
" Adding Task Versions of Applications is easy, contact the developers to request an inclusion"
)
from Ganga.Utility.ColourText import ANSIMarkup, fgcol, Effects
markup = ANSIMarkup()
fx = Effects()
from Ganga.Core.exceptions import ApplicationConfigurationError
from GangaAtlas.Lib.Credentials.ProxyHelper import getNickname
o = [""]
def c(s):
return markup(s, fgcol("blue"))
o.append(markup(" *** Task for Athena Analysis ***", fgcol("blue")))
o.append(" Analysis Application : " + c("t.analysis.application"))
o.append(" Set Dataset : " + c("t.setDataset('my.dataset')"))
o.append(" Input Dataset Object : " + c("t.analysis.inputdata"))
o.append(" Output Dataset Object : " + c("t.analysis.outputdata"))
o.append(" Files processed per job : " + c("t.analysis.files_per_job = 10"))
o.append("")
o.append(markup("Procedure to do a usual analysis:", fgcol("red")))
#o.append("config.Tasks.merged_files_per_job = 1 # default files per job for merged datasets")
#o.append("config.Tasks.recon_files_per_job = 10 # default files per job for recon (non-merged) datasets")
o.append("t = AnaTask()")
o.append('t.name = "MyAnalysisR1"')
o.append("t.analysis.outputdata.outputdata = ['nTuple.root' ]")
o.append("t.analysis.application.option_file = ['./myTopOptions.py' ]")
o.append("t.analysis.application.prepare()")
o.append("t.float = 10")
from Ganga.GPIDev.Lib.Tasks.common import *
from Ganga.GPIDev.Lib.Tasks import Task
from MCTransforms import EvgenTransform, SimulTransform, ReconTransform
from GangaAtlas.Lib.AthenaMC.AthenaMCDatasets import AthenaMCInputDatasets
from Ganga.GPIDev.Schema import *
from Ganga.Utility.ColourText import ANSIMarkup, fgcol, Effects
markup = ANSIMarkup()
fx = Effects()
from Ganga.GPIDev.Base.Objects import Node
o = ""
o += "\n" + markup(" *** Task for Athena Monte Carlo production ***",
fgcol("blue"))
def c(s):
return markup(s, fgcol("blue"))
o += "\n" + " Set the total number of events to be processed: " + c(
"t.total_events = 10000")
o += "\n" + " Applications for evgen/simul/recon : " + c(
"t.evgen.application, t.simul.application, t.recon.application")
o += "\n" + " Events processed per evgen/simul/recon : " + c(
"t.evgen.application.number_events_job = 10000")
o += "\n" + " Skip events / files from the previous dataset : " + c(
"t.simul.inputdata.skip_events = 1000")
o += "\n" + " Use an input dataset for the event generator : " + c(
"t.initializeFromGenerator(dataset, events_per_file_in_dataset)")
from GangaLHCb.Lib.LHCbDataset.BKQuery import BKQuery
from GangaLHCb.Lib.LHCbDataset.OutputData import OutputData
from LHCbTaskDummySplitter import LHCbTaskDummySplitter
from Ganga.Core import GangaException, GangaAttributeError
from GangaLHCb.Lib.LHCbDataset import LHCbDataset
import Ganga.Utility.Config
from copy import deepcopy
import os
from Ganga.Utility.logging import getLogger
markup = ANSIMarkup()
logger = getLogger(modulename=True)
config = Ganga.Utility.Config.getConfig('Configuration')
partition_colours = {
'ignored': "",
'hold': fgcol("lgray"),
'ready': fgcol("lgreen"),
'running': fgcol("green"),
'completed': fgcol("blue"),
'attempted': fgcol("yellow"),
'failed': fgcol("lred"),
'bad': fgcol("red"),
'unknown': fgcol("white"),
}
job_colours = {
'new': col("black", "white"),
'submitting': col("black", "orange"),
'submitted': col("white", "orange"),
'running': col("black", "green"),
'completing': col("white", "green"),
from Ganga.GPIDev.Lib.Tasks.common import *
from Ganga.GPIDev.Lib.Tasks import Task
from MCTransforms import EvgenTransform, SimulTransform, ReconTransform
from GangaAtlas.Lib.AthenaMC.AthenaMCDatasets import AthenaMCInputDatasets
from Ganga.GPIDev.Schema import *
from Ganga.Utility.ColourText import ANSIMarkup, fgcol, Effects
markup = ANSIMarkup()
fx = Effects()
from Ganga.GPIDev.Base.Objects import Node
o = ""
o+="\n"+ markup(" *** Task for Athena Monte Carlo production ***", fgcol("blue"))
def c(s):
return markup(s,fgcol("blue"))
o+="\n"+ " Set the total number of events to be processed: "+c("t.total_events = 10000")
o+="\n"+ " Applications for evgen/simul/recon : "+c("t.evgen.application, t.simul.application, t.recon.application")
o+="\n"+ " Events processed per evgen/simul/recon : "+c("t.evgen.application.number_events_job = 10000")
o+="\n"+ " Skip events / files from the previous dataset : "+c("t.simul.inputdata.skip_events = 1000")
o+="\n"+ " Use an input dataset for the event generator : "+c("t.initializeFromGenerator(dataset, events_per_file_in_dataset)")
o+="\n"+ " Use an existing evgen EVNT dataset : "+c("t.initializeFromEvgen(dataset, events_per_file_in_dataset)")
o+="\n"+ " Use an existing simul RDO dataset : "+c("t.initializeFromSimul(dataset, events_per_file_in_dataset)")
o+="\n"
o+="\n"+ markup("Procedure to do a usual production:", fgcol("red"))
o+="\n"+ "t = MCTask()"
o+="\n"+ 't.name = "MyFirstTask"'
o+="\n"+ 't.evgen.application.evgen_job_option = "CSC.005145.PythiaZmumu.py" # or "path/to/my_evgen_job_option.py"'
o+="\n"+ 't.evgen.application.transform_script = "csc_evgen08_trf.py"'
o+="\n"+ "t.total_events = 1000"
o+="\n"+ 't.setParameter(run_number="5145", production_name="MyProd-01", process_name="PythiaZmumu")'
from AnaTransform import AnaTransform
from Ganga.GPIDev.Schema import *
from Ganga.Utility.ColourText import ANSIMarkup, fgcol, Effects
markup = ANSIMarkup()
fx = Effects()
from Ganga.Core.exceptions import ApplicationConfigurationError
from GangaAtlas.Lib.Credentials.ProxyHelper import getNickname
o = [""]
def c(s):
return markup(s,fgcol("blue"))
o.append(markup(" *** Task for Athena Analysis ***", fgcol("blue")))
o.append(" Analysis Application : "+c("t.analysis.application"))
o.append(" Set Dataset : "+c("t.setDataset('my.dataset')"))
o.append(" Input Dataset Object : "+c("t.analysis.inputdata"))
o.append(" Output Dataset Object : "+c("t.analysis.outputdata"))
o.append(" Files processed per job : "+c("t.analysis.files_per_job = 10"))
o.append("")
o.append(markup("Procedure to do a usual analysis:", fgcol("red")))
#o.append("config.Tasks.merged_files_per_job = 1 # default files per job for merged datasets")
#o.append("config.Tasks.recon_files_per_job = 10 # default files per job for recon (non-merged) datasets")
o.append("t = AnaTask()")
o.append('t.name = "MyAnalysisR1"')
o.append("t.analysis.outputdata.outputdata = ['nTuple.root' ]")
o.append("t.analysis.application.option_file = ['./myTopOptions.py' ]")
o.append("t.analysis.application.prepare()")
o.append("t.float = 10")
