def help(self, short=False): """Print a short introduction and 'cheat sheet' for the Ganga Tasks package""" print('') print(markup(" *** Ganga Tasks: Short Introduction and 'Cheat Sheet' ***", fgcol("blue"))) print('') print(markup("Definitions: ", fgcol("red")) + "'Partition' - A unit of processing, for example processing a file or processing some events from a file") print(" 'Transform' - A group of partitions that have a common Ganga Application and Backend.") print(" 'Task' - A group of one or more 'Transforms' that can have dependencies on each other") print('') print(markup("Possible status values for partitions:", fgcol("red"))) print(' * "' + markup("ready", overview_colours["ready"]) + '" - ready to be executed ') print(' * "' + markup("hold", overview_colours["hold"]) + '" - dependencies not completed') print(' * "' + markup("running", overview_colours["running"]) + '" - at least one job tries to process this partition') print(' * "' + markup("attempted", overview_colours["attempted"]) + '"- tasks tried to process this partition, but has not yet succeeded') print(' * "' + markup("failed", overview_colours["failed"]) + '" - tasks failed to process this partition several times') print(' * "' + markup("bad", overview_colours["bad"]) + '" - this partition is excluded from further processing and will not be used as input to subsequent transforms') print(' * "' + markup("completed", overview_colours["completed"]) + '" ') print('') def c(s): return markup(s, fgcol("blue")) print(markup("Important commands:", fgcol("red"))) print(" Get a quick overview : " + c("tasks") + " Get a detailed view : " + c("tasks.table()")) print(" Access an existing task : " + c("t = tasks(id)") + " Remove a Task : " + c("tasks(id).remove()")) print(" Create a new (MC) Task : " + c("t = MCTask()") + " Copy a Task : " + c("nt = t.copy()")) print(" Show task configuration : " + c("t.info()") + " Show processing status : " + c("t.overview()")) print(" Set the float of a Task : " + c("t.float = 100") + " Set the name of a task : " + c("t.name = 'My Own Task v1'")) print(" Start processing : " + c("t.run()") + " Pause processing : " + c("t.pause()")) print(" Access Transform id N : " + c("tf = t.transforms[N]") + " Pause processing of tf : " + c( "tf.pause()") + " # This command is reverted by using t.run()") print(" Transform Application : " + c("tf.application") + " Transform Backend : " + c("tf.backend")) print('') print(" Set parameter in all applications : " + c("t.setParameter(my_software_version='1.42.0')")) print(" Set backend for all transforms : " + c("t.setBackend(backend) , p.e. t.setBackend(LCG())")) print(" Limit on how often jobs are resubmitted : " + c("tf.run_limit = 4")) print(" Manually change the status of partitions: " + c("tf.setPartitionStatus(partition, 'status')")) print('') print(" For an ATLAS Monte Carlo Production Example and specific help type: " + c("MCTask?")) print(" For an ATLAS Analysis Example and help type: " + c("AnaTask?")) print('') if not True: # if not short: print("ADVANCED COMMANDS:") print("Add Transform at position N : t.insertTransform(N, transform)") print("Remove Transform at position N : t.removeTransform(N)") print("Set Transform Application : tf.application = TaskApp() #This Application must be a 'Task Version' of the usual application") print(" Adding Task Versions of Applications is easy, contact the developers to request an inclusion")
def __str__(self, short=True, id=None): """Prints an overview over the currently running tasks""" if Ganga.GPIDev.Lib.Registry.RegistrySlice.config["tasks_show_help"]: self.help(short=True) Ganga.GPIDev.Lib.Registry.RegistrySlice.config.setUserValue("tasks_show_help", False) print("To show this help message again, type 'tasks.help()'.") print('') print(" The following is the output of " + markup("tasks.table()", fgcol("blue"))) lenfstring = 0 flist = [] for thing in Ganga.GPIDev.Lib.Registry.RegistrySlice.config["tasks_columns"]: width = Ganga.GPIDev.Lib.Registry.RegistrySlice.config["tasks_columns_width"][thing] lenfstring += width flist.append("%" + str(width) + "s ") fstring = "|".join(flist) fstring += '\n' lenfstring += 27 ds = fstring % ("#", "Type", "Name", "State", "Comment", "%4s: %4s/ %4s/ %4s/ %4s/ %4s/ %4s/ %4s" % ("Jobs", markup("done", overview_colours["completed"]), " " + markup("run", overview_colours["running"]), " " + markup("subd", overview_colours["submitted"]), " " + markup("attd", overview_colours["attempted"]), markup("fail", overview_colours["failed"]), markup("hold", overview_colours["hold"]), " " + markup("bad", overview_colours["bad"])), "Float") ds += "-" * lenfstring + "\n" from Ganga.GPIDev.Lib.Tasks import ITask if id is not None and type(id) is int: iterable_list = [self[id]] else: iterable_list = stripProxy(self).objects.values() for p in iterable_list: if isType(p, ITask): stat = "%4i: %4i/ %4i/ %4i/ --/ %4i/ %4i/ %4i" % (p.n_all(), p.n_status("completed"), p.n_status("running"), p.n_status("submitted"), p.n_status("failed"), p.n_status("hold"), p.n_status("bad")) ds += markup(fstring % (p.id, getName(p), p.name[0:Ganga.GPIDev.Lib.Registry.RegistrySlice.config['tasks_columns_width']['Name']], p.status, p.comment, stat, p.float), status_colours[p.status]) else: stat = "%4i: %4i/ %4i/ --/ %4i/ %4i/ %4i/ %4i" % (p.n_all(), p.n_status("completed"), p.n_status("running"), p.n_status("attempted"), p.n_status("failed"), p.n_status("hold"), p.n_status("bad")) ds += markup(fstring % (p.id, getName(p), p.name[0:Ganga.GPIDev.Lib.Registry.RegistrySlice.config['tasks_columns_width']['Name']], p.status, p.comment, stat, p.float), status_colours[p.status]) if short is True: continue for ti in range(0, len(p.transforms)): t = p.transforms[ti] if isType(p, ITask): stat = "%4i: %4i/ %4i/ %4i/ --/ %4i/ %4i/ %4s" % (t.n_all(), t.n_status("completed"), t.n_status("running"), t.n_status("submitted"), t.n_status("failed"), t.n_status("hold"), t.n_status("bad")) ds += "\n" + markup(fstring % ("%i.%i" % (p.id, ti), getName(t), t.name[0:Ganga.GPIDev.Lib.Registry.RegistrySlice.config['tasks_columns_width']['Name']], t.status, "", stat, ""), status_colours[t.status]) else: stat = "%4i: %4i/ %4i/ --/ %4i/ %4i/ %4i/ %4s" % (t.n_all(), t.n_status("completed"), t.n_status("running"), t.n_status("attempted"), t.n_status("failed"), t.n_status("hold"), t.n_status("bad")) ds += "zn" + markup(fstring % ("%i.%i" % (p.id, ti), getName(t), t.name[0:Ganga.GPIDev.Lib.Registry.RegistrySlice.config['tasks_columns_width']['Name']], t.status, "", stat, ""), status_colours[t.status]) ds += "-" * lenfstring + "\n" return ds + "\n"
def c(s): return markup(s, fgcol("blue"))
from GangaAtlas.Lib.Credentials.ProxyHelper import getNickname from dq2.clientapi.DQ2 import DQ2, DQUnknownDatasetException, DQDatasetExistsException, DQFileExistsInDatasetException, DQInvalidRequestException from dq2.container.exceptions import DQContainerAlreadyHasDataset, DQContainerDoesNotHaveDataset from dq2.common.DQException import DQException from Ganga.GPIDev.Schema import * import copy o = [""] def c(s): return markup(s, fgcol("blue")) o.append(markup(" *** Task for Multiple Athena Analyses ***", fgcol("blue"))) #o.append(" Analysis Application : "+c("t.analysis.application")) #o.append(" Set Dataset : "+c("t.setDataset('my.dataset')")) #o.append(" Input Dataset Object : "+c("t.analysis.inputdata")) #o.append(" Output Dataset Object : "+c("t.analysis.outputdata")) #o.append(" Files processed per job : "+c("t.analysis.files_per_job = 10")) #o.append("") #o.append(markup("Procedure to do a usual analysis:", fgcol("red"))) ##o.append("config.Tasks.merged_files_per_job = 1 # default files per job for merged datasets") ##o.append("config.Tasks.recon_files_per_job = 10 # default files per job for recon (non-merged) datasets") #o.append("t = AnaTask()") #o.append('t.name = "MyAnalysisR1"') #o.append("t.analysis.outputdata.outputdata = ['nTuple.root' ]") #o.append("t.analysis.application.option_file = ['./myTopOptions.py' ]") #o.append("t.analysis.application.prepare()") #o.append("t.float = 10")
from GangaLHCb.Lib.LHCbDataset.OutputData import OutputData from LHCbTaskDummySplitter import LHCbTaskDummySplitter from Ganga.Core import GangaException, GangaAttributeError from GangaLHCb.Lib.LHCbDataset import LHCbDataset import Ganga.Utility.Config from copy import deepcopy import os from Ganga.Utility.logging import getLogger markup = ANSIMarkup() logger = getLogger(modulename=True) config = Ganga.Utility.Config.getConfig('Configuration') partition_colours = { 'ignored': "", 'hold': fgcol("lgray"), 'ready': fgcol("lgreen"), 'running': fgcol("green"), 'completed': fgcol("blue"), 'attempted': fgcol("yellow"), 'failed': fgcol("lred"), 'bad': fgcol("red"), 'unknown': fgcol("white"), } job_colours = { 'new': col("black", "white"), 'submitting': col("black", "orange"), 'submitted': col("white", "orange"), 'running': col("black", "green"), 'completing': col("white", "green"),
def __str__(self, short=True, id=None): """Prints an overview over the currently running tasks""" if Ganga.GPIDev.Lib.Registry.RegistrySlice.config["tasks_show_help"]: self.help(short=True) Ganga.GPIDev.Lib.Registry.RegistrySlice.config.setUserValue( "tasks_show_help", False) print("To show this help message again, type 'tasks.help()'.") print('') print(" The following is the output of " + markup("tasks.table()", fgcol("blue"))) lenfstring = 0 flist = [] for thing in Ganga.GPIDev.Lib.Registry.RegistrySlice.config[ "tasks_columns"]: width = Ganga.GPIDev.Lib.Registry.RegistrySlice.config[ "tasks_columns_width"][thing] lenfstring += width flist.append("%" + str(width) + "s ") fstring = "|".join(flist) fstring += '\n' lenfstring += 27 ds = fstring % ("#", "Type", "Name", "State", "Comment", "%4s: %4s/ %4s/ %4s/ %4s/ %4s/ %4s/ %4s" % ("Jobs", markup("done", overview_colours["completed"]), " " + markup("run", overview_colours["running"]), " " + markup("subd", overview_colours["submitted"]), " " + markup("attd", overview_colours["attempted"]), markup("fail", overview_colours["failed"]), markup("hold", overview_colours["hold"]), " " + markup("bad", overview_colours["bad"])), "Float") ds += "-" * lenfstring + "\n" from Ganga.GPIDev.Lib.Tasks import ITask if id is not None and type(id) is int: iterable_list = [self[id]] else: iterable_list = stripProxy(self).objects.values() for p in iterable_list: if isType(p, ITask): stat = "%4i: %4i/ %4i/ %4i/ --/ %4i/ %4i/ %4i" % ( p.n_all(), p.n_status("completed"), p.n_status("running"), p.n_status("submitted"), p.n_status("failed"), p.n_status("hold"), p.n_status("bad")) ds += markup( fstring % (p.id, getName(p), p.name[0:Ganga.GPIDev.Lib.Registry.RegistrySlice. config['tasks_columns_width']['Name']], p.status, p.comment, stat, p.float), status_colours[p.status]) else: stat = "%4i: %4i/ %4i/ --/ %4i/ %4i/ %4i/ %4i" % ( p.n_all(), p.n_status("completed"), p.n_status("running"), p.n_status("attempted"), p.n_status("failed"), p.n_status("hold"), p.n_status("bad")) ds += markup( fstring % (p.id, getName(p), p.name[0:Ganga.GPIDev.Lib.Registry.RegistrySlice. config['tasks_columns_width']['Name']], p.status, p.comment, stat, p.float), status_colours[p.status]) if short is True: continue for ti in range(0, len(p.transforms)): t = p.transforms[ti] if isType(p, ITask): stat = "%4i: %4i/ %4i/ %4i/ --/ %4i/ %4i/ %4s" % ( t.n_all(), t.n_status("completed"), t.n_status("running"), t.n_status("submitted"), t.n_status("failed"), t.n_status("hold"), t.n_status("bad")) ds += "\n" + markup( fstring % ("%i.%i" % (p.id, ti), getName(t), t.name[0:Ganga.GPIDev.Lib.Registry.RegistrySlice. config['tasks_columns_width']['Name']], t.status, "", stat, ""), status_colours[t.status]) else: stat = "%4i: %4i/ %4i/ --/ %4i/ %4i/ %4i/ %4s" % ( t.n_all(), t.n_status("completed"), t.n_status("running"), t.n_status("attempted"), t.n_status("failed"), t.n_status("hold"), t.n_status("bad")) ds += "zn" + markup( fstring % ("%i.%i" % (p.id, ti), getName(t), t.name[0:Ganga.GPIDev.Lib.Registry.RegistrySlice. config['tasks_columns_width']['Name']], t.status, "", stat, ""), status_colours[t.status]) ds += "-" * lenfstring + "\n" return ds + "\n"
def help(self, short=False): """Print a short introduction and 'cheat sheet' for the Ganga Tasks package""" print('') print( markup( " *** Ganga Tasks: Short Introduction and 'Cheat Sheet' ***", fgcol("blue"))) print('') print( markup("Definitions: ", fgcol("red")) + "'Partition' - A unit of processing, for example processing a file or processing some events from a file" ) print( " 'Transform' - A group of partitions that have a common Ganga Application and Backend." ) print( " 'Task' - A group of one or more 'Transforms' that can have dependencies on each other" ) print('') print(markup("Possible status values for partitions:", fgcol("red"))) print(' * "' + markup("ready", overview_colours["ready"]) + '" - ready to be executed ') print(' * "' + markup("hold", overview_colours["hold"]) + '" - dependencies not completed') print(' * "' + markup("running", overview_colours["running"]) + '" - at least one job tries to process this partition') print( ' * "' + markup("attempted", overview_colours["attempted"]) + '"- tasks tried to process this partition, but has not yet succeeded' ) print(' * "' + markup("failed", overview_colours["failed"]) + '" - tasks failed to process this partition several times') print( ' * "' + markup("bad", overview_colours["bad"]) + '" - this partition is excluded from further processing and will not be used as input to subsequent transforms' ) print(' * "' + markup("completed", overview_colours["completed"]) + '" ') print('') def c(s): return markup(s, fgcol("blue")) print(markup("Important commands:", fgcol("red"))) print(" Get a quick overview : " + c("tasks") + " Get a detailed view : " + c("tasks.table()")) print(" Access an existing task : " + c("t = tasks(id)") + " Remove a Task : " + c("tasks(id).remove()")) print(" Create a new (MC) Task : " + c("t = MCTask()") + " Copy a Task : " + c("nt = t.copy()")) print(" Show task configuration : " + c("t.info()") + " Show processing status : " + c("t.overview()")) print(" Set the float of a Task : " + c("t.float = 100") + " Set the name of a task : " + c("t.name = 'My Own Task v1'")) print(" Start processing : " + c("t.run()") + " Pause processing : " + c("t.pause()")) print(" Access Transform id N : " + c("tf = t.transforms[N]") + " Pause processing of tf : " + c("tf.pause()") + " # This command is reverted by using t.run()") print(" Transform Application : " + c("tf.application") + " Transform Backend : " + c("tf.backend")) print('') print(" Set parameter in all applications : " + c("t.setParameter(my_software_version='1.42.0')")) print(" Set backend for all transforms : " + c("t.setBackend(backend) , p.e. t.setBackend(LCG())")) print(" Limit on how often jobs are resubmitted : " + c("tf.run_limit = 4")) print(" Manually change the status of partitions: " + c("tf.setPartitionStatus(partition, 'status')")) print('') print( " For an ATLAS Monte Carlo Production Example and specific help type: " + c("MCTask?")) print(" For an ATLAS Analysis Example and help type: " + c("AnaTask?")) print('') if not True: # if not short: print("ADVANCED COMMANDS:") print( "Add Transform at position N : t.insertTransform(N, transform)" ) print("Remove Transform at position N : t.removeTransform(N)") print( "Set Transform Application : tf.application = TaskApp() #This Application must be a 'Task Version' of the usual application" ) print( " Adding Task Versions of Applications is easy, contact the developers to request an inclusion" )
from Ganga.Utility.ColourText import ANSIMarkup, fgcol, Effects markup = ANSIMarkup() fx = Effects() from Ganga.Core.exceptions import ApplicationConfigurationError from GangaAtlas.Lib.Credentials.ProxyHelper import getNickname o = [""] def c(s): return markup(s, fgcol("blue")) o.append(markup(" *** Task for Athena Analysis ***", fgcol("blue"))) o.append(" Analysis Application : " + c("t.analysis.application")) o.append(" Set Dataset : " + c("t.setDataset('my.dataset')")) o.append(" Input Dataset Object : " + c("t.analysis.inputdata")) o.append(" Output Dataset Object : " + c("t.analysis.outputdata")) o.append(" Files processed per job : " + c("t.analysis.files_per_job = 10")) o.append("") o.append(markup("Procedure to do a usual analysis:", fgcol("red"))) #o.append("config.Tasks.merged_files_per_job = 1 # default files per job for merged datasets") #o.append("config.Tasks.recon_files_per_job = 10 # default files per job for recon (non-merged) datasets") o.append("t = AnaTask()") o.append('t.name = "MyAnalysisR1"') o.append("t.analysis.outputdata.outputdata = ['nTuple.root' ]") o.append("t.analysis.application.option_file = ['./myTopOptions.py' ]") o.append("t.analysis.application.prepare()") o.append("t.float = 10")
from Ganga.GPIDev.Lib.Tasks.common import * from Ganga.GPIDev.Lib.Tasks import Task from MCTransforms import EvgenTransform, SimulTransform, ReconTransform from GangaAtlas.Lib.AthenaMC.AthenaMCDatasets import AthenaMCInputDatasets from Ganga.GPIDev.Schema import * from Ganga.Utility.ColourText import ANSIMarkup, fgcol, Effects markup = ANSIMarkup() fx = Effects() from Ganga.GPIDev.Base.Objects import Node o = "" o += "\n" + markup(" *** Task for Athena Monte Carlo production ***", fgcol("blue")) def c(s): return markup(s, fgcol("blue")) o += "\n" + " Set the total number of events to be processed: " + c( "t.total_events = 10000") o += "\n" + " Applications for evgen/simul/recon : " + c( "t.evgen.application, t.simul.application, t.recon.application") o += "\n" + " Events processed per evgen/simul/recon : " + c( "t.evgen.application.number_events_job = 10000") o += "\n" + " Skip events / files from the previous dataset : " + c( "t.simul.inputdata.skip_events = 1000") o += "\n" + " Use an input dataset for the event generator : " + c( "t.initializeFromGenerator(dataset, events_per_file_in_dataset)")
from GangaLHCb.Lib.LHCbDataset.BKQuery import BKQuery from GangaLHCb.Lib.LHCbDataset.OutputData import OutputData from LHCbTaskDummySplitter import LHCbTaskDummySplitter from Ganga.Core import GangaException, GangaAttributeError from GangaLHCb.Lib.LHCbDataset import LHCbDataset import Ganga.Utility.Config from copy import deepcopy import os from Ganga.Utility.logging import getLogger markup = ANSIMarkup() logger = getLogger(modulename=True) config = Ganga.Utility.Config.getConfig('Configuration') partition_colours = { 'ignored': "", 'hold': fgcol("lgray"), 'ready': fgcol("lgreen"), 'running': fgcol("green"), 'completed': fgcol("blue"), 'attempted': fgcol("yellow"), 'failed': fgcol("lred"), 'bad': fgcol("red"), 'unknown': fgcol("white"), } job_colours = { 'new': col("black", "white"), 'submitting': col("black", "orange"), 'submitted': col("white", "orange"), 'running': col("black", "green"), 'completing': col("white", "green"),
from Ganga.GPIDev.Lib.Tasks.common import * from Ganga.GPIDev.Lib.Tasks import Task from MCTransforms import EvgenTransform, SimulTransform, ReconTransform from GangaAtlas.Lib.AthenaMC.AthenaMCDatasets import AthenaMCInputDatasets from Ganga.GPIDev.Schema import * from Ganga.Utility.ColourText import ANSIMarkup, fgcol, Effects markup = ANSIMarkup() fx = Effects() from Ganga.GPIDev.Base.Objects import Node o = "" o+="\n"+ markup(" *** Task for Athena Monte Carlo production ***", fgcol("blue")) def c(s): return markup(s,fgcol("blue")) o+="\n"+ " Set the total number of events to be processed: "+c("t.total_events = 10000") o+="\n"+ " Applications for evgen/simul/recon : "+c("t.evgen.application, t.simul.application, t.recon.application") o+="\n"+ " Events processed per evgen/simul/recon : "+c("t.evgen.application.number_events_job = 10000") o+="\n"+ " Skip events / files from the previous dataset : "+c("t.simul.inputdata.skip_events = 1000") o+="\n"+ " Use an input dataset for the event generator : "+c("t.initializeFromGenerator(dataset, events_per_file_in_dataset)") o+="\n"+ " Use an existing evgen EVNT dataset : "+c("t.initializeFromEvgen(dataset, events_per_file_in_dataset)") o+="\n"+ " Use an existing simul RDO dataset : "+c("t.initializeFromSimul(dataset, events_per_file_in_dataset)") o+="\n" o+="\n"+ markup("Procedure to do a usual production:", fgcol("red")) o+="\n"+ "t = MCTask()" o+="\n"+ 't.name = "MyFirstTask"' o+="\n"+ 't.evgen.application.evgen_job_option = "CSC.005145.PythiaZmumu.py" # or "path/to/my_evgen_job_option.py"' o+="\n"+ 't.evgen.application.transform_script = "csc_evgen08_trf.py"' o+="\n"+ "t.total_events = 1000" o+="\n"+ 't.setParameter(run_number="5145", production_name="MyProd-01", process_name="PythiaZmumu")'
from AnaTransform import AnaTransform from Ganga.GPIDev.Schema import * from Ganga.Utility.ColourText import ANSIMarkup, fgcol, Effects markup = ANSIMarkup() fx = Effects() from Ganga.Core.exceptions import ApplicationConfigurationError from GangaAtlas.Lib.Credentials.ProxyHelper import getNickname o = [""] def c(s): return markup(s,fgcol("blue")) o.append(markup(" *** Task for Athena Analysis ***", fgcol("blue"))) o.append(" Analysis Application : "+c("t.analysis.application")) o.append(" Set Dataset : "+c("t.setDataset('my.dataset')")) o.append(" Input Dataset Object : "+c("t.analysis.inputdata")) o.append(" Output Dataset Object : "+c("t.analysis.outputdata")) o.append(" Files processed per job : "+c("t.analysis.files_per_job = 10")) o.append("") o.append(markup("Procedure to do a usual analysis:", fgcol("red"))) #o.append("config.Tasks.merged_files_per_job = 1 # default files per job for merged datasets") #o.append("config.Tasks.recon_files_per_job = 10 # default files per job for recon (non-merged) datasets") o.append("t = AnaTask()") o.append('t.name = "MyAnalysisR1"') o.append("t.analysis.outputdata.outputdata = ['nTuple.root' ]") o.append("t.analysis.application.option_file = ['./myTopOptions.py' ]") o.append("t.analysis.application.prepare()") o.append("t.float = 10")