def start_element(name, attrs):
#logger.debug('Start element: name=%s attrs=%s', name, attrs) #FIXME: for 2.4 use CurrentColumnNumber and CurrentLineNumber
# if higher level element had error, ignore the corresponding part
# of the XML tree as we go down
if self.ignore_count:
self.ignore_count += 1
return
# initialize object stack
if name == 'root':
assert self.stack is None, "duplicated <root> element"
self.stack = []
return
assert not self.stack is None, "missing <root> element"
# load a class, make empty object and push it as the current object
# on the stack
if name == 'class':
try:
cls = allPlugins.find(attrs['category'], attrs['name'])
except PluginManagerError as e:
self.errors.append(e)
#self.errors.append('Unknown class: %(name)s'%attrs)
obj = EmptyGangaObject()
# ignore all elemenents until the corresponding ending
# element (</class>) is reached
self.ignore_count = 1
else:
version = Version(
*[int(v) for v in attrs['version'].split('.')])
if not cls._schema.version.isCompatible(version):
attrs['currversion'] = '%s.%s' % (
cls._schema.version.major,
cls._schema.version.minor)
self.errors.append(
SchemaVersionError(
'Incompatible schema of %(name)s, repository is %(version)s currently in use is %(currversion)s'
% attrs))
obj = EmptyGangaObject()
# ignore all elemenents until the corresponding ending
# element (</class>) is reached
self.ignore_count = 1
else:
# make a new ganga object
obj = cls()
self.stack.append(obj)
# push the attribute name on the stack
if name == 'attribute':
self.stack.append(attrs['name'])
# start value_contruct mode and initialize the value buffer
if name == 'value':
self.value_construct = ''
# save a marker where the sequence begins on the stack
if name == 'sequence':
self.sequence_start.append(len(self.stack))
class ICommandSet(GangaObject):
"""
Class used to define shell commands and options for working with credentials
"""
_schema = Schema(
Version(1, 0), {
"init":
SimpleItem(defvalue="",
doc="Command for creating/initialising credential"),
"info":
SimpleItem(
defvalue="",
doc="Command for obtaining information about credential"),
"destroy":
SimpleItem(defvalue="", doc="Command for destroying credential"),
"init_parameters":
SimpleItem(
defvalue={},
doc=
"Dictionary of parameter-value pairs to pass to init command"),
"destroy_parameters":
SimpleItem(
defvalue={},
doc=
"Dictionary of parameter-value pairs to pass to destroy command"
),
"info_parameters":
SimpleItem(
defvalue={},
doc=
"Dictionary mapping from Ganga credential properties to command-line options"
),
})
_category = "credential_commands"
_name = "ICommandSet"
_hidden = 1
_enable_config = 1
def __init__(self):
super(ICommandSet, self).__init__()
return
def setConfigValues(self):
"""
Update properties using values from relevant section of config file.
"""
section = "defaults_%s" % self._name
config = getConfig(section)
for attribute in self._schema.datadict.keys():
try:
value = config[attribute]
try:
value = eval(value)
except:
pass
setattr(self, attribute, value)
except ConfigError:
pass
class ThreadedTestGangaObject(GangaObject):
_schema = Schema(Version(1, 0), {
'a': SimpleItem(42, typelist=[int]),
'b': ComponentItem('TestGangaObject', defvalue='SimpleGangaObject'),
})
_category = 'TestGangaObject'
_hidden = True
_enable_plugin = True
class EmptyGangaObject(GangaObject):
"""Empty Ganga Object. Is used to construct incomplete jobs"""
_schema = Schema(Version(0, 0), {})
_name = "EmptyGangaObject"
_category = "internal"
_hidden = 1
def __init__(self):
super(EmptyGangaObject, self).__init__()
class ArgSplitter(ISplitter):
"""
Split job by changing the args attribute of the application.
This splitter only applies to the applications which have args attribute (e.g. Executable, Root).
It is a special case of the GenericSplitter.
This splitter allows the creation of a series of subjobs where
the only difference between different jobs are their
arguments. Below is an example that executes a ROOT script ~/analysis.C
void analysis(const char* type, int events) {
std::cout << type << " " << events << std::endl;
}
with 3 different sets of arguments.
s = ArgSplitter(args=[['AAA',1],['BBB',2],['CCC',3]])
r = Root(version='5.10.00',script='~/analysis.C')
j.Job(application=r, splitter=s)
Notice how each job takes a list of arguments (in this case a list
with a string and an integer). The splitter thus takes a list of
lists, in this case with 3 elements so there will be 3 subjobs.
Running the subjobs will produce the output:
subjob 1 : AAA 1
subjob 2 : BBB 2
subjob 3 : CCC 3
"""
_name = "ArgSplitter"
_schema = Schema(
Version(1, 0), {
'args':
SimpleItem(defvalue=[],
typelist=[
'list',
'Ganga.GPIDev.Lib.GangaList.GangaList.GangaList'
],
sequence=1,
doc='A list of lists of arguments to pass to script')
})
def split(self, job):
subjobs = []
for arg in self.args:
j = self.createSubjob(job, ['application'])
# Add new arguments to subjob
app = copy.deepcopy(job.application)
app.args = arg
j.application = app
logger.debug('Arguments for split job is: ' + str(arg))
subjobs.append(stripProxy(j))
return subjobs
class Im3ShapeSplitter(ISplitter):
"""
This splitter splits jobs using the Im3ShapeApp application using the size parameter.
If a splitter is configured with size = 5, split_by_file = True, then it will create 5 subjobs per file in the master_job.inputdata
If a splitter is configured with size = 5, split_by_file = False, then it will create 5 subjobs total and configure all subjobs to use all given data.
In the future there may be support for splitting based upon regex and namePatterns in the inputdata to allow a certain subset of data to be put in each subjob.
"""
_name = "Im3ShapeSplitter"
_schema = Schema(
Version(1, 0), {
'size':
SimpleItem(defvalue=5,
doc='Size of the tiles which are to be split.'),
'split_by_file':
SimpleItem(
defvalue=True,
doc=
'Should we auto-split into subjobs here on a per-file basis?')
})
def split(self, job):
"""
Actually perform the splitting of the given master job. The generated subjobs of the splitting are returned
Args:
job (Job): This is the master job object which is to be split and the subjobs of which are returned
"""
assert isinstance(job.application, Im3ShapeApp)
subjobs = []
def getApp(job, rank, size):
app = copy.deepcopy(job.application)
app.rank = rank
app.size = size
return app
if self.split_by_file:
for this_file in job.inputdata:
for rank in range(0, self.size):
j = self.createSubjob(job, ['application'])
# Add new arguments to subjob
j.application = getApp(job, rank, self.size)
j.inputdata = GangaDataset(files=[stripProxy(this_file)])
subjobs.append(j)
else:
for rank in range(0, self.size):
j = self.createSubjob(job, ['application'])
j.application = getApp(job, rank, self.size)
j.inputdata = job.inputdata
logger.debug('Rank for split job is: ' + str(rank))
subjobs.append(j)
return subjobs
class IncompleteObject(GangaObject):
""" This class represents an object that could not be loaded on startup"""
_schema = Schema(Version(0, 0), {})
_name = "IncompleteObject"
_category = "internal"
_hidden = 1
_exportmethods = ['reload', 'remove', '__repr__']
def __init__(self, registry, this_id):
"""
This constructs an object which is placed into the objects dict when a repo fails to load an object due to some error
Args:
registry (Registry): This is the registry the object belongs to
this_id (int): This is the registry/repo id of the object in the objects dict
"""
super(IncompleteObject, self).__init__()
self.registry = registry
self.id = this_id
def reload(self):
"""
This will trigger a re-load of the object from disk which is useful if the object was locked but accessible by Ganga
TODO work ouf if this is still called anywhere
"""
with self.registry._flush_lock:
with self.registry._read_lock:
self.registry._load(self)
logger.debug("Successfully reloaded '%s' object #%i!" %
(self.registry.name, self.id))
def remove(self):
"""
This will trigger a delete of the the object itself from within the given Repository but not registry
TODO work out if this is safe and still called
"""
with self.registry._flush_lock:
with self.registry._read_lock:
if len(self.registry.repository.lock([self.id])) == 0:
errstr = "Could not lock '%s' object #%i!" % (
self.registry.name, self.id)
try:
errstr += " Object is locked by session '%s' " % self.registry.repository.get_lock_session(
self.id)
except Exception as err:
logger.debug("Remove Lock error: %s" % err)
raise RegistryLockError(errstr)
self.registry.repository.delete([self.id])
def __repr__(self):
"""
This returns a repr of the object in question as inaccessible
"""
return "Incomplete object in '%s', ID %i. Try reload() or remove()." % (
self.registry.name, self.id)
class GangaDatasetSplitter(ISplitter):
""" Split job based on files given in GangaDataset inputdata field """
_name = "GangaDatasetSplitter"
_schema = Schema(Version(1, 0), {
'files_per_subjob': SimpleItem(defvalue=5, doc='the number of files per subjob', typelist=[int]),
'maxFiles': SimpleItem(defvalue=-1,
doc='Maximum number of files to use in a masterjob (None or -1 = all files)',
typelist=[int, None]),
})
def split(self, job):
subjobs = []
if not job.inputdata or not isType(job.inputdata, GangaDataset):
raise ApplicationConfigurationError(
"No GangaDataset given for GangaDatasetSplitter")
# find the full file list
full_list = []
for f in job.inputdata.files:
if f.containsWildcards():
# we have a wildcard so grab the subfiles
for sf in f.getSubFiles(process_wildcards=True):
full_list.append(sf)
else:
# no wildcards so just add the file
full_list.append(f)
if len(full_list) == 0:
raise ApplicationConfigurationError(
"GangaDatasetSplitter couldn't find any files to split over")
masterType = type(job.inputdata)
# split based on all the sub files
fid = 0
subjobs = []
filesToRun = len(full_list)
if not self.maxFiles == -1:
filesToRun = min(self.maxFiles, filesToRun)
while fid < filesToRun:
j = self.createSubjob(job)
j.inputdata = masterType()
j.inputdata.treat_as_inputfiles = job.inputdata.treat_as_inputfiles
for sf in full_list[fid:fid + self.files_per_subjob]:
j.inputdata.files.append(sf)
fid += self.files_per_subjob
subjobs.append(j)
return subjobs
class CoreTask(ITask):
"""General non-experimentally specific Task"""
_schema = Schema(Version(1, 0),
dict(ITask._schema.datadict.items() + {}.items()))
_category = 'tasks'
_name = 'CoreTask'
_exportmethods = ITask._exportmethods + []
_tasktype = "ITask"
default_registry = "tasks"
class Dataset(GangaObject):
_schema = Schema(Version(1, 0), {})
_category = 'datasets'
_name = "EmptyDataset"
def __init__(self):
super(Dataset, self).__init__()
# Return true if the dataset is an instance of the default base class.
# You may override it in your dataset definition but it is not mandatory.
def isEmpty(self):
return self._name == Dataset._name
class TestGangaObject(GangaObject):
_schema = Schema(Version(1, 0), {'a': SimpleItem(42, typelist=['int'])})
_category = 'TestGangaObject'
_name = 'TestGangaObject'
_exportmethods = ['example']
def example(self):
return 'example_string'
def not_proxied(self):
return 'example_string'
class TaskLocalCopy(Dataset):
"""Dummy dataset to force Tasks to copy the output from a job to local storage somewhere"""
_schema = Schema(
Version(1, 0), {
'local_location':
SimpleItem(defvalue="", doc="Local location to copy files to"),
'include_file_mask':
SimpleItem(
defvalue=[],
typelist=['str'],
sequence=1,
doc=
'List of Regular expressions of which files to include in copy'
),
'exclude_file_mask':
SimpleItem(
defvalue=[],
typelist=['str'],
sequence=1,
doc=
'List of Regular expressions of which files to exclude from copy'
),
'files':
SimpleItem(defvalue=[],
typelist=['str'],
sequence=1,
doc='List of successfully downloaded files'),
})
_category = 'datasets'
_name = 'TaskLocalCopy'
_exportmethods = ["isValid", "isDownloaded"]
def __init__(self):
super(TaskLocalCopy, self).__init__()
def isValid(self, fname):
"""Check if this file should be downloaded"""
for in_re in self.include_file_mask:
if not re.search(in_re, fname):
return False
for out_re in self.exclude_file_mask:
if re.search(out_re, fname):
return False
return True
def isDownloaded(self, fname):
"""Check if this file is present at the local_location"""
return os.path.exists(os.path.join(self.local_location, fname))
class CRABSplitter(ISplitter):
"""Splitter object for CRAB jobs."""
schemadic = {}
schemadic['maxevents'] = SimpleItem(defvalue=None,
typelist=['type(None)', 'int'],
doc='Maximum number of events/task')
schemadic['inputfiles'] = SimpleItem(defvalue=None,
typelist=['type(None)', 'str'],
doc='Number of input files')
schemadic['skipevents'] = SimpleItem(defvalue=None,
typelist=['type(None)', 'int'],
doc='Offset for the events')
_name = 'CRABSplitter'
_schema = Schema(Version(1, 0), schemadic)
def parseArguments(self, path):
"""Gets some job arguments from the FJR."""
splittingData = []
for job in parse(path).getElementsByTagName("Job"):
splittingData.append([
job.getAttribute("MaxEvents"),
job.getAttribute("InputFiles"),
job.getAttribute("SkipEvents")
])
return splittingData
def split(self, job):
"""Main splitter for the job."""
try:
arguments_path = os.path.join(job.inputdata.ui_working_dir,
'share/arguments.xml')
splittingData = self.parseArguments(arguments_path)
except IOError, e:
raise SplitterError(e)
subjobs = []
for maxevents, inputfiles, skipevents in splittingData:
j = self.createSubjob(job)
j.master = job
j.application = job.application
j.inputdata = job.inputdata
j.backend = job.backend
splitter = CRABSplitter()
splitter.maxevents = maxevents
splitter.inputfiles = inputfiles
splitter.skipevents = skipevents
j.splitter = splitter
subjobs.append(j)
return subjobs
class BoxMetadataObject(GangaObject):
"""Internal object to store names"""
_schema = Schema(
Version(1, 0), {
"name":
SimpleItem(defvalue="",
copyable=1,
doc='the name of this object',
typelist=[str])
})
_name = "BoxMetadataObject"
_category = "internal"
_enable_plugin = True
_hidden = 1
def setUp(self):
self.dd = {
'application': ComponentItem(category='applications'),
'backend': ComponentItem(category='backends'),
'name': SimpleItem('', comparable=0),
'workdir': SimpleItem(defvalue=None, type='string', transient=1, protected=1, comparable=0),
'status': SimpleItem(defvalue='new', protected=1, comparable=0),
'id': SimpleItem(defvalue=None, typelist=[str], protected=1, comparable=0),
'inputbox': FileItem(defvalue=[], sequence=1),
'outputbox': FileItem(defvalue=[], sequence=1),
'overriden_copyable': SimpleItem(defvalue=None, protected=1, copyable=1),
'plain_copyable': SimpleItem(defvalue=None, copyable=0)
}
self.s = Schema(Version(1, 0), self.dd)
class CRABDataset(Dataset):
"""Dataset definition for CRAB jobsets."""
schemadic = {}
schemadic.update(CRAB().schemadic)
schemadic.update(TASK().schemadic)
#schemadic['target_site'] = SimpleItem(defvalue=None,
# typelist=['type(None)', 'str'],
# doc='Target site name for the job.')
#schemadic['taskname'] = SimpleItem(defvalue=None, typelist=['type(None)','str'], doc='TaskName of the submitted task, returned from CRAB3 server')
_schema = Schema(Version(1, 0), schemadic)
_category = 'datasets'
_name = 'CRABDataset'
def __init__(self):
super(CRABDataset, self).__init__()
class CRABDataset(Dataset):
"""Dataset definition for CRAB jobsets."""
schemadic = {}
schemadic.update(CMSSW().schemadic)
schemadic.update(CRAB().schemadic)
schemadic.update(GRID().schemadic)
schemadic.update(USER().schemadic)
schemadic['target_site'] = SimpleItem(defvalue=None,
typelist=['type(None)', 'str'],
doc='Target site name for the job.')
_schema = Schema(Version(1, 0), schemadic)
_category = 'datasets'
_name = 'CRABDataset'
def __init__(self):
super(CRABDataset, self).__init__()
class OptionsFileSplitter(ISplitter):
'''Split a jobs based on a list of option file fragments
This Splitter takes as argument a list of option file statements and will
generate a job for each item in this list. The value of the indevidual list
item will be appended to the master options file. A use case of this
splitter would be to change a parameter in an algorithm (e.g. a cut) and to
recreate a set of jobs with different cuts
'''
_name = "OptionsFileSplitter"
docstr = "List of option-file strings, each list item creates a new subjob"
_schema = Schema(Version(1, 0),
{'optsArray': SimpleItem(defvalue=[], doc=docstr)})
def _create_subjob(self, job, inputdata):
j = Job()
j.copyFrom(job)
j.splitter = None
j.merger = None
j.inputsandbox = [] # master added automatically
j.inputfiles = []
j.inputdata = inputdata
return j
def split(self, job):
subjobs = []
inputdata = job.inputdata
if not job.inputdata:
share_path = os.path.join(
expandfilename(getConfig('Configuration')['gangadir']),
'shared',
getConfig('Configuration')['user'],
job.application.is_prepared.name, 'inputdata',
'options_data.pkl')
if os.path.exists(share_path):
f = open(share_path, 'r+b')
inputdata = pickle.load(f)
f.close()
for i in self.optsArray:
j = self._create_subjob(job, inputdata)
j._splitter_data = i
subjobs.append(j)
return subjobs
class OutputData(GangaObject):
'''Class for handling outputdata for LHCb jobs.
Example Usage:
od = OutputData(["file.1","file.2"])
od[0] # "file.1"
[...etc...]
'''
schema = {}
schema['files'] = SimpleItem(defvalue=[], typelist=['str'], sequence=1)
schema['location'] = SimpleItem(defvalue='', typelist=['str'])
_schema = Schema(Version(1, 1), schema)
_category = 'datasets'
_name = "OutputData"
_exportmethods = ['__len__', '__getitem__']
def __init__(self, files=None):
if files is None:
files = []
super(OutputData, self).__init__()
self.files = files
def __construct__(self, args):
if (len(args) != 1) or (type(args[0]) not in [list, tuple]):
super(OutputData, self).__construct__(args)
else:
self.files = args[0]
def __len__(self):
"""The number of files in the dataset."""
result = 0
if self.files:
result = len(self.files)
return result
def __nonzero__(self):
"""This is always True, as with an object."""
return True
def __getitem__(self, i):
'''Proivdes scripting (e.g. od[2] returns the 3rd file name) '''
if type(i) == type(slice(0)):
return GPIProxyObjectFactory(OutputData(files=self.files[i]))
else:
return self.files[i]
class IChecker(IPostProcessor):
"""
Abstract class which all checkers inherit from.
"""
_schema = Schema(
Version(1, 0), {
'checkSubjobs': SimpleItem(defvalue=True, doc='Run on subjobs'),
'checkMaster': SimpleItem(defvalue=True, doc='Run on master')
})
_category = 'postprocessor'
_name = 'IChecker'
_hidden = 1
order = 2
def execute(self, job, newstatus):
"""
Execute the check method, if check fails pass the check and issue an ERROR message. Message is also added to the debug folder.
"""
if newstatus == 'completed':
# If we're master job and check master check.
# If not master job and check subjobs check
if (job.master is None and self.checkMaster) or\
((job.master is not None) and self.checkSubjobs):
try:
return self.check(job)
except Exception as e:
with open(
os.path.join(job.getDebugWorkspace().getPath(),
'checker_errors.txt'),
'a') as debug_file:
debug_file.write(
'\n Checker has failed with the following error: \n'
)
debug_file.write(str(e))
logger.error("%s" % e)
return True
else:
return True
def check(self, job):
"""
Method to check the output of jobs.
Should be overidden.
"""
raise NotImplementedError
class SkbNotifier(IPostProcessor):
"""Ganga notifier for skybeard telegram API"""
_schema = Schema(Version(1, 0), {
'verbose': SimpleItem(
defvalue = False,
doc = 'Send telegram message on subjob completion'),
'address': SimpleItem(
defvalue = '',
doc = 'Skb server address or IP',
optional = False),
'key': SimpleItem(
defvalue = ''
doc = 'Auth key for skb server',
optional = False),
'chat_id': SimpleItem(
defvalue = '',
doc = 'ID of chat to send notifications to',
optional = False)
})
class GridftpFileIndex(GridFileIndex):
"""
Data object containing Gridftp file index information.
- id: gsiftp URI
- name: basename of the file
- md5sum: md5 checksum
- attributes['fpath']: path of the file on local machine
@author: Hurng-Chun Lee
@contact: [email protected]
"""
_schema = Schema(Version(1, 0), gridftp_file_index_schema_datadict)
_category = 'GridFileIndex'
_name = 'GridftpFileIndex'
def __init__(self):
super(GridftpFileIndex, self).__init__()
class LCGFileIndex(GridFileIndex):
"""
Data object containing LCG file index information.
@author: Hurng-Chun Lee
@contact: [email protected]
"""
lcg_file_index_schema_datadict.update({
'lfc_host': SimpleItem(defvalue='', copyable=1, doc='the LFC hostname'),
'local_fpath': SimpleItem(defvalue='', copyable=1, doc='the original file path on local machine')
})
_schema = Schema(Version(1, 0), lcg_file_index_schema_datadict)
_category = 'GridFileIndex'
_name = 'LCGFileIndex'
def __init__(self):
super(LCGFileIndex, self).__init__()
class TaskChainInput(Dataset):
"""Dummy dataset to map the output of a transform to the input of another transform"""
_schema = Schema(
Version(1, 0), {
'input_trf_id':
SimpleItem(defvalue=-1, doc="Input Transform ID"),
'single_unit':
SimpleItem(
defvalue=False,
doc='Create a single unit from all inputs in the transform'),
'use_copy_output':
SimpleItem(
defvalue=True,
doc=
'Use the copied output instead of default output (e.g. use local copy instead of grid copy)'
),
'include_file_mask':
SimpleItem(
defvalue=[],
typelist=['str'],
sequence=1,
doc=
'List of Regular expressions of which files to include for input'
),
'exclude_file_mask':
SimpleItem(
defvalue=[],
typelist=['str'],
sequence=1,
doc=
'List of Regular expressions of which files to exclude for input'
),
})
_category = 'datasets'
_name = 'TaskChainInput'
_exportmethods = []
def __init__(self):
super(TaskChainInput, self).__init__()
class IPostProcessor(GangaObject):
"""
Abstract base class for all classes with the postprocessor category.
"""
_schema = Schema(Version(0, 0), {})
_category = 'postprocessor'
_hidden = 1
order = 0
success = True
failure = False
def __init__(self):
super(IPostProcessor, self).__init__()
def execute(self, job, **options):
"""
To be overidden by inherited class
"""
raise NotImplementedError
class ExeSplitter(ISplitter):
""" Split executable applications (OBSOLETE).
This splitter allows the creation of subjobs where each subjob has a different Executable application.
This splitter is OBSOLETED use GenericSplitter or ArgSplitter instead.
"""
_name = "ExeSplitter"
_schema = Schema(Version(1, 0), {
'apps': ComponentItem('applications', defvalue=[], sequence=1, doc='a list of Executable app objects')
})
def split(self, job):
subjobs = []
for a in self.apps:
# for each subjob make a full copy of the master job
j = self.createSubjob(job)
j.application = a
if not a.exe:
j.application.exe = job.application.exe
subjobs.append(j)
return subjobs
class SandboxFile(LocalFile):
_schema = Schema(
Version(1, 1), {
'namePattern':
SimpleItem(defvalue="", doc='pattern of the file name'),
'localDir':
SimpleItem(
defvalue="",
doc=
'local dir where the file is stored, used from get and put methods'
),
'subfiles':
ComponentItem(category='gangafiles',
defvalue=[],
hidden=1,
typelist=['Ganga.GPIDev.Lib.File.SandboxFile'],
sequence=1,
copyable=0,
doc="collected files from the wildcard namePattern"),
'compressed':
SimpleItem(
defvalue=False,
typelist=['bool'],
protected=0,
doc=
'wheather the output file should be compressed before sending somewhere'
)
})
_category = 'gangafiles'
_name = "SandboxFile"
def __init__(self, namePattern='', localDir='', **kwds):
""" name is the name of the output file that is going to be processed
in some way defined by the derived class
"""
logger.warning(
"SandboxFile is now deprecated please change your configuration to use LocalFile instead!"
)
super(SandboxFile, self).__init__(namePattern, localDir, **kwds)
class CopySplitter(ISplitter):
"""A simple splitter for testing. Does nothing at all clever"""
_category = 'splitters'
_name = 'CopySplitter'
_schema = Schema(
Version(1, 0), {
'number':
SimpleItem(defvalue=5),
'function_hook':
SimpleItem(defvalue=None,
doc='Name of function in global namespace to call')
})
def split(self, job):
subjobs = []
for i in range(self.number):
j = self.createSubjob(job)
j.splitter = None
j.application = job.application
j.backend = job.backend
from Ganga.Utility.Config import getConfig
if not getConfig('Output')['ForbidLegacyInput']:
j.inputsandbox = job.inputsandbox[:]
else:
j.inputfiles = job.inputfiles[:]
j.outputfiles = job.outputfiles[:]
# gives tests a chance to alter subjobs
if self.function_hook:
g = globals()
if self.function_hook in g:
g[self.function_hook](j, i)
subjobs.append(j)
return subjobs
class SampleGangaObject(GangaObject):
_schema = Schema(Version(1, 0), {
'a': SimpleItem(42, typelist=[int]),
# 'b' is skipped on purpose
'c': ComponentItem('gangafiles'),
})
_category = 'TestGangaObject'
_name = 'TestGangaObject'
_exportmethods = ['example', 'check_not_proxy']
def example(self):
return 'example_string'
def check_not_proxy(self, obj):
assert not Ganga.GPIDev.Base.Proxy.isProxy(obj), 'incoming argument should be proxy-stripped'
ret = SampleGangaObject()
assert not Ganga.GPIDev.Base.Proxy.isProxy(ret), 'new object should not be proxy-wrapped'
return ret
def not_proxied(self):
return 'example_string'
class CoreUnit(IUnit):
_schema = Schema(Version(1, 0),
dict(IUnit._schema.datadict.items() + {}.items()))
_category = 'units'
_name = 'CoreUnit'
_exportmethods = IUnit._exportmethods + []
def __init__(self):
super(CoreUnit, self).__init__()
def createNewJob(self):
"""Create any jobs required for this unit"""
j = GPI.Job()
j.backend = self._getParent().backend.clone()
# copy form ourselves or the parent transform depending on what's
# specified
fields = [
'application', 'splitter', 'inputfiles', 'inputdata',
'inputsandbox', 'outputfiles', 'postprocessors'
]
for f in fields:
if (f == "postprocessors"
and len(getattr(self, f).process_objects) > 0):
j.postprocessors = copy.deepcopy(addProxy(self).postprocessors)
elif (f != "postprocessors" and getattr(self, f)):
setattr(j, f, copy.deepcopy(getattr(self, f)))
elif (f == "postprocessors"
and len(getattr(self._getParent(), f).process_objects) > 0):
j.postprocessors = copy.deepcopy(
addProxy(self._getParent()).postprocessors)
elif (f != "postprocessors" and getattr(self._getParent(), f)):
setattr(j, f, copy.deepcopy(getattr(self._getParent(), f)))
return j
