class ND280Kin_Checker(IFileChecker):
"""
o Checks .log file (not impl. yet)
o Checks if .kin file is present and non-zero
o Moves output files to their destinations
"""
_schema = IFileChecker._schema.inherit_copy()
_schema.datadict['prfx'] = SimpleItem(defvalue = None, typelist=['str','type(None)'], doc='Path prefix to store output files')
_schema.datadict['path'] = SimpleItem(defvalue = None, typelist=['str','type(None)'], doc='Middle path to store output files')
_category = 'postprocessor'
_name = 'ND280Kin_Checker'
_exportmethods = ['check']
def check(self,job):
"""
Checks if .kin file is present and non-zero
"""
logger.info("Checking/moving outputs of run "+job.name)
self.files = ['*.kin']
filepaths = self.findFiles(job)
if len(filepaths) != 1:
logger.error('Something wrong with kin file(s) '+filepaths+'. CANNOT CONTINUE')
self.move_output(job,ok=False)
return False
kinf = filepaths[0]
if os.stat(kinf).st_size == 0:
logger.error('Zero kin file '+kinf+'. CANNOT CONTINUE')
self.move_output(job,ok=False)
return False
logger.info('OK')
self.move_output(job,ok=True)
return True
def move_output(self,job,ok=True):
dest = os.path.join(self.prfx,self.path)
jout = 'KIN_'+job.name
task = {'*.kin':'kin','*.root':'kin','*.txt':'aux','*.conf':'aux','stdout':'jobOutput'}
if not ok:
task = {'*.kin':'errors','*.root':'errors','*.txt':'errors','*.conf':'errors','stdout':'errors'}
for p in task.keys():
odir = os.path.join(dest,task[p])
self.files = [p]
filepaths = self.findFiles(job)
for f in filepaths:
if not os.path.isdir(odir):
os.makedirs(odir)
shutil.move(f,odir)
# rename stdout in odir
if p == 'stdout':
shutil.move(os.path.join(odir,p),os.path.join(odir,jout))
class GridFileIndex(GangaObject):
'''
Data object for indexing a file on the grid.
@author: Hurng-Chun Lee
@contact: [email protected]
'''
_schema = Schema(
Version(1, 0), {
'id':
SimpleItem(defvalue='', doc='the main identity of the file'),
'name':
SimpleItem(defvalue='', doc='the name of the file'),
'md5sum':
SimpleItem(defvalue='', doc='the md5sum of the file'),
'attributes':
SimpleItem(defvalue={}, doc='a key:value pairs of file metadata')
})
_category = 'GridFileIndex'
_name = 'GridFileIndex'
logger = getLogger()
def __init__(self):
super(GridFileIndex, self).__init__()
def __eq__(self, other):
return other.id == self.id
class DiracProxy(ICredentialRequirement):
"""
An object specifying the requirements of a DIRAC proxy file
"""
_schema = ICredentialRequirement._schema.inherit_copy()
_schema.datadict['group'] = SimpleItem(defvalue=None, typelist=[str, None], doc='Group for the proxy')
_schema.datadict['encodeDefaultProxyFileName'] = \
SimpleItem(defvalue=True, doc='Should the proxy be generated with the group encoded onto the end of the proxy filename')
_category = 'CredentialRequirement'
info_class = DiracProxyInfo
def __init__(self, **kwargs):
"""
Constructs a dirac proxy requirement and assigns the default group if none is provided
"""
super(DiracProxy, self).__init__(**kwargs)
if self.group is None:
raise GangaValueError('DIRAC Proxy `group` is not set. Set this in ~/.gangarc in `[defaults_DiracProxy]/group`')
def encoded(self):
"""
This returns the encoding used to store a unique DIRAC proxy for each group
"""
my_config = getConfig('defaults_DiracProxy')
default_group = my_config['group']
if (my_config['encodeDefaultProxyFileName'] and self.group == default_group) or self.group != default_group:
return ':'.join(requirement for requirement in [self.group] if requirement) # filter out the empties
else:
return ''
class TestGangaObject(GangaObject):
"""Test Ganga Object. Is used to construct test jobs"""
_schema = Schema(
Version(1, 0), {
'id':
SimpleItem('0', doc='ID Needed for tests'),
'name':
SimpleItem(
'',
doc=
'optional label which may be any combination of ASCII characters',
typelist=['str']),
'subjobs':
ComponentItem('internal',
defvalue=[],
sequence=1,
protected=1,
load_default=0,
copyable=0,
optional=1,
doc='test subjobs'),
})
_name = "TestGangaObject"
_category = "internal"
def __init__(self, name='TestObjectName', sj=0):
super(TestGangaObject, self).__init__()
self.name = name
for i in range(sj):
self.subjobs.append(TestGangaObject(name + "." + str(i)))
class Notifier(IPostProcessor):
"""
Object which emails a user about jobs status are they have finished. The default behaviour is to email when a job has failed or when a master job has completed.
Notes:
* Ganga must be running to send the email, so this object is only really useful if you have a ganga session running the background (e.g. screen session).
* Will not send emails about failed subjobs if autoresubmit is on.
"""
_schema = Schema(
Version(1, 0), {
'verbose': SimpleItem(defvalue=False,
doc='Email on subjob completion'),
'address': SimpleItem(
defvalue='', doc='Email address', optional=False)
})
_category = 'postprocessor'
_name = 'Notifier'
order = 3
def execute(self, job, newstatus):
"""
Email user if:
* job is a master job, or
* job has failed but do not have auto resubmit
* job has not failed but verbose is set to true
"""
if len(job.subjobs) or (newstatus == 'failed' and job.do_auto_resubmit
is False) or (newstatus != 'failed'
and self.verbose is True):
return self.email(job, newstatus)
return True
def email(self, job, newstatus):
"""
Method to email a user about a job
"""
sender = '*****@*****.**'
receivers = self.address
subject = 'Ganga Notification: Job(%s) has %s.' % (job.fqid, newstatus)
msg_string = """
Dear User,\n
Job(%s) has gone into %s state.\n
Regards,
Ganga\n
PS: This is an automated notification from Ganga,
if you would like these messages to stop please
remove the notifier object from future jobs.
""" % (job.fqid, newstatus)
msg = email.message_from_string(msg_string)
msg['Subject'] = subject
msg['From'] = sender
msg['To'] = receivers
string_message = msg.as_string()
try:
smtpObj = smtplib.SMTP(config['SMTPHost'])
smtpObj.sendmail(sender, receivers, string_message)
except smtplib.SMTPException as e:
raise PostProcessException(str(e))
return True
class IFileChecker(IChecker):
"""
Abstract class which all checkers inherit from.
"""
_schema = IChecker._schema.inherit_copy()
_schema.datadict['files'] = SimpleItem(defvalue=[], doc='File to search in')
_schema.datadict['filesMustExist'] = SimpleItem(True, doc='Toggle whether to fail job if a file isn\'t found.')
_category = 'postprocessor'
_name = 'IFileChecker'
_hidden = 1
result = True
order = 2
def findFiles(self, job):
if not len(self.files):
raise PostProcessException(
'No files specified, %s will do nothing!' % getName(self))
filepaths = []
for f in self.files:
filepath = os.path.join(job.outputdir, f)
for expanded_file in glob.glob(filepath):
filepaths.append(expanded_file)
if not len(glob.glob(filepath)):
if (self.filesMustExist):
logger.info(
'The files %s does not exist, %s will fail job(%s) (to ignore missing files set filesMustExist to False)', filepath, self._name, job.fqid)
self.result = False
else:
logger.warning(
'Ignoring file %s as it does not exist.', filepath)
return filepaths
class Im3ShapeSplitter(ISplitter):
"""
This splitter splits jobs using the Im3ShapeApp application using the size parameter.
If a splitter is configured with size = 5, split_by_file = True, then it will create 5 subjobs per file in the master_job.inputdata
If a splitter is configured with size = 5, split_by_file = False, then it will create 5 subjobs total and configure all subjobs to use all given data.
In the future there may be support for splitting based upon regex and namePatterns in the inputdata to allow a certain subset of data to be put in each subjob.
"""
_name = "Im3ShapeSplitter"
_schema = Schema(
Version(1, 0), {
'size':
SimpleItem(defvalue=5,
doc='Size of the tiles which are to be split.'),
'split_by_file':
SimpleItem(
defvalue=True,
doc=
'Should we auto-split into subjobs here on a per-file basis?')
})
def split(self, job):
"""
Actually perform the splitting of the given master job. The generated subjobs of the splitting are returned
Args:
job (Job): This is the master job object which is to be split and the subjobs of which are returned
"""
assert isinstance(job.application, Im3ShapeApp)
subjobs = []
def getApp(job, rank, size):
app = copy.deepcopy(job.application)
app.rank = rank
app.size = size
return app
if self.split_by_file:
for this_file in job.inputdata:
for rank in range(0, self.size):
j = self.createSubjob(job, ['application'])
# Add new arguments to subjob
j.application = getApp(job, rank, self.size)
j.inputdata = GangaDataset(files=[stripProxy(this_file)])
subjobs.append(j)
else:
for rank in range(0, self.size):
j = self.createSubjob(job, ['application'])
j.application = getApp(job, rank, self.size)
j.inputdata = job.inputdata
logger.debug('Rank for split job is: ' + str(rank))
subjobs.append(j)
return subjobs
class GangaDatasetSplitter(ISplitter):
""" Split job based on files given in GangaDataset inputdata field """
_name = "GangaDatasetSplitter"
_schema = Schema(Version(1, 0), {
'files_per_subjob': SimpleItem(defvalue=5, doc='the number of files per subjob', typelist=[int]),
'maxFiles': SimpleItem(defvalue=-1,
doc='Maximum number of files to use in a masterjob (None or -1 = all files)',
typelist=[int, None]),
})
def split(self, job):
subjobs = []
if not job.inputdata or not isType(job.inputdata, GangaDataset):
raise ApplicationConfigurationError(
"No GangaDataset given for GangaDatasetSplitter")
# find the full file list
full_list = []
for f in job.inputdata.files:
if f.containsWildcards():
# we have a wildcard so grab the subfiles
for sf in f.getSubFiles(process_wildcards=True):
full_list.append(sf)
else:
# no wildcards so just add the file
full_list.append(f)
if len(full_list) == 0:
raise ApplicationConfigurationError(
"GangaDatasetSplitter couldn't find any files to split over")
masterType = type(job.inputdata)
# split based on all the sub files
fid = 0
subjobs = []
filesToRun = len(full_list)
if not self.maxFiles == -1:
filesToRun = min(self.maxFiles, filesToRun)
while fid < filesToRun:
j = self.createSubjob(job)
j.inputdata = masterType()
j.inputdata.treat_as_inputfiles = job.inputdata.treat_as_inputfiles
for sf in full_list[fid:fid + self.files_per_subjob]:
j.inputdata.files.append(sf)
fid += self.files_per_subjob
subjobs.append(j)
return subjobs
class TaskLocalCopy(Dataset):
"""Dummy dataset to force Tasks to copy the output from a job to local storage somewhere"""
_schema = Schema(
Version(1, 0), {
'local_location':
SimpleItem(defvalue="", doc="Local location to copy files to"),
'include_file_mask':
SimpleItem(
defvalue=[],
typelist=['str'],
sequence=1,
doc=
'List of Regular expressions of which files to include in copy'
),
'exclude_file_mask':
SimpleItem(
defvalue=[],
typelist=['str'],
sequence=1,
doc=
'List of Regular expressions of which files to exclude from copy'
),
'files':
SimpleItem(defvalue=[],
typelist=['str'],
sequence=1,
doc='List of successfully downloaded files'),
})
_category = 'datasets'
_name = 'TaskLocalCopy'
_exportmethods = ["isValid", "isDownloaded"]
def __init__(self):
super(TaskLocalCopy, self).__init__()
def isValid(self, fname):
"""Check if this file should be downloaded"""
for in_re in self.include_file_mask:
if not re.search(in_re, fname):
return False
for out_re in self.exclude_file_mask:
if re.search(out_re, fname):
return False
return True
def isDownloaded(self, fname):
"""Check if this file is present at the local_location"""
return os.path.exists(os.path.join(self.local_location, fname))
class CRABSplitter(ISplitter):
"""Splitter object for CRAB jobs."""
schemadic = {}
schemadic['maxevents'] = SimpleItem(defvalue=None,
typelist=['type(None)', 'int'],
doc='Maximum number of events/task')
schemadic['inputfiles'] = SimpleItem(defvalue=None,
typelist=['type(None)', 'str'],
doc='Number of input files')
schemadic['skipevents'] = SimpleItem(defvalue=None,
typelist=['type(None)', 'int'],
doc='Offset for the events')
_name = 'CRABSplitter'
_schema = Schema(Version(1, 0), schemadic)
def parseArguments(self, path):
"""Gets some job arguments from the FJR."""
splittingData = []
for job in parse(path).getElementsByTagName("Job"):
splittingData.append([
job.getAttribute("MaxEvents"),
job.getAttribute("InputFiles"),
job.getAttribute("SkipEvents")
])
return splittingData
def split(self, job):
"""Main splitter for the job."""
try:
arguments_path = os.path.join(job.inputdata.ui_working_dir,
'share/arguments.xml')
splittingData = self.parseArguments(arguments_path)
except IOError, e:
raise SplitterError(e)
subjobs = []
for maxevents, inputfiles, skipevents in splittingData:
j = self.createSubjob(job)
j.master = job
j.application = job.application
j.inputdata = job.inputdata
j.backend = job.backend
splitter = CRABSplitter()
splitter.maxevents = maxevents
splitter.inputfiles = inputfiles
splitter.skipevents = skipevents
j.splitter = splitter
subjobs.append(j)
return subjobs
class FileChecker(IFileChecker):
"""
Checks if string is in file.
self.searchStrings are the files you would like to check for.
self.files are the files you would like to check.
self.failIfFound (default = True) decides whether to fail the job if the string is found. If you set this to false the job will fail if the string *isnt* found.
self.fileMustExist toggles whether to fail the job if the specified file doesn't exist (default is True).
"""
_schema = IFileChecker._schema.inherit_copy()
_schema.datadict['searchStrings'] = SimpleItem(defvalue=[],
doc='String to search for')
_schema.datadict['failIfFound'] = SimpleItem(
True, doc='Toggle whether job fails if string is found or not found.')
_category = 'postprocessor'
_name = 'FileChecker'
_exportmethods = ['check']
def check(self, job):
"""
Check that a string is in a file, takes the job object as input.
"""
if not len(self.searchStrings):
raise PostProcessException(
'No searchStrings specified, FileChecker will do nothing!')
filepaths = self.findFiles(job)
if not len(filepaths):
raise PostProcessException(
'None of the files to check exist, FileChecker will do nothing!'
)
for filepath in filepaths:
for searchString in self.searchStrings:
stringFound = False
# self.findFiles() guarantees that file at filepath exists,
# hence no exception handling
with open(filepath) as file:
for line in file:
if re.search(searchString, line):
if self.failIfFound is True:
logger.info(
'The string %s has been found in file %s, FileChecker will fail job(%s)',
searchString, filepath, job.fqid)
return self.failure
stringFound = True
if not stringFound and self.failIfFound is False:
logger.info(
'The string %s has not been found in file %s, FileChecker will fail job(%s)',
searchString, filepath, job.fqid)
return self.failure
return self.result
class SimpleGangaObject(GangaObject):
_schema = Schema(Version(1, 0), {
'a': SimpleItem(42, typelist=[int]),
})
_category = 'TestGangaObject'
_hidden = True
_enable_plugin = True
class DefaultSplitter(ISplitter):
"""
The DefaultSplitter is assigned to all jobs by default and is intended to provide a single subjob for every job on submit.
This has been implemented as it potentially simplifies the internal logic in job managment significantly.
This splitter is not expected to be configurable or to split a dataset based upon any input.
In order to do this please make use of another splitter.
"""
_name = "DefaultSplitter"
## A dummy value is required to not get a bug in writing the object to an XML repo.
## The nature of the problem of writing an empty schema should probably be understood more correctly but is difficult to track down -rcurrie
_schema = Schema(
Version(1, 0), {
'dummy_value':
SimpleItem(defvalue=1,
hidden=1,
visitable=0,
doc='the number of files per subjob',
typelist=["int"])
})
def split(self, job):
subjobs = []
sj = self.createSubjob(job)
subjobs.append(sj)
return subjobs
class SampleGangaObject(GangaObject):
_schema = Schema(
Version(1, 0),
{
'a': SimpleItem(42, typelist=[int]),
# 'b' is skipped on purpose
'c': ComponentItem('gangafiles'),
})
_category = 'TestGangaObject'
_name = 'TestGangaObject'
_exportmethods = ['example', 'check_not_proxy']
def example(self):
return 'example_string'
def check_not_proxy(self, obj):
assert not Ganga.GPIDev.Base.Proxy.isProxy(
obj), 'incoming argument should be proxy-stripped'
ret = SampleGangaObject()
assert not Ganga.GPIDev.Base.Proxy.isProxy(
ret), 'new object should not be proxy-wrapped'
return ret
def not_proxied(self):
return 'example_string'
class CRABBackend(IBackend):
"""Backend implementation for CRAB."""
schemadic = {}
schemadic['verbose'] = SimpleItem(defvalue=1,
typelist=['int'],
doc='Set to 0 to disable CRAB logging')
schemadic['statusLog'] = SimpleItem(defvalue=0,
typelist=['int'],
doc='Set to 1 to keep -status logs')
def __init__(self):
super(CRABBackend, self).__init__()
config = Config.getConfig('CMSSW')
shell = Shell(os.path.join(config['CMSSW_SETUP'], 'CMSSW_generic.sh'),
[config['CMSSW_VERSION'], config['CRAB_VERSION']])
self.crab_env = shell.env
class ThreadedTestGangaObject(GangaObject):
_schema = Schema(Version(1, 0), {
'a': SimpleItem(42, typelist=[int]),
'b': ComponentItem('TestGangaObject', defvalue='SimpleGangaObject'),
})
_category = 'TestGangaObject'
_hidden = True
_enable_plugin = True
class OutputData(GangaObject):
'''Class for handling outputdata for LHCb jobs.
Example Usage:
od = OutputData(["file.1","file.2"])
od[0] # "file.1"
[...etc...]
'''
schema = {}
schema['files'] = SimpleItem(defvalue=[], typelist=['str'], sequence=1)
schema['location'] = SimpleItem(defvalue='', typelist=['str'])
_schema = Schema(Version(1, 1), schema)
_category = 'datasets'
_name = "OutputData"
_exportmethods = ['__len__', '__getitem__']
def __init__(self, files=None):
if files is None:
files = []
super(OutputData, self).__init__()
self.files = files
def __construct__(self, args):
if (len(args) != 1) or (type(args[0]) not in [list, tuple]):
super(OutputData, self).__construct__(args)
else:
self.files = args[0]
def __len__(self):
"""The number of files in the dataset."""
result = 0
if self.files:
result = len(self.files)
return result
def __nonzero__(self):
"""This is always True, as with an object."""
return True
def __getitem__(self, i):
'''Proivdes scripting (e.g. od[2] returns the 3rd file name) '''
if type(i) == type(slice(0)):
return GPIProxyObjectFactory(OutputData(files=self.files[i]))
else:
return self.files[i]
class IChecker(IPostProcessor):
"""
Abstract class which all checkers inherit from.
"""
_schema = Schema(
Version(1, 0), {
'checkSubjobs': SimpleItem(defvalue=True, doc='Run on subjobs'),
'checkMaster': SimpleItem(defvalue=True, doc='Run on master')
})
_category = 'postprocessor'
_name = 'IChecker'
_hidden = 1
order = 2
def execute(self, job, newstatus):
"""
Execute the check method, if check fails pass the check and issue an ERROR message. Message is also added to the debug folder.
"""
if newstatus == 'completed':
# If we're master job and check master check.
# If not master job and check subjobs check
if (job.master is None and self.checkMaster) or\
((job.master is not None) and self.checkSubjobs):
try:
return self.check(job)
except Exception as e:
with open(
os.path.join(job.getDebugWorkspace().getPath(),
'checker_errors.txt'),
'a') as debug_file:
debug_file.write(
'\n Checker has failed with the following error: \n'
)
debug_file.write(str(e))
logger.error("%s" % e)
return True
else:
return True
def check(self, job):
"""
Method to check the output of jobs.
Should be overidden.
"""
raise NotImplementedError
class ArgSplitter(ISplitter):
"""
Split job by changing the args attribute of the application.
This splitter only applies to the applications which have args attribute (e.g. Executable, Root).
It is a special case of the GenericSplitter.
This splitter allows the creation of a series of subjobs where
the only difference between different jobs are their
arguments. Below is an example that executes a ROOT script ~/analysis.C
void analysis(const char* type, int events) {
std::cout << type << " " << events << std::endl;
}
with 3 different sets of arguments.
s = ArgSplitter(args=[['AAA',1],['BBB',2],['CCC',3]])
r = Root(version='5.10.00',script='~/analysis.C')
j.Job(application=r, splitter=s)
Notice how each job takes a list of arguments (in this case a list
with a string and an integer). The splitter thus takes a list of
lists, in this case with 3 elements so there will be 3 subjobs.
Running the subjobs will produce the output:
subjob 1 : AAA 1
subjob 2 : BBB 2
subjob 3 : CCC 3
"""
_name = "ArgSplitter"
_schema = Schema(
Version(1, 0), {
'args':
SimpleItem(defvalue=[],
typelist=[
'list',
'Ganga.GPIDev.Lib.GangaList.GangaList.GangaList'
],
sequence=1,
doc='A list of lists of arguments to pass to script')
})
def split(self, job):
subjobs = []
for arg in self.args:
j = self.createSubjob(job, ['application'])
# Add new arguments to subjob
app = copy.deepcopy(job.application)
app.args = arg
j.application = app
logger.debug('Arguments for split job is: ' + str(arg))
subjobs.append(stripProxy(j))
return subjobs
class MetaDataChecker(IChecker):
"""
Checks the meta data of a job This class must be overidden to convert the experiment specific metadata.
"""
_schema = IChecker._schema.inherit_copy()
_schema.datadict['expression'] = SimpleItem(defvalue=None,
typelist=['str', 'type(None)'],
doc='The metadata attribute')
_schema.datadict['result'] = SimpleItem(defvalue=None,
typelist=['bool', 'type(None)'],
hidden=1,
doc='Check result')
_category = 'postprocessor'
_name = 'MetaDataChecker'
_hidden = 1
def calculateResult(self, job):
"""
To be overidden by experiment specific class
"""
raise NotImplementedError
def check(self, job):
"""
Checks metadata of job is within a certain range.
"""
if self.expression == None:
raise PostProcessException(
'No expression is set. MetaDataChecker will do nothing!')
try:
self.result = self.calculateResult(job)
except Exception as e:
raise PostProcessException(
'There was an error parsing the checker expression: %s - MetaDataChecker will do nothing!'
% e)
if self.result is not True and self.result is not False:
raise PostProcessException(
'The expression "%s" did not evaluate to True or False, MetaDataChecker will do nothing!'
% self.expression)
if self.result is False:
logger.info(
'MetaDataChecker has failed job(%s) because the expression "%s" is False'
% (job.fqid, self.expression))
return self.result
class SkbNotifier(IPostProcessor):
"""Ganga notifier for skybeard telegram API"""
_schema = Schema(Version(1, 0), {
'verbose': SimpleItem(
defvalue = False,
doc = 'Send telegram message on subjob completion'),
'address': SimpleItem(
defvalue = '',
doc = 'Skb server address or IP',
optional = False),
'key': SimpleItem(
defvalue = ''
doc = 'Auth key for skb server',
optional = False),
'chat_id': SimpleItem(
defvalue = '',
doc = 'ID of chat to send notifications to',
optional = False)
})
class VomsProxy(ICredentialRequirement):
"""
An object specifying the requirements of a VOMS proxy file
"""
_schema = ICredentialRequirement._schema.inherit_copy()
_schema.datadict['identity'] = SimpleItem(defvalue=None,
typelist=[str, None],
doc='Identity for the proxy')
_schema.datadict['vo'] = SimpleItem(
defvalue=None,
typelist=[str, None],
doc=
'Virtual Organisation for the proxy. Defaults to LGC/VirtualOrganisation'
)
_schema.datadict['role'] = SimpleItem(defvalue=None,
typelist=[str, None],
doc='Role that the proxy must have')
_schema.datadict['group'] = SimpleItem(
defvalue=None,
typelist=[str, None],
doc='Group for the proxy - either "group" or "group/subgroup"')
_category = 'CredentialRequirement'
info_class = VomsProxyInfo
def __init__(self, **kwargs):
"""
Construct a voms proxy requirement and assign the default vo from the config if none has been provided
"""
super(VomsProxy, self).__init__(**kwargs)
if 'vo' not in kwargs and getConfig('LCG')['VirtualOrganisation']:
self.vo = getConfig('LCG')['VirtualOrganisation']
def encoded(self):
"""
This returns the additional encoding of the identity, vo, role and group which are to be encoded into the voms file location
"""
return ':'.join(
requirement
for requirement in [self.identity, self.vo, self.role, self.group]
if requirement) # filter out the empties
class LCGFileIndex(GridFileIndex):
"""
Data object containing LCG file index information.
@author: Hurng-Chun Lee
@contact: [email protected]
"""
lcg_file_index_schema_datadict.update({
'lfc_host': SimpleItem(defvalue='', copyable=1, doc='the LFC hostname'),
'local_fpath': SimpleItem(defvalue='', copyable=1, doc='the original file path on local machine')
})
_schema = Schema(Version(1, 0), lcg_file_index_schema_datadict)
_category = 'GridFileIndex'
_name = 'LCGFileIndex'
def __init__(self):
super(LCGFileIndex, self).__init__()
class TaskChainInput(Dataset):
"""Dummy dataset to map the output of a transform to the input of another transform"""
_schema = Schema(
Version(1, 0), {
'input_trf_id':
SimpleItem(defvalue=-1, doc="Input Transform ID"),
'single_unit':
SimpleItem(
defvalue=False,
doc='Create a single unit from all inputs in the transform'),
'use_copy_output':
SimpleItem(
defvalue=True,
doc=
'Use the copied output instead of default output (e.g. use local copy instead of grid copy)'
),
'include_file_mask':
SimpleItem(
defvalue=[],
typelist=['str'],
sequence=1,
doc=
'List of Regular expressions of which files to include for input'
),
'exclude_file_mask':
SimpleItem(
defvalue=[],
typelist=['str'],
sequence=1,
doc=
'List of Regular expressions of which files to exclude for input'
),
})
_category = 'datasets'
_name = 'TaskChainInput'
_exportmethods = []
def __init__(self):
super(TaskChainInput, self).__init__()
class MergerTester(IMerger):
_category = 'postprocessor'
_exportmethods = ['merge']
_name = 'MergerTester'
_schema = IMerger._schema.inherit_copy()
_schema.datadict['alwaysfail'] = SimpleItem(
defvalue=True, doc='Flag to set if the merge should always fail')
_schema.datadict['wait'] = SimpleItem(
defvalue=-1, doc='Time in seconds that the merge should sleep for.')
def mergefiles(self, file_list, output_file):
logger.info('merging')
if self.wait > 0:
logger.info('sleeping for %d seconds' % self.wait)
import time
time.sleep(self.wait)
if self.alwaysfail:
raise PostProcessException(
'This merge will always fail as this is a test')
class TestGangaObject(GangaObject):
_schema = Schema(Version(1, 0), {'a': SimpleItem(42, typelist=['int'])})
_category = 'TestGangaObject'
_name = 'TestGangaObject'
_exportmethods = ['example']
def example(self):
return 'example_string'
def not_proxied(self):
return 'example_string'
class FakeCred(ICredentialRequirement):
"""
An object specifying the requirements of a VOMS proxy file for testing
"""
_schema = ICredentialRequirement._schema.inherit_copy()
_schema.datadict['vo'] = SimpleItem(defvalue=None, typelist=[str, None])
_schema.datadict['role'] = SimpleItem(defvalue=None, typelist=[str, None])
_category = 'CredentialRequirement'
_name = 'TestCred'
info_class = FakeCredInfo
def __init__(self, **kwargs):
super(FakeCred, self).__init__(**kwargs)
def encoded(self):
return ':'.join(requirement for requirement in [self.vo, self.role] if requirement)
def is_empty(self):
return not (self.vo or self.role)
def setUp(self):
self.dd = {
'application':
ComponentItem(category='applications'),
'backend':
ComponentItem(category='backends'),
'name':
SimpleItem('', comparable=0),
'workdir':
SimpleItem(defvalue=None,
type='string',
transient=1,
protected=1,
comparable=0),
'status':
SimpleItem(defvalue='new', protected=1, comparable=0),
'id':
SimpleItem(defvalue=None,
typelist=[str],
protected=1,
comparable=0),
'inputbox':
FileItem(defvalue=[], sequence=1),
'outputbox':
FileItem(defvalue=[], sequence=1),
'overriden_copyable':
SimpleItem(defvalue=None, protected=1, copyable=1),
'plain_copyable':
SimpleItem(defvalue=None, copyable=0)
}
self.s = Schema(Version(1, 0), self.dd)
class CopySplitter(ISplitter):
"""A simple splitter for testing. Does nothing at all clever"""
_category = 'splitters'
_name = 'CopySplitter'
_schema = Schema(
Version(1, 0), {
'number':
SimpleItem(defvalue=5),
'function_hook':
SimpleItem(defvalue=None,
doc='Name of function in global namespace to call')
})
def split(self, job):
subjobs = []
for i in range(self.number):
j = self.createSubjob(job)
j.splitter = None
j.application = job.application
j.backend = job.backend
from Ganga.Utility.Config import getConfig
if not getConfig('Output')['ForbidLegacyInput']:
j.inputsandbox = job.inputsandbox[:]
else:
j.inputfiles = job.inputfiles[:]
j.outputfiles = job.outputfiles[:]
# gives tests a chance to alter subjobs
if self.function_hook:
g = globals()
if self.function_hook in g:
g[self.function_hook](j, i)
subjobs.append(j)
return subjobs
class SandboxFile(LocalFile):
_schema = Schema(
Version(1, 1), {
'namePattern':
SimpleItem(defvalue="", doc='pattern of the file name'),
'localDir':
SimpleItem(
defvalue="",
doc=
'local dir where the file is stored, used from get and put methods'
),
'subfiles':
ComponentItem(category='gangafiles',
defvalue=[],
hidden=1,
typelist=['Ganga.GPIDev.Lib.File.SandboxFile'],
sequence=1,
copyable=0,
doc="collected files from the wildcard namePattern"),
'compressed':
SimpleItem(
defvalue=False,
typelist=['bool'],
protected=0,
doc=
'wheather the output file should be compressed before sending somewhere'
)
})
_category = 'gangafiles'
_name = "SandboxFile"
def __init__(self, namePattern='', localDir='', **kwds):
""" name is the name of the output file that is going to be processed
in some way defined by the derived class
"""
logger.warning(
"SandboxFile is now deprecated please change your configuration to use LocalFile instead!"
)
super(SandboxFile, self).__init__(namePattern, localDir, **kwds)
