def _init(self):
global xml_summary
global xml_schema
if not self.env_var:
raise GangaException('XMLSummary env not set!')
if not self.file:
raise GangaException('File not specified!')
if not os.path.exists(self.file):
raise GangaException('%s does not exist!' % self.file)
p = self._xmlPath()
v = self.env_var
if v not in xml_schema:
if 'schema' in sys.modules:
del sys.modules['schema']
xml_schema[v] = imp.load_source('schema', p + '/schema.py')
if 'summary' in sys.modules:
del sys.modules['summary']
xml_summary[v] = imp.load_source('summary', p + '/summary.py')
xml_summary[v].__schema__ = xml_schema[v]
sum = xml_summary[v].Summary(self._xmlSchema())
sum.parse(self.file)
self.data = sum
if 'schema' in sys.modules:
del sys.modules['schema']
if 'summary' in sys.modules:
del sys.modules['summary']
return
def get_result(command,
logger_message=None,
exception_message=None,
eval_includes=None,
retry_limit=5):
retries = 0
while retries < retry_limit:
try:
result = execute(command, eval_includes=eval_includes)
if not result_ok(result):
if logger_message is not None:
logger.warning('%s: %s' % (logger_message, str(result)))
if exception_message is not None:
logger.warning("Failed to run: %s" % str(command))
logger.warning("includes:\n%s" % str(eval_includes))
logger.warning("Result: '%s'" % str(result))
raise GangaException(exception_message)
raise GangaException("Failed to return result of '%s': %s" % (command, result))
return result
except Exception as x:
import time
logger.debug("Sleeping for 5 additional seconds to reduce possible overloading")
time.sleep(5.)
if retries == retry_limit - 1:
raise x
retries = retries + 1
logger.error("An Error Occured: %s" % str(x))
logger.error("Retrying: %s / %s " % (str(retries + 1), str(retry_limit)))
def uploadLocalFile(job, namePattern, localDir, should_del=True):
"""
Upload a locally available file to the grid as a DiracFile.
Randomly chooses an SE.
Args:
namePattern (str): name of the file
localDir (str): localDir of the file
should_del = (bool): should we delete the local file?
Return
DiracFile: a DiracFile of the uploaded LFN on the grid
"""
new_df = DiracFile(namePattern, localDir=localDir)
trySEs = getConfig('DIRAC')['allDiracSE']
random.shuffle(trySEs)
new_lfn = os.path.join(getInputFileDir(job), namePattern)
returnable = None
for SE in trySEs:
#Check that the SE is writable
if execute('checkSEStatus("%s", "%s")' % (SE, 'Write')):
try:
returnable = new_df.put(force=True, uploadSE=SE,
lfn=new_lfn)[0]
break
except GangaDiracError as err:
raise GangaException(
"Upload of input file as LFN %s to SE %s failed" %
(new_lfn, SE))
if not returnable:
raise GangaException("Failed to upload input file to any SE")
if should_del:
os.unlink(os.path.join(localDir, namePattern))
return returnable
def _optionallyUploadLocalFile(self):
"""
"""
if self.lfn != "":
return
if self.namePattern != "" and self.lfn == "":
logger.info("I have a local DiracFile, however you're requesting it's location on the grid")
logger.info("Shall I upload it to the grid before I continue?")
decision = raw_input('[y] / n:')
while not (decision.lower() in ['y', 'n'] or decision.lower() == ''):
decision = raw_input('[y] / n:')
if decision.lower() in ['y', '']:
# upload namePattern to grid
logger.debug("Uploading the file first")
self.put()
elif decision == 'n':
logger.debug("Not uploading now")
return
else:
# do Nothing
logger.debug("Continuing without uploading file")
if self.lfn == "":
raise GangaException('Uploading of namePattern: %s failed' % self.namePattern)
if self.namePattern == "" and self.lfn == "":
raise GangaException('Cannot do anything if I don\'t have an lfn or a namePattern!')
return
def replicate(self, destSE):
"""
Replicate this file from self.locations[0] to destSE
"""
if not self.locations:
if self.lfn != '':
self.getReplicas()
else:
raise GangaException(
'Can\'t replicate a file if it isn\'t already on a DIRAC SE, upload it first'
)
if self.lfn == '':
raise GangaException('Must supply an lfn to replicate')
logger.info("Replicating file %s to %s" % (self.lfn, destSE))
stdout = execute('replicateFile("%s", "%s", "%s")' %
(self.lfn, destSE, self.locations[0]))
if isinstance(stdout, dict) and stdout.get(
'OK', False) and self.lfn in stdout.get(
'Value', {'Successful': {}})['Successful']:
self.locations.append(destSE)
self.getReplicas(forceRefresh=True)
return
logger.error("Error in replicating file '%s' : %s" %
(self.lfn, stdout))
return stdout
def doesFileExist(input_file=None, input_list=None):
if input_file is None:
input_file = 'some.file'
if input_list is None:
input_list = []
test_fileName = ''
if type(input_file) == type(''):
test_fileName = input_file
elif hasattr(input_file, 'namePattern'):
test_fileName = input_file.namePattern
else:
raise GangaException("Failed to understand file: %s" % str(input_file))
have_matched = False
for input_fileName in input_list:
check_str = ''
if type(input_fileName) == type(''):
check_str = input_fileName
elif hasattr(input_fileName, 'namePattern'):
check_str = input_fileName.namePattern
else:
raise GangaException("Failed to understand file or pattern: %s" %
str(input_fileName))
if fnmatch.fnmatch(test_fileName, check_str):
have_matched = True
break
return have_matched
def execCmd(self, cmd):
"""
This method executes a command within the namespace of the project. The cmd is placed in a bash script which is executed within the env
This will adopt the platform associated with this application.
e.g. The following will execute a 'make' command within the given project dir
app = GaudiExec('some/path')
app.execCmd('make')
Args:
cmd (str): This is the command(s) which are to be executed within the project environment and directory
"""
if not self.directory:
raise GangaException("Cannot run a command using GaudiExec without a directory first being set!")
if not path.isdir(self.directory):
raise GangaException("The given directory: '%s' doesn't exist!" % self.directory)
cmd_file = tempfile.NamedTemporaryFile(suffix='.sh', delete=False)
if not cmd.startswith('./run '):
cmd = './run ' + cmd
cmd_file.write("#!/bin/bash")
cmd_file.write("\n")
cmd_file.write(self.getEnvScript())
cmd_file.write(cmd)
cmd_file.flush()
cmd_file.close()
st = os_stat(cmd_file.name)
chmod(cmd_file.name, st.st_mode | stat.S_IEXEC)
logger.debug("Running: %s" % cmd_file.name)
# I would have preferred to execute all commands against inside `./run` so we have some sane behaviour
# but this requires a build to have been run before we can use this command reliably... so we're just going to be explicit
if not path.isfile(path.join(self.directory, 'build.%s' %self.platform, 'run')):
rc, stdout, stderr = _exec_cmd('make', self.directory)
if rc != 0:
logger.error("Failed to perform initial make on a Cmake based project")
logger.error("This is required so that the './run' target exists and is callable within the project")
logger.error("StdErr: %s" % str(stderr))
raise GangaException("Failed to execute command")
if cmd != 'make':
rc, stdout, stderr = _exec_cmd(cmd_file.name, self.directory)
else:
rc, stdout, stderr = _exec_cmd(cmd_file.name, self.directory)
if rc != 0:
logger.error("Failed to execute command: %s" % cmd_file.name)
logger.error("Tried to execute command in: %s" % self.directory)
logger.error("StdErr: %s" % str(stderr))
raise GangaException("Failed to Execute command")
unlink(cmd_file.name)
return rc, stdout, stderr
def _create_subjob(self, job, dataset):
logger.debug("_create_subjob")
datatmp = []
logger.debug("dataset size: %s" % str(len(dataset)))
#logger.debug( "dataset: %s" % str(dataset) )
from GangaLHCb.Lib.LHCbDataset.LHCbDataset import LHCbDataset
if isinstance(dataset, LHCbDataset):
for i in dataset:
if isType(i, DiracFile):
datatmp.append(i)
else:
logger.error("Unkown file-type %s, cannot perform split with file %s" % (type(i), str(i)))
from Ganga.Core.exceptions import GangaException
raise GangaException("Unkown file-type %s, cannot perform split with file %s" % (type(i), str(i)))
elif isinstance(dataset, (list, GangaList)):
for this_file in dataset:
if type(this_file) is str:
datatmp.append(allComponentFilters['gangafiles'](this_file, None))
elif isType(this_file, IGangaFile):
datatmp.append(this_file)
else:
logger.error("Unexpected type: %s" % str(type(this_file)))
logger.error("Wanted object to inherit from type: %s: %s" % (str(type(IGangaFile()))))
from Ganga.Core.exceptions import GangaException
x = GangaException("Unknown(unexpected) file object: %s" % this_file)
raise x
elif type(dataset) is str:
datatmp.append(DiracFile(lfn=dataset))
else:
logger.error("Unkown dataset type, cannot perform split here")
from Ganga.Core.exceptions import GangaException
logger.error("Dataset found: " + str(dataset))
raise GangaException("Unkown dataset type, cannot perform split here")
logger.debug("Creating new Job in Splitter")
j = Job()
logger.debug("Copying From Job")
j.copyFrom(stripProxy(job), ['splitter', 'subjobs', 'inputdata', 'inputsandbox', 'inputfiles'])
logger.debug("Unsetting Splitter")
j.splitter = None
#logger.debug("Unsetting Merger")
#j.merger = None
#j.inputsandbox = [] ## master added automatically
#j.inputfiles = []
logger.debug("Setting InputData")
j.inputdata = LHCbDataset(files=datatmp[:],
persistency=self.persistency,
depth=self.depth)
#j.inputdata.XMLCatalogueSlice = self.XMLCatalogueSlice
logger.debug("Returning new subjob")
return j
def master_sandbox_prepare(app, appmasterconfig, sharedir_roots=None):
if sharedir_roots is None:
sharedir_roots = ['']
logger.debug("RTUTils master_sandbox_prepare")
# catch errors from not preparing properly
if not hasattr(stripProxy(app), 'is_prepared') or app.is_prepared is None:
logger.warning('Application is not prepared properly')
if hasattr(stripProxy(app), 'is_prepared'):
logger.warning("app.is_prepared: %s" % str(app.is_prepared))
import traceback
traceback.print_stack()
raise GangaException(None, 'Application not prepared properly')
# Note EITHER the master inputsandbox OR the job.inputsandbox is added to
# the subjob inputsandbox depending if the jobmasterconfig object is present
# or not... Therefore combine the job.inputsandbox with appmasterconfig.
job = stripProxy(app).getJobObject()
# user added items from the interactive GPI
from Ganga.Utility.Config import getConfig
if not getConfig('Output')['ForbidLegacyInput']:
inputsandbox = job.inputsandbox[:]
else:
if len(job.inputsandbox) > 0:
from Ganga.GPIDev.Lib.Job import JobError
raise JobError(
"InputFiles have been requested but there are objects in the inputSandBox... Aborting Job Prepare!"
)
inputsandbox = []
for filepattern in getInputFilesPatterns(job)[0]:
inputsandbox.append(File(filepattern))
if len(inputsandbox) > 100:
logger.warning(
'InputSandbox exceeds maximum size (100) supported by the Dirac backend'
)
raise GangaException(None, 'InputSandbox exceed maximum size')
outputsandbox = getOutputSandboxPatterns(job) # job.outputsandbox[:]
# inputsandbox files stored in share_dir from prepare method
sharedir_handler(app, sharedir_roots, inputsandbox)
# Here add any sandbox files/data coming from the appmasterconfig
# from master_configure. Catch the case where None is passed (as in tests)
if appmasterconfig:
inputsandbox += appmasterconfig.getSandboxFiles()
outputsandbox += appmasterconfig.getOutputSandboxFiles()
return unique(inputsandbox), unique(outputsandbox)
def getReplicas(self, forceRefresh=False):
"""
Get the list of all SE where this file has a replica
This relies on an internally stored list of replicas, (SE and unless forceRefresh = True
"""
if self.lfn == '':
self._optionallyUploadLocalFile()
if self.lfn == '':
raise GangaException("Can't find replicas for file which has no LFN!")
these_replicas = None
if len(self.subfiles) != 0:
allReplicas = []
for i in self.subfiles:
allReplicas.append(i.getReplicas())
these_replicas = allReplicas
else:
# deep copy just before wer change it incase we're pointing to the
# data stored in original from a copy
if self._have_copied:
self._storedReplicas = copy.deepcopy(self._storedReplicas)
if (self._storedReplicas == {} and len(self.subfiles) == 0) or forceRefresh:
self._storedReplicas = execute('getReplicas("%s")' % self.lfn)
if self._storedReplicas.get('OK', False) is True:
try:
self._storedReplicas = self._storedReplicas['Value']['Successful']
except Exception as err:
logger.error("Unknown Error: %s" % str(err))
raise err
else:
logger.error("Couldn't find replicas for: %s" % str(self.lfn))
raise GangaException("Couldn't find replicas for: %s" % str(self.lfn))
logger.debug("getReplicas: %s" % str(self._storedReplicas))
if self.lfn in self._storedReplicas:
self._updateRemoteURLs(self._storedReplicas)
these_replicas = [self._storedReplicas[self.lfn]]
else:
these_replicas = {}
elif self._storedReplicas != {}:
these_replicas = [self._storedReplicas[self.lfn]]
return these_replicas
def getGaudiExecInputData(optsfiles, app):
'''Returns a LHCbDataSet object from a list of options files. The
optional argument extraopts will decide if the extraopts string inside
the application is considered or not.
Usage example:
# Get the data from an options file and assign it to the jobs inputdata field
j.inputdata = getGaudiExecInputData([\"~/cmtuser/DaVinci_v22r0p2/Tutorial/Analysis/options/Bs2JpsiPhi2008.py\"], j.application)
This is also called behind the scenes for j.readInputData([\"~/cmtuser/DaVinci_v22r0p2/Tutorial/Analysis/options/Bs2JpsiPhi2008.py\"])
'''
if not isinstance(optsfiles, list):
optsfiles = [optsfiles]
# use a dummy file to keep the parser happy
if len(optsfiles) == 0:
raise GangaException("Need a file to parse to call this method")
try:
parser = PythonOptsCmakeParser(optsfiles, app)
except Exception as err:
msg = 'Unable to parse the job options. Please check options files and extraopts.'
logger.error("PythonOptsCmakeParserError:\n%s" % str(err))
raise ApplicationConfigurationError(None, msg)
return parser.get_input_data()
def do_collective_operation(self, keep_going, method, *args, **kwds):
"""
"""
if not isinstance(keep_going, bool):
raise GangaException(
"The variable 'keep_going' must be a boolean. Probably you wanted to do %s(%s).%s()"
% (self.name, keep_going, method))
result = []
for _id in self.objects.keys():
obj = self.objects[_id]
try:
if isinstance(method, str):
doc = method
result.append(getattr(obj, method)(*args, **kwds))
else:
try:
doc = method.__doc__
except AttributeError:
doc = str(method)
result.append(method(obj, *args, **kwds))
except GangaException as x:
if not keep_going:
raise
except Exception as x:
logger.exception('%s %s %s: %s %s', doc, self.name, _id,
getName(x), x)
if not keep_going:
raise
return result
def asksParameter(parameter):
'''Interactive method requesting user the value of each parameter
per session (FastSim, FullSim, Analysis)'''
if parameter['customValue'] and len(parameter['values']) == 0:
value = raw_input('\nEnter %s: ' % parameter['label'])
elif not parameter['customValue'] and len(
parameter['values']) == 0:
raise GangaException(
'Invalid rule (customValue:False and values=0).')
else:
table = list()
i = 0
for value in parameter['values']:
table.append({'id': i, 'value': value})
i += 1
if parameter['customValue']:
table.append({'id': i, 'value': 'Enter a custom value'})
print('\nChoose %s:' % parameter['label'])
column_names = ('id', 'value')
print(utils.format_dict_table(table, column_names))
index = utils.getIndex(maxExclusive=len(table))
if parameter['customValue'] and index == len(table) - 1:
value = raw_input('Custom value: ')
else:
value = table[index]['value']
# parameter insertion in dictionary. It will be subsequently
#inserted into dataset analysis bookkeeping table, hstore field
new_dataset['parameters'][parameter['name']] = value
return value
def replicate(self, destSE=''):
'''Replicate all LFNs to destSE. For a list of valid SE\'s, type
ds.replicate().'''
if not destSE:
from GangaDirac.Lib.Files.DiracFile import DiracFile
DiracFile().replicate('')
return
if not self.hasLFNs():
raise GangaException('Cannot replicate dataset w/ no LFNs.')
retry_files = []
for f in self.files:
if not isDiracFile(f):
continue
try:
result = f.replicate(destSE=destSE)
except Exception as err:
msg = 'Replication error for file %s (will retry in a bit).' % f.lfn
logger.warning(msg)
logger.warning("Error: %s" % str(err))
retry_files.append(f)
for f in retry_files:
try:
result = f.replicate(destSE=destSE)
except Exception as err:
msg = '2nd replication attempt failed for file %s. (will not retry)' % f.lfn
logger.warning(msg)
logger.warning(str(err))
def enableInternalServices():
"""
activates the internal services previously disabled due to expired credentials
"""
global servicesEnabled
if servicesEnabled:
log.error("Cannot (re)enable services, they're already running")
from Ganga.Core.exceptions import GangaException
raise GangaException("Cannot (re)enable services")
# startup the registries
from Ganga.Runtime import Repository_runtime
Repository_runtime.bootstrap()
# make sure all required credentials are valid
invalid_afs = [
afsToken
for afsToken in credential_store.get_all_matching_type(AfsToken)
if not afsToken.is_valid()
]
if invalid_afs:
log.error(
'No valid AFS token was found. Please re-authorise before reactivating this session.'
)
return
log.debug("Enabling the internal services")
# re-enable the monitoring loop as it's been explicityly requested here
enableMonitoringService()
servicesEnabled = True
log.info('Internal services reactivated successfuly')
def flush(self, ignore_disk=False):
"""Flush all subjobs to disk using XML methods
Args:
ignore_disk (bool): Optional flag to force the class to ignore all on-disk data when flushing
"""
from Ganga.Core.GangaRepository.GangaRepositoryXML import safe_save
from Ganga.Core.GangaRepository.VStreamer import to_file
if ignore_disk:
range_limit = self._cachedJobs.keys()
else:
range_limit = range(len(self))
for index in range_limit:
if index in self._cachedJobs:
## If it ain't dirty skip it
if not self._cachedJobs[index]._dirty:
continue
subjob_data = self.__get_dataFile(str(index))
subjob_obj = self._cachedJobs[index]
if subjob_obj is subjob_obj._getRoot():
raise GangaException(
self, "Subjob parent not set correctly in flush.")
safe_save(subjob_data, subjob_obj, to_file)
self.write_subJobIndex(ignore_disk)
def master_resubmit(self, rjobs):
'''Resubmit the master job to the grid'''
profiler = ElapsedTimeProfiler(getLogger(name='Profile.LCG'))
profiler.start()
job = self.getJobObject()
ick = False
if not job.master and len(job.subjobs) == 0:
# case 1: master job normal resubmission
logger.debug('rjobs: %s' % str(rjobs))
logger.debug('mode: master job normal resubmission')
ick = IBackend.master_resubmit(self, rjobs)
elif job.master:
# case 2: individual subjob resubmission
logger.debug('mode: individual subjob resubmission')
ick = IBackend.master_resubmit(self, rjobs)
else:
# case 3: master job bulk resubmission
logger.debug('mode: master job resubmission')
ick = self.master_bulk_resubmit(rjobs)
if not ick:
raise GangaException('ARC bulk submission failure')
profiler.check('job re-submission elapsed time')
return ick
def GPIProxyObjectFactory(_obj):
# type: (GangaObject) -> GPIProxyObject
"""
This function _must_ be passed a raw GangaObject. Use :function:`addProxy` for a safe version
Args:
_obj (GangaObject): the object to wrap
Returns:
a proxy object
"""
from Ganga.GPIDev.Base.Objects import GangaObject
if not isType(_obj, GangaObject):
from Ganga.Core.exceptions import GangaException
raise GangaException("%s is NOT a Proxyable object" % type(_obj))
if hasattr(_obj, proxyObject):
return getattr(_obj, proxyObject)
## This is defined within Objects.py, we could probably store this elsehere
## (We probably do) but as this is guaranteed to be accessible for GangaObjects I will use this
this_class = getattr(type(_obj), proxyClass)
proxy_class = this_class(_proxy_impl_obj_to_wrap=_obj)
return proxy_class
def replicateJobFile(fileToReplicate):
"""
A method to replicate a file to a random SE.
"""
if not isinstance(fileToReplicate, DiracFile):
raise GangaDiracError(
"Can only request replicas of DiracFiles. %s is not a DiracFile" %
fileToReplicate)
if len(fileToReplicate.locations) == 0:
fileToReplicate.getReplicas()
trySEs = [
SE for SE in getConfig('DIRAC')['allDiracSE']
if SE not in fileToReplicate.locations
]
random.shuffle(trySEs)
success = None
for SE in trySEs:
if execute('checkSEStatus("%s", "%s")' % (SE, 'Write')):
try:
fileToReplicate.replicate(SE)
success = True
break
except (GangaFileError, GangaDiracError) as err:
raise err
if not success:
raise GangaException("Failed to replicate %s to any SE" %
fileToReplicate.lfn)
def sharedir_handler(app, root_dir_names, output):
share_path = get_share_path(app)
if '' in root_dir_names:
# get the '' entry and only that entry so dont waste time walking
# others before root.
root_dir_names = ['']
for share_dir in root_dir_names:
if share_dir == '':
share_dir = share_path
else:
share_dir = os.path.join(share_path, share_dir)
for root, dirs, files in os.walk(share_dir):
# [1:] removes the preceeding /
subdir = root.replace(share_dir, '')[1:]
if isType(output, (list, tuple, GangaList)):
output += [File(name=os.path.join(root, f), subdir=subdir) for f in files]
# for f in files:
## output += [File(name=os.path.join(root,f),subdir=subdir)]
elif type(output) is type(''):
for d in dirs:
if not os.path.isdir(d):
os.makedirs(d)
for f in files:
shutil.copy(os.path.join(root, f), os.path.join(output, subdir, f))
else:
raise GangaException('output must be either a list to append to or a path string to copy to')
def setOutputDataset(self, **kwargs):
'''Through this method it is possible to define the partern of output files (like *.root) and the corresponding output dataset.
To choose the desired dataset, a list of all datasets of the chosen session is printed.'''
key = raw_input('Enter a pattern (eg. *.root): ')
j = self.getJobObject()
if isinstance(j.inputdata, SBInputDataset.SBInputPersonalProduction):
if j.inputdata.session == 'FastSim':
kwargs['session'] = 'fastsim'
elif j.inputdata.session == 'FullSim':
kwargs['session'] = 'fullsim'
else:
raise GangaException(
'j.inputdata.session is \'%s\'. It must be \'FastSim\' or \'FullSim\''
% j.inputdata.session)
else:
kwargs['session'] = 'analysis'
kwargs['owner'] = utils.getOwner()
kwargs['status'] = ['open', 'prepared']
manager = SBDatasetManager.SBDatasetManager()
datasets = manager.getDataset(**kwargs)
dataset = manager.printDatasets(
datasets) # print dataset and choose one of them
self.pairs[key] = dataset['dataset_id']
def enableInternalServices():
"""
activates the internal services previously disabled due to expired credentials
"""
global servicesEnabled
if servicesEnabled:
log.error("Cannot (re)enable services, they're already running")
from Ganga.Core.exceptions import GangaException
raise GangaException("Cannot (re)enable services")
# startup the registries
from Ganga.Runtime import Repository_runtime
Repository_runtime.bootstrap()
# make sure all required credentials are valid
missing_cred = getMissingCredentials()
if missing_cred:
log.error("The following credentials are still required: %s."
"Make sure you renew them before reactivating this session" %
','.join(missing_cred))
return
log.debug("Enabling the internal services")
# re-enable the monitoring loop as it's been explicityly requested here
enableMonitoringService()
servicesEnabled = True
log.info('Internal services reactivated successfuly')
def _checkConfig(self):
"""
Check that the MassStorageFile configuration is correct
"""
if not getConfig('Output')[_getName(self)]['uploadOptions']['path']:
raise GangaException(
'Unable to create MassStorageFile. Check your configuration!')
def _dirac_check_proxy(renew=True, shouldRaise=True):
"""
This function checks the validity of the DIRAC proxy
Args:
renew (bool): When True this will require a proxy to be valid before we proceed. False means raise Exception when expired
"""
global last_modified_valid
global proxy
if proxy is None:
proxy = getCredential('GridProxy')
_isValid = proxy.isValid()
if not _isValid:
if renew is True:
proxy.renew()
if not proxy.isValid():
last_modified_valid = False
if shouldRaise:
raise GangaException(
'Can not execute DIRAC API code w/o a valid grid proxy.'
)
else:
last_modified_valid = True
else:
last_modified_valid = False
else:
last_modified_valid = True
def __init__(self, files=None, persistency=None, depth=0, fromRef=False):
super(LHCbDataset, self).__init__()
if files is None:
files = []
self.files = GangaList()
process_files = True
if fromRef:
self.files._list.extend(files)
process_files = False
elif isinstance(files, GangaList):
def isFileTest(_file):
return isinstance(_file, IGangaFile)
areFiles = all([isFileTest(f) for f in files._list])
if areFiles:
self.files._list.extend(files._list)
process_files = False
elif isinstance(files, LHCbDataset):
self.files._list.extend(files.files._list)
process_files = False
if process_files:
if isType(files, LHCbDataset):
for this_file in files:
self.files.append(deepcopy(this_file))
elif isType(files, IGangaFile):
self.files.append(deepcopy(files))
elif isType(files, (list, tuple, GangaList)):
new_list = []
for this_file in files:
if type(this_file) is str:
new_file = string_datafile_shortcut_lhcb(
this_file, None)
elif isType(this_file, IGangaFile):
new_file = stripProxy(this_file)
else:
new_file = strToDataFile(this_file)
new_list.append(new_file)
self.files.extend(new_list)
elif type(files) is str:
self.files.append(string_datafile_shortcut_lhcb(files, None),
False)
else:
raise GangaException(
"Unknown object passed to LHCbDataset constructor!")
self.files._setParent(self)
logger.debug("Processed inputs, assigning files")
# Feel free to turn this on again for debugging but it's potentially quite expensive
#logger.debug( "Creating dataset with:\n%s" % self.files )
logger.debug("Assigned files")
self.persistency = persistency
self.depth = depth
logger.debug("Dataset Created")
def get(self, localPath=''):
"""
Retrieves locally the file matching this DiracFile object pattern.
If localPath is specified
"""
if localPath == '':
to_location = self.localDir
if self.localDir is None:
#to_location = os.getcwd()
if self._parent is not None and os.path.isdir(
self.getJobObject().outputdir):
to_location = self.getJobObject().outputdir
else:
to_location = os.getcwd()
else:
to_location = localPath
self.localDir = to_location
if not os.path.isdir(to_location):
raise GangaException(
'"%s" is not a valid directory... Please set the localDir attribute to a valid directory'
% self.localDir)
if self.lfn == "":
raise GangaException('Can\'t download a file without an LFN.')
logger.info("Getting file %s" % self.lfn)
stdout = execute('getFile("%s", destDir="%s")' %
(self.lfn, to_location))
if isinstance(stdout, dict) and stdout.get(
'OK', False) and self.lfn in stdout.get(
'Value', {'Successful': {}})['Successful']:
if self.namePattern == "":
name = os.path.basename(self.lfn)
if self.compressed:
name = name[:-3]
self.namePattern = name
if self.guid == "" or not self.locations:
self.getMetadata()
return
logger.error("Error in getting file '%s' : %s" %
(self.lfn, str(stdout)))
return stdout
def strip_filename(name):
if len(name) >= 4 and name[0:4].upper() == 'PFN:':
msg = 'Can not create LogicalFile from string that begins w/ "PFN:".'\
' You probably want to create a MassStorageFile.'
raise GangaException(msg)
if len(name) >= 4 and name[0:4].upper() == 'LFN:':
name = name[4:]
return name
def _create_subjob(self, job, dataset):
logger.debug("_create_subjob")
datatmp = []
if isinstance(dataset, LHCbDataset):
for i in dataset:
if isinstance(i, DiracFile):
datatmp.extend(i)
else:
logger.error(
"Unkown file-type %s, cannot perform split with file %s"
% (type(i), str(i)))
from Ganga.Core.exceptions import GangaException
raise GangaException(
"Unkown file-type %s, cannot perform split with file %s"
% (type(i), str(i)))
elif isinstance(dataset, list):
from Ganga.GPIDev.Base.Proxy import isType
for i in dataset:
if type(i) is str:
datatmp.append(DiracFile(lfn=i))
elif isType(i, DiracFile()):
datatmp.extend(i)
else:
x = GangaException("Unknown(unexpected) file object: %s" %
i)
raise x
else:
logger.error("Unkown dataset type, cannot perform split here")
from Ganga.Core.exceptions import GangaException
raise GangaException(
"Unkown dataset type, cannot perform split here")
logger.debug("Creating new Job in Splitter")
j = Job()
j.copyFrom(stripProxy(job))
j.splitter = None
j.merger = None
j.inputsandbox = [] # master added automatically
j.inputfiles = []
j.inputdata = LHCbDataset(files=datatmp[:],
persistency=self.persistency,
depth=self.depth)
j.inputdata.XMLCatalogueSlice = self.XMLCatalogueSlice
return j
def _checkOtherFiles(self, other):
if isType(other, GangaList) or isType(other, []):
other_files = LHCbDataset(other).getFullFileNames()
elif isType(other, LHCbDataset):
other_files = other.getFullFileNames()
else:
raise GangaException("Unknown type for difference")
return other_files
def getDatasetMetadata(self):
'''Gets the dataset from the bookkeeping for current path, etc.'''
if not self.path:
return None
if not self.type in ['Path', 'RunsByDate', 'Run', 'Production']:
raise GangaException('Type="%s" is not valid.' % self.type)
if not self.type is 'RunsByDate':
if self.startDate:
msg = 'startDate not supported for type="%s".' % self.type
raise GangaException(msg)
if self.endDate:
msg = 'endDate not supported for type="%s".' % self.type
raise GangaException(msg)
if self.selection:
msg = 'selection not supported for type="%s".' % self.type
raise GangaException(msg)
cmd = "getDataset('%s','%s','%s','%s','%s','%s')" % (
self.path, self.dqflag, self.type, self.startDate, self.endDate,
self.selection)
from Ganga.GPIDev.Lib.GangaList.GangaList import GangaList
knownLists = [tuple, list, GangaList]
if isType(self.dqflag, knownLists):
cmd = "getDataset('%s',%s,'%s','%s','%s','%s')" % (
self.path, self.dqflag, self.type, self.startDate,
self.endDate, self.selection)
try:
value = get_result(
cmd,
'BK query error.',
credential_requirements=self.credential_requirements)
except GangaDiracError as err:
return {'OK': False, 'Value': str(err)}
files = []
metadata = {}
if 'LFNs' in value:
files = value['LFNs']
if not type(files) is list: # i.e. a dict of LFN:Metadata
# if 'LFNs' in files: # i.e. a dict of LFN:Metadata
metadata = files.copy()
if metadata:
return {'OK': True, 'Value': metadata}
return {'OK': False, 'Value': metadata}
def __str__(self):
if self.job_id and self.dirac_id:
return "GangaDiracError, Job %s with Dirac ID %s : %s" % (self.job_id, self.dirac_id, self.message)
else:
return GangaException.__str__(self)
def __init__(self, x): GangaException.__init__(self, x) class ISplitter(GangaObject):
def __init__(self, x=''):
GangaException.__init__(self, x)
def __init__(self, message=''):
GangaException.__init__(self, message)
self.message = message
def __init__(self, txt=''):
GangaException.__init__(self, txt)
self.txt = txt
def __init__(self,what):
GangaException.__init__(self)
self.what = what
def __init__(self, message, dirac_id = None, job_id = None):
GangaException.__init__(self, message)
self.dirac_id = dirac_id
self.job_id = job_id
def __init__(self, excpt, message):
GangaException.__init__(self, excpt, message)
self.message = message
self.excpt = excpt
