def classicSinglePython(self, filename, fast=False):
if filename not in self._fileArg.value:
raise trfExceptions.TransformReportException(
trfExit.nameToCode('TRF_INTERNAL_REPORT_ERROR'),
'Unknown file ({0}) in the file report for {1}'.format(
filename, self._fileArg))
# Direct population of some keys
fileDict = {
'lfn': filename,
'dataset': self._fileArg.dataset,
}
# Fill in the mapped 'primary' keys
for myKey, classicKey in iteritems(self._internalToGpickleMap):
fileDict[classicKey] = self._fileArg.getSingleMetadata(
fname=filename, metadataKey=myKey, populate=not fast)
if classicKey == 'checkSum' and fileDict[classicKey] == 'UNDEFINED':
# Old style is that we give back None when we don't know
fileDict[classicKey] = None
elif fileDict[classicKey] == 'UNDEFINED':
# Suppress things we don't generally expect to know
del fileDict[classicKey]
# Base 'more' stuff which is known by the argFile itself
fileDict['more'] = {'metadata': {'fileType': self._fileArg.type}}
for myKey, classicKey in iteritems(self._internalToGpickleMoreMap):
value = self._fileArg.getSingleMetadata(fname=filename,
metadataKey=myKey,
populate=not fast)
if value != 'UNDEFINED':
fileDict['more']['metadata'][classicKey] = value
return fileDict
def classicSingleEltree(self, filename, fast=False):
if filename not in self._fileArg.value:
raise trfExceptions.TransformReportException(
trfExit.nameToCode('TRF_INTERNAL_REPORT_ERROR'),
'Unknown file ({0}) in the file report for {1}'.format(
filename, self._fileArg))
tree = ElementTree.Element('File',
ID=str(
self._fileArg.getSingleMetadata(
fname=filename,
metadataKey='file_guid',
populate=not fast)))
logical = ElementTree.SubElement(tree, 'logical')
lfn = ElementTree.SubElement(logical, 'lfn', name=filename)
for myKey, classicKey in self._internalToClassicMap.iteritems():
# beam_type is tricky - we return only the first list value,
# (but remember, protect against funny stuff!)
if myKey is 'beam_type':
beamType = self._fileArg.getSingleMetadata(fname=filename,
metadataKey=myKey,
populate=not fast)
if isinstance(beamType, list):
if len(beamType) is 0:
ElementTree.SubElement(tree,
'metadata',
att_name=classicKey,
att_value='')
else:
ElementTree.SubElement(tree,
'metadata',
att_name=classicKey,
att_value=str(beamType[0]))
else:
# This is really not normal, but best we can do is str conversion
ElementTree.SubElement(tree,
'metadata',
att_name=classicKey,
att_value=str(beamType))
else:
ElementTree.SubElement(tree,
'metadata',
att_name=classicKey,
att_value=str(
self._fileArg.getSingleMetadata(
fname=filename,
metadataKey=myKey,
populate=not fast)))
# Now add the metadata which is stored at the whole argument level
ElementTree.SubElement(tree,
'metadata',
att_name='fileType',
att_value=str(self._fileArg.type))
if self._fileArg.dataset is not None:
ElementTree.SubElement(tree,
'metadata',
att_name='dataset',
att_value=self._fileArg.dataset)
return tree
def singleFilePython(self,
filename,
fast=False,
type='full',
basename=True):
if filename not in self._fileArg.value:
raise trfExceptions.TransformReportException(
trfExit.nameToCode('TRF_INTERNAL_REPORT_ERROR'),
'Unknown file ({0}) in the file report for {1}'.format(
filename, self._fileArg))
if basename:
entry = {'name': os.path.basename(filename)}
else:
entry = {'name': os.path.relpath(os.path.normpath(filename))}
if type == 'name':
# For 'name' we return only the GUID
entry.update(
self._fileArg.getMetadata(files=filename,
populate=not fast,
metadataKeys=['file_guid'
])[filename])
elif type == 'full':
# Suppress io because it's the key at a higher level and _exists because it's internal
entry.update(
self._fileArg.getMetadata(files=filename,
populate=not fast,
maskMetadataKeys=[
'io', '_exists', 'integrity',
'file_type'
])[filename])
else:
raise trfExceptions.TransformReportException(
trfExit.nameToCode('TRF_INTERNAL_REPORT_ERROR'),
'Unknown file report type ({0}) in the file report for {1}'.
format(type, self._fileArg))
return entry
def writeJSONReport(self,
filename,
sort_keys=True,
indent=2,
fast=False,
fileReport=defaultFileReport):
with open(filename, 'w') as report:
try:
if not self._dataDictionary:
self._dataDictionary = self.python(fast=fast,
fileReport=fileReport)
json.dump(self._dataDictionary,
report,
sort_keys=sort_keys,
indent=indent)
except TypeError as e:
# TypeError means we had an unserialisable object - re-raise as a trf internal
message = 'TypeError raised during JSON report output: {0!s}'.format(
e)
msg.error(message)
raise trfExceptions.TransformReportException(
trfExit.nameToCode('TRF_INTERNAL_REPORT_ERROR'), message)
def python(self, fast=False, type='full'):
# First entity contains shared properties - same for all files in this argFile
if type == 'name':
fileArgProps = {
'dataset': self._fileArg.dataset,
'nentries': self._fileArg.getnentries(fast),
'subFiles': []
}
elif type == 'full':
fileArgProps = {
'dataset': self._fileArg.dataset,
'type': self._fileArg.type,
'subFiles': [],
'argName': self._fileArg.name,
}
else:
raise trfExceptions.TransformReportException(
trfExit.nameToCode('TRF_INTERNAL_REPORT_ERROR'),
'Unknown file report type ({0}) in the file report for {1}'.
format(type, self._fileArg))
## @note We try to strip off the path when there are multiple files to be reported on,
# however we should not do this if any of the files share a basename or anything is
# in a different directory
uniqueBasenames = set(
[os.path.basename(fname) for fname in self._fileArg.value])
uniqueDirectories = set([
os.path.dirname(os.path.relpath(os.path.normpath(fname)))
for fname in self._fileArg.value
])
if len(uniqueBasenames) != len(self._fileArg.value):
msg.info(
'Detected two files with the same basename in a file argument - report for file {0} will be produced with the path as a key'
.format(self._fileArg))
basenameReport = False
elif len(uniqueDirectories) > 1:
msg.warning(
'Detected output files in different directories - report for file {0} will be produced with the path as a key'
.format(self._fileArg))
basenameReport = False
else:
basenameReport = True
suppressed = []
for fname in self._fileArg.value:
subFile = None
if basenameReport:
subFile = self.singleFilePython(fname, fast=fast, type=type)
else:
subFile = self.singleFilePython(fname,
fast=fast,
type=type,
basename=False)
if subFile is not None:
# if nentries == 0 for DRAW, suppress subfile from report
if 'nentries' in subFile and subFile[
'nentries'] == 0 and isinstance(
self._fileArg, trfArgClasses.argBSFile):
msg.info('Suppressing file {0}, nentries is 0'.format(
subFile['name']))
suppressed.append(subFile['name'])
else:
fileArgProps['subFiles'].append(subFile)
return fileArgProps