def classicSinglePython(self, filename, fast=False): if filename not in self._fileArg.value: raise trfExceptions.TransformReportException( trfExit.nameToCode('TRF_INTERNAL_REPORT_ERROR'), 'Unknown file ({0}) in the file report for {1}'.format( filename, self._fileArg)) # Direct population of some keys fileDict = { 'lfn': filename, 'dataset': self._fileArg.dataset, } # Fill in the mapped 'primary' keys for myKey, classicKey in iteritems(self._internalToGpickleMap): fileDict[classicKey] = self._fileArg.getSingleMetadata( fname=filename, metadataKey=myKey, populate=not fast) if classicKey == 'checkSum' and fileDict[classicKey] == 'UNDEFINED': # Old style is that we give back None when we don't know fileDict[classicKey] = None elif fileDict[classicKey] == 'UNDEFINED': # Suppress things we don't generally expect to know del fileDict[classicKey] # Base 'more' stuff which is known by the argFile itself fileDict['more'] = {'metadata': {'fileType': self._fileArg.type}} for myKey, classicKey in iteritems(self._internalToGpickleMoreMap): value = self._fileArg.getSingleMetadata(fname=filename, metadataKey=myKey, populate=not fast) if value != 'UNDEFINED': fileDict['more']['metadata'][classicKey] = value return fileDict
def classicSingleEltree(self, filename, fast=False): if filename not in self._fileArg.value: raise trfExceptions.TransformReportException( trfExit.nameToCode('TRF_INTERNAL_REPORT_ERROR'), 'Unknown file ({0}) in the file report for {1}'.format( filename, self._fileArg)) tree = ElementTree.Element('File', ID=str( self._fileArg.getSingleMetadata( fname=filename, metadataKey='file_guid', populate=not fast))) logical = ElementTree.SubElement(tree, 'logical') lfn = ElementTree.SubElement(logical, 'lfn', name=filename) for myKey, classicKey in self._internalToClassicMap.iteritems(): # beam_type is tricky - we return only the first list value, # (but remember, protect against funny stuff!) if myKey is 'beam_type': beamType = self._fileArg.getSingleMetadata(fname=filename, metadataKey=myKey, populate=not fast) if isinstance(beamType, list): if len(beamType) is 0: ElementTree.SubElement(tree, 'metadata', att_name=classicKey, att_value='') else: ElementTree.SubElement(tree, 'metadata', att_name=classicKey, att_value=str(beamType[0])) else: # This is really not normal, but best we can do is str conversion ElementTree.SubElement(tree, 'metadata', att_name=classicKey, att_value=str(beamType)) else: ElementTree.SubElement(tree, 'metadata', att_name=classicKey, att_value=str( self._fileArg.getSingleMetadata( fname=filename, metadataKey=myKey, populate=not fast))) # Now add the metadata which is stored at the whole argument level ElementTree.SubElement(tree, 'metadata', att_name='fileType', att_value=str(self._fileArg.type)) if self._fileArg.dataset is not None: ElementTree.SubElement(tree, 'metadata', att_name='dataset', att_value=self._fileArg.dataset) return tree
def singleFilePython(self, filename, fast=False, type='full', basename=True): if filename not in self._fileArg.value: raise trfExceptions.TransformReportException( trfExit.nameToCode('TRF_INTERNAL_REPORT_ERROR'), 'Unknown file ({0}) in the file report for {1}'.format( filename, self._fileArg)) if basename: entry = {'name': os.path.basename(filename)} else: entry = {'name': os.path.relpath(os.path.normpath(filename))} if type == 'name': # For 'name' we return only the GUID entry.update( self._fileArg.getMetadata(files=filename, populate=not fast, metadataKeys=['file_guid' ])[filename]) elif type == 'full': # Suppress io because it's the key at a higher level and _exists because it's internal entry.update( self._fileArg.getMetadata(files=filename, populate=not fast, maskMetadataKeys=[ 'io', '_exists', 'integrity', 'file_type' ])[filename]) else: raise trfExceptions.TransformReportException( trfExit.nameToCode('TRF_INTERNAL_REPORT_ERROR'), 'Unknown file report type ({0}) in the file report for {1}'. format(type, self._fileArg)) return entry
def writeJSONReport(self, filename, sort_keys=True, indent=2, fast=False, fileReport=defaultFileReport): with open(filename, 'w') as report: try: if not self._dataDictionary: self._dataDictionary = self.python(fast=fast, fileReport=fileReport) json.dump(self._dataDictionary, report, sort_keys=sort_keys, indent=indent) except TypeError as e: # TypeError means we had an unserialisable object - re-raise as a trf internal message = 'TypeError raised during JSON report output: {0!s}'.format( e) msg.error(message) raise trfExceptions.TransformReportException( trfExit.nameToCode('TRF_INTERNAL_REPORT_ERROR'), message)
def python(self, fast=False, type='full'): # First entity contains shared properties - same for all files in this argFile if type == 'name': fileArgProps = { 'dataset': self._fileArg.dataset, 'nentries': self._fileArg.getnentries(fast), 'subFiles': [] } elif type == 'full': fileArgProps = { 'dataset': self._fileArg.dataset, 'type': self._fileArg.type, 'subFiles': [], 'argName': self._fileArg.name, } else: raise trfExceptions.TransformReportException( trfExit.nameToCode('TRF_INTERNAL_REPORT_ERROR'), 'Unknown file report type ({0}) in the file report for {1}'. format(type, self._fileArg)) ## @note We try to strip off the path when there are multiple files to be reported on, # however we should not do this if any of the files share a basename or anything is # in a different directory uniqueBasenames = set( [os.path.basename(fname) for fname in self._fileArg.value]) uniqueDirectories = set([ os.path.dirname(os.path.relpath(os.path.normpath(fname))) for fname in self._fileArg.value ]) if len(uniqueBasenames) != len(self._fileArg.value): msg.info( 'Detected two files with the same basename in a file argument - report for file {0} will be produced with the path as a key' .format(self._fileArg)) basenameReport = False elif len(uniqueDirectories) > 1: msg.warning( 'Detected output files in different directories - report for file {0} will be produced with the path as a key' .format(self._fileArg)) basenameReport = False else: basenameReport = True suppressed = [] for fname in self._fileArg.value: subFile = None if basenameReport: subFile = self.singleFilePython(fname, fast=fast, type=type) else: subFile = self.singleFilePython(fname, fast=fast, type=type, basename=False) if subFile is not None: # if nentries == 0 for DRAW, suppress subfile from report if 'nentries' in subFile and subFile[ 'nentries'] == 0 and isinstance( self._fileArg, trfArgClasses.argBSFile): msg.info('Suppressing file {0}, nentries is 0'.format( subFile['name'])) suppressed.append(subFile['name']) else: fileArgProps['subFiles'].append(subFile) return fileArgProps