def add_process(self, name, func, *args, **kwargs): """Add a task to the queue, args are passed directly to subprocess.Popen""" if func is None: prettyoutput.LogErr( "Process pool add task {0} called with 'None' as function". format(name)) if callable(func) == False: prettyoutput.LogErr( "Process pool add task {0} parameter was non-callable value {1} when it should be passed a function" .format(name, func)) assert (callable(func)) # keep_alive_thread is a non-daemon thread started when the queue is non-empty. # Python will not shut down while non-daemon threads are alive. When the queue empties the thread exits. # When items are added to the queue we create a new keep_alive_thread as needed if isinstance(kwargs, dict): if not 'shell' in kwargs: kwargs['shell'] = True else: kwargs = {} kwargs['shell'] = True entry = ProcessTask(name, func, *args, **kwargs) self.tasks.put(entry) self.add_threads_if_needed() return entry
def __CreateOutputBufferForArea(Height, Width, target_space_scale=None): '''Create output images using the passed width and height ''' global use_memmap fullImage = None fullImage_shape = ( int(Height), int(Width) ) #(int(np.ceil(target_space_scale * Height)), int(np.ceil(target_space_scale * Width))) if False: #use_memmap: try: fullimage_array_path = os.path.join( tempfile.gettempdir(), 'image_%dx%d_%s.npy' % (fullImage_shape[0], fullImage_shape[1], GetProcessAndThreadUniqueString())) # print("Open %s" % (fullimage_array_path)) fullImage = np.memmap(fullimage_array_path, dtype=np.float16, mode='w+', shape=fullImage_shape) fullImage[:] = 0 except: prettyoutput.LogErr("Unable to open memory mapped file %s." % (fullimage_array_path)) raise else: fullImage = np.zeros(fullImage_shape, dtype=np.float16) fullImageZbuffer = EmptyDistanceBuffer(fullImage.shape, dtype=fullImage.dtype) return (fullImage, fullImageZbuffer)
def WritePruneMap(self, MapImageToScoreFile): if len(self.MapImageToScore) == 0: prettyoutput.LogErr('No prune scores to write to file ' + MapImageToScoreFile) return with open(MapImageToScoreFile, 'w') as outfile: if (len(list(self.MapImageToScore.keys())) == 0): if (os.path.exists(MapImageToScoreFile)): os.remove(MapImageToScoreFile) prettyoutput.Log( "No prune scores present in PruneMap being saved: " + MapImageToScoreFile) outfile.close() return for f in sorted(self.MapImageToScore.keys()): score = self.MapImageToScore[f] outfile.write(f + '\t' + str(score) + '\n') outfile.close()
def ReplaceVariable(self, xpath, iStart): '''Replace # variable names in an xpath with variable string values''' # Find the next # if it exists iStart = iStart + 1 # Skip the # symbol iEndVar = xpath[iStart + 1:].find('#') if iEndVar < 0: # OK, just check to the end of the string iEndVar = len(xpath) else: iEndVar = iStart + iEndVar + 1 while iEndVar > iStart: keyCheck = xpath[iStart:iEndVar] try: value = self.TryGetValueForKey(keyCheck) xpath = xpath[:iStart - 1] + str(value) + xpath[iEndVar:] return xpath except KeyError as e: pass iEndVar = iEndVar - 1 logger = logging.getLogger(__name__ + ".ReplaceVariable") logger.error("nornir_buildmanager XPath variable not defined.\nXPath: " + xpath) prettyoutput.LogErr("nornir_buildmanager XPath variable not defined.\nXPath: " + xpath) sys.exit()
def Execute(self, args): '''This executes the loaded pipeline on the specified volume of data. parser is an instance of the argparser class which should be extended with any pipeline specific arguments args are the parameters from the command line''' # DOM = self.PipelineData.toDOM() # PipelineElement = DOM.firstChild ArgSet = ArgumentSet() PipelineElement = self.PipelineData prettyoutput.Log("Adding pipeline arguments") # parser = self.GetArgParser(parser) # (args, unused) = parser.parse_known_args(passedArgs) ArgSet.AddArguments(args) ArgSet.AddParameters(PipelineElement) # Load the Volume.XML file in the output directory self.VolumeTree = VolumeManagerETree.VolumeManager.Load(args.volumepath, Create=True) if(self.VolumeTree is None): PipelineManager.logger.critical("Could not load or create volume.xml " + args.outputpath) prettyoutput.LogErr("Could not load or create volume.xml " + args.outputpath) sys.exit() # dargs = copy.deepcopy(defaultDargs) self.ExecuteChildPipelines(ArgSet, self.VolumeTree, PipelineElement) nornir_pools.WaitOnAllPools()
def ParseTransform(TransformNode, OutputSectionNode): mFile = mosaicfile.MosaicFile.Load(TransformNode.FullPath) if (mFile is None): prettyoutput.LogErr("Unable to load transform: " + TransformNode.FullPath) return if (mFile.NumberOfImages < 1): prettyoutput.LogErr("Not including empty .mosaic file") return # Figure out what the tile prefix and postfix are for this mosaic file by extrapolating from the first tile filename for k in list(mFile.ImageToTransformString.keys()): TileFileName = k break # TileFileName = mFile.ImageToTransformString.keys()[0] # Figure out prefix and postfix parts of filenames parts = TileFileName.split('.') Postfix = parts[len(parts) - 1] # Two conventions are commonly used Section#.Tile#.png or Tile#.png if (len(parts) == 3): Prefix = parts[0] else: Prefix = '' UseForVolume = 'false' if ('grid' in TransformNode.Name.lower()): UseForVolume = 'true' #Viking needs the transform names to be consistent, and if transforms are built with different spacings, for TEM and CMP, Viking can't display #So we simplify the transform name TransformName = TransformNode.Name return ETree.SubElement( OutputSectionNode, 'Transform', { 'FilePostfix': Postfix, 'FilePrefix': Prefix, 'Path': TransformNode.Path, 'Name': TransformName, 'UseForVolume': UseForVolume })
def _CheckPipelineXMLExists(cls, PipelineXmlFile): if not os.path.exists(PipelineXmlFile): PipelineManager.logger.critical("Provided pipeline filename does not exist: " + PipelineXmlFile) prettyoutput.LogErr("Provided pipeline filename does not exist: " + PipelineXmlFile) sys.exit() return False return True
def XPathIterator(XPath): '''A Very limited iterator which takes xpath strings as input and iterates over each subpath. Returns an object with the following properties: The following XPath syntax elements are supported: / .. . [] @ = ''' # Get the name of the child node we are looking for PathParts = XPath.split('/') pat = re.compile( r""" (?P<Path>[^\[/\]]+) (\[ (?P<IsAttribute>@)? (?P<Name>[^=\]]+) ((?P<Operator>[=<>]) (?P<Value>[^\]]+))? \])? """, re.VERBOSE) # prettyoutput.Log(XPath); for subpath in PathParts: StartFromRoot = len(subpath) == 0 if (StartFromRoot): continue # prettyoutput.Log("\n" + subpath); matches = pat.match(subpath) if matches is None: prettyoutput.LogErr("Error in xpath subpath: " + subpath) sys.exit() # Figure out if Value is a string (Starts with quotes) Obj = XSubPath() Obj.RawPath = subpath for item in list(matches.groupdict().items()): Obj.__dict__[item[0]] = item[1] Obj.IsAttribute = not matches.group('IsAttribute') is None if (Obj.Value is not None): if Obj.Value[0] == "'" and Obj.Value[-1] == "'" and len( Obj.Value) >= 3: Obj.Value = Obj.Value[1:-2] elif Obj.Value[0] == '"' and Obj.Value[-1] == '"' and len( Obj.Value) >= 3: Obj.Value = Obj.Value[1:-2] else: try: Obj.Value = float(Obj.Value) except: pass yield Obj
def WritePruneMosaic(self, path, SourceMosaic, TargetMosaic='prune.mosaic', Tolerance='5'): ''' Remove tiles from the source mosaic with scores less than Tolerance and write the new mosaic to TargetMosaic. Raises a key error if image in prude scores does not exist in .mosaic file Raises a value error if the threshold removes all tiles in the mosaic. :return: Number of tiles removed ''' if (not isinstance(Tolerance, float)): Tolerance = float(Tolerance) SourceMosaicFullPath = os.path.join(path, SourceMosaic) TargetMosaicFullPath = os.path.join(path, TargetMosaic) mosaic = mosaicfile.MosaicFile.Load(SourceMosaicFullPath) # We copy this because if an input image is missing there will not be a prune score and it should be removed from the .mosaic file inputImageToTransforms = copy.deepcopy(mosaic.ImageToTransformString) mosaic.ImageToTransformString.clear() numRemoved = 0 for item in list(self.MapImageToScore.items()): filename = item[0] score = item[1] if (score >= Tolerance): keyVal = filename if not keyVal in inputImageToTransforms: keyVal = os.path.basename(filename) if not keyVal in inputImageToTransforms: raise KeyError( "PruneObj: Cannot locate image file in .mosaic " + keyVal) mosaic.ImageToTransformString[keyVal] = inputImageToTransforms[ keyVal] else: numRemoved = numRemoved + 1 if (len(mosaic.ImageToTransformString) <= 0): errMsg = "All tiles removed when using threshold = " + str( Tolerance) + "\nThe prune request was ignored" prettyoutput.LogErr(errMsg) raise ValueError(errMsg) else: prettyoutput.Log("Removed " + str(numRemoved) + " tiles pruning mosaic " + TargetMosaic) mosaic.Save(TargetMosaicFullPath) return numRemoved
def _LoadImageByExtension(ImageFullPath, dtype): ''' Loads an image file and returns an ndarray of dtype :param dtype dtype: Numpy datatype of returned array. If the type is a float then the returned array is in the range 0 to 1. Otherwise it is whatever pillow and numpy decide. ''' (root, ext) = os.path.splitext(ImageFullPath) image = None try: if ext == '.npy': image = np.load(ImageFullPath, 'c').astype(dtype) else: #image = plt.imread(ImageFullPath) with Image.open(ImageFullPath) as im: expected_dtype = pillow_helpers.dtype_for_pillow_image(im) image = np.array(im, dtype=expected_dtype) max_pixel_val = nornir_imageregistration.ImageMaxPixelValue(image) if dtype is not None: image = image.astype(dtype) # else: # #Reduce to smallest integer type that can hold the data # if im.mode[0] == 'I' and (np.issubdtype(image.dtype, np.int32) or np.issubdtype(image.dtype, np.uint32)): # (min_val, max_val) = im.getextrema() # smallest_dtype = np.uint32 # if max_val <= 65535: # smallest_dtype = np.uint16 # if max_val <= 255: # smallest_dtype = np.uint8 # # image = image.astype(smallest_dtype) # # dtype = image.dtype #Ensure data is in the range 0 to 1 for floating types if nornir_imageregistration.IsFloatArray(dtype): if im.mode[0] == 'F': (_, im_max_val) = im.getextrema() if im_max_val <= 1.0: return image max_val = max_pixel_val if max_val > 0: image = image / max_val im.close() except IOError as E: prettyoutput.LogErr("Unable to load image {0}".format(ImageFullPath)) raise E return image
def Save(self, filename): if (len(self.ImageToTransformString) <= 0): prettyoutput.LogErr("No tiles present in mosaic " + filename + "\nThe save request was aborted") return fMosaic = open(filename, 'w') fMosaic.write(self.MosaicStr()) fMosaic.close()
def SaveImage(ImageFullPath, image, bpp=None, **kwargs): '''Saves the image as greyscale with no contrast-stretching :param str ImageFullPath: The filename to save :param ndarray image: The image data to save :param int bpp: The bit depth to save, if the image data bpp is higher than this value it will be reduced. Otherwise only the bpp required to preserve the image data will be used. (8-bit data will not be upsampled to 16-bit) ''' dirname = os.path.dirname(ImageFullPath) if dirname is not None and len(dirname) > 0: os.makedirs(dirname, exist_ok=True) if bpp is None: bpp = nornir_imageregistration.ImageBpp(image) if bpp > 16: prettyoutput.LogErr("Saving image at 32 bits-per-pixel, check SaveImageParameters for efficiency:\n{0}".format(ImageFullPath)) if bpp > 8: #Ensure we even have the data to bother saving a higher bit depth detected_bpp = nornir_imageregistration.ImageBpp(image) if detected_bpp < bpp: bpp = detected_bpp (root, ext) = os.path.splitext(ImageFullPath) if ext == '.jp2': SaveImage_JPeg2000(ImageFullPath, image, **kwargs) elif ext == '.npy': np.save(ImageFullPath, image) else: if image.dtype == np.bool or bpp == 1: #Covers for pillow bug with bit images #https://stackoverflow.com/questions/50134468/convert-boolean-numpy-array-to-pillow-image #im = Image.fromarray(image.astype(np.uint8) * 255, mode='L').convert('1') im = OneBit_img_from_bool_array(image) elif bpp == 8: Uint8_image = _Image_To_Uint8(image) del image im = Image.fromarray(Uint8_image, mode="L") elif nornir_imageregistration.IsFloatArray(image): #TODO: I believe Pillow-SIMD finally added the ability to save I;16 for 16bpp PNG images if image.dtype == np.float16: image = image.astype(np.float32) im = Image.fromarray(image * ((1 << bpp)-1)) im = im.convert('I') else: if bpp < 32: if ext.lower() == '.png': im = uint16_img_from_uint16_array(image) else: im = Image.fromarray(image, mode="I;{0}".format(bpp)) else: im = Image.fromarray(image, mode="I".format(bpp)) im.save(ImageFullPath, **kwargs) return
def add_task(self, name, func, *args, **kwargs): """Add a task to the queue""" if func is None: prettyoutput.LogErr( "Multiprocess pool add task {0} called with 'None' as function" .format(name)) if callable(func) == False: prettyoutput.LogErr( "Multiprocess pool add task {0} parameter was non-callable value {1} when it should be passed a function" .format(name, func)) assert (callable(func)) # I've seen an issue here were apply_async prints an exception about not being able to import a module. It then swallows the exception. # The returned task seems valid and not complete, but the MultiprocessThreadTask's event is never set because the callback isn't used. # This hangs the caller if they wait on the task. retval_task = MultiprocessThreadTask(name, None, args, kwargs) retval_task.asyncresult = self.tasks.apply_async( func, args, kwargs, callback=self.callback_wrapper(retval_task.task_id, retval_task.callback), error_callback=self.callback_wrapper( retval_task.task_id, retval_task.callbackontaskfail)) if retval_task.asyncresult is None: raise ValueError( "apply_async returned None instead of an asyncresult object") retval_task.asyncresult._nornir_task_id_ = retval_task.task_id self._active_tasks[retval_task.task_id] = retval_task #print("Added task #{0}".format(retval_task.task_id)) self.TryReportActiveTaskCount() return retval_task
def add_task(self, name, func, *args, **kwargs): if func is None: prettyoutput.LogErr( "Thread pool add task {0} called with 'None' as function". format(name)) if callable(func) == False: prettyoutput.LogErr( "Thread pool add task {0} parameter was non-callable value {1} when it should be passed a function" .format(name, func)) assert (callable(func)) """Add a task to the queue""" assert (False == self.shutdown_event.isSet()) # keep_alive_thread is a non-daemon thread started when the queue is non-empty. # Python will not shut down while non-daemon threads are alive. When the queue empties the thread exits. # When items are added to the queue we create a new keep_alive_thread as needed entry = ThreadTask(name, func, *args, **kwargs) self.tasks.put(entry) self.add_threads_if_needed() return entry
def GetInfo(cls, filename): '''Parses a filename and returns what we have learned if it follows the convention <SECTION>_<CHANNEL>_<MosaicSource>_<DOWNSAMPLE>.mosaic [SectionNumber, Channel, MosaicSource, Downsample]''' if (os.path.exists(filename) == False): prettyoutput.LogErr("mosaic file not found: " + filename) return baseName = os.path.basename(filename) [baseName, ext] = os.path.splitext(baseName) parts = baseName.split("_") try: SectionNumber = int(parts[0]) except: # We really can't recover from this, so maybe an exception should be thrown instead SectionNumber = None prettyoutput.Log('Could not determine mosaic section number: ' + str(filename)) try: MosaicType = parts[-2] except: MosaicType = None prettyoutput.Log('Could not determine mosaic source: ' + str(filename)) if (len(parts) > 3): try: Channel = parts[1] except: Channel = None prettyoutput.Log('Could not determine mosaic channel: ' + str(filename)) # If we don't have a valid downsample value we assume 1 try: DownsampleStrings = parts[-1].split(".") Downsample = int(DownsampleStrings[0]) except: Downsample = None prettyoutput.Log('Could not determine mosaic downsample: ' + str(filename)) return [SectionNumber, Channel, MosaicType, Downsample]
def EmailIndex(path=None, subject=None, **kwargs): import nornir_shared.emaillib if (path is None): VolumeNode = kwargs.get('ReportingElement', None) if VolumeNode is None: PrettyOutput.LogErr("Path attribute not found for VolumeFinder") return while not VolumeNode.Parent is None: VolumeNode = VolumeNode.Parent path = VolumeNode.attrib['Path'] ServerHostname = FindServerFromAboutXML(path) if ServerHostname is None: ServerHostname = "" VolumeName = None VolumeXMLPath = os.path.join(path, 'Volume.xml') if os.path.exists(VolumeXMLPath): VolumeXML = ElementTree.parse(VolumeXMLPath) VolumeRoot = VolumeXML.getroot() if 'Name' in VolumeRoot.attrib: VolumeName = VolumeRoot.attrib['Name'] # Find all of the .html in the root directory reports = glob.glob(os.path.join(path, '*.html')) if not VolumeName is None: if not subject is None: subject = VolumeName + ": " + subject kwargs['subject'] = subject message = 'The following reports are available for this volume\n\n' for report in reports: RelativeReportPath = report[len(path) + 1:] reportURL = ServerHostname + '/' + RelativeReportPath message = message + "\n\t" + reportURL kwargs['message'] = message nornir_shared.emaillib.SendMail(**kwargs)
def Load(cls, filename): if (os.path.exists(filename) == False): prettyoutput.LogErr("Mosaic file not found: " + filename) return lines = [] with open(filename, 'r') as fMosaic: lines = fMosaic.readlines() fMosaic.close() obj = MosaicFile() iLine = 0 while iLine < len(lines): line = lines[iLine] line = line.strip() [text, value] = line.split(':', 1) if (text.startswith('number_of_images')): value.strip() obj.FileReportedNumberOfImages = int(value) elif (text.startswith('pixel_spacing')): value.strip() obj.pixel_spacing = int(float(value)) elif (text.startswith('use_std_mask')): obj.use_std_mask = int(value) elif (text.startswith('format_version_number')): obj.format_version_number = int(value) elif (text.startswith('image')): if (obj.format_version_number == 0): [filename, transform] = value.split(None, 1) filename = filename.strip() filename = os.path.basename(filename) transform = transform.strip() obj.ImageToTransformString[filename] = transform else: filename = os.path.basename(lines[iLine + 1].strip()) transform = lines[iLine + 2].strip() obj.ImageToTransformString[filename] = transform iLine = iLine + 2 iLine = iLine + 1 return obj
def _AddArgumentNodeToParser(parser, argNode): '''Returns a dictionary that can be added to a parser''' attribDictCopy = copy.copy(argNode.attrib) Flag = "" for key in attribDictCopy: val = attribDictCopy[key] # Starts as a string, try to convert to bool, int, or float if key == 'flag': Flag = nornir_shared.misc.ListFromDelimited(val) continue elif key == 'type': if val in __builtins__: val = __builtins__[val] elif val in globals(): val = globals()[val] else: logger = logging.getLogger(__name__ + "._AddArgumentNodeToParser") logger.error('Type not found in __builtins__ or module __dict__' + val) prettyoutput.LogErr('Type not found in __builtins__ or module __dict__ ' + val) raise Exception(message="%s type specified by argument node is not present in __builtins__ or module dictionary. Must use a standard python type." % val) continue attribDictCopy[key] = val elif key == 'default': attribDictCopy[key] = _ConvertValueToPythonType(val) elif key == 'required': attribDictCopy[key] = _ConvertValueToPythonType(val) elif key == 'choices': listOfChoices = nornir_shared.misc.ListFromDelimited(val) if len(listOfChoices) < 2: raise Exception(message="Flag %s does not specify multiple choices. Must use a comma delimited list to provide multiple choice options.\nCurrent choice string is: %s" % (attribDictCopy['flag'], val)) attribDictCopy[key] = listOfChoices if 'flag' in attribDictCopy: del attribDictCopy['flag'] parser.add_argument(*Flag, **attribDictCopy)
def Load(cls, PipelineXml, PipelineName=None): # PipelineData = Pipelines.CreateFromDOM(XMLDoc) SelectedPipeline = None XMLDoc = cls.LoadPipelineXML(PipelineXml) if PipelineName is None: PipelineManager.logger.warning("No pipeline name specified.") prettyoutput.Log("No pipeline name specified") cls.PrintPipelineEnumeration(XMLDoc) return None else: SelectedPipeline = XMLDoc.find("Pipeline[@Name='" + PipelineName + "']") if(SelectedPipeline is None): PipelineManager.logger.critical("No pipeline found named " + PipelineName) prettyoutput.LogErr("No pipeline found named " + PipelineName) cls.PrintPipelineEnumeration(XMLDoc) return None return PipelineManager(pipelinesRoot=XMLDoc.getroot(), pipelineData=SelectedPipeline)
def Load(filename): if not os.path.exists(filename): PrettyOutput.Log("Stos file not found: " + filename) return None obj = StosFile() try: [ obj.MappedSectionNumber, obj.ControlSectionNumber, Channels, Filters, obj.StosSource, obj._Downsample ] = StosFile.GetInfo(filename) except: pass with open(filename, 'r') as fMosaic: lines = fMosaic.readlines() if len(lines) < 7: PrettyOutput.LogErr("%s is not a valid stos file" % (filename)) raise ValueError("%s is not a valid stos file" % (filename)) obj.ControlImageFullPath = lines[0].strip() obj.MappedImageFullPath = lines[1].strip() ControlDims = lines[4].split() MappedDims = lines[5].split() obj.ControlImageDim = [float(x) for x in ControlDims] obj.MappedImageDim = [float(x) for x in MappedDims] obj.Transform = lines[6].strip() if len(lines) > 8: obj.ControlMaskFullPath = lines[8] obj.MappedMaskFullPath = lines[9] return obj
def __InvokeFunctionOnImageList__(listfilenames, Function=None, Pool=None, **kwargs): '''Return a number indicating how interesting the image is using SciPy ''' if Pool is None: TPool = nornir_pools.GetGlobalMultithreadingPool() else: TPool = Pool TileToScore = dict() tasklist = [] for filename in listfilenames: task = TPool.add_task('Calc Feature Score: ' + os.path.basename(filename), Function, filename, **kwargs) task.filename = filename tasklist.append(task) TPool.wait_completion() numTasks = len(tasklist) iTask = 0 for task in tasklist: Result = task.wait_return() iTask = iTask + 1 if Result is None: PrettyOutput.LogErr('No return value for ' + task.filename) continue # if Result[0] is None: # PrettyOutput.LogErr('No filename for ' + task.name) # continue PrettyOutput.CurseProgress("ImageStats", iTask, numTasks) filename = task.filename TileToScore[filename] = Result return TileToScore
def CalculatePruneScores(cls, Parameters, FilterNode, Downsample, TransformNode, OutputFile=None, Logger=None, **kwargs): '''@FilterNode Calculate the prune scores for a filter and level''' # VolumeManager.NodeManager.GetParent(Entry.NodeList) # Check for an existing prune map Overlap = float(Parameters.get('Overlap', 0.1)) if OutputFile is None: OutputFile = 'PruneScores' assert (isinstance(Downsample, int) or isinstance(Downsample, float)) [created, LevelNode] = FilterNode.TilePyramid.GetOrCreateLevel(Downsample) if (LevelNode is None): prettyoutput.LogErr( "Missing InputPyramidLevelNode attribute on PruneTiles") Logger.error( "Missing InputPyramidLevelNode attribute on PruneTiles") return if (TransformNode is None): prettyoutput.LogErr( "Missing TransformNode attribute on PruneTiles") Logger.error("Missing TransformNode attribute on PruneTiles") return FilterNode = LevelNode.FindParent("Filter") # Record the downsample level the values are calculated at: Parameters['Level'] = str(LevelNode.Downsample) Parameters['Filter'] = FilterNode.Name MangledName = nornir_shared.misc.GenNameFromDict( Parameters) + '_' + TransformNode.Type OutputFile = OutputFile + MangledName + '.txt' SaveRequired = False PruneMapElement = FilterNode.GetChildByAttrib('Prune', 'Overlap', Overlap) PruneMapElement = transforms.RemoveOnMismatch( PruneMapElement, 'InputTransformChecksum', TransformNode.Checksum) if not PruneMapElement is None: if hasattr(LevelNode, 'TilesValidated'): PruneMapElement = transforms.RemoveOnMismatch( PruneMapElement, 'NumImages', LevelNode.TilesValidated) if PruneMapElement is None: PruneMapElement = VolumeManagerETree.PruneNode.Create( Overlap=Overlap, Type=MangledName) [SaveRequired, PruneMapElement ] = FilterNode.UpdateOrAddChildByAttrib(PruneMapElement, 'Overlap') else: # If meta-data and the data file exist, nothing to do if os.path.exists(PruneMapElement.DataFullPath): return # Create file holders for the .xml and .png files PruneDataNode = VolumeManagerETree.DataNode.Create(OutputFile) [added, PruneDataNode] = PruneMapElement.UpdateOrAddChild(PruneDataNode) FullTilePath = LevelNode.FullPath TransformObj = mosaicfile.MosaicFile.Load(TransformNode.FullPath) TransformObj.RemoveInvalidMosaicImages(FullTilePath) files = [] for f in list(TransformObj.ImageToTransformString.keys()): files.append(os.path.join(FullTilePath, f)) TileToScore = Prune(files, Overlap) prune = PruneObj(TileToScore) prune.WritePruneMap(PruneDataNode.FullPath) PruneMapElement.InputTransformChecksum = TransformNode.Checksum PruneMapElement.NumImages = len(TileToScore) return FilterNode
def ConvertImagesInDict(ImagesToConvertDict, Flip=False, Flop=False, InputBpp=None, OutputBpp=None, Invert=False, bDeleteOriginal=False, RightLeftShift=None, AndValue=None, MinMax=None, Gamma=None): ''' The key and value in the dictionary have the full path of an image to convert. MinMax is a tuple [Min,Max] passed to the -level parameter if it is not None RightLeftShift is a tuple containing a right then left then return to center shift which should be done to remove useless bits from the data I do not use an and because I do not calculate ImageMagick's quantum size yet. Every image must share the same colorspace :return: True if images were converted :rtype: bool ''' if len(ImagesToConvertDict) == 0: return False if InputBpp is None: for k in ImagesToConvertDict.keys(): if os.path.exists(k): InputBpp = nornir_shared.images.GetImageBpp(k) break prettyoutput.CurseString('Stage', "ConvertImagesInDict") if MinMax is not None: if(MinMax[0] > MinMax[1]): raise ValueError("Invalid MinMax parameter passed to ConvertImagesInDict") pool = nornir_pools.GetMultithreadingPool("ConvertImagesInDict", num_threads=multiprocessing.cpu_count() * 2) #pool = nornir_pools.GetGlobalSerialPool() tasks = [] for (input_image, output_image) in ImagesToConvertDict.items(): task = pool.add_task("{0} -> {1}".format(input_image, output_image), _ConvertSingleImageToFile, input_image_param=input_image, output_filename=output_image, Flip=Flip, Flop=Flop, InputBpp=InputBpp, OutputBpp=OutputBpp, Invert=Invert, MinMax=MinMax, Gamma=Gamma) tasks.append(task) while len(tasks) > 0: t = tasks.pop(0) try: t.wait() except Exception as e: prettyoutput.LogErr("Failed to convert " + t.name) if bDeleteOriginal: for (input_image, output_image) in ImagesToConvertDict.items(): if input_image != output_image: pool.add_task("Delete {0}".format(input_image), os.remove, input_image) while len(tasks) > 0: t = tasks.pop(0) try: t.wait() except OSError as e: prettyoutput.LogErr("Unable to delete {0}\n{1}".format(t.name, e)) pass except IOError as e: prettyoutput.LogErr("Unable to delete {0}\n{1}".format(t.name, e)) pass if not pool is None: pool.wait_completion() pool.shutdown() pool = None del tasks
def PruneMosaic(cls, Parameters, PruneNode, TransformNode, OutputTransformName=None, Logger=None, **kwargs): '''@ChannelNode Uses a PruneData node to prune the specified mosaic file''' threshold_precision = VolumeManagerETree.TransformNode.get_threshold_precision( ) #Number of digits to save in XML file if (Logger is None): Logger = logging.getLogger(__name__ + '.PruneMosaic') Threshold = cls._GetThreshold(PruneNode, Parameters.get('Threshold', None)) if not Threshold is None: Threshold = TransformNode.round_precision_value( Threshold) #round(Threshold, threshold_precision) cls._TryUpdateUndefinedThresholdFromParameter(PruneNode, Threshold) if OutputTransformName is None: OutputTransformName = 'Prune' InputTransformNode = TransformNode TransformParent = InputTransformNode.Parent OutputMosaicName = OutputTransformName + nornir_shared.misc.GenNameFromDict( Parameters) + '.mosaic' MangledName = nornir_shared.misc.GenNameFromDict(Parameters) HistogramXMLFile = PruneObj.HistogramXMLFileTemplate % PruneNode.Type HistogramImageFile = PruneObj.HistogramSVGFileTemplate % PruneNode.Type MosaicDir = os.path.dirname(InputTransformNode.FullPath) OutputMosaicFullPath = os.path.join(MosaicDir, OutputMosaicName) # Check if there is an existing prune map, and if it exists if it is out of date PruneNodeParent = PruneNode.Parent transforms.RemoveWhere( TransformParent, 'Transform[@Name="' + OutputTransformName + '"]', lambda t: (t.Threshold != Threshold) or (t.Type != MangledName)) '''TODO: Add function to remove duplicate Prune Transforms with different thresholds''' TransformParent.RemoveOldChildrenByAttrib('Transform', 'Name', OutputTransformName) PruneDataNode = PruneNode.find('Data') if (PruneDataNode is None): Logger.warning("Did not find expected prune data node") return None OutputTransformNode = transforms.LoadOrCleanExistingTransformForInputTransform( channel_node=TransformParent, InputTransformNode=InputTransformNode, OutputTransformPath=OutputMosaicName) if not OutputTransformNode is None: if OutputTransformNode.Locked: Logger.info("Skipping locked transform %s" % OutputTransformNode.FullPath) return None OutputTransformNode = transforms.RemoveOnMismatch( OutputTransformNode, 'InputPruneDataChecksum', PruneDataNode.Checksum) OutputTransformNode = transforms.RemoveOnMismatch( OutputTransformNode, 'Threshold', Threshold, Precision=threshold_precision) # Add the Prune Transform node if it is missing if OutputTransformNode is None: OutputTransformNode = VolumeManagerETree.TransformNode.Create( Name=OutputTransformName, Type=MangledName, InputTransformChecksum=InputTransformNode.Checksum) TransformParent.append(OutputTransformNode) elif os.path.exists(OutputTransformNode.FullPath): # The meta-data and output exist, do nothing return None OutputTransformNode.SetTransform(InputTransformNode) OutputTransformNode.InputPruneDataType = PruneNode.Type OutputTransformNode.attrib[ 'InputPruneDataChecksum'] = PruneDataNode.Checksum if not Threshold is None: OutputTransformNode.Threshold = Threshold PruneObjInstance = cls.ReadPruneMap(PruneDataNode.FullPath) PruneObjInstance.Tolerance = Threshold assert (not PruneObjInstance is None) PruneObjInstance.HistogramXMLFileFullPath = os.path.join( PruneNodeParent.FullPath, HistogramXMLFile) PruneObjInstance.HistogramImageFileFullPath = os.path.join( PruneNodeParent.FullPath, HistogramImageFile) try: RemoveOutdatedFile(PruneDataNode.FullPath, PruneObjInstance.HistogramImageFileFullPath) RemoveOutdatedFile(PruneDataNode.FullPath, PruneObjInstance.HistogramXMLFileFullPath) HistogramImageNode = PruneNode.find('Image') if not HistogramImageNode is None: HistogramImageNode = transforms.RemoveOnMismatch( HistogramImageNode, 'Threshold', Threshold, Precision=threshold_precision) if HistogramImageNode is None or not os.path.exists( PruneObjInstance.HistogramImageFileFullPath): HistogramImageNode = VolumeManagerETree.ImageNode.Create( HistogramImageFile) (added, HistogramImageNode ) = PruneNode.UpdateOrAddChild(HistogramImageNode) if not added: #Handle the case where the path is different, such as when we change the extension type if os.path.exists(HistogramImageNode.FullPath): os.remove(HistogramImageNode.FullPath) HistogramImageNode.Path = HistogramImageFile HistogramImageNode.Threshold = Threshold PruneObjInstance.CreateHistogram( PruneObjInstance.HistogramXMLFileFullPath) assert (HistogramImageNode.FullPath == PruneObjInstance.HistogramImageFileFullPath) #if Async: #pool = nornir_pools.GetMultithreadingPool("Histograms") #pool.add_task("Create Histogram %s" % HistogramImageFile, plot.Histogram, HistogramXMLFile, HistogramImageFile, LinePosList=self.Tolerance, Title=Title) #else: plot.Histogram(PruneObjInstance.HistogramXMLFileFullPath, HistogramImageNode.FullPath, LinePosList=PruneObjInstance.Tolerance, Title="Threshold " + str(Threshold)) print("Done!") except Exception as E: prettyoutput.LogErr("Exception creating prunemap histogram:" + str(E.message)) pass if (OutputTransformNode is None): if (not hasattr(OutputTransformNode, Threshold)): OutputTransformNode.Threshold = Threshold if (OutputTransformNode.Threshold != Threshold): if (os.path.exists(OutputMosaicFullPath)): os.remove(OutputMosaicFullPath) if not os.path.exists(OutputMosaicFullPath): try: PruneObjInstance.WritePruneMosaic(PruneNodeParent.FullPath, InputTransformNode.FullPath, OutputMosaicFullPath, Tolerance=Threshold) except (KeyError, ValueError): os.remove(PruneDataNode.FullPath) PruneNode.remove(PruneDataNode) prettyoutput.LogErr("Remove prune data for section " + PruneDataNode.FullPath) return PruneNodeParent OutputTransformNode.Type = MangledName OutputTransformNode.Name = OutputTransformName # Setting this value automatically converts the double to a string using the %g formatter. This is a precision of two. The RemoveOnMismatch value needs to use a matching precision OutputTransformNode.Threshold = Threshold OutputTransformNode.ResetChecksum() # OutputTransformNode.Checksum = mosaicfile.MosaicFile.LoadChecksum(OutputTransformNode.FullPath) return [TransformParent, PruneNodeParent]
def VolumeFinder(path=None, OutputFile=None, VolumeNode=None, requiredFiles=None, **kwargs): '''Expects a 'Path' and 'RequiredFiles' keyword argument' produces an HTML index of all volumes under the path''' if requiredFiles is None: requiredFiles = [] if (path is None): VolumeNode = kwargs.get('VolumeNode', None) if VolumeNode is None: PrettyOutput.LogErr("Path attribute not found for VolumeFinder") return else: path = VolumeNode.attrib['Path'] requiredFiles = kwargs.get('RequiredFiles', []) if not isinstance(requiredFiles, list): RequiredFileStrs = str(requiredFiles).strip().split(',') requiredFiles = list() for fileStr in RequiredFileStrs: requiredFiles.append(fileStr) ServerHostname = FindServerFromAboutXML(path) if ServerHostname is None: ServerHostname = "" dirs = nornir_shared.files.RecurseSubdirectories( path, RequiredFiles=requiredFiles, ExcludeNames="") HTMLHeader = "<!DOCTYPE html> \n" + "<html>\n " + "<body>\n" HTMLFooter = "</body>\n" + "</html>\n" HTMLString = HTMLHeader dirDict = {} # Make sure required files is a list type if a single string was passed if isinstance(requiredFiles, str): requiredFiles = [requiredFiles] # Seperate out specifically named files from regular expressions RegExpPatterns = [] i = 0 while i < len(requiredFiles): filename = requiredFiles[i] if '*' in filename or '?' in filename: RegExpPatterns.append(filename) requiredFiles.pop(i) else: i = i + 1 for directory in dirs: # Take the path from the list and find files. dirImageList = dirDict.get(directory, []) for pattern in RegExpPatterns: filesMatchingPattern = glob.glob(os.path.join(directory, pattern)) for filename in filesMatchingPattern: filenameFullPath = os.path.join(directory, filename) dirImageList.append( filenameFullPath) # Should check if it exists maybe for filename in requiredFiles: filenameFullPath = os.path.join(directory, filename) if (os.path.exists(filenameFullPath)): dirImageList.append( filenameFullPath) # Should check if it exists maybe dirDict[directory] = dirImageList # Remove the path we were passed from the strings HTMLString = HTMLString.replace(path, '') RowNames = list(dirDict.keys()) RowNames.sort() RowNames.reverse() HTMLTable = HTMLTableForImageList(path, dirDict, RowNames, **kwargs) HTMLString = HTMLString + HTMLTable + '\n' HTMLString = HTMLString + HTMLFooter if (len(RowNames) == 0): PrettyOutput.Log("Report generator could not find matching files " + str(requiredFiles)) return "" if not OutputFile is None: webPageFullPath = os.path.join(path, OutputFile) f = open(webPageFullPath, 'w') f.write(HTMLString) f.close() return None # HTMLString