def RemoveInvalidMosaicImages(self, TileDir): '''Runs a time consuming check that verifies all the image files are valid. Returns true if invalid images were found.''' ImageList = [] ImageList.extend(list(self.ImageToTransformString.keys())) InvalidImages = images.AreValidImages(ImageList, TileDir) FoundInvalid = False for InvalidImage in InvalidImages: if InvalidImage in self.ImageToTransformString: InvalidImageFullPath = os.path.join(TileDir, InvalidImage) if os.path.exists(InvalidImageFullPath): prettyoutput.Log('Removing invalid image from disk:') os.remove(InvalidImageFullPath) prettyoutput.Log('Removing invalid image from mosaic: %s' % InvalidImage) del self.ImageToTransformString[InvalidImage] FoundInvalid = True # prettyoutput.Log('Removing invalid image from mosaic: %s' % InvalidImage) # del self.ImageToTransformString[InvalidImage] return FoundInvalid
def CreateVikingXML(StosMapName=None, StosGroupName=None, OutputFile=None, Host=None, **kwargs): '''When passed a volume node, creates a VikingXML file''' InputVolumeNode = kwargs.get('VolumeNode') path = InputVolumeNode.Path if OutputFile is None: OutputFile = "Volume.VikingXML" if not OutputFile.lower().endswith('.vikingxml'): OutputFile = OutputFile + ".VikingXML" # Create our XML File OutputXMLFilename = os.path.join(path, OutputFile) # Load the inputXML file and begin parsing # Create the root output node OutputVolumeNode = ETree.Element( 'Volume', { 'Name': InputVolumeNode.Name, 'num_stos': '0', 'num_sections': '0', 'InputChecksum': InputVolumeNode.Checksum }) (units_of_measure, units_per_pixel) = DetermineVolumeScale(InputVolumeNode) if units_of_measure is not None: AddScaleData(OutputVolumeNode, units_of_measure, units_per_pixel) ParseSections(InputVolumeNode, OutputVolumeNode) RemoveDuplicateScaleEntries(OutputVolumeNode, units_of_measure, units_per_pixel) ParseStos(InputVolumeNode, OutputVolumeNode, StosMapName, StosGroupName) OutputXML = ETree.tostring(OutputVolumeNode).decode('utf-8') # prettyoutput.Log(OutputXML) hFile = open(OutputXMLFilename, 'w') hFile.write(OutputXML) hFile.close() # Walk down to the path from the root directory, merging about.xml's as we go Url = RecursiveMergeAboutXML(path, OutputXMLFilename) if not Host is None and len(Host) > 0: OutputVolumeNode.attrib['host'] = Host prettyoutput.Log("Launch string:") prettyoutput.Log(Url) finalUrl = url_join(Url, OutputFile) vikingUrl = "http://connectomes.utah.edu/Software/Viking4/viking.application?" + finalUrl prettyoutput.Log(vikingUrl) return
def TryAddNotes(containerObj, InputPath, logger): '''Check the path for a notes.txt file. If found, add a <Notes> element to the passed containerObj''' NotesFiles = glob.glob(os.path.join(InputPath, '*notes*.*')) NotesAdded = False if len(NotesFiles) > 0: for filename in NotesFiles: try: from xml.sax.saxutils import escape NotesFilename = os.path.basename(filename) CopiedNotesFullPath = os.path.join(containerObj.FullPath, NotesFilename) if not os.path.exists(CopiedNotesFullPath): os.makedirs(containerObj.FullPath, exist_ok=True) shutil.copyfile(filename, CopiedNotesFullPath) NotesAdded = True with open(filename, 'r') as f: notesTxt = f.read() (base, ext) = os.path.splitext(filename) encoding = "utf-8" ext = ext.lower() # notesTxt = notesTxt.encode(encoding) notesTxt = notesTxt.replace('\0', '') if len(notesTxt) > 0: # XMLnotesTxt = notesTxt # notesTxt = notesTxt.encode('utf-8') XMLnotesTxt = escape(notesTxt) # Create a Notes node to save the notes into NotesNodeObj = NotesNode.Create( Text=XMLnotesTxt, SourceFilename=NotesFilename) containerObj.RemoveOldChildrenByAttrib( 'Notes', 'SourceFilename', NotesFilename) [added, NotesNodeObj] = containerObj.UpdateOrAddChildByAttrib( NotesNodeObj, 'SourceFilename') if added: # Try to copy the notes to the output dir if we created a node if not os.path.exists(CopiedNotesFullPath): shutil.copyfile(filename, CopiedNotesFullPath) NotesNodeObj.text = XMLnotesTxt NotesNodeObj.encoding = encoding NotesAdded = NotesAdded or added except: (etype, evalue, etraceback) = sys.exc_info() prettyoutput.Log("Attempt to include notes from " + filename + " failed.\n" + evalue.message) prettyoutput.Log(etraceback) return NotesAdded
def PrintPipelineEnumeration(cls, PipelineXML): PipelineXML = cls.LoadPipelineXML(PipelineXML) cls.logger.info("Enumerating available pipelines") prettyoutput.Log("Enumerating available pipelines") for pipeline in PipelineXML.getroot(): cls.logger.info(' ' + pipeline.attrib.get('Name', "")) prettyoutput.Log(' ' + pipeline.attrib.get('Name', "")) prettyoutput.Log(' ' + pipeline.attrib.get('Description', "") + '\n')
def FromXML(self, xml): obj = Histogram() if isinstance(xml, str): xmlDoc = xml.dom.minidom.parseString(xml) else: xmlDoc = xml #else: #raise InvalidOperation("xml parameter was not a string or XML minidom object") Elems = xmlDoc.getElementsByTagName('Histogram') if (len(Elems) != 1): prettyoutput.Log("Histogram tag not found in histogram XML") return HistogramElem = Elems[0] if (HistogramElem.hasAttribute('NumBins')): obj.NumBins = int(HistogramElem.getAttribute('NumBins')) if (HistogramElem.hasAttribute('NumSamples')): obj.NumSamples = int(HistogramElem.getAttribute('NumSamples')) if (HistogramElem.hasAttribute('MinValue')): obj.MinValue = float(HistogramElem.getAttribute('MinValue')) if (HistogramElem.hasAttribute('MaxValue')): obj.MaxValue = float(HistogramElem.getAttribute('MaxValue')) ChannelElems = HistogramElem.getElementsByTagName('Channel') if (len(ChannelElems) == 0): prettyoutput.Log("No channel element found in histogram") return ChannelElem = ChannelElems[0] BinNode = ChannelElem.firstChild BinString = BinNode.data BinStrings = BinString.split() obj.Bins = list() for i in range(0, len(BinStrings)): obj.Bins.append(int(BinStrings[i])) if (len(obj.Bins) != obj.NumBins): prettyoutput.Log("ERROR: obj.Bins != obj.NumBins") prettyoutput.Log(str(obj)) return return obj
def GetImageBpp(cls, IDocData, sectionDir): ImageBpp = IDocData.GetImageBpp() if ImageBpp is None: # Figure out the bit depth of the input tile = IDocData.tiles[0] SourceImageFullPath = os.path.join(sectionDir, tile.Image) ImageBpp = GetImageBpp(SourceImageFullPath) if(not ImageBpp is None): prettyoutput.Log("Source images are " + str(ImageBpp) + " bits per pixel") else: prettyoutput.Log("Could not determine source image BPP") raise ValueError("Could not determine source image BPP") return ImageBpp
def _GetMinMaxCutoffs(listfilenames, MinCutoff, MaxCutoff, idoc_data, histogramFullPath=None): histogramObj = None if not histogramFullPath is None: if os.path.exists(histogramFullPath): histogramObj = Histogram.Load(histogramFullPath) if histogramObj is None: prettyoutput.Log("Collecting mosaic min/max data") Bpp = idoc_data.GetImageBpp() if Bpp is None: Bpp = nornir_shared.images.GetImageBpp(listfilenames[0]) numBins = 256 if Bpp >= 11: numBins = 2048 maxVal = idoc_data.Max if idoc_data.CameraBpp is not None: if (1 << idoc_data.CameraBpp) - 1 < maxVal: maxVal = (1 << idoc_data.CameraBpp) - 1 histogramObj = image_stats.Histogram(listfilenames, Bpp=Bpp, MinVal=idoc_data.Min, MaxVal=idoc_data.Max, numBins=numBins) if not histogramFullPath is None: histogramObj = _CleanOutliersFromIDocHistogram(histogramObj) histogramObj.Save(histogramFullPath) assert(not histogramObj is None) # I am willing to clip 1 pixel every hundred thousand on the dark side, and one every ten thousand on the light return histogramObj.AutoLevel(MinCutoff, MaxCutoff)
def WritePruneMap(self, MapImageToScoreFile): if len(self.MapImageToScore) == 0: prettyoutput.LogErr('No prune scores to write to file ' + MapImageToScoreFile) return with open(MapImageToScoreFile, 'w') as outfile: if (len(list(self.MapImageToScore.keys())) == 0): if (os.path.exists(MapImageToScoreFile)): os.remove(MapImageToScoreFile) prettyoutput.Log( "No prune scores present in PruneMap being saved: " + MapImageToScoreFile) outfile.close() return for f in sorted(self.MapImageToScore.keys()): score = self.MapImageToScore[f] outfile.write(f + '\t' + str(score) + '\n') outfile.close()
def CreateTilesFromIDocTileData(cls, tiles, InputTileDir, OutputTileDir, OutputImageExt): ''' :param tiles IDocTileData: List of tiles to build dictionaries from ''' # SerialEM begins numbering file names from zero. So we will too. ImageNumber = -1 obj = NornirTileset(OutputImageExt) for tile in tiles: [ImageRoot, ImageExt] = os.path.splitext(tile.Image) ImageExt = ImageExt.strip('.') ImageExt = ImageExt.lower() ImageNumber = ImageNumber + 1 SourceImageFullPath = os.path.join(InputTileDir, tile.Image) if not os.path.exists(SourceImageFullPath): prettyoutput.Log("Could not locate import image: " + SourceImageFullPath) obj.MissingInputImage = True continue # I rename the converted image because I haven't checked how robust viking is with non-numbered images. I'm 99% sure it can handle it, but I don't want to test now. ConvertedImageName = (nornir_buildmanager.templates.Current.TileCoordFormat % ImageNumber) + '.' + OutputImageExt TargetImageFullPath = os.path.join(OutputTileDir, ConvertedImageName) obj.AddTile(NornirTileset.Tile(SourceImageFullPath, TargetImageFullPath, Position=tile.PieceCoordinates[0:2])) return obj
def Execute(self, args): '''This executes the loaded pipeline on the specified volume of data. parser is an instance of the argparser class which should be extended with any pipeline specific arguments args are the parameters from the command line''' # DOM = self.PipelineData.toDOM() # PipelineElement = DOM.firstChild ArgSet = ArgumentSet() PipelineElement = self.PipelineData prettyoutput.Log("Adding pipeline arguments") # parser = self.GetArgParser(parser) # (args, unused) = parser.parse_known_args(passedArgs) ArgSet.AddArguments(args) ArgSet.AddParameters(PipelineElement) # Load the Volume.XML file in the output directory self.VolumeTree = VolumeManagerETree.VolumeManager.Load(args.volumepath, Create=True) if(self.VolumeTree is None): PipelineManager.logger.critical("Could not load or create volume.xml " + args.outputpath) prettyoutput.LogErr("Could not load or create volume.xml " + args.outputpath) sys.exit() # dargs = copy.deepcopy(defaultDargs) self.ExecuteChildPipelines(ArgSet, self.VolumeTree, PipelineElement) nornir_pools.WaitOnAllPools()
def TryAddLogs(containerObj, InputPath, logger): '''Copy log files to output directories, and store select meta-data in the containerObj if it exists''' LogsFiles = glob.glob(os.path.join(InputPath, '*.log')) LogsAdded = False if len(LogsFiles) == 0: print("NO LOG FILE FOUND FOR CAPTURE: " % InputPath) elif len(LogsFiles) > 0: for filename in LogsFiles: NotesFilename = os.path.basename(filename) CopiedLogsFullPath = os.path.join(containerObj.FullPath, NotesFilename) if not os.path.exists(CopiedLogsFullPath): os.makedirs(containerObj.FullPath, exist_ok=True) shutil.copyfile(filename, CopiedLogsFullPath) LogsAdded = True # OK, try to parse the logs try: LogData = serialemlog.SerialEMLog.Load(filename) if LogData is None: continue if LogData.NumTiles == 0: continue # Create a Notes node to save the logs into LogNodeObj = DataNode.Create(Path=NotesFilename, attrib={'Name': 'Log'}) containerObj.RemoveOldChildrenByAttrib('Data', 'Name', 'Log') [added, LogNodeObj ] = containerObj.UpdateOrAddChildByAttrib(LogNodeObj, 'Name') LogsAdded = LogsAdded or added LogNodeObj.AverageTileTime = '%g' % LogData.AverageTileTime LogNodeObj.AverageTileDrift = '%g' % LogData.AverageTileDrift LogNodeObj.CaptureTime = '%g' % (LogData.MontageEnd - LogData.MontageStart) except: (etype, evalue, etraceback) = sys.exc_info() prettyoutput.Log("Attempt to include logs from " + filename + " failed.\n" + str(evalue)) prettyoutput.Log(str(etraceback)) return LogsAdded
def GetInfo(cls, filename): '''Parses a filename and returns what we have learned if it follows the convention <SECTION>_<CHANNEL>_<MosaicSource>_<DOWNSAMPLE>.mosaic [SectionNumber, Channel, MosaicSource, Downsample]''' if (os.path.exists(filename) == False): prettyoutput.LogErr("mosaic file not found: " + filename) return baseName = os.path.basename(filename) [baseName, ext] = os.path.splitext(baseName) parts = baseName.split("_") try: SectionNumber = int(parts[0]) except: # We really can't recover from this, so maybe an exception should be thrown instead SectionNumber = None prettyoutput.Log('Could not determine mosaic section number: ' + str(filename)) try: MosaicType = parts[-2] except: MosaicType = None prettyoutput.Log('Could not determine mosaic source: ' + str(filename)) if (len(parts) > 3): try: Channel = parts[1] except: Channel = None prettyoutput.Log('Could not determine mosaic channel: ' + str(filename)) # If we don't have a valid downsample value we assume 1 try: DownsampleStrings = parts[-1].split(".") Downsample = int(DownsampleStrings[0]) except: Downsample = None prettyoutput.Log('Could not determine mosaic downsample: ' + str(filename)) return [SectionNumber, Channel, MosaicType, Downsample]
def CreateHistogram(self, HistogramXMLFile, MapImageToScoreFile=None): if (len(list(self.MapImageToScore.items())) == 0 and MapImageToScoreFile is not None): # prettyoutput.Log( "Reading scores, MapImageToScore Empty " + MapImageToScoreFile) PruneObj.ReadPruneMap(MapImageToScoreFile) # prettyoutput.Log( "Read scores complete: " + str(self.MapImageToScore)) if (len(list(self.MapImageToScore.items())) == 0): prettyoutput.Log("No prune scores to create histogram with") return scores = [None] * len(list(self.MapImageToScore.items())) numScores = len(scores) i = 0 for pair in list(self.MapImageToScore.items()): # prettyoutput.Log("pair: " + str(pair)) scores[i] = pair[1] i = i + 1 # Figure out what type of histogram we should create # prettyoutput.Log('Scores: ' + str(scores)) minVal = min(scores) # prettyoutput.Log("MinVal: " + str(minVal)) maxVal = max(scores) # prettyoutput.Log("MaxVal: " + str(maxVal)) mean = sum(scores) / len(scores) # prettyoutput.Log("Mean: " + str(mean)) StdDevScalar = 1.0 / float(numScores - 1.0) total = 0 # Calc the std deviation for score in scores: temp = score - mean temp = temp * temp total = total + (temp * StdDevScalar) StdDev = math.sqrt(total) # prettyoutput.Log("StdDev: " + str(StdDev)) numBins = int(math.ceil((maxVal - minVal) / (StdDev / 10.0))) # prettyoutput.Log("NumBins: " + str(numBins)) if (numBins < 10): numBins = 10 if (numBins > len(scores)): numBins = len(scores) # prettyoutput.Log("Final NumBins: " + str(numBins)) H = Histogram.Init(minVal, maxVal, numBins) H.Add(scores) H.Save(HistogramXMLFile) print("Created Histogram %s" % HistogramXMLFile)
def WritePruneMosaic(self, path, SourceMosaic, TargetMosaic='prune.mosaic', Tolerance='5'): ''' Remove tiles from the source mosaic with scores less than Tolerance and write the new mosaic to TargetMosaic. Raises a key error if image in prude scores does not exist in .mosaic file Raises a value error if the threshold removes all tiles in the mosaic. :return: Number of tiles removed ''' if (not isinstance(Tolerance, float)): Tolerance = float(Tolerance) SourceMosaicFullPath = os.path.join(path, SourceMosaic) TargetMosaicFullPath = os.path.join(path, TargetMosaic) mosaic = mosaicfile.MosaicFile.Load(SourceMosaicFullPath) # We copy this because if an input image is missing there will not be a prune score and it should be removed from the .mosaic file inputImageToTransforms = copy.deepcopy(mosaic.ImageToTransformString) mosaic.ImageToTransformString.clear() numRemoved = 0 for item in list(self.MapImageToScore.items()): filename = item[0] score = item[1] if (score >= Tolerance): keyVal = filename if not keyVal in inputImageToTransforms: keyVal = os.path.basename(filename) if not keyVal in inputImageToTransforms: raise KeyError( "PruneObj: Cannot locate image file in .mosaic " + keyVal) mosaic.ImageToTransformString[keyVal] = inputImageToTransforms[ keyVal] else: numRemoved = numRemoved + 1 if (len(mosaic.ImageToTransformString) <= 0): errMsg = "All tiles removed when using threshold = " + str( Tolerance) + "\nThe prune request was ignored" prettyoutput.LogErr(errMsg) raise ValueError(errMsg) else: prettyoutput.Log("Removed " + str(numRemoved) + " tiles pruning mosaic " + TargetMosaic) mosaic.Save(TargetMosaicFullPath) return numRemoved
def Execute(buildArgs=None): # Spend more time on each thread before switching #sys.setswitchinterval(500) if buildArgs is None: buildArgs = sys.argv[1:] InitLogging(buildArgs) Timer = TaskTimer() parser = BuildParserRoot() args = parser.parse_args(buildArgs) if args.lowpriority: lowpriority() print( "Warning, using low priority flag. This can make builds much slower" ) # SetupLogging(OutputPath=args.volumepath) try: Timer.Start(args.PipelineName) args.func(args) Timer.End(args.PipelineName) finally: OutStr = str(Timer) prettyoutput.Log(OutStr) timeTextFullPath = os.path.join(args.volumepath, 'Timing.txt') try: with open(timeTextFullPath, 'w+') as OutputFile: OutputFile.writelines(OutStr) OutputFile.close() except: prettyoutput.Log('Could not write %s' % (timeTextFullPath))
def MergeAboutXML(volumeXML, aboutXML): import xml.dom.minidom prettyoutput.Log('MergeAboutXML ' + str(volumeXML) + ' ' + str(aboutXML)) if (os.path.exists(volumeXML) == False): return if (os.path.exists(aboutXML) == False): return aboutDom = xml.dom.minidom.parse(aboutXML) volumeDom = xml.dom.minidom.parse(volumeXML) # Figure out which elements are contained in the about dom which need to be injected into the volumeXML # If element names match, attributes are added which are missing from the volumeXML. # If element names do not match, they are injected into the volumeXML at the appropriate level aboutNode = aboutDom.documentElement volumeNode = volumeDom.documentElement Url = None # Volume path is a special case so we append the path to the host name if (volumeNode.nodeName == "Volume" and aboutNode.nodeName == "Volume"): baseVolumeDir = os.path.dirname(volumeXML) baseAboutDir = os.path.dirname(aboutXML) relPath = baseVolumeDir.replace(baseAboutDir, '') prettyoutput.Log("Relative path: " + relPath) Url = UpdateVolumePath(volumeNode, aboutNode, relPath) MergeElements(volumeNode, aboutNode) prettyoutput.Log("") xmlFile = open(volumeXML, "w") xmlFile.write(volumeDom.toxml()) xmlFile.close() return Url
def ElementsEqual(volumeElement, aboutElement): '''Return true if the elements have the same tag, and the attributes found in both elements have same value''' if (aboutElement.nodeName != volumeElement.nodeName): return False for attrib_key in list(aboutElement.attributes.keys()): if not (volumeElement.hasAttribute(attrib_key) and aboutElement.hasAttribute(attrib_key)): continue if not volumeElement.getAttribute( attrib_key) == aboutElement.getAttribute(attrib_key): return False return True # Volume is the root element so it is always a match if (aboutElement.nodeName == "Volume"): return True # Nodes only match if their attributes match if (aboutElement.nodeName == "Section"): aboutNumber = aboutElement.getAttribute("number") volNumber = volumeElement.getAttribute("number") if (aboutNumber != volNumber): return False else: prettyoutput.Log("Equal:") prettyoutput.Log('v: ' + volumeElement.nodeName + ' ' + str(volNumber)) prettyoutput.Log('a: ' + aboutElement.nodeName + ' ' + str(aboutNumber)) prettyoutput.Log('') return True return False
def MergeChildren(volumeParent, aboutParent): aboutElement = aboutParent.firstChild while (aboutElement is not None): if (aboutElement.nodeName is None): break volumeElements = volumeParent.getElementsByTagName( aboutElement.nodeName) # The volume doesn't have any elements like this. Add them if (volumeElements.length == 0): newNode = aboutElement.cloneNode(True) prettyoutput.Log('NewNode' + newNode.toxml()) volumeParent.insertBefore(newNode, volumeParent.firstChild) else: for volElement in volumeElements: MergeElements(volElement, aboutElement) aboutElement = aboutElement.nextSibling
def Load(cls, PipelineXml, PipelineName=None): # PipelineData = Pipelines.CreateFromDOM(XMLDoc) SelectedPipeline = None XMLDoc = cls.LoadPipelineXML(PipelineXml) if PipelineName is None: PipelineManager.logger.warning("No pipeline name specified.") prettyoutput.Log("No pipeline name specified") cls.PrintPipelineEnumeration(XMLDoc) return None else: SelectedPipeline = XMLDoc.find("Pipeline[@Name='" + PipelineName + "']") if(SelectedPipeline is None): PipelineManager.logger.critical("No pipeline found named " + PipelineName) prettyoutput.LogErr("No pipeline found named " + PipelineName) cls.PrintPipelineEnumeration(XMLDoc) return None return PipelineManager(pipelinesRoot=XMLDoc.getroot(), pipelineData=SelectedPipeline)
def Load(filename): if not os.path.exists(filename): PrettyOutput.Log("Stos file not found: " + filename) return None obj = StosFile() try: [ obj.MappedSectionNumber, obj.ControlSectionNumber, Channels, Filters, obj.StosSource, obj._Downsample ] = StosFile.GetInfo(filename) except: pass with open(filename, 'r') as fMosaic: lines = fMosaic.readlines() if len(lines) < 7: PrettyOutput.LogErr("%s is not a valid stos file" % (filename)) raise ValueError("%s is not a valid stos file" % (filename)) obj.ControlImageFullPath = lines[0].strip() obj.MappedImageFullPath = lines[1].strip() ControlDims = lines[4].split() MappedDims = lines[5].split() obj.ControlImageDim = [float(x) for x in ControlDims] obj.MappedImageDim = [float(x) for x in MappedDims] obj.Transform = lines[6].strip() if len(lines) > 8: obj.ControlMaskFullPath = lines[8] obj.MappedMaskFullPath = lines[9] return obj
def EqualizeStosGridPixelSpacing(self, control_spacing, mapped_spacing, MappedImageFullPath, MappedMaskFullPath, create_copy=True): ''' Used to correct a mismatch between pixel spacings of the mapped and control images in a stos file. This was originally occuring when aligning light microscopy images to TEM images. Nornir expects the spacings for Stos files to be equal. This function is implemented to keep the control spacing the same and adjust the mapped spacing to match. Stos files have no way to encode the spacing in the file itself unfortunately. ''' if control_spacing == mapped_spacing: if create_copy: return copy.deepcopy(self) else: return self PrettyOutput.Log( "ChangeStosGridPixelSpacing from {0:d} to {1:d}".format( mapped_spacing, control_spacing)) control_spacing = float(control_spacing) mapped_spacing = float(mapped_spacing) mapped_space_scalar = mapped_spacing / control_spacing NewStosFile = StosFile() NewStosFile.ControlImageDim = copy.copy(self.ControlImageDim) # NewStosFile.MappedImageDim = [x * scale for x in self.MappedImageDim] # Update the filenames which are the first two lines of the file NewStosFile.MappedImageDim = copy.copy(self.MappedImageDim) NewStosFile.MappedImageDim[ 2] = self.MappedImageDim[2] * mapped_space_scalar NewStosFile.MappedImageDim[ 3] = self.MappedImageDim[3] * mapped_space_scalar NewStosFile.ControlImageFullPath = self.ControlImageFullPath NewStosFile.ControlMaskFullPath = self.ControlMaskFullPath NewStosFile.MappedImageFullPath = MappedImageFullPath NewStosFile.MappedMaskFullPath = MappedMaskFullPath if os.path.exists(MappedImageFullPath): NewStosFile.MappedImageDim = StosFile.__GetImageDimsArray( MappedImageFullPath) # Adjust the transform points transformObj = nornir_imageregistration.transforms.LoadTransform( self.Transform, pixelSpacing=1.0) assert (not transformObj is None) transformObj.ScaleWarped(scalar=mapped_space_scalar) NewStosFile._Downsample = control_spacing if hasattr(transformObj, 'gridWidth'): # Save as a stos grid if we can bounds = (NewStosFile.MappedImageDim[1], NewStosFile.MappedImageDim[0], NewStosFile.MappedImageDim[3], NewStosFile.MappedImageDim[2]) NewStosFile.Transform = nornir_imageregistration.transforms.TransformToIRToolsString( transformObj, bounds=bounds) else: NewStosFile.Transform = nornir_imageregistration.transforms.TransformToIRToolsString( transformObj) # , bounds=NewStosFile.MappedImageDim) return NewStosFile
def ToMosaic(cls, VolumeObj, idocFileFullPath, ContrastCutoffs, OutputImageExt=None, TargetBpp=None, FlipList=None, ContrastMap=None, CameraBpp=None, debug=None): ''' This function will convert an idoc file in the given path to a .mosaic file. It will also rename image files to the requested extension and subdirectory. TargetBpp is calculated based on the number of bits required to encode the values between the median min and max values :param list FlipList: List of section numbers which should have images flipped :param dict ContrastMap: Dictionary mapping section number to (Min, Max, Gamma) tuples ''' if(OutputImageExt is None): OutputImageExt = 'png' if TargetBpp is None: TargetBpp = 8 if FlipList is None: FlipList = [] if ContrastMap is None: ContrastMap = {} SaveChannel = False idocFilePath = serialem_utils.GetPathWithoutSpaces(idocFileFullPath) OutputPath = VolumeObj.FullPath os.makedirs(OutputPath, exist_ok=True) logger = logging.getLogger(__name__ + '.' + str(cls.__name__) + "ToMosaic") # Report the current stage to the user prettyoutput.CurseString('Stage', "SerialEM to Mosaic " + str(idocFileFullPath)) SectionNumber = 0 sectionDir = os.path.dirname(idocFileFullPath) #serialem_utils.GetDirectories(idocFileFullPath) BlockObj = BlockNode.Create('TEM') [saveBlock, BlockObj] = VolumeObj.UpdateOrAddChild(BlockObj) # If the parent directory doesn't have the section number in the name, change it ExistingSectionInfo = shared.GetSectionInfo(sectionDir) if(ExistingSectionInfo[0] < 0): i = 5 # SectionNumber = SectionNumber + 1 # newPathName = ('%' + nornir_buildmanager.templates.Current.SectionFormat) % SectionNumber + '_' + sectionDir # newPath = os.path.join(ParentDir, newPathName) # prettyoutput.Log('Moving: ' + InputPath + ' -> ' + newPath) # shutil.move(InputPath, newPath) # # InputPath = newPath # # #Run glob again because the dir changes # idocFiles = glob.glob(os.path.join(InputPath,'*.idoc')) else: SectionNumber = ExistingSectionInfo.number prettyoutput.CurseString('Section', str(SectionNumber)) # Check for underscores. If there is an underscore and the first part is the sectionNumber, then use everything after as the section name SectionName = ('%' + nornir_buildmanager.templates.Current.SectionFormat) % ExistingSectionInfo.number SectionPath = ('%' + nornir_buildmanager.templates.Current.SectionFormat) % ExistingSectionInfo.number try: parts = sectionDir.partition("_") if not parts is None: if len(parts[2]) > 0: SectionName = parts[2] except: pass sectionObj = SectionNode.Create(SectionNumber, SectionName, SectionPath) [saveSection, sectionObj] = BlockObj.UpdateOrAddChildByAttrib(sectionObj, 'Number') sectionObj.Name = SectionName # Create a channel group [saveChannel, channelObj] = sectionObj.UpdateOrAddChildByAttrib(ChannelNode.Create('TEM'), 'Name') # Create a channel group for the section # I started ignoring existing supertile.mosaic files so I could rebuild sections where # a handful of tiles were corrupt # if(os.path.exists(SupertilePath)): # continue Flip = SectionNumber in FlipList; if(Flip): prettyoutput.Log("Found in FlipList.txt, flopping images") IDocData = IDoc.Load(idocFilePath, CameraBpp=CameraBpp) assert(hasattr(IDocData, 'PixelSpacing')) assert(hasattr(IDocData, 'DataMode')) assert(hasattr(IDocData, 'ImageSize')) # If there are no tiles... return if IDocData.NumTiles == 0: prettyoutput.Log("No tiles found in IDoc: " + idocFilePath) return # See if we can find a notes file... shared.TryAddNotes(channelObj, sectionDir, logger) serialem_utils.TryAddLogs(channelObj, sectionDir, logger) AddIdocNode(channelObj, idocFilePath, IDocData, logger) # Set the scale [added, ScaleObj] = cls.CreateScaleNode(IDocData, channelObj) # Parse the images ImageBpp = IDocData.GetImageBpp() if ImageBpp is None: ImageBpp = cls.GetImageBpp(IDocData, sectionDir) FilterName = 'Raw' + str(TargetBpp) if(TargetBpp is None): FilterName = 'Raw' histogramFullPath = os.path.join(sectionDir, 'Histogram.xml') IDocData.RemoveMissingTiles(sectionDir) source_tile_list = [os.path.join(sectionDir, t.Image) for t in IDocData.tiles ] (ActualMosaicMin, ActualMosaicMax, Gamma) = cls.GetSectionContrastSettings(SectionNumber, ContrastMap, ContrastCutoffs, source_tile_list, IDocData, histogramFullPath) ActualMosaicMax = numpy.around(ActualMosaicMax) ActualMosaicMin = numpy.around(ActualMosaicMin) contrast_mismatch = channelObj.RemoveFilterOnContrastMismatch(FilterName, ActualMosaicMin, ActualMosaicMax, Gamma) Pool = nornir_pools.GetGlobalThreadPool() #_PlotHistogram(histogramFullPath, SectionNumber, ActualMosaicMin, ActualMosaicMax) Pool.add_task(histogramFullPath, _PlotHistogram, histogramFullPath, SectionNumber, ActualMosaicMin, ActualMosaicMax, force_recreate=contrast_mismatch) ImageConversionRequired = contrast_mismatch # Create a channel for the Raw data [added_filter, filterObj] = channelObj.UpdateOrAddChildByAttrib(FilterNode.Create(Name=FilterName), 'Name') if added_filter: ImageConversionRequired = True filterObj.SetContrastValues(ActualMosaicMin, ActualMosaicMax, Gamma) filterObj.BitsPerPixel = TargetBpp SupertileName = 'Stage' SupertileTransform = SupertileName + '.mosaic' SupertilePath = os.path.join(channelObj.FullPath, SupertileTransform) # Check to make sure our supertile mosaic file is valid RemoveOutdatedFile(idocFilePath, SupertilePath) [added_transform, transformObj] = channelObj.UpdateOrAddChildByAttrib(TransformNode.Create(Name=SupertileName, Path=SupertileTransform, Type='Stage'), 'Path') [added_tilepyramid, PyramidNodeObj] = filterObj.UpdateOrAddChildByAttrib(TilePyramidNode.Create(Type='stage', NumberOfTiles=IDocData.NumTiles), 'Path') [added_level, LevelObj] = PyramidNodeObj.GetOrCreateLevel(1, GenerateData=False) Tileset = NornirTileset.CreateTilesFromIDocTileData(IDocData.tiles, InputTileDir=sectionDir, OutputTileDir=LevelObj.FullPath, OutputImageExt=OutputImageExt) # Make sure the target LevelObj is verified if not os.path.exists(LevelObj.FullPath): os.makedirs(LevelObj.FullPath, exist_ok=True) else: Tileset.RemoveStaleTilesFromOutputDir(SupertilePath=SupertilePath) VerifyTiles(filterObj.TilePyramid.GetLevel(1)) SourceToMissingTargetMap = Tileset.GetSourceToMissingTargetMap() # Figure out if we have to move or convert images if len(SourceToMissingTargetMap) == 0: ImageConversionRequired = False else: ImageConversionRequired = (not ImageBpp == TargetBpp) or (ImageConversionRequired or Tileset.ImageConversionRequired) if(ImageConversionRequired): Invert = False filterObj.SetContrastValues(ActualMosaicMin, ActualMosaicMax, Gamma) filterObj.TilePyramid.NumberOfTiles = IDocData.NumTiles # andValue = cls.GetBitmask(ActualMosaicMin, ActualMosaicMax, TargetBpp) #nornir_shared.images.ConvertImagesInDict(SourceToMissingTargetMap, Flip=Flip, Bpp=TargetBpp, Invert=Invert, bDeleteOriginal=False, MinMax=[ActualMosaicMin, ActualMosaicMax]) nornir_imageregistration.ConvertImagesInDict(SourceToMissingTargetMap, Flip=Flip, InputBpp=ImageBpp, OutputBpp=TargetBpp, Invert=Invert, bDeleteOriginal=False, MinMax=[ActualMosaicMin, ActualMosaicMax], Gamma=Gamma) elif(Tileset.ImageMoveRequired): for f in SourceToMissingTargetMap: shutil.copy(f, SourceToMissingTargetMap[f]) # If we wrote new images replace the .mosaic file if len(SourceToMissingTargetMap) > 0 or not os.path.exists(SupertilePath): # Writing this file indicates import succeeded and we don't need to repeat these steps, writing it will possibly invalidate a lot of downstream data # We need to flip the images. This may be a Utah scope issue, our Y coordinates are inverted relative to the images. To fix this # we flop instead of flip and reverse when writing the coordinates mosaicfile.MosaicFile.Write(SupertilePath, Entries=Tileset.GetPositionsForTargets(), Flip=not Flip, ImageSize=IDocData.ImageSize, Downsample=1); MFile = mosaicfile.MosaicFile.Load(SupertilePath) # Sometimes files fail to convert, when this occurs remove them from the .mosaic if MFile.RemoveInvalidMosaicImages(LevelObj.FullPath): MFile.Save(SupertilePath) Mosaic.TranslateMosaicFileToZeroOrigin(SupertilePath) transformObj.ResetChecksum() SaveChannel = True # transformObj.Checksum = MFile.Checksum if saveBlock: return VolumeObj elif saveSection: return BlockObj elif saveChannel: return sectionObj elif added_transform or added_tilepyramid or added_level or ImageConversionRequired or SaveChannel or contrast_mismatch: return channelObj return None
def Write(cls, OutfilePath, Entries, Flip=False, Flop=False, ImageSize=None, Downsample=1): ''' Creates a .mosaic file in the specified directory containing the specified Dictionary (Entries) at the specified Downsample rate. Entries should have the key=filename and values should be a tuple with pixel coordinates Setting Flip to True will invert all X coordinates Setting Flop to True will invert all Y coordinates ''' if ImageSize is None: ImageSize = [(4080, 4080)] * len(Entries) elif not isinstance(ImageSize, list): assert (len(ImageSize) == 2 ) # Expect tuple or list indicating image size ImageSize = ImageSize * len(Entries) else: # A list of two entries for the size if len(ImageSize) == 2 and not isinstance(ImageSize[0], list): ImageSize = [(ImageSize[0], ImageSize[1])] * len(Entries) elif len(ImageSize) == len(Entries): ImageSize = ImageSize else: raise Exception( "Unexpected list format for ImageSize argument") prettyoutput.CurseString('Stage', "WriteMosaicFile " + OutfilePath) # TODO - find min/max values and center image in .mosaic file minX = sys.maxsize minY = sys.maxsize maxX = -sys.maxsize - 1 maxY = -sys.maxsize - 1 keys = list(Entries.keys()) keys.sort() for key in keys: Coord = Entries[key] if (Coord[0] < minX): minX = Coord[0] if (Coord[0] > maxX): maxX = Coord[0] if (Coord[1] < minY): minY = Coord[1] if (Coord[1] > maxY): maxY = Coord[1] prettyoutput.Log('MinX:\t' + str(minX) + '\tMaxX:\t' + str(maxX)) prettyoutput.Log('MinY:\t' + str(minY) + '\tMaxY:\t' + str(maxY)) # Write standard .mosiac file header with open(OutfilePath, 'w+') as OutFile: OutFile.write('format_version_number: 1\n') OutFile.write('number_of_images: ' + str(len(Entries)) + '\n') OutFile.write('pixel_spacing: ' + str(Downsample) + '\n') OutFile.write('use_std_mask: 0\n') # prettyoutput.Log( "Keys: " + str(Entries.keys()) keys = list(Entries.keys()) keys.sort() for i, key in enumerate(keys): Coord = Entries[key] tilesize = ImageSize[i] # prettyoutput.Log( str(key) + " : " + str(Coord) # Centering is nice, but it seems to break multi mrc sections # Coord = (Coord[0] - (minX + CenterXOffset), Coord[1] - (minY + CenterYOffset)) Coord = (Coord[0], Coord[1]) X = Coord[0] Y = Coord[1] if Flip: Y = -Y if Flop: X = -X # Remove dirname from key key = os.path.basename(key) outstr = 'image:\n' + key + '\n' + 'LegendrePolynomialTransform_double_2_2_1 vp 6 1 0 1 1 1 0 fp 4 ' + str( X) + ' ' + str(Y) + ' ' + str(tilesize[0] / 2) + ' ' + str( tilesize[1] / 2) OutFile.write(outstr + '\n') OutFile.close()
def VolumeFinder(path=None, OutputFile=None, VolumeNode=None, requiredFiles=None, **kwargs): '''Expects a 'Path' and 'RequiredFiles' keyword argument' produces an HTML index of all volumes under the path''' if requiredFiles is None: requiredFiles = [] if (path is None): VolumeNode = kwargs.get('VolumeNode', None) if VolumeNode is None: PrettyOutput.LogErr("Path attribute not found for VolumeFinder") return else: path = VolumeNode.attrib['Path'] requiredFiles = kwargs.get('RequiredFiles', []) if not isinstance(requiredFiles, list): RequiredFileStrs = str(requiredFiles).strip().split(',') requiredFiles = list() for fileStr in RequiredFileStrs: requiredFiles.append(fileStr) ServerHostname = FindServerFromAboutXML(path) if ServerHostname is None: ServerHostname = "" dirs = nornir_shared.files.RecurseSubdirectories( path, RequiredFiles=requiredFiles, ExcludeNames="") HTMLHeader = "<!DOCTYPE html> \n" + "<html>\n " + "<body>\n" HTMLFooter = "</body>\n" + "</html>\n" HTMLString = HTMLHeader dirDict = {} # Make sure required files is a list type if a single string was passed if isinstance(requiredFiles, str): requiredFiles = [requiredFiles] # Seperate out specifically named files from regular expressions RegExpPatterns = [] i = 0 while i < len(requiredFiles): filename = requiredFiles[i] if '*' in filename or '?' in filename: RegExpPatterns.append(filename) requiredFiles.pop(i) else: i = i + 1 for directory in dirs: # Take the path from the list and find files. dirImageList = dirDict.get(directory, []) for pattern in RegExpPatterns: filesMatchingPattern = glob.glob(os.path.join(directory, pattern)) for filename in filesMatchingPattern: filenameFullPath = os.path.join(directory, filename) dirImageList.append( filenameFullPath) # Should check if it exists maybe for filename in requiredFiles: filenameFullPath = os.path.join(directory, filename) if (os.path.exists(filenameFullPath)): dirImageList.append( filenameFullPath) # Should check if it exists maybe dirDict[directory] = dirImageList # Remove the path we were passed from the strings HTMLString = HTMLString.replace(path, '') RowNames = list(dirDict.keys()) RowNames.sort() RowNames.reverse() HTMLTable = HTMLTableForImageList(path, dirDict, RowNames, **kwargs) HTMLString = HTMLString + HTMLTable + '\n' HTMLString = HTMLString + HTMLFooter if (len(RowNames) == 0): PrettyOutput.Log("Report generator could not find matching files " + str(requiredFiles)) return "" if not OutputFile is None: webPageFullPath = os.path.join(path, OutputFile) f = open(webPageFullPath, 'w') f.write(HTMLString) f.close() return None # HTMLString
def ToMosaic(cls, VolumeObj, InputPath, scaleValueInNm, OutputPath=None, OutputImageExt=None, TargetBpp=None, debug=None, *args, **kwargs): '''#Converts a directory of images to sections, each represented by a single image. Each image should have the format Section#_ChannelText''' # Default to the directory above ours if an output path is not specified if OutputPath is None: OutputPath = os.path.join(InputPath, "..") # If the user did not supply a value, use a default if (TargetBpp is None): TargetBpp = 8 # Report the current stage to the user # prettyoutput.CurseString('Stage', "PMGToMosaic " + InputPath); # Find the files with a .pmg extension filename = InputPath if 'histogram' in filename.lower(): prettyoutput.Log( "PNG importer ignoring probable histogram file: " + filename) return None # TODO wrap in try except and print nice error on badly named files? fileData = filenameparser.ParseFilename(filename, imageNameMappings) fileData = FillInMissingImageNameData(fileData) imageDir = os.path.dirname(filename) imagebasename = os.path.basename(filename) BlockName = os.path.basename(imageDir) if BlockName is None or len(BlockName) == 0: BlockName = 'LM' BlockObj = BlockNode.Create(BlockName) [addedBlock, BlockObj] = VolumeObj.UpdateOrAddChild(BlockObj) if (fileData is None): raise Exception("Could not parse section from PMG filename: " + filename) SectionNumber = fileData.Section sectionObj = SectionNode.Create(SectionNumber) [addedSection, sectionObj] = BlockObj.UpdateOrAddChildByAttrib(sectionObj, 'Number') ChannelName = fileData.Channel ChannelName = ChannelName.replace(' ', '_') channelObj = ChannelNode.Create(ChannelName) channelObj.SetScale(scaleValueInNm) [channelAdded, channelObj] = sectionObj.UpdateOrAddChildByAttrib(channelObj, 'Name') # Create a filter for the images # Create a filter and mosaic FilterName = fileData.Filter if FilterName is None: if (TargetBpp is None): FilterName = 'Import' else: FilterName = 'Import' + str(TargetBpp) (added_filter, filterObj) = channelObj.GetOrCreateFilter(FilterName) filterObj.BitsPerPixel = TargetBpp # Create an image for the filter ImageSetNode = metadatautils.CreateImageSetForImage( filterObj, filename, Downsample=fileData.Downsample) # Find the image node levelNode = ImageSetNode.GetChildByAttrib('Level', 'Downsample', fileData.Downsample) imageNode = levelNode.GetChildByAttrib('Image', 'Path', imagebasename) os.makedirs(os.path.dirname(imageNode.FullPath), exist_ok=True) prettyoutput.Log("Copying file: " + imageNode.FullPath) shutil.copy(filename, imageNode.FullPath) if addedBlock: return VolumeObj elif addedSection: return BlockObj elif channelAdded: return sectionObj else: return channelObj
def ParsePMG(filename, TileOverlapPercent=None): if TileOverlapPercent is None: TileOverlapPercent = 0.1 # Create a dictionary to store tile position Tiles = dict() # OutFilepath = os.path.join(SectionPath, OutFilename) PMGDir = os.path.dirname(filename) if (DEBUG): prettyoutput.Log("Filename: " + filename) # prettyoutput.Log("PMG to: " + OutFilepath) Data = '' with open(filename, 'r') as SourceFile: Data = SourceFile.read() SourceFile.close() Data = Data.replace('\r\n', '\n') Tuple = Data.partition('VISPIECES') Tuple = Tuple[2].partition('\n') NumPieces = int(Tuple[0]) Tuple = Data.partition('VISSCALE') Tuple = Tuple[2].partition('\n') ScaleFactor = 1 / float(Tuple[0]) Tuple = Data.partition('IMAGEREDUCTION') Tuple = Tuple[2].partition('\n') ReductionFactor = 1 / float(Tuple[0]) Data = Tuple[2] if (DEBUG): prettyoutput.Log(("Num Tiles: " + str(NumPieces))) prettyoutput.Log(("Scale : " + str(ScaleFactor))) prettyoutput.Log(("Reduction: " + str(ReductionFactor))) Tuple = Data.partition('PIECE') Data = Tuple[2] # What type of files did Syncroscan write? We have to assume BMP and then switch to tif # if the BMP's do not exist. FileTypeExt = "BMP" TileWidth = 0 TileHeight = 0 while Data: Tuple = Data.partition('ENDPIECE') # Remaining unprocessed file goes into data Data = Tuple[2] Entry = Tuple[0] # Find filename Tuple = Entry.partition('PATH') # Probably means we skipped the last tile in a PMG if (Tuple[1] != "PATH"): continue Tuple = Tuple[2].partition('<') Tuple = Tuple[2].partition('>') TileFilename = Tuple[0] TileFullPath = os.path.join(PMGDir, TileFilename) # PMG files for some reason always claim the tile is a .bmp. So we trust them first and if it doesn't # exist we try to find a .tif if not os.path.exists(TileFullPath): [base, ext] = os.path.splitext(TileFilename) TileFilename = base + '.tif' TileFullPath = os.path.join(PMGDir, TileFilename) FileTypeExt = 'tif' if not os.path.exists(TileFullPath): prettyoutput.Log('Skipping missing tile in PMG: ' + TileFilename) continue if (TileWidth == 0): try: [TileHeight, TileWidth ] = nornir_imageregistration.GetImageSize(TileFullPath) if (DEBUG): prettyoutput.Log( str(TileWidth) + ',' + str(TileHeight) + " " + TileFilename) except: prettyoutput.Log('Could not determine size of tile: ' + TileFilename) continue # prettyoutput.Log("Adding tile: " + TileFilename) # Prevent rounding errors later when we divide these numbers TileWidth = float(TileWidth) TileHeight = float(TileHeight) TileWidthMinusOverlap = TileWidth - (TileWidth * TileOverlapPercent) TileHeightMinusOverlap = TileHeight - (TileHeight * TileOverlapPercent) # Find Position Tuple = Entry.partition('CORNER') Tuple = Tuple[2].partition('\n') Position = Tuple[0].split() # prettyoutput.Log( Position X = float(Position[0]) Y = float(Position[1]) # Convert Position into pixel units using Reduction and Scale factors X = X * ScaleFactor * ReductionFactor Y = Y * ScaleFactor * ReductionFactor # Syncroscan lays out tiles in grids, so find the nearest grid coordinates of this tile # This lets us use the last captured tile per grid cell, so users can manually correct # focus iX = round(X / TileWidthMinusOverlap) iY = round(Y / TileHeightMinusOverlap) if (DEBUG): prettyoutput.Log(("Name,iX,iY: " + TileFilename + " " + str( (iX, iY)))) # Add tile to dictionary # Using the indicies will replace any existing tiles in that location. # Syncroscan adds the tiles to the file in order of capture, so the older, blurry # tiles will be replaced. Tiles[TileFilename] = X, Y # Prime for next iteration Tuple = Data.partition('PIECE') Data = Tuple[2] return Tiles
def ChangeStosGridPixelSpacing(self, oldspacing, newspacing, ControlImageFullPath, MappedImageFullPath, ControlMaskFullPath, MappedMaskFullPath, create_copy=True): ''' :param bool create_copy: True if a copy of the transform should be scaled, otherwise scales the transform we were called on ''' if oldspacing == newspacing and \ ControlImageFullPath == self.ControlImageFullPath and \ MappedImageFullPath == self.MappedImageFullPath and \ ControlMaskFullPath == self.ControlMaskFullPath and \ MappedMaskFullPath == self.MappedMaskFullPath: if create_copy: return copy.deepcopy(self) else: return self PrettyOutput.Log("ChangeStosGridPixelSpacing from " + str(oldspacing) + " to " + str(newspacing)) scale = float(oldspacing) / float(newspacing) NewStosFile = StosFile() # NewStosFile.ControlImageDim = [x * scale for x in self.ControlImageDim] NewStosFile.ControlImageDim = copy.copy(self.ControlImageDim) NewStosFile.ControlImageDim[2] = self.ControlImageDim[2] * scale NewStosFile.ControlImageDim[3] = self.ControlImageDim[3] * scale # NewStosFile.MappedImageDim = [x * scale for x in self.MappedImageDim] # Update the filenames which are the first two lines of the file NewStosFile.MappedImageDim = copy.copy(self.MappedImageDim) NewStosFile.MappedImageDim[2] = self.MappedImageDim[2] * scale NewStosFile.MappedImageDim[3] = self.MappedImageDim[3] * scale NewStosFile.ControlImageFullPath = ControlImageFullPath NewStosFile.MappedImageFullPath = MappedImageFullPath NewStosFile.ControlMaskFullPath = ControlMaskFullPath NewStosFile.MappedMaskFullPath = MappedMaskFullPath if os.path.exists(ControlImageFullPath): NewStosFile.ControlImageDim = StosFile.__GetImageDimsArray( ControlImageFullPath) if os.path.exists(MappedImageFullPath): NewStosFile.MappedImageDim = StosFile.__GetImageDimsArray( MappedImageFullPath) # Adjust the transform points if scale == 1.0: NewStosFile.Transform = self.Transform else: transformObj = nornir_imageregistration.transforms.LoadTransform( self.Transform, pixelSpacing=1.0) assert (not transformObj is None) transformObj.Scale(scalar=scale) NewStosFile._Downsample = newspacing if hasattr(transformObj, 'gridWidth'): # Save as a stos grid if we can bounds = (NewStosFile.MappedImageDim[1], NewStosFile.MappedImageDim[0], NewStosFile.MappedImageDim[3], NewStosFile.MappedImageDim[2]) NewStosFile.Transform = nornir_imageregistration.transforms.TransformToIRToolsString( transformObj, bounds=bounds) else: NewStosFile.Transform = nornir_imageregistration.transforms.TransformToIRToolsString( transformObj) # , bounds=NewStosFile.MappedImageDim) return NewStosFile
def Load(cls, filename): if (os.path.exists(filename) == False): prettyoutput.Log("Mosaic file not found: " + filename) return return Histogram.FromXML(xml.dom.minidom.parse(filename))
def Histogram(filenames, Bpp=None, Scale=None, **kwargs): '''Returns a single histogram built by combining histograms of all images If scale is not none the images are scaled before the histogram is collected''' if isinstance(filenames, str): listfilenames = [filenames] else: listfilenames = filenames numTiles = len(listfilenames) if numTiles == 0: return dict() if Bpp is None: Bpp = nornir_shared.images.GetImageBpp(listfilenames[0]) assert isinstance(listfilenames, list) FilenameToTask = {} if len(listfilenames) > 2: pool = nornir_pools.GetGlobalLocalMachinePool() else: pool = nornir_pools.GetGlobalSerialPool() for f in listfilenames: # (root, ext) = os.path.splitext(f) # __HistogramFilePillow__(f, Bpp=Bpp, Scale=Scale) task = pool.add_task(f, __HistogramFileSciPy__, f, Bpp=Bpp, Scale=Scale, **kwargs) # if ext == '.npy': # task = __HistogramFileSciPy__(f, Bpp=Bpp, Scale=Scale) # else: # #task = __HistogramFilePillow__(f, ProcPool=pool, Bpp=Bpp, Scale=Scale) # task = pool.add_task(f, __HistogramFilePillow__,f, Bpp=Bpp, Scale=Scale) # #task = __HistogramFileImageMagick__(f, ProcPool=pool, Bpp=Bpp, Scale=Scale) FilenameToTask[f] = task minVal = None maxVal = None histlist = [] numBins = None for f in list(FilenameToTask.keys()): task = FilenameToTask[f] try: h = task.wait_return() except IOError as e: PrettyOutput.Log("File not found " + f) continue histlist.append(h) # lines = taskOutput.splitlines() # # OutputMap[f] = lines # # (fminVal, fmaxVal) = nornir_imageregistration.im_histogram_parser.MinMaxValues(lines) if minVal is None: minVal = h.MinValue else: minVal = min(minVal, h.MinValue) if maxVal is None: maxVal = h.MaxValue else: maxVal = max(maxVal, h.MaxValue) numBins = len(h.Bins) # # threadTasks = [] # # thread_pool = nornir_pools.GetGlobalThreadPool() # for f in list(OutputMap.keys()): # threadTask = thread_pool.add_task(f, nornir_imageregistration.im_histogram_parser.Parse, OutputMap[f], minVal=minVal, maxVal=maxVal, numBins=numBins) # threadTasks.append(threadTask) # HistogramComposite = nornir_shared.histogram.Histogram.Init(minVal=minVal, maxVal=maxVal, numBins=numBins) for h in histlist: # hist = t.wait_return() HistogramComposite.AddHistogram(h) # histogram = IMHistogramOutput.Parse(taskOutput, minVal=minVal, maxVal=maxVal, numBins=numBins) # FilenameToResult[f] = [histogram, None, None] if Bpp > 8: HistogramComposite = nornir_shared.histogram.Histogram.Trim(HistogramComposite) # del threadTasks # FilenameToResult = __InvokeFunctionOnImageList__(listfilenames, Function=__HistogramFileImageMagick__, Pool=nornir_pools.GetGlobalThreadPool(), ProcPool = nornir_pools.GetGlobalClusterPool(), Bpp=Bpp, Scale=Scale)#, NumSamples=SamplesPerImage) # maxVal = 1 << Bpp # numBins = 256 # if Bpp > 8: # numBins = 1024 # Sum all of the result arrays together # for filename in listfilenames: # if filename in FilenameToResult: # Result = FilenameToResult[filename] # histogram = Result[0] # # # # if HistogramComposite is None: # # HistogramComposite = numpy.zeros(histogram.shape, dtype=numpy.int0) # # # # HistogramComposite = numpy.add(HistogramComposite, histogram) # # HistogramComposite.AddHistogram(histogram.Bins) return HistogramComposite
def ToMosaic(cls, VolumeObj, PMGFullPath, scaleValueInNm, OutputPath=None, Extension=None, OutputImageExt=None, TileOverlap=None, TargetBpp=None, debug=None, *args, **kwargs): '''#Converts a PMG #PMG files are created by Objective Imaging's Surveyor. #This function expects a directory to contain a single PMG file with the tile images in the same directory #Returns the SectionNumber and ChannelName of PMG processed. Otherwise [None,None]''' ParentDir = os.path.dirname(PMGFullPath) sectionDir = os.path.basename(PMGFullPath) # Default to the directory above ours if an output path is not specified if OutputPath is None: OutputPath = os.path.join(PMGFullPath, "..") # If the user did not supply a value, use a default if (TileOverlap is None): TileOverlap = 0.10 if (TargetBpp is None): TargetBpp = 8 # Report the current stage to the user # prettyoutput.CurseString('Stage', "PMGToMosaic " + InputPath) BlockName = 'TEM' BlockObj = BlockNode.Create('TEM') [addedBlock, BlockObj] = VolumeObj.UpdateOrAddChild(BlockObj) ChannelName = None # TODO wrap in try except and print nice error on badly named files? PMG = ParseFilename(PMGFullPath, pmgMappings) PMGDir = os.path.dirname(PMGFullPath) if (PMG is None): raise Exception("Could not parse section from PMG filename: %s" + PMGFullPath) if PMG.Section is None: PMG.Section = PMG.Spot sectionObj = SectionNode.Create(PMG.Section) [addedSection, sectionObj] = BlockObj.UpdateOrAddChildByAttrib(sectionObj, 'Number') # Calculate our output directory. The scripts expect directories to have section numbers, so use that. ChannelName = PMG.Probe ChannelName = ChannelName.replace(' ', '_') channelObj = ChannelNode.Create(ChannelName) channelObj.SetScale(scaleValueInNm) [channelAdded, channelObj] = sectionObj.UpdateOrAddChildByAttrib(channelObj, 'Name') channelObj.Initials = PMG.Initials channelObj.Mag = PMG.Mag channelObj.Spot = PMG.Spot channelObj.Slide = PMG.Slide channelObj.Block = PMG.Block FlipList = GetFlipList(ParentDir) Flip = PMG.Section in FlipList if (Flip): prettyoutput.Log("Flipping") # TODO: Add scale element # OutFilename = ChannelName + "_supertile.mosaic" # OutFilepath = os.path.join(SectionPath, OutFilename) # Preserve the PMG file # PMGBasename = os.path.basename(filename) # PMGOutputFile = os.path.join(OutputPath, PMGBasename) # ir.RemoveOutdatedFile(filename, PMGOutputFile) # if not os.path.exists(PMGOutputFile): # shutil.copy(filename, PMGOutputFile) # # #See if we need to remove the old supertile mosaic # ir.RemoveOutdatedFile(filename, OutFilepath) # if(os.path.exists(OutFilepath)): # continue # Tiles = ParsePMG(PMGFullPath) if len(Tiles) == 0: raise Exception("No tiles found within PMG file") NumImages = len(Tiles) # Create a filter and mosaic FilterName = 'Raw' + str(TargetBpp) if (TargetBpp is None): FilterName = 'Raw' [added_filter, filterObj] = channelObj.GetOrCreateFilter(FilterName) filterObj.BitsPerPixel = TargetBpp SupertileName = 'Stage' SupertileTransform = SupertileName + '.mosaic' [addedTransform, transformObj] = channelObj.UpdateOrAddChildByAttrib( TransformNode.Create(Name=SupertileName, Path=SupertileTransform, Type='Stage'), 'Path') [added, PyramidNodeObj] = filterObj.UpdateOrAddChildByAttrib( TilePyramidNode.Create(Type='stage', NumberOfTiles=NumImages), 'Path') [added, LevelObj ] = PyramidNodeObj.UpdateOrAddChildByAttrib(LevelNode.Create(Level=1), 'Downsample') # Make sure the target LevelObj is verified VerifyTiles(LevelNode=LevelObj) OutputImagePath = os.path.join(channelObj.FullPath, filterObj.Path, PyramidNodeObj.Path, LevelObj.Path) os.makedirs(OutputImagePath, exist_ok=True) InputTileToOutputTile = {} PngTiles = {} TileKeys = list(Tiles.keys()) imageSize = [] for inputTile in TileKeys: [base, ext] = os.path.splitext(inputTile) pngMosaicTile = base + '.png' OutputTileFullPath = os.path.join(LevelObj.FullPath, pngMosaicTile) InputTileFullPath = os.path.join(PMGDir, inputTile) if not os.path.exists(OutputTileFullPath): InputTileToOutputTile[InputTileFullPath] = OutputTileFullPath PngTiles[pngMosaicTile] = Tiles[inputTile] (Height, Width) = nornir_imageregistration.GetImageSize(InputTileFullPath) imageSize.append((Width, Height)) nornir_imageregistration.ConvertImagesInDict(InputTileToOutputTile, Flip=False, OutputBpp=TargetBpp) if not os.path.exists(transformObj.FullPath): mosaicfile.MosaicFile.Write(transformObj.FullPath, PngTiles, Flip=Flip, ImageSize=imageSize) return [PMG.Section, ChannelName]