def Import(VolumeElement, ImportPath, extension=None, *args, **kwargs): '''Import the specified directory into the volume''' if extension is None: extension = 'dm4' if not os.path.exists(ImportPath): raise Exception("Import directory not found: " + ImportPath) return tile_overlap = kwargs.get('tile_overlap', None) if tile_overlap is not None: #Convert the percentage parameter to a 0-1.0 float, and reverse the X,Y ordering to match the rest of Nornir tile_overlap = np.asarray((tile_overlap[1], tile_overlap[0]), np.float32) / 100.0 FlipList = nornir_buildmanager.importers.GetFlipList(ImportPath) histogramFilename = os.path.join(ImportPath, nornir_buildmanager.importers.DefaultHistogramFilename) ContrastMap = nornir_buildmanager.importers.LoadHistogramCutoffs(histogramFilename) if len(ContrastMap) == 0: nornir_buildmanager.importers.CreateDefaultHistogramCutoffFile(histogramFilename) DirList = files.RecurseSubdirectoriesGenerator(ImportPath, RequiredFiles="*." + extension, ExcludeNames=[], ExcludedDownsampleLevels=[]) for path in DirList: prettyoutput.CurseString("DM4Import", "Importing *.dm4 from {0}".format(path)) for idocFullPath in glob.glob(os.path.join(path, '*.dm4')): yield from DigitalMicrograph4Import.ToMosaic(VolumeElement, idocFullPath, VolumeElement.FullPath, FlipList=FlipList, ContrastMap=ContrastMap, tile_overlap=tile_overlap) nornir_pools.WaitOnAllPools() for transform_fullpath in mosaics_loaded: mosaicObj = mosaics_loaded[transform_fullpath] mosaicObj.SaveToMosaicFile(transform_fullpath) for transform_fullpath in transforms_changed: transformObj = transforms_changed[transform_fullpath] transformObj.ResetChecksum() yield transformObj.Parent
def Write(cls, OutfilePath, Entries, Flip=False, Flop=False, ImageSize=None, Downsample=1): ''' Creates a .mosaic file in the specified directory containing the specified Dictionary (Entries) at the specified Downsample rate. Entries should have the key=filename and values should be a tuple with pixel coordinates Setting Flip to True will invert all X coordinates Setting Flop to True will invert all Y coordinates ''' if ImageSize is None: ImageSize = [(4080, 4080)] * len(Entries) elif not isinstance(ImageSize, list): assert (len(ImageSize) == 2 ) # Expect tuple or list indicating image size ImageSize = ImageSize * len(Entries) else: # A list of two entries for the size if len(ImageSize) == 2 and not isinstance(ImageSize[0], list): ImageSize = [(ImageSize[0], ImageSize[1])] * len(Entries) elif len(ImageSize) == len(Entries): ImageSize = ImageSize else: raise Exception( "Unexpected list format for ImageSize argument") prettyoutput.CurseString('Stage', "WriteMosaicFile " + OutfilePath) # TODO - find min/max values and center image in .mosaic file minX = sys.maxsize minY = sys.maxsize maxX = -sys.maxsize - 1 maxY = -sys.maxsize - 1 keys = list(Entries.keys()) keys.sort() for key in keys: Coord = Entries[key] if (Coord[0] < minX): minX = Coord[0] if (Coord[0] > maxX): maxX = Coord[0] if (Coord[1] < minY): minY = Coord[1] if (Coord[1] > maxY): maxY = Coord[1] prettyoutput.Log('MinX:\t' + str(minX) + '\tMaxX:\t' + str(maxX)) prettyoutput.Log('MinY:\t' + str(minY) + '\tMaxY:\t' + str(maxY)) # Write standard .mosiac file header with open(OutfilePath, 'w+') as OutFile: OutFile.write('format_version_number: 1\n') OutFile.write('number_of_images: ' + str(len(Entries)) + '\n') OutFile.write('pixel_spacing: ' + str(Downsample) + '\n') OutFile.write('use_std_mask: 0\n') # prettyoutput.Log( "Keys: " + str(Entries.keys()) keys = list(Entries.keys()) keys.sort() for i, key in enumerate(keys): Coord = Entries[key] tilesize = ImageSize[i] # prettyoutput.Log( str(key) + " : " + str(Coord) # Centering is nice, but it seems to break multi mrc sections # Coord = (Coord[0] - (minX + CenterXOffset), Coord[1] - (minY + CenterYOffset)) Coord = (Coord[0], Coord[1]) X = Coord[0] Y = Coord[1] if Flip: Y = -Y if Flop: X = -X # Remove dirname from key key = os.path.basename(key) outstr = 'image:\n' + key + '\n' + 'LegendrePolynomialTransform_double_2_2_1 vp 6 1 0 1 1 1 0 fp 4 ' + str( X) + ' ' + str(Y) + ' ' + str(tilesize[0] / 2) + ' ' + str( tilesize[1] / 2) OutFile.write(outstr + '\n') OutFile.close()
def MultipleIntensityAverageCmd(self): return "MultipleIntensityAverage" def StosBruteCmd(self): # return "ir-stos-brute -sh 1 -clahe 2 -refine -cubic -regularize " return "ir-stos-brute -sh 1 -refine -regularize " def StosGridCmd(self, Spacing=None, Neighborhood=None): if (Spacing is None): Spacing = 128 if (Neighborhood is None): Neighborhood = 128 return "ir-stos-grid -sh 1 -fft 0 0.25 -grid_spacing " + str( Spacing) + " -neighborhood " + str(Neighborhood) + " -it 10 " def StosAddTransform(self): return "ir-add-transforms " # Users calling this library can change this list to adjust the downsample levels used Current = __Config() if __name__ == '__main__': try: from nornir_shared import prettyoutput prettyoutput.CurseString("# of Cores", str(Current.NumProcs)) except: pass
def ConvertImagesInDict(ImagesToConvertDict, Flip=False, Flop=False, InputBpp=None, OutputBpp=None, Invert=False, bDeleteOriginal=False, RightLeftShift=None, AndValue=None, MinMax=None, Gamma=None): ''' The key and value in the dictionary have the full path of an image to convert. MinMax is a tuple [Min,Max] passed to the -level parameter if it is not None RightLeftShift is a tuple containing a right then left then return to center shift which should be done to remove useless bits from the data I do not use an and because I do not calculate ImageMagick's quantum size yet. Every image must share the same colorspace :return: True if images were converted :rtype: bool ''' if len(ImagesToConvertDict) == 0: return False if InputBpp is None: for k in ImagesToConvertDict.keys(): if os.path.exists(k): InputBpp = nornir_shared.images.GetImageBpp(k) break prettyoutput.CurseString('Stage', "ConvertImagesInDict") if MinMax is not None: if(MinMax[0] > MinMax[1]): raise ValueError("Invalid MinMax parameter passed to ConvertImagesInDict") pool = nornir_pools.GetMultithreadingPool("ConvertImagesInDict", num_threads=multiprocessing.cpu_count() * 2) #pool = nornir_pools.GetGlobalSerialPool() tasks = [] for (input_image, output_image) in ImagesToConvertDict.items(): task = pool.add_task("{0} -> {1}".format(input_image, output_image), _ConvertSingleImageToFile, input_image_param=input_image, output_filename=output_image, Flip=Flip, Flop=Flop, InputBpp=InputBpp, OutputBpp=OutputBpp, Invert=Invert, MinMax=MinMax, Gamma=Gamma) tasks.append(task) while len(tasks) > 0: t = tasks.pop(0) try: t.wait() except Exception as e: prettyoutput.LogErr("Failed to convert " + t.name) if bDeleteOriginal: for (input_image, output_image) in ImagesToConvertDict.items(): if input_image != output_image: pool.add_task("Delete {0}".format(input_image), os.remove, input_image) while len(tasks) > 0: t = tasks.pop(0) try: t.wait() except OSError as e: prettyoutput.LogErr("Unable to delete {0}\n{1}".format(t.name, e)) pass except IOError as e: prettyoutput.LogErr("Unable to delete {0}\n{1}".format(t.name, e)) pass if not pool is None: pool.wait_completion() pool.shutdown() pool = None del tasks
def ToMosaic(cls, VolumeObj, idocFileFullPath, ContrastCutoffs, OutputImageExt=None, TargetBpp=None, FlipList=None, ContrastMap=None, CameraBpp=None, debug=None): ''' This function will convert an idoc file in the given path to a .mosaic file. It will also rename image files to the requested extension and subdirectory. TargetBpp is calculated based on the number of bits required to encode the values between the median min and max values :param list FlipList: List of section numbers which should have images flipped :param dict ContrastMap: Dictionary mapping section number to (Min, Max, Gamma) tuples ''' if(OutputImageExt is None): OutputImageExt = 'png' if TargetBpp is None: TargetBpp = 8 if FlipList is None: FlipList = [] if ContrastMap is None: ContrastMap = {} SaveChannel = False idocFilePath = serialem_utils.GetPathWithoutSpaces(idocFileFullPath) OutputPath = VolumeObj.FullPath os.makedirs(OutputPath, exist_ok=True) logger = logging.getLogger(__name__ + '.' + str(cls.__name__) + "ToMosaic") # Report the current stage to the user prettyoutput.CurseString('Stage', "SerialEM to Mosaic " + str(idocFileFullPath)) SectionNumber = 0 sectionDir = os.path.dirname(idocFileFullPath) #serialem_utils.GetDirectories(idocFileFullPath) BlockObj = BlockNode.Create('TEM') [saveBlock, BlockObj] = VolumeObj.UpdateOrAddChild(BlockObj) # If the parent directory doesn't have the section number in the name, change it ExistingSectionInfo = shared.GetSectionInfo(sectionDir) if(ExistingSectionInfo[0] < 0): i = 5 # SectionNumber = SectionNumber + 1 # newPathName = ('%' + nornir_buildmanager.templates.Current.SectionFormat) % SectionNumber + '_' + sectionDir # newPath = os.path.join(ParentDir, newPathName) # prettyoutput.Log('Moving: ' + InputPath + ' -> ' + newPath) # shutil.move(InputPath, newPath) # # InputPath = newPath # # #Run glob again because the dir changes # idocFiles = glob.glob(os.path.join(InputPath,'*.idoc')) else: SectionNumber = ExistingSectionInfo.number prettyoutput.CurseString('Section', str(SectionNumber)) # Check for underscores. If there is an underscore and the first part is the sectionNumber, then use everything after as the section name SectionName = ('%' + nornir_buildmanager.templates.Current.SectionFormat) % ExistingSectionInfo.number SectionPath = ('%' + nornir_buildmanager.templates.Current.SectionFormat) % ExistingSectionInfo.number try: parts = sectionDir.partition("_") if not parts is None: if len(parts[2]) > 0: SectionName = parts[2] except: pass sectionObj = SectionNode.Create(SectionNumber, SectionName, SectionPath) [saveSection, sectionObj] = BlockObj.UpdateOrAddChildByAttrib(sectionObj, 'Number') sectionObj.Name = SectionName # Create a channel group [saveChannel, channelObj] = sectionObj.UpdateOrAddChildByAttrib(ChannelNode.Create('TEM'), 'Name') # Create a channel group for the section # I started ignoring existing supertile.mosaic files so I could rebuild sections where # a handful of tiles were corrupt # if(os.path.exists(SupertilePath)): # continue Flip = SectionNumber in FlipList; if(Flip): prettyoutput.Log("Found in FlipList.txt, flopping images") IDocData = IDoc.Load(idocFilePath, CameraBpp=CameraBpp) assert(hasattr(IDocData, 'PixelSpacing')) assert(hasattr(IDocData, 'DataMode')) assert(hasattr(IDocData, 'ImageSize')) # If there are no tiles... return if IDocData.NumTiles == 0: prettyoutput.Log("No tiles found in IDoc: " + idocFilePath) return # See if we can find a notes file... shared.TryAddNotes(channelObj, sectionDir, logger) serialem_utils.TryAddLogs(channelObj, sectionDir, logger) AddIdocNode(channelObj, idocFilePath, IDocData, logger) # Set the scale [added, ScaleObj] = cls.CreateScaleNode(IDocData, channelObj) # Parse the images ImageBpp = IDocData.GetImageBpp() if ImageBpp is None: ImageBpp = cls.GetImageBpp(IDocData, sectionDir) FilterName = 'Raw' + str(TargetBpp) if(TargetBpp is None): FilterName = 'Raw' histogramFullPath = os.path.join(sectionDir, 'Histogram.xml') IDocData.RemoveMissingTiles(sectionDir) source_tile_list = [os.path.join(sectionDir, t.Image) for t in IDocData.tiles ] (ActualMosaicMin, ActualMosaicMax, Gamma) = cls.GetSectionContrastSettings(SectionNumber, ContrastMap, ContrastCutoffs, source_tile_list, IDocData, histogramFullPath) ActualMosaicMax = numpy.around(ActualMosaicMax) ActualMosaicMin = numpy.around(ActualMosaicMin) contrast_mismatch = channelObj.RemoveFilterOnContrastMismatch(FilterName, ActualMosaicMin, ActualMosaicMax, Gamma) Pool = nornir_pools.GetGlobalThreadPool() #_PlotHistogram(histogramFullPath, SectionNumber, ActualMosaicMin, ActualMosaicMax) Pool.add_task(histogramFullPath, _PlotHistogram, histogramFullPath, SectionNumber, ActualMosaicMin, ActualMosaicMax, force_recreate=contrast_mismatch) ImageConversionRequired = contrast_mismatch # Create a channel for the Raw data [added_filter, filterObj] = channelObj.UpdateOrAddChildByAttrib(FilterNode.Create(Name=FilterName), 'Name') if added_filter: ImageConversionRequired = True filterObj.SetContrastValues(ActualMosaicMin, ActualMosaicMax, Gamma) filterObj.BitsPerPixel = TargetBpp SupertileName = 'Stage' SupertileTransform = SupertileName + '.mosaic' SupertilePath = os.path.join(channelObj.FullPath, SupertileTransform) # Check to make sure our supertile mosaic file is valid RemoveOutdatedFile(idocFilePath, SupertilePath) [added_transform, transformObj] = channelObj.UpdateOrAddChildByAttrib(TransformNode.Create(Name=SupertileName, Path=SupertileTransform, Type='Stage'), 'Path') [added_tilepyramid, PyramidNodeObj] = filterObj.UpdateOrAddChildByAttrib(TilePyramidNode.Create(Type='stage', NumberOfTiles=IDocData.NumTiles), 'Path') [added_level, LevelObj] = PyramidNodeObj.GetOrCreateLevel(1, GenerateData=False) Tileset = NornirTileset.CreateTilesFromIDocTileData(IDocData.tiles, InputTileDir=sectionDir, OutputTileDir=LevelObj.FullPath, OutputImageExt=OutputImageExt) # Make sure the target LevelObj is verified if not os.path.exists(LevelObj.FullPath): os.makedirs(LevelObj.FullPath, exist_ok=True) else: Tileset.RemoveStaleTilesFromOutputDir(SupertilePath=SupertilePath) VerifyTiles(filterObj.TilePyramid.GetLevel(1)) SourceToMissingTargetMap = Tileset.GetSourceToMissingTargetMap() # Figure out if we have to move or convert images if len(SourceToMissingTargetMap) == 0: ImageConversionRequired = False else: ImageConversionRequired = (not ImageBpp == TargetBpp) or (ImageConversionRequired or Tileset.ImageConversionRequired) if(ImageConversionRequired): Invert = False filterObj.SetContrastValues(ActualMosaicMin, ActualMosaicMax, Gamma) filterObj.TilePyramid.NumberOfTiles = IDocData.NumTiles # andValue = cls.GetBitmask(ActualMosaicMin, ActualMosaicMax, TargetBpp) #nornir_shared.images.ConvertImagesInDict(SourceToMissingTargetMap, Flip=Flip, Bpp=TargetBpp, Invert=Invert, bDeleteOriginal=False, MinMax=[ActualMosaicMin, ActualMosaicMax]) nornir_imageregistration.ConvertImagesInDict(SourceToMissingTargetMap, Flip=Flip, InputBpp=ImageBpp, OutputBpp=TargetBpp, Invert=Invert, bDeleteOriginal=False, MinMax=[ActualMosaicMin, ActualMosaicMax], Gamma=Gamma) elif(Tileset.ImageMoveRequired): for f in SourceToMissingTargetMap: shutil.copy(f, SourceToMissingTargetMap[f]) # If we wrote new images replace the .mosaic file if len(SourceToMissingTargetMap) > 0 or not os.path.exists(SupertilePath): # Writing this file indicates import succeeded and we don't need to repeat these steps, writing it will possibly invalidate a lot of downstream data # We need to flip the images. This may be a Utah scope issue, our Y coordinates are inverted relative to the images. To fix this # we flop instead of flip and reverse when writing the coordinates mosaicfile.MosaicFile.Write(SupertilePath, Entries=Tileset.GetPositionsForTargets(), Flip=not Flip, ImageSize=IDocData.ImageSize, Downsample=1); MFile = mosaicfile.MosaicFile.Load(SupertilePath) # Sometimes files fail to convert, when this occurs remove them from the .mosaic if MFile.RemoveInvalidMosaicImages(LevelObj.FullPath): MFile.Save(SupertilePath) Mosaic.TranslateMosaicFileToZeroOrigin(SupertilePath) transformObj.ResetChecksum() SaveChannel = True # transformObj.Checksum = MFile.Checksum if saveBlock: return VolumeObj elif saveSection: return BlockObj elif saveChannel: return sectionObj elif added_transform or added_tilepyramid or added_level or ImageConversionRequired or SaveChannel or contrast_mismatch: return channelObj return None
def AssembleTilesetFromImageSet(Parameters, ImageSetNode, TileShape=None, Logger=None, **kwargs): '''Create full resolution tiles of specfied size for the mosaics''' prettyoutput.CurseString('Stage', "Assemble Tile Pyramids") TileWidth = TileShape[0] TileHeight = TileShape[1] FilterNode = ImageSetNode.FindParent('Filter') InputLevelNode = ImageSetNode.MaxResLevel if InputLevelNode is None: Logger.warning( "No input image level found for AssembleTilesetFromImageSet") return # Remove our tileset if our image is newer than the tileset ImageNode = ImageSetNode.GetImage(InputLevelNode.Downsample) if ImageNode is None: Logger.warning("No input image found for AssembleTilesetFromImageSet") return if FilterNode.HasTileset: TileSetNode = FilterNode.Tileset # if files.IsOutdated(ImageNode.FullPath, TileSetNode.Levels[0].FullPath): # TileSetNode.Clean("Input image was newer than tileset") # else: # return if not FilterNode.HasTileset: TileSetNode = nornir_buildmanager.VolumeManager.TilesetNode.Create() [added, TileSetNode ] = FilterNode.UpdateOrAddChildByAttrib(TileSetNode, 'Path') TileSetNode.TileXDim = TileWidth TileSetNode.TileYDim = TileHeight TileSetNode.FilePostfix = '.png' TileSetNode.FilePrefix = FilterNode.Name + '_' TileSetNode.CoordFormat = nornir_buildmanager.templates.Current.GridTileCoordFormat os.makedirs(TileSetNode.FullPath, exist_ok=True) # OK, check if the first level of the tileset exists (added_outputlevel, OutputLevel) = TileSetNode.GetOrCreateLevel(InputLevelNode.Downsample, GenerateData=False) added_outputlevel = True if (added_outputlevel): [YDim, XDim] = nornir_shared.images.GetImageSize(ImageNode.FullPath) tile_output = os.path.join(TileSetNode.FullPath, 'Tile%d.png') # Need to call ir-assemble cmd_template = 'magick convert %(InputPath)s -background black -crop %(TileSizeX)dx%(TileSizeY)d -depth 8 -quality 106 -type Grayscale -extent %(XDim)dx%(YDim)d %(TilePrefix)s' cmd = cmd_template % { 'InputPath': ImageNode.FullPath, 'TileSizeX': TileWidth, 'TileSizeY': TileHeight, 'XDim': TileWidth, 'YDim': TileHeight, 'TilePrefix': tile_output } prettyoutput.CurseString('Cmd', cmd) subprocess.call(cmd + ' && exit', shell=True) FilePostfix = '' GridDimY = int(math.ceil(YDim / float(TileHeight))) GridDimX = int(math.ceil(XDim / float(TileWidth))) GridTileNameTemplate = nornir_buildmanager.templates.Current.GridTileNameTemplate os.makedirs(OutputLevel.FullPath, exist_ok=True) iFile = 0 for iY in range(0, GridDimY): for iX in range(0, GridDimX): tileFileName = os.path.join(TileSetNode.FullPath, tile_output % (iFile)) gridTileFileName = GridTileNameTemplate % { 'prefix': TileSetNode.FilePrefix, 'X': iX, 'Y': iY, 'postfix': TileSetNode.FilePostfix } gridTileFileName = os.path.join(OutputLevel.FullPath, gridTileFileName) shutil.move(tileFileName, gridTileFileName) iFile = iFile + 1 OutputLevel.GridDimX = GridDimX OutputLevel.GridDimY = GridDimY return FilterNode