Exemplo n.º 1
0
def Import(VolumeElement, ImportPath, extension=None, *args, **kwargs):
    '''Import the specified directory into the volume'''
    
    if extension is None:
        extension = 'dm4'
        
    if not os.path.exists(ImportPath):
        raise Exception("Import directory not found: " + ImportPath)
        return
    
    tile_overlap = kwargs.get('tile_overlap', None)
    if tile_overlap is not None:
        #Convert the percentage parameter to a 0-1.0 float, and reverse the X,Y ordering to match the rest of Nornir
        tile_overlap = np.asarray((tile_overlap[1], tile_overlap[0]), np.float32) / 100.0
        
    FlipList = nornir_buildmanager.importers.GetFlipList(ImportPath)
    histogramFilename = os.path.join(ImportPath, nornir_buildmanager.importers.DefaultHistogramFilename)
    ContrastMap = nornir_buildmanager.importers.LoadHistogramCutoffs(histogramFilename)
    if len(ContrastMap) == 0:
        nornir_buildmanager.importers.CreateDefaultHistogramCutoffFile(histogramFilename)

    DirList = files.RecurseSubdirectoriesGenerator(ImportPath, RequiredFiles="*." + extension, ExcludeNames=[], ExcludedDownsampleLevels=[])
    for path in DirList:
        prettyoutput.CurseString("DM4Import", "Importing *.dm4 from {0}".format(path))
        for idocFullPath in glob.glob(os.path.join(path, '*.dm4')):
            yield from DigitalMicrograph4Import.ToMosaic(VolumeElement, idocFullPath, VolumeElement.FullPath, FlipList=FlipList, ContrastMap=ContrastMap, tile_overlap=tile_overlap)
    
    nornir_pools.WaitOnAllPools()
    
    for transform_fullpath in mosaics_loaded:
        mosaicObj = mosaics_loaded[transform_fullpath]
        mosaicObj.SaveToMosaicFile(transform_fullpath)
        
    for transform_fullpath in transforms_changed:
        transformObj = transforms_changed[transform_fullpath]
        transformObj.ResetChecksum()
        yield transformObj.Parent
    def Write(cls,
              OutfilePath,
              Entries,
              Flip=False,
              Flop=False,
              ImageSize=None,
              Downsample=1):
        '''
        Creates a .mosaic file in the specified directory containing the specified
        Dictionary (Entries) at the specified Downsample rate.  Entries should have the key=filename
        and values should be a tuple with pixel coordinates
    
        Setting Flip to True will invert all X coordinates
        Setting Flop to True will invert all Y coordinates
        '''

        if ImageSize is None:
            ImageSize = [(4080, 4080)] * len(Entries)
        elif not isinstance(ImageSize, list):
            assert (len(ImageSize) == 2
                    )  # Expect tuple or list indicating image size
            ImageSize = ImageSize * len(Entries)
        else:
            # A list of two entries for the size
            if len(ImageSize) == 2 and not isinstance(ImageSize[0], list):
                ImageSize = [(ImageSize[0], ImageSize[1])] * len(Entries)
            elif len(ImageSize) == len(Entries):
                ImageSize = ImageSize
            else:
                raise Exception(
                    "Unexpected list format for ImageSize argument")

        prettyoutput.CurseString('Stage', "WriteMosaicFile " + OutfilePath)

        # TODO - find min/max values and center image in .mosaic file
        minX = sys.maxsize
        minY = sys.maxsize
        maxX = -sys.maxsize - 1
        maxY = -sys.maxsize - 1

        keys = list(Entries.keys())
        keys.sort()

        for key in keys:
            Coord = Entries[key]
            if (Coord[0] < minX):
                minX = Coord[0]
            if (Coord[0] > maxX):
                maxX = Coord[0]
            if (Coord[1] < minY):
                minY = Coord[1]
            if (Coord[1] > maxY):
                maxY = Coord[1]

        prettyoutput.Log('MinX:\t' + str(minX) + '\tMaxX:\t' + str(maxX))
        prettyoutput.Log('MinY:\t' + str(minY) + '\tMaxY:\t' + str(maxY))

        # Write standard .mosiac file header
        with open(OutfilePath, 'w+') as OutFile:
            OutFile.write('format_version_number: 1\n')

            OutFile.write('number_of_images: ' + str(len(Entries)) + '\n')
            OutFile.write('pixel_spacing: ' + str(Downsample) + '\n')
            OutFile.write('use_std_mask: 0\n')

            #    prettyoutput.Log( "Keys: " + str(Entries.keys())
            keys = list(Entries.keys())
            keys.sort()

            for i, key in enumerate(keys):
                Coord = Entries[key]
                tilesize = ImageSize[i]

                # prettyoutput.Log( str(key) + " : " + str(Coord)

                # Centering is nice, but it seems to break multi mrc sections
                #            Coord = (Coord[0] - (minX + CenterXOffset), Coord[1] - (minY + CenterYOffset))
                Coord = (Coord[0], Coord[1])

                X = Coord[0]
                Y = Coord[1]
                if Flip:
                    Y = -Y
                if Flop:
                    X = -X

                # Remove dirname from key
                key = os.path.basename(key)

                outstr = 'image:\n' + key + '\n' + 'LegendrePolynomialTransform_double_2_2_1 vp 6 1 0 1 1 1 0 fp 4 ' + str(
                    X) + ' ' + str(Y) + ' ' + str(tilesize[0] / 2) + ' ' + str(
                        tilesize[1] / 2)
                OutFile.write(outstr + '\n')

            OutFile.close()
Exemplo n.º 3
0
    def MultipleIntensityAverageCmd(self):
        return "MultipleIntensityAverage"

    def StosBruteCmd(self):
        #    return "ir-stos-brute -sh 1 -clahe 2 -refine -cubic -regularize "
        return "ir-stos-brute -sh 1 -refine -regularize "

    def StosGridCmd(self, Spacing=None, Neighborhood=None):
        if (Spacing is None):
            Spacing = 128
        if (Neighborhood is None):
            Neighborhood = 128

        return "ir-stos-grid -sh 1 -fft 0 0.25 -grid_spacing " + str(
            Spacing) + " -neighborhood " + str(Neighborhood) + " -it 10 "

    def StosAddTransform(self):
        return "ir-add-transforms "

    # Users calling this library can change this list to adjust the downsample levels used


Current = __Config()

if __name__ == '__main__':
    try:
        from nornir_shared import prettyoutput
        prettyoutput.CurseString("# of Cores", str(Current.NumProcs))
    except:
        pass
Exemplo n.º 4
0
def ConvertImagesInDict(ImagesToConvertDict, Flip=False, Flop=False, InputBpp=None, OutputBpp=None, Invert=False, bDeleteOriginal=False, RightLeftShift=None, AndValue=None, MinMax=None, Gamma=None):
    '''
    The key and value in the dictionary have the full path of an image to convert.
    MinMax is a tuple [Min,Max] passed to the -level parameter if it is not None
    RightLeftShift is a tuple containing a right then left then return to center shift which should be done to remove useless bits from the data
    I do not use an and because I do not calculate ImageMagick's quantum size yet.
    Every image must share the same colorspace
     
    :return: True if images were converted
    :rtype: bool 
    '''

    if len(ImagesToConvertDict) == 0:
        return False
    
    if InputBpp is None:
        for k in ImagesToConvertDict.keys():
            if os.path.exists(k):
                InputBpp = nornir_shared.images.GetImageBpp(k)
                break
    
    prettyoutput.CurseString('Stage', "ConvertImagesInDict")
    
    if MinMax is not None:
        if(MinMax[0] > MinMax[1]):
            raise ValueError("Invalid MinMax parameter passed to ConvertImagesInDict")
     
    pool = nornir_pools.GetMultithreadingPool("ConvertImagesInDict", num_threads=multiprocessing.cpu_count() * 2)
    #pool = nornir_pools.GetGlobalSerialPool()
    tasks = []
    
    for (input_image, output_image) in ImagesToConvertDict.items():
        task = pool.add_task("{0} -> {1}".format(input_image, output_image), 
                      _ConvertSingleImageToFile,
                      input_image_param=input_image,
                      output_filename=output_image,
                      Flip=Flip,
                      Flop=Flop,
                      InputBpp=InputBpp,
                      OutputBpp=OutputBpp,
                      Invert=Invert,
                      MinMax=MinMax,
                      Gamma=Gamma)
        tasks.append(task)
        
    while len(tasks) > 0:
        t = tasks.pop(0)
        try:
            t.wait()
        except Exception as e:
            prettyoutput.LogErr("Failed to convert " + t.name)
              
    if bDeleteOriginal:
        for (input_image, output_image) in ImagesToConvertDict.items():
            if input_image != output_image:
                pool.add_task("Delete {0}".format(input_image), os.remove, input_image)
                      
        while len(tasks) > 0:
            t = tasks.pop(0)
            try:
                t.wait()
            except OSError as e:
                prettyoutput.LogErr("Unable to delete {0}\n{1}".format(t.name, e))
                pass
            except IOError as e:
                prettyoutput.LogErr("Unable to delete {0}\n{1}".format(t.name, e))
                pass
            
    if not pool is None:
        pool.wait_completion()
        pool.shutdown()
        pool = None
            
    del tasks
Exemplo n.º 5
0
    def ToMosaic(cls, VolumeObj, idocFileFullPath, ContrastCutoffs, OutputImageExt=None, TargetBpp=None, FlipList=None, ContrastMap=None, CameraBpp=None, debug=None):
        '''
        This function will convert an idoc file in the given path to a .mosaic file.
        It will also rename image files to the requested extension and subdirectory.
        TargetBpp is calculated based on the number of bits required to encode the values
        between the median min and max values
        :param list FlipList: List of section numbers which should have images flipped
        :param dict ContrastMap: Dictionary mapping section number to (Min, Max, Gamma) tuples 
        '''
        if(OutputImageExt is None):
            OutputImageExt = 'png'
  
        if TargetBpp is None:
            TargetBpp = 8

        if FlipList is None:
            FlipList = []

        if ContrastMap is None:
            ContrastMap = {}
   
        SaveChannel = False

        idocFilePath = serialem_utils.GetPathWithoutSpaces(idocFileFullPath)
 
        OutputPath = VolumeObj.FullPath

        os.makedirs(OutputPath, exist_ok=True)

        logger = logging.getLogger(__name__ + '.' + str(cls.__name__) + "ToMosaic")

        # Report the current stage to the user
        prettyoutput.CurseString('Stage', "SerialEM to Mosaic " + str(idocFileFullPath))

        SectionNumber = 0
        sectionDir = os.path.dirname(idocFileFullPath) #serialem_utils.GetDirectories(idocFileFullPath)

        BlockObj = BlockNode.Create('TEM')
        [saveBlock, BlockObj] = VolumeObj.UpdateOrAddChild(BlockObj)

        # If the parent directory doesn't have the section number in the name, change it
        ExistingSectionInfo = shared.GetSectionInfo(sectionDir)
        if(ExistingSectionInfo[0] < 0):
            i = 5
#            SectionNumber = SectionNumber + 1
#            newPathName = ('%' + nornir_buildmanager.templates.Current.SectionFormat) % SectionNumber + '_' + sectionDir
#            newPath = os.path.join(ParentDir, newPathName)
#            prettyoutput.Log('Moving: ' + InputPath + ' -> ' + newPath)
#            shutil.move(InputPath, newPath)
#
#            InputPath = newPath
#
#            #Run glob again because the dir changes
#            idocFiles = glob.glob(os.path.join(InputPath,'*.idoc'))
        else:
            SectionNumber = ExistingSectionInfo.number

        prettyoutput.CurseString('Section', str(SectionNumber))

        # Check for underscores.  If there is an underscore and the first part is the sectionNumber, then use everything after as the section name
        SectionName = ('%' + nornir_buildmanager.templates.Current.SectionFormat) % ExistingSectionInfo.number
        SectionPath = ('%' + nornir_buildmanager.templates.Current.SectionFormat) % ExistingSectionInfo.number
        try:
            parts = sectionDir.partition("_")
            if not parts is None:
                if len(parts[2]) > 0:
                    SectionName = parts[2]
        except:
            pass

        sectionObj = SectionNode.Create(SectionNumber,
                                        SectionName,
                                        SectionPath)

        [saveSection, sectionObj] = BlockObj.UpdateOrAddChildByAttrib(sectionObj, 'Number')
        sectionObj.Name = SectionName

        # Create a channel group 
        [saveChannel, channelObj] = sectionObj.UpdateOrAddChildByAttrib(ChannelNode.Create('TEM'), 'Name')
   
        # Create a channel group for the section

        # I started ignoring existing supertile.mosaic files so I could rebuild sections where
        # a handful of tiles were corrupt
        # if(os.path.exists(SupertilePath)):
        #    continue

        Flip = SectionNumber in FlipList;
        if(Flip):
            prettyoutput.Log("Found in FlipList.txt, flopping images")

        IDocData = IDoc.Load(idocFilePath, CameraBpp=CameraBpp)

        assert(hasattr(IDocData, 'PixelSpacing'))
        assert(hasattr(IDocData, 'DataMode'))
        assert(hasattr(IDocData, 'ImageSize'))

        # If there are no tiles... return
        if IDocData.NumTiles == 0:
            prettyoutput.Log("No tiles found in IDoc: " + idocFilePath)
            return

        # See if we can find a notes file...
        shared.TryAddNotes(channelObj, sectionDir, logger)
        serialem_utils.TryAddLogs(channelObj, sectionDir, logger)

        AddIdocNode(channelObj, idocFilePath, IDocData, logger)

        # Set the scale
        [added, ScaleObj] = cls.CreateScaleNode(IDocData, channelObj)
        
        # Parse the images
        ImageBpp = IDocData.GetImageBpp()            
        if ImageBpp is None:
            ImageBpp = cls.GetImageBpp(IDocData, sectionDir) 

        FilterName = 'Raw' + str(TargetBpp)
        if(TargetBpp is None):
            FilterName = 'Raw'

        histogramFullPath = os.path.join(sectionDir, 'Histogram.xml')
        
        IDocData.RemoveMissingTiles(sectionDir)
        source_tile_list = [os.path.join(sectionDir, t.Image) for t in IDocData.tiles ]
          
        (ActualMosaicMin, ActualMosaicMax, Gamma) = cls.GetSectionContrastSettings(SectionNumber, ContrastMap, ContrastCutoffs, source_tile_list, IDocData, histogramFullPath)
        ActualMosaicMax = numpy.around(ActualMosaicMax)
        ActualMosaicMin = numpy.around(ActualMosaicMin)
        
        contrast_mismatch = channelObj.RemoveFilterOnContrastMismatch(FilterName, ActualMosaicMin, ActualMosaicMax, Gamma)
    
        Pool = nornir_pools.GetGlobalThreadPool()
        #_PlotHistogram(histogramFullPath, SectionNumber, ActualMosaicMin, ActualMosaicMax)
        Pool.add_task(histogramFullPath, _PlotHistogram, histogramFullPath, SectionNumber, ActualMosaicMin, ActualMosaicMax, force_recreate=contrast_mismatch)
        
        
        ImageConversionRequired = contrast_mismatch

        # Create a channel for the Raw data 
        [added_filter, filterObj] = channelObj.UpdateOrAddChildByAttrib(FilterNode.Create(Name=FilterName), 'Name')
        if added_filter:
            ImageConversionRequired = True

        filterObj.SetContrastValues(ActualMosaicMin, ActualMosaicMax, Gamma)
        filterObj.BitsPerPixel = TargetBpp

        SupertileName = 'Stage'
        SupertileTransform = SupertileName + '.mosaic'
        SupertilePath = os.path.join(channelObj.FullPath, SupertileTransform)

        # Check to make sure our supertile mosaic file is valid
        RemoveOutdatedFile(idocFilePath, SupertilePath)

        [added_transform, transformObj] = channelObj.UpdateOrAddChildByAttrib(TransformNode.Create(Name=SupertileName,
                                                                         Path=SupertileTransform,
                                                                         Type='Stage'),
                                                                         'Path')

        [added_tilepyramid, PyramidNodeObj] = filterObj.UpdateOrAddChildByAttrib(TilePyramidNode.Create(Type='stage',
                                                                            NumberOfTiles=IDocData.NumTiles),
                                                                            'Path')

        [added_level, LevelObj] = PyramidNodeObj.GetOrCreateLevel(1, GenerateData=False)

        Tileset = NornirTileset.CreateTilesFromIDocTileData(IDocData.tiles, InputTileDir=sectionDir, OutputTileDir=LevelObj.FullPath, OutputImageExt=OutputImageExt)

        # Make sure the target LevelObj is verified        
        if not os.path.exists(LevelObj.FullPath):
            os.makedirs(LevelObj.FullPath, exist_ok=True)
        else:
            Tileset.RemoveStaleTilesFromOutputDir(SupertilePath=SupertilePath)
            VerifyTiles(filterObj.TilePyramid.GetLevel(1))

        SourceToMissingTargetMap = Tileset.GetSourceToMissingTargetMap()

        # Figure out if we have to move or convert images
        if len(SourceToMissingTargetMap) == 0:
            ImageConversionRequired = False
        else:
            ImageConversionRequired = (not ImageBpp == TargetBpp) or (ImageConversionRequired or Tileset.ImageConversionRequired)

        if(ImageConversionRequired):
            Invert = False 
            filterObj.SetContrastValues(ActualMosaicMin, ActualMosaicMax, Gamma)
            filterObj.TilePyramid.NumberOfTiles = IDocData.NumTiles
            # andValue = cls.GetBitmask(ActualMosaicMin, ActualMosaicMax, TargetBpp)
            #nornir_shared.images.ConvertImagesInDict(SourceToMissingTargetMap, Flip=Flip, Bpp=TargetBpp, Invert=Invert, bDeleteOriginal=False, MinMax=[ActualMosaicMin, ActualMosaicMax])
            nornir_imageregistration.ConvertImagesInDict(SourceToMissingTargetMap, Flip=Flip, InputBpp=ImageBpp, OutputBpp=TargetBpp, Invert=Invert, bDeleteOriginal=False, MinMax=[ActualMosaicMin, ActualMosaicMax], Gamma=Gamma)

        elif(Tileset.ImageMoveRequired):
            for f in SourceToMissingTargetMap:
                shutil.copy(f, SourceToMissingTargetMap[f])

        # If we wrote new images replace the .mosaic file
        if len(SourceToMissingTargetMap) > 0 or not os.path.exists(SupertilePath):
            # Writing this file indicates import succeeded and we don't need to repeat these steps, writing it will possibly invalidate a lot of downstream data
            # We need to flip the images.  This may be a Utah scope issue, our Y coordinates are inverted relative to the images.  To fix this
            # we flop instead of flip and reverse when writing the coordinates
            mosaicfile.MosaicFile.Write(SupertilePath, Entries=Tileset.GetPositionsForTargets(), Flip=not Flip, ImageSize=IDocData.ImageSize, Downsample=1);
            MFile = mosaicfile.MosaicFile.Load(SupertilePath)

            # Sometimes files fail to convert, when this occurs remove them from the .mosaic
            if MFile.RemoveInvalidMosaicImages(LevelObj.FullPath):
                MFile.Save(SupertilePath)
 
            Mosaic.TranslateMosaicFileToZeroOrigin(SupertilePath)
            transformObj.ResetChecksum()
            SaveChannel = True
            # transformObj.Checksum = MFile.Checksum

        if saveBlock:
            return VolumeObj
        elif saveSection:
            return BlockObj
        elif saveChannel:
            return sectionObj
        elif  added_transform or added_tilepyramid or added_level or ImageConversionRequired or SaveChannel or contrast_mismatch:
            return channelObj
        return None
Exemplo n.º 6
0
def AssembleTilesetFromImageSet(Parameters,
                                ImageSetNode,
                                TileShape=None,
                                Logger=None,
                                **kwargs):
    '''Create full resolution tiles of specfied size for the mosaics'''
    prettyoutput.CurseString('Stage', "Assemble Tile Pyramids")

    TileWidth = TileShape[0]
    TileHeight = TileShape[1]

    FilterNode = ImageSetNode.FindParent('Filter')

    InputLevelNode = ImageSetNode.MaxResLevel
    if InputLevelNode is None:
        Logger.warning(
            "No input image level found for AssembleTilesetFromImageSet")
        return

    # Remove our tileset if our image is newer than the tileset
    ImageNode = ImageSetNode.GetImage(InputLevelNode.Downsample)
    if ImageNode is None:
        Logger.warning("No input image found for AssembleTilesetFromImageSet")
        return

    if FilterNode.HasTileset:
        TileSetNode = FilterNode.Tileset

#         if files.IsOutdated(ImageNode.FullPath, TileSetNode.Levels[0].FullPath):
#             TileSetNode.Clean("Input image was newer than tileset")
#         else:
#             return

    if not FilterNode.HasTileset:
        TileSetNode = nornir_buildmanager.VolumeManager.TilesetNode.Create()
        [added, TileSetNode
         ] = FilterNode.UpdateOrAddChildByAttrib(TileSetNode, 'Path')

        TileSetNode.TileXDim = TileWidth
        TileSetNode.TileYDim = TileHeight
        TileSetNode.FilePostfix = '.png'
        TileSetNode.FilePrefix = FilterNode.Name + '_'
        TileSetNode.CoordFormat = nornir_buildmanager.templates.Current.GridTileCoordFormat

    os.makedirs(TileSetNode.FullPath, exist_ok=True)

    # OK, check if the first level of the tileset exists
    (added_outputlevel,
     OutputLevel) = TileSetNode.GetOrCreateLevel(InputLevelNode.Downsample,
                                                 GenerateData=False)

    added_outputlevel = True
    if (added_outputlevel):
        [YDim, XDim] = nornir_shared.images.GetImageSize(ImageNode.FullPath)

        tile_output = os.path.join(TileSetNode.FullPath, 'Tile%d.png')

        # Need to call ir-assemble
        cmd_template = 'magick convert %(InputPath)s -background black -crop %(TileSizeX)dx%(TileSizeY)d -depth 8 -quality 106 -type Grayscale -extent %(XDim)dx%(YDim)d %(TilePrefix)s'

        cmd = cmd_template % {
            'InputPath': ImageNode.FullPath,
            'TileSizeX': TileWidth,
            'TileSizeY': TileHeight,
            'XDim': TileWidth,
            'YDim': TileHeight,
            'TilePrefix': tile_output
        }

        prettyoutput.CurseString('Cmd', cmd)
        subprocess.call(cmd + ' && exit', shell=True)

        FilePostfix = ''

        GridDimY = int(math.ceil(YDim / float(TileHeight)))
        GridDimX = int(math.ceil(XDim / float(TileWidth)))

        GridTileNameTemplate = nornir_buildmanager.templates.Current.GridTileNameTemplate

        os.makedirs(OutputLevel.FullPath, exist_ok=True)

        iFile = 0
        for iY in range(0, GridDimY):
            for iX in range(0, GridDimX):
                tileFileName = os.path.join(TileSetNode.FullPath,
                                            tile_output % (iFile))
                gridTileFileName = GridTileNameTemplate % {
                    'prefix': TileSetNode.FilePrefix,
                    'X': iX,
                    'Y': iY,
                    'postfix': TileSetNode.FilePostfix
                }

                gridTileFileName = os.path.join(OutputLevel.FullPath,
                                                gridTileFileName)

                shutil.move(tileFileName, gridTileFileName)

                iFile = iFile + 1

        OutputLevel.GridDimX = GridDimX
        OutputLevel.GridDimY = GridDimY

    return FilterNode