def Import(VolumeElement, ImportPath, extension=None, *args, **kwargs):
'''Import the specified directory into the volume'''
if extension is None:
extension = 'dm4'
if not os.path.exists(ImportPath):
raise Exception("Import directory not found: " + ImportPath)
return
tile_overlap = kwargs.get('tile_overlap', None)
if tile_overlap is not None:
#Convert the percentage parameter to a 0-1.0 float, and reverse the X,Y ordering to match the rest of Nornir
tile_overlap = np.asarray((tile_overlap[1], tile_overlap[0]), np.float32) / 100.0
FlipList = nornir_buildmanager.importers.GetFlipList(ImportPath)
histogramFilename = os.path.join(ImportPath, nornir_buildmanager.importers.DefaultHistogramFilename)
ContrastMap = nornir_buildmanager.importers.LoadHistogramCutoffs(histogramFilename)
if len(ContrastMap) == 0:
nornir_buildmanager.importers.CreateDefaultHistogramCutoffFile(histogramFilename)
DirList = files.RecurseSubdirectoriesGenerator(ImportPath, RequiredFiles="*." + extension, ExcludeNames=[], ExcludedDownsampleLevels=[])
for path in DirList:
prettyoutput.CurseString("DM4Import", "Importing *.dm4 from {0}".format(path))
for idocFullPath in glob.glob(os.path.join(path, '*.dm4')):
yield from DigitalMicrograph4Import.ToMosaic(VolumeElement, idocFullPath, VolumeElement.FullPath, FlipList=FlipList, ContrastMap=ContrastMap, tile_overlap=tile_overlap)
nornir_pools.WaitOnAllPools()
for transform_fullpath in mosaics_loaded:
mosaicObj = mosaics_loaded[transform_fullpath]
mosaicObj.SaveToMosaicFile(transform_fullpath)
for transform_fullpath in transforms_changed:
transformObj = transforms_changed[transform_fullpath]
transformObj.ResetChecksum()
yield transformObj.Parent
def Write(cls,
OutfilePath,
Entries,
Flip=False,
Flop=False,
ImageSize=None,
Downsample=1):
'''
Creates a .mosaic file in the specified directory containing the specified
Dictionary (Entries) at the specified Downsample rate. Entries should have the key=filename
and values should be a tuple with pixel coordinates
Setting Flip to True will invert all X coordinates
Setting Flop to True will invert all Y coordinates
'''
if ImageSize is None:
ImageSize = [(4080, 4080)] * len(Entries)
elif not isinstance(ImageSize, list):
assert (len(ImageSize) == 2
) # Expect tuple or list indicating image size
ImageSize = ImageSize * len(Entries)
else:
# A list of two entries for the size
if len(ImageSize) == 2 and not isinstance(ImageSize[0], list):
ImageSize = [(ImageSize[0], ImageSize[1])] * len(Entries)
elif len(ImageSize) == len(Entries):
ImageSize = ImageSize
else:
raise Exception(
"Unexpected list format for ImageSize argument")
prettyoutput.CurseString('Stage', "WriteMosaicFile " + OutfilePath)
# TODO - find min/max values and center image in .mosaic file
minX = sys.maxsize
minY = sys.maxsize
maxX = -sys.maxsize - 1
maxY = -sys.maxsize - 1
keys = list(Entries.keys())
keys.sort()
for key in keys:
Coord = Entries[key]
if (Coord[0] < minX):
minX = Coord[0]
if (Coord[0] > maxX):
maxX = Coord[0]
if (Coord[1] < minY):
minY = Coord[1]
if (Coord[1] > maxY):
maxY = Coord[1]
prettyoutput.Log('MinX:\t' + str(minX) + '\tMaxX:\t' + str(maxX))
prettyoutput.Log('MinY:\t' + str(minY) + '\tMaxY:\t' + str(maxY))
# Write standard .mosiac file header
with open(OutfilePath, 'w+') as OutFile:
OutFile.write('format_version_number: 1\n')
OutFile.write('number_of_images: ' + str(len(Entries)) + '\n')
OutFile.write('pixel_spacing: ' + str(Downsample) + '\n')
OutFile.write('use_std_mask: 0\n')
# prettyoutput.Log( "Keys: " + str(Entries.keys())
keys = list(Entries.keys())
keys.sort()
for i, key in enumerate(keys):
Coord = Entries[key]
tilesize = ImageSize[i]
# prettyoutput.Log( str(key) + " : " + str(Coord)
# Centering is nice, but it seems to break multi mrc sections
# Coord = (Coord[0] - (minX + CenterXOffset), Coord[1] - (minY + CenterYOffset))
Coord = (Coord[0], Coord[1])
X = Coord[0]
Y = Coord[1]
if Flip:
Y = -Y
if Flop:
X = -X
# Remove dirname from key
key = os.path.basename(key)
outstr = 'image:\n' + key + '\n' + 'LegendrePolynomialTransform_double_2_2_1 vp 6 1 0 1 1 1 0 fp 4 ' + str(
X) + ' ' + str(Y) + ' ' + str(tilesize[0] / 2) + ' ' + str(
tilesize[1] / 2)
OutFile.write(outstr + '\n')
OutFile.close()
def MultipleIntensityAverageCmd(self):
return "MultipleIntensityAverage"
def StosBruteCmd(self):
# return "ir-stos-brute -sh 1 -clahe 2 -refine -cubic -regularize "
return "ir-stos-brute -sh 1 -refine -regularize "
def StosGridCmd(self, Spacing=None, Neighborhood=None):
if (Spacing is None):
Spacing = 128
if (Neighborhood is None):
Neighborhood = 128
return "ir-stos-grid -sh 1 -fft 0 0.25 -grid_spacing " + str(
Spacing) + " -neighborhood " + str(Neighborhood) + " -it 10 "
def StosAddTransform(self):
return "ir-add-transforms "
# Users calling this library can change this list to adjust the downsample levels used
Current = __Config()
if __name__ == '__main__':
try:
from nornir_shared import prettyoutput
prettyoutput.CurseString("# of Cores", str(Current.NumProcs))
except:
pass
def ConvertImagesInDict(ImagesToConvertDict, Flip=False, Flop=False, InputBpp=None, OutputBpp=None, Invert=False, bDeleteOriginal=False, RightLeftShift=None, AndValue=None, MinMax=None, Gamma=None):
'''
The key and value in the dictionary have the full path of an image to convert.
MinMax is a tuple [Min,Max] passed to the -level parameter if it is not None
RightLeftShift is a tuple containing a right then left then return to center shift which should be done to remove useless bits from the data
I do not use an and because I do not calculate ImageMagick's quantum size yet.
Every image must share the same colorspace
:return: True if images were converted
:rtype: bool
'''
if len(ImagesToConvertDict) == 0:
return False
if InputBpp is None:
for k in ImagesToConvertDict.keys():
if os.path.exists(k):
InputBpp = nornir_shared.images.GetImageBpp(k)
break
prettyoutput.CurseString('Stage', "ConvertImagesInDict")
if MinMax is not None:
if(MinMax[0] > MinMax[1]):
raise ValueError("Invalid MinMax parameter passed to ConvertImagesInDict")
pool = nornir_pools.GetMultithreadingPool("ConvertImagesInDict", num_threads=multiprocessing.cpu_count() * 2)
#pool = nornir_pools.GetGlobalSerialPool()
tasks = []
for (input_image, output_image) in ImagesToConvertDict.items():
task = pool.add_task("{0} -> {1}".format(input_image, output_image),
_ConvertSingleImageToFile,
input_image_param=input_image,
output_filename=output_image,
Flip=Flip,
Flop=Flop,
InputBpp=InputBpp,
OutputBpp=OutputBpp,
Invert=Invert,
MinMax=MinMax,
Gamma=Gamma)
tasks.append(task)
while len(tasks) > 0:
t = tasks.pop(0)
try:
t.wait()
except Exception as e:
prettyoutput.LogErr("Failed to convert " + t.name)
if bDeleteOriginal:
for (input_image, output_image) in ImagesToConvertDict.items():
if input_image != output_image:
pool.add_task("Delete {0}".format(input_image), os.remove, input_image)
while len(tasks) > 0:
t = tasks.pop(0)
try:
t.wait()
except OSError as e:
prettyoutput.LogErr("Unable to delete {0}\n{1}".format(t.name, e))
pass
except IOError as e:
prettyoutput.LogErr("Unable to delete {0}\n{1}".format(t.name, e))
pass
if not pool is None:
pool.wait_completion()
pool.shutdown()
pool = None
del tasks
def ToMosaic(cls, VolumeObj, idocFileFullPath, ContrastCutoffs, OutputImageExt=None, TargetBpp=None, FlipList=None, ContrastMap=None, CameraBpp=None, debug=None):
'''
This function will convert an idoc file in the given path to a .mosaic file.
It will also rename image files to the requested extension and subdirectory.
TargetBpp is calculated based on the number of bits required to encode the values
between the median min and max values
:param list FlipList: List of section numbers which should have images flipped
:param dict ContrastMap: Dictionary mapping section number to (Min, Max, Gamma) tuples
'''
if(OutputImageExt is None):
OutputImageExt = 'png'
if TargetBpp is None:
TargetBpp = 8
if FlipList is None:
FlipList = []
if ContrastMap is None:
ContrastMap = {}
SaveChannel = False
idocFilePath = serialem_utils.GetPathWithoutSpaces(idocFileFullPath)
OutputPath = VolumeObj.FullPath
os.makedirs(OutputPath, exist_ok=True)
logger = logging.getLogger(__name__ + '.' + str(cls.__name__) + "ToMosaic")
# Report the current stage to the user
prettyoutput.CurseString('Stage', "SerialEM to Mosaic " + str(idocFileFullPath))
SectionNumber = 0
sectionDir = os.path.dirname(idocFileFullPath) #serialem_utils.GetDirectories(idocFileFullPath)
BlockObj = BlockNode.Create('TEM')
[saveBlock, BlockObj] = VolumeObj.UpdateOrAddChild(BlockObj)
# If the parent directory doesn't have the section number in the name, change it
ExistingSectionInfo = shared.GetSectionInfo(sectionDir)
if(ExistingSectionInfo[0] < 0):
i = 5
# SectionNumber = SectionNumber + 1
# newPathName = ('%' + nornir_buildmanager.templates.Current.SectionFormat) % SectionNumber + '_' + sectionDir
# newPath = os.path.join(ParentDir, newPathName)
# prettyoutput.Log('Moving: ' + InputPath + ' -> ' + newPath)
# shutil.move(InputPath, newPath)
#
# InputPath = newPath
#
# #Run glob again because the dir changes
# idocFiles = glob.glob(os.path.join(InputPath,'*.idoc'))
else:
SectionNumber = ExistingSectionInfo.number
prettyoutput.CurseString('Section', str(SectionNumber))
# Check for underscores. If there is an underscore and the first part is the sectionNumber, then use everything after as the section name
SectionName = ('%' + nornir_buildmanager.templates.Current.SectionFormat) % ExistingSectionInfo.number
SectionPath = ('%' + nornir_buildmanager.templates.Current.SectionFormat) % ExistingSectionInfo.number
try:
parts = sectionDir.partition("_")
if not parts is None:
if len(parts[2]) > 0:
SectionName = parts[2]
except:
pass
sectionObj = SectionNode.Create(SectionNumber,
SectionName,
SectionPath)
[saveSection, sectionObj] = BlockObj.UpdateOrAddChildByAttrib(sectionObj, 'Number')
sectionObj.Name = SectionName
# Create a channel group
[saveChannel, channelObj] = sectionObj.UpdateOrAddChildByAttrib(ChannelNode.Create('TEM'), 'Name')
# Create a channel group for the section
# I started ignoring existing supertile.mosaic files so I could rebuild sections where
# a handful of tiles were corrupt
# if(os.path.exists(SupertilePath)):
# continue
Flip = SectionNumber in FlipList;
if(Flip):
prettyoutput.Log("Found in FlipList.txt, flopping images")
IDocData = IDoc.Load(idocFilePath, CameraBpp=CameraBpp)
assert(hasattr(IDocData, 'PixelSpacing'))
assert(hasattr(IDocData, 'DataMode'))
assert(hasattr(IDocData, 'ImageSize'))
# If there are no tiles... return
if IDocData.NumTiles == 0:
prettyoutput.Log("No tiles found in IDoc: " + idocFilePath)
return
# See if we can find a notes file...
shared.TryAddNotes(channelObj, sectionDir, logger)
serialem_utils.TryAddLogs(channelObj, sectionDir, logger)
AddIdocNode(channelObj, idocFilePath, IDocData, logger)
# Set the scale
[added, ScaleObj] = cls.CreateScaleNode(IDocData, channelObj)
# Parse the images
ImageBpp = IDocData.GetImageBpp()
if ImageBpp is None:
ImageBpp = cls.GetImageBpp(IDocData, sectionDir)
FilterName = 'Raw' + str(TargetBpp)
if(TargetBpp is None):
FilterName = 'Raw'
histogramFullPath = os.path.join(sectionDir, 'Histogram.xml')
IDocData.RemoveMissingTiles(sectionDir)
source_tile_list = [os.path.join(sectionDir, t.Image) for t in IDocData.tiles ]
(ActualMosaicMin, ActualMosaicMax, Gamma) = cls.GetSectionContrastSettings(SectionNumber, ContrastMap, ContrastCutoffs, source_tile_list, IDocData, histogramFullPath)
ActualMosaicMax = numpy.around(ActualMosaicMax)
ActualMosaicMin = numpy.around(ActualMosaicMin)
contrast_mismatch = channelObj.RemoveFilterOnContrastMismatch(FilterName, ActualMosaicMin, ActualMosaicMax, Gamma)
Pool = nornir_pools.GetGlobalThreadPool()
#_PlotHistogram(histogramFullPath, SectionNumber, ActualMosaicMin, ActualMosaicMax)
Pool.add_task(histogramFullPath, _PlotHistogram, histogramFullPath, SectionNumber, ActualMosaicMin, ActualMosaicMax, force_recreate=contrast_mismatch)
ImageConversionRequired = contrast_mismatch
# Create a channel for the Raw data
[added_filter, filterObj] = channelObj.UpdateOrAddChildByAttrib(FilterNode.Create(Name=FilterName), 'Name')
if added_filter:
ImageConversionRequired = True
filterObj.SetContrastValues(ActualMosaicMin, ActualMosaicMax, Gamma)
filterObj.BitsPerPixel = TargetBpp
SupertileName = 'Stage'
SupertileTransform = SupertileName + '.mosaic'
SupertilePath = os.path.join(channelObj.FullPath, SupertileTransform)
# Check to make sure our supertile mosaic file is valid
RemoveOutdatedFile(idocFilePath, SupertilePath)
[added_transform, transformObj] = channelObj.UpdateOrAddChildByAttrib(TransformNode.Create(Name=SupertileName,
Path=SupertileTransform,
Type='Stage'),
'Path')
[added_tilepyramid, PyramidNodeObj] = filterObj.UpdateOrAddChildByAttrib(TilePyramidNode.Create(Type='stage',
NumberOfTiles=IDocData.NumTiles),
'Path')
[added_level, LevelObj] = PyramidNodeObj.GetOrCreateLevel(1, GenerateData=False)
Tileset = NornirTileset.CreateTilesFromIDocTileData(IDocData.tiles, InputTileDir=sectionDir, OutputTileDir=LevelObj.FullPath, OutputImageExt=OutputImageExt)
# Make sure the target LevelObj is verified
if not os.path.exists(LevelObj.FullPath):
os.makedirs(LevelObj.FullPath, exist_ok=True)
else:
Tileset.RemoveStaleTilesFromOutputDir(SupertilePath=SupertilePath)
VerifyTiles(filterObj.TilePyramid.GetLevel(1))
SourceToMissingTargetMap = Tileset.GetSourceToMissingTargetMap()
# Figure out if we have to move or convert images
if len(SourceToMissingTargetMap) == 0:
ImageConversionRequired = False
else:
ImageConversionRequired = (not ImageBpp == TargetBpp) or (ImageConversionRequired or Tileset.ImageConversionRequired)
if(ImageConversionRequired):
Invert = False
filterObj.SetContrastValues(ActualMosaicMin, ActualMosaicMax, Gamma)
filterObj.TilePyramid.NumberOfTiles = IDocData.NumTiles
# andValue = cls.GetBitmask(ActualMosaicMin, ActualMosaicMax, TargetBpp)
#nornir_shared.images.ConvertImagesInDict(SourceToMissingTargetMap, Flip=Flip, Bpp=TargetBpp, Invert=Invert, bDeleteOriginal=False, MinMax=[ActualMosaicMin, ActualMosaicMax])
nornir_imageregistration.ConvertImagesInDict(SourceToMissingTargetMap, Flip=Flip, InputBpp=ImageBpp, OutputBpp=TargetBpp, Invert=Invert, bDeleteOriginal=False, MinMax=[ActualMosaicMin, ActualMosaicMax], Gamma=Gamma)
elif(Tileset.ImageMoveRequired):
for f in SourceToMissingTargetMap:
shutil.copy(f, SourceToMissingTargetMap[f])
# If we wrote new images replace the .mosaic file
if len(SourceToMissingTargetMap) > 0 or not os.path.exists(SupertilePath):
# Writing this file indicates import succeeded and we don't need to repeat these steps, writing it will possibly invalidate a lot of downstream data
# We need to flip the images. This may be a Utah scope issue, our Y coordinates are inverted relative to the images. To fix this
# we flop instead of flip and reverse when writing the coordinates
mosaicfile.MosaicFile.Write(SupertilePath, Entries=Tileset.GetPositionsForTargets(), Flip=not Flip, ImageSize=IDocData.ImageSize, Downsample=1);
MFile = mosaicfile.MosaicFile.Load(SupertilePath)
# Sometimes files fail to convert, when this occurs remove them from the .mosaic
if MFile.RemoveInvalidMosaicImages(LevelObj.FullPath):
MFile.Save(SupertilePath)
Mosaic.TranslateMosaicFileToZeroOrigin(SupertilePath)
transformObj.ResetChecksum()
SaveChannel = True
# transformObj.Checksum = MFile.Checksum
if saveBlock:
return VolumeObj
elif saveSection:
return BlockObj
elif saveChannel:
return sectionObj
elif added_transform or added_tilepyramid or added_level or ImageConversionRequired or SaveChannel or contrast_mismatch:
return channelObj
return None
def AssembleTilesetFromImageSet(Parameters,
ImageSetNode,
TileShape=None,
Logger=None,
**kwargs):
'''Create full resolution tiles of specfied size for the mosaics'''
prettyoutput.CurseString('Stage', "Assemble Tile Pyramids")
TileWidth = TileShape[0]
TileHeight = TileShape[1]
FilterNode = ImageSetNode.FindParent('Filter')
InputLevelNode = ImageSetNode.MaxResLevel
if InputLevelNode is None:
Logger.warning(
"No input image level found for AssembleTilesetFromImageSet")
return
# Remove our tileset if our image is newer than the tileset
ImageNode = ImageSetNode.GetImage(InputLevelNode.Downsample)
if ImageNode is None:
Logger.warning("No input image found for AssembleTilesetFromImageSet")
return
if FilterNode.HasTileset:
TileSetNode = FilterNode.Tileset
# if files.IsOutdated(ImageNode.FullPath, TileSetNode.Levels[0].FullPath):
# TileSetNode.Clean("Input image was newer than tileset")
# else:
# return
if not FilterNode.HasTileset:
TileSetNode = nornir_buildmanager.VolumeManager.TilesetNode.Create()
[added, TileSetNode
] = FilterNode.UpdateOrAddChildByAttrib(TileSetNode, 'Path')
TileSetNode.TileXDim = TileWidth
TileSetNode.TileYDim = TileHeight
TileSetNode.FilePostfix = '.png'
TileSetNode.FilePrefix = FilterNode.Name + '_'
TileSetNode.CoordFormat = nornir_buildmanager.templates.Current.GridTileCoordFormat
os.makedirs(TileSetNode.FullPath, exist_ok=True)
# OK, check if the first level of the tileset exists
(added_outputlevel,
OutputLevel) = TileSetNode.GetOrCreateLevel(InputLevelNode.Downsample,
GenerateData=False)
added_outputlevel = True
if (added_outputlevel):
[YDim, XDim] = nornir_shared.images.GetImageSize(ImageNode.FullPath)
tile_output = os.path.join(TileSetNode.FullPath, 'Tile%d.png')
# Need to call ir-assemble
cmd_template = 'magick convert %(InputPath)s -background black -crop %(TileSizeX)dx%(TileSizeY)d -depth 8 -quality 106 -type Grayscale -extent %(XDim)dx%(YDim)d %(TilePrefix)s'
cmd = cmd_template % {
'InputPath': ImageNode.FullPath,
'TileSizeX': TileWidth,
'TileSizeY': TileHeight,
'XDim': TileWidth,
'YDim': TileHeight,
'TilePrefix': tile_output
}
prettyoutput.CurseString('Cmd', cmd)
subprocess.call(cmd + ' && exit', shell=True)
FilePostfix = ''
GridDimY = int(math.ceil(YDim / float(TileHeight)))
GridDimX = int(math.ceil(XDim / float(TileWidth)))
GridTileNameTemplate = nornir_buildmanager.templates.Current.GridTileNameTemplate
os.makedirs(OutputLevel.FullPath, exist_ok=True)
iFile = 0
for iY in range(0, GridDimY):
for iX in range(0, GridDimX):
tileFileName = os.path.join(TileSetNode.FullPath,
tile_output % (iFile))
gridTileFileName = GridTileNameTemplate % {
'prefix': TileSetNode.FilePrefix,
'X': iX,
'Y': iY,
'postfix': TileSetNode.FilePostfix
}
gridTileFileName = os.path.join(OutputLevel.FullPath,
gridTileFileName)
shutil.move(tileFileName, gridTileFileName)
iFile = iFile + 1
OutputLevel.GridDimX = GridDimX
OutputLevel.GridDimY = GridDimY
return FilterNode