def RemoveInvalidMosaicImages(self, TileDir):
'''Runs a time consuming check that verifies all the image files are valid. Returns true if invalid images were found.'''
ImageList = []
ImageList.extend(list(self.ImageToTransformString.keys()))
InvalidImages = images.AreValidImages(ImageList, TileDir)
FoundInvalid = False
for InvalidImage in InvalidImages:
if InvalidImage in self.ImageToTransformString:
InvalidImageFullPath = os.path.join(TileDir, InvalidImage)
if os.path.exists(InvalidImageFullPath):
prettyoutput.Log('Removing invalid image from disk:')
os.remove(InvalidImageFullPath)
prettyoutput.Log('Removing invalid image from mosaic: %s' %
InvalidImage)
del self.ImageToTransformString[InvalidImage]
FoundInvalid = True
# prettyoutput.Log('Removing invalid image from mosaic: %s' % InvalidImage)
# del self.ImageToTransformString[InvalidImage]
return FoundInvalid
def CreateVikingXML(StosMapName=None,
StosGroupName=None,
OutputFile=None,
Host=None,
**kwargs):
'''When passed a volume node, creates a VikingXML file'''
InputVolumeNode = kwargs.get('VolumeNode')
path = InputVolumeNode.Path
if OutputFile is None:
OutputFile = "Volume.VikingXML"
if not OutputFile.lower().endswith('.vikingxml'):
OutputFile = OutputFile + ".VikingXML"
# Create our XML File
OutputXMLFilename = os.path.join(path, OutputFile)
# Load the inputXML file and begin parsing
# Create the root output node
OutputVolumeNode = ETree.Element(
'Volume', {
'Name': InputVolumeNode.Name,
'num_stos': '0',
'num_sections': '0',
'InputChecksum': InputVolumeNode.Checksum
})
(units_of_measure, units_per_pixel) = DetermineVolumeScale(InputVolumeNode)
if units_of_measure is not None:
AddScaleData(OutputVolumeNode, units_of_measure, units_per_pixel)
ParseSections(InputVolumeNode, OutputVolumeNode)
RemoveDuplicateScaleEntries(OutputVolumeNode, units_of_measure,
units_per_pixel)
ParseStos(InputVolumeNode, OutputVolumeNode, StosMapName, StosGroupName)
OutputXML = ETree.tostring(OutputVolumeNode).decode('utf-8')
# prettyoutput.Log(OutputXML)
hFile = open(OutputXMLFilename, 'w')
hFile.write(OutputXML)
hFile.close()
# Walk down to the path from the root directory, merging about.xml's as we go
Url = RecursiveMergeAboutXML(path, OutputXMLFilename)
if not Host is None and len(Host) > 0:
OutputVolumeNode.attrib['host'] = Host
prettyoutput.Log("Launch string:")
prettyoutput.Log(Url)
finalUrl = url_join(Url, OutputFile)
vikingUrl = "http://connectomes.utah.edu/Software/Viking4/viking.application?" + finalUrl
prettyoutput.Log(vikingUrl)
return
def TryAddNotes(containerObj, InputPath, logger):
'''Check the path for a notes.txt file. If found, add a <Notes> element to the passed containerObj'''
NotesFiles = glob.glob(os.path.join(InputPath, '*notes*.*'))
NotesAdded = False
if len(NotesFiles) > 0:
for filename in NotesFiles:
try:
from xml.sax.saxutils import escape
NotesFilename = os.path.basename(filename)
CopiedNotesFullPath = os.path.join(containerObj.FullPath,
NotesFilename)
if not os.path.exists(CopiedNotesFullPath):
os.makedirs(containerObj.FullPath, exist_ok=True)
shutil.copyfile(filename, CopiedNotesFullPath)
NotesAdded = True
with open(filename, 'r') as f:
notesTxt = f.read()
(base, ext) = os.path.splitext(filename)
encoding = "utf-8"
ext = ext.lower()
# notesTxt = notesTxt.encode(encoding)
notesTxt = notesTxt.replace('\0', '')
if len(notesTxt) > 0:
# XMLnotesTxt = notesTxt
# notesTxt = notesTxt.encode('utf-8')
XMLnotesTxt = escape(notesTxt)
# Create a Notes node to save the notes into
NotesNodeObj = NotesNode.Create(
Text=XMLnotesTxt, SourceFilename=NotesFilename)
containerObj.RemoveOldChildrenByAttrib(
'Notes', 'SourceFilename', NotesFilename)
[added,
NotesNodeObj] = containerObj.UpdateOrAddChildByAttrib(
NotesNodeObj, 'SourceFilename')
if added:
# Try to copy the notes to the output dir if we created a node
if not os.path.exists(CopiedNotesFullPath):
shutil.copyfile(filename, CopiedNotesFullPath)
NotesNodeObj.text = XMLnotesTxt
NotesNodeObj.encoding = encoding
NotesAdded = NotesAdded or added
except:
(etype, evalue, etraceback) = sys.exc_info()
prettyoutput.Log("Attempt to include notes from " + filename +
" failed.\n" + evalue.message)
prettyoutput.Log(etraceback)
return NotesAdded
def PrintPipelineEnumeration(cls, PipelineXML):
PipelineXML = cls.LoadPipelineXML(PipelineXML)
cls.logger.info("Enumerating available pipelines")
prettyoutput.Log("Enumerating available pipelines")
for pipeline in PipelineXML.getroot():
cls.logger.info(' ' + pipeline.attrib.get('Name', ""))
prettyoutput.Log(' ' + pipeline.attrib.get('Name', ""))
prettyoutput.Log(' ' + pipeline.attrib.get('Description', "") + '\n')
def FromXML(self, xml):
obj = Histogram()
if isinstance(xml, str):
xmlDoc = xml.dom.minidom.parseString(xml)
else:
xmlDoc = xml
#else:
#raise InvalidOperation("xml parameter was not a string or XML minidom object")
Elems = xmlDoc.getElementsByTagName('Histogram')
if (len(Elems) != 1):
prettyoutput.Log("Histogram tag not found in histogram XML")
return
HistogramElem = Elems[0]
if (HistogramElem.hasAttribute('NumBins')):
obj.NumBins = int(HistogramElem.getAttribute('NumBins'))
if (HistogramElem.hasAttribute('NumSamples')):
obj.NumSamples = int(HistogramElem.getAttribute('NumSamples'))
if (HistogramElem.hasAttribute('MinValue')):
obj.MinValue = float(HistogramElem.getAttribute('MinValue'))
if (HistogramElem.hasAttribute('MaxValue')):
obj.MaxValue = float(HistogramElem.getAttribute('MaxValue'))
ChannelElems = HistogramElem.getElementsByTagName('Channel')
if (len(ChannelElems) == 0):
prettyoutput.Log("No channel element found in histogram")
return
ChannelElem = ChannelElems[0]
BinNode = ChannelElem.firstChild
BinString = BinNode.data
BinStrings = BinString.split()
obj.Bins = list()
for i in range(0, len(BinStrings)):
obj.Bins.append(int(BinStrings[i]))
if (len(obj.Bins) != obj.NumBins):
prettyoutput.Log("ERROR: obj.Bins != obj.NumBins")
prettyoutput.Log(str(obj))
return
return obj
def GetImageBpp(cls, IDocData, sectionDir):
ImageBpp = IDocData.GetImageBpp()
if ImageBpp is None:
# Figure out the bit depth of the input
tile = IDocData.tiles[0]
SourceImageFullPath = os.path.join(sectionDir, tile.Image)
ImageBpp = GetImageBpp(SourceImageFullPath)
if(not ImageBpp is None):
prettyoutput.Log("Source images are " + str(ImageBpp) + " bits per pixel")
else:
prettyoutput.Log("Could not determine source image BPP")
raise ValueError("Could not determine source image BPP")
return ImageBpp
def _GetMinMaxCutoffs(listfilenames, MinCutoff, MaxCutoff, idoc_data, histogramFullPath=None):
histogramObj = None
if not histogramFullPath is None:
if os.path.exists(histogramFullPath):
histogramObj = Histogram.Load(histogramFullPath)
if histogramObj is None:
prettyoutput.Log("Collecting mosaic min/max data")
Bpp = idoc_data.GetImageBpp()
if Bpp is None:
Bpp = nornir_shared.images.GetImageBpp(listfilenames[0])
numBins = 256
if Bpp >= 11:
numBins = 2048
maxVal = idoc_data.Max
if idoc_data.CameraBpp is not None:
if (1 << idoc_data.CameraBpp) - 1 < maxVal:
maxVal = (1 << idoc_data.CameraBpp) - 1
histogramObj = image_stats.Histogram(listfilenames, Bpp=Bpp, MinVal=idoc_data.Min, MaxVal=idoc_data.Max, numBins=numBins)
if not histogramFullPath is None:
histogramObj = _CleanOutliersFromIDocHistogram(histogramObj)
histogramObj.Save(histogramFullPath)
assert(not histogramObj is None)
# I am willing to clip 1 pixel every hundred thousand on the dark side, and one every ten thousand on the light
return histogramObj.AutoLevel(MinCutoff, MaxCutoff)
def WritePruneMap(self, MapImageToScoreFile):
if len(self.MapImageToScore) == 0:
prettyoutput.LogErr('No prune scores to write to file ' +
MapImageToScoreFile)
return
with open(MapImageToScoreFile, 'w') as outfile:
if (len(list(self.MapImageToScore.keys())) == 0):
if (os.path.exists(MapImageToScoreFile)):
os.remove(MapImageToScoreFile)
prettyoutput.Log(
"No prune scores present in PruneMap being saved: " +
MapImageToScoreFile)
outfile.close()
return
for f in sorted(self.MapImageToScore.keys()):
score = self.MapImageToScore[f]
outfile.write(f + '\t' + str(score) + '\n')
outfile.close()
def CreateTilesFromIDocTileData(cls, tiles, InputTileDir, OutputTileDir, OutputImageExt):
'''
:param tiles IDocTileData: List of tiles to build dictionaries from
'''
# SerialEM begins numbering file names from zero. So we will too.
ImageNumber = -1
obj = NornirTileset(OutputImageExt)
for tile in tiles:
[ImageRoot, ImageExt] = os.path.splitext(tile.Image)
ImageExt = ImageExt.strip('.')
ImageExt = ImageExt.lower()
ImageNumber = ImageNumber + 1
SourceImageFullPath = os.path.join(InputTileDir, tile.Image)
if not os.path.exists(SourceImageFullPath):
prettyoutput.Log("Could not locate import image: " + SourceImageFullPath)
obj.MissingInputImage = True
continue
# I rename the converted image because I haven't checked how robust viking is with non-numbered images. I'm 99% sure it can handle it, but I don't want to test now.
ConvertedImageName = (nornir_buildmanager.templates.Current.TileCoordFormat % ImageNumber) + '.' + OutputImageExt
TargetImageFullPath = os.path.join(OutputTileDir, ConvertedImageName)
obj.AddTile(NornirTileset.Tile(SourceImageFullPath, TargetImageFullPath, Position=tile.PieceCoordinates[0:2]))
return obj
def Execute(self, args):
'''This executes the loaded pipeline on the specified volume of data.
parser is an instance of the argparser class which should be
extended with any pipeline specific arguments args are the parameters from the command line'''
# DOM = self.PipelineData.toDOM()
# PipelineElement = DOM.firstChild
ArgSet = ArgumentSet()
PipelineElement = self.PipelineData
prettyoutput.Log("Adding pipeline arguments")
# parser = self.GetArgParser(parser)
# (args, unused) = parser.parse_known_args(passedArgs)
ArgSet.AddArguments(args)
ArgSet.AddParameters(PipelineElement)
# Load the Volume.XML file in the output directory
self.VolumeTree = VolumeManagerETree.VolumeManager.Load(args.volumepath, Create=True)
if(self.VolumeTree is None):
PipelineManager.logger.critical("Could not load or create volume.xml " + args.outputpath)
prettyoutput.LogErr("Could not load or create volume.xml " + args.outputpath)
sys.exit()
# dargs = copy.deepcopy(defaultDargs)
self.ExecuteChildPipelines(ArgSet, self.VolumeTree, PipelineElement)
nornir_pools.WaitOnAllPools()
def TryAddLogs(containerObj, InputPath, logger):
'''Copy log files to output directories, and store select meta-data in the containerObj if it exists'''
LogsFiles = glob.glob(os.path.join(InputPath, '*.log'))
LogsAdded = False
if len(LogsFiles) == 0:
print("NO LOG FILE FOUND FOR CAPTURE: " % InputPath)
elif len(LogsFiles) > 0:
for filename in LogsFiles:
NotesFilename = os.path.basename(filename)
CopiedLogsFullPath = os.path.join(containerObj.FullPath,
NotesFilename)
if not os.path.exists(CopiedLogsFullPath):
os.makedirs(containerObj.FullPath, exist_ok=True)
shutil.copyfile(filename, CopiedLogsFullPath)
LogsAdded = True
# OK, try to parse the logs
try:
LogData = serialemlog.SerialEMLog.Load(filename)
if LogData is None:
continue
if LogData.NumTiles == 0:
continue
# Create a Notes node to save the logs into
LogNodeObj = DataNode.Create(Path=NotesFilename,
attrib={'Name': 'Log'})
containerObj.RemoveOldChildrenByAttrib('Data', 'Name', 'Log')
[added, LogNodeObj
] = containerObj.UpdateOrAddChildByAttrib(LogNodeObj, 'Name')
LogsAdded = LogsAdded or added
LogNodeObj.AverageTileTime = '%g' % LogData.AverageTileTime
LogNodeObj.AverageTileDrift = '%g' % LogData.AverageTileDrift
LogNodeObj.CaptureTime = '%g' % (LogData.MontageEnd -
LogData.MontageStart)
except:
(etype, evalue, etraceback) = sys.exc_info()
prettyoutput.Log("Attempt to include logs from " + filename +
" failed.\n" + str(evalue))
prettyoutput.Log(str(etraceback))
return LogsAdded
def GetInfo(cls, filename):
'''Parses a filename and returns what we have learned if it follows the convention
<SECTION>_<CHANNEL>_<MosaicSource>_<DOWNSAMPLE>.mosaic
[SectionNumber, Channel, MosaicSource, Downsample]'''
if (os.path.exists(filename) == False):
prettyoutput.LogErr("mosaic file not found: " + filename)
return
baseName = os.path.basename(filename)
[baseName, ext] = os.path.splitext(baseName)
parts = baseName.split("_")
try:
SectionNumber = int(parts[0])
except:
# We really can't recover from this, so maybe an exception should be thrown instead
SectionNumber = None
prettyoutput.Log('Could not determine mosaic section number: ' +
str(filename))
try:
MosaicType = parts[-2]
except:
MosaicType = None
prettyoutput.Log('Could not determine mosaic source: ' +
str(filename))
if (len(parts) > 3):
try:
Channel = parts[1]
except:
Channel = None
prettyoutput.Log('Could not determine mosaic channel: ' +
str(filename))
# If we don't have a valid downsample value we assume 1
try:
DownsampleStrings = parts[-1].split(".")
Downsample = int(DownsampleStrings[0])
except:
Downsample = None
prettyoutput.Log('Could not determine mosaic downsample: ' +
str(filename))
return [SectionNumber, Channel, MosaicType, Downsample]
def CreateHistogram(self, HistogramXMLFile, MapImageToScoreFile=None):
if (len(list(self.MapImageToScore.items())) == 0
and MapImageToScoreFile is not None):
# prettyoutput.Log( "Reading scores, MapImageToScore Empty " + MapImageToScoreFile)
PruneObj.ReadPruneMap(MapImageToScoreFile)
# prettyoutput.Log( "Read scores complete: " + str(self.MapImageToScore))
if (len(list(self.MapImageToScore.items())) == 0):
prettyoutput.Log("No prune scores to create histogram with")
return
scores = [None] * len(list(self.MapImageToScore.items()))
numScores = len(scores)
i = 0
for pair in list(self.MapImageToScore.items()):
# prettyoutput.Log("pair: " + str(pair))
scores[i] = pair[1]
i = i + 1
# Figure out what type of histogram we should create
# prettyoutput.Log('Scores: ' + str(scores))
minVal = min(scores)
# prettyoutput.Log("MinVal: " + str(minVal))
maxVal = max(scores)
# prettyoutput.Log("MaxVal: " + str(maxVal))
mean = sum(scores) / len(scores)
# prettyoutput.Log("Mean: " + str(mean))
StdDevScalar = 1.0 / float(numScores - 1.0)
total = 0
# Calc the std deviation
for score in scores:
temp = score - mean
temp = temp * temp
total = total + (temp * StdDevScalar)
StdDev = math.sqrt(total)
# prettyoutput.Log("StdDev: " + str(StdDev))
numBins = int(math.ceil((maxVal - minVal) / (StdDev / 10.0)))
# prettyoutput.Log("NumBins: " + str(numBins))
if (numBins < 10):
numBins = 10
if (numBins > len(scores)):
numBins = len(scores)
# prettyoutput.Log("Final NumBins: " + str(numBins))
H = Histogram.Init(minVal, maxVal, numBins)
H.Add(scores)
H.Save(HistogramXMLFile)
print("Created Histogram %s" % HistogramXMLFile)
def WritePruneMosaic(self,
path,
SourceMosaic,
TargetMosaic='prune.mosaic',
Tolerance='5'):
'''
Remove tiles from the source mosaic with scores less than Tolerance and
write the new mosaic to TargetMosaic.
Raises a key error if image in prude scores does not exist in .mosaic file
Raises a value error if the threshold removes all tiles in the mosaic.
:return: Number of tiles removed
'''
if (not isinstance(Tolerance, float)):
Tolerance = float(Tolerance)
SourceMosaicFullPath = os.path.join(path, SourceMosaic)
TargetMosaicFullPath = os.path.join(path, TargetMosaic)
mosaic = mosaicfile.MosaicFile.Load(SourceMosaicFullPath)
# We copy this because if an input image is missing there will not be a prune score and it should be removed from the .mosaic file
inputImageToTransforms = copy.deepcopy(mosaic.ImageToTransformString)
mosaic.ImageToTransformString.clear()
numRemoved = 0
for item in list(self.MapImageToScore.items()):
filename = item[0]
score = item[1]
if (score >= Tolerance):
keyVal = filename
if not keyVal in inputImageToTransforms:
keyVal = os.path.basename(filename)
if not keyVal in inputImageToTransforms:
raise KeyError(
"PruneObj: Cannot locate image file in .mosaic " +
keyVal)
mosaic.ImageToTransformString[keyVal] = inputImageToTransforms[
keyVal]
else:
numRemoved = numRemoved + 1
if (len(mosaic.ImageToTransformString) <= 0):
errMsg = "All tiles removed when using threshold = " + str(
Tolerance) + "\nThe prune request was ignored"
prettyoutput.LogErr(errMsg)
raise ValueError(errMsg)
else:
prettyoutput.Log("Removed " + str(numRemoved) +
" tiles pruning mosaic " + TargetMosaic)
mosaic.Save(TargetMosaicFullPath)
return numRemoved
def Execute(buildArgs=None):
# Spend more time on each thread before switching
#sys.setswitchinterval(500)
if buildArgs is None:
buildArgs = sys.argv[1:]
InitLogging(buildArgs)
Timer = TaskTimer()
parser = BuildParserRoot()
args = parser.parse_args(buildArgs)
if args.lowpriority:
lowpriority()
print(
"Warning, using low priority flag. This can make builds much slower"
)
# SetupLogging(OutputPath=args.volumepath)
try:
Timer.Start(args.PipelineName)
args.func(args)
Timer.End(args.PipelineName)
finally:
OutStr = str(Timer)
prettyoutput.Log(OutStr)
timeTextFullPath = os.path.join(args.volumepath, 'Timing.txt')
try:
with open(timeTextFullPath, 'w+') as OutputFile:
OutputFile.writelines(OutStr)
OutputFile.close()
except:
prettyoutput.Log('Could not write %s' % (timeTextFullPath))
def MergeAboutXML(volumeXML, aboutXML):
import xml.dom.minidom
prettyoutput.Log('MergeAboutXML ' + str(volumeXML) + ' ' + str(aboutXML))
if (os.path.exists(volumeXML) == False):
return
if (os.path.exists(aboutXML) == False):
return
aboutDom = xml.dom.minidom.parse(aboutXML)
volumeDom = xml.dom.minidom.parse(volumeXML)
# Figure out which elements are contained in the about dom which need to be injected into the volumeXML
# If element names match, attributes are added which are missing from the volumeXML.
# If element names do not match, they are injected into the volumeXML at the appropriate level
aboutNode = aboutDom.documentElement
volumeNode = volumeDom.documentElement
Url = None
# Volume path is a special case so we append the path to the host name
if (volumeNode.nodeName == "Volume" and aboutNode.nodeName == "Volume"):
baseVolumeDir = os.path.dirname(volumeXML)
baseAboutDir = os.path.dirname(aboutXML)
relPath = baseVolumeDir.replace(baseAboutDir, '')
prettyoutput.Log("Relative path: " + relPath)
Url = UpdateVolumePath(volumeNode, aboutNode, relPath)
MergeElements(volumeNode, aboutNode)
prettyoutput.Log("")
xmlFile = open(volumeXML, "w")
xmlFile.write(volumeDom.toxml())
xmlFile.close()
return Url
def ElementsEqual(volumeElement, aboutElement):
'''Return true if the elements have the same tag, and the attributes found in both elements have same value'''
if (aboutElement.nodeName != volumeElement.nodeName):
return False
for attrib_key in list(aboutElement.attributes.keys()):
if not (volumeElement.hasAttribute(attrib_key)
and aboutElement.hasAttribute(attrib_key)):
continue
if not volumeElement.getAttribute(
attrib_key) == aboutElement.getAttribute(attrib_key):
return False
return True
# Volume is the root element so it is always a match
if (aboutElement.nodeName == "Volume"):
return True
# Nodes only match if their attributes match
if (aboutElement.nodeName == "Section"):
aboutNumber = aboutElement.getAttribute("number")
volNumber = volumeElement.getAttribute("number")
if (aboutNumber != volNumber):
return False
else:
prettyoutput.Log("Equal:")
prettyoutput.Log('v: ' + volumeElement.nodeName + ' ' +
str(volNumber))
prettyoutput.Log('a: ' + aboutElement.nodeName + ' ' +
str(aboutNumber))
prettyoutput.Log('')
return True
return False
def MergeChildren(volumeParent, aboutParent):
aboutElement = aboutParent.firstChild
while (aboutElement is not None):
if (aboutElement.nodeName is None):
break
volumeElements = volumeParent.getElementsByTagName(
aboutElement.nodeName)
# The volume doesn't have any elements like this. Add them
if (volumeElements.length == 0):
newNode = aboutElement.cloneNode(True)
prettyoutput.Log('NewNode' + newNode.toxml())
volumeParent.insertBefore(newNode, volumeParent.firstChild)
else:
for volElement in volumeElements:
MergeElements(volElement, aboutElement)
aboutElement = aboutElement.nextSibling
def Load(cls, PipelineXml, PipelineName=None):
# PipelineData = Pipelines.CreateFromDOM(XMLDoc)
SelectedPipeline = None
XMLDoc = cls.LoadPipelineXML(PipelineXml)
if PipelineName is None:
PipelineManager.logger.warning("No pipeline name specified.")
prettyoutput.Log("No pipeline name specified")
cls.PrintPipelineEnumeration(XMLDoc)
return None
else:
SelectedPipeline = XMLDoc.find("Pipeline[@Name='" + PipelineName + "']")
if(SelectedPipeline is None):
PipelineManager.logger.critical("No pipeline found named " + PipelineName)
prettyoutput.LogErr("No pipeline found named " + PipelineName)
cls.PrintPipelineEnumeration(XMLDoc)
return None
return PipelineManager(pipelinesRoot=XMLDoc.getroot(), pipelineData=SelectedPipeline)
def Load(filename):
if not os.path.exists(filename):
PrettyOutput.Log("Stos file not found: " + filename)
return None
obj = StosFile()
try:
[
obj.MappedSectionNumber, obj.ControlSectionNumber, Channels,
Filters, obj.StosSource, obj._Downsample
] = StosFile.GetInfo(filename)
except:
pass
with open(filename, 'r') as fMosaic:
lines = fMosaic.readlines()
if len(lines) < 7:
PrettyOutput.LogErr("%s is not a valid stos file" % (filename))
raise ValueError("%s is not a valid stos file" % (filename))
obj.ControlImageFullPath = lines[0].strip()
obj.MappedImageFullPath = lines[1].strip()
ControlDims = lines[4].split()
MappedDims = lines[5].split()
obj.ControlImageDim = [float(x) for x in ControlDims]
obj.MappedImageDim = [float(x) for x in MappedDims]
obj.Transform = lines[6].strip()
if len(lines) > 8:
obj.ControlMaskFullPath = lines[8]
obj.MappedMaskFullPath = lines[9]
return obj
def EqualizeStosGridPixelSpacing(self,
control_spacing,
mapped_spacing,
MappedImageFullPath,
MappedMaskFullPath,
create_copy=True):
'''
Used to correct a mismatch between pixel spacings of the mapped and control images in a stos file.
This was originally occuring when aligning light microscopy images to TEM images.
Nornir expects the spacings for Stos files to be equal.
This function is implemented to keep the control spacing the same and adjust the mapped spacing to match.
Stos files have no way to encode the spacing in the file itself unfortunately.
'''
if control_spacing == mapped_spacing:
if create_copy:
return copy.deepcopy(self)
else:
return self
PrettyOutput.Log(
"ChangeStosGridPixelSpacing from {0:d} to {1:d}".format(
mapped_spacing, control_spacing))
control_spacing = float(control_spacing)
mapped_spacing = float(mapped_spacing)
mapped_space_scalar = mapped_spacing / control_spacing
NewStosFile = StosFile()
NewStosFile.ControlImageDim = copy.copy(self.ControlImageDim)
# NewStosFile.MappedImageDim = [x * scale for x in self.MappedImageDim]
# Update the filenames which are the first two lines of the file
NewStosFile.MappedImageDim = copy.copy(self.MappedImageDim)
NewStosFile.MappedImageDim[
2] = self.MappedImageDim[2] * mapped_space_scalar
NewStosFile.MappedImageDim[
3] = self.MappedImageDim[3] * mapped_space_scalar
NewStosFile.ControlImageFullPath = self.ControlImageFullPath
NewStosFile.ControlMaskFullPath = self.ControlMaskFullPath
NewStosFile.MappedImageFullPath = MappedImageFullPath
NewStosFile.MappedMaskFullPath = MappedMaskFullPath
if os.path.exists(MappedImageFullPath):
NewStosFile.MappedImageDim = StosFile.__GetImageDimsArray(
MappedImageFullPath)
# Adjust the transform points
transformObj = nornir_imageregistration.transforms.LoadTransform(
self.Transform, pixelSpacing=1.0)
assert (not transformObj is None)
transformObj.ScaleWarped(scalar=mapped_space_scalar)
NewStosFile._Downsample = control_spacing
if hasattr(transformObj, 'gridWidth'):
# Save as a stos grid if we can
bounds = (NewStosFile.MappedImageDim[1],
NewStosFile.MappedImageDim[0],
NewStosFile.MappedImageDim[3],
NewStosFile.MappedImageDim[2])
NewStosFile.Transform = nornir_imageregistration.transforms.TransformToIRToolsString(
transformObj, bounds=bounds)
else:
NewStosFile.Transform = nornir_imageregistration.transforms.TransformToIRToolsString(
transformObj) # , bounds=NewStosFile.MappedImageDim)
return NewStosFile
def ToMosaic(cls, VolumeObj, idocFileFullPath, ContrastCutoffs, OutputImageExt=None, TargetBpp=None, FlipList=None, ContrastMap=None, CameraBpp=None, debug=None):
'''
This function will convert an idoc file in the given path to a .mosaic file.
It will also rename image files to the requested extension and subdirectory.
TargetBpp is calculated based on the number of bits required to encode the values
between the median min and max values
:param list FlipList: List of section numbers which should have images flipped
:param dict ContrastMap: Dictionary mapping section number to (Min, Max, Gamma) tuples
'''
if(OutputImageExt is None):
OutputImageExt = 'png'
if TargetBpp is None:
TargetBpp = 8
if FlipList is None:
FlipList = []
if ContrastMap is None:
ContrastMap = {}
SaveChannel = False
idocFilePath = serialem_utils.GetPathWithoutSpaces(idocFileFullPath)
OutputPath = VolumeObj.FullPath
os.makedirs(OutputPath, exist_ok=True)
logger = logging.getLogger(__name__ + '.' + str(cls.__name__) + "ToMosaic")
# Report the current stage to the user
prettyoutput.CurseString('Stage', "SerialEM to Mosaic " + str(idocFileFullPath))
SectionNumber = 0
sectionDir = os.path.dirname(idocFileFullPath) #serialem_utils.GetDirectories(idocFileFullPath)
BlockObj = BlockNode.Create('TEM')
[saveBlock, BlockObj] = VolumeObj.UpdateOrAddChild(BlockObj)
# If the parent directory doesn't have the section number in the name, change it
ExistingSectionInfo = shared.GetSectionInfo(sectionDir)
if(ExistingSectionInfo[0] < 0):
i = 5
# SectionNumber = SectionNumber + 1
# newPathName = ('%' + nornir_buildmanager.templates.Current.SectionFormat) % SectionNumber + '_' + sectionDir
# newPath = os.path.join(ParentDir, newPathName)
# prettyoutput.Log('Moving: ' + InputPath + ' -> ' + newPath)
# shutil.move(InputPath, newPath)
#
# InputPath = newPath
#
# #Run glob again because the dir changes
# idocFiles = glob.glob(os.path.join(InputPath,'*.idoc'))
else:
SectionNumber = ExistingSectionInfo.number
prettyoutput.CurseString('Section', str(SectionNumber))
# Check for underscores. If there is an underscore and the first part is the sectionNumber, then use everything after as the section name
SectionName = ('%' + nornir_buildmanager.templates.Current.SectionFormat) % ExistingSectionInfo.number
SectionPath = ('%' + nornir_buildmanager.templates.Current.SectionFormat) % ExistingSectionInfo.number
try:
parts = sectionDir.partition("_")
if not parts is None:
if len(parts[2]) > 0:
SectionName = parts[2]
except:
pass
sectionObj = SectionNode.Create(SectionNumber,
SectionName,
SectionPath)
[saveSection, sectionObj] = BlockObj.UpdateOrAddChildByAttrib(sectionObj, 'Number')
sectionObj.Name = SectionName
# Create a channel group
[saveChannel, channelObj] = sectionObj.UpdateOrAddChildByAttrib(ChannelNode.Create('TEM'), 'Name')
# Create a channel group for the section
# I started ignoring existing supertile.mosaic files so I could rebuild sections where
# a handful of tiles were corrupt
# if(os.path.exists(SupertilePath)):
# continue
Flip = SectionNumber in FlipList;
if(Flip):
prettyoutput.Log("Found in FlipList.txt, flopping images")
IDocData = IDoc.Load(idocFilePath, CameraBpp=CameraBpp)
assert(hasattr(IDocData, 'PixelSpacing'))
assert(hasattr(IDocData, 'DataMode'))
assert(hasattr(IDocData, 'ImageSize'))
# If there are no tiles... return
if IDocData.NumTiles == 0:
prettyoutput.Log("No tiles found in IDoc: " + idocFilePath)
return
# See if we can find a notes file...
shared.TryAddNotes(channelObj, sectionDir, logger)
serialem_utils.TryAddLogs(channelObj, sectionDir, logger)
AddIdocNode(channelObj, idocFilePath, IDocData, logger)
# Set the scale
[added, ScaleObj] = cls.CreateScaleNode(IDocData, channelObj)
# Parse the images
ImageBpp = IDocData.GetImageBpp()
if ImageBpp is None:
ImageBpp = cls.GetImageBpp(IDocData, sectionDir)
FilterName = 'Raw' + str(TargetBpp)
if(TargetBpp is None):
FilterName = 'Raw'
histogramFullPath = os.path.join(sectionDir, 'Histogram.xml')
IDocData.RemoveMissingTiles(sectionDir)
source_tile_list = [os.path.join(sectionDir, t.Image) for t in IDocData.tiles ]
(ActualMosaicMin, ActualMosaicMax, Gamma) = cls.GetSectionContrastSettings(SectionNumber, ContrastMap, ContrastCutoffs, source_tile_list, IDocData, histogramFullPath)
ActualMosaicMax = numpy.around(ActualMosaicMax)
ActualMosaicMin = numpy.around(ActualMosaicMin)
contrast_mismatch = channelObj.RemoveFilterOnContrastMismatch(FilterName, ActualMosaicMin, ActualMosaicMax, Gamma)
Pool = nornir_pools.GetGlobalThreadPool()
#_PlotHistogram(histogramFullPath, SectionNumber, ActualMosaicMin, ActualMosaicMax)
Pool.add_task(histogramFullPath, _PlotHistogram, histogramFullPath, SectionNumber, ActualMosaicMin, ActualMosaicMax, force_recreate=contrast_mismatch)
ImageConversionRequired = contrast_mismatch
# Create a channel for the Raw data
[added_filter, filterObj] = channelObj.UpdateOrAddChildByAttrib(FilterNode.Create(Name=FilterName), 'Name')
if added_filter:
ImageConversionRequired = True
filterObj.SetContrastValues(ActualMosaicMin, ActualMosaicMax, Gamma)
filterObj.BitsPerPixel = TargetBpp
SupertileName = 'Stage'
SupertileTransform = SupertileName + '.mosaic'
SupertilePath = os.path.join(channelObj.FullPath, SupertileTransform)
# Check to make sure our supertile mosaic file is valid
RemoveOutdatedFile(idocFilePath, SupertilePath)
[added_transform, transformObj] = channelObj.UpdateOrAddChildByAttrib(TransformNode.Create(Name=SupertileName,
Path=SupertileTransform,
Type='Stage'),
'Path')
[added_tilepyramid, PyramidNodeObj] = filterObj.UpdateOrAddChildByAttrib(TilePyramidNode.Create(Type='stage',
NumberOfTiles=IDocData.NumTiles),
'Path')
[added_level, LevelObj] = PyramidNodeObj.GetOrCreateLevel(1, GenerateData=False)
Tileset = NornirTileset.CreateTilesFromIDocTileData(IDocData.tiles, InputTileDir=sectionDir, OutputTileDir=LevelObj.FullPath, OutputImageExt=OutputImageExt)
# Make sure the target LevelObj is verified
if not os.path.exists(LevelObj.FullPath):
os.makedirs(LevelObj.FullPath, exist_ok=True)
else:
Tileset.RemoveStaleTilesFromOutputDir(SupertilePath=SupertilePath)
VerifyTiles(filterObj.TilePyramid.GetLevel(1))
SourceToMissingTargetMap = Tileset.GetSourceToMissingTargetMap()
# Figure out if we have to move or convert images
if len(SourceToMissingTargetMap) == 0:
ImageConversionRequired = False
else:
ImageConversionRequired = (not ImageBpp == TargetBpp) or (ImageConversionRequired or Tileset.ImageConversionRequired)
if(ImageConversionRequired):
Invert = False
filterObj.SetContrastValues(ActualMosaicMin, ActualMosaicMax, Gamma)
filterObj.TilePyramid.NumberOfTiles = IDocData.NumTiles
# andValue = cls.GetBitmask(ActualMosaicMin, ActualMosaicMax, TargetBpp)
#nornir_shared.images.ConvertImagesInDict(SourceToMissingTargetMap, Flip=Flip, Bpp=TargetBpp, Invert=Invert, bDeleteOriginal=False, MinMax=[ActualMosaicMin, ActualMosaicMax])
nornir_imageregistration.ConvertImagesInDict(SourceToMissingTargetMap, Flip=Flip, InputBpp=ImageBpp, OutputBpp=TargetBpp, Invert=Invert, bDeleteOriginal=False, MinMax=[ActualMosaicMin, ActualMosaicMax], Gamma=Gamma)
elif(Tileset.ImageMoveRequired):
for f in SourceToMissingTargetMap:
shutil.copy(f, SourceToMissingTargetMap[f])
# If we wrote new images replace the .mosaic file
if len(SourceToMissingTargetMap) > 0 or not os.path.exists(SupertilePath):
# Writing this file indicates import succeeded and we don't need to repeat these steps, writing it will possibly invalidate a lot of downstream data
# We need to flip the images. This may be a Utah scope issue, our Y coordinates are inverted relative to the images. To fix this
# we flop instead of flip and reverse when writing the coordinates
mosaicfile.MosaicFile.Write(SupertilePath, Entries=Tileset.GetPositionsForTargets(), Flip=not Flip, ImageSize=IDocData.ImageSize, Downsample=1);
MFile = mosaicfile.MosaicFile.Load(SupertilePath)
# Sometimes files fail to convert, when this occurs remove them from the .mosaic
if MFile.RemoveInvalidMosaicImages(LevelObj.FullPath):
MFile.Save(SupertilePath)
Mosaic.TranslateMosaicFileToZeroOrigin(SupertilePath)
transformObj.ResetChecksum()
SaveChannel = True
# transformObj.Checksum = MFile.Checksum
if saveBlock:
return VolumeObj
elif saveSection:
return BlockObj
elif saveChannel:
return sectionObj
elif added_transform or added_tilepyramid or added_level or ImageConversionRequired or SaveChannel or contrast_mismatch:
return channelObj
return None
def Write(cls,
OutfilePath,
Entries,
Flip=False,
Flop=False,
ImageSize=None,
Downsample=1):
'''
Creates a .mosaic file in the specified directory containing the specified
Dictionary (Entries) at the specified Downsample rate. Entries should have the key=filename
and values should be a tuple with pixel coordinates
Setting Flip to True will invert all X coordinates
Setting Flop to True will invert all Y coordinates
'''
if ImageSize is None:
ImageSize = [(4080, 4080)] * len(Entries)
elif not isinstance(ImageSize, list):
assert (len(ImageSize) == 2
) # Expect tuple or list indicating image size
ImageSize = ImageSize * len(Entries)
else:
# A list of two entries for the size
if len(ImageSize) == 2 and not isinstance(ImageSize[0], list):
ImageSize = [(ImageSize[0], ImageSize[1])] * len(Entries)
elif len(ImageSize) == len(Entries):
ImageSize = ImageSize
else:
raise Exception(
"Unexpected list format for ImageSize argument")
prettyoutput.CurseString('Stage', "WriteMosaicFile " + OutfilePath)
# TODO - find min/max values and center image in .mosaic file
minX = sys.maxsize
minY = sys.maxsize
maxX = -sys.maxsize - 1
maxY = -sys.maxsize - 1
keys = list(Entries.keys())
keys.sort()
for key in keys:
Coord = Entries[key]
if (Coord[0] < minX):
minX = Coord[0]
if (Coord[0] > maxX):
maxX = Coord[0]
if (Coord[1] < minY):
minY = Coord[1]
if (Coord[1] > maxY):
maxY = Coord[1]
prettyoutput.Log('MinX:\t' + str(minX) + '\tMaxX:\t' + str(maxX))
prettyoutput.Log('MinY:\t' + str(minY) + '\tMaxY:\t' + str(maxY))
# Write standard .mosiac file header
with open(OutfilePath, 'w+') as OutFile:
OutFile.write('format_version_number: 1\n')
OutFile.write('number_of_images: ' + str(len(Entries)) + '\n')
OutFile.write('pixel_spacing: ' + str(Downsample) + '\n')
OutFile.write('use_std_mask: 0\n')
# prettyoutput.Log( "Keys: " + str(Entries.keys())
keys = list(Entries.keys())
keys.sort()
for i, key in enumerate(keys):
Coord = Entries[key]
tilesize = ImageSize[i]
# prettyoutput.Log( str(key) + " : " + str(Coord)
# Centering is nice, but it seems to break multi mrc sections
# Coord = (Coord[0] - (minX + CenterXOffset), Coord[1] - (minY + CenterYOffset))
Coord = (Coord[0], Coord[1])
X = Coord[0]
Y = Coord[1]
if Flip:
Y = -Y
if Flop:
X = -X
# Remove dirname from key
key = os.path.basename(key)
outstr = 'image:\n' + key + '\n' + 'LegendrePolynomialTransform_double_2_2_1 vp 6 1 0 1 1 1 0 fp 4 ' + str(
X) + ' ' + str(Y) + ' ' + str(tilesize[0] / 2) + ' ' + str(
tilesize[1] / 2)
OutFile.write(outstr + '\n')
OutFile.close()
def VolumeFinder(path=None,
OutputFile=None,
VolumeNode=None,
requiredFiles=None,
**kwargs):
'''Expects a 'Path' and 'RequiredFiles' keyword argument'
produces an HTML index of all volumes under the path'''
if requiredFiles is None:
requiredFiles = []
if (path is None):
VolumeNode = kwargs.get('VolumeNode', None)
if VolumeNode is None:
PrettyOutput.LogErr("Path attribute not found for VolumeFinder")
return
else:
path = VolumeNode.attrib['Path']
requiredFiles = kwargs.get('RequiredFiles', [])
if not isinstance(requiredFiles, list):
RequiredFileStrs = str(requiredFiles).strip().split(',')
requiredFiles = list()
for fileStr in RequiredFileStrs:
requiredFiles.append(fileStr)
ServerHostname = FindServerFromAboutXML(path)
if ServerHostname is None:
ServerHostname = ""
dirs = nornir_shared.files.RecurseSubdirectories(
path, RequiredFiles=requiredFiles, ExcludeNames="")
HTMLHeader = "<!DOCTYPE html> \n" + "<html>\n " + "<body>\n"
HTMLFooter = "</body>\n" + "</html>\n"
HTMLString = HTMLHeader
dirDict = {}
# Make sure required files is a list type if a single string was passed
if isinstance(requiredFiles, str):
requiredFiles = [requiredFiles]
# Seperate out specifically named files from regular expressions
RegExpPatterns = []
i = 0
while i < len(requiredFiles):
filename = requiredFiles[i]
if '*' in filename or '?' in filename:
RegExpPatterns.append(filename)
requiredFiles.pop(i)
else:
i = i + 1
for directory in dirs:
# Take the path from the list and find files.
dirImageList = dirDict.get(directory, [])
for pattern in RegExpPatterns:
filesMatchingPattern = glob.glob(os.path.join(directory, pattern))
for filename in filesMatchingPattern:
filenameFullPath = os.path.join(directory, filename)
dirImageList.append(
filenameFullPath) # Should check if it exists maybe
for filename in requiredFiles:
filenameFullPath = os.path.join(directory, filename)
if (os.path.exists(filenameFullPath)):
dirImageList.append(
filenameFullPath) # Should check if it exists maybe
dirDict[directory] = dirImageList
# Remove the path we were passed from the strings
HTMLString = HTMLString.replace(path, '')
RowNames = list(dirDict.keys())
RowNames.sort()
RowNames.reverse()
HTMLTable = HTMLTableForImageList(path, dirDict, RowNames, **kwargs)
HTMLString = HTMLString + HTMLTable + '\n'
HTMLString = HTMLString + HTMLFooter
if (len(RowNames) == 0):
PrettyOutput.Log("Report generator could not find matching files " +
str(requiredFiles))
return ""
if not OutputFile is None:
webPageFullPath = os.path.join(path, OutputFile)
f = open(webPageFullPath, 'w')
f.write(HTMLString)
f.close()
return None # HTMLString
def ToMosaic(cls,
VolumeObj,
InputPath,
scaleValueInNm,
OutputPath=None,
OutputImageExt=None,
TargetBpp=None,
debug=None,
*args,
**kwargs):
'''#Converts a directory of images to sections, each represented by a single image.
Each image should have the format Section#_ChannelText'''
# Default to the directory above ours if an output path is not specified
if OutputPath is None:
OutputPath = os.path.join(InputPath, "..")
# If the user did not supply a value, use a default
if (TargetBpp is None):
TargetBpp = 8
# Report the current stage to the user
# prettyoutput.CurseString('Stage', "PMGToMosaic " + InputPath);
# Find the files with a .pmg extension
filename = InputPath
if 'histogram' in filename.lower():
prettyoutput.Log(
"PNG importer ignoring probable histogram file: " + filename)
return None
# TODO wrap in try except and print nice error on badly named files?
fileData = filenameparser.ParseFilename(filename, imageNameMappings)
fileData = FillInMissingImageNameData(fileData)
imageDir = os.path.dirname(filename)
imagebasename = os.path.basename(filename)
BlockName = os.path.basename(imageDir)
if BlockName is None or len(BlockName) == 0:
BlockName = 'LM'
BlockObj = BlockNode.Create(BlockName)
[addedBlock, BlockObj] = VolumeObj.UpdateOrAddChild(BlockObj)
if (fileData is None):
raise Exception("Could not parse section from PMG filename: " +
filename)
SectionNumber = fileData.Section
sectionObj = SectionNode.Create(SectionNumber)
[addedSection,
sectionObj] = BlockObj.UpdateOrAddChildByAttrib(sectionObj, 'Number')
ChannelName = fileData.Channel
ChannelName = ChannelName.replace(' ', '_')
channelObj = ChannelNode.Create(ChannelName)
channelObj.SetScale(scaleValueInNm)
[channelAdded,
channelObj] = sectionObj.UpdateOrAddChildByAttrib(channelObj, 'Name')
# Create a filter for the images
# Create a filter and mosaic
FilterName = fileData.Filter
if FilterName is None:
if (TargetBpp is None):
FilterName = 'Import'
else:
FilterName = 'Import' + str(TargetBpp)
(added_filter, filterObj) = channelObj.GetOrCreateFilter(FilterName)
filterObj.BitsPerPixel = TargetBpp
# Create an image for the filter
ImageSetNode = metadatautils.CreateImageSetForImage(
filterObj, filename, Downsample=fileData.Downsample)
# Find the image node
levelNode = ImageSetNode.GetChildByAttrib('Level', 'Downsample',
fileData.Downsample)
imageNode = levelNode.GetChildByAttrib('Image', 'Path', imagebasename)
os.makedirs(os.path.dirname(imageNode.FullPath), exist_ok=True)
prettyoutput.Log("Copying file: " + imageNode.FullPath)
shutil.copy(filename, imageNode.FullPath)
if addedBlock:
return VolumeObj
elif addedSection:
return BlockObj
elif channelAdded:
return sectionObj
else:
return channelObj
def ParsePMG(filename, TileOverlapPercent=None):
if TileOverlapPercent is None:
TileOverlapPercent = 0.1
# Create a dictionary to store tile position
Tiles = dict()
# OutFilepath = os.path.join(SectionPath, OutFilename)
PMGDir = os.path.dirname(filename)
if (DEBUG):
prettyoutput.Log("Filename: " + filename)
# prettyoutput.Log("PMG to: " + OutFilepath)
Data = ''
with open(filename, 'r') as SourceFile:
Data = SourceFile.read()
SourceFile.close()
Data = Data.replace('\r\n', '\n')
Tuple = Data.partition('VISPIECES')
Tuple = Tuple[2].partition('\n')
NumPieces = int(Tuple[0])
Tuple = Data.partition('VISSCALE')
Tuple = Tuple[2].partition('\n')
ScaleFactor = 1 / float(Tuple[0])
Tuple = Data.partition('IMAGEREDUCTION')
Tuple = Tuple[2].partition('\n')
ReductionFactor = 1 / float(Tuple[0])
Data = Tuple[2]
if (DEBUG):
prettyoutput.Log(("Num Tiles: " + str(NumPieces)))
prettyoutput.Log(("Scale : " + str(ScaleFactor)))
prettyoutput.Log(("Reduction: " + str(ReductionFactor)))
Tuple = Data.partition('PIECE')
Data = Tuple[2]
# What type of files did Syncroscan write? We have to assume BMP and then switch to tif
# if the BMP's do not exist.
FileTypeExt = "BMP"
TileWidth = 0
TileHeight = 0
while Data:
Tuple = Data.partition('ENDPIECE')
# Remaining unprocessed file goes into data
Data = Tuple[2]
Entry = Tuple[0]
# Find filename
Tuple = Entry.partition('PATH')
# Probably means we skipped the last tile in a PMG
if (Tuple[1] != "PATH"):
continue
Tuple = Tuple[2].partition('<')
Tuple = Tuple[2].partition('>')
TileFilename = Tuple[0]
TileFullPath = os.path.join(PMGDir, TileFilename)
# PMG files for some reason always claim the tile is a .bmp. So we trust them first and if it doesn't
# exist we try to find a .tif
if not os.path.exists(TileFullPath):
[base, ext] = os.path.splitext(TileFilename)
TileFilename = base + '.tif'
TileFullPath = os.path.join(PMGDir, TileFilename)
FileTypeExt = 'tif'
if not os.path.exists(TileFullPath):
prettyoutput.Log('Skipping missing tile in PMG: ' + TileFilename)
continue
if (TileWidth == 0):
try:
[TileHeight, TileWidth
] = nornir_imageregistration.GetImageSize(TileFullPath)
if (DEBUG):
prettyoutput.Log(
str(TileWidth) + ',' + str(TileHeight) + " " +
TileFilename)
except:
prettyoutput.Log('Could not determine size of tile: ' +
TileFilename)
continue
# prettyoutput.Log("Adding tile: " + TileFilename)
# Prevent rounding errors later when we divide these numbers
TileWidth = float(TileWidth)
TileHeight = float(TileHeight)
TileWidthMinusOverlap = TileWidth - (TileWidth * TileOverlapPercent)
TileHeightMinusOverlap = TileHeight - (TileHeight * TileOverlapPercent)
# Find Position
Tuple = Entry.partition('CORNER')
Tuple = Tuple[2].partition('\n')
Position = Tuple[0].split()
# prettyoutput.Log( Position
X = float(Position[0])
Y = float(Position[1])
# Convert Position into pixel units using Reduction and Scale factors
X = X * ScaleFactor * ReductionFactor
Y = Y * ScaleFactor * ReductionFactor
# Syncroscan lays out tiles in grids, so find the nearest grid coordinates of this tile
# This lets us use the last captured tile per grid cell, so users can manually correct
# focus
iX = round(X / TileWidthMinusOverlap)
iY = round(Y / TileHeightMinusOverlap)
if (DEBUG):
prettyoutput.Log(("Name,iX,iY: " + TileFilename + " " + str(
(iX, iY))))
# Add tile to dictionary
# Using the indicies will replace any existing tiles in that location.
# Syncroscan adds the tiles to the file in order of capture, so the older, blurry
# tiles will be replaced.
Tiles[TileFilename] = X, Y
# Prime for next iteration
Tuple = Data.partition('PIECE')
Data = Tuple[2]
return Tiles
def ChangeStosGridPixelSpacing(self,
oldspacing,
newspacing,
ControlImageFullPath,
MappedImageFullPath,
ControlMaskFullPath,
MappedMaskFullPath,
create_copy=True):
'''
:param bool create_copy: True if a copy of the transform should be scaled, otherwise scales the transform we were called on
'''
if oldspacing == newspacing and \
ControlImageFullPath == self.ControlImageFullPath and \
MappedImageFullPath == self.MappedImageFullPath and \
ControlMaskFullPath == self.ControlMaskFullPath and \
MappedMaskFullPath == self.MappedMaskFullPath:
if create_copy:
return copy.deepcopy(self)
else:
return self
PrettyOutput.Log("ChangeStosGridPixelSpacing from " + str(oldspacing) +
" to " + str(newspacing))
scale = float(oldspacing) / float(newspacing)
NewStosFile = StosFile()
# NewStosFile.ControlImageDim = [x * scale for x in self.ControlImageDim]
NewStosFile.ControlImageDim = copy.copy(self.ControlImageDim)
NewStosFile.ControlImageDim[2] = self.ControlImageDim[2] * scale
NewStosFile.ControlImageDim[3] = self.ControlImageDim[3] * scale
# NewStosFile.MappedImageDim = [x * scale for x in self.MappedImageDim]
# Update the filenames which are the first two lines of the file
NewStosFile.MappedImageDim = copy.copy(self.MappedImageDim)
NewStosFile.MappedImageDim[2] = self.MappedImageDim[2] * scale
NewStosFile.MappedImageDim[3] = self.MappedImageDim[3] * scale
NewStosFile.ControlImageFullPath = ControlImageFullPath
NewStosFile.MappedImageFullPath = MappedImageFullPath
NewStosFile.ControlMaskFullPath = ControlMaskFullPath
NewStosFile.MappedMaskFullPath = MappedMaskFullPath
if os.path.exists(ControlImageFullPath):
NewStosFile.ControlImageDim = StosFile.__GetImageDimsArray(
ControlImageFullPath)
if os.path.exists(MappedImageFullPath):
NewStosFile.MappedImageDim = StosFile.__GetImageDimsArray(
MappedImageFullPath)
# Adjust the transform points
if scale == 1.0:
NewStosFile.Transform = self.Transform
else:
transformObj = nornir_imageregistration.transforms.LoadTransform(
self.Transform, pixelSpacing=1.0)
assert (not transformObj is None)
transformObj.Scale(scalar=scale)
NewStosFile._Downsample = newspacing
if hasattr(transformObj, 'gridWidth'):
# Save as a stos grid if we can
bounds = (NewStosFile.MappedImageDim[1],
NewStosFile.MappedImageDim[0],
NewStosFile.MappedImageDim[3],
NewStosFile.MappedImageDim[2])
NewStosFile.Transform = nornir_imageregistration.transforms.TransformToIRToolsString(
transformObj, bounds=bounds)
else:
NewStosFile.Transform = nornir_imageregistration.transforms.TransformToIRToolsString(
transformObj) # , bounds=NewStosFile.MappedImageDim)
return NewStosFile
def Load(cls, filename):
if (os.path.exists(filename) == False):
prettyoutput.Log("Mosaic file not found: " + filename)
return
return Histogram.FromXML(xml.dom.minidom.parse(filename))
def Histogram(filenames, Bpp=None, Scale=None, **kwargs):
'''Returns a single histogram built by combining histograms of all images
If scale is not none the images are scaled before the histogram is collected'''
if isinstance(filenames, str):
listfilenames = [filenames]
else:
listfilenames = filenames
numTiles = len(listfilenames)
if numTiles == 0:
return dict()
if Bpp is None:
Bpp = nornir_shared.images.GetImageBpp(listfilenames[0])
assert isinstance(listfilenames, list)
FilenameToTask = {}
if len(listfilenames) > 2:
pool = nornir_pools.GetGlobalLocalMachinePool()
else:
pool = nornir_pools.GetGlobalSerialPool()
for f in listfilenames:
# (root, ext) = os.path.splitext(f)
# __HistogramFilePillow__(f, Bpp=Bpp, Scale=Scale)
task = pool.add_task(f, __HistogramFileSciPy__, f, Bpp=Bpp, Scale=Scale, **kwargs)
# if ext == '.npy':
# task = __HistogramFileSciPy__(f, Bpp=Bpp, Scale=Scale)
# else:
# #task = __HistogramFilePillow__(f, ProcPool=pool, Bpp=Bpp, Scale=Scale)
# task = pool.add_task(f, __HistogramFilePillow__,f, Bpp=Bpp, Scale=Scale)
# #task = __HistogramFileImageMagick__(f, ProcPool=pool, Bpp=Bpp, Scale=Scale)
FilenameToTask[f] = task
minVal = None
maxVal = None
histlist = []
numBins = None
for f in list(FilenameToTask.keys()):
task = FilenameToTask[f]
try:
h = task.wait_return()
except IOError as e:
PrettyOutput.Log("File not found " + f)
continue
histlist.append(h)
# lines = taskOutput.splitlines()
#
# OutputMap[f] = lines
#
# (fminVal, fmaxVal) = nornir_imageregistration.im_histogram_parser.MinMaxValues(lines)
if minVal is None:
minVal = h.MinValue
else:
minVal = min(minVal, h.MinValue)
if maxVal is None:
maxVal = h.MaxValue
else:
maxVal = max(maxVal, h.MaxValue)
numBins = len(h.Bins)
#
# threadTasks = []
#
# thread_pool = nornir_pools.GetGlobalThreadPool()
# for f in list(OutputMap.keys()):
# threadTask = thread_pool.add_task(f, nornir_imageregistration.im_histogram_parser.Parse, OutputMap[f], minVal=minVal, maxVal=maxVal, numBins=numBins)
# threadTasks.append(threadTask)
#
HistogramComposite = nornir_shared.histogram.Histogram.Init(minVal=minVal, maxVal=maxVal, numBins=numBins)
for h in histlist:
# hist = t.wait_return()
HistogramComposite.AddHistogram(h)
# histogram = IMHistogramOutput.Parse(taskOutput, minVal=minVal, maxVal=maxVal, numBins=numBins)
# FilenameToResult[f] = [histogram, None, None]
if Bpp > 8:
HistogramComposite = nornir_shared.histogram.Histogram.Trim(HistogramComposite)
# del threadTasks
# FilenameToResult = __InvokeFunctionOnImageList__(listfilenames, Function=__HistogramFileImageMagick__, Pool=nornir_pools.GetGlobalThreadPool(), ProcPool = nornir_pools.GetGlobalClusterPool(), Bpp=Bpp, Scale=Scale)#, NumSamples=SamplesPerImage)
# maxVal = 1 << Bpp
# numBins = 256
# if Bpp > 8:
# numBins = 1024
# Sum all of the result arrays together
# for filename in listfilenames:
# if filename in FilenameToResult:
# Result = FilenameToResult[filename]
# histogram = Result[0]
# #
# # if HistogramComposite is None:
# # HistogramComposite = numpy.zeros(histogram.shape, dtype=numpy.int0)
# #
# # HistogramComposite = numpy.add(HistogramComposite, histogram)
#
# HistogramComposite.AddHistogram(histogram.Bins)
return HistogramComposite
def ToMosaic(cls,
VolumeObj,
PMGFullPath,
scaleValueInNm,
OutputPath=None,
Extension=None,
OutputImageExt=None,
TileOverlap=None,
TargetBpp=None,
debug=None,
*args,
**kwargs):
'''#Converts a PMG
#PMG files are created by Objective Imaging's Surveyor.
#This function expects a directory to contain a single PMG file with the tile images in the same directory
#Returns the SectionNumber and ChannelName of PMG processed. Otherwise [None,None]'''
ParentDir = os.path.dirname(PMGFullPath)
sectionDir = os.path.basename(PMGFullPath)
# Default to the directory above ours if an output path is not specified
if OutputPath is None:
OutputPath = os.path.join(PMGFullPath, "..")
# If the user did not supply a value, use a default
if (TileOverlap is None):
TileOverlap = 0.10
if (TargetBpp is None):
TargetBpp = 8
# Report the current stage to the user
# prettyoutput.CurseString('Stage', "PMGToMosaic " + InputPath)
BlockName = 'TEM'
BlockObj = BlockNode.Create('TEM')
[addedBlock, BlockObj] = VolumeObj.UpdateOrAddChild(BlockObj)
ChannelName = None
# TODO wrap in try except and print nice error on badly named files?
PMG = ParseFilename(PMGFullPath, pmgMappings)
PMGDir = os.path.dirname(PMGFullPath)
if (PMG is None):
raise Exception("Could not parse section from PMG filename: %s" +
PMGFullPath)
if PMG.Section is None:
PMG.Section = PMG.Spot
sectionObj = SectionNode.Create(PMG.Section)
[addedSection,
sectionObj] = BlockObj.UpdateOrAddChildByAttrib(sectionObj, 'Number')
# Calculate our output directory. The scripts expect directories to have section numbers, so use that.
ChannelName = PMG.Probe
ChannelName = ChannelName.replace(' ', '_')
channelObj = ChannelNode.Create(ChannelName)
channelObj.SetScale(scaleValueInNm)
[channelAdded,
channelObj] = sectionObj.UpdateOrAddChildByAttrib(channelObj, 'Name')
channelObj.Initials = PMG.Initials
channelObj.Mag = PMG.Mag
channelObj.Spot = PMG.Spot
channelObj.Slide = PMG.Slide
channelObj.Block = PMG.Block
FlipList = GetFlipList(ParentDir)
Flip = PMG.Section in FlipList
if (Flip):
prettyoutput.Log("Flipping")
# TODO: Add scale element
# OutFilename = ChannelName + "_supertile.mosaic"
# OutFilepath = os.path.join(SectionPath, OutFilename)
# Preserve the PMG file
# PMGBasename = os.path.basename(filename)
# PMGOutputFile = os.path.join(OutputPath, PMGBasename)
# ir.RemoveOutdatedFile(filename, PMGOutputFile)
# if not os.path.exists(PMGOutputFile):
# shutil.copy(filename, PMGOutputFile)
#
# #See if we need to remove the old supertile mosaic
# ir.RemoveOutdatedFile(filename, OutFilepath)
# if(os.path.exists(OutFilepath)):
# continue
#
Tiles = ParsePMG(PMGFullPath)
if len(Tiles) == 0:
raise Exception("No tiles found within PMG file")
NumImages = len(Tiles)
# Create a filter and mosaic
FilterName = 'Raw' + str(TargetBpp)
if (TargetBpp is None):
FilterName = 'Raw'
[added_filter, filterObj] = channelObj.GetOrCreateFilter(FilterName)
filterObj.BitsPerPixel = TargetBpp
SupertileName = 'Stage'
SupertileTransform = SupertileName + '.mosaic'
[addedTransform, transformObj] = channelObj.UpdateOrAddChildByAttrib(
TransformNode.Create(Name=SupertileName,
Path=SupertileTransform,
Type='Stage'), 'Path')
[added, PyramidNodeObj] = filterObj.UpdateOrAddChildByAttrib(
TilePyramidNode.Create(Type='stage', NumberOfTiles=NumImages),
'Path')
[added, LevelObj
] = PyramidNodeObj.UpdateOrAddChildByAttrib(LevelNode.Create(Level=1),
'Downsample')
# Make sure the target LevelObj is verified
VerifyTiles(LevelNode=LevelObj)
OutputImagePath = os.path.join(channelObj.FullPath, filterObj.Path,
PyramidNodeObj.Path, LevelObj.Path)
os.makedirs(OutputImagePath, exist_ok=True)
InputTileToOutputTile = {}
PngTiles = {}
TileKeys = list(Tiles.keys())
imageSize = []
for inputTile in TileKeys:
[base, ext] = os.path.splitext(inputTile)
pngMosaicTile = base + '.png'
OutputTileFullPath = os.path.join(LevelObj.FullPath, pngMosaicTile)
InputTileFullPath = os.path.join(PMGDir, inputTile)
if not os.path.exists(OutputTileFullPath):
InputTileToOutputTile[InputTileFullPath] = OutputTileFullPath
PngTiles[pngMosaicTile] = Tiles[inputTile]
(Height,
Width) = nornir_imageregistration.GetImageSize(InputTileFullPath)
imageSize.append((Width, Height))
nornir_imageregistration.ConvertImagesInDict(InputTileToOutputTile,
Flip=False,
OutputBpp=TargetBpp)
if not os.path.exists(transformObj.FullPath):
mosaicfile.MosaicFile.Write(transformObj.FullPath,
PngTiles,
Flip=Flip,
ImageSize=imageSize)
return [PMG.Section, ChannelName]
