def add_process(self, name, func, *args, **kwargs):
"""Add a task to the queue, args are passed directly to subprocess.Popen"""
if func is None:
prettyoutput.LogErr(
"Process pool add task {0} called with 'None' as function".
format(name))
if callable(func) == False:
prettyoutput.LogErr(
"Process pool add task {0} parameter was non-callable value {1} when it should be passed a function"
.format(name, func))
assert (callable(func))
# keep_alive_thread is a non-daemon thread started when the queue is non-empty.
# Python will not shut down while non-daemon threads are alive. When the queue empties the thread exits.
# When items are added to the queue we create a new keep_alive_thread as needed
if isinstance(kwargs, dict):
if not 'shell' in kwargs:
kwargs['shell'] = True
else:
kwargs = {}
kwargs['shell'] = True
entry = ProcessTask(name, func, *args, **kwargs)
self.tasks.put(entry)
self.add_threads_if_needed()
return entry
def __CreateOutputBufferForArea(Height, Width, target_space_scale=None):
'''Create output images using the passed width and height
'''
global use_memmap
fullImage = None
fullImage_shape = (
int(Height), int(Width)
) #(int(np.ceil(target_space_scale * Height)), int(np.ceil(target_space_scale * Width)))
if False: #use_memmap:
try:
fullimage_array_path = os.path.join(
tempfile.gettempdir(),
'image_%dx%d_%s.npy' % (fullImage_shape[0], fullImage_shape[1],
GetProcessAndThreadUniqueString()))
# print("Open %s" % (fullimage_array_path))
fullImage = np.memmap(fullimage_array_path,
dtype=np.float16,
mode='w+',
shape=fullImage_shape)
fullImage[:] = 0
except:
prettyoutput.LogErr("Unable to open memory mapped file %s." %
(fullimage_array_path))
raise
else:
fullImage = np.zeros(fullImage_shape, dtype=np.float16)
fullImageZbuffer = EmptyDistanceBuffer(fullImage.shape,
dtype=fullImage.dtype)
return (fullImage, fullImageZbuffer)
def WritePruneMap(self, MapImageToScoreFile):
if len(self.MapImageToScore) == 0:
prettyoutput.LogErr('No prune scores to write to file ' +
MapImageToScoreFile)
return
with open(MapImageToScoreFile, 'w') as outfile:
if (len(list(self.MapImageToScore.keys())) == 0):
if (os.path.exists(MapImageToScoreFile)):
os.remove(MapImageToScoreFile)
prettyoutput.Log(
"No prune scores present in PruneMap being saved: " +
MapImageToScoreFile)
outfile.close()
return
for f in sorted(self.MapImageToScore.keys()):
score = self.MapImageToScore[f]
outfile.write(f + '\t' + str(score) + '\n')
outfile.close()
def ReplaceVariable(self, xpath, iStart):
'''Replace # variable names in an xpath with variable string values'''
# Find the next # if it exists
iStart = iStart + 1 # Skip the # symbol
iEndVar = xpath[iStart + 1:].find('#')
if iEndVar < 0:
# OK, just check to the end of the string
iEndVar = len(xpath)
else:
iEndVar = iStart + iEndVar + 1
while iEndVar > iStart:
keyCheck = xpath[iStart:iEndVar]
try:
value = self.TryGetValueForKey(keyCheck)
xpath = xpath[:iStart - 1] + str(value) + xpath[iEndVar:]
return xpath
except KeyError as e:
pass
iEndVar = iEndVar - 1
logger = logging.getLogger(__name__ + ".ReplaceVariable")
logger.error("nornir_buildmanager XPath variable not defined.\nXPath: " + xpath)
prettyoutput.LogErr("nornir_buildmanager XPath variable not defined.\nXPath: " + xpath)
sys.exit()
def Execute(self, args):
'''This executes the loaded pipeline on the specified volume of data.
parser is an instance of the argparser class which should be
extended with any pipeline specific arguments args are the parameters from the command line'''
# DOM = self.PipelineData.toDOM()
# PipelineElement = DOM.firstChild
ArgSet = ArgumentSet()
PipelineElement = self.PipelineData
prettyoutput.Log("Adding pipeline arguments")
# parser = self.GetArgParser(parser)
# (args, unused) = parser.parse_known_args(passedArgs)
ArgSet.AddArguments(args)
ArgSet.AddParameters(PipelineElement)
# Load the Volume.XML file in the output directory
self.VolumeTree = VolumeManagerETree.VolumeManager.Load(args.volumepath, Create=True)
if(self.VolumeTree is None):
PipelineManager.logger.critical("Could not load or create volume.xml " + args.outputpath)
prettyoutput.LogErr("Could not load or create volume.xml " + args.outputpath)
sys.exit()
# dargs = copy.deepcopy(defaultDargs)
self.ExecuteChildPipelines(ArgSet, self.VolumeTree, PipelineElement)
nornir_pools.WaitOnAllPools()
def ParseTransform(TransformNode, OutputSectionNode):
mFile = mosaicfile.MosaicFile.Load(TransformNode.FullPath)
if (mFile is None):
prettyoutput.LogErr("Unable to load transform: " +
TransformNode.FullPath)
return
if (mFile.NumberOfImages < 1):
prettyoutput.LogErr("Not including empty .mosaic file")
return
# Figure out what the tile prefix and postfix are for this mosaic file by extrapolating from the first tile filename
for k in list(mFile.ImageToTransformString.keys()):
TileFileName = k
break
# TileFileName = mFile.ImageToTransformString.keys()[0]
# Figure out prefix and postfix parts of filenames
parts = TileFileName.split('.')
Postfix = parts[len(parts) - 1]
# Two conventions are commonly used Section#.Tile#.png or Tile#.png
if (len(parts) == 3):
Prefix = parts[0]
else:
Prefix = ''
UseForVolume = 'false'
if ('grid' in TransformNode.Name.lower()):
UseForVolume = 'true'
#Viking needs the transform names to be consistent, and if transforms are built with different spacings, for TEM and CMP, Viking can't display
#So we simplify the transform name
TransformName = TransformNode.Name
return ETree.SubElement(
OutputSectionNode, 'Transform', {
'FilePostfix': Postfix,
'FilePrefix': Prefix,
'Path': TransformNode.Path,
'Name': TransformName,
'UseForVolume': UseForVolume
})
def _CheckPipelineXMLExists(cls, PipelineXmlFile):
if not os.path.exists(PipelineXmlFile):
PipelineManager.logger.critical("Provided pipeline filename does not exist: " + PipelineXmlFile)
prettyoutput.LogErr("Provided pipeline filename does not exist: " + PipelineXmlFile)
sys.exit()
return False
return True
def XPathIterator(XPath):
'''A Very limited iterator which takes xpath strings as input and iterates over each subpath.
Returns an object with the following properties:
The following XPath syntax elements are supported: / .. . [] @ = '''
# Get the name of the child node we are looking for
PathParts = XPath.split('/')
pat = re.compile(
r"""
(?P<Path>[^\[/\]]+)
(\[
(?P<IsAttribute>@)?
(?P<Name>[^=\]]+)
((?P<Operator>[=<>])
(?P<Value>[^\]]+))?
\])?
""", re.VERBOSE)
# prettyoutput.Log(XPath);
for subpath in PathParts:
StartFromRoot = len(subpath) == 0
if (StartFromRoot):
continue
# prettyoutput.Log("\n" + subpath);
matches = pat.match(subpath)
if matches is None:
prettyoutput.LogErr("Error in xpath subpath: " + subpath)
sys.exit()
# Figure out if Value is a string (Starts with quotes)
Obj = XSubPath()
Obj.RawPath = subpath
for item in list(matches.groupdict().items()):
Obj.__dict__[item[0]] = item[1]
Obj.IsAttribute = not matches.group('IsAttribute') is None
if (Obj.Value is not None):
if Obj.Value[0] == "'" and Obj.Value[-1] == "'" and len(
Obj.Value) >= 3:
Obj.Value = Obj.Value[1:-2]
elif Obj.Value[0] == '"' and Obj.Value[-1] == '"' and len(
Obj.Value) >= 3:
Obj.Value = Obj.Value[1:-2]
else:
try:
Obj.Value = float(Obj.Value)
except:
pass
yield Obj
def WritePruneMosaic(self,
path,
SourceMosaic,
TargetMosaic='prune.mosaic',
Tolerance='5'):
'''
Remove tiles from the source mosaic with scores less than Tolerance and
write the new mosaic to TargetMosaic.
Raises a key error if image in prude scores does not exist in .mosaic file
Raises a value error if the threshold removes all tiles in the mosaic.
:return: Number of tiles removed
'''
if (not isinstance(Tolerance, float)):
Tolerance = float(Tolerance)
SourceMosaicFullPath = os.path.join(path, SourceMosaic)
TargetMosaicFullPath = os.path.join(path, TargetMosaic)
mosaic = mosaicfile.MosaicFile.Load(SourceMosaicFullPath)
# We copy this because if an input image is missing there will not be a prune score and it should be removed from the .mosaic file
inputImageToTransforms = copy.deepcopy(mosaic.ImageToTransformString)
mosaic.ImageToTransformString.clear()
numRemoved = 0
for item in list(self.MapImageToScore.items()):
filename = item[0]
score = item[1]
if (score >= Tolerance):
keyVal = filename
if not keyVal in inputImageToTransforms:
keyVal = os.path.basename(filename)
if not keyVal in inputImageToTransforms:
raise KeyError(
"PruneObj: Cannot locate image file in .mosaic " +
keyVal)
mosaic.ImageToTransformString[keyVal] = inputImageToTransforms[
keyVal]
else:
numRemoved = numRemoved + 1
if (len(mosaic.ImageToTransformString) <= 0):
errMsg = "All tiles removed when using threshold = " + str(
Tolerance) + "\nThe prune request was ignored"
prettyoutput.LogErr(errMsg)
raise ValueError(errMsg)
else:
prettyoutput.Log("Removed " + str(numRemoved) +
" tiles pruning mosaic " + TargetMosaic)
mosaic.Save(TargetMosaicFullPath)
return numRemoved
def _LoadImageByExtension(ImageFullPath, dtype):
'''
Loads an image file and returns an ndarray of dtype
:param dtype dtype: Numpy datatype of returned array. If the type is a float then the returned array is in the range 0 to 1. Otherwise it is whatever pillow and numpy decide.
'''
(root, ext) = os.path.splitext(ImageFullPath)
image = None
try:
if ext == '.npy':
image = np.load(ImageFullPath, 'c').astype(dtype)
else:
#image = plt.imread(ImageFullPath)
with Image.open(ImageFullPath) as im:
expected_dtype = pillow_helpers.dtype_for_pillow_image(im)
image = np.array(im, dtype=expected_dtype)
max_pixel_val = nornir_imageregistration.ImageMaxPixelValue(image)
if dtype is not None:
image = image.astype(dtype)
# else:
# #Reduce to smallest integer type that can hold the data
# if im.mode[0] == 'I' and (np.issubdtype(image.dtype, np.int32) or np.issubdtype(image.dtype, np.uint32)):
# (min_val, max_val) = im.getextrema()
# smallest_dtype = np.uint32
# if max_val <= 65535:
# smallest_dtype = np.uint16
# if max_val <= 255:
# smallest_dtype = np.uint8
#
# image = image.astype(smallest_dtype)
#
# dtype = image.dtype
#Ensure data is in the range 0 to 1 for floating types
if nornir_imageregistration.IsFloatArray(dtype):
if im.mode[0] == 'F':
(_, im_max_val) = im.getextrema()
if im_max_val <= 1.0:
return image
max_val = max_pixel_val
if max_val > 0:
image = image / max_val
im.close()
except IOError as E:
prettyoutput.LogErr("Unable to load image {0}".format(ImageFullPath))
raise E
return image
def Save(self, filename):
if (len(self.ImageToTransformString) <= 0):
prettyoutput.LogErr("No tiles present in mosaic " + filename +
"\nThe save request was aborted")
return
fMosaic = open(filename, 'w')
fMosaic.write(self.MosaicStr())
fMosaic.close()
def SaveImage(ImageFullPath, image, bpp=None, **kwargs):
'''Saves the image as greyscale with no contrast-stretching
:param str ImageFullPath: The filename to save
:param ndarray image: The image data to save
:param int bpp: The bit depth to save, if the image data bpp is higher than this value it will be reduced. Otherwise only the bpp required to preserve the image data will be used. (8-bit data will not be upsampled to 16-bit)
'''
dirname = os.path.dirname(ImageFullPath)
if dirname is not None and len(dirname) > 0:
os.makedirs(dirname, exist_ok=True)
if bpp is None:
bpp = nornir_imageregistration.ImageBpp(image)
if bpp > 16:
prettyoutput.LogErr("Saving image at 32 bits-per-pixel, check SaveImageParameters for efficiency:\n{0}".format(ImageFullPath))
if bpp > 8:
#Ensure we even have the data to bother saving a higher bit depth
detected_bpp = nornir_imageregistration.ImageBpp(image)
if detected_bpp < bpp:
bpp = detected_bpp
(root, ext) = os.path.splitext(ImageFullPath)
if ext == '.jp2':
SaveImage_JPeg2000(ImageFullPath, image, **kwargs)
elif ext == '.npy':
np.save(ImageFullPath, image)
else:
if image.dtype == np.bool or bpp == 1:
#Covers for pillow bug with bit images
#https://stackoverflow.com/questions/50134468/convert-boolean-numpy-array-to-pillow-image
#im = Image.fromarray(image.astype(np.uint8) * 255, mode='L').convert('1')
im = OneBit_img_from_bool_array(image)
elif bpp == 8:
Uint8_image = _Image_To_Uint8(image)
del image
im = Image.fromarray(Uint8_image, mode="L")
elif nornir_imageregistration.IsFloatArray(image):
#TODO: I believe Pillow-SIMD finally added the ability to save I;16 for 16bpp PNG images
if image.dtype == np.float16:
image = image.astype(np.float32)
im = Image.fromarray(image * ((1 << bpp)-1))
im = im.convert('I')
else:
if bpp < 32:
if ext.lower() == '.png':
im = uint16_img_from_uint16_array(image)
else:
im = Image.fromarray(image, mode="I;{0}".format(bpp))
else:
im = Image.fromarray(image, mode="I".format(bpp))
im.save(ImageFullPath, **kwargs)
return
def add_task(self, name, func, *args, **kwargs):
"""Add a task to the queue"""
if func is None:
prettyoutput.LogErr(
"Multiprocess pool add task {0} called with 'None' as function"
.format(name))
if callable(func) == False:
prettyoutput.LogErr(
"Multiprocess pool add task {0} parameter was non-callable value {1} when it should be passed a function"
.format(name, func))
assert (callable(func))
# I've seen an issue here were apply_async prints an exception about not being able to import a module. It then swallows the exception.
# The returned task seems valid and not complete, but the MultiprocessThreadTask's event is never set because the callback isn't used.
# This hangs the caller if they wait on the task.
retval_task = MultiprocessThreadTask(name, None, args, kwargs)
retval_task.asyncresult = self.tasks.apply_async(
func,
args,
kwargs,
callback=self.callback_wrapper(retval_task.task_id,
retval_task.callback),
error_callback=self.callback_wrapper(
retval_task.task_id, retval_task.callbackontaskfail))
if retval_task.asyncresult is None:
raise ValueError(
"apply_async returned None instead of an asyncresult object")
retval_task.asyncresult._nornir_task_id_ = retval_task.task_id
self._active_tasks[retval_task.task_id] = retval_task
#print("Added task #{0}".format(retval_task.task_id))
self.TryReportActiveTaskCount()
return retval_task
def add_task(self, name, func, *args, **kwargs):
if func is None:
prettyoutput.LogErr(
"Thread pool add task {0} called with 'None' as function".
format(name))
if callable(func) == False:
prettyoutput.LogErr(
"Thread pool add task {0} parameter was non-callable value {1} when it should be passed a function"
.format(name, func))
assert (callable(func))
"""Add a task to the queue"""
assert (False == self.shutdown_event.isSet())
# keep_alive_thread is a non-daemon thread started when the queue is non-empty.
# Python will not shut down while non-daemon threads are alive. When the queue empties the thread exits.
# When items are added to the queue we create a new keep_alive_thread as needed
entry = ThreadTask(name, func, *args, **kwargs)
self.tasks.put(entry)
self.add_threads_if_needed()
return entry
def GetInfo(cls, filename):
'''Parses a filename and returns what we have learned if it follows the convention
<SECTION>_<CHANNEL>_<MosaicSource>_<DOWNSAMPLE>.mosaic
[SectionNumber, Channel, MosaicSource, Downsample]'''
if (os.path.exists(filename) == False):
prettyoutput.LogErr("mosaic file not found: " + filename)
return
baseName = os.path.basename(filename)
[baseName, ext] = os.path.splitext(baseName)
parts = baseName.split("_")
try:
SectionNumber = int(parts[0])
except:
# We really can't recover from this, so maybe an exception should be thrown instead
SectionNumber = None
prettyoutput.Log('Could not determine mosaic section number: ' +
str(filename))
try:
MosaicType = parts[-2]
except:
MosaicType = None
prettyoutput.Log('Could not determine mosaic source: ' +
str(filename))
if (len(parts) > 3):
try:
Channel = parts[1]
except:
Channel = None
prettyoutput.Log('Could not determine mosaic channel: ' +
str(filename))
# If we don't have a valid downsample value we assume 1
try:
DownsampleStrings = parts[-1].split(".")
Downsample = int(DownsampleStrings[0])
except:
Downsample = None
prettyoutput.Log('Could not determine mosaic downsample: ' +
str(filename))
return [SectionNumber, Channel, MosaicType, Downsample]
def EmailIndex(path=None, subject=None, **kwargs):
import nornir_shared.emaillib
if (path is None):
VolumeNode = kwargs.get('ReportingElement', None)
if VolumeNode is None:
PrettyOutput.LogErr("Path attribute not found for VolumeFinder")
return
while not VolumeNode.Parent is None:
VolumeNode = VolumeNode.Parent
path = VolumeNode.attrib['Path']
ServerHostname = FindServerFromAboutXML(path)
if ServerHostname is None:
ServerHostname = ""
VolumeName = None
VolumeXMLPath = os.path.join(path, 'Volume.xml')
if os.path.exists(VolumeXMLPath):
VolumeXML = ElementTree.parse(VolumeXMLPath)
VolumeRoot = VolumeXML.getroot()
if 'Name' in VolumeRoot.attrib:
VolumeName = VolumeRoot.attrib['Name']
# Find all of the .html in the root directory
reports = glob.glob(os.path.join(path, '*.html'))
if not VolumeName is None:
if not subject is None:
subject = VolumeName + ": " + subject
kwargs['subject'] = subject
message = 'The following reports are available for this volume\n\n'
for report in reports:
RelativeReportPath = report[len(path) + 1:]
reportURL = ServerHostname + '/' + RelativeReportPath
message = message + "\n\t" + reportURL
kwargs['message'] = message
nornir_shared.emaillib.SendMail(**kwargs)
def Load(cls, filename):
if (os.path.exists(filename) == False):
prettyoutput.LogErr("Mosaic file not found: " + filename)
return
lines = []
with open(filename, 'r') as fMosaic:
lines = fMosaic.readlines()
fMosaic.close()
obj = MosaicFile()
iLine = 0
while iLine < len(lines):
line = lines[iLine]
line = line.strip()
[text, value] = line.split(':', 1)
if (text.startswith('number_of_images')):
value.strip()
obj.FileReportedNumberOfImages = int(value)
elif (text.startswith('pixel_spacing')):
value.strip()
obj.pixel_spacing = int(float(value))
elif (text.startswith('use_std_mask')):
obj.use_std_mask = int(value)
elif (text.startswith('format_version_number')):
obj.format_version_number = int(value)
elif (text.startswith('image')):
if (obj.format_version_number == 0):
[filename, transform] = value.split(None, 1)
filename = filename.strip()
filename = os.path.basename(filename)
transform = transform.strip()
obj.ImageToTransformString[filename] = transform
else:
filename = os.path.basename(lines[iLine + 1].strip())
transform = lines[iLine + 2].strip()
obj.ImageToTransformString[filename] = transform
iLine = iLine + 2
iLine = iLine + 1
return obj
def _AddArgumentNodeToParser(parser, argNode):
'''Returns a dictionary that can be added to a parser'''
attribDictCopy = copy.copy(argNode.attrib)
Flag = ""
for key in attribDictCopy:
val = attribDictCopy[key]
# Starts as a string, try to convert to bool, int, or float
if key == 'flag':
Flag = nornir_shared.misc.ListFromDelimited(val)
continue
elif key == 'type':
if val in __builtins__:
val = __builtins__[val]
elif val in globals():
val = globals()[val]
else:
logger = logging.getLogger(__name__ + "._AddArgumentNodeToParser")
logger.error('Type not found in __builtins__ or module __dict__' + val)
prettyoutput.LogErr('Type not found in __builtins__ or module __dict__ ' + val)
raise Exception(message="%s type specified by argument node is not present in __builtins__ or module dictionary. Must use a standard python type." % val)
continue
attribDictCopy[key] = val
elif key == 'default':
attribDictCopy[key] = _ConvertValueToPythonType(val)
elif key == 'required':
attribDictCopy[key] = _ConvertValueToPythonType(val)
elif key == 'choices':
listOfChoices = nornir_shared.misc.ListFromDelimited(val)
if len(listOfChoices) < 2:
raise Exception(message="Flag %s does not specify multiple choices. Must use a comma delimited list to provide multiple choice options.\nCurrent choice string is: %s" % (attribDictCopy['flag'], val))
attribDictCopy[key] = listOfChoices
if 'flag' in attribDictCopy:
del attribDictCopy['flag']
parser.add_argument(*Flag, **attribDictCopy)
def Load(cls, PipelineXml, PipelineName=None):
# PipelineData = Pipelines.CreateFromDOM(XMLDoc)
SelectedPipeline = None
XMLDoc = cls.LoadPipelineXML(PipelineXml)
if PipelineName is None:
PipelineManager.logger.warning("No pipeline name specified.")
prettyoutput.Log("No pipeline name specified")
cls.PrintPipelineEnumeration(XMLDoc)
return None
else:
SelectedPipeline = XMLDoc.find("Pipeline[@Name='" + PipelineName + "']")
if(SelectedPipeline is None):
PipelineManager.logger.critical("No pipeline found named " + PipelineName)
prettyoutput.LogErr("No pipeline found named " + PipelineName)
cls.PrintPipelineEnumeration(XMLDoc)
return None
return PipelineManager(pipelinesRoot=XMLDoc.getroot(), pipelineData=SelectedPipeline)
def Load(filename):
if not os.path.exists(filename):
PrettyOutput.Log("Stos file not found: " + filename)
return None
obj = StosFile()
try:
[
obj.MappedSectionNumber, obj.ControlSectionNumber, Channels,
Filters, obj.StosSource, obj._Downsample
] = StosFile.GetInfo(filename)
except:
pass
with open(filename, 'r') as fMosaic:
lines = fMosaic.readlines()
if len(lines) < 7:
PrettyOutput.LogErr("%s is not a valid stos file" % (filename))
raise ValueError("%s is not a valid stos file" % (filename))
obj.ControlImageFullPath = lines[0].strip()
obj.MappedImageFullPath = lines[1].strip()
ControlDims = lines[4].split()
MappedDims = lines[5].split()
obj.ControlImageDim = [float(x) for x in ControlDims]
obj.MappedImageDim = [float(x) for x in MappedDims]
obj.Transform = lines[6].strip()
if len(lines) > 8:
obj.ControlMaskFullPath = lines[8]
obj.MappedMaskFullPath = lines[9]
return obj
def __InvokeFunctionOnImageList__(listfilenames, Function=None, Pool=None, **kwargs):
'''Return a number indicating how interesting the image is using SciPy
'''
if Pool is None:
TPool = nornir_pools.GetGlobalMultithreadingPool()
else:
TPool = Pool
TileToScore = dict()
tasklist = []
for filename in listfilenames:
task = TPool.add_task('Calc Feature Score: ' + os.path.basename(filename), Function, filename, **kwargs)
task.filename = filename
tasklist.append(task)
TPool.wait_completion()
numTasks = len(tasklist)
iTask = 0
for task in tasklist:
Result = task.wait_return()
iTask = iTask + 1
if Result is None:
PrettyOutput.LogErr('No return value for ' + task.filename)
continue
# if Result[0] is None:
# PrettyOutput.LogErr('No filename for ' + task.name)
# continue
PrettyOutput.CurseProgress("ImageStats", iTask, numTasks)
filename = task.filename
TileToScore[filename] = Result
return TileToScore
def CalculatePruneScores(cls,
Parameters,
FilterNode,
Downsample,
TransformNode,
OutputFile=None,
Logger=None,
**kwargs):
'''@FilterNode
Calculate the prune scores for a filter and level'''
# VolumeManager.NodeManager.GetParent(Entry.NodeList)
# Check for an existing prune map
Overlap = float(Parameters.get('Overlap', 0.1))
if OutputFile is None:
OutputFile = 'PruneScores'
assert (isinstance(Downsample, int) or isinstance(Downsample, float))
[created,
LevelNode] = FilterNode.TilePyramid.GetOrCreateLevel(Downsample)
if (LevelNode is None):
prettyoutput.LogErr(
"Missing InputPyramidLevelNode attribute on PruneTiles")
Logger.error(
"Missing InputPyramidLevelNode attribute on PruneTiles")
return
if (TransformNode is None):
prettyoutput.LogErr(
"Missing TransformNode attribute on PruneTiles")
Logger.error("Missing TransformNode attribute on PruneTiles")
return
FilterNode = LevelNode.FindParent("Filter")
# Record the downsample level the values are calculated at:
Parameters['Level'] = str(LevelNode.Downsample)
Parameters['Filter'] = FilterNode.Name
MangledName = nornir_shared.misc.GenNameFromDict(
Parameters) + '_' + TransformNode.Type
OutputFile = OutputFile + MangledName + '.txt'
SaveRequired = False
PruneMapElement = FilterNode.GetChildByAttrib('Prune', 'Overlap',
Overlap)
PruneMapElement = transforms.RemoveOnMismatch(
PruneMapElement, 'InputTransformChecksum', TransformNode.Checksum)
if not PruneMapElement is None:
if hasattr(LevelNode, 'TilesValidated'):
PruneMapElement = transforms.RemoveOnMismatch(
PruneMapElement, 'NumImages', LevelNode.TilesValidated)
if PruneMapElement is None:
PruneMapElement = VolumeManagerETree.PruneNode.Create(
Overlap=Overlap, Type=MangledName)
[SaveRequired, PruneMapElement
] = FilterNode.UpdateOrAddChildByAttrib(PruneMapElement,
'Overlap')
else:
# If meta-data and the data file exist, nothing to do
if os.path.exists(PruneMapElement.DataFullPath):
return
# Create file holders for the .xml and .png files
PruneDataNode = VolumeManagerETree.DataNode.Create(OutputFile)
[added,
PruneDataNode] = PruneMapElement.UpdateOrAddChild(PruneDataNode)
FullTilePath = LevelNode.FullPath
TransformObj = mosaicfile.MosaicFile.Load(TransformNode.FullPath)
TransformObj.RemoveInvalidMosaicImages(FullTilePath)
files = []
for f in list(TransformObj.ImageToTransformString.keys()):
files.append(os.path.join(FullTilePath, f))
TileToScore = Prune(files, Overlap)
prune = PruneObj(TileToScore)
prune.WritePruneMap(PruneDataNode.FullPath)
PruneMapElement.InputTransformChecksum = TransformNode.Checksum
PruneMapElement.NumImages = len(TileToScore)
return FilterNode
def ConvertImagesInDict(ImagesToConvertDict, Flip=False, Flop=False, InputBpp=None, OutputBpp=None, Invert=False, bDeleteOriginal=False, RightLeftShift=None, AndValue=None, MinMax=None, Gamma=None):
'''
The key and value in the dictionary have the full path of an image to convert.
MinMax is a tuple [Min,Max] passed to the -level parameter if it is not None
RightLeftShift is a tuple containing a right then left then return to center shift which should be done to remove useless bits from the data
I do not use an and because I do not calculate ImageMagick's quantum size yet.
Every image must share the same colorspace
:return: True if images were converted
:rtype: bool
'''
if len(ImagesToConvertDict) == 0:
return False
if InputBpp is None:
for k in ImagesToConvertDict.keys():
if os.path.exists(k):
InputBpp = nornir_shared.images.GetImageBpp(k)
break
prettyoutput.CurseString('Stage', "ConvertImagesInDict")
if MinMax is not None:
if(MinMax[0] > MinMax[1]):
raise ValueError("Invalid MinMax parameter passed to ConvertImagesInDict")
pool = nornir_pools.GetMultithreadingPool("ConvertImagesInDict", num_threads=multiprocessing.cpu_count() * 2)
#pool = nornir_pools.GetGlobalSerialPool()
tasks = []
for (input_image, output_image) in ImagesToConvertDict.items():
task = pool.add_task("{0} -> {1}".format(input_image, output_image),
_ConvertSingleImageToFile,
input_image_param=input_image,
output_filename=output_image,
Flip=Flip,
Flop=Flop,
InputBpp=InputBpp,
OutputBpp=OutputBpp,
Invert=Invert,
MinMax=MinMax,
Gamma=Gamma)
tasks.append(task)
while len(tasks) > 0:
t = tasks.pop(0)
try:
t.wait()
except Exception as e:
prettyoutput.LogErr("Failed to convert " + t.name)
if bDeleteOriginal:
for (input_image, output_image) in ImagesToConvertDict.items():
if input_image != output_image:
pool.add_task("Delete {0}".format(input_image), os.remove, input_image)
while len(tasks) > 0:
t = tasks.pop(0)
try:
t.wait()
except OSError as e:
prettyoutput.LogErr("Unable to delete {0}\n{1}".format(t.name, e))
pass
except IOError as e:
prettyoutput.LogErr("Unable to delete {0}\n{1}".format(t.name, e))
pass
if not pool is None:
pool.wait_completion()
pool.shutdown()
pool = None
del tasks
def PruneMosaic(cls,
Parameters,
PruneNode,
TransformNode,
OutputTransformName=None,
Logger=None,
**kwargs):
'''@ChannelNode
Uses a PruneData node to prune the specified mosaic file'''
threshold_precision = VolumeManagerETree.TransformNode.get_threshold_precision(
) #Number of digits to save in XML file
if (Logger is None):
Logger = logging.getLogger(__name__ + '.PruneMosaic')
Threshold = cls._GetThreshold(PruneNode,
Parameters.get('Threshold', None))
if not Threshold is None:
Threshold = TransformNode.round_precision_value(
Threshold) #round(Threshold, threshold_precision)
cls._TryUpdateUndefinedThresholdFromParameter(PruneNode, Threshold)
if OutputTransformName is None:
OutputTransformName = 'Prune'
InputTransformNode = TransformNode
TransformParent = InputTransformNode.Parent
OutputMosaicName = OutputTransformName + nornir_shared.misc.GenNameFromDict(
Parameters) + '.mosaic'
MangledName = nornir_shared.misc.GenNameFromDict(Parameters)
HistogramXMLFile = PruneObj.HistogramXMLFileTemplate % PruneNode.Type
HistogramImageFile = PruneObj.HistogramSVGFileTemplate % PruneNode.Type
MosaicDir = os.path.dirname(InputTransformNode.FullPath)
OutputMosaicFullPath = os.path.join(MosaicDir, OutputMosaicName)
# Check if there is an existing prune map, and if it exists if it is out of date
PruneNodeParent = PruneNode.Parent
transforms.RemoveWhere(
TransformParent, 'Transform[@Name="' + OutputTransformName + '"]',
lambda t: (t.Threshold != Threshold) or (t.Type != MangledName))
'''TODO: Add function to remove duplicate Prune Transforms with different thresholds'''
TransformParent.RemoveOldChildrenByAttrib('Transform', 'Name',
OutputTransformName)
PruneDataNode = PruneNode.find('Data')
if (PruneDataNode is None):
Logger.warning("Did not find expected prune data node")
return None
OutputTransformNode = transforms.LoadOrCleanExistingTransformForInputTransform(
channel_node=TransformParent,
InputTransformNode=InputTransformNode,
OutputTransformPath=OutputMosaicName)
if not OutputTransformNode is None:
if OutputTransformNode.Locked:
Logger.info("Skipping locked transform %s" %
OutputTransformNode.FullPath)
return None
OutputTransformNode = transforms.RemoveOnMismatch(
OutputTransformNode, 'InputPruneDataChecksum',
PruneDataNode.Checksum)
OutputTransformNode = transforms.RemoveOnMismatch(
OutputTransformNode,
'Threshold',
Threshold,
Precision=threshold_precision)
# Add the Prune Transform node if it is missing
if OutputTransformNode is None:
OutputTransformNode = VolumeManagerETree.TransformNode.Create(
Name=OutputTransformName,
Type=MangledName,
InputTransformChecksum=InputTransformNode.Checksum)
TransformParent.append(OutputTransformNode)
elif os.path.exists(OutputTransformNode.FullPath):
# The meta-data and output exist, do nothing
return None
OutputTransformNode.SetTransform(InputTransformNode)
OutputTransformNode.InputPruneDataType = PruneNode.Type
OutputTransformNode.attrib[
'InputPruneDataChecksum'] = PruneDataNode.Checksum
if not Threshold is None:
OutputTransformNode.Threshold = Threshold
PruneObjInstance = cls.ReadPruneMap(PruneDataNode.FullPath)
PruneObjInstance.Tolerance = Threshold
assert (not PruneObjInstance is None)
PruneObjInstance.HistogramXMLFileFullPath = os.path.join(
PruneNodeParent.FullPath, HistogramXMLFile)
PruneObjInstance.HistogramImageFileFullPath = os.path.join(
PruneNodeParent.FullPath, HistogramImageFile)
try:
RemoveOutdatedFile(PruneDataNode.FullPath,
PruneObjInstance.HistogramImageFileFullPath)
RemoveOutdatedFile(PruneDataNode.FullPath,
PruneObjInstance.HistogramXMLFileFullPath)
HistogramImageNode = PruneNode.find('Image')
if not HistogramImageNode is None:
HistogramImageNode = transforms.RemoveOnMismatch(
HistogramImageNode,
'Threshold',
Threshold,
Precision=threshold_precision)
if HistogramImageNode is None or not os.path.exists(
PruneObjInstance.HistogramImageFileFullPath):
HistogramImageNode = VolumeManagerETree.ImageNode.Create(
HistogramImageFile)
(added, HistogramImageNode
) = PruneNode.UpdateOrAddChild(HistogramImageNode)
if not added:
#Handle the case where the path is different, such as when we change the extension type
if os.path.exists(HistogramImageNode.FullPath):
os.remove(HistogramImageNode.FullPath)
HistogramImageNode.Path = HistogramImageFile
HistogramImageNode.Threshold = Threshold
PruneObjInstance.CreateHistogram(
PruneObjInstance.HistogramXMLFileFullPath)
assert (HistogramImageNode.FullPath ==
PruneObjInstance.HistogramImageFileFullPath)
#if Async:
#pool = nornir_pools.GetMultithreadingPool("Histograms")
#pool.add_task("Create Histogram %s" % HistogramImageFile, plot.Histogram, HistogramXMLFile, HistogramImageFile, LinePosList=self.Tolerance, Title=Title)
#else:
plot.Histogram(PruneObjInstance.HistogramXMLFileFullPath,
HistogramImageNode.FullPath,
LinePosList=PruneObjInstance.Tolerance,
Title="Threshold " + str(Threshold))
print("Done!")
except Exception as E:
prettyoutput.LogErr("Exception creating prunemap histogram:" +
str(E.message))
pass
if (OutputTransformNode is None):
if (not hasattr(OutputTransformNode, Threshold)):
OutputTransformNode.Threshold = Threshold
if (OutputTransformNode.Threshold != Threshold):
if (os.path.exists(OutputMosaicFullPath)):
os.remove(OutputMosaicFullPath)
if not os.path.exists(OutputMosaicFullPath):
try:
PruneObjInstance.WritePruneMosaic(PruneNodeParent.FullPath,
InputTransformNode.FullPath,
OutputMosaicFullPath,
Tolerance=Threshold)
except (KeyError, ValueError):
os.remove(PruneDataNode.FullPath)
PruneNode.remove(PruneDataNode)
prettyoutput.LogErr("Remove prune data for section " +
PruneDataNode.FullPath)
return PruneNodeParent
OutputTransformNode.Type = MangledName
OutputTransformNode.Name = OutputTransformName
# Setting this value automatically converts the double to a string using the %g formatter. This is a precision of two. The RemoveOnMismatch value needs to use a matching precision
OutputTransformNode.Threshold = Threshold
OutputTransformNode.ResetChecksum()
# OutputTransformNode.Checksum = mosaicfile.MosaicFile.LoadChecksum(OutputTransformNode.FullPath)
return [TransformParent, PruneNodeParent]
def VolumeFinder(path=None,
OutputFile=None,
VolumeNode=None,
requiredFiles=None,
**kwargs):
'''Expects a 'Path' and 'RequiredFiles' keyword argument'
produces an HTML index of all volumes under the path'''
if requiredFiles is None:
requiredFiles = []
if (path is None):
VolumeNode = kwargs.get('VolumeNode', None)
if VolumeNode is None:
PrettyOutput.LogErr("Path attribute not found for VolumeFinder")
return
else:
path = VolumeNode.attrib['Path']
requiredFiles = kwargs.get('RequiredFiles', [])
if not isinstance(requiredFiles, list):
RequiredFileStrs = str(requiredFiles).strip().split(',')
requiredFiles = list()
for fileStr in RequiredFileStrs:
requiredFiles.append(fileStr)
ServerHostname = FindServerFromAboutXML(path)
if ServerHostname is None:
ServerHostname = ""
dirs = nornir_shared.files.RecurseSubdirectories(
path, RequiredFiles=requiredFiles, ExcludeNames="")
HTMLHeader = "<!DOCTYPE html> \n" + "<html>\n " + "<body>\n"
HTMLFooter = "</body>\n" + "</html>\n"
HTMLString = HTMLHeader
dirDict = {}
# Make sure required files is a list type if a single string was passed
if isinstance(requiredFiles, str):
requiredFiles = [requiredFiles]
# Seperate out specifically named files from regular expressions
RegExpPatterns = []
i = 0
while i < len(requiredFiles):
filename = requiredFiles[i]
if '*' in filename or '?' in filename:
RegExpPatterns.append(filename)
requiredFiles.pop(i)
else:
i = i + 1
for directory in dirs:
# Take the path from the list and find files.
dirImageList = dirDict.get(directory, [])
for pattern in RegExpPatterns:
filesMatchingPattern = glob.glob(os.path.join(directory, pattern))
for filename in filesMatchingPattern:
filenameFullPath = os.path.join(directory, filename)
dirImageList.append(
filenameFullPath) # Should check if it exists maybe
for filename in requiredFiles:
filenameFullPath = os.path.join(directory, filename)
if (os.path.exists(filenameFullPath)):
dirImageList.append(
filenameFullPath) # Should check if it exists maybe
dirDict[directory] = dirImageList
# Remove the path we were passed from the strings
HTMLString = HTMLString.replace(path, '')
RowNames = list(dirDict.keys())
RowNames.sort()
RowNames.reverse()
HTMLTable = HTMLTableForImageList(path, dirDict, RowNames, **kwargs)
HTMLString = HTMLString + HTMLTable + '\n'
HTMLString = HTMLString + HTMLFooter
if (len(RowNames) == 0):
PrettyOutput.Log("Report generator could not find matching files " +
str(requiredFiles))
return ""
if not OutputFile is None:
webPageFullPath = os.path.join(path, OutputFile)
f = open(webPageFullPath, 'w')
f.write(HTMLString)
f.close()
return None # HTMLString
