def get_source(self, environment, template):
from PYME.IO import unifiedIO
try:
if os.path.exists(os.path.join(os.path.dirname(__file__), template)):
source = unifiedIO.read(os.path.join(os.path.dirname(__file__), template)).decode('utf-8')
else:
source = unifiedIO.read(template).decode('utf-8')
except:
logger.exception('Error loading template')
raise jinja2.TemplateNotFound
return source, template, lambda: False
def run_template(request):
from PYME import config
from PYME.IO import unifiedIO
from PYME.recipes.modules import ModuleCollection
if config.get('PYMERuleserver-use', True):
from PYME.cluster.HTTPRulePusher import RecipePusher
else:
from PYME.cluster.HTTPTaskPusher import RecipePusher
recipeURI = 'pyme-cluster://%s/%s' % (
server_filter, request.POST.get('recipeURL').lstrip('/'))
output_directory = 'pyme-cluster://%s/%s' % (
server_filter, request.POST.get('recipeOutputPath').lstrip('/'))
recipe_text = unifiedIO.read(recipeURI)
recipe = ModuleCollection.fromYAML(recipe_text)
for file_input in recipe.file_inputs:
input_url = 'pyme-cluster://%s/%s' % (
server_filter, request.POST.get('%sURL' % file_input).lstrip('/'))
recipe_text = recipe_text.replace('{' + file_input + '}', input_url)
pusher = RecipePusher(recipe=recipe_text, output_dir=output_directory)
fileNames = request.POST.getlist('files', [])
pusher.fileTasksForInputs(input=fileNames)
return HttpResponseRedirect('/status/queues/')
def __init__(self, recipe=None, recipeURI=None, output_dir=None):
from PYME.recipes.modules import ModuleCollection
if recipe:
if isinstance(recipe, string_types):
self.recipe_text = recipe
self.recipe = ModuleCollection.fromYAML(recipe)
else:
self.recipe_text = recipe.toYAML()
self.recipe = recipe
self.recipeURI = None
else:
self.recipe = None
if recipeURI is None:
raise ValueError(
'recipeURI must be defined if no recipe given')
else:
from PYME.IO import unifiedIO
self.recipeURI = recipeURI
self.recipe = ModuleCollection.fromYAML(
unifiedIO.read(recipeURI))
self.output_dir = output_dir
self.taskQueueURI = _getTaskQueueURI()
#generate a queue ID as a hash of the recipe and the current time
to_hash = self.recipeURI if self.recipeURI else self.recipe_text
try: # hashlib requires bytes on py3
to_hash = to_hash.encode()
except TypeError: # encoding without a string argument, i.e. already bytes
pass
h = hashlib.md5(to_hash)
h.update(str(time.time()).encode())
self.queueID = h.hexdigest() # hexdigest returns str
def load_json(filename):
import json
from PYME.IO import unifiedIO
mdh = NestedClassMDHandler()
mdh.update(json.loads(unifiedIO.read(filename)))
return mdh
def load_shiftmap(uri):
"""
helper function to handle I/O of two versions of shiftmaps. Note that HDF is prefered
:param uri: str
path or url to shiftmap-containing file (hdf, or [less ideal] json)
:return: dict
shiftmap
"""
from PYME.IO import unifiedIO, tabular
from PYME.IO.MetaDataHandler import HDFMDHandler
import tables
import json
try: # try loading shift map as hdf file
with unifiedIO.local_or_temp_filename(uri) as f:
t = tables.open_file(f)
shift_map_source = tabular.HDFSource(t, 'shift_map') # todo - is there a cleaner way to do this?
shift_map_source.mdh = HDFMDHandler(t)
# build dict of dicts so we can easily rebuild shiftfield objects in multiview.calc_shifts_for_points
shift_map = {'shiftModel': shift_map_source.mdh['Multiview.shift_map.model']}
legend = shift_map_source.mdh['Multiview.shift_map.legend']
for l in legend.keys():
keys = shift_map_source.keys()
shift_map[l] = dict(zip(keys, [shift_map_source[k][legend[l]] for k in keys]))
t.close()
except tables.HDF5ExtError: # file is probably saved as json (legacy)
s = unifiedIO.read(uri)
shift_map = json.loads(s)
return shift_map
def execute(self, namespace):
from PYME.Analysis.points.astigmatism import astigTools
from PYME.IO import unifiedIO
import json
inp = namespace[self.input_name]
if 'mdh' not in dir(inp):
raise RuntimeError('MapAstigZ needs metadata')
if self.astigmatism_calibration_location == '': # grab calibration from the metadata
calibration_location = inp.mdh['Analysis.AstigmatismMapID']
else:
calibration_location = self.astigmatism_calibration_location
s = unifiedIO.read(calibration_location)
astig_calibrations = json.loads(s)
mapped = tabular.MappingFilter(inp)
z, zerr = astigTools.lookup_astig_z(mapped,
astig_calibrations,
self.rough_knot_spacing,
plot=False)
mapped.addColumn('astigmatic_z', z)
mapped.addColumn('astigmatic_z_lookup_error', zerr)
mapped.setMapping('z', 'astigmatic_z + z')
mapped.mdh = inp.mdh
mapped.mdh[
'Analysis.astigmatism_calibration_used'] = calibration_location
namespace[self.output_name] = mapped
def run_template(request):
from PYME import config
from PYME.IO import unifiedIO
from PYME.recipes import Recipe
from PYME.recipes import modules
from PYME.cluster.rules import RecipeRule
recipeURI = 'pyme-cluster://%s/%s' % (
server_filter, request.POST.get('recipeURL').lstrip('/'))
output_directory = 'pyme-cluster://%s/%s' % (
server_filter, request.POST.get('recipeOutputPath').lstrip('/'))
recipe_text = unifiedIO.read(recipeURI).decode('utf-8')
recipe = Recipe.fromYAML(recipe_text)
# handle templated userfile inputs - these will be loaded by e.g. unifiedIO later
for file_input in recipe.file_inputs:
input_url = 'pyme-cluster://%s/%s' % (
server_filter, request.POST.get('%sURL' % file_input).lstrip('/'))
recipe_text = recipe_text.replace('{' + file_input + '}', input_url)
rule = RecipeRule(recipe=recipe_text,
output_dir=output_directory,
inputs={'input': request.POST.getlist('files', [])})
rule.push()
return HttpResponseRedirect('/status/queues/')
def __init__(self, recipe=None, recipeURI=None):
from PYME.recipes.modules import ModuleCollection
if recipe:
if isinstance(recipe, string_types):
self.recipe_text = recipe
self.recipe = ModuleCollection.fromYAML(recipe)
else:
self.recipe_text = recipe.toYAML()
self.recipe = recipe
self.recipeURI = None
else:
self.recipe = None
if recipeURI is None:
raise ValueError('recipeURI must be defined if no recipe given')
else:
from PYME.IO import unifiedIO
self.recipeURI = recipeURI
self.recipe = ModuleCollection.fromYAML(unifiedIO.read(recipeURI))
self.taskQueueURI = _getTaskQueueURI()
#generate a queue ID as a hash of the recipe and the current time
h = hashlib.md5(self.recipeURI if self.recipeURI else self.recipe_text)
h.update('%s' % time.time())
self.queueID = h.hexdigest()
def view_svg(request):
from PYME.IO import unifiedIO
from PYME.recipes.modules import ModuleCollection
from PYME.recipes import recipeLayout
recipeURI = ('pyme-cluster://%s/' %
server_filter) + request.GET.get('recipeURL').lstrip('/')
recipe = ModuleCollection.fromYAML(unifiedIO.read(recipeURI))
svg = recipeLayout.to_svg(recipe.dependancyGraph())
return HttpResponse(svg, content_type='image/svg+xml')
def extra_inputs(request):
from PYME.IO import unifiedIO
from PYME.recipes.modules import ModuleCollection
recipeURI = ('pyme-cluster://%s/' %
server_filter) + request.GET.get('recipeURL').lstrip('/')
recipe = ModuleCollection.fromYAML(unifiedIO.read(recipeURI))
return render(request, 'recipes/extra_inputs.html', {
'file_inputs': recipe.file_inputs,
'serverfilter': server_filter
})
def __init__(self,
seriesName,
analysisMetadata,
resultsFilename=None,
startAt=0,
dataSourceModule=None,
serverfilter=clusterIO.local_serverfilter,
**kwargs):
from PYME.IO import MetaDataHandler
from PYME.Analysis import MetaData
from PYME.IO.FileUtils.nameUtils import genClusterResultFileName
from PYME.IO import unifiedIO
unifiedIO.assert_uri_ok(seriesName)
if resultsFilename is None:
resultsFilename = genClusterResultFileName(seriesName)
resultsFilename = verify_cluster_results_filename(resultsFilename)
logger.info('Results file: ' + resultsFilename)
resultsMdh = MetaDataHandler.NestedClassMDHandler()
# NB - anything passed in analysis MDH will wipe out corresponding entries in the series metadata
resultsMdh.update(
json.loads(unifiedIO.read(seriesName + '/metadata.json')))
resultsMdh.update(analysisMetadata)
resultsMdh['EstimatedLaserOnFrameNo'] = resultsMdh.getOrDefault(
'EstimatedLaserOnFrameNo',
resultsMdh.getOrDefault('Analysis.StartAt', 0))
MetaData.fixEMGain(resultsMdh)
self._setup(seriesName, resultsMdh, resultsFilename, startAt,
serverfilter)
#load data source
if dataSourceModule is None:
DataSource = DataSources.getDataSourceForFilename(seriesName)
else:
DataSource = __import__(
'PYME.IO.DataSources.' + dataSourceModule,
fromlist=['PYME', 'io',
'DataSources']).DataSource #import our data source
self.ds = DataSource(seriesName)
logger.debug('DataSource.__class__: %s' % self.ds.__class__)
Rule.__init__(self, **kwargs)
def createFitTaskFromTaskDef(task):
"""
Creates a fit task from a new-style json task definition
Parameters
----------
task : dict
The parsed task definition. As the task definition will need to be parsed by the worker before we get here,
we expect this to take the form of a python dictionary.
Returns
-------
a fitTask instance
"""
from PYME.IO import MetaDataHandler
dataSourceID = task['inputs']['frames']
frameIndex = int(task['taskdef']['frameIndex'])
#logger.debug('Creating a task for %s - frame %d' % (dataSourceID, frameIndex))
md = task['taskdef']['metadata']
#sort out our metadata
#TODO - Move this somewhere saner - e.g. a helper function in the MetaDataHandler module
mdh = MetaDataHandler.NestedClassMDHandler()
if isinstance(md, dict):
#metadata was parsed with the enclosing json
mdh.update(md)
elif isinstance(md, six.string_types):
if md.startswith('{'):
#metadata is a quoted json dump
import json
mdh.update(json.loads(md))
else:
#metadata entry is a filename/URI
from PYME.IO import unifiedIO
if md.endswith('.json'):
import json
mdh.update(json.loads(unifiedIO.read(md)))
else:
raise NotImplementedError(
'Loading metadata from a URI in task description is not yet supported'
)
return fitTask(dataSourceID=dataSourceID,
frameIndex=frameIndex,
metadata=mdh)
def launch_localize(analysisMDH, seriesName):
"""
Pushes an analysis task for a given series to the distributor
Parameters
----------
analysisMDH : dictionary-like
MetaDataHandler describing the analysis tasks to launch
seriesName : str
cluster path, e.g. pyme-cluster:///example_folder/series
Returns
-------
"""
#import logging
import json
#from PYME.ParallelTasks import HTTPTaskPusher
from PYME.IO import MetaDataHandler
from PYME.Analysis import MetaData
from PYME.IO.FileUtils.nameUtils import genClusterResultFileName
from PYME.IO import unifiedIO
unifiedIO.assert_uri_ok(seriesName)
seriesName = seriesName
resultsFilename = verify_cluster_results_filename(
genClusterResultFileName(seriesName))
logger.info('Results file: ' + resultsFilename)
resultsMdh = MetaDataHandler.NestedClassMDHandler()
# NB - anything passed in analysis MDH will wipe out corresponding entries in the series metadata
resultsMdh.update(json.loads(unifiedIO.read(seriesName +
'/metadata.json')))
resultsMdh.update(analysisMDH)
resultsMdh['EstimatedLaserOnFrameNo'] = resultsMdh.getOrDefault(
'EstimatedLaserOnFrameNo',
resultsMdh.getOrDefault('Analysis.StartAt', 0))
MetaData.fixEMGain(resultsMdh)
# resultsMdh['DataFileID'] = fileID.genDataSourceID(image.dataSource)
# TODO - do we need to keep track of the pushers in some way (we currently rely on the fact that the pushing thread
# will hold a reference
pusher = HTTPRulePusher(dataSourceID=seriesName,
metadata=resultsMdh,
resultsFilename=resultsFilename)
logging.debug('Queue created')
def execute(self, namespace):
from PYME.Analysis.points import multiview
from PYME.IO import unifiedIO
from PYME.IO.MetaDataHandler import HDFMDHandler
import tables
import json
inp = namespace[self.input_name]
if 'mdh' not in dir(inp):
raise RuntimeError('ShiftCorrect needs metadata')
if self.shift_map_path == '': # grab shftmap from the metadata
loc = inp.mdh['Shiftmap']
else:
loc = self.shift_map_path
try: # try loading shift map as hdf file
with unifiedIO.local_or_temp_filename(loc) as f:
t = tables.open_file(f)
shift_map_source = tabular.HDFSource(
t,
'shift_map') # todo - is there a cleaner way to do this?
shift_map_source.mdh = HDFMDHandler(t)
# build dict of dicts so we can easily rebuild shiftfield objects in multiview.calc_shifts_for_points
shift_map = {
'shiftModel': shift_map_source.mdh['Multiview.shift_map.model']
}
legend = shift_map_source.mdh['Multiview.shift_map.legend']
for l in legend.keys():
keys = shift_map_source.keys()
shift_map[l] = dict(
zip(keys, [shift_map_source[k][legend[l]] for k in keys]))
t.close()
except tables.HDF5ExtError: # file is probably saved as json (legacy)
s = unifiedIO.read(self.shift_map_path)
shift_map = json.loads(s)
mapped = tabular.MappingFilter(inp)
multiview.apply_shifts_to_points(mapped, shift_map)
# propagate metadata
mapped.mdh = inp.mdh
mapped.mdh['Multiview.shift_map.location'] = loc
namespace[self.output_name] = mapped
def update_from_file(self, filename):
"""
Update the contents of the recipe from a .yaml file
WARNING: This function will likely be REMOVED WITHOUT NOTICE. It is a quick hack to get the prototype web-based
recipe editor working, but will be surplus to requirements once we have a proper recipe manager in the web based
editor. It's logically obtuse to consider something the same recipe once you've completely replaced it with a
recipe that has been loaded from file. It is much more sensible to create a new recipe instance when loading
a recipe from file, and this is the recommended approach.
Parameters
----------
filename: str
filename or PYME-CLUSTER:// URI
"""
from PYME.IO import unifiedIO
self.update_from_yaml(unifiedIO.read(filename).decode())
def computeLoop(self):
while self._loop_alive:
#loop over tasks - we pop each task and then delete it after processing
#to keep memory usage down
queueURL, taskDescr = self.inputQueue.get()
if taskDescr['type'] == 'localization':
try:
task = remFitBuf.createFitTaskFromTaskDef(taskDescr)
res = task()
self.resultsQueue.put((queueURL, taskDescr, res))
except:
import traceback
traceback.print_exc()
tb = traceback.format_exc()
logger.exception(tb)
self.resultsQueue.put(
(queueURL, taskDescr, TaskError(taskDescr, tb)))
#self.resultsQueue.put((queueURL, taskDescr, None))
elif taskDescr['type'] == 'recipe':
from PYME.recipes import Recipe
from PYME.recipes import modules
try:
taskdefRef = taskDescr.get('taskdefRef', None)
if taskdefRef: #recipe is defined in a file - go find it
recipe_yaml = unifiedIO.read(taskdefRef)
else: #recipe is defined in the task
recipe_yaml = taskDescr['taskdef']['recipe']
recipe = Recipe.fromYAML(recipe_yaml)
#load recipe inputs
logging.debug(taskDescr)
for key, url in taskDescr['inputs'].items():
logging.debug('RECIPE: loading %s as %s' % (url, key))
recipe.loadInput(url, key)
#print recipe.namespace
recipe.execute()
#save results
context = {
'data_root': clusterIO.local_dataroot,
'task_id': taskDescr['id'].split('~')[0]
}
#update context with file stub and input directory
try:
principle_input = taskDescr['inputs'][
'input'] #default input
context['file_stub'] = os.path.splitext(
os.path.basename(principle_input))[0]
context['input_dir'] = unifiedIO.dirname(
principle_input)
except KeyError:
pass
try:
od = taskDescr['output_dir']
# make sure we have a trailing slash
# TODO - this should be fine for most windows use cases, as you should generally
# use POSIX urls for the cluster/cluster of one, but might need checking
if not od.endswith('/'):
od = od + '/'
context['output_dir'] = unifiedIO.dirname(od)
except KeyError:
pass
#print taskDescr['inputs']
#print context
#abuse outputs as context
outputs = taskDescr.get('outputs', None)
if not outputs is None:
context.update(outputs)
#print context, context['input_dir']
recipe.save(context)
self.resultsQueue.put((queueURL, taskDescr, True))
except Exception:
import traceback
traceback.print_exc()
tb = traceback.format_exc()
logger.exception(tb)
self.resultsQueue.put(
(queueURL, taskDescr, TaskError(taskDescr, tb)))
def _findAndParseMetadata(self, filename):
"""Try and find and load a .xml or .md metadata file that might be ascociated
with a given image filename. See the relevant metadatahandler classes
for details."""
import xml.parsers.expat
from PYME.IO import unifiedIO
if not self.mdh is None:
return #we already have metadata (probably passed in on command line)
mdf = None
xmlfn = os.path.splitext(filename)[0] + '.xml'
xmlfnmc = os.path.splitext(filename)[0].split('__')[0] + '.xml'
if os.path.exists(xmlfn):
try:
self.mdh = MetaDataHandler.NestedClassMDHandler(
MetaData.TIRFDefault)
self.mdh.copyEntriesFrom(MetaDataHandler.XMLMDHandler(xmlfn))
mdf = xmlfn
except xml.parsers.expat.ExpatError:
#fix for bug in which PYME .md was written with a .xml extension
self.mdh = MetaDataHandler.NestedClassMDHandler(
MetaData.BareBones)
self.mdh.copyEntriesFrom(
MetaDataHandler.SimpleMDHandler(xmlfn))
mdf = xmlfn
elif os.path.exists(
xmlfnmc): #this is a single colour channel of a pair
self.mdh = MetaDataHandler.NestedClassMDHandler(
MetaData.TIRFDefault)
self.mdh.copyEntriesFrom(MetaDataHandler.XMLMDHandler(xmlfnmc))
mdf = xmlfnmc
else:
self.mdh = MetaDataHandler.NestedClassMDHandler(MetaData.BareBones)
#check for simple metadata (python code with an .md extension which
#fills a dictionary called md)
mdfn = os.path.splitext(filename)[0] + '.md'
jsonfn = os.path.splitext(filename)[0] + '.json'
if os.path.exists(mdfn):
self.mdh.copyEntriesFrom(MetaDataHandler.SimpleMDHandler(mdfn))
mdf = mdfn
elif os.path.exists(jsonfn):
import json
with open(jsonfn, 'r') as f:
mdd = json.load(f)
self.mdh.update(mdd)
elif filename.endswith('.lsm'):
#read lsm metadata
from PYME.contrib.gohlke.tifffile import TIFFfile
tf = TIFFfile(filename)
lsm_info = tf[0].cz_lsm_scan_information
self.mdh['voxelsize.x'] = lsm_info['line_spacing']
self.mdh['voxelsize.y'] = lsm_info['line_spacing']
self.mdh['voxelsize.z'] = lsm_info['plane_spacing']
def lsm_pop(basename, dic):
for k, v in dic.items():
if isinstance(v, list):
#print k, v
for i, l_i in enumerate(v):
#print i, l_i, basename
lsm_pop(
basename + k + '.' + k[:-1] + '%i.' % i,
l_i)
else:
self.mdh[basename + k] = v
lsm_pop('LSM.', lsm_info)
elif filename.endswith('.tif'):
#look for OME data...
from PYME.contrib.gohlke.tifffile import TIFFfile
tf = TIFFfile(filename)
if tf.is_ome:
try:
omemdh = MetaDataHandler.OMEXMLMDHandler(
tf.pages[0].tags['image_description'].value)
self.mdh.copyEntriesFrom(omemdh)
except IndexError:
pass
elif filename.endswith('.dcimg'): #Bewersdorf lab Biplane
# FIXME load seriesXX.json for seriesXX_chunkXX.dcimg files more elegantly
jsonfn = filename[:-22] + '.json'
import json
try:
mdd = json.loads(unifiedIO.read(jsonfn))
self.mdh.update(mdd)
except IOError:
pass
elif filename.endswith('.dbl'): #Bewersdorf lab STED
mdfn = filename[:-4] + '.txt'
entrydict = {}
try: #try to read in extra metadata if possible
with unifiedIO.openFile(mdfn, 'r') as mf:
for line in mf:
s = line.split(':')
if len(s) == 2:
entrydict[s[0]] = s[1]
except IOError:
pass
# vx, vy = entrydict['Pixel size (um)'].split('x')
# self.mdh['voxelsize.x'] = float(vx)
# self.mdh['voxelsize.y'] = float(vy)
# self.mdh['voxelsize.z'] = 0.2 #FIXME for stacks ...
#
# sx, sy = entrydict['Image format'].split('x')
# self.mdh['Camera.ROIWidth'] = int(sx)
# self.mdh['Camera.ROIHeight'] = int(sy)
#
# self.mdh['NumImages'] = int(entrydict['# Images'])
with unifiedIO.openFile(filename) as df:
s = df.read(8)
Z, X, Y, T = numpy.fromstring(s, '>u2')
s = df.read(16)
depth, width, height, elapsed = numpy.fromstring(s, '<f4')
self.mdh['voxelsize.x'] = width / X
self.mdh['voxelsize.y'] = height / Y
self.mdh['voxelsize.z'] = depth
self.mdh['Camera.ROIWidth'] = X
self.mdh['Camera.ROIHeight'] = Y
self.mdh['NumImages'] = Z * T
def _sanitise_key(key):
k = key.replace('#', 'Num')
k = k.replace('(%)', '')
k = k.replace('(', '')
k = k.replace(')', '')
k = k.replace('.', '')
k = k.replace('/', '')
k = k.replace('?', '')
k = k.replace(' ', '')
if not k[0].isalpha():
k = 's' + k
return k
for k, v in entrydict.items():
self.mdh['STED.%s' % _sanitise_key(k)] = v
#else: #try bioformats
# OMEXML = bioformats.get_omexml_metadata(filename).encode('utf8')
# OMEmd = MetaDataHandler.OMEXMLMDHandler(OMEXML)
# self.mdh.copyEntriesFrom(OMEmd)
if self.haveGUI and not ('voxelsize.x' in self.mdh.keys()
and 'voxelsize.y' in self.mdh.keys()):
from PYME.DSView.voxSizeDialog import VoxSizeDialog
dlg = VoxSizeDialog(None)
dlg.ShowModal()
self.mdh.setEntry('voxelsize.x', dlg.GetVoxX())
self.mdh.setEntry('voxelsize.y', dlg.GetVoxY())
self.mdh.setEntry('voxelsize.z', dlg.GetVoxZ())
return mdf
