def matlabFindTargets(self):
pymat.put(self.handle, 'focus', [])
pymat.put(self.handle, 'acquisition', [])
d, f = os.path.split(self.settings['module path'])
if d:
pymat.eval(self.handle, 'path(path, \'%s\')' % d)
if not f[:-2]:
raise RuntimeError
pymat.eval(self.handle,
'[acquisition, focus] = %s(image,image_id)' % f[:-2])
focus = pymat.get(self.handle, 'focus')
acquisition = pymat.get(self.handle, 'acquisition')
self.setTargets(acquisition, 'acquisition')
self.setTargets(focus, 'focus')
import time
time.sleep(1)
if self.settings['user check']:
self.panel.foundTargets()
def get3d(handle, name):
mlabraw.eval(handle, """
flat_array = %s(:);
shape = size(%s);
"""%(name, name))
flat_array = mlabraw.get(handle, "flat_array")
shape = map(int, mlabraw.get(handle, "shape").flat)
return np.ndarray(buffer = flat_array, shape = shape, order="F")
def _get(self, name, remove=False):
varname = name
vartype = self._var_type(varname)
if vartype in self._mlabraw_can_convert:
var = mlabraw.get(self._session, varname)
if type(var) is Numeric.ArrayType:
if self._flatten_row_vecs and Numeric.shape(var)[0] == 1:
var.shape = var.shape[1:2]
elif self._flatten_col_vecs and Numeric.shape(var)[1] == 1:
var.shape = var.shape[0:1]
if self._array_cast:
var = self._array_cast(var)
else:
var = None
if self._optionally_convert.get(vartype):
# manual conversions may fail (e.g. for multidimensional
# cell arrays), in that case just fall back on proxying.
try:
var = self._manually_convert(varname, vartype)
except MlabConversionError: pass
if var is None:
# we can't convert this to a python object, so we just
# create a proxy, and don't delete the real matlab
# reference until the proxy is garbage collected
var = self._make_proxy(varname)
if remove:
mlabraw.eval(self._session, "clear('%s');" % varname)
return var
def _get(self, name, remove=False):
r"""Directly access a variable in matlab space.
This should normally not be used by user code."""
# FIXME should this really be needed in normal operation?
if name in self._proxies: return self._proxies[name]
varname = name
vartype = self._var_type(varname)
if vartype in self._mlabraw_can_convert:
var = mlabraw.get(self._session, varname)
if isinstance(var, ndarray):
if self._flatten_row_vecs and numpy.shape(var)[0] == 1:
var.shape = var.shape[1:2]
elif self._flatten_col_vecs and numpy.shape(var)[1] == 1:
var.shape = var.shape[0:1]
if self._array_cast:
var = self._array_cast(var)
else:
var = None
if self._dont_proxy.get(vartype):
# manual conversions may fail (e.g. for multidimensional
# cell arrays), in that case just fall back on proxying.
try:
var = self._manually_convert(varname, vartype)
except MlabConversionError:
pass
if var is None:
# we can't convert this to a python object, so we just
# create a proxy, and don't delete the real matlab
# reference until the proxy is garbage collected
var = self._make_proxy(varname)
if remove:
mlabraw.eval(self._session, "clear('%s');" % varname)
return var
def _get(self, name, remove=False):
r"""Directly access a variable in matlab space.
This should normally not be used by user code."""
# FIXME should this really be needed in normal operation?
if name in self._proxies: return self._proxies[name]
varname = name
vartype = self._var_type(varname)
if vartype in self._mlabraw_can_convert:
var = mlabraw.get(self._session, varname)
if isinstance(var, ndarray):
if var.shape:
if self._flatten_row_vecs and numpy.shape(var)[0] == 1:
var.shape = var.shape[1:2]
elif len(var.shape) > 1 and self._flatten_col_vecs and numpy.shape(var)[1] == 1:
var.shape = var.shape[0:1]
if self._array_cast:
var = self._array_cast(var)
else:
var = None
if self._dont_proxy.get(vartype):
# manual conversions may fail (e.g. for multidimensional
# cell arrays), in that case just fall back on proxying.
try:
var = self._manually_convert(varname, vartype)
except MlabConversionError: pass
if var is None:
# we can't convert this to a python object, so we just
# create a proxy, and don't delete the real matlab
# reference until the proxy is garbage collected
var = self._make_proxy(varname)
if remove:
mlabraw.eval(self._session, "clear('%s');" % varname)
return var
def _var_type(self, varname):
mlabraw.eval(self._session,
"TMP_CLS__ = class(%(x)s); if issparse(%(x)s),"
"TMP_CLS__ = [TMP_CLS__,'-sparse']; end;" % dict(x=varname))
res_type = mlabraw.get(self._session, "TMP_CLS__")
mlabraw.eval(self._session, "clear TMP_CLS__;") # unlikely to need try/finally to ensure clear
return res_type
def transform_points(self, points):
mlabraw.put(self.handle, "points", points)
mlabraw.eval(self.handle,"""
points_result = tps_eval(points, params);
""")
points_result = mlabraw.get(self.handle, "points_result")
return points_result
def _var_type(self, varname):
mlabraw.eval(
self._session, "TMP_CLS__ = class(%(x)s); if issparse(%(x)s),"
"TMP_CLS__ = [TMP_CLS__,'-sparse']; end;" % dict(x=varname))
res_type = mlabraw.get(self._session, "TMP_CLS__")
mlabraw.eval(
self._session,
"clear TMP_CLS__;") # unlikely to need try/finally to ensure clear
return res_type
def branch_points(bw):
initialize()
mlabraw.put(MATLAB, "bw",bw)
mlabraw.eval(MATLAB, """
bp = bwmorph(bw,'branchpoints')
bp_d = double(bp);
""")
bp_d = mlabraw.get(MATLAB, "bp_d")
bp = bp_d.astype('uint8')
return bp
def branch_points(bw):
initialize()
mlabraw.put(MATLAB, "bw", bw)
mlabraw.eval(
MATLAB, """
bp = bwmorph(bw,'branchpoints')
bp_d = double(bp);
""")
bp_d = mlabraw.get(MATLAB, "bp_d")
bp = bp_d.astype('uint8')
return bp
def _var_type(self, varname):
"""Ask matlab what the type of varname is.
:param varname: string variable
:return: string type, e.g. ``double`` or ``char``.
"""
mlabraw.eval(self._session,
"TMP_CLS__ = class(%(x)s); if issparse(%(x)s),"
"TMP_CLS__ = [TMP_CLS__,'-sparse']; end;" % dict(x=varname))
res_type = mlabraw.get(self._session, "TMP_CLS__")
mlabraw.eval(self._session, "clear TMP_CLS__;") # unlikely to need try/finally to ensure clear
return res_type
def _var_type(self, varname):
"""Ask matlab what the type of varname is.
:param varname: string variable
:return: string type, e.g. ``double`` or ``char``.
"""
mlabraw.eval(self._session,
"TMP_CLS__ = class(%(x)s); if issparse(%(x)s),"
"TMP_CLS__ = [TMP_CLS__,'-sparse']; end;" % dict(x=varname))
res_type = mlabraw.get(self._session, "TMP_CLS__")
mlabraw.eval(self._session, "clear TMP_CLS__;") # unlikely to need try/finally to ensure clear
return res_type
def transform_poses(self, points, rots):
mlabraw.put(self.handle, "points", points)
put3d(self.handle, "rots", rots)
mlabraw.eval(self.handle,"""
[points_result, rots_result] = tps_eval_frames(points, rots, params);
""")
points_result = mlabraw.get(self.handle,"points_result")
rots_result = get3d(self.handle, "rots_result")
return points_result, rots_result
def matlabFindTargets(self):
pymat.put(self.handle, 'focus', [])
pymat.put(self.handle, 'acquisition', [])
d, f = os.path.split(self.settings['module path'])
if d:
pymat.eval(self.handle, 'path(path, \'%s\')' % d)
if not f[:-2]:
raise RuntimeError
pymat.eval(self.handle, '[acquisition, focus] = %s(image,image_id)' % f[:-2])
focus = pymat.get(self.handle, 'focus')
acquisition = pymat.get(self.handle, 'acquisition')
self.setTargets(acquisition, 'acquisition')
self.setTargets(focus, 'focus')
import time
time.sleep(1)
if self.settings['user check']:
self.panel.foundTargets()
def testRawMlabraw(self):
"""A few explicit tests for mlabraw"""
import mlabraw
#print "test mlabraw"
self.assertRaises(TypeError, mlabraw.put, 33, 'a', 1)
self.assertRaises(TypeError, mlabraw.get, object(), 'a')
self.assertRaises(TypeError, mlabraw.eval, object(), '1')
# -100 is picked kinda arbitrarily to account for internal "overhead";
# I don't want to hardcode the exact value; users can assume 1000
# chars is safe
mlabraw.eval(mlab._session, '1' * (BUFSIZE - 100))
assert numpy.inf == mlabraw.get(mlab._session, 'ans');
# test for buffer overflow detection
self.assertRaises(Exception, mlabraw.eval, mlab._session, '1' * BUFSIZE)
def remove_holes(labels,min_size):
initialize()
mlabraw.put(MATLAB, "L",labels)
mlabraw.put(MATLAB, "min_size",min_size)
mlabraw.eval(MATLAB, """
max_label = max(L(:));
good_pix = L==0;
for label = 1:max_label
good_pix = good_pix | bwareaopen(L==label,min_size,4);
end
bad_pix = ~logical(good_pix);
[~,I] = bwdist(good_pix,'Chessboard');
NewL = L;
NewL(bad_pix) = L(I(bad_pix));
NewL_d = double(NewL);
""")
NewL_d = mlabraw.get(MATLAB, "NewL_d")
return NewL_d.astype('uint8')
def remove_holes(labels, min_size):
initialize()
mlabraw.put(MATLAB, "L", labels)
mlabraw.put(MATLAB, "min_size", min_size)
mlabraw.eval(
MATLAB, """
max_label = max(L(:));
good_pix = L==0;
for label = 1:max_label
good_pix = good_pix | bwareaopen(L==label,min_size,4);
end
bad_pix = ~logical(good_pix);
[~,I] = bwdist(good_pix,'Chessboard');
NewL = L;
NewL(bad_pix) = L(I(bad_pix));
NewL_d = double(NewL);
""")
NewL_d = mlabraw.get(MATLAB, "NewL_d")
return NewL_d.astype('uint8')
def testRawMlabraw(self):
"""A few explicit tests for mlabraw"""
import mlabraw
#print "test mlabraw"
self.assertRaises(TypeError, mlabraw.put, 33, 'a', 1)
self.assertRaises(TypeError, mlabraw.get, object(), 'a')
self.assertRaises(TypeError, mlabraw.eval, object(), '1')
# -100 is picked kinda arbitrarily to account for internal "overhead";
# I don't want to hardcode the exact value; users can assume 1000
# chars is safe
mlabraw.eval(mlab._session, '1' * (BUFSIZE - 100))
assert numpy.inf == mlabraw.get(mlab._session, 'ans')
# test for buffer overflow detection
self.assertRaises(Exception, mlabraw.eval, mlab._session,
'1' * BUFSIZE)
self.assertEqual(mlabraw.eval(mlab._session, r"fprintf('1\n')"), '1\n')
try:
self.assertEqual(mlabraw.eval(mlab._session, r"1"), '')
finally:
mlabraw.eval(mlab._session, 'clear ans')
def runAce(matlab, imgdata, params, showprev=True):
imgname = imgdata['filename']
if showprev is True:
bestctfvalue = ctfdb.getBestCtfByResolution(imgdata)
if bestctfvalue:
bestconf = ctfdb.calculateConfidenceScore(bestctfvalue)
print ( "Prev best: '"+bestctfvalue['acerun']['name']+"', conf="+
apDisplay.colorProb(bestconf)+", defocus="+str(round(-1.0*abs(bestctfvalue['defocus1']*1.0e6),2))+
" microns" )
if params['uncorrected']:
tmpname='temporaryCorrectedImage.mrc'
imgarray = apDBImage.correctImage(imgdata)
imgpath = os.path.join(params['rundir'],tmpname)
apImage.arrayToMrc(imgarray, imgpath)
print "processing", imgpath
else:
imgpath = os.path.join(imgdata['session']['image path'], imgname+'.mrc')
nominal = None
if params['nominal'] is not None:
nominal=params['nominal']
elif params['newnominal'] is True:
bestctfvalue = ctfdb.getBestCtfByResolution(imgdata)
nominal = bestctfvalue['defocus1']
if nominal is None:
nominal = imgdata['scope']['defocus']
if nominal is None or nominal > 0 or nominal < -15e-6:
apDisplay.printWarning("Nominal should be of the form nominal=-1.2e-6"+\
" for -1.2 microns NOT:"+str(nominal))
#Neil's Hack
#if 'autosample' in params and params['autosample']:
# x = abs(nominal*1.0e6)
# val = 1.585 + 0.057587 * x - 0.044106 * x**2 + 0.010877 * x**3
# resamplefr_override = round(val,3)
# print "resamplefr_override=",resamplefr_override
# pymat.eval(matlab, "resamplefr="+str(resamplefr_override)+";")
pymat.eval(matlab,("dforig = %e;" % nominal))
if params['stig'] == 0:
plist = (imgpath, params['outtextfile'], params['display'], params['stig'],\
params['medium'], -nominal, params['tempdir']+"/")
acecmd = makeMatlabCmd("ctfparams = ace(",");",plist)
else:
plist = (imgname, imgpath, params['outtextfile'], params['opimagedir'], \
params['matdir'], params['display'], params['stig'],\
params['medium'], -nominal, params['tempdir']+"/", params['resamplefr'])
acecmd = makeMatlabCmd("ctfparams = measureAstigmatism(",");",plist)
#print acecmd
pymat.eval(matlab,acecmd)
matfile = os.path.join(params['matdir'], imgname+".mrc.mat")
if params['stig']==0:
savematcmd = "save('"+matfile+"','ctfparams','scopeparams', 'dforig');"
pymat.eval(matlab,savematcmd)
ctfvalue = pymat.get(matlab, 'ctfparams')
ctfvalue=ctfvalue[0]
printResults(params, nominal, ctfvalue)
return ctfvalue
def get(name):
return mlabraw.get(MATLAB, name)
def get(self, name):
if DEBUG: print "getting '%s'"%name
return mlabraw.get(self._engine, name)
def get(name):
return mlabraw.get(MATLAB, name)
def _execute(self):
batch_size = self.batch_size
pooling_region_counts = self.pooling_region_counts
dataset_family = self.dataset_family
which_set = self.which_set
size = self.size
nan = 0
dataset_descriptor = dataset_family[which_set][size]
dataset = dataset_descriptor.dataset_maker()
expected_num_examples = dataset_descriptor.num_examples
full_X = dataset.get_design_matrix()
num_examples = full_X.shape[0]
assert num_examples == expected_num_examples
if self.restrict is not None:
assert self.restrict[1] <= full_X.shape[0]
print 'restricting to examples ',self.restrict[0],' through ',self.restrict[1],' exclusive'
full_X = full_X[self.restrict[0]:self.restrict[1],:]
assert self.restrict[1] > self.restrict[0]
#update for after restriction
num_examples = full_X.shape[0]
assert num_examples > 0
dataset.X = None
dataset.design_loc = None
dataset.compress = False
patchifier = ExtractGridPatches( patch_shape = (size,size), patch_stride = (1,1) )
pipeline = serial.load(dataset_descriptor.pipeline_path)
assert isinstance(pipeline.items[0], ExtractPatches)
pipeline.items[0] = patchifier
print 'defining features'
Z = T.matrix('Z')
if self.one_sided:
feat = abs(Z)
else:
pos = T.clip(Z,0.,1e30)
neg = T.clip(-Z,0.,1e30)
feat = T.concatenate((pos, neg), axis=1)
print 'compiling theano function'
f = function([Z],feat)
nfeat = self.W.shape[1] * (2 - self.one_sided)
if not (nfeat == 1600 or nfeat == 3200):
print nfeat
assert False
if config.device.startswith('gpu') and nfeat >= 4000:
f = halver(f, nfeat)
topo_feat_var = T.TensorType(broadcastable = (False,False,False,False), dtype='float32')()
region_features = function([topo_feat_var],
topo_feat_var.mean(axis=(1,2)) )
def average_pool( stride ):
def point( p ):
return p * ns / stride
rval = np.zeros( (topo_feat.shape[0], stride, stride, topo_feat.shape[3] ) , dtype = 'float32')
for i in xrange(stride):
for j in xrange(stride):
rval[:,i,j,:] = region_features( topo_feat[:,point(i):point(i+1), point(j):point(j+1),:] )
return rval
outputs = [ np.zeros((num_examples,count,count,nfeat),dtype='float32') for count in pooling_region_counts ]
assert len(outputs) > 0
fd = DenseDesignMatrix(X = np.zeros((1,1),dtype='float32'), view_converter = DefaultViewConverter([1, 1, nfeat] ) )
ns = 32 - size + 1
depatchifier = ReassembleGridPatches( orig_shape = (ns, ns), patch_shape=(1,1) )
if len(range(0,num_examples-batch_size+1,batch_size)) <= 0:
print num_examples
print batch_size
for i in xrange(0,num_examples-batch_size+1,batch_size):
print i
t1 = time.time()
d = copy.copy(dataset)
d.set_design_matrix(full_X[i:i+batch_size,:])
t2 = time.time()
#print '\tapplying preprocessor'
d.apply_preprocessor(pipeline, can_fit = False)
X2 = d.get_design_matrix()
t3 = time.time()
M.put(s,'batch',X2)
M.eval(s, 'Z = sparse_codes(batch, dictionary, lambda)')
Z = M.get(s, 'Z')
feat = f(np.cast['float32'](Z))
t4 = time.time()
assert feat.dtype == 'float32'
feat_dataset = copy.copy(fd)
if np.any(np.isnan(feat)):
nan += np.isnan(feat).sum()
feat[np.isnan(feat)] = 0
feat_dataset.set_design_matrix(feat)
#print '\treassembling features'
feat_dataset.apply_preprocessor(depatchifier)
#print '\tmaking topological view'
topo_feat = feat_dataset.get_topological_view()
assert topo_feat.shape[0] == batch_size
t5 = time.time()
#average pooling
for output, count in zip(outputs, pooling_region_counts):
output[i:i+batch_size,...] = average_pool(count)
t6 = time.time()
print (t6-t1, t2-t1, t3-t2, t4-t3, t5-t4, t6-t5)
for output, save_path in zip(outputs, self.save_paths):
if self.chunk_size is not None:
assert save_path.endswith('.npy')
save_path_pieces = save_path.split('.npy')
assert len(save_path_pieces) == 2
assert save_path_pieces[1] == ''
save_path = save_path_pieces[0] + '_' + chr(ord('A')+self.chunk_id)+'.npy'
np.save(save_path,output)
if nan > 0:
warnings.warn(str(nan)+' features were nan')
def runAce(matlab, imgdata, params, showprev=True):
imgname = imgdata['filename']
if showprev is True:
bestctfvalue, bestconf = ctfdb.getBestCtfValueForImage(imgdata)
if bestctfvalue:
print ( "Prev best: '"+bestctfvalue['acerun']['name']+"', conf="+
apDisplay.colorProb(bestconf)+", defocus="+str(round(-1.0*abs(bestctfvalue['defocus1']*1.0e6),2))+
" microns" )
if params['uncorrected']:
tmpname='temporaryCorrectedImage.mrc'
imgarray = apDBImage.correctImage(imgdata)
imgpath = os.path.join(params['rundir'],tmpname)
apImage.arrayToMrc(imgarray, imgpath)
print "processing", imgpath
else:
imgpath = os.path.join(imgdata['session']['image path'], imgname+'.mrc')
nominal = None
if params['nominal'] is not None:
nominal=params['nominal']
elif params['newnominal'] is True:
nominal = ctfdb.getBestDefocusForImage(imgdata, msg=True)
if nominal is None:
nominal = imgdata['scope']['defocus']
if nominal is None or nominal > 0 or nominal < -15e-6:
apDisplay.printWarning("Nominal should be of the form nominal=-1.2e-6"+\
" for -1.2 microns NOT:"+str(nominal))
#Neil's Hack
#if 'autosample' in params and params['autosample']:
# x = abs(nominal*1.0e6)
# val = 1.585 + 0.057587 * x - 0.044106 * x**2 + 0.010877 * x**3
# resamplefr_override = round(val,3)
# print "resamplefr_override=",resamplefr_override
# pymat.eval(matlab, "resamplefr="+str(resamplefr_override)+";")
pymat.eval(matlab,("dforig = %e;" % nominal))
if params['stig'] == 0:
plist = (imgpath, params['outtextfile'], params['display'], params['stig'],\
params['medium'], -nominal, params['tempdir']+"/")
acecmd = makeMatlabCmd("ctfparams = ace(",");",plist)
else:
plist = (imgname, imgpath, params['outtextfile'], params['opimagedir'], \
params['matdir'], params['display'], params['stig'],\
params['medium'], -nominal, params['tempdir']+"/", params['resamplefr'])
acecmd = makeMatlabCmd("ctfparams = measureAstigmatism(",");",plist)
#print acecmd
pymat.eval(matlab,acecmd)
matfile = os.path.join(params['matdir'], imgname+".mrc.mat")
if params['stig']==0:
savematcmd = "save('"+matfile+"','ctfparams','scopeparams', 'dforig');"
pymat.eval(matlab,savematcmd)
ctfvalue = pymat.get(matlab, 'ctfparams')
ctfvalue=ctfvalue[0]
ctfdb.printResults(params, nominal, ctfvalue)
return ctfvalue
def _var_type(self, varname):
mlabraw.eval(self._session, "TMP_CLS__ = class(%s);" % varname) #FIXME for funcs we would need ''s
res_type = mlabraw.get(self._session, "TMP_CLS__")
mlabraw.eval(self._session, "clear TMP_CLS__;")
return res_type