def _get(self, name, remove=False):
varname = name
vartype = self._var_type(varname)
if vartype in self._mlabraw_can_convert:
var = mlabraw.get(self._session, varname)
if type(var) is Numeric.ArrayType:
if self._flatten_row_vecs and Numeric.shape(var)[0] == 1:
var.shape = var.shape[1:2]
elif self._flatten_col_vecs and Numeric.shape(var)[1] == 1:
var.shape = var.shape[0:1]
if self._array_cast:
var = self._array_cast(var)
else:
var = None
if self._optionally_convert.get(vartype):
# manual conversions may fail (e.g. for multidimensional
# cell arrays), in that case just fall back on proxying.
try:
var = self._manually_convert(varname, vartype)
except MlabConversionError: pass
if var is None:
# we can't convert this to a python object, so we just
# create a proxy, and don't delete the real matlab
# reference until the proxy is garbage collected
var = self._make_proxy(varname)
if remove:
mlabraw.eval(self._session, "clear('%s');" % varname)
return var
def matlabFindTargets(self):
pymat.put(self.handle, 'focus', [])
pymat.put(self.handle, 'acquisition', [])
d, f = os.path.split(self.settings['module path'])
if d:
pymat.eval(self.handle, 'path(path, \'%s\')' % d)
if not f[:-2]:
raise RuntimeError
pymat.eval(self.handle,
'[acquisition, focus] = %s(image,image_id)' % f[:-2])
focus = pymat.get(self.handle, 'focus')
acquisition = pymat.get(self.handle, 'acquisition')
self.setTargets(acquisition, 'acquisition')
self.setTargets(focus, 'focus')
import time
time.sleep(1)
if self.settings['user check']:
self.panel.foundTargets()
def runAceCorrect(imgdict,params):
imgname = imgdict['filename']
imgpath = os.path.join(imgdict['session']['image path'], imgname+'.mrc')
voltage = (imgdict['scope']['high tension'])
apix = apDatabase.getPixelSize(imgdict)
ctfvalues, conf = ctfdb.getBestCtfValueForImage(imgdict)
ctdimname = imgname
ctdimpath = os.path.join(params['rundir'],ctdimname)
print "Corrected Image written to " + ctdimpath
#pdb.set_trace()
acecorrectcommand=("ctfcorrect1('%s', '%s', '%.32f', '%.32f', '%f', '%f', '%f');" % \
(imgpath, ctdimpath, ctfvalues['defocus1'], ctfvalues['defocus2'], -ctfvalues['angle_astigmatism'], voltage, apix))
print acecorrectcommand
try:
matlab = pymat.open("matlab -nosplash")
except:
apDisplay.environmentError()
raise
pymat.eval(matlab, acecorrectcommand)
pymat.close(matlab)
return
def __init__(self, display = False):
import mlabraw
if display:
self.handle = mlabraw.open("matlab -logfile /tmp/matlablog")
else:
self.handle = mlabraw.open("matlab -nodesktop -nodisplay -nojvm -logfile /tmp/matlablog")
mlabraw.eval(self.handle, "addpath('%s');"%os.path.join(os.path.dirname(lfd.__file__), "matlab"))
def eval_code(self, builder):
self.builder = builder
#should get a list of func names
func_names = []
for program in builder.project:
print program.summary()
#flatten tree and find Func.names
nodes = program.flatten(False, False, False)
for node in nodes:
if node.cls == "Func":
func_names.append(node.name)
#print func_names
self.func_names = func_names
self.called_func_names = []
#check if main
func = builder[0][1][0]
if func.name == 'main':
code_block = func[3]
#evaluate, recursive function
self._evaluate(code_block)
mlabraw.eval(self.session, 'whos_f')
else:
print 'matlab have to run script file'
def runAceCorrect(imgdict,params):
imgname = imgdict['filename']
imgpath = os.path.join(imgdict['session']['image path'], imgname+'.mrc')
voltage = (imgdict['scope']['high tension'])
apix = apDatabase.getPixelSize(imgdict)
ctfvalues = ctfdb.getBestCtfByResolution(imgdata)
conf = ctfdb.calculateConfidenceScore(bestctfvalue)
ctdimname = imgname
ctdimpath = os.path.join(params['rundir'],ctdimname)
print "Corrected Image written to " + ctdimpath
#pdb.set_trace()
acecorrectcommand=("ctfcorrect1('%s', '%s', '%.32f', '%.32f', '%f', '%f', '%f');" % \
(imgpath, ctdimpath, ctfvalues['defocus1'], ctfvalues['defocus2'], -ctfvalues['angle_astigmatism'], voltage, apix))
print acecorrectcommand
try:
matlab = pymat.open("matlab -nosplash")
except:
apDisplay.environmentError()
raise
pymat.eval(matlab, acecorrectcommand)
pymat.close(matlab)
return
def _get_part(self, to_get):
if self._mlabwrap._var_type(to_get) in self._mlabwrap._mlabraw_can_convert:
#!!! need assignment to TMP_VAL__ because `mlabraw.get` only works
# with 'atomic' values like ``foo`` and not e.g. ``foo.bar``.
mlabraw.eval(self._mlabwrap._session, "TMP_VAL__=%s" % to_get)
return self._mlabwrap._get('TMP_VAL__', remove=True)
return type(self)(self._mlabwrap, to_get, self)
def eval_code(self, builder):
self.builder = builder
#should get a list of func names
func_names = []
for program in builder.project:
print program.summary()
#flatten tree and find Func.names
nodes = program.flatten(False, False, False)
for node in nodes:
if node.cls == "Func":
func_names.append(node.name)
#print func_names
self.func_names = func_names
self.called_func_names = []
#check if main
func = builder[0][1][0]
if func.name == 'main':
code_block = func[3]
#evaluate, recursive function
self._evaluate(code_block)
mlabraw.eval(self.session, 'whos_f')
else:
print 'matlab have to run script file'
def _get(self, name, remove=False):
r"""Directly access a variable in matlab space.
This should normally not be used by user code."""
# FIXME should this really be needed in normal operation?
if name in self._proxies: return self._proxies[name]
varname = name
vartype = self._var_type(varname)
if vartype in self._mlabraw_can_convert:
var = mlabraw.get(self._session, varname)
if isinstance(var, ndarray):
if self._flatten_row_vecs and numpy.shape(var)[0] == 1:
var.shape = var.shape[1:2]
elif self._flatten_col_vecs and numpy.shape(var)[1] == 1:
var.shape = var.shape[0:1]
if self._array_cast:
var = self._array_cast(var)
else:
var = None
if self._dont_proxy.get(vartype):
# manual conversions may fail (e.g. for multidimensional
# cell arrays), in that case just fall back on proxying.
try:
var = self._manually_convert(varname, vartype)
except MlabConversionError:
pass
if var is None:
# we can't convert this to a python object, so we just
# create a proxy, and don't delete the real matlab
# reference until the proxy is garbage collected
var = self._make_proxy(varname)
if remove:
mlabraw.eval(self._session, "clear('%s');" % varname)
return var
def _var_type(self, varname):
mlabraw.eval(self._session,
"TMP_CLS__ = class(%(x)s); if issparse(%(x)s),"
"TMP_CLS__ = [TMP_CLS__,'-sparse']; end;" % dict(x=varname))
res_type = mlabraw.get(self._session, "TMP_CLS__")
mlabraw.eval(self._session, "clear TMP_CLS__;") # unlikely to need try/finally to ensure clear
return res_type
def _get_values(self, varnames):
if not varnames: raise ValueError("No varnames") #to prevent clear('')
res = []
for varname in varnames:
res.append(self._get(varname))
mlabraw.eval(self._session, "clear('%s');" % "','".join(varnames)) #FIXME wrap try/finally?
return res
def _get(self, name, remove=False):
r"""Directly access a variable in matlab space.
This should normally not be used by user code."""
# FIXME should this really be needed in normal operation?
if name in self._proxies: return self._proxies[name]
varname = name
vartype = self._var_type(varname)
if vartype in self._mlabraw_can_convert:
var = mlabraw.get(self._session, varname)
if isinstance(var, ndarray):
if var.shape:
if self._flatten_row_vecs and numpy.shape(var)[0] == 1:
var.shape = var.shape[1:2]
elif len(var.shape) > 1 and self._flatten_col_vecs and numpy.shape(var)[1] == 1:
var.shape = var.shape[0:1]
if self._array_cast:
var = self._array_cast(var)
else:
var = None
if self._dont_proxy.get(vartype):
# manual conversions may fail (e.g. for multidimensional
# cell arrays), in that case just fall back on proxying.
try:
var = self._manually_convert(varname, vartype)
except MlabConversionError: pass
if var is None:
# we can't convert this to a python object, so we just
# create a proxy, and don't delete the real matlab
# reference until the proxy is garbage collected
var = self._make_proxy(varname)
if remove:
mlabraw.eval(self._session, "clear('%s');" % varname)
return var
def _get_part(self, to_get):
if self._mlabwrap._var_type(to_get) in self._mlabwrap._mlabraw_can_convert:
#!!! need assignment to TMP_VAL__ because `mlabraw.get` only works
# with 'atomic' values like ``foo`` and not e.g. ``foo.bar``.
mlabraw.eval(self._mlabwrap._session, "TMP_VAL__=%s" % to_get)
return self._mlabwrap._get('TMP_VAL__', remove=True)
return type(self)(self._mlabwrap, to_get, self)
def get3d(handle, name):
mlabraw.eval(handle, """
flat_array = %s(:);
shape = size(%s);
"""%(name, name))
flat_array = mlabraw.get(handle, "flat_array")
shape = map(int, mlabraw.get(handle, "shape").flat)
return np.ndarray(buffer = flat_array, shape = shape, order="F")
def transform_points(self, points):
mlabraw.put(self.handle, "points", points)
mlabraw.eval(self.handle,"""
points_result = tps_eval(points, params);
""")
points_result = mlabraw.get(self.handle, "points_result")
return points_result
def _get_values(self, varnames):
if not varnames: raise ValueError("No varnames") #to prevent clear('')
res = []
for varname in varnames:
res.append(self._get(varname))
mlabraw.eval(self._session, "clear('%s');" %
"','".join(varnames)) #FIXME wrap try/finally?
return res
def _set(self, name, value):
r"""Directly set a variable `name` in matlab space to `value`.
This should normally not be used in user code."""
if isinstance(value, MlabObjectProxy):
mlabraw.eval(self._session, "%s = %s;" % (name, value._name))
else:
## mlabraw.put(self._session, name, self._as_mlabable_type(value))
mlabraw.put(self._session, name, value)
def _var_type(self, varname):
mlabraw.eval(
self._session, "TMP_CLS__ = class(%(x)s); if issparse(%(x)s),"
"TMP_CLS__ = [TMP_CLS__,'-sparse']; end;" % dict(x=varname))
res_type = mlabraw.get(self._session, "TMP_CLS__")
mlabraw.eval(
self._session,
"clear TMP_CLS__;") # unlikely to need try/finally to ensure clear
return res_type
def _set(self, name, value):
r"""Directly set a variable `name` in matlab space to `value`.
This should normally not be used in user code."""
if isinstance(value, MlabObjectProxy):
mlabraw.eval(self._session, "%s = %s;" % (name, value._name))
else:
## mlabraw.put(self._session, name, self._as_mlabable_type(value))
mlabraw.put(self._session, name, value)
def branch_points(bw):
initialize()
mlabraw.put(MATLAB, "bw",bw)
mlabraw.eval(MATLAB, """
bp = bwmorph(bw,'branchpoints')
bp_d = double(bp);
""")
bp_d = mlabraw.get(MATLAB, "bp_d")
bp = bp_d.astype('uint8')
return bp
def _make_proxy(self, varname, parent=None, constructor=MlabObjectProxy):
"""Creates a proxy for a variable.
XXX create and cache nested proxies also here.
"""
proxy_val_name = "PROXY_VAL%d__" % self._proxy_count
self._proxy_count += 1
mlabraw.eval(self._session, "%s = %s;" % (proxy_val_name, varname))
res = constructor(self, proxy_val_name, parent)
self._proxies[proxy_val_name] = res
return res
def branch_points(bw):
initialize()
mlabraw.put(MATLAB, "bw", bw)
mlabraw.eval(
MATLAB, """
bp = bwmorph(bw,'branchpoints')
bp_d = double(bp);
""")
bp_d = mlabraw.get(MATLAB, "bp_d")
bp = bp_d.astype('uint8')
return bp
def fit_transformation(self, points0, points1):
assert len(points0) == len(points1)
mlabraw.put(self.handle, "points0", points0)
mlabraw.put(self.handle, "points1", points1)
mlabraw.eval(self.handle, """
opts = opts_fit;
opts.reg = .01;
params = tps_fit(points0, points1, opts);
save('/tmp/after_fitting.mat');
""")
def _make_proxy(self, varname, parent=None, constructor=MlabObjectProxy):
"""Creates a proxy for a variable.
XXX create and cache nested proxies also here.
"""
# FIXME why not just use gensym here?
proxy_val_name = "PROXY_VAL%d__" % self._proxy_count
self._proxy_count += 1
mlabraw.eval(self._session, "%s = %s;" % (proxy_val_name, varname))
res = constructor(self, proxy_val_name, parent)
self._proxies[proxy_val_name] = res
return res
def _set(self, name, value):
r"""Directly set a variable `name` in matlab space to `value`.
This should normally not be used in user code."""
if isinstance(value, MlabObjectProxy):
mlabraw.eval(self._session, "%s = %s;" % (name, value._name))
else:
## mlabraw.put(self._session, name, self._as_mlabable_type(value))
if isinstance(value, numpy.ndarray):
value = value.copy() # XXX: mlabraw seems to badly handle strides.
mlabraw.put(self._session, name, value)
def _var_type(self, varname):
"""Ask matlab what the type of varname is.
:param varname: string variable
:return: string type, e.g. ``double`` or ``char``.
"""
mlabraw.eval(self._session,
"TMP_CLS__ = class(%(x)s); if issparse(%(x)s),"
"TMP_CLS__ = [TMP_CLS__,'-sparse']; end;" % dict(x=varname))
res_type = mlabraw.get(self._session, "TMP_CLS__")
mlabraw.eval(self._session, "clear TMP_CLS__;") # unlikely to need try/finally to ensure clear
return res_type
def _var_type(self, varname):
"""Ask matlab what the type of varname is.
:param varname: string variable
:return: string type, e.g. ``double`` or ``char``.
"""
mlabraw.eval(self._session,
"TMP_CLS__ = class(%(x)s); if issparse(%(x)s),"
"TMP_CLS__ = [TMP_CLS__,'-sparse']; end;" % dict(x=varname))
res_type = mlabraw.get(self._session, "TMP_CLS__")
mlabraw.eval(self._session, "clear TMP_CLS__;") # unlikely to need try/finally to ensure clear
return res_type
def __setstate__(self, state):
"Experimental unpickling support."
global mlab #XXX this should be dealt with correctly
old_name = state['name']
mlab_name = "UNPICKLED%s__" % gensym('')
tmp_filename = None
try:
tmp_filename = strToTempfile(state['mlab_contents'],
suffix='.mat',
binary=1)
mlabraw.eval(
mlab._session, "TMP_UNPICKLE_STRUCT__ = load('%s', '%s');" %
(tmp_filename, old_name))
mlabraw.eval(
mlab._session,
"%s = TMP_UNPICKLE_STRUCT__.%s;" % (mlab_name, old_name))
mlabraw.eval(mlab._session, "clear TMP_UNPICKLE_STRUCT__;")
# XXX
mlab._make_proxy(
mlab_name,
constructor=lambda *args: self.__init__(*args) or self)
mlabraw.eval(mlab._session, 'clear %s;' % mlab_name)
finally:
if tmp_filename and os.path.exists(tmp_filename):
os.remove(tmp_filename)
def transform_poses(self, points, rots):
mlabraw.put(self.handle, "points", points)
put3d(self.handle, "rots", rots)
mlabraw.eval(self.handle,"""
[points_result, rots_result] = tps_eval_frames(points, rots, params);
""")
points_result = mlabraw.get(self.handle,"points_result")
rots_result = get3d(self.handle, "rots_result")
return points_result, rots_result
def setScopeParams(matlab,params):
tempdir = params['tempdir']+"/"
if os.path.isdir(tempdir):
plist = (params['kv'],params['cs'],params['apix'],tempdir)
acecmd1 = makeMatlabCmd("setscopeparams(",");",plist)
pymat.eval(matlab,acecmd1)
plist = (params['kv'],params['cs'],params['apix'])
acecmd2 = makeMatlabCmd("scopeparams = [","];",plist)
pymat.eval(matlab,acecmd2)
else:
apDisplay.printError("Temp directory, '"+params['tempdir']+"' not present.")
return
def setScopeParams(matlab,params):
tempdir = params['tempdir']+"/"
if os.path.isdir(tempdir):
plist = (params['kv'],params['cs'],params['apix'],tempdir)
acecmd1 = makeMatlabCmd("setscopeparams(",");",plist)
pymat.eval(matlab,acecmd1)
plist = (params['kv'],params['cs'],params['apix'])
acecmd2 = makeMatlabCmd("scopeparams = [","];",plist)
pymat.eval(matlab,acecmd2)
else:
apDisplay.printError("Temp directory, '"+params['tempdir']+"' not present.")
return
def onRunParamGUI(self, evt):
# added for GUI parametrization:
self.handle = pymat.open()
d, f = os.path.split(self.widgets["parametergui path"].GetValue()) # self.settings['parametergui path'])
if d:
pymat.eval(self.handle, "path(path, '%s')" % d)
if not f[:-2]:
raise RuntimeError
pymat.eval(self.handle, "%s" % f[:-2])
def onRunParamGUI(self, evt):
# added for GUI parametrization:
self.handle = pymat.open()
d, f = os.path.split(self.widgets['parametergui path'].GetValue()
) #self.settings['parametergui path'])
if d:
pymat.eval(self.handle, 'path(path, \'%s\')' % d)
if not f[:-2]:
raise RuntimeError
pymat.eval(self.handle, '%s' % f[:-2])
def testRawMlabraw(self):
"""A few explicit tests for mlabraw"""
import mlabraw
#print "test mlabraw"
self.assertRaises(TypeError, mlabraw.put, 33, 'a', 1)
self.assertRaises(TypeError, mlabraw.get, object(), 'a')
self.assertRaises(TypeError, mlabraw.eval, object(), '1')
# -100 is picked kinda arbitrarily to account for internal "overhead";
# I don't want to hardcode the exact value; users can assume 1000
# chars is safe
mlabraw.eval(mlab._session, '1' * (BUFSIZE - 100))
assert numpy.inf == mlabraw.get(mlab._session, 'ans');
# test for buffer overflow detection
self.assertRaises(Exception, mlabraw.eval, mlab._session, '1' * BUFSIZE)
def test_mlab_raw_eval(self):
"""Test evaluating strings in mlabraw module.
N.B. This is currently known to fail.
It appears that doing
mlabraw.eval(mlab._session, r"'x'")
does not return the string 'x' as exected.
"""
import mlabraw
self.assertEqual(mlabraw.eval(mlab._session, r"fprintf('1\n')"), '1\n')
try:
self.assertEqual(mlabraw.eval(mlab._session, r"1"), '')
finally:
mlabraw.eval(mlab._session, 'clear ans')
def fit_transformation_icp(self, points0, points1):
mlabraw.put(self.handle, "points0", points0)
mlabraw.put(self.handle, "points1", points1)
mlabraw.eval(self.handle, """
opts = opts_icp;
opts.n_iter=6;
opts.lines_from_orig=1;
opts.corr_opts.method='bipartite';
opts.fit_opts.reg = .01;
%opts.plot_grid = 1;
opts.corr_opts.bipartite_opts.deficient_col_penalty=1;
opts.corr_opts.bipartite_opts.deficient_row_penalty=1;
params = tps_rpm(points0, points1, opts);
save('/tmp/after_fitting.mat');
""")
def _get_cell(self, varname):
# XXX can currently only handle 1D
mlabraw.eval(self._session,
"TMP_SIZE_INFO__ = \
[min(size(%(vn)s)) == 1 & ndims(%(vn)s) == 2, \
max(size(%(vn)s))];" % {'vn':varname})
is_rank1, cell_len = self._get("TMP_SIZE_INFO__", remove=True).flat
if is_rank1:
cell_bits = (["TMP%i%s__" % (i, gensym('_'))
for i in range(cell_len)])
mlabraw.eval(self._session, '[%s] = deal(%s{:});' %
(",".join(cell_bits), varname))
# !!! this recursive call means we have to take care with
# overwriting temps!!!
return self._get_values(cell_bits)
else:
raise MlabConversionError("Not a 1D cell array")
def runAceDrift(matlab,imgdict,params):
imgname = imgdict['filename']
imgpath = os.path.join(imgdict['session']['image path'], imgname+'.mrc')
if params['nominal']:
nominal=params['nominal']
else:
nominal=imgdict['scope']['defocus']
#pdb.set_trace()
acecommand=("measureAnisotropy('%s','%s',%d,'%s',%e,'%s','%s','%s', '%s');" % \
( imgpath, params['outtextfile'], params['display'],\
params['medium'], -nominal, params['tempdir']+"/", params['opimagedir'], params['matdir'], imgname))
#~ acecommand=("mnUpCut = measureDrift('%s','%s',%d,%d,'%s',%e,'%s');" % \
#~ ( imgpath, params['outtextfile'], params['display'], params['stig'],\
#~ params['medium'], -nominal, params['tempdir']))
pymat.eval(matlab,acecommand)
def remove_holes(labels,min_size):
initialize()
mlabraw.put(MATLAB, "L",labels)
mlabraw.put(MATLAB, "min_size",min_size)
mlabraw.eval(MATLAB, """
max_label = max(L(:));
good_pix = L==0;
for label = 1:max_label
good_pix = good_pix | bwareaopen(L==label,min_size,4);
end
bad_pix = ~logical(good_pix);
[~,I] = bwdist(good_pix,'Chessboard');
NewL = L;
NewL(bad_pix) = L(I(bad_pix));
NewL_d = double(NewL);
""")
NewL_d = mlabraw.get(MATLAB, "NewL_d")
return NewL_d.astype('uint8')
def runAceDrift(matlab,imgdict,params):
imgname = imgdict['filename']
imgpath = os.path.join(imgdict['session']['image path'], imgname+'.mrc')
if params['nominal']:
nominal=params['nominal']
else:
nominal=imgdict['scope']['defocus']
#pdb.set_trace()
acecommand=("measureAnisotropy('%s','%s',%d,'%s',%e,'%s','%s','%s', '%s');" % \
( imgpath, params['outtextfile'], params['display'],\
params['medium'], -nominal, params['tempdir']+"/", params['opimagedir'], params['matdir'], imgname))
#~ acecommand=("mnUpCut = measureDrift('%s','%s',%d,%d,'%s',%e,'%s');" % \
#~ ( imgpath, params['outtextfile'], params['display'], params['stig'],\
#~ params['medium'], -nominal, params['tempdir']))
pymat.eval(matlab,acecommand)
def remove_holes(labels, min_size):
initialize()
mlabraw.put(MATLAB, "L", labels)
mlabraw.put(MATLAB, "min_size", min_size)
mlabraw.eval(
MATLAB, """
max_label = max(L(:));
good_pix = L==0;
for label = 1:max_label
good_pix = good_pix | bwareaopen(L==label,min_size,4);
end
bad_pix = ~logical(good_pix);
[~,I] = bwdist(good_pix,'Chessboard');
NewL = L;
NewL(bad_pix) = L(I(bad_pix));
NewL_d = double(NewL);
""")
NewL_d = mlabraw.get(MATLAB, "NewL_d")
return NewL_d.astype('uint8')
def _get_cell(self, varname):
# XXX can currently only handle ``{}`` and 1D cells
mlabraw.eval(self._session,
"TMP_SIZE_INFO__ = \
[all(size(%(vn)s) == 0), \
min(size(%(vn)s)) == 1 & ndims(%(vn)s) == 2, \
max(size(%(vn)s))];" % {'vn':varname})
is_empty, is_rank1, cell_len = map(int,
self._get("TMP_SIZE_INFO__", remove=True).flat)
if is_empty:
return []
elif is_rank1:
cell_bits = (["TMP%i%s__" % (i, gensym('_'))
for i in range(cell_len)])
mlabraw.eval(self._session, '[%s] = deal(%s{:});' %
(",".join(cell_bits), varname))
# !!! this recursive call means we have to take care with
# overwriting temps!!!
return self._get_values(cell_bits)
else:
raise MlabConversionError("Not a 1D cell array")
def _get_cell(self, varname):
# XXX can currently only handle ``{}`` and 1D cells
mlabraw.eval(self._session,
"TMP_SIZE_INFO__ = \
[all(size(%(vn)s) == 0), \
min(size(%(vn)s)) == 1 & ndims(%(vn)s) == 2, \
max(size(%(vn)s))];" % {'vn':varname})
is_empty, is_rank1, cell_len = map(int,
self._get("TMP_SIZE_INFO__", remove=True).flat)
if is_empty:
return []
elif is_rank1:
cell_bits = (["TMP%i__" % (self.proxy_count+i)
for i in range(cell_len)])
self._proxy_count += cell_len
mlabraw.eval(self._session, '[%s] = deal(%s{:});' %
(",".join(cell_bits), varname))
# !!! this recursive call means we have to take care with
# overwriting temps!!!
return self._get_values(cell_bits)
else:
raise MlabConversionError("Not a 1D cell array")
def _set_part(self, to_set, value):
#FIXME s.a.
if isinstance(value, MlabObjectProxy):
mlabraw.eval(self._mlabwrap._session, "%s = %s;" % (to_set, value._name))
else:
self._mlabwrap._set("TMP_VAL__", value)
mlabraw.eval(self._mlabwrap._session, "%s = TMP_VAL__;" % to_set)
mlabraw.eval(self._mlabwrap._session, 'clear TMP_VAL__;')
def _set_part(self, to_set, value):
#FIXME s.a.
if isinstance(value, MlabObjectProxy):
mlabraw.eval(self._mlabwrap._session, "%s = %s;" % (to_set, value._name))
else:
self._mlabwrap._set("TMP_VAL__", value)
mlabraw.eval(self._mlabwrap._session, "%s = TMP_VAL__;" % to_set)
mlabraw.eval(self._mlabwrap._session, 'clear TMP_VAL__;')
def matlabFindTargets(self):
pymat.put(self.handle, 'focus', [])
pymat.put(self.handle, 'acquisition', [])
d, f = os.path.split(self.settings['module path'])
if d:
pymat.eval(self.handle, 'path(path, \'%s\')' % d)
if not f[:-2]:
raise RuntimeError
pymat.eval(self.handle, '[acquisition, focus] = %s(image,image_id)' % f[:-2])
focus = pymat.get(self.handle, 'focus')
acquisition = pymat.get(self.handle, 'acquisition')
self.setTargets(acquisition, 'acquisition')
self.setTargets(focus, 'focus')
import time
time.sleep(1)
if self.settings['user check']:
self.panel.foundTargets()
def __setstate__(self, state):
"Experimental unpickling support."
global mlab #XXX this should be dealt with correctly
old_name = state['name']
mlab_name = "UNPICKLED%s__" % gensym('')
try:
tmp_filename = tempfile.mktemp('.mat')
spitOut(state['mlab_contents'], tmp_filename, binary=1)
mlabraw.eval(mlab._session,
"TMP_UNPICKLE_STRUCT__ = load('%s', '%s');" % (
tmp_filename, old_name))
mlabraw.eval(mlab._session,
"%s = TMP_UNPICKLE_STRUCT__.%s;" % (mlab_name, old_name))
mlabraw.eval(mlab._session, "clear TMP_UNPICKLE_STRUCT__;")
# XXX
mlab._make_proxy(mlab_name, constructor=lambda *args: self.__init__(*args) or self)
mlabraw.eval(mlab._session, 'clear %s;' % mlab_name)
finally:
if os.path.exists(tmp_filename): os.remove(tmp_filename)
def testRawMlabraw(self):
"""A few explicit tests for mlabraw"""
import mlabraw
#print "test mlabraw"
self.assertRaises(TypeError, mlabraw.put, 33, 'a', 1)
self.assertRaises(TypeError, mlabraw.get, object(), 'a')
self.assertRaises(TypeError, mlabraw.eval, object(), '1')
# -100 is picked kinda arbitrarily to account for internal "overhead";
# I don't want to hardcode the exact value; users can assume 1000
# chars is safe
mlabraw.eval(mlab._session, '1' * (BUFSIZE - 100))
assert numpy.inf == mlabraw.get(mlab._session, 'ans')
# test for buffer overflow detection
self.assertRaises(Exception, mlabraw.eval, mlab._session,
'1' * BUFSIZE)
self.assertEqual(mlabraw.eval(mlab._session, r"fprintf('1\n')"), '1\n')
try:
self.assertEqual(mlabraw.eval(mlab._session, r"1"), '')
finally:
mlabraw.eval(mlab._session, 'clear ans')
def __del__(self):
if self._parent is None:
mlabraw.eval(self._mlabwrap._session, 'clear %s;' % self._name)
def evaluate(string):
mlabraw.eval(MATLAB, string)
def _execute(self):
batch_size = self.batch_size
pooling_region_counts = self.pooling_region_counts
dataset_family = self.dataset_family
which_set = self.which_set
size = self.size
nan = 0
dataset_descriptor = dataset_family[which_set][size]
dataset = dataset_descriptor.dataset_maker()
expected_num_examples = dataset_descriptor.num_examples
full_X = dataset.get_design_matrix()
num_examples = full_X.shape[0]
assert num_examples == expected_num_examples
if self.restrict is not None:
assert self.restrict[1] <= full_X.shape[0]
print 'restricting to examples ',self.restrict[0],' through ',self.restrict[1],' exclusive'
full_X = full_X[self.restrict[0]:self.restrict[1],:]
assert self.restrict[1] > self.restrict[0]
#update for after restriction
num_examples = full_X.shape[0]
assert num_examples > 0
dataset.X = None
dataset.design_loc = None
dataset.compress = False
patchifier = ExtractGridPatches( patch_shape = (size,size), patch_stride = (1,1) )
pipeline = serial.load(dataset_descriptor.pipeline_path)
assert isinstance(pipeline.items[0], ExtractPatches)
pipeline.items[0] = patchifier
print 'defining features'
Z = T.matrix('Z')
if self.one_sided:
feat = abs(Z)
else:
pos = T.clip(Z,0.,1e30)
neg = T.clip(-Z,0.,1e30)
feat = T.concatenate((pos, neg), axis=1)
print 'compiling theano function'
f = function([Z],feat)
nfeat = self.W.shape[1] * (2 - self.one_sided)
if not (nfeat == 1600 or nfeat == 3200):
print nfeat
assert False
if config.device.startswith('gpu') and nfeat >= 4000:
f = halver(f, nfeat)
topo_feat_var = T.TensorType(broadcastable = (False,False,False,False), dtype='float32')()
region_features = function([topo_feat_var],
topo_feat_var.mean(axis=(1,2)) )
def average_pool( stride ):
def point( p ):
return p * ns / stride
rval = np.zeros( (topo_feat.shape[0], stride, stride, topo_feat.shape[3] ) , dtype = 'float32')
for i in xrange(stride):
for j in xrange(stride):
rval[:,i,j,:] = region_features( topo_feat[:,point(i):point(i+1), point(j):point(j+1),:] )
return rval
outputs = [ np.zeros((num_examples,count,count,nfeat),dtype='float32') for count in pooling_region_counts ]
assert len(outputs) > 0
fd = DenseDesignMatrix(X = np.zeros((1,1),dtype='float32'), view_converter = DefaultViewConverter([1, 1, nfeat] ) )
ns = 32 - size + 1
depatchifier = ReassembleGridPatches( orig_shape = (ns, ns), patch_shape=(1,1) )
if len(range(0,num_examples-batch_size+1,batch_size)) <= 0:
print num_examples
print batch_size
for i in xrange(0,num_examples-batch_size+1,batch_size):
print i
t1 = time.time()
d = copy.copy(dataset)
d.set_design_matrix(full_X[i:i+batch_size,:])
t2 = time.time()
#print '\tapplying preprocessor'
d.apply_preprocessor(pipeline, can_fit = False)
X2 = d.get_design_matrix()
t3 = time.time()
M.put(s,'batch',X2)
M.eval(s, 'Z = sparse_codes(batch, dictionary, lambda)')
Z = M.get(s, 'Z')
feat = f(np.cast['float32'](Z))
t4 = time.time()
assert feat.dtype == 'float32'
feat_dataset = copy.copy(fd)
if np.any(np.isnan(feat)):
nan += np.isnan(feat).sum()
feat[np.isnan(feat)] = 0
feat_dataset.set_design_matrix(feat)
#print '\treassembling features'
feat_dataset.apply_preprocessor(depatchifier)
#print '\tmaking topological view'
topo_feat = feat_dataset.get_topological_view()
assert topo_feat.shape[0] == batch_size
t5 = time.time()
#average pooling
for output, count in zip(outputs, pooling_region_counts):
output[i:i+batch_size,...] = average_pool(count)
t6 = time.time()
print (t6-t1, t2-t1, t3-t2, t4-t3, t5-t4, t6-t5)
for output, save_path in zip(outputs, self.save_paths):
if self.chunk_size is not None:
assert save_path.endswith('.npy')
save_path_pieces = save_path.split('.npy')
assert len(save_path_pieces) == 2
assert save_path_pieces[1] == ''
save_path = save_path_pieces[0] + '_' + chr(ord('A')+self.chunk_id)+'.npy'
np.save(save_path,output)
if nan > 0:
warnings.warn(str(nan)+' features were nan')
def _do(self, cmd, *args, **kwargs):
"""Semi-raw execution of a matlab command.
Smartly handle calls to matlab, figure out what to do with `args`,
and when to use function call syntax and not.
If no `args` are specified, the ``cmd`` not ``result = cmd()`` form is
used in Matlab -- this also makes literal Matlab commands legal
(eg. cmd=``get(gca, 'Children')``).
If ``nout=0`` is specified, the Matlab command is executed as
procedure, otherwise it is executed as function (default), nout
specifying how many values should be returned (default 1).
**Beware that if you use don't specify ``nout=0`` for a `cmd` that
never returns a value will raise an error** (because assigning a
variable to a call that doesn't return a value is illegal in matlab).
``cast`` specifies which typecast should be applied to the result
(e.g. `int`), it defaults to none.
XXX: should we add ``parens`` parameter?
"""
handle_out = kwargs.get('handle_out', _flush_write_stdout)
#self._session = self._session or mlabraw.open()
# HACK
if self._autosync_dirs:
mlabraw.eval(self._session,
"cd('%s');" % os.getcwd().replace("'", "''"))
nout = kwargs.get('nout', 1)
#XXX what to do with matlab screen output
argnames = []
tempargs = []
try:
for count, arg in enumerate(args):
if isinstance(arg, MlabObjectProxy):
argnames.append(arg._name)
else:
nextName = 'arg%d__' % count
argnames.append(nextName)
tempargs.append(nextName)
# have to convert these by hand
## try:
## arg = self._as_mlabable_type(arg)
## except TypeError:
## raise TypeError("Illegal argument type (%s.:) for %d. argument" %
## (type(arg), type(count)))
mlabraw.put(self._session, argnames[-1], arg)
if args:
cmd = "%s(%s)%s" % (cmd, ", ".join(argnames),
('', ';')[kwargs.get('show', 0)])
# got three cases for nout:
# 0 -> None, 1 -> val, >1 -> [val1, val2, ...]
if nout == 0:
handle_out(mlabraw.eval(self._session, cmd))
return
# deal with matlab-style multiple value return
resSL = ((["RES%d__" % i for i in range(nout)]))
handle_out(
mlabraw.eval(self._session,
'[%s]=%s;' % (", ".join(resSL), cmd)))
res = self._get_values(resSL)
if nout == 1: res = res[0]
else: res = tuple(res)
if kwargs.has_key('cast'):
if nout == 0: raise TypeError("Can't cast: 0 nout")
return kwargs['cast'](res)
else:
return res
finally:
if len(tempargs) and self._clear_call_args:
mlabraw.eval(self._session,
"clear('%s');" % "','".join(tempargs))
def _do(self, cmd, *args, **kwargs):
"""Semi-raw execution of a matlab command.
Smartly handle calls to matlab, figure out what to do with `args`,
and when to use function call syntax and not.
If no `args` are specified, the ``cmd`` not ``result = cmd()`` form is
used in Matlab -- this also makes literal Matlab commands legal
(eg. cmd=``get(gca, 'Children')``).
If ``nout=0`` is specified, the Matlab command is executed as
procedure, otherwise it is executed as function (default), nout
specifying how many values should be returned (default 1).
**Beware that if you use don't specify ``nout=0`` for a `cmd` that
never returns a value will raise an error** (because assigning a
variable to a call that doesn't return a value is illegal in matlab).
``cast`` specifies which typecast should be applied to the result
(e.g. `int`), it defaults to none.
XXX: should we add ``parens`` parameter?
"""
handle_out = kwargs.get('handle_out', _flush_write_stdout)
#self._session = self._session or mlabraw.open()
# HACK
if self._autosync_dirs:
mlabraw.eval(self._session, "cd('%s');" % os.getcwd().replace("'", "''"))
nout = kwargs.get('nout', 1)
#XXX what to do with matlab screen output
argnames = []
tempargs = []
try:
for count, arg in enumerate(args):
if isinstance(arg, MlabObjectProxy):
argnames.append(arg._name)
else:
nextName = 'arg%d__' % count
argnames.append(nextName)
tempargs.append(nextName)
# have to convert these by hand
## try:
## arg = self._as_mlabable_type(arg)
## except TypeError:
## raise TypeError("Illegal argument type (%s.:) for %d. argument" %
## (type(arg), type(count)))
mlabraw.put(self._session, argnames[-1], arg)
if args:
cmd = "%s(%s)%s" % (cmd, ", ".join(argnames),
('',';')[kwargs.get('show',0)])
# got three cases for nout:
# 0 -> None, 1 -> val, >1 -> [val1, val2, ...]
if nout == 0:
handle_out(mlabraw.eval(self._session, cmd))
return
# deal with matlab-style multiple value return
resSL = ((["RES%d__" % i for i in range(nout)]))
handle_out(mlabraw.eval(self._session, '[%s]=%s;' % (", ".join(resSL), cmd)))
res = self._get_values(resSL)
if nout == 1: res = res[0]
else: res = tuple(res)
if kwargs.has_key('cast'):
if nout == 0: raise TypeError("Can't cast: 0 nout")
return kwargs['cast'](res)
else:
return res
finally:
if len(tempargs) and self._clear_call_args:
mlabraw.eval(self._session, "clear('%s');" %
"','".join(tempargs))
def setAceConfig(matlab,params):
tempdir=params['tempdir']+"/"
if os.path.isdir(tempdir):
pymat.eval(matlab, "edgethcarbon="+str(params['edgethcarbon'])+";")
pymat.eval(matlab, "edgethice="+str(params['edgethice'])+";")
pymat.eval(matlab, "pfcarbon="+str(params['pfcarbon'])+";")
pymat.eval(matlab, "pfice="+str(params['pfice'])+";")
pymat.eval(matlab, "overlap="+str(params['overlap'])+";")
pymat.eval(matlab, "fieldsize="+str(params['fieldsize'])+";")
pymat.eval(matlab, "resamplefr="+str(params['resamplefr'])+";")
pymat.eval(matlab, "drange="+str(params['drange'])+";")
aceconfig=os.path.join(tempdir,"aceconfig.mat")
acecmd = "save('"+aceconfig+"','edgethcarbon','edgethice','pfcarbon','pfice',"+\
"'overlap','fieldsize','resamplefr','drange');"
pymat.eval(matlab, acecmd)
else:
apDisplay.printError("Temp directory, '"+tempdir+"' not present.")
return
def runAce(matlab, imgdata, params, showprev=True):
imgname = imgdata['filename']
if showprev is True:
bestctfvalue = ctfdb.getBestCtfByResolution(imgdata)
if bestctfvalue:
bestconf = ctfdb.calculateConfidenceScore(bestctfvalue)
print ( "Prev best: '"+bestctfvalue['acerun']['name']+"', conf="+
apDisplay.colorProb(bestconf)+", defocus="+str(round(-1.0*abs(bestctfvalue['defocus1']*1.0e6),2))+
" microns" )
if params['uncorrected']:
tmpname='temporaryCorrectedImage.mrc'
imgarray = apDBImage.correctImage(imgdata)
imgpath = os.path.join(params['rundir'],tmpname)
apImage.arrayToMrc(imgarray, imgpath)
print "processing", imgpath
else:
imgpath = os.path.join(imgdata['session']['image path'], imgname+'.mrc')
nominal = None
if params['nominal'] is not None:
nominal=params['nominal']
elif params['newnominal'] is True:
bestctfvalue = ctfdb.getBestCtfByResolution(imgdata)
nominal = bestctfvalue['defocus1']
if nominal is None:
nominal = imgdata['scope']['defocus']
if nominal is None or nominal > 0 or nominal < -15e-6:
apDisplay.printWarning("Nominal should be of the form nominal=-1.2e-6"+\
" for -1.2 microns NOT:"+str(nominal))
#Neil's Hack
#if 'autosample' in params and params['autosample']:
# x = abs(nominal*1.0e6)
# val = 1.585 + 0.057587 * x - 0.044106 * x**2 + 0.010877 * x**3
# resamplefr_override = round(val,3)
# print "resamplefr_override=",resamplefr_override
# pymat.eval(matlab, "resamplefr="+str(resamplefr_override)+";")
pymat.eval(matlab,("dforig = %e;" % nominal))
if params['stig'] == 0:
plist = (imgpath, params['outtextfile'], params['display'], params['stig'],\
params['medium'], -nominal, params['tempdir']+"/")
acecmd = makeMatlabCmd("ctfparams = ace(",");",plist)
else:
plist = (imgname, imgpath, params['outtextfile'], params['opimagedir'], \
params['matdir'], params['display'], params['stig'],\
params['medium'], -nominal, params['tempdir']+"/", params['resamplefr'])
acecmd = makeMatlabCmd("ctfparams = measureAstigmatism(",");",plist)
#print acecmd
pymat.eval(matlab,acecmd)
matfile = os.path.join(params['matdir'], imgname+".mrc.mat")
if params['stig']==0:
savematcmd = "save('"+matfile+"','ctfparams','scopeparams', 'dforig');"
pymat.eval(matlab,savematcmd)
ctfvalue = pymat.get(matlab, 'ctfparams')
ctfvalue=ctfvalue[0]
printResults(params, nominal, ctfvalue)
return ctfvalue
from theano import tensor as T
#from theano.sandbox.neighbours import images2neibs
from theano import function
from pylearn2.datasets.preprocessing import ExtractPatches, ExtractGridPatches, ReassembleGridPatches
from pylearn2.utils import serial
from pylearn2.datasets.dense_design_matrix import DenseDesignMatrix, DefaultViewConverter
from pylearn2.datasets.cifar10 import CIFAR10
from pylearn2.datasets.cifar100 import CIFAR100
from pylearn2.datasets.tl_challenge import TL_Challenge
import sys
config.floatX = 'float32'
from mlabwrap import mlab
s = mlab._session
import mlabraw as M
M.eval(s, "addpath('/u/goodfeli/SPAMS/release/atlas64')")
M.eval(s, "addpath('/u/goodfeli/galatea/s3c/sc_vq_demo')")
class halver:
def __init__(self,f,nhid):
self.f = f
self.nhid = nhid
def __call__(self,X):
m = X.shape[0]
#mid = m/2
m1 = m/3
m2 = 2*m/3
rval = np.zeros((m,self.nhid),dtype='float32')
rval[0:m1,:] = self.f(X[0:m1,:])
def _evaluate(self, code_block):
#eval line by line in code_block
for node in code_block:
func_nodes = []
#multiline code -> for, while, if, case
#if statement
#one line code
if node.cls in ('Assign', 'Assigns', 'Statement'):
#flatten the node, check for function: Var/Get unknown type name
sub_nodes = node.flatten(False, False, False)
#check if Var/Get unknown type name -> self written function
for sub_node in sub_nodes:
if ((sub_node.cls == 'Get' or sub_node.cls == 'Var') and \
sub_node.backend == 'unknown' and \
sub_node.type == 'TYPE' and \
sub_node.name in self.func_names):
#Test if function processed before
if sub_node.name not in self.called_func_names:
func_nodes.append(sub_node)
self.called_func_names.append(sub_node.name)
#loop over func_nodes
for func_node in func_nodes:
#print func_node.name
params = []
function = False
#save and store variables, if entered function
# node.func.name is name of caller function
mlabraw.eval(self.session,
'save mconvert_' + code_block.func.name)
#clear variables, set params, Get function have params
mlabraw.eval(self.session, 'clear all')
#find function node that is called
#check current file for function -> code_block
funcs = code_block.func.parent #funcs
for func in funcs:
if func_node.name == func.name:
new_code_block = func[3]
function = True
#check other file for function -> code_block
if not function:
for program in node.project:
func = program[1][0]
if func_node.name == func.name:
params = func[2]
new_code_block = func[3]
function = True
#set params
for var, param in zip(func_node, params):
my_string = "load (\'mconvert_" + code_block.func.name + "\', \'" + var.name + "\')"
mlabraw.eval(self.session, my_string)
mlabraw.eval(self.session,
param.name + ' = ' + var.name)
if var.name != param.name:
mlabraw.eval(self.session, 'clear ' + var.name)
#jump into code_block of the function
self._evaluate(new_code_block)
mlabraw.eval(self.session, 'whos_f')
#load previous variables
mlabraw.eval(self.session, 'clear all')
mlabraw.eval(self.session,
'load mconvert_' + code_block.func.name)
#eval code line
mlabraw.eval(self.session, node.code)