def run():
"""
<Description>
Args:
param1: This is the first param.
Returns:
This is a description of what is returned.
"""
base_dir = RetinalUtil._landscape_base()
step = Pipeline.Step.REDUCED
in_dir = Pipeline._cache_dir(base=base_dir, enum=Pipeline.Step.SANITIZED)
out_dir = Pipeline._cache_dir(base=base_dir, enum=step)
force = True
limit = None
functor = lambda: to_iwt(in_dir)
data = CheckpointUtilities.multi_load(cache_dir=out_dir,
load_func=functor,
force=force,
limit=limit,
name_func=FEC_Util.fec_name_func)
ProcessingUtil.plot_data(base_dir, step, data, xlim=[-50, 150])
plot_subdir = Pipeline._plot_subdir(base_dir, step)
out_name = plot_subdir + "heatmap.png"
ProcessingUtil.heatmap_ensemble_plot(data, out_name=out_name)
def run():
"""
<Description>
Args:
param1: This is the first param.
Returns:
This is a description of what is returned.
"""
base_dir = RetinalUtil._landscape_base()
step = Pipeline.Step.SANITIZED
in_dir = Pipeline._cache_dir(base=base_dir, enum=Pipeline.Step.MANUAL)
out_dir = Pipeline._cache_dir(base=base_dir, enum=step)
force = True
limit = None
min_sep = RetinalUtil.min_sep_landscape()
max_sep = min_sep + 100e-9
functor = lambda: slice_data(in_dir, min_sep=min_sep, max_sep=max_sep)
data = CheckpointUtilities.multi_load(cache_dir=out_dir,
load_func=functor,
force=force,
limit=limit,
name_func=FEC_Util.fec_name_func)
plot_dir = Pipeline._plot_subdir(base=base_dir, enum=step)
ProcessingUtil.heatmap_ensemble_plot(data,
out_name=plot_dir + "heatmap.png",
xlim=[-20, max_sep * 1e9])
# plot each individual
ProcessingUtil.plot_data(base_dir, step, data)
def run():
"""
<Description>
Args:
param1: This is the first param.
Returns:
This is a description of what is returned.
"""
default_base = "../../../Data/170321FEC/"
base_dir = Pipeline._base_dir_from_cmd(default=default_base)
step = Pipeline.Step.FILTERED
in_dir = Pipeline._cache_dir(base=base_dir, enum=Pipeline.Step.READ)
out_dir = Pipeline._cache_dir(base=base_dir, enum=step)
force = True
limit = None
f_filter_Hz = 5e3
# filter to X s
t_filter_s = 1 / f_filter_Hz
# t_filter -> n_filter_points
# after filtering, take every N points, where
# N = f_decimate * n_filter_points
# in other words, we oversample by 1/f_decimate
f_decimate = 0.33
assert f_decimate < 1 and f_decimate > 0
data = filter_data(in_dir, out_dir, force, t_filter_s, f_decimate)
ProcessingUtil.plot_data(base_dir, step, data, markevery=1)
def run():
"""
<Description>
Args:
param1: This is the first param.
Returns:
This is a description of what is returned.
"""
base_input_processing = RetinalUtil._processing_base()
base_dir = RetinalUtil._landscape_base()
step = Pipeline.Step.MANUAL
in_dir = Pipeline._cache_dir(base=base_input_processing,
enum=Pipeline.Step.POLISH)
out_dir = Pipeline._cache_dir(base=base_dir,enum=step)
data_input = CheckpointUtilities.lazy_multi_load(in_dir)
force = True
functor = lambda : ProcessingUtil.\
_filter_by_bl(data_input,base_input_processing)
data = CheckpointUtilities.multi_load(cache_dir=out_dir,load_func=functor,
force=force,
name_func=FEC_Util.fec_name_func)
# plot each individual
ProcessingUtil.plot_data(base_dir,step,data,xlim=[-50,150])
plot_subdir = Pipeline._plot_subdir(base_dir, step)
out_name = plot_subdir + "heatmap.png"
ProcessingUtil.heatmap_ensemble_plot(data, out_name=out_name)
def polish_plot(ax1,ax2,d_unpolish,d_polish,xlim,ylim,f_x,plot_components_1,i):
plt.sca(ax1)
kw_plot = dict(f_x=f_x,xlim=xlim,ylim=ylim,use_shift=True,i=i)
ProcessingUtil._aligned_plot(d_unpolish, plot_components=plot_components_1,
**kw_plot)
PlotUtilities.xlabel("")
PlotUtilities.no_x_label(ax1)
plt.sca(ax2)
ProcessingUtil._aligned_plot(d_polish, plot_components=False,
**kw_plot)
def _feather_plot(data,plot_subdir,f_x = lambda x: x.Separation,
xlim=[-20, 150]):
_, ylim = ProcessingUtil.nm_and_pN_limits(data, f_x=f_x)
for i,d in enumerate(data):
fig = PlotUtilities.figure()
ProcessingUtil.plot_single_fec(d, f_x, xlim, ylim, markevery=1,
i=i)
x = f_x(d) * 1e9
for i in d.info_feather.event_idx:
plt.axvline(x[i])
name = FEC_Util.fec_name_func(0,d)
PlotUtilities.savefig(fig,plot_subdir + name + ".png")
def feather_single(d,force_no_adhesion=False,**kw):
"""
:param d: FEC to get FJC+WLC fit of
:param min_F_N: minimum force, in Newtons, for fitting event. helps avoid
occasional small force events
:param kw: keywords to use for fitting...
:return:
"""
force_N = d.Force
where_above_surface = np.where(force_N >= 0)[0]
assert where_above_surface.size > 0, "Force never above surface "
# use FEATHER; fit to the first event, don't look for adhesion
d_pred_only = d._slice(slice(0,None,1))
# first, try removing surface adhesions
is_600 = _is_PEG600(d)
skip_adhesion = force_no_adhesion or is_600
f_refs_initial = [Detector.delta_mask_function] if skip_adhesion else None
feather_kw = dict(d=d_pred_only,**kw)
pred_info,tau_n = _detect_retract_FEATHER(f_refs=f_refs_initial,
**feather_kw)
# if we removed more than 20nm or we didnt find any events, then
# FEATHER got confused by a near-surface BR. Tell it not to look for
# surface adhesions
expected_surface_m = d.Separation[pred_info.slice_fit.start]
expected_gf_m = 20e-9
if ((len(pred_info.event_idx) == 0) or (expected_surface_m > expected_gf_m)):
f_refs = [Detector.delta_mask_function]
pred_info,tau_n = _detect_retract_FEATHER(f_refs=f_refs,
**feather_kw)
pred_info.tau_n = tau_n
assert len(pred_info.event_idx) > 0 , "FEATHER can't find an event..."
# POST: found at least one event.
to_ret = ProcessingUtil.AlignedFEC(d,info_fit=None,feather_info=pred_info)
return to_ret
def run():
"""
<Description>
Args:
param1: This is the first param.
Returns:
This is a description of what is returned.
"""
default_base = "../../../Data/170321FEC/"
base_dir = Pipeline._base_dir_from_cmd(default=default_base)
step = Pipeline.Step.CORRECTED
in_dir = Pipeline._cache_dir(base=base_dir, enum=Pipeline.Step.FILTERED)
out_dir = Pipeline._cache_dir(base=base_dir, enum=step)
force = True
n_filter = 10
data = filter_data(in_dir, out_dir, force, n_filter)
ProcessingUtil.plot_data(base_dir, step, data, markevery=1)
def _plot_fec_list(list_v,
xlim,
ylim,
label=None,
color=None,
linewidth=0.3,
**kw):
f_x = lambda x_tmp: x_tmp.Separation
for i, d in enumerate(list_v):
label_tmp = label if i == 0 else None
ProcessingUtil.plot_single_fec(d,
f_x,
xlim,
ylim,
label=label_tmp,
style_data=dict(color=color,
alpha=0.3,
linewidth=linewidth),
i=i,
**kw)
def run():
"""
<Description>
Args:
param1: This is the first param.
Returns:
This is a description of what is returned.
"""
default_base = "../../../Data/170321FEC/"
base_dir = Pipeline._base_dir_from_cmd(default=default_base)
step = Pipeline.Step.READ
cache_dir = Pipeline._cache_dir(base=base_dir, enum=step)
force = True
limit = None
functor = lambda: read_all_data(base_dir)
data = CheckpointUtilities.multi_load(cache_dir=cache_dir,
load_func=functor,
force=force,
limit=limit,
name_func=FEC_Util.fec_name_func)
ProcessingUtil.plot_data(base_dir, step, data, markevery=100)
def run():
"""
<Description>
Args:
param1: This is the first param.
Returns:
This is a description of what is returned.
"""
default_base = "../../../Data/170321FEC/"
base_dir = Pipeline._base_dir_from_cmd(default=default_base)
step = Pipeline.Step.ALIGNED
in_dir = Pipeline._cache_dir(base=base_dir, enum=Pipeline.Step.SANITIZED)
out_dir = Pipeline._cache_dir(base=base_dir, enum=step)
force = True
max_n_pool = multiprocessing.cpu_count() - 1
n_pool = 6
N_fit_pts = 20
min_F_N = 175e-12 if "+Retinal" in base_dir else 90e-12
data = RetinalUtil.align_data(in_dir,
out_dir,
force=force,
n_pool=n_pool,
min_F_N=min_F_N,
N_fit_pts=N_fit_pts)
plot_subdir = Pipeline._plot_subdir(base_dir, step)
xlim_heatmap_nm = [-40, 100]
ProcessingUtil.heatmap_ensemble_plot(data,
xlim=xlim_heatmap_nm,
out_name=plot_subdir + "heatmap.png")
# get the post-blacklist heapmap, too..
data_filtered = ProcessingUtil._filter_by_bl(data, in_dir)
# align the data...
data_aligned = [RetinalUtil._polish_helper(d) for d in data_filtered]
out_name = plot_subdir + "heatmap_bl.png"
ProcessingUtil.heatmap_ensemble_plot(data_aligned,
xlim=xlim_heatmap_nm,
out_name=out_name)
# make individual plots
ProcessingUtil.make_aligned_plot(base_dir,
step,
data,
xlim=[-30, 150],
use_shift=True)
def align_single(d,min_F_N,**kw):
"""
:param d: FEC to get FJC+WLC fit of
:param min_F_N: minimum force, in Newtons, for fitting event. helps avoid
occasional small force events
:param kw: keywords to use for fitting...
:return:
"""
force_N = d.Force
pred_info = d.info_feather
max_fit_idx = GF2_event_idx(d,min_F_N)
# determine the minimum fit index by the maximum of...
# (1) the surface
where_above_surface = np.where(force_N >= 0)[0]
assert where_above_surface.size > 0 , "Never above surface"
first_time_above_surface = where_above_surface[0]
assert first_time_above_surface < max_fit_idx , \
"Couldn't find fitting region"
# (2) the last event *before* the current, if it exists
event_idx = pred_info.event_idx
idx_last_event_before = np.where(event_idx < max_fit_idx)[0]
start_idx = first_time_above_surface
if idx_last_event_before.size > 0:
# then we have an event *before* the final GC helix one.
# make that the start idx, if it is later (it should be!)
new_start = event_idx[idx_last_event_before[-1]]
assert new_start > start_idx , "First event is less than surface?!"
start_idx = new_start
# start the fit after any potential adhesions
fit_start = max(start_idx,pred_info.slice_fit.start)
fit_slice = slice(fit_start,max_fit_idx,1)
# slice the object to just the region we want
obj_slice = d._slice(fit_slice)
# fit wlc to the f vs x of that slice
info_fit = WLCHao.hao_fit(obj_slice.Separation,obj_slice.Force,**kw)
info_fit.fit_slice = fit_slice
to_ret = ProcessingUtil.AlignedFEC(d,info_fit,feather_info=pred_info)
return to_ret
def _multi_align(out_dir,kw,all_data,n_pool):
input_v = [ [d,out_dir,kw] for d in all_data]
to_ret = ProcessingUtil._multiproc(func_align, input_v, n_pool)
to_ret = [r for r in to_ret if r is not None]
return to_ret
def make_plot(dir_in, out_name):
dir_aligned = dir_in + "cache_8_aligned/"
dir_polished = dir_in + "cache_10_polish/"
data_unaligned = CheckpointUtilities.lazy_multi_load(dir_aligned)
data_polished = CheckpointUtilities.lazy_multi_load(dir_polished)
data_unaligned = ProcessingUtil._filter_by_bl(data_unaligned, dir_in)
data_polished = ProcessingUtil._filter_by_bl(data_polished, dir_in)
# scootch the data over slightly
for d in data_polished:
d.Separation -= 20e-9
to_x = lambda _x: _x * 1e9
to_y = lambda _y: _y * 1e12
xlim = [-10, 100]
ylim = [-20, 350]
plt.close()
fig = PlotUtilities.figure(figsize=(3.5, 4.5))
kw_plot = dict(linewidth=0.75)
ax1 = plt.subplot(2, 2, 1)
for d in data_unaligned:
plt.plot(to_x(d.Separation), to_y(d.Force), **kw_plot)
ax2 = plt.subplot(2, 2, 2)
for d in data_polished:
plt.plot(to_x(d.Separation), to_y(d.Force), **kw_plot)
for a in [ax1, ax2]:
a.set_ylim(ylim)
a.set_xlim(xlim)
PlotUtilities.lazyLabel("Extension (nm)", "$F$ (pN)", "", ax=a)
PlotUtilities.ylabel("", ax=ax2)
height_pN = 100
min_y, max_y, delta_y = Scalebar.offsets_zero_tick(
limits=ylim, range_scalebar=height_pN)
kw_scale = dict(offset_x=0.7,
offset_y=max_y,
x_kwargs=dict(width=20, unit="nm"),
y_kwargs=dict(height=height_pN, unit="pN"))
for a in [ax1, ax2]:
Scalebar.crossed_x_and_y_relative(ax=a, **kw_scale)
PlotUtilities.no_y_label(ax=a)
PlotUtilities.no_x_label(ax=a)
PlotUtilities.x_label_on_top(ax=a)
num = 200
bins_x = np.linspace(xlim[0], xlim[-1], endpoint=True, num=num)
bins_y = np.linspace(ylim[0], ylim[-1], endpoint=True, num=num)
kw = dict(color='w', use_colorbar=False, bins=(bins_x, bins_y), title="")
kw_scale_heat = dict(**kw_scale)
line_kw_def = dict(color='w', linewidth=2)
font_x, font_y = Scalebar.font_kwargs_modified(x_kwargs=dict(color='w'),
y_kwargs=dict(color='w'))
kw_scale_heat['x_kwargs']['line_kwargs'] = line_kw_def
kw_scale_heat['x_kwargs']['font_kwargs'] = font_x
kw_scale_heat['y_kwargs']['line_kwargs'] = line_kw_def
kw_scale_heat['y_kwargs']['font_kwargs'] = font_y
ax3 = plt.subplot(2, 2, 3)
Plotting.formatted_heatmap(data=data_unaligned, **kw)
Scalebar.crossed_x_and_y_relative(ax=ax3, **kw_scale_heat)
ax4 = plt.subplot(2, 2, 4)
Plotting.formatted_heatmap(data=data_polished, **kw)
Scalebar.crossed_x_and_y_relative(ax=ax4, **kw_scale_heat)
for a in [ax3, ax4]:
PlotUtilities.no_y_label(ax=a)
PlotUtilities.no_x_label(ax=a)
a.set_xlim(xlim)
a.set_ylim(ylim)
PlotUtilities.xlabel("", ax=a)
PlotUtilities.ylabel("", ax=ax4)
PlotUtilities.ylabel("$F$ (pN)", ax=ax3)
PlotUtilities.savefig(fig, out_name)
pass
def align_data(base_dir,out_dir,n_pool,**kw):
all_data = CheckpointUtilities.lazy_multi_load(base_dir)
input_v = [ [d,out_dir,kw] for d in all_data]
to_ret = ProcessingUtil._multiproc(func, input_v, n_pool)
to_ret = [r for r in to_ret if r is not None]
return to_ret
def filter_data(base_dir, out_dir, force, t_filter, f_decimate):
all_data = CheckpointUtilities.lazy_multi_load(base_dir)
input_v = [[d, out_dir, force, t_filter, f_decimate] for d in all_data]
to_ret = ProcessingUtil._multiproc(func, input_v, n_pool=3)
return to_ret
def func(args):
d, out_dir, force, t_filter, f_decimate = args
to_ret = ProcessingUtil._cache_individual(d, out_dir, _filter_single,
force, d, t_filter, f_decimate)
return to_ret
def func(args):
d, out_dir, force, n_filter = args
to_ret = ProcessingUtil._cache_individual(d, out_dir, _filter_single,
force, d, n_filter)
return to_ret
def run():
"""
<Description>
Args:
param1: This is the first param.
Returns:
This is a description of what is returned.
"""
default_base = "../../../Data/170321FEC/"
base_dir = Pipeline._base_dir_from_cmd(default=default_base)
step = Pipeline.Step.POLISH
in_dir = Pipeline._cache_dir(base=base_dir, enum=Pipeline.Step.ALIGNED)
out_dir = Pipeline._cache_dir(base=base_dir,enum=step)
plot_dir = Pipeline._plot_subdir(base=base_dir, enum=step)
force = True
limit = None
functor = lambda : polish_data(in_dir)
data =CheckpointUtilities.multi_load(cache_dir=out_dir,load_func=functor,
force=force,
limit=limit,
name_func=FEC_Util.fec_name_func)
sizes = [d.Force.size for d in data]
min_s = min(sizes)
sliced_data = [d._slice(slice(0,min_s,1)) for d in data]
for d in sliced_data:
d.Offset = d.ZSnsr[0]
d.Extension = d.Separation
d.kT = 4.1e-21
data_wham = UtilWHAM.to_wham_input(objs=sliced_data, n_ext_bins=75)
data_wham.z = np.array([d.ZSnsr for d in sliced_data])
obj_wham = WeightedHistogram.wham(data_wham)
data_unpolished = CheckpointUtilities.lazy_multi_load(in_dir)
f_x_zsnsr = lambda x: x.ZSnsr
xlim = [-40,150]
ProcessingUtil.heatmap_ensemble_plot(data,out_name=plot_dir + "heatmap.png",
xlim=xlim)
ProcessingUtil.heatmap_ensemble_plot(data,f_x=f_x_zsnsr,
out_name=plot_dir + "heatmap_Z.png",
kw_map=dict(x_func=f_x_zsnsr),
xlim=xlim)
# plot each individual
f_x = lambda x_tmp : x_tmp.Separation
plot_subdir = Pipeline._plot_subdir(base_dir, step)
name_func = FEC_Util.fec_name_func
_, ylim = ProcessingUtil.nm_and_pN_limits(data,f_x)
for i,(d_unpolish,d_polish) in enumerate(zip(data_unpolished,data)):
kw_plot = dict(d_unpolish=d_unpolish,
d_polish=d_polish,
xlim=xlim,ylim=ylim,i=i)
fig = PlotUtilities.figure((6,6))
# make the Separation column
ax1,ax2 = plt.subplot(2,2,1), plt.subplot(2,2,3)
polish_plot(ax1, ax2, f_x = lambda x: x.Separation,
plot_components_1=True,**kw_plot)
# make the ZSnsr column
ax3,ax4 = plt.subplot(2,2,2), plt.subplot(2,2,4)
polish_plot(ax3, ax4, f_x = lambda x: x.ZSnsr,plot_components_1=False,
**kw_plot)
PlotUtilities.xlabel("Stage Position (nm)", ax=ax4)
for a in [ax3,ax4]:
PlotUtilities.no_y_label(ax=a)
PlotUtilities.ylabel("",ax=a)
name = plot_subdir + name_func(0, d_polish) + ".png"
PlotUtilities.savefig(fig,name)
def _align_and_cache(d,out_dir,force=False,**kw):
return ProcessingUtil._cache_individual(d, out_dir,
RetinalUtil.feather_single,
force,d, **kw)