def run(): """ <Description> Args: param1: This is the first param. Returns: This is a description of what is returned. """ base_dir = RetinalUtil._landscape_base() step = Pipeline.Step.REDUCED in_dir = Pipeline._cache_dir(base=base_dir, enum=Pipeline.Step.SANITIZED) out_dir = Pipeline._cache_dir(base=base_dir, enum=step) force = True limit = None functor = lambda: to_iwt(in_dir) data = CheckpointUtilities.multi_load(cache_dir=out_dir, load_func=functor, force=force, limit=limit, name_func=FEC_Util.fec_name_func) ProcessingUtil.plot_data(base_dir, step, data, xlim=[-50, 150]) plot_subdir = Pipeline._plot_subdir(base_dir, step) out_name = plot_subdir + "heatmap.png" ProcessingUtil.heatmap_ensemble_plot(data, out_name=out_name)
def run(): """ <Description> Args: param1: This is the first param. Returns: This is a description of what is returned. """ base_dir = RetinalUtil._landscape_base() step = Pipeline.Step.SANITIZED in_dir = Pipeline._cache_dir(base=base_dir, enum=Pipeline.Step.MANUAL) out_dir = Pipeline._cache_dir(base=base_dir, enum=step) force = True limit = None min_sep = RetinalUtil.min_sep_landscape() max_sep = min_sep + 100e-9 functor = lambda: slice_data(in_dir, min_sep=min_sep, max_sep=max_sep) data = CheckpointUtilities.multi_load(cache_dir=out_dir, load_func=functor, force=force, limit=limit, name_func=FEC_Util.fec_name_func) plot_dir = Pipeline._plot_subdir(base=base_dir, enum=step) ProcessingUtil.heatmap_ensemble_plot(data, out_name=plot_dir + "heatmap.png", xlim=[-20, max_sep * 1e9]) # plot each individual ProcessingUtil.plot_data(base_dir, step, data)
def run(): """ <Description> Args: param1: This is the first param. Returns: This is a description of what is returned. """ default_base = "../../../Data/170321FEC/" base_dir = Pipeline._base_dir_from_cmd(default=default_base) step = Pipeline.Step.FILTERED in_dir = Pipeline._cache_dir(base=base_dir, enum=Pipeline.Step.READ) out_dir = Pipeline._cache_dir(base=base_dir, enum=step) force = True limit = None f_filter_Hz = 5e3 # filter to X s t_filter_s = 1 / f_filter_Hz # t_filter -> n_filter_points # after filtering, take every N points, where # N = f_decimate * n_filter_points # in other words, we oversample by 1/f_decimate f_decimate = 0.33 assert f_decimate < 1 and f_decimate > 0 data = filter_data(in_dir, out_dir, force, t_filter_s, f_decimate) ProcessingUtil.plot_data(base_dir, step, data, markevery=1)
def run(): """ <Description> Args: param1: This is the first param. Returns: This is a description of what is returned. """ base_input_processing = RetinalUtil._processing_base() base_dir = RetinalUtil._landscape_base() step = Pipeline.Step.MANUAL in_dir = Pipeline._cache_dir(base=base_input_processing, enum=Pipeline.Step.POLISH) out_dir = Pipeline._cache_dir(base=base_dir,enum=step) data_input = CheckpointUtilities.lazy_multi_load(in_dir) force = True functor = lambda : ProcessingUtil.\ _filter_by_bl(data_input,base_input_processing) data = CheckpointUtilities.multi_load(cache_dir=out_dir,load_func=functor, force=force, name_func=FEC_Util.fec_name_func) # plot each individual ProcessingUtil.plot_data(base_dir,step,data,xlim=[-50,150]) plot_subdir = Pipeline._plot_subdir(base_dir, step) out_name = plot_subdir + "heatmap.png" ProcessingUtil.heatmap_ensemble_plot(data, out_name=out_name)
def polish_plot(ax1,ax2,d_unpolish,d_polish,xlim,ylim,f_x,plot_components_1,i): plt.sca(ax1) kw_plot = dict(f_x=f_x,xlim=xlim,ylim=ylim,use_shift=True,i=i) ProcessingUtil._aligned_plot(d_unpolish, plot_components=plot_components_1, **kw_plot) PlotUtilities.xlabel("") PlotUtilities.no_x_label(ax1) plt.sca(ax2) ProcessingUtil._aligned_plot(d_polish, plot_components=False, **kw_plot)
def _feather_plot(data,plot_subdir,f_x = lambda x: x.Separation, xlim=[-20, 150]): _, ylim = ProcessingUtil.nm_and_pN_limits(data, f_x=f_x) for i,d in enumerate(data): fig = PlotUtilities.figure() ProcessingUtil.plot_single_fec(d, f_x, xlim, ylim, markevery=1, i=i) x = f_x(d) * 1e9 for i in d.info_feather.event_idx: plt.axvline(x[i]) name = FEC_Util.fec_name_func(0,d) PlotUtilities.savefig(fig,plot_subdir + name + ".png")
def feather_single(d,force_no_adhesion=False,**kw): """ :param d: FEC to get FJC+WLC fit of :param min_F_N: minimum force, in Newtons, for fitting event. helps avoid occasional small force events :param kw: keywords to use for fitting... :return: """ force_N = d.Force where_above_surface = np.where(force_N >= 0)[0] assert where_above_surface.size > 0, "Force never above surface " # use FEATHER; fit to the first event, don't look for adhesion d_pred_only = d._slice(slice(0,None,1)) # first, try removing surface adhesions is_600 = _is_PEG600(d) skip_adhesion = force_no_adhesion or is_600 f_refs_initial = [Detector.delta_mask_function] if skip_adhesion else None feather_kw = dict(d=d_pred_only,**kw) pred_info,tau_n = _detect_retract_FEATHER(f_refs=f_refs_initial, **feather_kw) # if we removed more than 20nm or we didnt find any events, then # FEATHER got confused by a near-surface BR. Tell it not to look for # surface adhesions expected_surface_m = d.Separation[pred_info.slice_fit.start] expected_gf_m = 20e-9 if ((len(pred_info.event_idx) == 0) or (expected_surface_m > expected_gf_m)): f_refs = [Detector.delta_mask_function] pred_info,tau_n = _detect_retract_FEATHER(f_refs=f_refs, **feather_kw) pred_info.tau_n = tau_n assert len(pred_info.event_idx) > 0 , "FEATHER can't find an event..." # POST: found at least one event. to_ret = ProcessingUtil.AlignedFEC(d,info_fit=None,feather_info=pred_info) return to_ret
def run(): """ <Description> Args: param1: This is the first param. Returns: This is a description of what is returned. """ default_base = "../../../Data/170321FEC/" base_dir = Pipeline._base_dir_from_cmd(default=default_base) step = Pipeline.Step.CORRECTED in_dir = Pipeline._cache_dir(base=base_dir, enum=Pipeline.Step.FILTERED) out_dir = Pipeline._cache_dir(base=base_dir, enum=step) force = True n_filter = 10 data = filter_data(in_dir, out_dir, force, n_filter) ProcessingUtil.plot_data(base_dir, step, data, markevery=1)
def _plot_fec_list(list_v, xlim, ylim, label=None, color=None, linewidth=0.3, **kw): f_x = lambda x_tmp: x_tmp.Separation for i, d in enumerate(list_v): label_tmp = label if i == 0 else None ProcessingUtil.plot_single_fec(d, f_x, xlim, ylim, label=label_tmp, style_data=dict(color=color, alpha=0.3, linewidth=linewidth), i=i, **kw)
def run(): """ <Description> Args: param1: This is the first param. Returns: This is a description of what is returned. """ default_base = "../../../Data/170321FEC/" base_dir = Pipeline._base_dir_from_cmd(default=default_base) step = Pipeline.Step.READ cache_dir = Pipeline._cache_dir(base=base_dir, enum=step) force = True limit = None functor = lambda: read_all_data(base_dir) data = CheckpointUtilities.multi_load(cache_dir=cache_dir, load_func=functor, force=force, limit=limit, name_func=FEC_Util.fec_name_func) ProcessingUtil.plot_data(base_dir, step, data, markevery=100)
def run(): """ <Description> Args: param1: This is the first param. Returns: This is a description of what is returned. """ default_base = "../../../Data/170321FEC/" base_dir = Pipeline._base_dir_from_cmd(default=default_base) step = Pipeline.Step.ALIGNED in_dir = Pipeline._cache_dir(base=base_dir, enum=Pipeline.Step.SANITIZED) out_dir = Pipeline._cache_dir(base=base_dir, enum=step) force = True max_n_pool = multiprocessing.cpu_count() - 1 n_pool = 6 N_fit_pts = 20 min_F_N = 175e-12 if "+Retinal" in base_dir else 90e-12 data = RetinalUtil.align_data(in_dir, out_dir, force=force, n_pool=n_pool, min_F_N=min_F_N, N_fit_pts=N_fit_pts) plot_subdir = Pipeline._plot_subdir(base_dir, step) xlim_heatmap_nm = [-40, 100] ProcessingUtil.heatmap_ensemble_plot(data, xlim=xlim_heatmap_nm, out_name=plot_subdir + "heatmap.png") # get the post-blacklist heapmap, too.. data_filtered = ProcessingUtil._filter_by_bl(data, in_dir) # align the data... data_aligned = [RetinalUtil._polish_helper(d) for d in data_filtered] out_name = plot_subdir + "heatmap_bl.png" ProcessingUtil.heatmap_ensemble_plot(data_aligned, xlim=xlim_heatmap_nm, out_name=out_name) # make individual plots ProcessingUtil.make_aligned_plot(base_dir, step, data, xlim=[-30, 150], use_shift=True)
def align_single(d,min_F_N,**kw): """ :param d: FEC to get FJC+WLC fit of :param min_F_N: minimum force, in Newtons, for fitting event. helps avoid occasional small force events :param kw: keywords to use for fitting... :return: """ force_N = d.Force pred_info = d.info_feather max_fit_idx = GF2_event_idx(d,min_F_N) # determine the minimum fit index by the maximum of... # (1) the surface where_above_surface = np.where(force_N >= 0)[0] assert where_above_surface.size > 0 , "Never above surface" first_time_above_surface = where_above_surface[0] assert first_time_above_surface < max_fit_idx , \ "Couldn't find fitting region" # (2) the last event *before* the current, if it exists event_idx = pred_info.event_idx idx_last_event_before = np.where(event_idx < max_fit_idx)[0] start_idx = first_time_above_surface if idx_last_event_before.size > 0: # then we have an event *before* the final GC helix one. # make that the start idx, if it is later (it should be!) new_start = event_idx[idx_last_event_before[-1]] assert new_start > start_idx , "First event is less than surface?!" start_idx = new_start # start the fit after any potential adhesions fit_start = max(start_idx,pred_info.slice_fit.start) fit_slice = slice(fit_start,max_fit_idx,1) # slice the object to just the region we want obj_slice = d._slice(fit_slice) # fit wlc to the f vs x of that slice info_fit = WLCHao.hao_fit(obj_slice.Separation,obj_slice.Force,**kw) info_fit.fit_slice = fit_slice to_ret = ProcessingUtil.AlignedFEC(d,info_fit,feather_info=pred_info) return to_ret
def _multi_align(out_dir,kw,all_data,n_pool): input_v = [ [d,out_dir,kw] for d in all_data] to_ret = ProcessingUtil._multiproc(func_align, input_v, n_pool) to_ret = [r for r in to_ret if r is not None] return to_ret
def make_plot(dir_in, out_name): dir_aligned = dir_in + "cache_8_aligned/" dir_polished = dir_in + "cache_10_polish/" data_unaligned = CheckpointUtilities.lazy_multi_load(dir_aligned) data_polished = CheckpointUtilities.lazy_multi_load(dir_polished) data_unaligned = ProcessingUtil._filter_by_bl(data_unaligned, dir_in) data_polished = ProcessingUtil._filter_by_bl(data_polished, dir_in) # scootch the data over slightly for d in data_polished: d.Separation -= 20e-9 to_x = lambda _x: _x * 1e9 to_y = lambda _y: _y * 1e12 xlim = [-10, 100] ylim = [-20, 350] plt.close() fig = PlotUtilities.figure(figsize=(3.5, 4.5)) kw_plot = dict(linewidth=0.75) ax1 = plt.subplot(2, 2, 1) for d in data_unaligned: plt.plot(to_x(d.Separation), to_y(d.Force), **kw_plot) ax2 = plt.subplot(2, 2, 2) for d in data_polished: plt.plot(to_x(d.Separation), to_y(d.Force), **kw_plot) for a in [ax1, ax2]: a.set_ylim(ylim) a.set_xlim(xlim) PlotUtilities.lazyLabel("Extension (nm)", "$F$ (pN)", "", ax=a) PlotUtilities.ylabel("", ax=ax2) height_pN = 100 min_y, max_y, delta_y = Scalebar.offsets_zero_tick( limits=ylim, range_scalebar=height_pN) kw_scale = dict(offset_x=0.7, offset_y=max_y, x_kwargs=dict(width=20, unit="nm"), y_kwargs=dict(height=height_pN, unit="pN")) for a in [ax1, ax2]: Scalebar.crossed_x_and_y_relative(ax=a, **kw_scale) PlotUtilities.no_y_label(ax=a) PlotUtilities.no_x_label(ax=a) PlotUtilities.x_label_on_top(ax=a) num = 200 bins_x = np.linspace(xlim[0], xlim[-1], endpoint=True, num=num) bins_y = np.linspace(ylim[0], ylim[-1], endpoint=True, num=num) kw = dict(color='w', use_colorbar=False, bins=(bins_x, bins_y), title="") kw_scale_heat = dict(**kw_scale) line_kw_def = dict(color='w', linewidth=2) font_x, font_y = Scalebar.font_kwargs_modified(x_kwargs=dict(color='w'), y_kwargs=dict(color='w')) kw_scale_heat['x_kwargs']['line_kwargs'] = line_kw_def kw_scale_heat['x_kwargs']['font_kwargs'] = font_x kw_scale_heat['y_kwargs']['line_kwargs'] = line_kw_def kw_scale_heat['y_kwargs']['font_kwargs'] = font_y ax3 = plt.subplot(2, 2, 3) Plotting.formatted_heatmap(data=data_unaligned, **kw) Scalebar.crossed_x_and_y_relative(ax=ax3, **kw_scale_heat) ax4 = plt.subplot(2, 2, 4) Plotting.formatted_heatmap(data=data_polished, **kw) Scalebar.crossed_x_and_y_relative(ax=ax4, **kw_scale_heat) for a in [ax3, ax4]: PlotUtilities.no_y_label(ax=a) PlotUtilities.no_x_label(ax=a) a.set_xlim(xlim) a.set_ylim(ylim) PlotUtilities.xlabel("", ax=a) PlotUtilities.ylabel("", ax=ax4) PlotUtilities.ylabel("$F$ (pN)", ax=ax3) PlotUtilities.savefig(fig, out_name) pass
def align_data(base_dir,out_dir,n_pool,**kw): all_data = CheckpointUtilities.lazy_multi_load(base_dir) input_v = [ [d,out_dir,kw] for d in all_data] to_ret = ProcessingUtil._multiproc(func, input_v, n_pool) to_ret = [r for r in to_ret if r is not None] return to_ret
def filter_data(base_dir, out_dir, force, t_filter, f_decimate): all_data = CheckpointUtilities.lazy_multi_load(base_dir) input_v = [[d, out_dir, force, t_filter, f_decimate] for d in all_data] to_ret = ProcessingUtil._multiproc(func, input_v, n_pool=3) return to_ret
def func(args): d, out_dir, force, t_filter, f_decimate = args to_ret = ProcessingUtil._cache_individual(d, out_dir, _filter_single, force, d, t_filter, f_decimate) return to_ret
def func(args): d, out_dir, force, n_filter = args to_ret = ProcessingUtil._cache_individual(d, out_dir, _filter_single, force, d, n_filter) return to_ret
def run(): """ <Description> Args: param1: This is the first param. Returns: This is a description of what is returned. """ default_base = "../../../Data/170321FEC/" base_dir = Pipeline._base_dir_from_cmd(default=default_base) step = Pipeline.Step.POLISH in_dir = Pipeline._cache_dir(base=base_dir, enum=Pipeline.Step.ALIGNED) out_dir = Pipeline._cache_dir(base=base_dir,enum=step) plot_dir = Pipeline._plot_subdir(base=base_dir, enum=step) force = True limit = None functor = lambda : polish_data(in_dir) data =CheckpointUtilities.multi_load(cache_dir=out_dir,load_func=functor, force=force, limit=limit, name_func=FEC_Util.fec_name_func) sizes = [d.Force.size for d in data] min_s = min(sizes) sliced_data = [d._slice(slice(0,min_s,1)) for d in data] for d in sliced_data: d.Offset = d.ZSnsr[0] d.Extension = d.Separation d.kT = 4.1e-21 data_wham = UtilWHAM.to_wham_input(objs=sliced_data, n_ext_bins=75) data_wham.z = np.array([d.ZSnsr for d in sliced_data]) obj_wham = WeightedHistogram.wham(data_wham) data_unpolished = CheckpointUtilities.lazy_multi_load(in_dir) f_x_zsnsr = lambda x: x.ZSnsr xlim = [-40,150] ProcessingUtil.heatmap_ensemble_plot(data,out_name=plot_dir + "heatmap.png", xlim=xlim) ProcessingUtil.heatmap_ensemble_plot(data,f_x=f_x_zsnsr, out_name=plot_dir + "heatmap_Z.png", kw_map=dict(x_func=f_x_zsnsr), xlim=xlim) # plot each individual f_x = lambda x_tmp : x_tmp.Separation plot_subdir = Pipeline._plot_subdir(base_dir, step) name_func = FEC_Util.fec_name_func _, ylim = ProcessingUtil.nm_and_pN_limits(data,f_x) for i,(d_unpolish,d_polish) in enumerate(zip(data_unpolished,data)): kw_plot = dict(d_unpolish=d_unpolish, d_polish=d_polish, xlim=xlim,ylim=ylim,i=i) fig = PlotUtilities.figure((6,6)) # make the Separation column ax1,ax2 = plt.subplot(2,2,1), plt.subplot(2,2,3) polish_plot(ax1, ax2, f_x = lambda x: x.Separation, plot_components_1=True,**kw_plot) # make the ZSnsr column ax3,ax4 = plt.subplot(2,2,2), plt.subplot(2,2,4) polish_plot(ax3, ax4, f_x = lambda x: x.ZSnsr,plot_components_1=False, **kw_plot) PlotUtilities.xlabel("Stage Position (nm)", ax=ax4) for a in [ax3,ax4]: PlotUtilities.no_y_label(ax=a) PlotUtilities.ylabel("",ax=a) name = plot_subdir + name_func(0, d_polish) + ".png" PlotUtilities.savefig(fig,name)
def _align_and_cache(d,out_dir,force=False,**kw): return ProcessingUtil._cache_individual(d, out_dir, RetinalUtil.feather_single, force,d, **kw)