def main(_high_temporal_resolution=False, _pair_distance_range=None): if _pair_distance_range is None: _pair_distance_range = PAIR_DISTANCE_RANGE _experiments = all_experiments() _experiments = filtering.by_categories( _experiments=_experiments, _is_single_cell=False, _is_high_temporal_resolution=_high_temporal_resolution, _is_bleb=False, _is_dead_dead=False, _is_live_dead=False, _is_bead=False, _is_metastasis=False) _experiments = ['SN16'] _tuples = load.experiments_groups_as_tuples(_experiments) _tuples = filtering.by_pair_distance_range(_tuples, _pair_distance_range) _tuples = filtering.by_real_pairs(_tuples) _tuples = filtering.by_band(_tuples) print('Total tuples:', len(_tuples)) _arguments = [] for _tuple in _tuples: _experiment, _series_id, _group = _tuple _latest_time_frame = compute.latest_time_frame_before_overlapping( _experiment, _series_id, _group, OFFSET_X) for _cell_id, _offset_y in product(['left_cell', 'right_cell'], [0, OFFSET_Y]): _arguments.append({ 'experiment': _experiment, 'series_id': _series_id, 'group': _group, 'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER, 'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER, 'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER, 'offset_x': OFFSET_X, 'offset_y': _offset_y, 'offset_z': OFFSET_Z, 'cell_id': _cell_id, 'direction': 'inside', 'time_points': _latest_time_frame }) _windows_dictionary, _windows_to_compute = compute.windows( _arguments, _keys=['experiment', 'series_id', 'group', 'cell_id', 'offset_y']) _fiber_densities = compute.fiber_densities(_windows_to_compute) _experiments_fiber_densities = { _key: [_fiber_densities[_tuple] for _tuple in _windows_dictionary[_key]] for _key in _windows_dictionary } _no_offset_derivatives_array = [] _offset_derivatives_array = [] for _tuple in tqdm(_tuples, desc='Main loop'): _experiment, _series_id, _group = _tuple _no_offset_left_cell_fiber_densities = _experiments_fiber_densities[( _experiment, _series_id, _group, 'left_cell', 0)] _no_offset_right_cell_fiber_densities = _experiments_fiber_densities[( _experiment, _series_id, _group, 'right_cell', 0)] _offset_left_cell_fiber_densities = _experiments_fiber_densities[( _experiment, _series_id, _group, 'left_cell', OFFSET_Y)] _offset_right_cell_fiber_densities = _experiments_fiber_densities[( _experiment, _series_id, _group, 'right_cell', OFFSET_Y)] _properties = \ load.group_properties(_experiment, _series_id, _group) _no_offset_left_cell_fiber_densities = compute.remove_blacklist( _experiment, _series_id, _properties['cells_ids']['left_cell'], _no_offset_left_cell_fiber_densities) _no_offset_right_cell_fiber_densities = compute.remove_blacklist( _experiment, _series_id, _properties['cells_ids']['right_cell'], _no_offset_right_cell_fiber_densities) _offset_left_cell_fiber_densities = compute.remove_blacklist( _experiment, _series_id, _properties['cells_ids']['left_cell'], _offset_left_cell_fiber_densities) _offset_right_cell_fiber_densities = compute.remove_blacklist( _experiment, _series_id, _properties['cells_ids']['right_cell'], _offset_right_cell_fiber_densities) _normalization = load.normalization_series_file_data( _experiment, _series_id) for _cell_pair in [(_no_offset_left_cell_fiber_densities, _offset_left_cell_fiber_densities), (_no_offset_right_cell_fiber_densities, _offset_right_cell_fiber_densities)]: for _time_frame in range( 1, min(len(_cell_pair[0]), len(_cell_pair[1]))): _no_offset_cell_fiber_density_previous = _cell_pair[0][ _time_frame - 1] _no_offset_cell_fiber_density = _cell_pair[0][_time_frame] _offset_cell_fiber_density_previous = _cell_pair[1][_time_frame - 1] _offset_cell_fiber_density = _cell_pair[1][_time_frame] if any([ _no_offset_cell_fiber_density_previous[1], _no_offset_cell_fiber_density[1], _offset_cell_fiber_density_previous[1], _offset_cell_fiber_density[1] ]): continue _no_offset_cell_fiber_density_previous_z_score = compute_lib.z_score( _x=_no_offset_cell_fiber_density_previous[0], _average=_normalization['average'], _std=_normalization['std']) _no_offset_cell_fiber_density_z_score = compute_lib.z_score( _x=_no_offset_cell_fiber_density[0], _average=_normalization['average'], _std=_normalization['std']) _offset_cell_fiber_density_previous_z_score = compute_lib.z_score( _x=_offset_cell_fiber_density_previous[0], _average=_normalization['average'], _std=_normalization['std']) _offset_cell_fiber_density_z_score = compute_lib.z_score( _x=_offset_cell_fiber_density[0], _average=_normalization['average'], _std=_normalization['std']) _no_offset_derivative = _no_offset_cell_fiber_density_z_score - _no_offset_cell_fiber_density_previous_z_score _offset_derivative = _offset_cell_fiber_density_z_score - _offset_cell_fiber_density_previous_z_score _no_offset_derivatives_array.append(_no_offset_derivative) _offset_derivatives_array.append(_offset_derivative) print('Total points:', len(_no_offset_derivatives_array)) _offset_minus_no_offset = \ np.array(_offset_derivatives_array) - np.array(_no_offset_derivatives_array) print('Wilcoxon of offset minus no offset around the zero:') print(wilcoxon(_offset_minus_no_offset)) print('Higher offset amount:', (_offset_minus_no_offset > 0).sum() / len(_offset_minus_no_offset)) # plot _fig = go.Figure( data=go.Scatter(x=_no_offset_derivatives_array, y=_offset_derivatives_array, mode='markers', marker={ 'size': 5, 'color': '#ea8500' }, showlegend=False), layout={ 'xaxis': { 'title': 'No offset derivative', # 'zeroline': False, # 'range': [-1.1, 1.2], # 'tickmode': 'array', # 'tickvals': [-1, -0.5, 0, 0.5, 1] }, 'yaxis': { 'title': 'Offset derivative', # 'zeroline': False, # 'range': [-1.1, 1.2], # 'tickmode': 'array', # 'tickvals': [-1, -0.5, 0, 0.5, 1] }, 'shapes': [ # { # 'type': 'line', # 'x0': -1, # 'y0': -1, # 'x1': -1, # 'y1': 1, # 'line': { # 'color': 'black', # 'width': 2 # } # }, # { # 'type': 'line', # 'x0': -1, # 'y0': -1, # 'x1': 1, # 'y1': -1, # 'line': { # 'color': 'black', # 'width': 2 # } # }, { 'type': 'line', 'x0': -5, 'y0': -5, 'x1': 5, 'y1': 5, 'line': { 'color': 'red', 'width': 2 } } ] }) save.to_html(_fig=_fig, _path=os.path.join(paths.PLOTS, save.get_module_name()), _filename='plot')
def main(_band=True, _high_temporal_resolution=False): _y_arrays = [[] for _i in PAIR_DISTANCE_RANGES] _names_array = [] for _distances_index, _distances_range in enumerate(PAIR_DISTANCE_RANGES): print('Pair distance range:', str(_distances_range)) _experiments = all_experiments() _experiments = filtering.by_categories( _experiments=_experiments, _is_single_cell=False, _is_high_temporal_resolution=_high_temporal_resolution, _is_bleb=False, _is_dead_dead=False, _is_live_dead=False, _is_bead=False, _is_metastasis=False) _tuples = load.experiments_groups_as_tuples(_experiments) _tuples = filtering.by_pair_distance_range(_tuples, _distances_range) _tuples = filtering.by_real_pairs(_tuples) _tuples = filtering.by_band(_tuples, _band=_band) print('Total tuples:', len(_tuples)) _arguments = [] for _tuple in _tuples: _experiment, _series_id, _group = _tuple _latest_time_frame = compute.latest_time_frame_before_overlapping( _experiment, _series_id, _group, OFFSET_X) for _cell_id in ['left_cell', 'right_cell']: _arguments.append({ 'experiment': _experiment, 'series_id': _series_id, 'group': _group, 'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER, 'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER, 'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER, 'offset_x': OFFSET_X, 'offset_y': OFFSET_Y, 'offset_z': OFFSET_Z, 'cell_id': _cell_id, 'direction': 'inside', 'time_points': _latest_time_frame }) _windows_dictionary, _windows_to_compute = compute.windows( _arguments, _keys=['experiment', 'series_id', 'group', 'cell_id']) _fiber_densities = compute.fiber_densities(_windows_to_compute, _subtract_border=True) _experiments_fiber_densities = { _key: [_fiber_densities[_tuple] for _tuple in _windows_dictionary[_key]] for _key in _windows_dictionary } _tuples_by_experiment = organize.by_experiment(_tuples) _higher_same_counter = 0 _valid_tuples = [] for _experiment in _tuples_by_experiment: print('Experiment:', _experiment) _experiment_tuples = _tuples_by_experiment[_experiment] for _same_index in tqdm(range(len(_experiment_tuples)), desc='Main loop'): _same_tuple = _experiment_tuples[_same_index] _same_experiment, _same_series, _same_group = _same_tuple _same_left_cell_fiber_densities = \ _experiments_fiber_densities[ (_same_experiment, _same_series, _same_group, 'left_cell') ] _same_right_cell_fiber_densities = \ _experiments_fiber_densities[ (_same_experiment, _same_series, _same_group, 'right_cell') ] _same_properties = \ load.group_properties(_same_experiment, _same_series, _same_group) _same_left_cell_fiber_densities = compute.remove_blacklist( _same_experiment, _same_series, _same_properties['cells_ids']['left_cell'], _same_left_cell_fiber_densities) _same_right_cell_fiber_densities = compute.remove_blacklist( _same_experiment, _same_series, _same_properties['cells_ids']['right_cell'], _same_right_cell_fiber_densities) _same_left_cell_fiber_densities_filtered, _same_right_cell_fiber_densities_filtered = \ compute.longest_same_indices_shared_in_borders_sub_array( _same_left_cell_fiber_densities, _same_right_cell_fiber_densities ) # ignore small arrays if len(_same_left_cell_fiber_densities_filtered ) < compute.minimum_time_frames_for_correlation( _same_experiment): continue _same_correlation = compute_lib.correlation( compute_lib.derivative( _same_left_cell_fiber_densities_filtered, _n=DERIVATIVE), compute_lib.derivative( _same_right_cell_fiber_densities_filtered, _n=DERIVATIVE)) for _different_index in range(len(_experiment_tuples)): if _same_index != _different_index: _different_tuple = _experiment_tuples[_different_index] _different_experiment, _different_series, _different_group = \ _different_tuple for _same_cell_id, _different_cell_id in product( ['left_cell', 'right_cell'], ['left_cell', 'right_cell']): _same_fiber_densities = _experiments_fiber_densities[ (_same_experiment, _same_series, _same_group, _same_cell_id)] _different_fiber_densities = _experiments_fiber_densities[ (_different_experiment, _different_series, _different_group, _different_cell_id)] _different_properties = load.group_properties( _different_experiment, _different_series, _different_group) _same_fiber_densities = compute.remove_blacklist( _same_experiment, _same_series, _same_properties['cells_ids'][_same_cell_id], _same_fiber_densities) _different_fiber_densities = compute.remove_blacklist( _different_experiment, _different_series, _different_properties['cells_ids'] [_different_cell_id], _different_fiber_densities) _same_fiber_densities_filtered, _different_fiber_densities_filtered = \ compute.longest_same_indices_shared_in_borders_sub_array( _same_fiber_densities, _different_fiber_densities ) # ignore small arrays if len( _same_fiber_densities_filtered ) < compute.minimum_time_frames_for_correlation( _different_experiment): continue _different_correlation = compute_lib.correlation( compute_lib.derivative( _same_fiber_densities_filtered, _n=DERIVATIVE), compute_lib.derivative( _different_fiber_densities_filtered, _n=DERIVATIVE)) _point_distance = compute_lib.distance_from_a_point_to_a_line( _line=[-1, -1, 1, 1], _point=[ _same_correlation, _different_correlation ]) if _same_correlation > _different_correlation: _y_arrays[_distances_index].append( _point_distance) _higher_same_counter += 1 else: _y_arrays[_distances_index].append( -_point_distance) if _same_tuple not in _valid_tuples: _valid_tuples.append(_same_tuple) print('Total tuples:', len(_valid_tuples)) print('Total points:', len(_y_arrays[_distances_index])) print('Wilcoxon around the zero:') print(wilcoxon(_y_arrays[_distances_index])) print('Higher same amount:', _higher_same_counter / len(_y_arrays[_distances_index])) _names_array.append( str(_distances_range[0]) + '-' + str(_distances_range[1])) # plot _colors_array = config.colors(3) _fig = go.Figure(data=[ go.Box(y=_y, name=_name, boxpoints=False, line={'width': 1}, marker={ 'size': 10, 'color': _color }, showlegend=False) for _y, _name, _color in zip(_y_arrays, _names_array, _colors_array) ], layout={ 'xaxis': { 'title': 'Pair distance (cell diameter)', 'zeroline': False, 'type': 'category' }, 'yaxis': { 'title': 'Same minus different correlation', 'range': [-1, 1.1], 'zeroline': False, 'tickmode': 'array', 'tickvals': [-1, -0.5, 0, 0.5, 1] } }) save.to_html(_fig=_fig, _path=os.path.join(paths.PLOTS, save.get_module_name()), _filename='plot_band_' + str(_band) + '_high_temporal_res_' + str(_high_temporal_resolution))
def main(_band=None, _high_temporal_resolution=True, _tuples_to_mark=None, _tuples_to_plot=None, _plots=None): if _plots is None: _plots = ['whiteness', 'granger'] _experiments = all_experiments() _experiments = filtering.by_categories( _experiments=_experiments, _is_single_cell=False, _is_high_temporal_resolution=_high_temporal_resolution, _is_bleb=False, _is_dead_dead=False, _is_live_dead=False, _is_bead=False, _is_metastasis=False) _tuples = load.experiments_groups_as_tuples(_experiments) _tuples = filtering.by_time_frames_amount(_tuples, _time_frames=MINIMUM_TIME_FRAMES) _tuples = filtering.by_pair_distance_range( _tuples, _distance_range=PAIR_DISTANCE_RANGE) _tuples = filtering.by_real_pairs(_tuples) _tuples = filtering.by_band(_tuples, _band=_band) print('Total tuples:', len(_tuples)) _arguments = [] for _tuple in _tuples: _experiment, _series_id, _group = _tuple _latest_time_frame = compute.latest_time_frame_before_overlapping( _experiment, _series_id, _group, OFFSET_X) for _cell_id in ['left_cell', 'right_cell']: _arguments.append({ 'experiment': _experiment, 'series_id': _series_id, 'group': _group, 'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER, 'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER, 'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER, 'offset_x': OFFSET_X, 'offset_y': OFFSET_Y, 'offset_z': OFFSET_Z, 'cell_id': _cell_id, 'direction': 'inside', 'time_points': _latest_time_frame }) _windows_dictionary, _windows_to_compute = compute.windows( _arguments, _keys=['experiment', 'series_id', 'group', 'cell_id']) _fiber_densities = compute.fiber_densities(_windows_to_compute, _subtract_border=True) _experiments_fiber_densities = { _key: [_fiber_densities[_tuple] for _tuple in _windows_dictionary[_key]] for _key in _windows_dictionary } _n_pairs = 0 _n_pairs_with_band = 0 _whiteness_p_values = [] _n_passed_whiteness_with_band = 0 _granger_causality_p_values = [] _n_passed_granger_causality_with_band = 0 _correlations = [] _time_lag_correlations = [] _end_fiber_densities = [] for _tuple in _tuples: _experiment, _series_id, _group = _tuple _left_cell_fiber_densities = \ _experiments_fiber_densities[(_experiment, _series_id, _group, 'left_cell')] _right_cell_fiber_densities = \ _experiments_fiber_densities[(_experiment, _series_id, _group, 'right_cell')] _properties = load.group_properties(_experiment, _series_id, _group) _left_cell_fiber_densities = compute.remove_blacklist( _experiment, _series_id, _properties['cells_ids']['left_cell'], _left_cell_fiber_densities) _right_cell_fiber_densities = compute.remove_blacklist( _experiment, _series_id, _properties['cells_ids']['right_cell'], _right_cell_fiber_densities) _left_cell_fiber_densities_filtered, _right_cell_fiber_densities_filtered = \ compute.longest_same_indices_shared_in_borders_sub_array( _left_cell_fiber_densities, _right_cell_fiber_densities) # ignore small arrays if len(_left_cell_fiber_densities_filtered) < MINIMUM_TIME_FRAMES: continue _n_pairs += 1 if _properties['band']: _n_pairs_with_band += 1 _start_time_frame = 0 for _left in _left_cell_fiber_densities: if _left[0] == _left_cell_fiber_densities_filtered[0]: break _start_time_frame += 1 # stationary test with warnings.catch_warnings(): warnings.simplefilter('ignore', category=InterpolationWarning) # find derivative for stationary for _derivative in range(10): _left_cell_fiber_densities_derivative = \ compute_lib.derivative(_left_cell_fiber_densities_filtered, _n=_derivative) _right_cell_fiber_densities_derivative = \ compute_lib.derivative(_right_cell_fiber_densities_filtered, _n=_derivative) if ADF_TEST: _, _left_cell_adf_p_value, _, _, _, _ = adfuller( _left_cell_fiber_densities_derivative) _, _right_cell_adf_p_value, _, _, _, _ = adfuller( _right_cell_fiber_densities_derivative) if _left_cell_adf_p_value > 0.05 or _right_cell_adf_p_value > 0.05: continue if KPSS_TEST: _, _left_cell_kpss_p_value, _, _ = kpss( _left_cell_fiber_densities_derivative, nlags='legacy') _, _right_cell_kpss_p_value, _, _ = kpss( _right_cell_fiber_densities_derivative, nlags='legacy') if _left_cell_kpss_p_value < 0.05 or _right_cell_kpss_p_value < 0.05: continue # stationary break # causality try: _x = pd.DataFrame(data=[[_left_value, _right_value] for _left_value, _right_value in zip( _left_cell_fiber_densities_derivative, _right_cell_fiber_densities_derivative) ], columns=['left', 'right']) # var model to retrieve lag _var_model = VAR(_x) _lag_order_results = _var_model.select_order() _estimators_lags = [ _lag_order_results.aic, _lag_order_results.bic, _lag_order_results.fpe, _lag_order_results.hqic ] _min_estimator_lag = min(_estimators_lags) # found a lag if 0 < _min_estimator_lag <= MAXIMUM_LAG: _var_model_results = _var_model.fit(maxlags=_min_estimator_lag, ic=None) _whiteness = _var_model_results.test_whiteness( nlags=_min_estimator_lag + 1) _whiteness_p_values.append(_whiteness.pvalue) if _tuples_to_mark is not None and _tuple in _tuples_to_mark and _whiteness.pvalue > 0.05: print(_tuple, 'marked whiteness p-value:', _whiteness.pvalue) # no autocorrelation in the residuals if _whiteness.pvalue > 0.05: if _properties['band']: _n_passed_whiteness_with_band += 1 # time lag = 0 _correlation = compute_lib.correlation( _left_cell_fiber_densities_derivative, _right_cell_fiber_densities_derivative) # if _correlation < 0.5: # continue # granger causality for _caused, _causing in zip(['left', 'right'], ['right', 'left']): _granger = _var_model_results.test_causality( caused=_caused, causing=_causing) _granger_causality_p_values.append(_granger.pvalue) # time lag = 0 _correlations.append(_correlation) # time lag = min estimator if _causing == 'left': _left_fiber_densities_time_lag = \ _left_cell_fiber_densities_derivative[:-_min_estimator_lag] _right_fiber_densities_time_lag = \ _right_cell_fiber_densities_derivative[_min_estimator_lag:] else: _left_fiber_densities_time_lag = \ _left_cell_fiber_densities_derivative[_min_estimator_lag:] _right_fiber_densities_time_lag = \ _right_cell_fiber_densities_derivative[:-_min_estimator_lag] _time_lag_correlation = compute_lib.correlation( _left_fiber_densities_time_lag, _right_fiber_densities_time_lag) _time_lag_correlations.append(_time_lag_correlation) # end fiber density _time_frame = compute.density_time_frame(_experiment) if len(_left_cell_fiber_densities_filtered ) > _time_frame: _end_fiber_density = \ (_left_cell_fiber_densities_filtered[_time_frame] + _right_cell_fiber_densities_filtered[_time_frame]) / 2 else: _end_fiber_density = \ (_left_cell_fiber_densities_filtered[-1] + _right_cell_fiber_densities_filtered[-1]) / 2 _normalization = load.normalization_series_file_data( _experiment, _series_id) _normalized_fiber_density = compute_lib.z_score( _end_fiber_density, _normalization['average'], _normalization['std']) _end_fiber_densities.append(_normalized_fiber_density) # marking if _tuples_to_mark is not None and _tuple in _tuples_to_mark and _granger.pvalue < 0.05: print(_tuple, 'causing:', _causing, 'marked granger p-value:', _granger.pvalue) if _granger.pvalue < 0.05: if _properties['band']: _n_passed_granger_causality_with_band += 1 _normality = _var_model_results.test_normality() _inst_granger = _var_model_results.test_inst_causality( causing=_causing) print( _tuple, _causing.capitalize() + ' causes ' + _caused + '!', 'time-points: ' + str(len( _left_cell_fiber_densities_derivative)), 'stationary derivative: ' + str(_derivative), 'band:' + str(_properties['band']), 'p-value: ' + str(round(_granger.pvalue, 4)), 'lag: ' + str(_min_estimator_lag), 'normality p-value: ' + str(round(_normality.pvalue, 4)), 'inst p-value: ' + str(round(_inst_granger.pvalue, 4)), sep='\t') # lag = 0 print('Time lag = 0 correlation:', _correlation) # rest of lags for _lag in range(1, _min_estimator_lag + 1): if _causing == 'left': _left_fiber_densities_time_lag = _left_cell_fiber_densities_derivative[: -_lag] _right_fiber_densities_time_lag = _right_cell_fiber_densities_derivative[ _lag:] else: _left_fiber_densities_time_lag = _left_cell_fiber_densities_derivative[ _lag:] _right_fiber_densities_time_lag = _right_cell_fiber_densities_derivative[: -_lag] _correlation = compute_lib.correlation( _left_fiber_densities_time_lag, _right_fiber_densities_time_lag) print( 'Time lag = ' + str(_lag) + ' correlation:', _correlation) # plots if _tuples_to_plot is not None and _tuple in _tuples_to_plot: _y_arrays = [ _left_cell_fiber_densities_derivative, _right_cell_fiber_densities_derivative ] _names_array = ['Left cell', 'Right cell'] _colors_array = config.colors(2) _temporal_resolution = compute.temporal_resolution_in_minutes( _experiment) _fig = go.Figure(data=[ go.Scatter(x=np.arange( start=_start_time_frame, stop=_start_time_frame + len(_left_cell_fiber_densities_derivative ), step=1) * _temporal_resolution, y=_y, name=_name, mode='lines', line={ 'color': _color, 'width': 1 }) for _y, _name, _color in zip( _y_arrays, _names_array, _colors_array) ], layout={ 'xaxis': { 'title': 'Time (minutes)', 'zeroline': False }, 'yaxis': { 'title': 'Fiber density (z-score)' + '\'' * _derivative, 'zeroline': False }, 'legend': { 'xanchor': 'left', 'x': 0.1, 'yanchor': 'top', 'bordercolor': 'black', 'borderwidth': 2, 'bgcolor': 'white' }, }) _experiment, _series_id, _group = _tuple save.to_html( _fig=_fig, _path=os.path.join(paths.PLOTS, save.get_module_name()), _filename='plot_' + _experiment + '_' + str(_series_id) + '_' + _group) # residuals _y_arrays = \ [_var_model_results.resid.values[:, 0], _var_model_results.resid.values[:, 1]] _fig = go.Figure(data=[ go.Scatter(x=np.arange( start=_start_time_frame, stop=_start_time_frame + len(_y), step=1) * _temporal_resolution, y=_y, name=_name, mode='lines', line={ 'color': _color, 'width': 1 }) for _y, _name, _color in zip( _y_arrays, _names_array, _colors_array) ], layout={ 'xaxis': { 'title': 'Time (minutes)', 'zeroline': False }, 'yaxis': { 'title': 'Residual', 'zeroline': False }, 'legend': { 'xanchor': 'left', 'x': 0.1, 'yanchor': 'top', 'bordercolor': 'black', 'borderwidth': 2, 'bgcolor': 'white' }, }) _experiment, _series_id, _group = _tuple save.to_html( _fig=_fig, _path=os.path.join(paths.PLOTS, save.get_module_name()), _filename='plot_residuals_' + _experiment + '_' + str(_series_id) + '_' + _group) # not enough time points except ValueError: continue print('Total pairs:', _n_pairs) print('Total pairs with band:', _n_pairs_with_band) print('Total pairs passed whiteness:', (np.array(_whiteness_p_values) > 0.05).sum()) print('Total pairs passed whiteness with band:', _n_passed_whiteness_with_band) print('Total cells passed granger causality:', (np.array(_granger_causality_p_values) < 0.05).sum()) print('Total cells passed granger causality with band:', _n_passed_granger_causality_with_band) # p-value correction print('Corrections of GC p-value < 0.05:') _granger_causality_p_values_corrected = multipletests( pvals=_granger_causality_p_values, method='fdr_bh') for _p_value, _p_value_corrected in zip( _granger_causality_p_values, _granger_causality_p_values_corrected[1]): if _p_value < 0.05: print('Original GC p-value:', _p_value, 'corrected:', _p_value_corrected) # plots for _test_name, _y_title, _y_array in \ zip( ['whiteness', 'granger'], ['Whiteness p-value', 'Granger causality p-value'], [_whiteness_p_values, _granger_causality_p_values] ): if _test_name in _plots: _fig = go.Figure(data=go.Box(y=_y_array, boxpoints='all', jitter=1, pointpos=0, line={'width': 1}, fillcolor='white', marker={ 'size': 10, 'color': '#ea8500' }, opacity=0.7, showlegend=False), layout={ 'xaxis': { 'zeroline': False }, 'yaxis': { 'title': _y_title, 'zeroline': False, 'range': [-0.1, 1.1], 'tickmode': 'array', 'tickvals': [0.05, 1] }, 'shapes': [{ 'type': 'line', 'x0': -0.75, 'y0': 0.05, 'x1': 0.75, 'y1': 0.05, 'line': { 'color': 'red', 'width': 2, 'dash': 'dash' } }] }) save.to_html(_fig=_fig, _path=os.path.join(paths.PLOTS, save.get_module_name()), _filename='plot_' + _test_name) # granger versus correlation print( 'GC vs. correlation pearson correlation:', compute_lib.correlation(_granger_causality_p_values, _correlations, _with_p_value=True)) _fig = go.Figure(data=go.Scatter(x=_granger_causality_p_values, y=_correlations, mode='markers', marker={ 'size': 10, 'color': '#ea8500' }, showlegend=False), layout={ 'xaxis': { 'title': 'Granger causality p-value', 'zeroline': False, }, 'yaxis': { 'title': 'Inner correlation', 'zeroline': False, } }) save.to_html(_fig=_fig, _path=os.path.join(paths.PLOTS, save.get_module_name()), _filename='plot_gc_vs_correlation') # granger versus time lag correlation print( 'GC vs. time lag correlation pearson correlation:', compute_lib.correlation(_granger_causality_p_values, _time_lag_correlations, _with_p_value=True)) _fig = go.Figure(data=go.Scatter(x=_granger_causality_p_values, y=_time_lag_correlations, mode='markers', marker={ 'size': 10, 'color': '#ea8500' }, showlegend=False), layout={ 'xaxis': { 'title': 'Granger causality p-value', 'zeroline': False, }, 'yaxis': { 'title': 'GC lag inner correlation', 'zeroline': False, } }) save.to_html(_fig=_fig, _path=os.path.join(paths.PLOTS, save.get_module_name()), _filename='plot_gc_vs_time_lag_correlation') # granger versus end fiber density print( 'GC vs. end fiber density pearson correlation:', compute_lib.correlation(_granger_causality_p_values, _end_fiber_densities, _with_p_value=True)) _fig = go.Figure(data=go.Scatter(x=_granger_causality_p_values, y=_end_fiber_densities, mode='markers', marker={ 'size': 10, 'color': '#ea8500' }, showlegend=False), layout={ 'xaxis': { 'title': 'Granger causality p-value', 'zeroline': False, }, 'yaxis': { 'title': 'End fiber density (z-score)', 'zeroline': False, } }) save.to_html(_fig=_fig, _path=os.path.join(paths.PLOTS, save.get_module_name()), _filename='plot_gc_vs_end_density')
def main(_real_cells=True, _static=False, _dead_dead=False, _live_dead=False, _dead=False, _live=False, _bead=False, _metastasis=False, _bleb=False, _bleb_amount_um=None, _band=True, _offset_y=0.5, _high_temporal_resolution=False): _experiments = all_experiments() _experiments = filtering.by_categories( _experiments=_experiments, _is_single_cell=False, _is_high_temporal_resolution=_high_temporal_resolution, _is_bleb=_bleb, _is_dead_dead=_dead_dead, _is_live_dead=_live_dead, _is_bead=_bead, _is_metastasis=_metastasis) _tuples = load.experiments_groups_as_tuples(_experiments) _tuples = filtering.by_pair_distance_range(_tuples, PAIR_DISTANCE_RANGE) _tuples = filtering.by_real_pairs(_tuples, _real_pairs=_real_cells) _tuples = filtering.by_fake_static_pairs(_tuples, _fake_static_pairs=_static) if _dead_dead is not False or _live_dead is not False: _tuples = filtering.by_dead_live(_tuples, _dead=_dead, _live=_live) _tuples = filtering.by_band(_tuples, _band=_band) if _bleb: _tuples = filtering.by_bleb_amount_um(_tuples, _amount_um=_bleb_amount_um) _tuples_matched = organize.by_matched_real_real_and_real_fake(_tuples) print('Total matched pairs:', len(_tuples_matched)) _arguments = [] for _matched_tuple in _tuples_matched: for _tuple in _matched_tuple: _experiment, _series_id, _group = _tuple _latest_time_frame = compute.latest_time_frame_before_overlapping( _experiment, _series_id, _group, OFFSET_X) for _cell_id in ['left_cell', 'right_cell']: _arguments.append({ 'experiment': _experiment, 'series_id': _series_id, 'group': _group, 'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER, 'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER, 'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER, 'offset_x': OFFSET_X, 'offset_y': _offset_y, 'offset_z': OFFSET_Z, 'cell_id': _cell_id, 'direction': 'inside', 'time_points': _latest_time_frame }) _windows_dictionary, _windows_to_compute = compute.windows( _arguments, _keys=['experiment', 'series_id', 'group', 'cell_id']) _fiber_densities = compute.fiber_densities(_windows_to_compute, _subtract_border=True) _experiments_fiber_densities = { _key: [_fiber_densities[_tuple] for _tuple in _windows_dictionary[_key]] for _key in _windows_dictionary } _real_real_correlations_array = [] _real_fake_correlations_array = [] _valid_real_real_tuples = [] for _tuple_matched in tqdm(_tuples_matched, desc='Main loop'): _real_real_tuple, _real_fake_tuple = _tuple_matched _real_real_experiment, _real_real_series, _real_real_group = _real_real_tuple _, _, _real_fake_group = _real_fake_tuple _real_real_left_cell_fiber_densities = \ _experiments_fiber_densities[ (_real_real_experiment, _real_real_series, _real_real_group, 'left_cell') ] _real_real_right_cell_fiber_densities = \ _experiments_fiber_densities[ (_real_real_experiment, _real_real_series, _real_real_group, 'right_cell') ] _real_fake_left_cell_fiber_densities = \ _experiments_fiber_densities[ (_real_real_experiment, _real_real_series, _real_fake_group, 'left_cell') ] _real_fake_right_cell_fiber_densities = \ _experiments_fiber_densities[ (_real_real_experiment, _real_real_series, _real_fake_group, 'right_cell') ] _properties = \ load.group_properties(_real_real_experiment, _real_real_series, _real_real_group) _real_real_left_cell_fiber_densities = compute.remove_blacklist( _real_real_experiment, _real_real_series, _properties['cells_ids']['left_cell'], _real_real_left_cell_fiber_densities) _real_real_right_cell_fiber_densities = compute.remove_blacklist( _real_real_experiment, _real_real_series, _properties['cells_ids']['right_cell'], _real_real_right_cell_fiber_densities) _real_fake_left_cell_fiber_densities = compute.remove_blacklist( _real_real_experiment, _real_real_series, _properties['cells_ids']['left_cell'], _real_fake_left_cell_fiber_densities) _real_fake_right_cell_fiber_densities = compute.remove_blacklist( _real_real_experiment, _real_real_series, _properties['cells_ids']['right_cell'], _real_fake_right_cell_fiber_densities) _real_real_left_cell_fiber_densities_filtered, _real_real_right_cell_fiber_densities_filtered = \ compute.longest_same_indices_shared_in_borders_sub_array( _real_real_left_cell_fiber_densities, _real_real_right_cell_fiber_densities ) # ignore small arrays _minimum_time_frame_for_correlation = compute.minimum_time_frames_for_correlation( _real_real_experiment) if len(_real_real_left_cell_fiber_densities_filtered ) < _minimum_time_frame_for_correlation: continue _real_real_correlation = compute_lib.correlation( compute_lib.derivative( _real_real_left_cell_fiber_densities_filtered, _n=DERIVATIVE), compute_lib.derivative( _real_real_right_cell_fiber_densities_filtered, _n=DERIVATIVE)) _real_fake_left_cell_fiber_densities_filtered, _real_fake_right_cell_fiber_densities_filtered = \ compute.longest_same_indices_shared_in_borders_sub_array( _real_fake_left_cell_fiber_densities, _real_fake_right_cell_fiber_densities ) # ignore small arrays if len(_real_fake_left_cell_fiber_densities_filtered ) < _minimum_time_frame_for_correlation: continue _real_fake_correlation = compute_lib.correlation( compute_lib.derivative( _real_fake_left_cell_fiber_densities_filtered, _n=DERIVATIVE), compute_lib.derivative( _real_fake_right_cell_fiber_densities_filtered, _n=DERIVATIVE)) _real_real_correlations_array.append(_real_real_correlation) _real_fake_correlations_array.append(_real_fake_correlation) if _real_real_tuple not in _valid_real_real_tuples: _valid_real_real_tuples.append(_real_real_tuple) print('Total real-real pairs:', len(_valid_real_real_tuples)) _real_minus_fake = np.array(_real_real_correlations_array) - np.array( _real_fake_correlations_array) print('Wilcoxon of real-real minus real-fake around the zero:') print(wilcoxon(_real_minus_fake)) print('Higher real-real amount:', (_real_minus_fake > 0).sum() / len(_real_minus_fake)) # plot _fig = go.Figure(data=go.Scatter(x=_real_real_correlations_array, y=_real_fake_correlations_array, mode='markers', marker={ 'size': 5, 'color': '#ea8500' }, showlegend=False), layout={ 'xaxis': { 'title': 'Real-real correlation', 'zeroline': False, 'range': [-1.1, 1.2], 'tickmode': 'array', 'tickvals': [-1, -0.5, 0, 0.5, 1] }, 'yaxis': { 'title': 'Real-fake correlation', 'zeroline': False, 'range': [-1.1, 1.2], 'tickmode': 'array', 'tickvals': [-1, -0.5, 0, 0.5, 1] }, 'shapes': [{ 'type': 'line', 'x0': -1, 'y0': -1, 'x1': -1, 'y1': 1, 'line': { 'color': 'black', 'width': 2 } }, { 'type': 'line', 'x0': -1, 'y0': -1, 'x1': 1, 'y1': -1, 'line': { 'color': 'black', 'width': 2 } }, { 'type': 'line', 'x0': -1, 'y0': -1, 'x1': 1, 'y1': 1, 'line': { 'color': 'red', 'width': 2 } }] }) save.to_html(_fig=_fig, _path=os.path.join(paths.PLOTS, save.get_module_name()), _filename='plot_real_' + str(_real_cells) + '_static_' + str(_static) + '_dead_dead_' + str(_dead_dead) + '_live_dead_' + str(_live_dead) + '_dead_' + str(_dead) + '_live_' + str(_live) + '_bead_' + str(_bead) + '_metastasis_' + str(_metastasis) + '_bleb_' + str(_bleb) + str(_bleb_amount_um) + '_band_' + str(_band) + '_high_time_' + str(_high_temporal_resolution) + '_y_' + str(_offset_y))
def compute_fiber_densities(_real_cells=True, _static=False, _dead_dead=False, _live_dead=False, _dead=False, _live=False, _bead=False, _metastasis=False, _bleb=False, _bleb_amount_um=None, _band=True, _high_temporal_resolution=False, _pair_distance_range=None, _offset_y=0.5, _offset_z=0, _padding_y_by=0.5, _padding_z_by=0.0, _space_y_by=0.25, _space_z_by=0.0): if _pair_distance_range is None: _pair_distance_range = PAIR_DISTANCE_RANGE _experiments = all_experiments() _experiments = filtering.by_categories( _experiments=_experiments, _is_single_cell=False, _is_high_temporal_resolution=_high_temporal_resolution, _is_bleb=_bleb, _is_dead_dead=_dead_dead, _is_live_dead=_live_dead, _is_bead=_bead, _is_metastasis=_metastasis ) _tuples = load.experiments_groups_as_tuples(_experiments) _tuples = filtering.by_pair_distance_range(_tuples, _pair_distance_range) _tuples = filtering.by_real_pairs(_tuples, _real_pairs=_real_cells) _tuples = filtering.by_fake_static_pairs(_tuples, _fake_static_pairs=_static) if _dead_dead is not False or _live_dead is not False: _tuples = filtering.by_dead_live(_tuples, _dead=_dead, _live=_live) _tuples = filtering.by_band(_tuples, _band=_band) if _bleb: _tuples = filtering.by_bleb_amount_um(_tuples, _amount_um=_bleb_amount_um) print('Total tuples:', len(_tuples)) _arguments = [] for _tuple in _tuples: _experiment, _series_id, _group = _tuple _latest_time_frame = compute.latest_time_frame_before_overlapping(_experiment, _series_id, _group, OFFSET_X) for _cell_id in ['left_cell', 'right_cell']: _arguments.append({ 'experiment': _experiment, 'series_id': _series_id, 'group': _group, 'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER, 'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER, 'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER, 'offset_x': OFFSET_X, 'offset_y': _offset_y, 'offset_z': _offset_z, 'cell_id': _cell_id, 'direction': 'inside', 'time_points': _latest_time_frame }) _windows_dictionary, _windows_to_compute = compute.windows(_arguments, _keys=['experiment', 'series_id', 'group', 'cell_id']) _fiber_densities = compute.fiber_densities(_windows_to_compute, _subtract_border=True, _padding_y_by=_padding_y_by, _padding_z_by=_padding_z_by, _space_y_by=_space_y_by, _space_z_by=_space_z_by) _experiments_fiber_densities = { _key: [_fiber_densities[_tuple] for _tuple in _windows_dictionary[_key]] for _key in _windows_dictionary } _tuples_by_experiment = organize.by_experiment(_tuples) _same_correlations_array = [] _different_correlations_array = [] _valid_tuples = [] for _experiment in _tuples_by_experiment: print('Experiment:', _experiment) _experiment_tuples = _tuples_by_experiment[_experiment] for _same_index in tqdm(range(len(_experiment_tuples)), desc='Main loop'): _same_tuple = _experiment_tuples[_same_index] _same_experiment, _same_series, _same_group = _same_tuple _same_left_cell_fiber_densities = \ _experiments_fiber_densities[ (_same_experiment, _same_series, _same_group, 'left_cell') ] _same_right_cell_fiber_densities = \ _experiments_fiber_densities[ (_same_experiment, _same_series, _same_group, 'right_cell') ] _same_properties = \ load.group_properties(_same_experiment, _same_series, _same_group) _same_left_cell_fiber_densities = compute.remove_blacklist( _same_experiment, _same_series, _same_properties['cells_ids']['left_cell'], _same_left_cell_fiber_densities ) _same_right_cell_fiber_densities = compute.remove_blacklist( _same_experiment, _same_series, _same_properties['cells_ids']['right_cell'], _same_right_cell_fiber_densities ) _same_left_cell_fiber_densities_filtered, _same_right_cell_fiber_densities_filtered = \ compute.longest_same_indices_shared_in_borders_sub_array( _same_left_cell_fiber_densities, _same_right_cell_fiber_densities ) # ignore small arrays if len(_same_left_cell_fiber_densities_filtered) < compute.minimum_time_frames_for_correlation(_same_experiment): continue _same_correlation = compute_lib.correlation( compute_lib.derivative(_same_left_cell_fiber_densities_filtered, _n=DERIVATIVE), compute_lib.derivative(_same_right_cell_fiber_densities_filtered, _n=DERIVATIVE) ) for _different_index in range(len(_experiment_tuples)): if _same_index != _different_index: _different_tuple = _experiment_tuples[_different_index] _different_experiment, _different_series, _different_group = \ _different_tuple for _same_cell_id, _different_cell_id in product(['left_cell', 'right_cell'], ['left_cell', 'right_cell']): _same_fiber_densities = _experiments_fiber_densities[( _same_experiment, _same_series, _same_group, _same_cell_id )] _different_fiber_densities = _experiments_fiber_densities[( _different_experiment, _different_series, _different_group, _different_cell_id )] _different_properties = load.group_properties( _different_experiment, _different_series, _different_group ) _same_fiber_densities = compute.remove_blacklist( _same_experiment, _same_series, _same_properties['cells_ids'][_same_cell_id], _same_fiber_densities ) _different_fiber_densities = compute.remove_blacklist( _different_experiment, _different_series, _different_properties['cells_ids'][_different_cell_id], _different_fiber_densities ) _same_fiber_densities_filtered, _different_fiber_densities_filtered = \ compute.longest_same_indices_shared_in_borders_sub_array( _same_fiber_densities, _different_fiber_densities ) # ignore small arrays if len(_same_fiber_densities_filtered) < compute.minimum_time_frames_for_correlation(_different_experiment): continue _different_correlation = compute_lib.correlation( compute_lib.derivative(_same_fiber_densities_filtered, _n=DERIVATIVE), compute_lib.derivative(_different_fiber_densities_filtered, _n=DERIVATIVE) ) _same_correlations_array.append(_same_correlation) _different_correlations_array.append(_different_correlation) if _same_tuple not in _valid_tuples: _valid_tuples.append(_same_tuple) print('Offset Y:', _offset_y, 'Offset Z:', _offset_z) print('Total tuples:', len(_valid_tuples)) print('Total points:', len(_same_correlations_array)) _same_minus_different = \ np.array(_same_correlations_array) - np.array(_different_correlations_array) print('Wilcoxon of same minus different around the zero:') print(wilcoxon(_same_minus_different)) print('Higher same amount:', (_same_minus_different > 0).sum() / len(_same_minus_different)) return _same_correlations_array, _different_correlations_array