def main(_real_cells=True, _static=False, _band=True, _high_temporal_resolution=False): _z_array = np.zeros(shape=(len(BY), len(BY))) for (_padding_index, _padding_by), (_space_index, _space_by) in product(enumerate(BY), enumerate(BY)): print('Padding by: ', _padding_by, ', space by: ', _space_by) _same_correlations_array, _different_correlations_array = \ same_inner_correlation_vs_different_inner_correlation.compute_fiber_densities( _real_cells=_real_cells, _static=_static, _band=_band, _high_temporal_resolution=_high_temporal_resolution, _padding_y_by=_padding_by, _padding_z_by=_padding_by, _space_y_by=_space_by, _space_z_by=_space_by ) _same_minus_different = np.array(_same_correlations_array) - np.array( _different_correlations_array) _same_greater_than_different = (_same_minus_different > 0).sum() / len(_same_minus_different) _z_array[_space_index, _padding_index] = _same_greater_than_different # plot _colors_array = ['black', 'white', config.colors(1)] _fig = go.Figure(data=go.Heatmap( x=BY, y=BY, z=_z_array, colorscale=sns.color_palette(_colors_array).as_hex(), colorbar={ 'tickmode': 'array', 'tickvals': [0.5, 0.75, 1], 'ticktext': ['0.5', 'Same > different', '1'], 'tickangle': -90 }, showscale=True, zmin=0.5, zmax=1), layout={ 'xaxis': { 'title': 'Border size (cell diameter)', 'zeroline': False, 'tickmode': 'array', 'tickvals': [0, 0.5, 1, 1.5, 2] }, 'yaxis': { 'title': 'Space from window (cell diameter)', 'zeroline': False, 'tickmode': 'array', 'tickvals': [0, 0.5, 1, 1.5, 2] } }) save.to_html(_fig=_fig, _path=os.path.join(paths.PLOTS, save.get_module_name()), _filename='plot_real_' + str(_real_cells) + '_static_' + str(_static) + '_band_' + str(_band) + '_high_time_' + str(_high_temporal_resolution))
def main(_band=True, _high_temporal_resolution=False): print('Computing fiber densities vs. offsets in axes:') _fiber_densities_z_array = inner_density_vs_offsets_in_axes.compute_z_array( _band=_band, _high_temporal_resolution=_high_temporal_resolution) print('Computing fiber density changes vs. offsets in axes:') _fiber_density_changes_z_array = inner_density_change_vs_offsets_in_axes.compute_z_array( _band=_band, _high_temporal_resolution=_high_temporal_resolution) _z_array = _fiber_density_changes_z_array / _fiber_densities_z_array # plot _offsets_y = np.arange(start=OFFSET_Y_START, stop=OFFSET_Y_END + OFFSET_Y_STEP, step=OFFSET_Y_STEP) _offsets_z = np.arange(start=OFFSET_Z_START, stop=OFFSET_Z_END + OFFSET_Z_STEP, step=OFFSET_Z_STEP) _colors_array = ['white', config.colors(1)] _fig = go.Figure(data=go.Heatmap( x=_offsets_z, y=_offsets_y, z=_z_array, colorscale=sns.color_palette(_colors_array).as_hex(), colorbar={ 'tickmode': 'array', 'tickvals': [0, 0.25, 0.5], 'ticktext': ['0.0', 'Z-score change / z-score', '0.5'], 'tickangle': -90 }, showscale=True, zmin=-0, zmax=0.5), layout={ 'xaxis': { 'title': 'Offset in XY axis (cell diameter)', 'zeroline': False, 'tickmode': 'array', 'tickvals': [-4, -2, 0, 2, 4] }, 'yaxis': { 'title': 'Offset in Z axis (cell diameter)', 'zeroline': False, 'tickmode': 'array', 'tickvals': [-1, 0, 1, 2] } }) save.to_html(_fig=_fig, _path=os.path.join(paths.PLOTS, save.get_module_name()), _filename='plot_high_time_' + str(_high_temporal_resolution) + '_band_' + str(_band))
def main(_high_temporal_resolution=False): _y_arrays = [[], []] for _band_index, _band in enumerate([True, False]): print('Band:', _band) _same_correlations_array, _different_correlations_array = \ same_inner_correlation_vs_different_inner_correlation.compute_fiber_densities(_band=_band, _high_temporal_resolution=_high_temporal_resolution) for _same, _different in zip(_same_correlations_array, _different_correlations_array): _point_distance = compute_lib.distance_from_a_point_to_a_line( _line=[-1, -1, 1, 1], _point=[_same, _different] ) if _same > _different: _y_arrays[_band_index].append(_point_distance) else: _y_arrays[_band_index].append(-_point_distance) # plot _colors_array = config.colors(2) _names_array = ['Band', 'No Band'] _fig = go.Figure( data=[ go.Box( y=_y_array, name=_name, boxpoints=False, line={ 'width': 1, 'color': _color }, showlegend=False ) for _y_array, _name, _color in zip(_y_arrays, _names_array, _colors_array) ], layout={ 'xaxis': { 'zeroline': False }, 'yaxis': { 'title': 'Same minus different correlation', 'zeroline': False } } ) save.to_html( _fig=_fig, _path=os.path.join(paths.PLOTS, save.get_module_name()), _filename='plot_high_temporal_res_' + str(_high_temporal_resolution) )
def main(_band=True, _high_temporal_resolution=False, _offset_x=OFFSET_X, _offset_y_start=OFFSET_Y_START, _offset_y_end=OFFSET_Y_END, _offset_y_step=OFFSET_Y_STEP, _offset_z_start=OFFSET_Z_START, _offset_z_end=OFFSET_Z_END, _offset_z_step=OFFSET_Z_STEP): global _tuples, _experiments_fiber_densities, _z_array compute_z_array(_band=_band, _high_temporal_resolution=_high_temporal_resolution, _offset_x=OFFSET_X, _offset_y_start=OFFSET_Y_START, _offset_y_end=OFFSET_Y_END, _offset_y_step=OFFSET_Y_STEP, _offset_z_start=OFFSET_Z_START, _offset_z_end=OFFSET_Z_END, _offset_z_step=OFFSET_Z_STEP) # plot _offsets_y = np.arange(start=_offset_y_start, stop=_offset_y_end + _offset_y_step, step=_offset_y_step) _offsets_z = np.arange(start=_offset_z_start, stop=_offset_z_end + _offset_z_step, step=_offset_z_step) _colors_array = ['white', config.colors(1)] _fig = go.Figure(data=go.Heatmap( x=_offsets_z, y=_offsets_y, z=_z_array, colorscale=sns.color_palette(_colors_array).as_hex(), colorbar={ 'tickmode': 'array', 'tickvals': [0, 0.35, 0.7], 'ticktext': ['0.0', 'Z-score change', '0.7'], 'tickangle': -90 }, showscale=True, zmin=0, zmax=0.7), layout={ 'xaxis': { 'title': 'Offset in XY axis (cell diameter)', 'zeroline': False, 'tickmode': 'array', 'tickvals': [-4, -2, 0, 2, 4] }, 'yaxis': { 'title': 'Offset in Z axis (cell diameter)', 'zeroline': False, 'tickmode': 'array', 'tickvals': [-1, 0, 1, 2] } }) save.to_html(_fig=_fig, _path=os.path.join(paths.PLOTS, save.get_module_name()), _filename='plot_high_time_' + str(_high_temporal_resolution) + '_band_' + str(_band))
def main(): _experiments = all_experiments() _experiments = filtering.by_categories(_experiments=_experiments, _is_single_cell=False, _is_high_temporal_resolution=False, _is_bleb=False, _is_dead_dead=False, _is_live_dead=False, _is_bead=False, _is_metastasis=False) _tuples = load.experiments_groups_as_tuples(_experiments) _tuples = filtering.by_time_frames_amount( _tuples, compute.density_time_frame(_experiments[0])) _tuples = filtering.by_real_pairs(_tuples) _tuples = filtering.by_band(_tuples) _tuples = filtering.by_pair_distance_range(_tuples, PAIR_DISTANCE_RANGE) print('Total tuples:', len(_tuples)) _arguments = [] for _tuple in _tuples: _experiment, _series_id, _group = _tuple _latest_time_frame = compute.latest_time_frame_before_overlapping( _experiment, _series_id, _group, OFFSET_X) for _cell_id in ['left_cell', 'right_cell']: _arguments.append({ 'experiment': _experiment, 'series_id': _series_id, 'group': _group, 'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER, 'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER, 'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER, 'offset_x': OFFSET_X, 'offset_y': OFFSET_Y, 'offset_z': OFFSET_Z, 'cell_id': _cell_id, 'direction': 'inside', 'time_points': _latest_time_frame }) _windows_dictionary, _windows_to_compute = \ compute.windows(_arguments, _keys=['experiment', 'series_id', 'group', 'cell_id']) _fiber_densities = compute.fiber_densities(_windows_to_compute, _subtract_border=True) _experiments_fiber_densities = { _key: [_fiber_densities[_tuple] for _tuple in _windows_dictionary[_key]] for _key in _windows_dictionary } _kpss_y_arrays = [[] for _i in DERIVATIVES] _adf_y_arrays = [[] for _i in DERIVATIVES] for _tuple in tqdm(_tuples, desc='Experiments loop'): _experiment, _series_id, _group = _tuple _properties = load.group_properties(_experiment, _series_id, _group) _left_cell_fiber_densities = _experiments_fiber_densities[( _experiment, _series_id, _group, 'left_cell')] _left_cell_fiber_densities = compute.remove_blacklist( _experiment, _series_id, _properties['cells_ids']['left_cell'], _left_cell_fiber_densities) _right_cell_fiber_densities = _experiments_fiber_densities[( _experiment, _series_id, _group, 'right_cell')] _right_cell_fiber_densities = compute.remove_blacklist( _experiment, _series_id, _properties['cells_ids']['right_cell'], _right_cell_fiber_densities) if not OUT_OF_BOUNDARIES: _left_cell_fiber_densities = \ compute.longest_fiber_densities_ascending_sequence(_left_cell_fiber_densities) _right_cell_fiber_densities = \ compute.longest_fiber_densities_ascending_sequence(_right_cell_fiber_densities) else: _left_cell_fiber_densities = [ _fiber_density[0] for _fiber_density in _left_cell_fiber_densities ] _right_cell_fiber_densities = [ _fiber_density[0] for _fiber_density in _right_cell_fiber_densities ] # ignore small arrays _minimum_time_frames_for_correlation = compute.minimum_time_frames_for_correlation( _experiment) if len(_left_cell_fiber_densities) < _minimum_time_frames_for_correlation or \ len(_right_cell_fiber_densities) < _minimum_time_frames_for_correlation: continue for _derivative_index, _derivative in enumerate(DERIVATIVES): for _cell_fiber_densities in [ _left_cell_fiber_densities, _right_cell_fiber_densities ]: _cell_fiber_densities_derivative = compute_lib.derivative( _cell_fiber_densities, _n=_derivative) _, _kpss_p_value, _, _ = kpss(_cell_fiber_densities_derivative, nlags='legacy') _kpss_y_arrays[_derivative_index].append(_kpss_p_value) _, _adf_p_value, _, _, _, _ = adfuller( _cell_fiber_densities_derivative) _adf_y_arrays[_derivative_index].append(_adf_p_value) print('Total pairs:', len(_kpss_y_arrays[0]) / 2) # print results print('KPSS:') for _derivative_index, _derivative in enumerate(DERIVATIVES): _stationary_count = len([ _value for _value in _kpss_y_arrays[_derivative_index] if _value > 0.05 ]) print( 'Derivative:', _derivative, 'Stationary:', str(_stationary_count / len(_kpss_y_arrays[_derivative_index]) * 100) + '%') print('ADF:') for _derivative_index, _derivative in enumerate(DERIVATIVES): _stationary_count = len([ _value for _value in _adf_y_arrays[_derivative_index] if _value < 0.05 ]) print( 'Derivative:', _derivative, 'Stationary:', str(_stationary_count / len(_adf_y_arrays[_derivative_index]) * 100) + '%') # plot _colors_array = config.colors(3) for _test_name, _y_title, _y_tickvals, _p_value_line, _y_arrays in \ zip( ['kpss', 'adf'], ['KPSS test p-value', 'ADF test p-value'], [[0.05, 0.1], [0.05, 1]], [0.05, 0.05], [_kpss_y_arrays, _adf_y_arrays] ): _fig = go.Figure(data=[ go.Box(y=_y, name=_derivative, boxpoints='all', jitter=1, pointpos=0, line={'width': 1}, fillcolor='white', marker={ 'size': 10, 'color': _color }, opacity=0.7, showlegend=False) for _y, _derivative, _color in zip( _y_arrays, DERIVATIVES_TEXT, _colors_array) ], layout={ 'xaxis': { 'title': 'Fiber density derivative', 'zeroline': False }, 'yaxis': { 'title': _y_title, 'zeroline': False, 'tickmode': 'array', 'tickvals': _y_tickvals }, 'shapes': [{ 'type': 'line', 'x0': DERIVATIVES[0] - 0.75, 'y0': _p_value_line, 'x1': DERIVATIVES[-1] + 0.75, 'y1': _p_value_line, 'line': { 'color': 'red', 'width': 2, 'dash': 'dash' } }] }) save.to_html(_fig=_fig, _path=os.path.join(paths.PLOTS, save.get_module_name()), _filename='plot_' + _test_name)
def main(): print('Single cells') _single_cells_fiber_densities = inner_density_vs_window_distance_single_cells.compute_experiments_data() print('Cell pairs') _names_array, _x_array, _y_array = inner_density_vs_window_distance.compute_cell_pairs() # plot _colors_array = config.colors(len(_names_array) + 1) _fig = go.Figure( data=[ go.Scatter( x=_x, y=[np.mean(_array) for _array in _y], name=_name, error_y={ 'type': 'data', 'array': [np.std(_array) for _array in _y], 'thickness': 1, 'color': _color }, mode='markers', marker={ 'size': 15, 'color': _color }, opacity=0.7 ) for _x, _y, _name, _color in zip(_x_array, _y_array, _names_array, _colors_array[:-1]) ] + [ go.Scatter( x=inner_density_vs_window_distance_single_cells.OFFSETS_X, y=[np.mean(_array) for _array in _single_cells_fiber_densities], name='Single cells', error_y={ 'type': 'data', 'array': [np.std(_array) for _array in _single_cells_fiber_densities], 'thickness': 1, 'color': _colors_array[-1] }, mode='markers', marker={ 'size': 15, 'color': _colors_array[-1] }, opacity=0.7 ) ], layout={ 'xaxis': { 'title': 'Window distance (cell diameter)', 'zeroline': False }, 'yaxis': { 'title': 'Fiber density (z-score)', 'range': [-1.7, 13], 'zeroline': False, 'tickmode': 'array', 'tickvals': [0, 4, 8, 12] }, 'legend': { 'xanchor': 'right', 'yanchor': 'top', 'bordercolor': 'black', 'borderwidth': 2 }, 'shapes': [ { 'type': 'line', 'x0': -0.2, 'y0': -1.5, 'x1': 3.4, 'y1': -1.5, 'line': { 'color': 'black', 'width': 2 } }, { 'type': 'line', 'x0': -0.2, 'y0': -1.5, 'x1': -0.2, 'y1': 13, 'line': { 'color': 'black', 'width': 2 } } ] } ) save.to_html( _fig=_fig, _path=os.path.join(paths.PLOTS, save.get_module_name()), _filename='plot' )
def main(): _experiments = all_experiments() _experiments = filtering.by_categories( _experiments=_experiments, _is_single_cell=False, _is_high_temporal_resolution=False, _is_bleb=False, _is_dead_dead=False, _is_live_dead=False, _is_bead=False, _is_metastasis=False ) _tuples = load.experiments_groups_as_tuples(_experiments) _tuples = filtering.by_real_pairs(_tuples) _tuples = filtering.by_band(_tuples) _tuples = filtering.by_time_frames_amount(_tuples, MINIMUM_TIME_FRAMES) _tuples = filtering.by_pair_distance_range(_tuples, [MINIMUM_PAIR_DISTANCE, sys.maxsize]) _triplets = filtering.by_triplets(_tuples) print('Total triplets:', len(_triplets)) _arguments = [] for _triplet in _triplets: for _tuple in _triplet: _experiment, _series_id, _group = _tuple _latest_time_frame = compute.latest_time_frame_before_overlapping(_experiment, _series_id, _group, OFFSET_X) for _cell_id in ['left_cell', 'right_cell']: _arguments.append({ 'experiment': _experiment, 'series_id': _series_id, 'group': _group, 'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER, 'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER, 'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER, 'offset_x': OFFSET_X, 'offset_y': OFFSET_Y, 'offset_z': OFFSET_Z, 'cell_id': _cell_id, 'direction': 'inside', 'time_points': _latest_time_frame }) _windows_dictionary, _windows_to_compute = compute.windows(_arguments, _keys=['experiment', 'series_id', 'group', 'cell_id']) _fiber_densities = compute.fiber_densities(_windows_to_compute, _subtract_border=True) _experiments_fiber_densities = { _key: [_fiber_densities[_tuple] for _tuple in _windows_dictionary[_key]] for _key in _windows_dictionary } _same_correlations_arrays = [[] for _i in _triplets] _different_correlations_arrays = [[] for _i in _triplets] _names_array = [] for _triplet_index, _triplet in enumerate(_triplets): for _same_index in tqdm(range(len(_triplet)), desc='Main loop'): _same_tuple = _triplet[_same_index] _same_experiment, _same_series, _same_group = _same_tuple _same_left_cell_fiber_densities = \ _experiments_fiber_densities[ (_same_experiment, _same_series, _same_group, 'left_cell') ] _same_right_cell_fiber_densities = \ _experiments_fiber_densities[ (_same_experiment, _same_series, _same_group, 'right_cell') ] _same_properties = \ load.group_properties(_same_experiment, _same_series, _same_group) _same_left_cell_fiber_densities = compute.remove_blacklist( _same_experiment, _same_series, _same_properties['cells_ids']['left_cell'], _same_left_cell_fiber_densities ) _same_right_cell_fiber_densities = compute.remove_blacklist( _same_experiment, _same_series, _same_properties['cells_ids']['right_cell'], _same_right_cell_fiber_densities ) _same_left_cell_fiber_densities_filtered, _same_right_cell_fiber_densities_filtered = \ compute.longest_same_indices_shared_in_borders_sub_array( _same_left_cell_fiber_densities, _same_right_cell_fiber_densities ) _same_correlation = compute_lib.correlation( compute_lib.derivative(_same_left_cell_fiber_densities_filtered, _n=DERIVATIVE), compute_lib.derivative(_same_right_cell_fiber_densities_filtered, _n=DERIVATIVE) ) for _different_index in range(len(_triplet)): if _same_index != _different_index: _different_tuple = _triplet[_different_index] _different_experiment, _different_series, _different_group = \ _different_tuple for _same_cell_id, _different_cell_id in product(['left_cell', 'right_cell'], ['left_cell', 'right_cell']): _same_fiber_densities = _experiments_fiber_densities[( _same_experiment, _same_series, _same_group, _same_cell_id )] _different_fiber_densities = _experiments_fiber_densities[( _different_experiment, _different_series, _different_group, _different_cell_id )] _different_properties = load.group_properties( _different_experiment, _different_series, _different_group ) _same_fiber_densities = compute.remove_blacklist( _same_experiment, _same_series, _same_properties['cells_ids'][_same_cell_id], _same_fiber_densities ) _different_fiber_densities = compute.remove_blacklist( _different_experiment, _different_series, _different_properties['cells_ids'][_different_cell_id], _different_fiber_densities ) _same_fiber_densities_filtered, _different_fiber_densities_filtered = \ compute.longest_same_indices_shared_in_borders_sub_array( _same_fiber_densities, _different_fiber_densities ) _different_correlation = compute_lib.correlation( compute_lib.derivative(_same_fiber_densities_filtered, _n=DERIVATIVE), compute_lib.derivative(_different_fiber_densities_filtered, _n=DERIVATIVE) ) _same_correlations_arrays[_triplet_index].append(_same_correlation) _different_correlations_arrays[_triplet_index].append(_different_correlation) _names_array.append('Triplet #' + str(_triplet_index + 1)) print('Total points:', len(np.array(_same_correlations_arrays).flatten())) _same_minus_different = \ np.array(_same_correlations_arrays).flatten() - np.array(_different_correlations_arrays).flatten() print('Wilcoxon of same minus different around the zero:') print(wilcoxon(_same_minus_different)) print('Higher same amount:', (_same_minus_different > 0).sum() / len(_same_minus_different)) # plot _colors_array = ['green', 'blue', config.colors(1)] _fig = go.Figure( data=[ go.Scatter( x=_same_correlations_array, y=_different_correlations_array, name=_name, mode='markers', marker={ 'size': 15, 'color': _color }, opacity=0.7 ) for _same_correlations_array, _different_correlations_array, _name, _color in zip(_same_correlations_arrays, _different_correlations_arrays, _names_array, _colors_array) ], layout={ 'xaxis': { 'title': 'Same network correlation', 'zeroline': False, 'range': [-1.1, 1.2], 'tickmode': 'array', 'tickvals': [-1, -0.5, 0, 0.5, 1] }, 'yaxis': { 'title': 'Different network correlation', 'zeroline': False, 'range': [-1.1, 1.2], 'tickmode': 'array', 'tickvals': [-1, -0.5, 0, 0.5, 1] }, 'legend': { 'xanchor': 'left', 'x': 0.1, 'yanchor': 'top', 'bordercolor': 'black', 'borderwidth': 2, 'bgcolor': 'white' }, 'shapes': [ { 'type': 'line', 'x0': -1, 'y0': -1, 'x1': -1, 'y1': 1, 'line': { 'color': 'black', 'width': 2 } }, { 'type': 'line', 'x0': -1, 'y0': -1, 'x1': 1, 'y1': -1, 'line': { 'color': 'black', 'width': 2 } }, { 'type': 'line', 'x0': -1, 'y0': -1, 'x1': 1, 'y1': 1, 'line': { 'color': 'red', 'width': 2 } } ] } ) save.to_html( _fig=_fig, _path=os.path.join(paths.PLOTS, save.get_module_name()), _filename='plot' )
def main(_offset_y=0.5, _high_temporal_resolution=False): _distances_from_y_equal_x = compute_fiber_densities( _offset_y, _high_temporal_resolution) print('Total points:', len(_distances_from_y_equal_x[0])) print('Higher real same amount:', (np.array(_distances_from_y_equal_x[0]) > 0).sum() / len(_distances_from_y_equal_x[0])) print('Wilcoxon of real points:', wilcoxon(_distances_from_y_equal_x[0])) print('Higher fake same amount:', (np.array(_distances_from_y_equal_x[1]) > 0).sum() / len(_distances_from_y_equal_x[1])) print('Wilcoxon of fake points:', wilcoxon(_distances_from_y_equal_x[1])) _real_minus_fake = np.array(_distances_from_y_equal_x[0]) - np.array( _distances_from_y_equal_x[1]) print('Real > fake amount:', (_real_minus_fake > 0).sum() / len(_real_minus_fake)) print('Wilcoxon real & fake:', wilcoxon(_distances_from_y_equal_x[0], _distances_from_y_equal_x[1])) # box plot _colors_array = config.colors(2) _names_array = ['Real', 'Fake'] _fig = go.Figure(data=[ go.Box(y=_y_array, name=_name, boxpoints=False, line={'width': 1}, marker={'color': _color}, showlegend=False) for _y_array, _name, _color in zip( _distances_from_y_equal_x, _names_array, _colors_array) ], layout={ 'xaxis': { 'zeroline': False }, 'yaxis': { 'title': 'Same minus different correlation', 'zeroline': False, 'range': [-1, 1.1], 'tickmode': 'array', 'tickvals': [-1, -0.5, 0, 0.5, 1] } }) save.to_html(_fig=_fig, _path=os.path.join(paths.PLOTS, save.get_module_name()), _filename='plot_box_high_temporal_res_' + str(_high_temporal_resolution) + '_offset_y_' + str(_offset_y)) # scatter plot _fig = go.Figure(data=go.Scatter(x=_distances_from_y_equal_x[0], y=_distances_from_y_equal_x[1], mode='markers', marker={ 'size': 5, 'color': '#ea8500' }, showlegend=False), layout={ 'xaxis': { 'title': 'Real pair', 'zeroline': False, 'range': [-1.1, 1.2], 'tickmode': 'array', 'tickvals': [-1, -0.5, 0, 0.5, 1] }, 'yaxis': { 'title': 'Fake pair', 'zeroline': False, 'range': [-1.1, 1.2], 'tickmode': 'array', 'tickvals': [-1, -0.5, 0, 0.5, 1] }, 'shapes': [{ 'type': 'line', 'x0': -1, 'y0': -1, 'x1': -1, 'y1': 1, 'line': { 'color': 'black', 'width': 2 } }, { 'type': 'line', 'x0': -1, 'y0': -1, 'x1': 1, 'y1': -1, 'line': { 'color': 'black', 'width': 2 } }, { 'type': 'line', 'x0': -1, 'y0': -1, 'x1': 1, 'y1': 1, 'line': { 'color': 'red', 'width': 2 } }] }) save.to_html(_fig=_fig, _path=os.path.join(paths.PLOTS, save.get_module_name()), _filename='plot_high_temporal_res_' + str(_high_temporal_resolution) + '_offset_y_' + str(_offset_y))
def main(_band=True): _experiments = all_experiments() _experiments = filtering.by_categories(_experiments=_experiments, _is_single_cell=False, _is_high_temporal_resolution=None, _is_bleb=True, _is_dead_dead=False, _is_live_dead=False, _is_bead=False, _is_metastasis=False) _tuples = load.experiments_groups_as_tuples(_experiments) _tuples = filtering.by_pair_distance_range( _tuples, _distance_range=PAIR_DISTANCE_RANGE) _tuples = filtering.by_real_pairs(_tuples, _real_pairs=REAL_CELLS) _tuples = filtering.by_fake_static_pairs(_tuples, _fake_static_pairs=STATIC) _tuples = filtering.by_band(_tuples, _band=_band) _tuples = filtering.by_bleb_from_start(_tuples, _from_start=False) print('Total tuples:', len(_tuples)) _arguments = [] for _tuple in _tuples: _experiment, _series_id, _group = _tuple _latest_time_frame = compute.latest_time_frame_before_overlapping( _experiment, _series_id, _group, OFFSET_X) for _cell_id in ['left_cell', 'right_cell']: _arguments.append({ 'experiment': _experiment, 'series_id': _series_id, 'group': _group, 'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER, 'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER, 'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER, 'offset_x': OFFSET_X, 'offset_y': OFFSET_Y, 'offset_z': OFFSET_Z, 'cell_id': _cell_id, 'direction': 'inside', 'time_points': _latest_time_frame }) _windows_dictionary, _windows_to_compute = compute.windows( _arguments, _keys=['experiment', 'series_id', 'group', 'cell_id']) _fiber_densities = compute.fiber_densities(_windows_to_compute, _subtract_border=True) _experiments_fiber_densities = { _key: [_fiber_densities[_tuple] for _tuple in _windows_dictionary[_key]] for _key in _windows_dictionary } _tuples_by_experiment = organize.by_experiment(_tuples) # same (before, after), different (before, after) _correlations = [[[], []], [[], []]] _valid_real_tuples = [] for _experiment in _tuples_by_experiment: print('Experiment:', _experiment) _experiment_tuples = _tuples_by_experiment[_experiment] for _same_index in tqdm(range(len(_experiment_tuples)), desc='Main loop'): _same_tuple = _experiment_tuples[_same_index] _same_experiment, _same_series, _same_group = _same_tuple _same_left_cell_fiber_densities = \ _experiments_fiber_densities[ (_same_experiment, _same_series, _same_group, 'left_cell') ] _same_right_cell_fiber_densities = \ _experiments_fiber_densities[ (_same_experiment, _same_series, _same_group, 'right_cell') ] _same_properties = \ load.group_properties(_same_experiment, _same_series, _same_group) _same_left_cell_fiber_densities = compute.remove_blacklist( _same_experiment, _same_series, _same_properties['cells_ids']['left_cell'], _same_left_cell_fiber_densities) _same_right_cell_fiber_densities = compute.remove_blacklist( _same_experiment, _same_series, _same_properties['cells_ids']['right_cell'], _same_right_cell_fiber_densities) _same_before_left_cell_fiber_densities = \ _same_left_cell_fiber_densities[:AFTER_BLEB_INJECTION_FIRST_TIME_FRAME[_same_experiment]] _same_before_right_cell_fiber_densities = \ _same_right_cell_fiber_densities[:AFTER_BLEB_INJECTION_FIRST_TIME_FRAME[_same_experiment]] _same_after_left_cell_fiber_densities = \ _same_left_cell_fiber_densities[AFTER_BLEB_INJECTION_FIRST_TIME_FRAME[_same_experiment]:] _same_after_right_cell_fiber_densities = \ _same_right_cell_fiber_densities[AFTER_BLEB_INJECTION_FIRST_TIME_FRAME[_same_experiment]:] _same_before_left_cell_fiber_densities_filtered, _same_before_right_cell_fiber_densities_filtered = \ compute.longest_same_indices_shared_in_borders_sub_array( _same_before_left_cell_fiber_densities, _same_before_right_cell_fiber_densities ) _same_after_left_cell_fiber_densities_filtered, _same_after_right_cell_fiber_densities_filtered = \ compute.longest_same_indices_shared_in_borders_sub_array( _same_after_left_cell_fiber_densities, _same_after_right_cell_fiber_densities ) # ignore small arrays _minimum_time_frame_for_correlation = compute.minimum_time_frames_for_correlation( _same_experiment) if len(_same_before_left_cell_fiber_densities_filtered) < _minimum_time_frame_for_correlation or \ len(_same_after_left_cell_fiber_densities_filtered) < _minimum_time_frame_for_correlation: continue _same_before_correlation = compute_lib.correlation( compute_lib.derivative( _same_before_left_cell_fiber_densities_filtered, _n=DERIVATIVE), compute_lib.derivative( _same_before_right_cell_fiber_densities_filtered, _n=DERIVATIVE)) _same_after_correlation = compute_lib.correlation( compute_lib.derivative( _same_after_left_cell_fiber_densities_filtered, _n=DERIVATIVE), compute_lib.derivative( _same_after_right_cell_fiber_densities_filtered, _n=DERIVATIVE)) for _different_index in range(len(_experiment_tuples)): if _same_index != _different_index: _different_tuple = _experiment_tuples[_different_index] _different_experiment, _different_series, _different_group = _different_tuple for _same_cell_id, _different_cell_id in product( ['left_cell', 'right_cell'], ['left_cell', 'right_cell']): _same_fiber_densities = _experiments_fiber_densities[( _same_experiment, _same_series, _same_group, _same_cell_id)] _different_fiber_densities = _experiments_fiber_densities[ (_different_experiment, _different_series, _different_group, _different_cell_id)] _different_properties = load.group_properties( _different_experiment, _different_series, _different_group) _same_fiber_densities = compute.remove_blacklist( _same_experiment, _same_series, _same_properties['cells_ids'][_same_cell_id], _same_fiber_densities) _different_fiber_densities = compute.remove_blacklist( _different_experiment, _different_series, _different_properties['cells_ids'] [_different_cell_id], _different_fiber_densities) _same_before_fiber_densities = \ _same_fiber_densities[:AFTER_BLEB_INJECTION_FIRST_TIME_FRAME[_same_experiment]] _same_after_fiber_densities = \ _same_fiber_densities[AFTER_BLEB_INJECTION_FIRST_TIME_FRAME[_same_experiment]:] _different_before_fiber_densities = \ _different_fiber_densities[:AFTER_BLEB_INJECTION_FIRST_TIME_FRAME[_different_experiment]] _different_after_fiber_densities = \ _different_fiber_densities[AFTER_BLEB_INJECTION_FIRST_TIME_FRAME[_different_experiment]:] _same_before_fiber_densities_filtered, _different_before_fiber_densities_filtered = \ compute.longest_same_indices_shared_in_borders_sub_array( _same_before_fiber_densities, _different_before_fiber_densities ) _same_after_fiber_densities_filtered, _different_after_fiber_densities_filtered = \ compute.longest_same_indices_shared_in_borders_sub_array( _same_after_fiber_densities, _different_after_fiber_densities ) # ignore small arrays if len(_same_before_fiber_densities_filtered) < _minimum_time_frame_for_correlation or \ len(_same_after_fiber_densities_filtered) < _minimum_time_frame_for_correlation: continue _different_before_correlation = compute_lib.correlation( compute_lib.derivative( _same_before_fiber_densities_filtered, _n=DERIVATIVE), compute_lib.derivative( _different_before_fiber_densities_filtered, _n=DERIVATIVE)) _different_after_correlation = compute_lib.correlation( compute_lib.derivative( _same_after_fiber_densities_filtered, _n=DERIVATIVE), compute_lib.derivative( _different_after_fiber_densities_filtered, _n=DERIVATIVE)) _correlations[0][0].append(_same_before_correlation) _correlations[0][1].append(_same_after_correlation) _correlations[1][0].append( _different_before_correlation) _correlations[1][1].append( _different_after_correlation) if _same_tuple not in _valid_real_tuples: _valid_real_tuples.append(_same_tuple) print('Total tuples:', len(_valid_real_tuples)) _distances_from_y_equal_x = [[], []] _same_correlations, _different_correlations = _correlations _same_before_correlations, _same_after_correlations = _same_correlations _different_before_correlations, _different_after_correlations = _different_correlations for _same_before, _same_after, _different_before, _different_after in \ zip(_same_before_correlations, _same_after_correlations, _different_before_correlations, _different_after_correlations): for _group_type_index, _same, _different in \ zip([0, 1], [_same_before, _same_after], [_different_before, _different_after]): _point_distance = compute_lib.distance_from_a_point_to_a_line( _line=[-1, -1, 1, 1], _point=[_same, _different]) if _same > _different: _distances_from_y_equal_x[_group_type_index].append( _point_distance) else: _distances_from_y_equal_x[_group_type_index].append( -_point_distance) print('Total points:', len(_distances_from_y_equal_x[0])) print('Higher before same amount:', (np.array(_distances_from_y_equal_x[0]) > 0).sum() / len(_distances_from_y_equal_x[0])) print('Wilcoxon of before points:', wilcoxon(_distances_from_y_equal_x[0])) print('Higher after same amount:', (np.array(_distances_from_y_equal_x[1]) > 0).sum() / len(_distances_from_y_equal_x[1])) print('Wilcoxon of after points:', wilcoxon(_distances_from_y_equal_x[1])) _before_minus_after = np.array(_distances_from_y_equal_x[0]) - np.array( _distances_from_y_equal_x[1]) print('Before > after amount:', (_before_minus_after > 0).sum() / len(_before_minus_after)) print('Wilcoxon before & after:', wilcoxon(_distances_from_y_equal_x[0], _distances_from_y_equal_x[1])) # box plot _colors_array = config.colors(2) _names_array = ['Before', 'After'] _fig = go.Figure(data=[ go.Box(y=_y_array, name=_name, boxpoints=False, line={'width': 1}, marker={'color': _color}, showlegend=False) for _y_array, _name, _color in zip( _distances_from_y_equal_x, _names_array, _colors_array) ], layout={ 'xaxis': { 'zeroline': False }, 'yaxis': { 'title': 'Same minus different correlation', 'zeroline': False, 'range': [-1, 1.1], 'tickmode': 'array', 'tickvals': [-1, -0.5, 0, 0.5, 1] } }) save.to_html(_fig=_fig, _path=os.path.join(paths.PLOTS, save.get_module_name()), _filename='plot_box') # scatter plot _fig = go.Figure(data=go.Scatter(x=_distances_from_y_equal_x[0], y=_distances_from_y_equal_x[1], mode='markers', marker={ 'size': 5, 'color': '#ea8500' }, showlegend=False), layout={ 'xaxis': { 'title': 'Before bleb', 'zeroline': False, 'range': [-1.1, 1.2], 'tickmode': 'array', 'tickvals': [-1, -0.5, 0, 0.5, 1] }, 'yaxis': { 'title': 'After bleb', 'zeroline': False, 'range': [-1.1, 1.2], 'tickmode': 'array', 'tickvals': [-1, -0.5, 0, 0.5, 1] }, 'shapes': [{ 'type': 'line', 'x0': -1, 'y0': -1, 'x1': -1, 'y1': 1, 'line': { 'color': 'black', 'width': 2 } }, { 'type': 'line', 'x0': -1, 'y0': -1, 'x1': 1, 'y1': -1, 'line': { 'color': 'black', 'width': 2 } }, { 'type': 'line', 'x0': -1, 'y0': -1, 'x1': 1, 'y1': 1, 'line': { 'color': 'red', 'width': 2 } }] }) save.to_html(_fig=_fig, _path=os.path.join(paths.PLOTS, save.get_module_name()), _filename='plot')
def compute_tuples(_tuples): _arguments = [] for _tuple in _tuples: _experiment, _series_id, _group = _tuple # stop when windows are overlapping _properties = load.group_properties(_experiment, _series_id, _group) _latest_time_frame = len(_properties['time_points']) _middle_offsets_x = [] for _time_frame in range(len(_properties['time_points'])): _pair_distance = \ compute.pair_distance_in_cell_size_time_frame(_experiment, _series_id, _group, _time_frame) if _time_frame == 0: print(_tuple, 'Pair distance:', _pair_distance) _middle_offsets_x.append( (_time_frame, (_pair_distance - 1) / 2 - QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER / 2)) if _pair_distance - 1 - OFFSET_X * 2 < QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER * 2: _latest_time_frame = _time_frame - 1 break for _cell_id in ['left_cell', 'right_cell']: _arguments.append({ 'experiment': _experiment, 'series_id': _series_id, 'group': _group, 'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER, 'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER, 'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER, 'offset_x': OFFSET_X, 'offset_y': OFFSET_Y, 'offset_z': OFFSET_Z, 'cell_id': _cell_id, 'direction': 'inside', 'time_points': _latest_time_frame, 'middle_time_frame': -1 }) # middle one for (_time_frame, _offset_x) in _middle_offsets_x: _arguments.append({ 'experiment': _experiment, 'series_id': _series_id, 'group': _group, 'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER, 'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER, 'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER, 'offset_x': _offset_x, 'offset_y': OFFSET_Y, 'offset_z': OFFSET_Z, 'cell_id': 'left_cell', 'direction': 'inside', 'time_point': _time_frame, 'middle_time_frame': _time_frame }) _windows_dictionary, _windows_to_compute = \ compute.windows(_arguments, _keys=['experiment', 'series_id', 'group', 'cell_id', 'middle_time_frame']) _fiber_densities = compute.fiber_densities(_windows_to_compute) _experiments_fiber_densities = { _key: [_fiber_densities[_tuple] for _tuple in _windows_dictionary[_key]] for _key in _windows_dictionary } _tuples_data = [] for _tuple in tqdm(_tuples, desc='Main loop'): _experiment, _series_id, _group = _tuple _left_cell_fiber_densities = \ _experiments_fiber_densities[(_experiment, _series_id, _group, 'left_cell', -1)] _right_cell_fiber_densities = \ _experiments_fiber_densities[(_experiment, _series_id, _group, 'right_cell', -1)] _middle_fiber_densities = \ [_experiments_fiber_densities[(_experiment, _series_id, _group, 'left_cell', _time_frame)][0] for _time_frame in range(0, len(_left_cell_fiber_densities))] _properties = load.group_properties(_experiment, _series_id, _group) _left_cell_fiber_densities = compute.remove_blacklist( _experiment, _series_id, _properties['cells_ids']['left_cell'], _left_cell_fiber_densities) _right_cell_fiber_densities = compute.remove_blacklist( _experiment, _series_id, _properties['cells_ids']['right_cell'], _right_cell_fiber_densities) _left_cell_fiber_densities_filtered, _right_cell_fiber_densities_filtered = \ compute.longest_same_indices_shared_in_borders_sub_array( _left_cell_fiber_densities, _right_cell_fiber_densities) if len(_left_cell_fiber_densities_filtered) == 0: continue _start_time_frame = 0 for _left in _left_cell_fiber_densities: if _left[0] == _left_cell_fiber_densities_filtered[0]: break _start_time_frame += 1 _middle_fiber_densities_filtered = \ [_fiber_density[0] for _fiber_density in _middle_fiber_densities] _middle_fiber_densities_filtered = \ _middle_fiber_densities_filtered[_start_time_frame: _start_time_frame + len(_left_cell_fiber_densities_filtered)] _normalization = load.normalization_series_file_data( _experiment, _series_id) _left_cell_fiber_densities_normalized = compute_lib.z_score_array( _array=_left_cell_fiber_densities_filtered, _average=_normalization['average'], _std=_normalization['std']) _right_cell_fiber_densities_normalized = compute_lib.z_score_array( _array=_right_cell_fiber_densities_filtered, _average=_normalization['average'], _std=_normalization['std']) _middle_fiber_densities_normalized = compute_lib.z_score_array( _array=_middle_fiber_densities_filtered, _average=_normalization['average'], _std=_normalization['std']) _left_cell_fiber_densities_normalized = \ compute_lib.derivative(_left_cell_fiber_densities_normalized, _n=DERIVATIVE) _right_cell_fiber_densities_normalized = \ compute_lib.derivative(_right_cell_fiber_densities_normalized, _n=DERIVATIVE) _middle_fiber_densities_normalized = \ compute_lib.derivative(_middle_fiber_densities_normalized, _n=DERIVATIVE) _correlation = \ compute_lib.correlation( compute_lib.derivative(_left_cell_fiber_densities_normalized, _n=1), compute_lib.derivative(_right_cell_fiber_densities_normalized, _n=1) ) _tuples_data.append([ _tuple, _start_time_frame, (_left_cell_fiber_densities_normalized, _right_cell_fiber_densities_normalized, _middle_fiber_densities_normalized), _correlation ]) # plots _names_array = ['Left cell', 'Right cell', 'Middle'] _colors_array = config.colors(3) for _tuple_data in _tuples_data: _tuple, _start_time_frame, _y_arrays, _correlation = _tuple_data _fig = go.Figure(data=[ go.Scatter(x=np.arange( start=_start_time_frame, stop=_start_time_frame + len(_y_arrays[0]) - DERIVATIVE, step=1) * TEMPORAL_RESOLUTION, y=_y, name=_name, mode='lines', line={'color': _color}) for _y, _name, _color in zip( _y_arrays, _names_array, _colors_array) ], layout={ 'xaxis': { 'title': 'Time (minutes)', 'zeroline': False }, 'yaxis': { 'title': 'Fiber density (z-score)', 'zeroline': False, 'range': [-3, 8], 'tickmode': 'array', 'tickvals': [-3, 0, 3, 6] }, 'legend': { 'xanchor': 'left', 'x': 0.1, 'yanchor': 'top', 'bordercolor': 'black', 'borderwidth': 2, 'bgcolor': 'white' }, }) _experiment, _series_id, _group = _tuple save.to_html(_fig=_fig, _path=os.path.join(paths.PLOTS, save.get_module_name()), _filename='plot_' + _experiment + '_' + str(_series_id) + '_' + _group) print('Tuple:', _tuple, 'Correlation:', _correlation)
def main(): _experiments = all_experiments() _experiments = filtering.by_categories(_experiments=_experiments, _is_single_cell=True, _is_high_temporal_resolution=False, _is_bleb=False, _is_dead_dead=False, _is_live_dead=False, _is_bead=False, _is_metastasis=False) _tuples = load.experiments_groups_as_tuples(_experiments) _tuples = filtering.by_time_frames_amount( _tuples, compute.density_time_frame(_experiments[0])) _tuples = filtering.by_main_cell(_tuples) _arguments = [] for _tuple in _tuples: _experiment, _series_id, _group = _tuple _time_frame = compute.density_time_frame(_experiment) for _direction in ['left', 'right']: _arguments.append({ 'experiment': _experiment, 'series_id': _series_id, 'group': _group, 'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER, 'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER, 'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER, 'offset_x': OFFSET_X, 'offset_y': OFFSET_Y, 'offset_z': OFFSET_Z, 'cell_id': 'cell', 'direction': _direction, 'time_points': _time_frame }) _windows_dictionary, _windows_to_compute = \ compute.windows(_arguments, _keys=['experiment', 'series_id', 'group', 'direction']) _fiber_densities = compute.fiber_densities(_windows_to_compute, _subtract_border=True) _tuples = organize.by_single_cell_id(_tuples) print('Total tuples:', len(_tuples)) _experiments_ids = list(_tuples.keys()) _y_arrays = [[] for _i in DERIVATIVES] for _index_1 in tqdm(range(len(_experiments_ids)), desc='Main loop'): _tuple_1 = _experiments_ids[_index_1] _experiment_1, _series_id_1, _cell_id_1 = _tuple_1 _fiber_densities_1 = compute_single_cell_mean( _experiment=_experiment_1, _series_id=_series_id_1, _cell_tuples=_tuples[_tuple_1], _windows_dictionary=_windows_dictionary, _fiber_densities=_fiber_densities) for _index_2 in range(_index_1 + 1, len(_experiments_ids)): _tuple_2 = _experiments_ids[_index_2] _experiment_2, _series_id_2, _cell_id_2 = _tuple_2 _fiber_densities_2 = compute_single_cell_mean( _experiment=_experiment_2, _series_id=_series_id_2, _cell_tuples=_tuples[_tuple_2], _windows_dictionary=_windows_dictionary, _fiber_densities=_fiber_densities) for _derivative_index, _derivative in enumerate(DERIVATIVES): _y_arrays[_derivative_index].append( compute_lib.correlation( compute_lib.derivative(_fiber_densities_1, _n=_derivative), compute_lib.derivative(_fiber_densities_2, _n=_derivative))) print('Total points:', len(_y_arrays[0])) print('Wilcoxon around the zero') for _y_array, _derivative in zip(_y_arrays, DERIVATIVES): print('Derivative:', _derivative, wilcoxon(_y_array)) # plot _colors_array = config.colors(3) _fig = go.Figure(data=[ go.Box(y=_y, name=_derivative, boxpoints='all', jitter=1, pointpos=0, line={'width': 1}, fillcolor='white', marker={ 'size': 10, 'color': _color }, opacity=0.7, showlegend=False) for _y, _derivative, _color in zip( _y_arrays, DERIVATIVES_TEXT, _colors_array) ], layout={ 'xaxis': { 'title': 'Fiber density derivative', 'zeroline': False }, 'yaxis': { 'title': 'Correlation', 'range': [-1, 1], 'zeroline': False, 'tickmode': 'array', 'tickvals': [-1, -0.5, 0, 0.5, 1] } }) save.to_html(_fig=_fig, _path=os.path.join(paths.PLOTS, save.get_module_name()), _filename='plot')
def main(_directions=None): if _directions is None: _directions = ['inside', 'outside'] _experiments = all_experiments() _experiments = filtering.by_categories( _experiments=_experiments, _is_single_cell=False, _is_high_temporal_resolution=False, _is_bleb=False, _is_dead_dead=False, _is_live_dead=False, _is_bead=False, _is_metastasis=False ) _tuples = load.experiments_groups_as_tuples(_experiments) _tuples = filtering.by_real_pairs(_tuples) _tuples = filtering.by_band(_tuples) _tuples = filtering.by_pair_distance_range(_tuples, PAIR_DISTANCE_RANGE) print('Total tuples:', len(_tuples)) _arguments = [] for _tuple in _tuples: _experiment, _series_id, _group = _tuple _latest_time_frame = compute.latest_time_frame_before_overlapping(_experiment, _series_id, _group, OFFSET_X) for _cell_id, _direction in product(['left_cell', 'right_cell'], _directions): _arguments.append({ 'experiment': _experiment, 'series_id': _series_id, 'group': _group, 'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER, 'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER, 'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER, 'offset_x': OFFSET_X, 'offset_y': OFFSET_Y, 'offset_z': OFFSET_Z, 'cell_id': _cell_id, 'direction': _direction, 'time_points': _latest_time_frame }) _windows_dictionary, _windows_to_compute = \ compute.windows(_arguments, _keys=['experiment', 'series_id', 'group', 'cell_id', 'direction']) _fiber_densities = compute.fiber_densities(_windows_to_compute, _subtract_border=True) _experiments_fiber_densities = { _key: [_fiber_densities[_tuple] for _tuple in _windows_dictionary[_key]] for _key in _windows_dictionary } for _direction in _directions: _y_arrays = [[] for _i in DERIVATIVES] for _tuple in tqdm(_tuples, desc='Experiments loop'): _experiment, _series_id, _group = _tuple if (_experiment, _series_id, _group, 'left_cell', _direction) not in _windows_dictionary or \ (_experiment, _series_id, _group, 'right_cell', _direction) not in _windows_dictionary: continue _properties = load.group_properties(_experiment, _series_id, _group) _left_cell_fiber_densities = \ _experiments_fiber_densities[(_experiment, _series_id, _group, 'left_cell', _direction)] _right_cell_fiber_densities = \ _experiments_fiber_densities[(_experiment, _series_id, _group, 'right_cell', _direction)] _left_cell_fiber_densities = compute.remove_blacklist( _experiment, _series_id, _properties['cells_ids']['left_cell'], _left_cell_fiber_densities) _right_cell_fiber_densities = compute.remove_blacklist( _experiment, _series_id, _properties['cells_ids']['right_cell'], _right_cell_fiber_densities) if not OUT_OF_BOUNDARIES: _left_cell_fiber_densities, _right_cell_fiber_densities = \ compute.longest_same_indices_shared_in_borders_sub_array( _left_cell_fiber_densities, _right_cell_fiber_densities ) else: _left_cell_fiber_densities = [_fiber_density[0] for _fiber_density in _left_cell_fiber_densities] _right_cell_fiber_densities = [_fiber_density[0] for _fiber_density in _right_cell_fiber_densities] # ignore small arrays _minimum_time_frame_for_correlation = compute.minimum_time_frames_for_correlation(_experiment) if len(_left_cell_fiber_densities) < _minimum_time_frame_for_correlation or \ len(_right_cell_fiber_densities) < _minimum_time_frame_for_correlation: continue for _derivative_index, _derivative in enumerate(DERIVATIVES): _y_arrays[_derivative_index].append(compute_lib.correlation( compute_lib.derivative(_left_cell_fiber_densities, _n=_derivative), compute_lib.derivative(_right_cell_fiber_densities, _n=_derivative) )) print('Direction:', _direction) print('Total pairs:', len(_y_arrays[0])) print('Wilcoxon around the zero') for _y_array, _derivative in zip(_y_arrays, DERIVATIVES): print('Derivative:', _derivative, wilcoxon(_y_array)) # plot _y_title = 'Inner correlation' if _direction == 'inside' else 'Outer correlation' _colors_array = config.colors(3) _fig = go.Figure( data=[ go.Box( y=_y, name=_derivative, boxpoints='all', jitter=1, pointpos=0, line={ 'width': 1 }, fillcolor='white', marker={ 'size': 10, 'color': _color }, opacity=0.7, showlegend=False ) for _y, _derivative, _color in zip(_y_arrays, DERIVATIVES_TEXT, _colors_array) ], layout={ 'xaxis': { 'title': 'Fiber density derivative', 'zeroline': False }, 'yaxis': { 'title': _y_title, 'range': [-1, 1], 'zeroline': False, 'tickmode': 'array', 'tickvals': [-1, -0.5, 0, 0.5, 1] } } ) save.to_html( _fig=_fig, _path=os.path.join(paths.PLOTS, save.get_module_name()), _filename='plot_direction_' + _direction )
def main(_band=True, _high_temporal_resolution=False): _y_arrays = [[] for _i in PAIR_DISTANCE_RANGES] _names_array = [] for _distances_index, _distances_range in enumerate(PAIR_DISTANCE_RANGES): print('Pair distance range:', str(_distances_range)) _experiments = all_experiments() _experiments = filtering.by_categories( _experiments=_experiments, _is_single_cell=False, _is_high_temporal_resolution=_high_temporal_resolution, _is_bleb=False, _is_dead_dead=False, _is_live_dead=False, _is_bead=False, _is_metastasis=False) _tuples = load.experiments_groups_as_tuples(_experiments) _tuples = filtering.by_pair_distance_range(_tuples, _distances_range) _tuples = filtering.by_real_pairs(_tuples) _tuples = filtering.by_band(_tuples, _band=_band) print('Total tuples:', len(_tuples)) _arguments = [] for _tuple in _tuples: _experiment, _series_id, _group = _tuple _latest_time_frame = compute.latest_time_frame_before_overlapping( _experiment, _series_id, _group, OFFSET_X) for _cell_id in ['left_cell', 'right_cell']: _arguments.append({ 'experiment': _experiment, 'series_id': _series_id, 'group': _group, 'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER, 'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER, 'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER, 'offset_x': OFFSET_X, 'offset_y': OFFSET_Y, 'offset_z': OFFSET_Z, 'cell_id': _cell_id, 'direction': 'inside', 'time_points': _latest_time_frame }) _windows_dictionary, _windows_to_compute = compute.windows( _arguments, _keys=['experiment', 'series_id', 'group', 'cell_id']) _fiber_densities = compute.fiber_densities(_windows_to_compute, _subtract_border=True) _experiments_fiber_densities = { _key: [_fiber_densities[_tuple] for _tuple in _windows_dictionary[_key]] for _key in _windows_dictionary } _tuples_by_experiment = organize.by_experiment(_tuples) _higher_same_counter = 0 _valid_tuples = [] for _experiment in _tuples_by_experiment: print('Experiment:', _experiment) _experiment_tuples = _tuples_by_experiment[_experiment] for _same_index in tqdm(range(len(_experiment_tuples)), desc='Main loop'): _same_tuple = _experiment_tuples[_same_index] _same_experiment, _same_series, _same_group = _same_tuple _same_left_cell_fiber_densities = \ _experiments_fiber_densities[ (_same_experiment, _same_series, _same_group, 'left_cell') ] _same_right_cell_fiber_densities = \ _experiments_fiber_densities[ (_same_experiment, _same_series, _same_group, 'right_cell') ] _same_properties = \ load.group_properties(_same_experiment, _same_series, _same_group) _same_left_cell_fiber_densities = compute.remove_blacklist( _same_experiment, _same_series, _same_properties['cells_ids']['left_cell'], _same_left_cell_fiber_densities) _same_right_cell_fiber_densities = compute.remove_blacklist( _same_experiment, _same_series, _same_properties['cells_ids']['right_cell'], _same_right_cell_fiber_densities) _same_left_cell_fiber_densities_filtered, _same_right_cell_fiber_densities_filtered = \ compute.longest_same_indices_shared_in_borders_sub_array( _same_left_cell_fiber_densities, _same_right_cell_fiber_densities ) # ignore small arrays if len(_same_left_cell_fiber_densities_filtered ) < compute.minimum_time_frames_for_correlation( _same_experiment): continue _same_correlation = compute_lib.correlation( compute_lib.derivative( _same_left_cell_fiber_densities_filtered, _n=DERIVATIVE), compute_lib.derivative( _same_right_cell_fiber_densities_filtered, _n=DERIVATIVE)) for _different_index in range(len(_experiment_tuples)): if _same_index != _different_index: _different_tuple = _experiment_tuples[_different_index] _different_experiment, _different_series, _different_group = \ _different_tuple for _same_cell_id, _different_cell_id in product( ['left_cell', 'right_cell'], ['left_cell', 'right_cell']): _same_fiber_densities = _experiments_fiber_densities[ (_same_experiment, _same_series, _same_group, _same_cell_id)] _different_fiber_densities = _experiments_fiber_densities[ (_different_experiment, _different_series, _different_group, _different_cell_id)] _different_properties = load.group_properties( _different_experiment, _different_series, _different_group) _same_fiber_densities = compute.remove_blacklist( _same_experiment, _same_series, _same_properties['cells_ids'][_same_cell_id], _same_fiber_densities) _different_fiber_densities = compute.remove_blacklist( _different_experiment, _different_series, _different_properties['cells_ids'] [_different_cell_id], _different_fiber_densities) _same_fiber_densities_filtered, _different_fiber_densities_filtered = \ compute.longest_same_indices_shared_in_borders_sub_array( _same_fiber_densities, _different_fiber_densities ) # ignore small arrays if len( _same_fiber_densities_filtered ) < compute.minimum_time_frames_for_correlation( _different_experiment): continue _different_correlation = compute_lib.correlation( compute_lib.derivative( _same_fiber_densities_filtered, _n=DERIVATIVE), compute_lib.derivative( _different_fiber_densities_filtered, _n=DERIVATIVE)) _point_distance = compute_lib.distance_from_a_point_to_a_line( _line=[-1, -1, 1, 1], _point=[ _same_correlation, _different_correlation ]) if _same_correlation > _different_correlation: _y_arrays[_distances_index].append( _point_distance) _higher_same_counter += 1 else: _y_arrays[_distances_index].append( -_point_distance) if _same_tuple not in _valid_tuples: _valid_tuples.append(_same_tuple) print('Total tuples:', len(_valid_tuples)) print('Total points:', len(_y_arrays[_distances_index])) print('Wilcoxon around the zero:') print(wilcoxon(_y_arrays[_distances_index])) print('Higher same amount:', _higher_same_counter / len(_y_arrays[_distances_index])) _names_array.append( str(_distances_range[0]) + '-' + str(_distances_range[1])) # plot _colors_array = config.colors(3) _fig = go.Figure(data=[ go.Box(y=_y, name=_name, boxpoints=False, line={'width': 1}, marker={ 'size': 10, 'color': _color }, showlegend=False) for _y, _name, _color in zip(_y_arrays, _names_array, _colors_array) ], layout={ 'xaxis': { 'title': 'Pair distance (cell diameter)', 'zeroline': False, 'type': 'category' }, 'yaxis': { 'title': 'Same minus different correlation', 'range': [-1, 1.1], 'zeroline': False, 'tickmode': 'array', 'tickvals': [-1, -0.5, 0, 0.5, 1] } }) save.to_html(_fig=_fig, _path=os.path.join(paths.PLOTS, save.get_module_name()), _filename='plot_band_' + str(_band) + '_high_temporal_res_' + str(_high_temporal_resolution))
def main(): _arguments = [] for _tuple in TRIPLET: _experiment, _series_id, _group = _tuple _pair_distance = compute.pair_distance_in_cell_size_time_frame(_experiment, _series_id, _group, _time_frame=0) print(_tuple, 'pairs distance:', round(_pair_distance, 2)) _latest_time_frame = compute.latest_time_frame_before_overlapping(_experiment, _series_id, _group, OFFSET_X) for _cell_id in ['left_cell', 'right_cell']: _arguments.append({ 'experiment': _experiment, 'series_id': _series_id, 'group': _group, 'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER, 'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER, 'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER, 'offset_x': OFFSET_X, 'offset_y': OFFSET_Y, 'offset_z': OFFSET_Z, 'cell_id': _cell_id, 'direction': 'inside', 'time_points': _latest_time_frame }) _windows_dictionary, _windows_to_compute = compute.windows(_arguments, _keys=['experiment', 'series_id', 'group', 'cell_id']) _fiber_densities = compute.fiber_densities(_windows_to_compute, _subtract_border=True) _experiments_fiber_densities = { _key: [_fiber_densities[_tuple] for _tuple in _windows_dictionary[_key]] for _key in _windows_dictionary } _same_correlations_arrays = [[], [], []] _different_correlations_arrays = [[], [], []] _names_array = [] for _same_index in tqdm(range(3), desc='Main loop'): _same_tuple = TRIPLET[_same_index] _same_experiment, _same_series, _same_group = _same_tuple _same_left_cell_fiber_densities = \ _experiments_fiber_densities[ (_same_experiment, _same_series, _same_group, 'left_cell') ] _same_right_cell_fiber_densities = \ _experiments_fiber_densities[ (_same_experiment, _same_series, _same_group, 'right_cell') ] _same_properties = \ load.group_properties(_same_experiment, _same_series, _same_group) _same_left_cell_fiber_densities = compute.remove_blacklist( _same_experiment, _same_series, _same_properties['cells_ids']['left_cell'], _same_left_cell_fiber_densities ) _same_right_cell_fiber_densities = compute.remove_blacklist( _same_experiment, _same_series, _same_properties['cells_ids']['right_cell'], _same_right_cell_fiber_densities ) _same_left_cell_fiber_densities_filtered, _same_right_cell_fiber_densities_filtered = \ compute.longest_same_indices_shared_in_borders_sub_array( _same_left_cell_fiber_densities, _same_right_cell_fiber_densities ) _same_correlation = compute_lib.correlation( compute_lib.derivative(_same_left_cell_fiber_densities_filtered, _n=DERIVATIVE), compute_lib.derivative(_same_right_cell_fiber_densities_filtered, _n=DERIVATIVE) ) for _different_index in range(3): if _same_index != _different_index: _different_tuple = TRIPLET[_different_index] _different_experiment, _different_series, _different_group = \ _different_tuple for _same_cell_id, _different_cell_id in product(['left_cell', 'right_cell'], ['left_cell', 'right_cell']): _same_fiber_densities = _experiments_fiber_densities[( _same_experiment, _same_series, _same_group, _same_cell_id )] _different_fiber_densities = _experiments_fiber_densities[( _different_experiment, _different_series, _different_group, _different_cell_id )] _different_properties = load.group_properties( _different_experiment, _different_series, _different_group ) _same_fiber_densities = compute.remove_blacklist( _same_experiment, _same_series, _same_properties['cells_ids'][_same_cell_id], _same_fiber_densities ) _different_fiber_densities = compute.remove_blacklist( _different_experiment, _different_series, _different_properties['cells_ids'][_different_cell_id], _different_fiber_densities ) _same_fiber_densities_filtered, _different_fiber_densities_filtered = \ compute.longest_same_indices_shared_in_borders_sub_array( _same_fiber_densities, _different_fiber_densities ) _different_correlation = compute_lib.correlation( compute_lib.derivative(_same_fiber_densities_filtered, _n=DERIVATIVE), compute_lib.derivative(_different_fiber_densities_filtered, _n=DERIVATIVE) ) _same_correlations_arrays[_same_index].append(_same_correlation) _different_correlations_arrays[_same_index].append(_different_correlation) _names_array.append('Cells ' + _same_group.split('_')[1] + ' & ' + _same_group.split('_')[2]) print('Group:', TRIPLET[_same_index]) print('Points:', len(_same_correlations_arrays[_same_index])) _same_minus_different = \ np.array(_same_correlations_arrays[_same_index]) - np.array(_different_correlations_arrays[_same_index]) print('Wilcoxon of same minus different around the zero:') print(wilcoxon(_same_minus_different)) print('Higher same amount:', (_same_minus_different > 0).sum() / len(_same_minus_different)) print('Total points:', len(np.array(_same_correlations_arrays).flatten())) _same_minus_different = \ np.array(_same_correlations_arrays).flatten() - np.array(_different_correlations_arrays).flatten() print('Wilcoxon of same minus different around the zero:') print(wilcoxon(_same_minus_different)) print('Higher same amount:', (_same_minus_different > 0).sum() / len(_same_minus_different)) # plot _colors_array = config.colors(3) _fig = go.Figure( data=[ go.Scatter( x=_same_correlations_array, y=_different_correlations_array, name=_name, mode='markers', marker={ 'size': 15, 'color': _color }, opacity=0.7 ) for _same_correlations_array, _different_correlations_array, _name, _color in zip(_same_correlations_arrays, _different_correlations_arrays, _names_array, _colors_array) ], layout={ 'xaxis': { 'title': 'Same network correlation', 'zeroline': False, 'range': [-1.1, 1.2], 'tickmode': 'array', 'tickvals': [-1, -0.5, 0, 0.5, 1] }, 'yaxis': { 'title': 'Different network correlation', 'zeroline': False, 'range': [-1.1, 1.2], 'tickmode': 'array', 'tickvals': [-1, -0.5, 0, 0.5, 1] }, 'legend': { 'xanchor': 'left', 'x': 0.1, 'yanchor': 'top', 'bordercolor': 'black', 'borderwidth': 2, 'bgcolor': 'white' }, 'shapes': [ { 'type': 'line', 'x0': -1, 'y0': -1, 'x1': -1, 'y1': 1, 'line': { 'color': 'black', 'width': 2 } }, { 'type': 'line', 'x0': -1, 'y0': -1, 'x1': 1, 'y1': -1, 'line': { 'color': 'black', 'width': 2 } }, { 'type': 'line', 'x0': -1, 'y0': -1, 'x1': 1, 'y1': 1, 'line': { 'color': 'red', 'width': 2 } } ] } ) save.to_html( _fig=_fig, _path=os.path.join(paths.PLOTS, save.get_module_name()), _filename='plot' )
def main(): _experiments = all_experiments() _experiments = filtering.by_categories(_experiments=_experiments, _is_single_cell=True, _is_high_temporal_resolution=False, _is_bleb=False, _is_dead_dead=False, _is_live_dead=False, _is_bead=False, _is_metastasis=False) _tuples = load.experiments_groups_as_tuples(_experiments) _tuples = filtering.by_time_frames_amount( _tuples, compute.density_time_frame(_experiments[0])) _tuples = filtering.by_main_cell(_tuples) _arguments = [] for _tuple in _tuples: _experiment, _series_id, _group = _tuple _time_frame = compute.density_time_frame(_experiment) for _direction in ['left', 'right']: _arguments.append({ 'experiment': _experiment, 'series_id': _series_id, 'group': _group, 'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER, 'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER, 'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER, 'offset_x': OFFSET_X, 'offset_y': OFFSET_Y, 'offset_z': OFFSET_Z, 'cell_id': 'cell', 'direction': _direction, 'time_points': _time_frame }) _windows_dictionary, _windows_to_compute = \ compute.windows(_arguments, _keys=['experiment', 'series_id', 'group', 'direction']) _fiber_densities = compute.fiber_densities(_windows_to_compute, _subtract_border=True) _tuples = organize.by_single_cell_id(_tuples) print('Total experiments:', len(_tuples)) _kpss_y_arrays = [[] for _i in DERIVATIVES] _adf_y_arrays = [[] for _i in DERIVATIVES] for _tuple in tqdm(_tuples, desc='Experiments loop'): _experiment, _series_id, _cell_id = _tuple _cell_fiber_densities = compute_single_cell_mean( _experiment=_experiment, _series_id=_series_id, _cell_tuples=_tuples[_tuple], _windows_dictionary=_windows_dictionary, _fiber_densities=_fiber_densities) for _derivative_index, _derivative in enumerate(DERIVATIVES): _cell_fiber_densities_derivative = compute_lib.derivative( _cell_fiber_densities, _n=_derivative) _, _kpss_p_value, _, _ = kpss(_cell_fiber_densities_derivative, nlags='legacy') _kpss_y_arrays[_derivative_index].append(_kpss_p_value) _, _adf_p_value, _, _, _, _ = adfuller( _cell_fiber_densities_derivative) _adf_y_arrays[_derivative_index].append(_adf_p_value) print('Total cells:', len(_kpss_y_arrays[0])) # print results print('KPSS:') for _derivative_index, _derivative in enumerate(DERIVATIVES): _stationary_count = len([ _value for _value in _kpss_y_arrays[_derivative_index] if _value > 0.05 ]) print( 'Derivative:', _derivative, 'Stationary:', str(_stationary_count / len(_kpss_y_arrays[_derivative_index]) * 100) + '%') print('ADF:') for _derivative_index, _derivative in enumerate(DERIVATIVES): _stationary_count = len([ _value for _value in _adf_y_arrays[_derivative_index] if _value < 0.05 ]) print( 'Derivative:', _derivative, 'Stationary:', str(_stationary_count / len(_adf_y_arrays[_derivative_index]) * 100) + '%') # plot _colors_array = config.colors(3) for _test_name, _y_title, _y_tickvals, _p_value_line, _y_arrays in \ zip( ['kpss', 'adf'], ['KPSS test p-value', 'ADF test p-value'], [[0.05, 0.1], [0.05, 1]], [0.05, 0.05], [_kpss_y_arrays, _adf_y_arrays] ): _fig = go.Figure(data=[ go.Box(y=_y, name=_derivative, boxpoints='all', jitter=1, pointpos=0, line={'width': 1}, fillcolor='white', marker={ 'size': 10, 'color': _color }, opacity=0.7, showlegend=False) for _y, _derivative, _color in zip( _y_arrays, DERIVATIVES_TEXT, _colors_array) ], layout={ 'xaxis': { 'title': 'Fiber density derivative', 'zeroline': False }, 'yaxis': { 'title': _y_title, 'zeroline': False, 'tickmode': 'array', 'tickvals': _y_tickvals }, 'shapes': [{ 'type': 'line', 'x0': DERIVATIVES[0] - 0.75, 'y0': _p_value_line, 'x1': DERIVATIVES[-1] + 0.75, 'y1': _p_value_line, 'line': { 'color': 'red', 'width': 2, 'dash': 'dash' } }] }) save.to_html(_fig=_fig, _path=os.path.join(paths.PLOTS, save.get_module_name()), _filename='plot_' + _test_name)
def main(_real_cells=True, _static=False, _band=True, _high_temporal_resolution=False): _experiments = all_experiments() _experiments = filtering.by_categories( _experiments=_experiments, _is_single_cell=False, _is_high_temporal_resolution=_high_temporal_resolution, _is_bleb=False, _is_dead_dead=False, _is_live_dead=False, _is_bead=False, _is_metastasis=False ) _tuples = load.experiments_groups_as_tuples(_experiments) _tuples = filtering.by_pair_distance_range(_tuples, PAIR_DISTANCE_RANGE) _tuples = filtering.by_real_pairs(_tuples, _real_pairs=_real_cells) _tuples = filtering.by_fake_static_pairs(_tuples, _fake_static_pairs=_static) _tuples = filtering.by_band(_tuples, _band=_band) print('Total tuples:', len(_tuples)) _arguments = [] for _tuple in _tuples: _experiment, _series_id, _group = _tuple _latest_time_frame = compute.latest_time_frame_before_overlapping(_experiment, _series_id, _group, OFFSET_X) for _cell_id in ['left_cell', 'right_cell']: _arguments.append({ 'experiment': _experiment, 'series_id': _series_id, 'group': _group, 'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER, 'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER, 'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER, 'offset_x': OFFSET_X, 'offset_y': OFFSET_Y, 'offset_z': OFFSET_Z, 'cell_id': _cell_id, 'direction': 'inside', 'time_points': _latest_time_frame }) _z_array = np.zeros(shape=(len(BY), len(BY))) for (_padding_index, _padding_by), (_space_index, _space_by) in product(enumerate(BY), enumerate(BY)): print('Padding by: ', _padding_by, ', space by: ', _space_by) _correlation = compute_data(_tuples, _arguments, _padding_y_by=_padding_by, _padding_z_by=_padding_by, _space_y_by=_space_by, _space_z_by=_space_by) _z_array[_space_index, _padding_index] = _correlation # plot _colors_array = ['white', config.colors(1)] _fig = go.Figure( data=go.Heatmap( x=BY, y=BY, z=_z_array, colorscale=sns.color_palette(_colors_array).as_hex(), colorbar={ 'tickmode': 'array', 'tickvals': [0, 0.5, 1], 'ticktext': ['0', 'Correlation', '1'], 'tickangle': -90 }, showscale=True, zmin=0, zmax=1 ), layout={ 'xaxis': { 'title': 'Border size (cell diameter)', 'zeroline': False, 'tickmode': 'array', 'tickvals': [0, 0.5, 1, 1.5, 2] }, 'yaxis': { 'title': 'Space from quantification window (cell diameter)', 'zeroline': False, 'tickmode': 'array', 'tickvals': [0, 0.5, 1, 1.5, 2] } } ) save.to_html( _fig=_fig, _path=os.path.join(paths.PLOTS, save.get_module_name()), _filename='plot_high_time_' + str(_high_temporal_resolution) + '_band_' + str(_band) )
def main(_band=None, _high_temporal_resolution=True, _tuples_to_mark=None, _tuples_to_plot=None, _plots=None): if _plots is None: _plots = ['whiteness', 'granger'] _experiments = all_experiments() _experiments = filtering.by_categories( _experiments=_experiments, _is_single_cell=False, _is_high_temporal_resolution=_high_temporal_resolution, _is_bleb=False, _is_dead_dead=False, _is_live_dead=False, _is_bead=False, _is_metastasis=False) _tuples = load.experiments_groups_as_tuples(_experiments) _tuples = filtering.by_time_frames_amount(_tuples, _time_frames=MINIMUM_TIME_FRAMES) _tuples = filtering.by_pair_distance_range( _tuples, _distance_range=PAIR_DISTANCE_RANGE) _tuples = filtering.by_real_pairs(_tuples) _tuples = filtering.by_band(_tuples, _band=_band) print('Total tuples:', len(_tuples)) _arguments = [] for _tuple in _tuples: _experiment, _series_id, _group = _tuple _latest_time_frame = compute.latest_time_frame_before_overlapping( _experiment, _series_id, _group, OFFSET_X) for _cell_id in ['left_cell', 'right_cell']: _arguments.append({ 'experiment': _experiment, 'series_id': _series_id, 'group': _group, 'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER, 'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER, 'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER, 'offset_x': OFFSET_X, 'offset_y': OFFSET_Y, 'offset_z': OFFSET_Z, 'cell_id': _cell_id, 'direction': 'inside', 'time_points': _latest_time_frame }) _windows_dictionary, _windows_to_compute = compute.windows( _arguments, _keys=['experiment', 'series_id', 'group', 'cell_id']) _fiber_densities = compute.fiber_densities(_windows_to_compute, _subtract_border=True) _experiments_fiber_densities = { _key: [_fiber_densities[_tuple] for _tuple in _windows_dictionary[_key]] for _key in _windows_dictionary } _n_pairs = 0 _n_pairs_with_band = 0 _whiteness_p_values = [] _n_passed_whiteness_with_band = 0 _granger_causality_p_values = [] _n_passed_granger_causality_with_band = 0 _correlations = [] _time_lag_correlations = [] _end_fiber_densities = [] for _tuple in _tuples: _experiment, _series_id, _group = _tuple _left_cell_fiber_densities = \ _experiments_fiber_densities[(_experiment, _series_id, _group, 'left_cell')] _right_cell_fiber_densities = \ _experiments_fiber_densities[(_experiment, _series_id, _group, 'right_cell')] _properties = load.group_properties(_experiment, _series_id, _group) _left_cell_fiber_densities = compute.remove_blacklist( _experiment, _series_id, _properties['cells_ids']['left_cell'], _left_cell_fiber_densities) _right_cell_fiber_densities = compute.remove_blacklist( _experiment, _series_id, _properties['cells_ids']['right_cell'], _right_cell_fiber_densities) _left_cell_fiber_densities_filtered, _right_cell_fiber_densities_filtered = \ compute.longest_same_indices_shared_in_borders_sub_array( _left_cell_fiber_densities, _right_cell_fiber_densities) # ignore small arrays if len(_left_cell_fiber_densities_filtered) < MINIMUM_TIME_FRAMES: continue _n_pairs += 1 if _properties['band']: _n_pairs_with_band += 1 _start_time_frame = 0 for _left in _left_cell_fiber_densities: if _left[0] == _left_cell_fiber_densities_filtered[0]: break _start_time_frame += 1 # stationary test with warnings.catch_warnings(): warnings.simplefilter('ignore', category=InterpolationWarning) # find derivative for stationary for _derivative in range(10): _left_cell_fiber_densities_derivative = \ compute_lib.derivative(_left_cell_fiber_densities_filtered, _n=_derivative) _right_cell_fiber_densities_derivative = \ compute_lib.derivative(_right_cell_fiber_densities_filtered, _n=_derivative) if ADF_TEST: _, _left_cell_adf_p_value, _, _, _, _ = adfuller( _left_cell_fiber_densities_derivative) _, _right_cell_adf_p_value, _, _, _, _ = adfuller( _right_cell_fiber_densities_derivative) if _left_cell_adf_p_value > 0.05 or _right_cell_adf_p_value > 0.05: continue if KPSS_TEST: _, _left_cell_kpss_p_value, _, _ = kpss( _left_cell_fiber_densities_derivative, nlags='legacy') _, _right_cell_kpss_p_value, _, _ = kpss( _right_cell_fiber_densities_derivative, nlags='legacy') if _left_cell_kpss_p_value < 0.05 or _right_cell_kpss_p_value < 0.05: continue # stationary break # causality try: _x = pd.DataFrame(data=[[_left_value, _right_value] for _left_value, _right_value in zip( _left_cell_fiber_densities_derivative, _right_cell_fiber_densities_derivative) ], columns=['left', 'right']) # var model to retrieve lag _var_model = VAR(_x) _lag_order_results = _var_model.select_order() _estimators_lags = [ _lag_order_results.aic, _lag_order_results.bic, _lag_order_results.fpe, _lag_order_results.hqic ] _min_estimator_lag = min(_estimators_lags) # found a lag if 0 < _min_estimator_lag <= MAXIMUM_LAG: _var_model_results = _var_model.fit(maxlags=_min_estimator_lag, ic=None) _whiteness = _var_model_results.test_whiteness( nlags=_min_estimator_lag + 1) _whiteness_p_values.append(_whiteness.pvalue) if _tuples_to_mark is not None and _tuple in _tuples_to_mark and _whiteness.pvalue > 0.05: print(_tuple, 'marked whiteness p-value:', _whiteness.pvalue) # no autocorrelation in the residuals if _whiteness.pvalue > 0.05: if _properties['band']: _n_passed_whiteness_with_band += 1 # time lag = 0 _correlation = compute_lib.correlation( _left_cell_fiber_densities_derivative, _right_cell_fiber_densities_derivative) # if _correlation < 0.5: # continue # granger causality for _caused, _causing in zip(['left', 'right'], ['right', 'left']): _granger = _var_model_results.test_causality( caused=_caused, causing=_causing) _granger_causality_p_values.append(_granger.pvalue) # time lag = 0 _correlations.append(_correlation) # time lag = min estimator if _causing == 'left': _left_fiber_densities_time_lag = \ _left_cell_fiber_densities_derivative[:-_min_estimator_lag] _right_fiber_densities_time_lag = \ _right_cell_fiber_densities_derivative[_min_estimator_lag:] else: _left_fiber_densities_time_lag = \ _left_cell_fiber_densities_derivative[_min_estimator_lag:] _right_fiber_densities_time_lag = \ _right_cell_fiber_densities_derivative[:-_min_estimator_lag] _time_lag_correlation = compute_lib.correlation( _left_fiber_densities_time_lag, _right_fiber_densities_time_lag) _time_lag_correlations.append(_time_lag_correlation) # end fiber density _time_frame = compute.density_time_frame(_experiment) if len(_left_cell_fiber_densities_filtered ) > _time_frame: _end_fiber_density = \ (_left_cell_fiber_densities_filtered[_time_frame] + _right_cell_fiber_densities_filtered[_time_frame]) / 2 else: _end_fiber_density = \ (_left_cell_fiber_densities_filtered[-1] + _right_cell_fiber_densities_filtered[-1]) / 2 _normalization = load.normalization_series_file_data( _experiment, _series_id) _normalized_fiber_density = compute_lib.z_score( _end_fiber_density, _normalization['average'], _normalization['std']) _end_fiber_densities.append(_normalized_fiber_density) # marking if _tuples_to_mark is not None and _tuple in _tuples_to_mark and _granger.pvalue < 0.05: print(_tuple, 'causing:', _causing, 'marked granger p-value:', _granger.pvalue) if _granger.pvalue < 0.05: if _properties['band']: _n_passed_granger_causality_with_band += 1 _normality = _var_model_results.test_normality() _inst_granger = _var_model_results.test_inst_causality( causing=_causing) print( _tuple, _causing.capitalize() + ' causes ' + _caused + '!', 'time-points: ' + str(len( _left_cell_fiber_densities_derivative)), 'stationary derivative: ' + str(_derivative), 'band:' + str(_properties['band']), 'p-value: ' + str(round(_granger.pvalue, 4)), 'lag: ' + str(_min_estimator_lag), 'normality p-value: ' + str(round(_normality.pvalue, 4)), 'inst p-value: ' + str(round(_inst_granger.pvalue, 4)), sep='\t') # lag = 0 print('Time lag = 0 correlation:', _correlation) # rest of lags for _lag in range(1, _min_estimator_lag + 1): if _causing == 'left': _left_fiber_densities_time_lag = _left_cell_fiber_densities_derivative[: -_lag] _right_fiber_densities_time_lag = _right_cell_fiber_densities_derivative[ _lag:] else: _left_fiber_densities_time_lag = _left_cell_fiber_densities_derivative[ _lag:] _right_fiber_densities_time_lag = _right_cell_fiber_densities_derivative[: -_lag] _correlation = compute_lib.correlation( _left_fiber_densities_time_lag, _right_fiber_densities_time_lag) print( 'Time lag = ' + str(_lag) + ' correlation:', _correlation) # plots if _tuples_to_plot is not None and _tuple in _tuples_to_plot: _y_arrays = [ _left_cell_fiber_densities_derivative, _right_cell_fiber_densities_derivative ] _names_array = ['Left cell', 'Right cell'] _colors_array = config.colors(2) _temporal_resolution = compute.temporal_resolution_in_minutes( _experiment) _fig = go.Figure(data=[ go.Scatter(x=np.arange( start=_start_time_frame, stop=_start_time_frame + len(_left_cell_fiber_densities_derivative ), step=1) * _temporal_resolution, y=_y, name=_name, mode='lines', line={ 'color': _color, 'width': 1 }) for _y, _name, _color in zip( _y_arrays, _names_array, _colors_array) ], layout={ 'xaxis': { 'title': 'Time (minutes)', 'zeroline': False }, 'yaxis': { 'title': 'Fiber density (z-score)' + '\'' * _derivative, 'zeroline': False }, 'legend': { 'xanchor': 'left', 'x': 0.1, 'yanchor': 'top', 'bordercolor': 'black', 'borderwidth': 2, 'bgcolor': 'white' }, }) _experiment, _series_id, _group = _tuple save.to_html( _fig=_fig, _path=os.path.join(paths.PLOTS, save.get_module_name()), _filename='plot_' + _experiment + '_' + str(_series_id) + '_' + _group) # residuals _y_arrays = \ [_var_model_results.resid.values[:, 0], _var_model_results.resid.values[:, 1]] _fig = go.Figure(data=[ go.Scatter(x=np.arange( start=_start_time_frame, stop=_start_time_frame + len(_y), step=1) * _temporal_resolution, y=_y, name=_name, mode='lines', line={ 'color': _color, 'width': 1 }) for _y, _name, _color in zip( _y_arrays, _names_array, _colors_array) ], layout={ 'xaxis': { 'title': 'Time (minutes)', 'zeroline': False }, 'yaxis': { 'title': 'Residual', 'zeroline': False }, 'legend': { 'xanchor': 'left', 'x': 0.1, 'yanchor': 'top', 'bordercolor': 'black', 'borderwidth': 2, 'bgcolor': 'white' }, }) _experiment, _series_id, _group = _tuple save.to_html( _fig=_fig, _path=os.path.join(paths.PLOTS, save.get_module_name()), _filename='plot_residuals_' + _experiment + '_' + str(_series_id) + '_' + _group) # not enough time points except ValueError: continue print('Total pairs:', _n_pairs) print('Total pairs with band:', _n_pairs_with_band) print('Total pairs passed whiteness:', (np.array(_whiteness_p_values) > 0.05).sum()) print('Total pairs passed whiteness with band:', _n_passed_whiteness_with_band) print('Total cells passed granger causality:', (np.array(_granger_causality_p_values) < 0.05).sum()) print('Total cells passed granger causality with band:', _n_passed_granger_causality_with_band) # p-value correction print('Corrections of GC p-value < 0.05:') _granger_causality_p_values_corrected = multipletests( pvals=_granger_causality_p_values, method='fdr_bh') for _p_value, _p_value_corrected in zip( _granger_causality_p_values, _granger_causality_p_values_corrected[1]): if _p_value < 0.05: print('Original GC p-value:', _p_value, 'corrected:', _p_value_corrected) # plots for _test_name, _y_title, _y_array in \ zip( ['whiteness', 'granger'], ['Whiteness p-value', 'Granger causality p-value'], [_whiteness_p_values, _granger_causality_p_values] ): if _test_name in _plots: _fig = go.Figure(data=go.Box(y=_y_array, boxpoints='all', jitter=1, pointpos=0, line={'width': 1}, fillcolor='white', marker={ 'size': 10, 'color': '#ea8500' }, opacity=0.7, showlegend=False), layout={ 'xaxis': { 'zeroline': False }, 'yaxis': { 'title': _y_title, 'zeroline': False, 'range': [-0.1, 1.1], 'tickmode': 'array', 'tickvals': [0.05, 1] }, 'shapes': [{ 'type': 'line', 'x0': -0.75, 'y0': 0.05, 'x1': 0.75, 'y1': 0.05, 'line': { 'color': 'red', 'width': 2, 'dash': 'dash' } }] }) save.to_html(_fig=_fig, _path=os.path.join(paths.PLOTS, save.get_module_name()), _filename='plot_' + _test_name) # granger versus correlation print( 'GC vs. correlation pearson correlation:', compute_lib.correlation(_granger_causality_p_values, _correlations, _with_p_value=True)) _fig = go.Figure(data=go.Scatter(x=_granger_causality_p_values, y=_correlations, mode='markers', marker={ 'size': 10, 'color': '#ea8500' }, showlegend=False), layout={ 'xaxis': { 'title': 'Granger causality p-value', 'zeroline': False, }, 'yaxis': { 'title': 'Inner correlation', 'zeroline': False, } }) save.to_html(_fig=_fig, _path=os.path.join(paths.PLOTS, save.get_module_name()), _filename='plot_gc_vs_correlation') # granger versus time lag correlation print( 'GC vs. time lag correlation pearson correlation:', compute_lib.correlation(_granger_causality_p_values, _time_lag_correlations, _with_p_value=True)) _fig = go.Figure(data=go.Scatter(x=_granger_causality_p_values, y=_time_lag_correlations, mode='markers', marker={ 'size': 10, 'color': '#ea8500' }, showlegend=False), layout={ 'xaxis': { 'title': 'Granger causality p-value', 'zeroline': False, }, 'yaxis': { 'title': 'GC lag inner correlation', 'zeroline': False, } }) save.to_html(_fig=_fig, _path=os.path.join(paths.PLOTS, save.get_module_name()), _filename='plot_gc_vs_time_lag_correlation') # granger versus end fiber density print( 'GC vs. end fiber density pearson correlation:', compute_lib.correlation(_granger_causality_p_values, _end_fiber_densities, _with_p_value=True)) _fig = go.Figure(data=go.Scatter(x=_granger_causality_p_values, y=_end_fiber_densities, mode='markers', marker={ 'size': 10, 'color': '#ea8500' }, showlegend=False), layout={ 'xaxis': { 'title': 'Granger causality p-value', 'zeroline': False, }, 'yaxis': { 'title': 'End fiber density (z-score)', 'zeroline': False, } }) save.to_html(_fig=_fig, _path=os.path.join(paths.PLOTS, save.get_module_name()), _filename='plot_gc_vs_end_density')