def compute_z_array(_band=True, _high_temporal_resolution=False, _offset_x=OFFSET_X, _offset_y_start=OFFSET_Y_START, _offset_y_end=OFFSET_Y_END, _offset_y_step=OFFSET_Y_STEP, _offset_z_start=OFFSET_Z_START, _offset_z_end=OFFSET_Z_END, _offset_z_step=OFFSET_Z_STEP): global _tuples, _experiments_fiber_densities, _z_array _experiments = all_experiments() _experiments = filtering.by_categories( _experiments=_experiments, _is_single_cell=False, _is_high_temporal_resolution=_high_temporal_resolution, _is_bleb=False, _is_dead_dead=False, _is_live_dead=False, _is_bead=False, _is_metastasis=False) _tuples = load.experiments_groups_as_tuples(_experiments) _tuples = filtering.by_time_frames_amount( _tuples, compute.minimum_time_frames_for_correlation(_experiments[0])) _tuples = filtering.by_pair_distance_range(_tuples, PAIR_DISTANCE_RANGE) _tuples = filtering.by_real_pairs(_tuples) _tuples = filtering.by_band(_tuples, _band=_band) print('Total tuples:', len(_tuples)) _offsets_y = np.arange(start=_offset_y_start, stop=_offset_y_end + _offset_y_step, step=_offset_y_step) _offsets_z = np.arange(start=_offset_z_start, stop=_offset_z_end + _offset_z_step, step=_offset_z_step) _arguments = [] for _tuple in _tuples: _experiment, _series_id, _group = _tuple _latest_time_frame = compute.latest_time_frame_before_overlapping( _experiment, _series_id, _group, OFFSET_X) for _offset_y, _offset_z, _cell_id in product( _offsets_y, _offsets_z, ['left_cell', 'right_cell']): _arguments.append({ 'experiment': _experiment, 'series_id': _series_id, 'group': _group, 'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER, 'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER, 'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER, 'offset_x': OFFSET_X, 'offset_y': _offset_y, 'offset_z': _offset_z, 'cell_id': _cell_id, 'direction': 'inside', 'time_points': _latest_time_frame }) _windows_dictionary, _windows_to_compute = \ compute.windows(_arguments, _keys=['experiment', 'series_id', 'group', 'offset_y', 'offset_z', 'cell_id']) _fiber_densities = compute.fiber_densities(_windows_to_compute) _experiments_fiber_densities = { _key: [_fiber_densities[_tuple] for _tuple in _windows_dictionary[_key]] for _key in _windows_dictionary } # clean _fiber_densities = None _windows_dictionary = None _windows_to_compute = None _arguments = [] for (_offset_y_index, _offset_y), (_offset_z_index, _offset_z) in \ product(enumerate(_offsets_y), enumerate(_offsets_z)): _arguments.append( (_offset_y_index, _offset_y, _offset_z_index, _offset_z)) _z_array = np.zeros(shape=(len(_offsets_y), len(_offsets_z))) with Pool(CPUS_TO_USE) as _p: for _answer in tqdm(_p.imap_unordered(compute_data, _arguments), total=len(_arguments), desc='Computing heatmap'): _offset_y_index, _offset_z_index, _mean = _answer _z_array[_offset_y_index, _offset_z_index] = _mean _p.close() _p.join() return _z_array
def main(_high_temporal_resolution=True): _experiments = all_experiments() _experiments = filtering.by_categories( _experiments=_experiments, _is_single_cell=False, _is_high_temporal_resolution=_high_temporal_resolution, _is_bleb=False, _is_dead_dead=False, _is_live_dead=False, _is_bead=False, _is_metastasis=False) _tuples = load.experiments_groups_as_tuples(_experiments) _tuples = filtering.by_time_frames_amount( _tuples, _time_frames=MOVING_WINDOW_LENGTH[_high_temporal_resolution]) _tuples = filtering.by_pair_distance_range( _tuples, _distance_range=PAIR_DISTANCE_RANGE) _tuples = filtering.by_real_pairs(_tuples) _tuples = filtering.by_band(_tuples) print('Total tuples:', len(_tuples)) _arguments = [] for _tuple in _tuples: _experiment, _series_id, _group = _tuple _latest_time_frame = compute.latest_time_frame_before_overlapping( _experiment, _series_id, _group, OFFSET_X) for _cell_id in ['left_cell', 'right_cell']: _arguments.append({ 'experiment': _experiment, 'series_id': _series_id, 'group': _group, 'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER, 'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER, 'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER, 'offset_x': OFFSET_X, 'offset_y': OFFSET_Y, 'offset_z': OFFSET_Z, 'cell_id': _cell_id, 'direction': 'inside', 'time_points': _latest_time_frame }) _windows_dictionary, _windows_to_compute = compute.windows( _arguments, _keys=['experiment', 'series_id', 'group', 'cell_id']) _fiber_densities = compute.fiber_densities(_windows_to_compute, _subtract_border=True) _experiments_fiber_densities = { _key: [_fiber_densities[_tuple] for _tuple in _windows_dictionary[_key]] for _key in _windows_dictionary } for _tuple in _tuples: _correlations = [] _experiment, _series_id, _group = _tuple _left_cell_fiber_densities = \ _experiments_fiber_densities[(_experiment, _series_id, _group, 'left_cell')] _right_cell_fiber_densities = \ _experiments_fiber_densities[(_experiment, _series_id, _group, 'right_cell')] _properties = load.group_properties(_experiment, _series_id, _group) _left_cell_fiber_densities = compute.remove_blacklist( _experiment, _series_id, _properties['cells_ids']['left_cell'], _left_cell_fiber_densities) _right_cell_fiber_densities = compute.remove_blacklist( _experiment, _series_id, _properties['cells_ids']['right_cell'], _right_cell_fiber_densities) for _start_time_frame in \ range(0, END_TIME_FRAME[_high_temporal_resolution], TIME_FRAME_STEP[_high_temporal_resolution]): _left_cell_fiber_densities_window = _left_cell_fiber_densities[ _start_time_frame:_start_time_frame + MOVING_WINDOW_LENGTH[_high_temporal_resolution]] _right_cell_fiber_densities_window = _right_cell_fiber_densities[ _start_time_frame:_start_time_frame + MOVING_WINDOW_LENGTH[_high_temporal_resolution]] _left_cell_fiber_densities_filtered, _right_cell_fiber_densities_filtered = \ compute.longest_same_indices_shared_in_borders_sub_array( _left_cell_fiber_densities_window, _right_cell_fiber_densities_window) # ignore small arrays if len(_left_cell_fiber_densities_filtered ) < MOVING_WINDOW_LENGTH[_high_temporal_resolution]: _correlations.append(None) continue _correlations.append( compute_lib.correlation( compute_lib.derivative(_left_cell_fiber_densities_filtered, _n=DERIVATIVE), compute_lib.derivative( _right_cell_fiber_densities_filtered, _n=DERIVATIVE))) # plot _temporal_resolution = compute.temporal_resolution_in_minutes( _experiment) _fig = go.Figure(data=go.Scatter( x=np.arange(start=0, stop=len(_correlations), step=1) * _temporal_resolution * TIME_FRAME_STEP[_high_temporal_resolution], y=_correlations, mode='lines+markers', line={'dash': 'solid'}), layout={ 'xaxis': { 'title': 'Window start time (minutes)', 'zeroline': False }, 'yaxis': { 'title': 'Inner correlation', 'zeroline': False } }) save.to_html(_fig=_fig, _path=os.path.join(paths.PLOTS, save.get_module_name()), _filename='plot_' + str(_experiment) + '_' + str(_series_id) + '_' + str(_group))
def compute_matched_fiber(_tuples): _tuples = filtering.by_time_frames_amount( _tuples, compute.density_time_frame(_tuples[0])) _tuples = filtering.by_pair_distance_range(_tuples, PAIR_DISTANCE_RANGE) _experiments_matched = organize.by_matched_real_and_fake(_tuples) print('Total matched pairs:', len(_experiments_matched)) _max_offsets_x = [] _arguments = [] for _matched_tuple in _experiments_matched: for _tuple in _matched_tuple: _experiment, _series_id, _group = _tuple _time_frame = compute.density_time_frame(_experiment) _pair_distance = compute.pair_distance_in_cell_size_time_frame( _experiment, _series_id, _group, _time_frame - 1) _offsets_x = \ np.arange(start=0, stop=_pair_distance / 2 - 0.5 - QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER, step=OFFSET_X_STEP) if len(_offsets_x) > len(_max_offsets_x): _max_offsets_x = _offsets_x for _offset_x in _offsets_x: for _cell_id in ['left_cell', 'right_cell']: _arguments.append({ 'experiment': _experiment, 'series_id': _series_id, 'group': _group, 'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER, 'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER, 'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER, 'offset_x': _offset_x, 'offset_y': OFFSET_Y, 'offset_z': OFFSET_Z, 'cell_id': _cell_id, 'direction': 'inside', 'time_point': _time_frame - 1 }) _windows_dictionary, _windows_to_compute = \ compute.windows(_arguments, _keys=['experiment', 'series_id', 'group', 'offset_x', 'cell_id']) _fiber_densities = compute.fiber_densities(_windows_to_compute) _experiments_fiber_densities_real = [[] for _i in range(len(_max_offsets_x))] _experiments_fiber_densities_fake = [[] for _i in range(len(_max_offsets_x))] for _tuple in tqdm(_experiments_matched, desc='Experiments loop'): _tuple_real, _tuple_fake = _tuple _experiment_real, _series_id_real, _group_real = _tuple_real _experiment_fake, _series_id_fake, _group_fake = _tuple_fake for _offset_x_index, _offset_x in enumerate(_max_offsets_x): for _cell_id in ['left_cell', 'right_cell']: if (_experiment_real, _series_id_real, _group_real, _offset_x, _cell_id) and \ (_experiment_fake, _series_id_fake, _group_fake, _offset_x, _cell_id) in _windows_dictionary: _normalization = load.normalization_series_file_data( _experiment_real, _series_id_real) _window_tuple_real = \ _windows_dictionary[(_experiment_real, _series_id_real, _group_real, _offset_x, _cell_id)][0] _fiber_density_real = _fiber_densities[_window_tuple_real] _window_tuple_fake = \ _windows_dictionary[(_experiment_fake, _series_id_fake, _group_fake, _offset_x, _cell_id)][0] _fiber_density_fake = _fiber_densities[_window_tuple_fake] if not OUT_OF_BOUNDARIES and (_fiber_density_real[1] or _fiber_density_fake[1]): continue _normalized_fiber_density_real = compute_lib.z_score( _x=_fiber_density_real[0], _average=_normalization['average'], _std=_normalization['std']) _normalized_fiber_density_fake = compute_lib.z_score( _x=_fiber_density_fake[0], _average=_normalization['average'], _std=_normalization['std']) if not np.isnan( _normalized_fiber_density_real) and not np.isnan( _normalized_fiber_density_fake): _experiments_fiber_densities_real[ _offset_x_index].append( _normalized_fiber_density_real) _experiments_fiber_densities_fake[ _offset_x_index].append( _normalized_fiber_density_fake) return _experiments_fiber_densities_real, _experiments_fiber_densities_fake, _max_offsets_x
def main(_real_cells=True, _static=False, _band=True, _high_temporal_resolution=False): _experiments = all_experiments() _experiments = filtering.by_categories( _experiments=_experiments, _is_single_cell=False, _is_high_temporal_resolution=_high_temporal_resolution, _is_bleb=False, _is_dead_dead=False, _is_live_dead=False, _is_bead=False, _is_metastasis=False ) _tuples = load.experiments_groups_as_tuples(_experiments) _tuples = filtering.by_pair_distance_range(_tuples, PAIR_DISTANCE_RANGE) _tuples = filtering.by_real_pairs(_tuples, _real_pairs=_real_cells) _tuples = filtering.by_fake_static_pairs(_tuples, _fake_static_pairs=_static) _tuples = filtering.by_band(_tuples, _band=_band) _tuples = filtering.by_time_frames_amount(_tuples, compute.minimum_time_frames_for_correlation(_experiments[0])) print('Total tuples:', len(_tuples)) print('Density') _arguments = [] for _tuple in _tuples: _experiment, _series_id, _group = _tuple _time_frame = compute.minimum_time_frames_for_correlation(_experiment) for _cell_id in ['left_cell', 'right_cell']: _arguments.append({ 'experiment': _experiment, 'series_id': _series_id, 'group': _group, 'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER, 'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER, 'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER, 'offset_x': OFFSET_X, 'offset_y': OFFSET_Y_DENSITY, 'offset_z': OFFSET_Z, 'cell_id': _cell_id, 'direction': 'inside', 'time_point': _time_frame - 1 }) _windows_dictionary, _windows_to_compute = \ compute.windows(_arguments, _keys=['experiment', 'series_id', 'group', 'cell_id']) _densities_fiber_densities = compute.fiber_densities(_windows_to_compute, _subtract_border=False) _densities_experiments_fiber_densities = { _key: [_densities_fiber_densities[_tuple] for _tuple in _windows_dictionary[_key]] for _key in _windows_dictionary } print('Correlations') _arguments = [] for _tuple in _tuples: _experiment, _series_id, _group = _tuple _latest_time_frame = compute.latest_time_frame_before_overlapping(_experiment, _series_id, _group, OFFSET_X) for _cell_id in ['left_cell', 'right_cell']: _arguments.append({ 'experiment': _experiment, 'series_id': _series_id, 'group': _group, 'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER, 'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER, 'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER, 'offset_x': OFFSET_X, 'offset_y': OFFSET_Y_CORRELATION, 'offset_z': OFFSET_Z, 'cell_id': _cell_id, 'direction': 'inside', 'time_points': _latest_time_frame }) _windows_dictionary, _windows_to_compute = \ compute.windows(_arguments, _keys=['experiment', 'series_id', 'group', 'cell_id']) _correlations_fiber_densities = compute.fiber_densities(_windows_to_compute, _subtract_border=True) _correlations_experiments_fiber_densities = { _key: [_correlations_fiber_densities[_tuple] for _tuple in _windows_dictionary[_key]] for _key in _windows_dictionary } _densities = [] _correlations = [] for _tuple in tqdm(_tuples, desc='Main loop'): _experiment, _series_id, _group = _tuple # density _left_cell_fiber_density = \ _densities_experiments_fiber_densities[(_experiment, _series_id, _group, 'left_cell')][0] _right_cell_fiber_density = \ _densities_experiments_fiber_densities[(_experiment, _series_id, _group, 'right_cell')][0] # not relevant if _left_cell_fiber_density[1] or _right_cell_fiber_density[1]: continue _normalization = load.normalization_series_file_data(_experiment, _series_id) _left_cell_fiber_density_normalized = compute_lib.z_score( _x=_left_cell_fiber_density[0], _average=_normalization['average'], _std=_normalization['std'] ) _right_cell_fiber_density_normalized = compute_lib.z_score( _x=_right_cell_fiber_density[0], _average=_normalization['average'], _std=_normalization['std'] ) _density = (_left_cell_fiber_density_normalized + _right_cell_fiber_density_normalized) / 2 # correlation _left_cell_fiber_densities = \ _correlations_experiments_fiber_densities[(_experiment, _series_id, _group, 'left_cell')] _right_cell_fiber_densities = \ _correlations_experiments_fiber_densities[(_experiment, _series_id, _group, 'right_cell')] _properties = load.group_properties(_experiment, _series_id, _group) _left_cell_fiber_densities = compute.remove_blacklist( _experiment, _series_id, _properties['cells_ids']['left_cell'], _left_cell_fiber_densities) _right_cell_fiber_densities = compute.remove_blacklist( _experiment, _series_id, _properties['cells_ids']['right_cell'], _right_cell_fiber_densities) _left_cell_fiber_densities_filtered, _right_cell_fiber_densities_filtered = \ compute.longest_same_indices_shared_in_borders_sub_array( _left_cell_fiber_densities, _right_cell_fiber_densities ) # ignore small arrays if len(_left_cell_fiber_densities_filtered) < compute.minimum_time_frames_for_correlation(_experiment): continue _correlation = compute_lib.correlation( compute_lib.derivative(_left_cell_fiber_densities_filtered, _n=DERIVATIVE), compute_lib.derivative(_right_cell_fiber_densities_filtered, _n=DERIVATIVE) ) _densities.append(_density) _correlations.append(_correlation) print('Total tuples:', len(_densities)) print('Pearson correlation of densities and correlations:') print(compute_lib.correlation(_densities, _correlations, _with_p_value=True)) # plot _fig = go.Figure( data=go.Scatter( x=_densities, y=_correlations, mode='markers', marker={ 'size': 5, 'color': '#ea8500' }, showlegend=False ), layout={ 'xaxis': { 'title': 'Fiber density (z-score)', 'zeroline': False, 'range': [-1.1, 15.2], # 'tickmode': 'array', # 'tickvals': [-1, -0.5, 0, 0.5, 1] }, 'yaxis': { 'title': 'Correlation', 'zeroline': False, 'range': [-1.1, 1.2], 'tickmode': 'array', 'tickvals': [-1, -0.5, 0, 0.5, 1] }, 'shapes': [ { 'type': 'line', 'x0': 0, 'y0': -1, 'x1': 0, 'y1': 1, 'line': { 'color': 'black', 'width': 2 } }, { 'type': 'line', 'x0': 0, 'y0': -1, 'x1': 15, 'y1': -1, 'line': { 'color': 'black', 'width': 2 } } ] } ) save.to_html( _fig=_fig, _path=os.path.join(paths.PLOTS, save.get_module_name()), _filename='plot_real_' + str(_real_cells) + '_static_' + str(_static) + '_band_' + str(_band) + '_high_time_' + str(_high_temporal_resolution) + '_y_density_' + str(OFFSET_Y_DENSITY) + '_y_correlation_' + str(OFFSET_Y_CORRELATION) )
def main(): _experiments = all_experiments() _experiments = filtering.by_categories(_experiments=_experiments, _is_single_cell=False, _is_high_temporal_resolution=False, _is_bleb=False, _is_dead_dead=False, _is_live_dead=False, _is_bead=False, _is_metastasis=False) _tuples = load.experiments_groups_as_tuples(_experiments) _tuples = filtering.by_time_frames_amount( _tuples, compute.density_time_frame(_experiments[0])) _tuples = filtering.by_real_pairs(_tuples) _tuples = filtering.by_band(_tuples) _tuples = filtering.by_pair_distance_range(_tuples, PAIR_DISTANCE_RANGE) print('Total tuples:', len(_tuples)) _arguments = [] for _tuple in _tuples: _experiment, _series_id, _group = _tuple _latest_time_frame = compute.latest_time_frame_before_overlapping( _experiment, _series_id, _group, OFFSET_X) for _cell_id in ['left_cell', 'right_cell']: _arguments.append({ 'experiment': _experiment, 'series_id': _series_id, 'group': _group, 'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER, 'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER, 'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER, 'offset_x': OFFSET_X, 'offset_y': OFFSET_Y, 'offset_z': OFFSET_Z, 'cell_id': _cell_id, 'direction': 'inside', 'time_points': _latest_time_frame }) _windows_dictionary, _windows_to_compute = \ compute.windows(_arguments, _keys=['experiment', 'series_id', 'group', 'cell_id']) _fiber_densities = compute.fiber_densities(_windows_to_compute, _subtract_border=True) _experiments_fiber_densities = { _key: [_fiber_densities[_tuple] for _tuple in _windows_dictionary[_key]] for _key in _windows_dictionary } _kpss_y_arrays = [[] for _i in DERIVATIVES] _adf_y_arrays = [[] for _i in DERIVATIVES] for _tuple in tqdm(_tuples, desc='Experiments loop'): _experiment, _series_id, _group = _tuple _properties = load.group_properties(_experiment, _series_id, _group) _left_cell_fiber_densities = _experiments_fiber_densities[( _experiment, _series_id, _group, 'left_cell')] _left_cell_fiber_densities = compute.remove_blacklist( _experiment, _series_id, _properties['cells_ids']['left_cell'], _left_cell_fiber_densities) _right_cell_fiber_densities = _experiments_fiber_densities[( _experiment, _series_id, _group, 'right_cell')] _right_cell_fiber_densities = compute.remove_blacklist( _experiment, _series_id, _properties['cells_ids']['right_cell'], _right_cell_fiber_densities) if not OUT_OF_BOUNDARIES: _left_cell_fiber_densities = \ compute.longest_fiber_densities_ascending_sequence(_left_cell_fiber_densities) _right_cell_fiber_densities = \ compute.longest_fiber_densities_ascending_sequence(_right_cell_fiber_densities) else: _left_cell_fiber_densities = [ _fiber_density[0] for _fiber_density in _left_cell_fiber_densities ] _right_cell_fiber_densities = [ _fiber_density[0] for _fiber_density in _right_cell_fiber_densities ] # ignore small arrays _minimum_time_frames_for_correlation = compute.minimum_time_frames_for_correlation( _experiment) if len(_left_cell_fiber_densities) < _minimum_time_frames_for_correlation or \ len(_right_cell_fiber_densities) < _minimum_time_frames_for_correlation: continue for _derivative_index, _derivative in enumerate(DERIVATIVES): for _cell_fiber_densities in [ _left_cell_fiber_densities, _right_cell_fiber_densities ]: _cell_fiber_densities_derivative = compute_lib.derivative( _cell_fiber_densities, _n=_derivative) _, _kpss_p_value, _, _ = kpss(_cell_fiber_densities_derivative, nlags='legacy') _kpss_y_arrays[_derivative_index].append(_kpss_p_value) _, _adf_p_value, _, _, _, _ = adfuller( _cell_fiber_densities_derivative) _adf_y_arrays[_derivative_index].append(_adf_p_value) print('Total pairs:', len(_kpss_y_arrays[0]) / 2) # print results print('KPSS:') for _derivative_index, _derivative in enumerate(DERIVATIVES): _stationary_count = len([ _value for _value in _kpss_y_arrays[_derivative_index] if _value > 0.05 ]) print( 'Derivative:', _derivative, 'Stationary:', str(_stationary_count / len(_kpss_y_arrays[_derivative_index]) * 100) + '%') print('ADF:') for _derivative_index, _derivative in enumerate(DERIVATIVES): _stationary_count = len([ _value for _value in _adf_y_arrays[_derivative_index] if _value < 0.05 ]) print( 'Derivative:', _derivative, 'Stationary:', str(_stationary_count / len(_adf_y_arrays[_derivative_index]) * 100) + '%') # plot _colors_array = config.colors(3) for _test_name, _y_title, _y_tickvals, _p_value_line, _y_arrays in \ zip( ['kpss', 'adf'], ['KPSS test p-value', 'ADF test p-value'], [[0.05, 0.1], [0.05, 1]], [0.05, 0.05], [_kpss_y_arrays, _adf_y_arrays] ): _fig = go.Figure(data=[ go.Box(y=_y, name=_derivative, boxpoints='all', jitter=1, pointpos=0, line={'width': 1}, fillcolor='white', marker={ 'size': 10, 'color': _color }, opacity=0.7, showlegend=False) for _y, _derivative, _color in zip( _y_arrays, DERIVATIVES_TEXT, _colors_array) ], layout={ 'xaxis': { 'title': 'Fiber density derivative', 'zeroline': False }, 'yaxis': { 'title': _y_title, 'zeroline': False, 'tickmode': 'array', 'tickvals': _y_tickvals }, 'shapes': [{ 'type': 'line', 'x0': DERIVATIVES[0] - 0.75, 'y0': _p_value_line, 'x1': DERIVATIVES[-1] + 0.75, 'y1': _p_value_line, 'line': { 'color': 'red', 'width': 2, 'dash': 'dash' } }] }) save.to_html(_fig=_fig, _path=os.path.join(paths.PLOTS, save.get_module_name()), _filename='plot_' + _test_name)
def compute_experiments_data(): _experiments = all_experiments() _experiments = experiments_filtering.by_categories( _experiments=_experiments, _is_single_cell=False, _is_high_temporal_resolution=False, _is_bleb=False, _is_dead_dead=False, _is_live_dead=False, _is_bead=False, _is_metastasis=False ) _tuples = experiments_load.experiments_groups_as_tuples(_experiments) _tuples = experiments_filtering.by_time_frames_amount(_tuples, EXPERIMENTS_TIME_FRAMES) _tuples = experiments_filtering.by_real_pairs(_tuples) _tuples = experiments_filtering.by_pair_distance_range(_tuples, PAIR_DISTANCE_RANGE) _tuples = experiments_filtering.by_band(_tuples) print('Total tuples:', len(_tuples)) _arguments = [] for _tuple in _tuples: _experiment, _series_id, _group = _tuple _time_frame = experiments_compute.density_time_frame(_experiment) for _cell_id in ['left_cell', 'right_cell']: _arguments.append({ 'experiment': _experiment, 'series_id': _series_id, 'group': _group, 'length_x': experiments_config.QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER, 'length_y': experiments_config.QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER, 'length_z': experiments_config.QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER, 'offset_x': OFFSET_X, 'offset_y': OFFSET_Y, 'offset_z': OFFSET_Z, 'cell_id': _cell_id, 'direction': 'inside', 'time_points': _time_frame }) _windows_dictionary, _windows_to_compute = \ experiments_compute.windows(_arguments, _keys=['experiment', 'series_id', 'group', 'cell_id']) _fiber_densities = experiments_compute.fiber_densities(_windows_to_compute) _experiments_fiber_densities = [[] for _i in range(EXPERIMENTS_TIME_FRAMES)] for _tuple in tqdm(_tuples, desc='Experiments loop'): _experiment, _series_id, _group = _tuple _normalization = experiments_load.normalization_series_file_data(_experiment, _series_id) for _time_frame in range(EXPERIMENTS_TIME_FRAMES): for _cell_id in ['left_cell', 'right_cell']: _window_tuple = _windows_dictionary[(_experiment, _series_id, _group, _cell_id)][_time_frame] _fiber_density = _fiber_densities[_window_tuple] if not OUT_OF_BOUNDARIES and _fiber_density[1]: continue _normalized_fiber_density = compute_lib.z_score( _x=_fiber_density[0], _average=_normalization['average'], _std=_normalization['std'] ) if not np.isnan(_normalized_fiber_density): _experiments_fiber_densities[_time_frame].append(_normalized_fiber_density) print('Total experiments pairs:', len(_experiments_fiber_densities[0])) return _experiments_fiber_densities
def main(): _experiments = all_experiments() _experiments = filtering.by_categories( _experiments=_experiments, _is_single_cell=False, _is_high_temporal_resolution=False, _is_bleb=False, _is_dead_dead=False, _is_live_dead=False, _is_bead=False, _is_metastasis=False ) _tuples = load.experiments_groups_as_tuples(_experiments) _tuples = filtering.by_real_pairs(_tuples) _tuples = filtering.by_band(_tuples) _tuples = filtering.by_time_frames_amount(_tuples, MINIMUM_TIME_FRAMES) _tuples = filtering.by_pair_distance_range(_tuples, [MINIMUM_PAIR_DISTANCE, sys.maxsize]) _triplets = filtering.by_triplets(_tuples) print('Total triplets:', len(_triplets)) _arguments = [] for _triplet in _triplets: for _tuple in _triplet: _experiment, _series_id, _group = _tuple _latest_time_frame = compute.latest_time_frame_before_overlapping(_experiment, _series_id, _group, OFFSET_X) for _cell_id in ['left_cell', 'right_cell']: _arguments.append({ 'experiment': _experiment, 'series_id': _series_id, 'group': _group, 'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER, 'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER, 'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER, 'offset_x': OFFSET_X, 'offset_y': OFFSET_Y, 'offset_z': OFFSET_Z, 'cell_id': _cell_id, 'direction': 'inside', 'time_points': _latest_time_frame }) _windows_dictionary, _windows_to_compute = compute.windows(_arguments, _keys=['experiment', 'series_id', 'group', 'cell_id']) _fiber_densities = compute.fiber_densities(_windows_to_compute, _subtract_border=True) _experiments_fiber_densities = { _key: [_fiber_densities[_tuple] for _tuple in _windows_dictionary[_key]] for _key in _windows_dictionary } _same_correlations_arrays = [[] for _i in _triplets] _different_correlations_arrays = [[] for _i in _triplets] _names_array = [] for _triplet_index, _triplet in enumerate(_triplets): for _same_index in tqdm(range(len(_triplet)), desc='Main loop'): _same_tuple = _triplet[_same_index] _same_experiment, _same_series, _same_group = _same_tuple _same_left_cell_fiber_densities = \ _experiments_fiber_densities[ (_same_experiment, _same_series, _same_group, 'left_cell') ] _same_right_cell_fiber_densities = \ _experiments_fiber_densities[ (_same_experiment, _same_series, _same_group, 'right_cell') ] _same_properties = \ load.group_properties(_same_experiment, _same_series, _same_group) _same_left_cell_fiber_densities = compute.remove_blacklist( _same_experiment, _same_series, _same_properties['cells_ids']['left_cell'], _same_left_cell_fiber_densities ) _same_right_cell_fiber_densities = compute.remove_blacklist( _same_experiment, _same_series, _same_properties['cells_ids']['right_cell'], _same_right_cell_fiber_densities ) _same_left_cell_fiber_densities_filtered, _same_right_cell_fiber_densities_filtered = \ compute.longest_same_indices_shared_in_borders_sub_array( _same_left_cell_fiber_densities, _same_right_cell_fiber_densities ) _same_correlation = compute_lib.correlation( compute_lib.derivative(_same_left_cell_fiber_densities_filtered, _n=DERIVATIVE), compute_lib.derivative(_same_right_cell_fiber_densities_filtered, _n=DERIVATIVE) ) for _different_index in range(len(_triplet)): if _same_index != _different_index: _different_tuple = _triplet[_different_index] _different_experiment, _different_series, _different_group = \ _different_tuple for _same_cell_id, _different_cell_id in product(['left_cell', 'right_cell'], ['left_cell', 'right_cell']): _same_fiber_densities = _experiments_fiber_densities[( _same_experiment, _same_series, _same_group, _same_cell_id )] _different_fiber_densities = _experiments_fiber_densities[( _different_experiment, _different_series, _different_group, _different_cell_id )] _different_properties = load.group_properties( _different_experiment, _different_series, _different_group ) _same_fiber_densities = compute.remove_blacklist( _same_experiment, _same_series, _same_properties['cells_ids'][_same_cell_id], _same_fiber_densities ) _different_fiber_densities = compute.remove_blacklist( _different_experiment, _different_series, _different_properties['cells_ids'][_different_cell_id], _different_fiber_densities ) _same_fiber_densities_filtered, _different_fiber_densities_filtered = \ compute.longest_same_indices_shared_in_borders_sub_array( _same_fiber_densities, _different_fiber_densities ) _different_correlation = compute_lib.correlation( compute_lib.derivative(_same_fiber_densities_filtered, _n=DERIVATIVE), compute_lib.derivative(_different_fiber_densities_filtered, _n=DERIVATIVE) ) _same_correlations_arrays[_triplet_index].append(_same_correlation) _different_correlations_arrays[_triplet_index].append(_different_correlation) _names_array.append('Triplet #' + str(_triplet_index + 1)) print('Total points:', len(np.array(_same_correlations_arrays).flatten())) _same_minus_different = \ np.array(_same_correlations_arrays).flatten() - np.array(_different_correlations_arrays).flatten() print('Wilcoxon of same minus different around the zero:') print(wilcoxon(_same_minus_different)) print('Higher same amount:', (_same_minus_different > 0).sum() / len(_same_minus_different)) # plot _colors_array = ['green', 'blue', config.colors(1)] _fig = go.Figure( data=[ go.Scatter( x=_same_correlations_array, y=_different_correlations_array, name=_name, mode='markers', marker={ 'size': 15, 'color': _color }, opacity=0.7 ) for _same_correlations_array, _different_correlations_array, _name, _color in zip(_same_correlations_arrays, _different_correlations_arrays, _names_array, _colors_array) ], layout={ 'xaxis': { 'title': 'Same network correlation', 'zeroline': False, 'range': [-1.1, 1.2], 'tickmode': 'array', 'tickvals': [-1, -0.5, 0, 0.5, 1] }, 'yaxis': { 'title': 'Different network correlation', 'zeroline': False, 'range': [-1.1, 1.2], 'tickmode': 'array', 'tickvals': [-1, -0.5, 0, 0.5, 1] }, 'legend': { 'xanchor': 'left', 'x': 0.1, 'yanchor': 'top', 'bordercolor': 'black', 'borderwidth': 2, 'bgcolor': 'white' }, 'shapes': [ { 'type': 'line', 'x0': -1, 'y0': -1, 'x1': -1, 'y1': 1, 'line': { 'color': 'black', 'width': 2 } }, { 'type': 'line', 'x0': -1, 'y0': -1, 'x1': 1, 'y1': -1, 'line': { 'color': 'black', 'width': 2 } }, { 'type': 'line', 'x0': -1, 'y0': -1, 'x1': 1, 'y1': 1, 'line': { 'color': 'red', 'width': 2 } } ] } ) save.to_html( _fig=_fig, _path=os.path.join(paths.PLOTS, save.get_module_name()), _filename='plot' )
def main(): _experiments = all_experiments() _experiments = filtering.by_categories(_experiments=_experiments, _is_single_cell=False, _is_high_temporal_resolution=False, _is_bleb=False, _is_dead_dead=False, _is_live_dead=False, _is_bead=False, _is_metastasis=False) _tuples = load.experiments_groups_as_tuples(_experiments) _tuples = filtering.by_time_frames_amount( _tuples, compute.density_time_frame(_experiments[0])) _tuples = filtering.by_real_pairs(_tuples) _tuples = filtering.by_band(_tuples) _max_offsets_x = [] _arguments = [] for _tuple in _tuples: _experiment, _series_id, _group = _tuple _time_frame = compute.minimum_time_frames_for_correlation(_experiment) _pair_distance = \ compute.pair_distance_in_cell_size_time_frame(_experiment, _series_id, _group, _time_frame=_time_frame - 1) _offsets_x = \ np.arange(start=0, stop=_pair_distance / 2 - 0.5 - QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER, step=OFFSET_X_STEP) if len(_offsets_x) > len(_max_offsets_x): _max_offsets_x = _offsets_x for _offset_x in _offsets_x: for _cell_id in ['left_cell', 'right_cell']: _arguments.append({ 'experiment': _experiment, 'series_id': _series_id, 'group': _group, 'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER, 'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER, 'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER, 'offset_x': _offset_x, 'offset_y': OFFSET_Y, 'offset_z': OFFSET_Z, 'cell_id': _cell_id, 'direction': 'inside', 'time_point': _time_frame - 1 }) _windows_dictionary, _windows_to_compute = \ compute.windows(_arguments, _keys=['experiment', 'series_id', 'group', 'offset_x', 'cell_id']) _fiber_densities = compute.fiber_densities(_windows_to_compute) _x_array = [] _y_array = [] _n_array = [] _p_value_array = [] for _offset_x in _max_offsets_x: _pair_distances = [] _z_scores = [] for _tuple in _tuples: _experiment, _series_id, _group = _tuple for _cell_id in ['left_cell', 'right_cell']: if (_experiment, _series_id, _group, _offset_x, _cell_id) in _windows_dictionary: _normalization = load.normalization_series_file_data( _experiment, _series_id) _window_tuple = _windows_dictionary[(_experiment, _series_id, _group, _offset_x, _cell_id)][0] _fiber_density = _fiber_densities[_window_tuple] if not OUT_OF_BOUNDARIES and _fiber_density[1]: continue _normalized_fiber_density = compute_lib.z_score( _x=_fiber_density[0], _average=_normalization['average'], _std=_normalization['std']) if not np.isnan(_normalized_fiber_density): _pair_distances.append( compute.pair_distance_in_cell_size_time_frame( _experiment, _series_id, _group, _time_frame=0)) _z_scores.append(_normalized_fiber_density) if len(_pair_distances) > 2: _x_array.append(round(_offset_x, 1)) _correlation = compute_lib.correlation(_pair_distances, _z_scores, _with_p_value=True) _y_array.append(round(_correlation[0], 2)) _n_array.append(len(_pair_distances)) _p_value_array.append(round(_correlation[1], 2)) print('Pearson:') print(compute_lib.correlation(_x_array, _y_array, _with_p_value=True)) # plot _significant_x_array = [ _x for _x, _p_value in zip(_x_array, _p_value_array) if _p_value < 0.05 ] _fig = go.Figure(data=[ go.Scatter(x=_x_array, y=_y_array, mode='markers', marker={ 'size': 15, 'color': 'black' }, showlegend=False), go.Scatter(x=_significant_x_array, y=[-0.79] * len(_significant_x_array), mode='text', text='*', textfont={'color': 'red'}, showlegend=False) ], layout={ 'xaxis': { 'title': 'Window distance (cell diameter)', 'zeroline': False }, 'yaxis': { 'title': 'Correlation:<br>pair distance vs. fiber density', 'zeroline': False, 'range': [-0.82, 0.3], 'tickmode': 'array', 'tickvals': [-0.75, -0.25, 0.25] }, 'shapes': [{ 'type': 'line', 'x0': -0.2, 'y0': -0.8, 'x1': 3.2, 'y1': -0.8, 'line': { 'color': 'black', 'width': 2 } }, { 'type': 'line', 'x0': -0.2, 'y0': -0.8, 'x1': -0.2, 'y1': 0.3, 'line': { 'color': 'black', 'width': 2 } }] }) save.to_html(_fig=_fig, _path=os.path.join(paths.PLOTS, save.get_module_name()), _filename='plot') # table print('Window distance (cell diameter)', 'Correlation: pair distance vs. fiber density', 'N', 'P-value', sep='\t') for _x, _y, _n, _p_value in zip(_x_array, _y_array, _n_array, _p_value_array): print(_x, _y, _n, _p_value, sep='\t')
def main(): _experiments = all_experiments() _experiments = filtering.by_categories(_experiments=_experiments, _is_single_cell=True, _is_high_temporal_resolution=False, _is_bleb=False, _is_dead_dead=False, _is_live_dead=False, _is_bead=False, _is_metastasis=False) _tuples = load.experiments_groups_as_tuples(_experiments) _tuples = filtering.by_time_frames_amount( _tuples, compute.density_time_frame(_experiments[0])) _tuples = filtering.by_main_cell(_tuples) _arguments = [] for _tuple in _tuples: _experiment, _series_id, _group = _tuple _time_frame = compute.density_time_frame(_experiment) for _direction in ['left', 'right']: _arguments.append({ 'experiment': _experiment, 'series_id': _series_id, 'group': _group, 'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER, 'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER, 'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER, 'offset_x': OFFSET_X, 'offset_y': OFFSET_Y, 'offset_z': OFFSET_Z, 'cell_id': 'cell', 'direction': _direction, 'time_points': _time_frame }) _windows_dictionary, _windows_to_compute = \ compute.windows(_arguments, _keys=['experiment', 'series_id', 'group', 'direction']) _fiber_densities = compute.fiber_densities(_windows_to_compute, _subtract_border=True) _tuples = organize.by_single_cell_id(_tuples) print('Total tuples:', len(_tuples)) _experiments_ids = list(_tuples.keys()) _y_arrays = [[] for _i in DERIVATIVES] for _index_1 in tqdm(range(len(_experiments_ids)), desc='Main loop'): _tuple_1 = _experiments_ids[_index_1] _experiment_1, _series_id_1, _cell_id_1 = _tuple_1 _fiber_densities_1 = compute_single_cell_mean( _experiment=_experiment_1, _series_id=_series_id_1, _cell_tuples=_tuples[_tuple_1], _windows_dictionary=_windows_dictionary, _fiber_densities=_fiber_densities) for _index_2 in range(_index_1 + 1, len(_experiments_ids)): _tuple_2 = _experiments_ids[_index_2] _experiment_2, _series_id_2, _cell_id_2 = _tuple_2 _fiber_densities_2 = compute_single_cell_mean( _experiment=_experiment_2, _series_id=_series_id_2, _cell_tuples=_tuples[_tuple_2], _windows_dictionary=_windows_dictionary, _fiber_densities=_fiber_densities) for _derivative_index, _derivative in enumerate(DERIVATIVES): _y_arrays[_derivative_index].append( compute_lib.correlation( compute_lib.derivative(_fiber_densities_1, _n=_derivative), compute_lib.derivative(_fiber_densities_2, _n=_derivative))) print('Total points:', len(_y_arrays[0])) print('Wilcoxon around the zero') for _y_array, _derivative in zip(_y_arrays, DERIVATIVES): print('Derivative:', _derivative, wilcoxon(_y_array)) # plot _colors_array = config.colors(3) _fig = go.Figure(data=[ go.Box(y=_y, name=_derivative, boxpoints='all', jitter=1, pointpos=0, line={'width': 1}, fillcolor='white', marker={ 'size': 10, 'color': _color }, opacity=0.7, showlegend=False) for _y, _derivative, _color in zip( _y_arrays, DERIVATIVES_TEXT, _colors_array) ], layout={ 'xaxis': { 'title': 'Fiber density derivative', 'zeroline': False }, 'yaxis': { 'title': 'Correlation', 'range': [-1, 1], 'zeroline': False, 'tickmode': 'array', 'tickvals': [-1, -0.5, 0, 0.5, 1] } }) save.to_html(_fig=_fig, _path=os.path.join(paths.PLOTS, save.get_module_name()), _filename='plot')
def main(): _experiments = all_experiments() _experiments = filtering.by_categories(_experiments=_experiments, _is_single_cell=True, _is_high_temporal_resolution=False, _is_bleb=False, _is_dead_dead=False, _is_live_dead=False, _is_bead=False, _is_metastasis=False) _tuples = load.experiments_groups_as_tuples(_experiments) _tuples = filtering.by_time_frames_amount( _tuples, compute.density_time_frame(_experiments[0])) _tuples = filtering.by_main_cell(_tuples) _arguments = [] for _tuple in _tuples: _experiment, _series_id, _group = _tuple _time_frame = compute.density_time_frame(_experiment) for _direction in ['left', 'right']: _arguments.append({ 'experiment': _experiment, 'series_id': _series_id, 'group': _group, 'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER, 'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER, 'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER, 'offset_x': OFFSET_X, 'offset_y': OFFSET_Y, 'offset_z': OFFSET_Z, 'cell_id': 'cell', 'direction': _direction, 'time_points': _time_frame }) _windows_dictionary, _windows_to_compute = \ compute.windows(_arguments, _keys=['experiment', 'series_id', 'group', 'direction']) _fiber_densities = compute.fiber_densities(_windows_to_compute, _subtract_border=True) _tuples = organize.by_single_cell_id(_tuples) print('Total experiments:', len(_tuples)) _kpss_y_arrays = [[] for _i in DERIVATIVES] _adf_y_arrays = [[] for _i in DERIVATIVES] for _tuple in tqdm(_tuples, desc='Experiments loop'): _experiment, _series_id, _cell_id = _tuple _cell_fiber_densities = compute_single_cell_mean( _experiment=_experiment, _series_id=_series_id, _cell_tuples=_tuples[_tuple], _windows_dictionary=_windows_dictionary, _fiber_densities=_fiber_densities) for _derivative_index, _derivative in enumerate(DERIVATIVES): _cell_fiber_densities_derivative = compute_lib.derivative( _cell_fiber_densities, _n=_derivative) _, _kpss_p_value, _, _ = kpss(_cell_fiber_densities_derivative, nlags='legacy') _kpss_y_arrays[_derivative_index].append(_kpss_p_value) _, _adf_p_value, _, _, _, _ = adfuller( _cell_fiber_densities_derivative) _adf_y_arrays[_derivative_index].append(_adf_p_value) print('Total cells:', len(_kpss_y_arrays[0])) # print results print('KPSS:') for _derivative_index, _derivative in enumerate(DERIVATIVES): _stationary_count = len([ _value for _value in _kpss_y_arrays[_derivative_index] if _value > 0.05 ]) print( 'Derivative:', _derivative, 'Stationary:', str(_stationary_count / len(_kpss_y_arrays[_derivative_index]) * 100) + '%') print('ADF:') for _derivative_index, _derivative in enumerate(DERIVATIVES): _stationary_count = len([ _value for _value in _adf_y_arrays[_derivative_index] if _value < 0.05 ]) print( 'Derivative:', _derivative, 'Stationary:', str(_stationary_count / len(_adf_y_arrays[_derivative_index]) * 100) + '%') # plot _colors_array = config.colors(3) for _test_name, _y_title, _y_tickvals, _p_value_line, _y_arrays in \ zip( ['kpss', 'adf'], ['KPSS test p-value', 'ADF test p-value'], [[0.05, 0.1], [0.05, 1]], [0.05, 0.05], [_kpss_y_arrays, _adf_y_arrays] ): _fig = go.Figure(data=[ go.Box(y=_y, name=_derivative, boxpoints='all', jitter=1, pointpos=0, line={'width': 1}, fillcolor='white', marker={ 'size': 10, 'color': _color }, opacity=0.7, showlegend=False) for _y, _derivative, _color in zip( _y_arrays, DERIVATIVES_TEXT, _colors_array) ], layout={ 'xaxis': { 'title': 'Fiber density derivative', 'zeroline': False }, 'yaxis': { 'title': _y_title, 'zeroline': False, 'tickmode': 'array', 'tickvals': _y_tickvals }, 'shapes': [{ 'type': 'line', 'x0': DERIVATIVES[0] - 0.75, 'y0': _p_value_line, 'x1': DERIVATIVES[-1] + 0.75, 'y1': _p_value_line, 'line': { 'color': 'red', 'width': 2, 'dash': 'dash' } }] }) save.to_html(_fig=_fig, _path=os.path.join(paths.PLOTS, save.get_module_name()), _filename='plot_' + _test_name)
def main(): _experiments = all_experiments() _experiments = filtering.by_categories(_experiments=_experiments, _is_single_cell=False, _is_high_temporal_resolution=False, _is_bleb=False, _is_dead_dead=False, _is_live_dead=False, _is_bead=False, _is_metastasis=False) _tuples = load.experiments_groups_as_tuples(_experiments) _tuples = filtering.by_time_frames_amount( _tuples, compute.density_time_frame(_experiments[0])) _tuples = filtering.by_real_pairs(_tuples) _tuples = filtering.by_band(_tuples) print('Total tuples:', len(_tuples)) _arguments = [] for _tuple in _tuples: _experiment, _series_id, _group = _tuple _time_frame = compute.density_time_frame(_experiment) _pair_distance = \ compute.pair_distance_in_cell_size_time_frame(_experiment, _series_id, _group, _time_frame=_time_frame - 1) for _offset_x in OFFSETS_X: if _pair_distance / 2 - 0.5 - QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER >= _offset_x: for _cell_id in ['left_cell', 'right_cell']: _arguments.append({ 'experiment': _experiment, 'series_id': _series_id, 'group': _group, 'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER, 'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER, 'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER, 'offset_x': _offset_x, 'offset_y': OFFSET_Y, 'offset_z': OFFSET_Z, 'cell_id': _cell_id, 'direction': 'inside', 'time_point': _time_frame - 1 }) _windows_dictionary, _windows_to_compute = \ compute.windows(_arguments, _keys=['experiment', 'series_id', 'group', 'cell_id', 'offset_x']) _fiber_densities = compute.fiber_densities(_windows_to_compute) for _offset_x in OFFSETS_X: _x_array = [] _y_array = [] for _tuple in _tuples: _experiment, _series_id, _group = _tuple for _cell_id in ['left_cell', 'right_cell']: if (_experiment, _series_id, _group, _cell_id, _offset_x) in _windows_dictionary: _pair_distance = \ compute.pair_distance_in_cell_size_time_frame(_experiment, _series_id, _group, _time_frame=0) _normalization = load.normalization_series_file_data( _experiment, _series_id) _window_tuple = _windows_dictionary[(_experiment, _series_id, _group, _cell_id, _offset_x)][0] _fiber_density = _fiber_densities[_window_tuple] if not OUT_OF_BOUNDARIES and _fiber_density[1]: continue _normalized_fiber_density = compute_lib.z_score( _x=_fiber_density[0], _average=_normalization['average'], _std=_normalization['std']) if not np.isnan(_normalized_fiber_density): _x_array.append(_pair_distance) _y_array.append(_normalized_fiber_density) print('Offset x (cell diameter):', _offset_x) print('Total pairs:', len(_x_array)) print(compute_lib.correlation(_x_array, _y_array, _with_p_value=True)) # plot _fig = go.Figure(data=go.Scatter(x=_x_array, y=_y_array, mode='markers', marker={ 'size': 15, 'color': 'black' }), layout={ 'xaxis': { 'title': 'Pair distance (cell diameter)', 'zeroline': False }, 'yaxis': { 'title': 'Fiber density (z-score)', 'zeroline': False, 'range': [-2.2, 13], 'tickmode': 'array', 'tickvals': [0, 4, 8, 12] }, 'shapes': [{ 'type': 'line', 'x0': 4.5, 'y0': -2, 'x1': 9.5, 'y1': -2, 'line': { 'color': 'black', 'width': 2 } }, { 'type': 'line', 'x0': 4.5, 'y0': -2, 'x1': 4.5, 'y1': 13, 'line': { 'color': 'black', 'width': 2 } }] }) save.to_html(_fig=_fig, _path=os.path.join(paths.PLOTS, save.get_module_name()), _filename='plot_offset_x_' + str(_offset_x))
def compute_cell_pairs(): _x_array = [] _y_array = [] _names_array = [] for _distances_range in PAIR_DISTANCE_RANGES: print('Pair distance range:', str(_distances_range)) _experiments = all_experiments() _experiments = filtering.by_categories( _experiments=_experiments, _is_single_cell=False, _is_high_temporal_resolution=False, _is_bleb=False, _is_dead_dead=False, _is_live_dead=False, _is_bead=False, _is_metastasis=False) _tuples = load.experiments_groups_as_tuples(_experiments) _tuples = filtering.by_time_frames_amount( _tuples, compute.density_time_frame(_experiments[0])) _tuples = filtering.by_real_pairs(_tuples) _tuples = filtering.by_pair_distance_range(_tuples, _distances_range) _tuples = filtering.by_band(_tuples) print('Total tuples:', len(_tuples)) _max_offsets_x = [] _arguments = [] for _tuple in _tuples: _experiment, _series_id, _group = _tuple _time_frame = compute.minimum_time_frames_for_correlation( _experiment) _pair_distance = \ compute.pair_distance_in_cell_size_time_frame(_experiment, _series_id, _group, _time_frame - 1) _offsets_x = np.arange( start=0, stop=_pair_distance / 2 - 0.5 - QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER, step=OFFSET_X_STEP) if len(_offsets_x) > len(_max_offsets_x): _max_offsets_x = _offsets_x for _offset_x in _offsets_x: for _cell_id in ['left_cell', 'right_cell']: _arguments.append({ 'experiment': _experiment, 'series_id': _series_id, 'group': _group, 'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER, 'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER, 'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER, 'offset_x': _offset_x, 'offset_y': OFFSET_Y, 'offset_z': OFFSET_Z, 'cell_id': _cell_id, 'direction': 'inside', 'time_point': _time_frame - 1 }) _windows_dictionary, _windows_to_compute = \ compute.windows(_arguments, _keys=['experiment', 'series_id', 'group', 'offset_x', 'cell_id']) _fiber_densities = compute.fiber_densities(_windows_to_compute) _pair_distance_fiber_densities = [[] for _i in range(len(_max_offsets_x))] for _tuple in _tuples: _experiment, _series_id, _group = _tuple for _offset_x_index, _offset_x in enumerate(_max_offsets_x): for _cell_id in ['left_cell', 'right_cell']: if (_experiment, _series_id, _group, _offset_x, _cell_id) in _windows_dictionary: _normalization = load.normalization_series_file_data( _experiment, _series_id) _window_tuple = _windows_dictionary[(_experiment, _series_id, _group, _offset_x, _cell_id)][0] _fiber_density = _fiber_densities[_window_tuple] if not OUT_OF_BOUNDARIES and _fiber_density[1]: continue _normalized_fiber_density = compute_lib.z_score( _x=_fiber_density[0], _average=_normalization['average'], _std=_normalization['std']) if not np.isnan(_normalized_fiber_density): _pair_distance_fiber_densities[ _offset_x_index].append( _normalized_fiber_density) _x_array.append(_max_offsets_x) _y_array.append(_pair_distance_fiber_densities) _names_array.append('Pair distance ' + str(_distances_range[0]) + '-' + str(_distances_range[1])) return _names_array, _x_array, _y_array
def main(): print('Single Cell') _experiments = all_experiments() _experiments = filtering.by_categories(_experiments=_experiments, _is_single_cell=True, _is_high_temporal_resolution=False, _is_bleb=False, _is_dead_dead=False, _is_live_dead=False, _is_bead=False, _is_metastasis=False) _tuples = load.experiments_groups_as_tuples(_experiments) _tuples = filtering.by_time_frames_amount( _tuples, compute.minimum_time_frames_for_correlation(_experiments[0])) _arguments = [] for _tuple in _tuples: _experiment, _series_id, _group = _tuple _time_frame = compute.minimum_time_frames_for_correlation(_experiment) for _offset_x, _direction in product(OFFSETS_X, ['left', 'right']): _arguments.append({ 'experiment': _experiment, 'series_id': _series_id, 'group': _group, 'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER, 'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER, 'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER, 'offset_x': _offset_x, 'offset_y': OFFSET_Y, 'offset_z': OFFSET_Z, 'cell_id': 'cell', 'direction': _direction, 'time_point': _time_frame - 1 }) _windows_dictionary, _windows_to_compute = \ compute.windows(_arguments, _keys=['experiment', 'series_id', 'group', 'offset_x', 'direction']) _fiber_densities = compute.fiber_densities(_windows_to_compute) _tuples = organize.by_single_cell_id(_tuples) _single_cell_fiber_densities = [[] for _i in range(len(OFFSETS_X))] for _tuple in _tuples: _experiment, _series_id, _cell_id = _tuple print('Experiment:', _experiment, 'Series ID:', _series_id, 'Cell ID:', _cell_id, sep='\t') _offset_index = 0 _normalization = load.normalization_series_file_data( _experiment, _series_id) for _offset_x in OFFSETS_X: _cell_fiber_densities = [] for _cell_tuple in _tuples[_tuple]: _, _, _group = _cell_tuple for _direction in ['left', 'right']: _window_tuple = _windows_dictionary[(_experiment, _series_id, _group, _offset_x, _direction)][0] _fiber_density = _fiber_densities[_window_tuple] if not OUT_OF_BOUNDARIES and _fiber_density[1]: continue _normalized_fiber_density = compute_lib.z_score( _x=_fiber_density[0], _average=_normalization['average'], _std=_normalization['std']) _cell_fiber_densities.append(_normalized_fiber_density) if len(_cell_fiber_densities) > 0: _single_cell_fiber_densities[_offset_index].append( np.mean(_cell_fiber_densities)) _offset_index += 1 print('Pairs') _experiments = all_experiments() _experiments = filtering.by_categories(_experiments=_experiments, _is_single_cell=False, _is_high_temporal_resolution=False, _is_bleb=False, _is_dead_dead=False, _is_live_dead=False, _is_bead=False, _is_metastasis=False) _tuples = load.experiments_groups_as_tuples(_experiments) _tuples = filtering.by_time_frames_amount( _tuples, compute.minimum_time_frames_for_correlation(_experiments[0])) _tuples = filtering.by_real_pairs(_tuples) _tuples = filtering.by_pair_distance(_tuples, PAIR_DISTANCE) _tuples = filtering.by_band(_tuples) _arguments = [] for _tuple in _tuples: _experiment, _series_id, _group = _tuple _time_frame = compute.minimum_time_frames_for_correlation(_experiment) for _offset_x in OFFSETS_X: _arguments.append({ 'experiment': _experiment, 'series_id': _series_id, 'group': _group, 'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER, 'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER, 'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER, 'offset_x': _offset_x, 'offset_y': OFFSET_Y, 'offset_z': OFFSET_Z, 'cell_id': 'left_cell', 'direction': 'inside', 'time_point': _time_frame - 1 }) _windows_dictionary, _windows_to_compute = \ compute.windows(_arguments, _keys=['experiment', 'series_id', 'group', 'offset_x']) _fiber_densities = compute.fiber_densities(_windows_to_compute) _pairs_fiber_densities = [[] for _i in range(len(OFFSETS_X))] for _tuple in _tuples: _experiment, _series_id, _group = _tuple print('Experiment:', _experiment, 'Series ID:', _series_id, 'Group:', _group, sep='\t') _offset_index = 0 _normalization = load.normalization_series_file_data( _experiment, _series_id) # take offsets based on pair distance _properties = load.group_properties(_experiment, _series_id, _group) _left_cell_coordinates = [ list(_properties['time_points'][0]['left_cell'] ['coordinates'].values()) ] _right_cell_coordinates = [ list(_properties['time_points'][0]['right_cell'] ['coordinates'].values()) ] _pair_distance = compute.pair_distance_in_cell_size( _experiment, _series_id, _left_cell_coordinates, _right_cell_coordinates) _edges_distance = _pair_distance - 1 _max_x_offset = _edges_distance - QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER for _offset_x in OFFSETS_X: if _offset_x > _max_x_offset: break _fiber_density = _fiber_densities[_windows_dictionary[( _experiment, _series_id, _group, _offset_x)][0]] if not OUT_OF_BOUNDARIES and _fiber_density[1]: continue _normalized_fiber_density = compute_lib.z_score( _x=_fiber_density[0], _average=_normalization['average'], _std=_normalization['std']) _pairs_fiber_densities[_offset_index].append( _normalized_fiber_density) _offset_index += 1 # plot _fig = go.Figure(data=[ go.Scatter(x=OFFSETS_X, y=[np.mean(_array) for _array in _pairs_fiber_densities], name='Pairs', error_y={ 'type': 'data', 'array': [np.std(_array) for _array in _pairs_fiber_densities], 'thickness': 1 }, mode='lines+markers', line={'dash': 'solid'}), go.Scatter( x=OFFSETS_X, y=[np.mean(_array) for _array in _single_cell_fiber_densities], name='Single Cell', error_y={ 'type': 'data', 'array': [np.std(_array) for _array in _single_cell_fiber_densities], 'thickness': 1 }, mode='lines+markers', line={'dash': 'dash'}) ], layout={ 'xaxis_title': 'Distance from left cell (cell size)', 'yaxis_title': 'Fiber density (z-score)' }) save.to_html(_fig=_fig, _path=os.path.join(paths.PLOTS, save.get_module_name()), _filename='plot_distance_' + str(PAIR_DISTANCE))
def compute_experiments_data(): _experiments = all_experiments() _experiments = experiments_filtering.by_categories( _experiments=_experiments, _is_single_cell=True, _is_high_temporal_resolution=False, _is_bleb=False, _is_dead_dead=False, _is_live_dead=False, _is_bead=False, _is_metastasis=False ) _tuples = experiments_load.experiments_groups_as_tuples(_experiments) _tuples = experiments_filtering.by_time_frames_amount(_tuples, EXPERIMENTS_TIME_FRAME) _tuples = experiments_filtering.by_main_cell(_tuples) _arguments = [] for _tuple in _tuples: _experiment, _series_id, _group = _tuple for _offset_x, _direction in product(OFFSETS_X, ['left', 'right']): _arguments.append({ 'experiment': _experiment, 'series_id': _series_id, 'group': _group, 'length_x': experiments_config.QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER, 'length_y': experiments_config.QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER, 'length_z': experiments_config.QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER, 'offset_x': _offset_x, 'offset_y': OFFSET_Y, 'offset_z': OFFSET_Z, 'cell_id': 'cell', 'direction': _direction, 'time_point': EXPERIMENTS_TIME_FRAME - 1 }) _windows_dictionary, _windows_to_compute = \ experiments_compute.windows(_arguments, _keys=['experiment', 'series_id', 'group', 'offset_x', 'direction']) _fiber_densities = experiments_compute.fiber_densities(_windows_to_compute) _tuples = experiments_organize.by_single_cell_id(_tuples) _experiments_fiber_densities = [[] for _i in range(len(OFFSETS_X))] for _tuple in tqdm(_tuples, desc='Experiments loop'): _experiment, _series_id, _cell_id = _tuple _normalization = experiments_load.normalization_series_file_data(_experiment, _series_id) for _offset_x_index, _offset_x in enumerate(OFFSETS_X): _cell_fiber_densities = [] for _cell_tuple in _tuples[_tuple]: _, _, _group = _cell_tuple for _direction in ['left', 'right']: _window_tuple = _windows_dictionary[(_experiment, _series_id, _group, _offset_x, _direction)][0] _fiber_density = _fiber_densities[_window_tuple] if not OUT_OF_BOUNDARIES and _fiber_density[1]: continue _normalized_fiber_density = compute_lib.z_score( _x=_fiber_density[0], _average=_normalization['average'], _std=_normalization['std'] ) if not np.isnan(_normalized_fiber_density): _cell_fiber_densities.append(_normalized_fiber_density) if len(_cell_fiber_densities) > 0: _experiments_fiber_densities[_offset_x_index].append(np.mean(_cell_fiber_densities)) return _experiments_fiber_densities
def main(_band=None, _high_temporal_resolution=True, _tuples_to_mark=None, _tuples_to_plot=None, _plots=None): if _plots is None: _plots = ['whiteness', 'granger'] _experiments = all_experiments() _experiments = filtering.by_categories( _experiments=_experiments, _is_single_cell=False, _is_high_temporal_resolution=_high_temporal_resolution, _is_bleb=False, _is_dead_dead=False, _is_live_dead=False, _is_bead=False, _is_metastasis=False) _tuples = load.experiments_groups_as_tuples(_experiments) _tuples = filtering.by_time_frames_amount(_tuples, _time_frames=MINIMUM_TIME_FRAMES) _tuples = filtering.by_pair_distance_range( _tuples, _distance_range=PAIR_DISTANCE_RANGE) _tuples = filtering.by_real_pairs(_tuples) _tuples = filtering.by_band(_tuples, _band=_band) print('Total tuples:', len(_tuples)) _arguments = [] for _tuple in _tuples: _experiment, _series_id, _group = _tuple _latest_time_frame = compute.latest_time_frame_before_overlapping( _experiment, _series_id, _group, OFFSET_X) for _cell_id in ['left_cell', 'right_cell']: _arguments.append({ 'experiment': _experiment, 'series_id': _series_id, 'group': _group, 'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER, 'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER, 'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER, 'offset_x': OFFSET_X, 'offset_y': OFFSET_Y, 'offset_z': OFFSET_Z, 'cell_id': _cell_id, 'direction': 'inside', 'time_points': _latest_time_frame }) _windows_dictionary, _windows_to_compute = compute.windows( _arguments, _keys=['experiment', 'series_id', 'group', 'cell_id']) _fiber_densities = compute.fiber_densities(_windows_to_compute, _subtract_border=True) _experiments_fiber_densities = { _key: [_fiber_densities[_tuple] for _tuple in _windows_dictionary[_key]] for _key in _windows_dictionary } _n_pairs = 0 _n_pairs_with_band = 0 _whiteness_p_values = [] _n_passed_whiteness_with_band = 0 _granger_causality_p_values = [] _n_passed_granger_causality_with_band = 0 _correlations = [] _time_lag_correlations = [] _end_fiber_densities = [] for _tuple in _tuples: _experiment, _series_id, _group = _tuple _left_cell_fiber_densities = \ _experiments_fiber_densities[(_experiment, _series_id, _group, 'left_cell')] _right_cell_fiber_densities = \ _experiments_fiber_densities[(_experiment, _series_id, _group, 'right_cell')] _properties = load.group_properties(_experiment, _series_id, _group) _left_cell_fiber_densities = compute.remove_blacklist( _experiment, _series_id, _properties['cells_ids']['left_cell'], _left_cell_fiber_densities) _right_cell_fiber_densities = compute.remove_blacklist( _experiment, _series_id, _properties['cells_ids']['right_cell'], _right_cell_fiber_densities) _left_cell_fiber_densities_filtered, _right_cell_fiber_densities_filtered = \ compute.longest_same_indices_shared_in_borders_sub_array( _left_cell_fiber_densities, _right_cell_fiber_densities) # ignore small arrays if len(_left_cell_fiber_densities_filtered) < MINIMUM_TIME_FRAMES: continue _n_pairs += 1 if _properties['band']: _n_pairs_with_band += 1 _start_time_frame = 0 for _left in _left_cell_fiber_densities: if _left[0] == _left_cell_fiber_densities_filtered[0]: break _start_time_frame += 1 # stationary test with warnings.catch_warnings(): warnings.simplefilter('ignore', category=InterpolationWarning) # find derivative for stationary for _derivative in range(10): _left_cell_fiber_densities_derivative = \ compute_lib.derivative(_left_cell_fiber_densities_filtered, _n=_derivative) _right_cell_fiber_densities_derivative = \ compute_lib.derivative(_right_cell_fiber_densities_filtered, _n=_derivative) if ADF_TEST: _, _left_cell_adf_p_value, _, _, _, _ = adfuller( _left_cell_fiber_densities_derivative) _, _right_cell_adf_p_value, _, _, _, _ = adfuller( _right_cell_fiber_densities_derivative) if _left_cell_adf_p_value > 0.05 or _right_cell_adf_p_value > 0.05: continue if KPSS_TEST: _, _left_cell_kpss_p_value, _, _ = kpss( _left_cell_fiber_densities_derivative, nlags='legacy') _, _right_cell_kpss_p_value, _, _ = kpss( _right_cell_fiber_densities_derivative, nlags='legacy') if _left_cell_kpss_p_value < 0.05 or _right_cell_kpss_p_value < 0.05: continue # stationary break # causality try: _x = pd.DataFrame(data=[[_left_value, _right_value] for _left_value, _right_value in zip( _left_cell_fiber_densities_derivative, _right_cell_fiber_densities_derivative) ], columns=['left', 'right']) # var model to retrieve lag _var_model = VAR(_x) _lag_order_results = _var_model.select_order() _estimators_lags = [ _lag_order_results.aic, _lag_order_results.bic, _lag_order_results.fpe, _lag_order_results.hqic ] _min_estimator_lag = min(_estimators_lags) # found a lag if 0 < _min_estimator_lag <= MAXIMUM_LAG: _var_model_results = _var_model.fit(maxlags=_min_estimator_lag, ic=None) _whiteness = _var_model_results.test_whiteness( nlags=_min_estimator_lag + 1) _whiteness_p_values.append(_whiteness.pvalue) if _tuples_to_mark is not None and _tuple in _tuples_to_mark and _whiteness.pvalue > 0.05: print(_tuple, 'marked whiteness p-value:', _whiteness.pvalue) # no autocorrelation in the residuals if _whiteness.pvalue > 0.05: if _properties['band']: _n_passed_whiteness_with_band += 1 # time lag = 0 _correlation = compute_lib.correlation( _left_cell_fiber_densities_derivative, _right_cell_fiber_densities_derivative) # if _correlation < 0.5: # continue # granger causality for _caused, _causing in zip(['left', 'right'], ['right', 'left']): _granger = _var_model_results.test_causality( caused=_caused, causing=_causing) _granger_causality_p_values.append(_granger.pvalue) # time lag = 0 _correlations.append(_correlation) # time lag = min estimator if _causing == 'left': _left_fiber_densities_time_lag = \ _left_cell_fiber_densities_derivative[:-_min_estimator_lag] _right_fiber_densities_time_lag = \ _right_cell_fiber_densities_derivative[_min_estimator_lag:] else: _left_fiber_densities_time_lag = \ _left_cell_fiber_densities_derivative[_min_estimator_lag:] _right_fiber_densities_time_lag = \ _right_cell_fiber_densities_derivative[:-_min_estimator_lag] _time_lag_correlation = compute_lib.correlation( _left_fiber_densities_time_lag, _right_fiber_densities_time_lag) _time_lag_correlations.append(_time_lag_correlation) # end fiber density _time_frame = compute.density_time_frame(_experiment) if len(_left_cell_fiber_densities_filtered ) > _time_frame: _end_fiber_density = \ (_left_cell_fiber_densities_filtered[_time_frame] + _right_cell_fiber_densities_filtered[_time_frame]) / 2 else: _end_fiber_density = \ (_left_cell_fiber_densities_filtered[-1] + _right_cell_fiber_densities_filtered[-1]) / 2 _normalization = load.normalization_series_file_data( _experiment, _series_id) _normalized_fiber_density = compute_lib.z_score( _end_fiber_density, _normalization['average'], _normalization['std']) _end_fiber_densities.append(_normalized_fiber_density) # marking if _tuples_to_mark is not None and _tuple in _tuples_to_mark and _granger.pvalue < 0.05: print(_tuple, 'causing:', _causing, 'marked granger p-value:', _granger.pvalue) if _granger.pvalue < 0.05: if _properties['band']: _n_passed_granger_causality_with_band += 1 _normality = _var_model_results.test_normality() _inst_granger = _var_model_results.test_inst_causality( causing=_causing) print( _tuple, _causing.capitalize() + ' causes ' + _caused + '!', 'time-points: ' + str(len( _left_cell_fiber_densities_derivative)), 'stationary derivative: ' + str(_derivative), 'band:' + str(_properties['band']), 'p-value: ' + str(round(_granger.pvalue, 4)), 'lag: ' + str(_min_estimator_lag), 'normality p-value: ' + str(round(_normality.pvalue, 4)), 'inst p-value: ' + str(round(_inst_granger.pvalue, 4)), sep='\t') # lag = 0 print('Time lag = 0 correlation:', _correlation) # rest of lags for _lag in range(1, _min_estimator_lag + 1): if _causing == 'left': _left_fiber_densities_time_lag = _left_cell_fiber_densities_derivative[: -_lag] _right_fiber_densities_time_lag = _right_cell_fiber_densities_derivative[ _lag:] else: _left_fiber_densities_time_lag = _left_cell_fiber_densities_derivative[ _lag:] _right_fiber_densities_time_lag = _right_cell_fiber_densities_derivative[: -_lag] _correlation = compute_lib.correlation( _left_fiber_densities_time_lag, _right_fiber_densities_time_lag) print( 'Time lag = ' + str(_lag) + ' correlation:', _correlation) # plots if _tuples_to_plot is not None and _tuple in _tuples_to_plot: _y_arrays = [ _left_cell_fiber_densities_derivative, _right_cell_fiber_densities_derivative ] _names_array = ['Left cell', 'Right cell'] _colors_array = config.colors(2) _temporal_resolution = compute.temporal_resolution_in_minutes( _experiment) _fig = go.Figure(data=[ go.Scatter(x=np.arange( start=_start_time_frame, stop=_start_time_frame + len(_left_cell_fiber_densities_derivative ), step=1) * _temporal_resolution, y=_y, name=_name, mode='lines', line={ 'color': _color, 'width': 1 }) for _y, _name, _color in zip( _y_arrays, _names_array, _colors_array) ], layout={ 'xaxis': { 'title': 'Time (minutes)', 'zeroline': False }, 'yaxis': { 'title': 'Fiber density (z-score)' + '\'' * _derivative, 'zeroline': False }, 'legend': { 'xanchor': 'left', 'x': 0.1, 'yanchor': 'top', 'bordercolor': 'black', 'borderwidth': 2, 'bgcolor': 'white' }, }) _experiment, _series_id, _group = _tuple save.to_html( _fig=_fig, _path=os.path.join(paths.PLOTS, save.get_module_name()), _filename='plot_' + _experiment + '_' + str(_series_id) + '_' + _group) # residuals _y_arrays = \ [_var_model_results.resid.values[:, 0], _var_model_results.resid.values[:, 1]] _fig = go.Figure(data=[ go.Scatter(x=np.arange( start=_start_time_frame, stop=_start_time_frame + len(_y), step=1) * _temporal_resolution, y=_y, name=_name, mode='lines', line={ 'color': _color, 'width': 1 }) for _y, _name, _color in zip( _y_arrays, _names_array, _colors_array) ], layout={ 'xaxis': { 'title': 'Time (minutes)', 'zeroline': False }, 'yaxis': { 'title': 'Residual', 'zeroline': False }, 'legend': { 'xanchor': 'left', 'x': 0.1, 'yanchor': 'top', 'bordercolor': 'black', 'borderwidth': 2, 'bgcolor': 'white' }, }) _experiment, _series_id, _group = _tuple save.to_html( _fig=_fig, _path=os.path.join(paths.PLOTS, save.get_module_name()), _filename='plot_residuals_' + _experiment + '_' + str(_series_id) + '_' + _group) # not enough time points except ValueError: continue print('Total pairs:', _n_pairs) print('Total pairs with band:', _n_pairs_with_band) print('Total pairs passed whiteness:', (np.array(_whiteness_p_values) > 0.05).sum()) print('Total pairs passed whiteness with band:', _n_passed_whiteness_with_band) print('Total cells passed granger causality:', (np.array(_granger_causality_p_values) < 0.05).sum()) print('Total cells passed granger causality with band:', _n_passed_granger_causality_with_band) # p-value correction print('Corrections of GC p-value < 0.05:') _granger_causality_p_values_corrected = multipletests( pvals=_granger_causality_p_values, method='fdr_bh') for _p_value, _p_value_corrected in zip( _granger_causality_p_values, _granger_causality_p_values_corrected[1]): if _p_value < 0.05: print('Original GC p-value:', _p_value, 'corrected:', _p_value_corrected) # plots for _test_name, _y_title, _y_array in \ zip( ['whiteness', 'granger'], ['Whiteness p-value', 'Granger causality p-value'], [_whiteness_p_values, _granger_causality_p_values] ): if _test_name in _plots: _fig = go.Figure(data=go.Box(y=_y_array, boxpoints='all', jitter=1, pointpos=0, line={'width': 1}, fillcolor='white', marker={ 'size': 10, 'color': '#ea8500' }, opacity=0.7, showlegend=False), layout={ 'xaxis': { 'zeroline': False }, 'yaxis': { 'title': _y_title, 'zeroline': False, 'range': [-0.1, 1.1], 'tickmode': 'array', 'tickvals': [0.05, 1] }, 'shapes': [{ 'type': 'line', 'x0': -0.75, 'y0': 0.05, 'x1': 0.75, 'y1': 0.05, 'line': { 'color': 'red', 'width': 2, 'dash': 'dash' } }] }) save.to_html(_fig=_fig, _path=os.path.join(paths.PLOTS, save.get_module_name()), _filename='plot_' + _test_name) # granger versus correlation print( 'GC vs. correlation pearson correlation:', compute_lib.correlation(_granger_causality_p_values, _correlations, _with_p_value=True)) _fig = go.Figure(data=go.Scatter(x=_granger_causality_p_values, y=_correlations, mode='markers', marker={ 'size': 10, 'color': '#ea8500' }, showlegend=False), layout={ 'xaxis': { 'title': 'Granger causality p-value', 'zeroline': False, }, 'yaxis': { 'title': 'Inner correlation', 'zeroline': False, } }) save.to_html(_fig=_fig, _path=os.path.join(paths.PLOTS, save.get_module_name()), _filename='plot_gc_vs_correlation') # granger versus time lag correlation print( 'GC vs. time lag correlation pearson correlation:', compute_lib.correlation(_granger_causality_p_values, _time_lag_correlations, _with_p_value=True)) _fig = go.Figure(data=go.Scatter(x=_granger_causality_p_values, y=_time_lag_correlations, mode='markers', marker={ 'size': 10, 'color': '#ea8500' }, showlegend=False), layout={ 'xaxis': { 'title': 'Granger causality p-value', 'zeroline': False, }, 'yaxis': { 'title': 'GC lag inner correlation', 'zeroline': False, } }) save.to_html(_fig=_fig, _path=os.path.join(paths.PLOTS, save.get_module_name()), _filename='plot_gc_vs_time_lag_correlation') # granger versus end fiber density print( 'GC vs. end fiber density pearson correlation:', compute_lib.correlation(_granger_causality_p_values, _end_fiber_densities, _with_p_value=True)) _fig = go.Figure(data=go.Scatter(x=_granger_causality_p_values, y=_end_fiber_densities, mode='markers', marker={ 'size': 10, 'color': '#ea8500' }, showlegend=False), layout={ 'xaxis': { 'title': 'Granger causality p-value', 'zeroline': False, }, 'yaxis': { 'title': 'End fiber density (z-score)', 'zeroline': False, } }) save.to_html(_fig=_fig, _path=os.path.join(paths.PLOTS, save.get_module_name()), _filename='plot_gc_vs_end_density')