def align_by_z_score(_experiments, _experiments_fiber_densities): _experiments_fiber_densities_aligned = {} for _tuple in tqdm(_experiments, desc='Temporal aligning experiments'): _experiment, _series_id, _group = _tuple _left_cell_alignment_fiber_densities = \ _experiments_fiber_densities[(_experiment, _series_id, _group, 'left_cell', ALIGNMENT_OFFSET_Y)] _right_cell_alignment_fiber_densities = \ _experiments_fiber_densities[(_experiment, _series_id, _group, 'right_cell', ALIGNMENT_OFFSET_Y)] _left_cell_fiber_densities = \ _experiments_fiber_densities[(_experiment, _series_id, _group, 'left_cell', OFFSET_Y)] _right_cell_fiber_densities = \ _experiments_fiber_densities[(_experiment, _series_id, _group, 'right_cell', OFFSET_Y)] _normalization = load.normalization_series_file_data( _experiment, _series_id) _left_cell_value_aligned, _right_cell_value_aligned = None, None for _time_frame, (_left_cell_fiber_density, _right_cell_fiber_density) in \ enumerate(zip(_left_cell_alignment_fiber_densities, _right_cell_alignment_fiber_densities)): _normalized_left_cell_fiber_density = compute_lib.z_score( _x=_left_cell_fiber_density[0], _average=_normalization['average'], _std=_normalization['std']) _normalized_right_cell_fiber_density = compute_lib.z_score( _x=_right_cell_fiber_density[0], _average=_normalization['average'], _std=_normalization['std']) _mean_z_score = (_normalized_left_cell_fiber_density + _normalized_right_cell_fiber_density) / 2 if _mean_z_score > START_TIME_FRAME_Z_SCORE: if _time_frame == 0 and BLOCK_ALREADY_DENSE: _left_cell_value_aligned = [] _right_cell_value_aligned = [] else: _left_cell_value_aligned = _left_cell_fiber_densities[ _time_frame:] _right_cell_value_aligned = _right_cell_fiber_densities[ _time_frame:] break # in case z-score not reached if _left_cell_value_aligned is None: _left_cell_value_aligned = [] _right_cell_value_aligned = [] _experiments_fiber_densities_aligned[(_experiment, _series_id, _group, 'left_cell')] = \ _left_cell_value_aligned _experiments_fiber_densities_aligned[(_experiment, _series_id, _group, 'right_cell')] = \ _right_cell_value_aligned return _experiments_fiber_densities_aligned
def compute_data(): _simulations = load.structured() _simulations = filtering.by_time_points_amount(_simulations, _time_points=TIME_POINTS) _simulations = filtering.by_categories(_simulations, _is_single_cell=False, _is_heterogeneity=False, _is_low_connectivity=False, _is_causality=False, _is_dominant_passive=False, _is_fibrin=False) _simulations = filtering.by_pair_distance(_simulations, _distance=PAIR_DISTANCE) print('Total simulations:', len(_simulations)) _fiber_densities = compute_fiber_densities(_simulations) _simulations_fiber_densities = [[] for _i in range(TIME_POINTS)] for _simulation in tqdm(_simulations, desc='Main loop'): _normalization = load.normalization(_simulation) for _time_point in range(TIME_POINTS): for _cell_id in ['left_cell', 'right_cell']: _fiber_density = _fiber_densities[(_simulation, _cell_id)][_time_point] _normalized_fiber_density = compute_lib.z_score( _fiber_density, _normalization['average'], _normalization['std']) _simulations_fiber_densities[_time_point].append( _normalized_fiber_density) print('Total pairs:', len(_simulations_fiber_densities[0])) return _simulations_fiber_densities
def compute_data(_arguments): _offset_y_index, _offset_y, _offset_z_index, _offset_z = _arguments _fiber_densities_array = [] for _tuple in _tuples: _experiment, _series_id, _group = _tuple _normalization = load.normalization_series_file_data( _experiment, _series_id) for _cell_id in ['left_cell', 'right_cell']: _fiber_density = \ _experiments_fiber_densities[(_experiment, _series_id, _group, _offset_y, _offset_z, _cell_id)] # remove out of boundaries if _fiber_density[1]: continue _normalized_fiber_density = compute_lib.z_score( _x=_fiber_density[0], _average=_normalization['average'], _std=_normalization['std']) _fiber_densities_array.append(_normalized_fiber_density) if len(_fiber_densities_array) > 0: return _offset_y_index, _offset_z_index, np.mean( _fiber_densities_array) else: return _offset_y_index, _offset_z_index, None
def compute_data(_arguments): _offset_y_index, _offset_y, _offset_z_index, _offset_z = _arguments _fiber_density_changes_array = [] for _tuple in _tuples: _experiment, _series_id, _group = _tuple _normalization = load.normalization_series_file_data( _experiment, _series_id) _properties = load.group_properties(_experiment, _series_id, _group) for _cell_id in ['left_cell', 'right_cell']: _cell_fiber_densities = \ _experiments_fiber_densities[(_experiment, _series_id, _group, _offset_y, _offset_z, _cell_id)] _cell_fiber_densities = compute.remove_blacklist( _experiment, _series_id, _properties['cells_ids'][_cell_id], _cell_fiber_densities) _previous_cell_fiber_density_normalized = None _cell_values = [] for _time_frame, _cell_fiber_density in enumerate( _cell_fiber_densities): # not out of border if _cell_fiber_density[1]: _previous_cell_fiber_density_normalized = None continue # normalize _cell_fiber_density_normalized = compute_lib.z_score( _x=_cell_fiber_density[0], _average=_normalization['average'], _std=_normalization['std']) # no previous if _previous_cell_fiber_density_normalized is None: _previous_cell_fiber_density_normalized = _cell_fiber_density_normalized continue # change _cell_fiber_density_normalized_change = abs( _cell_fiber_density_normalized - _previous_cell_fiber_density_normalized) _previous_cell_fiber_density_normalized = _cell_fiber_density_normalized # save _cell_values.append(_cell_fiber_density_normalized_change) # save if len(_cell_values) > 0: _fiber_density_changes_array.append(np.mean(_cell_values)) if len(_fiber_density_changes_array) > 0: return _offset_y_index, _offset_z_index, np.mean( _fiber_density_changes_array) else: return _offset_y_index, _offset_z_index, None
def compute_cell_pairs(_low_connectivity): _x_array = [] _y_array = [] _names_array = [] _max_offsets_x = [] for _distance in PAIR_DISTANCE: print('Pair distance ' + str(_distance)) _simulations = load.structured() _simulations = filtering.by_categories( _simulations, _is_single_cell=False, _is_heterogeneity=False, _is_low_connectivity=_low_connectivity, _is_causality=False, _is_dominant_passive=False, _is_fibrin=False) _simulations = filtering.by_pair_distance(_simulations, _distance=_distance) _simulations = filtering.by_time_points_amount( _simulations, _time_points=TIME_POINT[_low_connectivity]) _offsets_x = np.arange(start=0, stop=_distance / 2 - 0.25 - QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER, step=OFFSET_X_STEP) if len(_offsets_x) > len(_max_offsets_x): _max_offsets_x = _offsets_x _fiber_densities = compute_fiber_densities(_simulations, _offsets_x, _low_connectivity) _pair_distance_fiber_densities = [[] for _i in range(len(_offsets_x))] for _simulation in _simulations: _offset_index = 0 _normalization = load.normalization(_simulation) for _offset_x in _offsets_x: for _cell_id in ['left_cell', 'right_cell']: _fiber_density = _fiber_densities[(_simulation, _offset_x, _cell_id)] _normalized_fiber_density = compute_lib.z_score( _fiber_density, _normalization['average'], _normalization['std']) _pair_distance_fiber_densities[_offset_index].append( _normalized_fiber_density) _offset_index += 1 _x_array.append(_offsets_x) _y_array.append(_pair_distance_fiber_densities) _names_array.append('Pair distance ' + str(_distance)) return _names_array, _x_array, _y_array
def compute_data(_arguments): _offset_y_index, _offset_y = _arguments _fiber_densities_array = [] for _simulation in _simulations: _simulation_normalization = load.normalization(_simulation) for _cell_id in ['left_cell', 'right_cell']: _fiber_density = _fiber_densities[(_simulation, _offset_y, _cell_id)] _normalized_fiber_density = compute_lib.z_score( _x=_fiber_density, _average=_simulation_normalization['average'], _std=_simulation_normalization['std'] ) _fiber_densities_array.append(_normalized_fiber_density) if len(_fiber_densities_array) > 0: return _offset_y_index, np.mean(_fiber_densities_array) else: return _offset_y_index, None
def compute_simulations_data(_low_connectivity): _simulations = simulations_load.structured() _simulations = simulations_filtering.by_time_points_amount( _simulations, SIMULATIONS_TIME_POINT[_low_connectivity]) _simulations = simulations_filtering.by_categories( _simulations, _is_single_cell=False, _is_heterogeneity=False, _is_low_connectivity=_low_connectivity, _is_causality=False, _is_dominant_passive=False, _is_fibrin=False) _simulations = simulations_filtering.by_pair_distance( _simulations, _distance=PAIR_DISTANCE) print('Total simulations:', len(_simulations)) _fiber_densities = compute_simulations_fiber_densities( _simulations, _low_connectivity) _simulations_fiber_densities = [[] for _i in range(len(SIMULATIONS_OFFSETS_X)) ] for _simulation in tqdm(_simulations, desc='Simulations Loop'): _normalization = simulations_load.normalization(_simulation) for _offset_x_index, _offset_x in enumerate(SIMULATIONS_OFFSETS_X): for _cell_id in ['left_cell', 'right_cell']: _fiber_density = _fiber_densities[(_simulation, _offset_x, _cell_id)] _normalized_fiber_density = compute_lib.z_score( _fiber_density, _normalization['average'], _normalization['std']) _simulations_fiber_densities[_offset_x_index].append( _normalized_fiber_density) return _simulations_fiber_densities
def compute_simulations_data(_low_connectivity): _simulations = simulations_load.structured() _simulations = simulations_filtering.by_time_points_amount( _simulations, _time_points=SIMULATIONS_TIME_POINT[_low_connectivity] ) _simulations = simulations_filtering.by_categories( _simulations, _is_single_cell=True, _is_heterogeneity=False, _is_low_connectivity=_low_connectivity, _is_causality=False, _is_dominant_passive=False, _is_fibrin=False ) _fiber_densities = compute_simulations_fiber_densities(_simulations, _low_connectivity) _simulations_fiber_densities = [[] for _i in range(len(OFFSETS_X))] for _simulation in tqdm(_simulations, desc='Simulations Loop'): _normalization = simulations_load.normalization(_simulation) for _offset_x_index, _offset_x in enumerate(OFFSETS_X): _direction_fiber_densities = [] for _direction in ['left', 'right', 'up', 'down']: _fiber_density = _fiber_densities[(_simulation, _offset_x, _direction)] _normalized_fiber_density = compute_lib.z_score( _fiber_density, _normalization['average'], _normalization['std'] ) _direction_fiber_densities.append(_normalized_fiber_density) _simulations_fiber_densities[_offset_x_index].append(np.mean(_direction_fiber_densities)) return _simulations_fiber_densities
def compute_experiments_data(): _experiments = all_experiments() _experiments = experiments_filtering.by_categories( _experiments=_experiments, _is_single_cell=True, _is_high_temporal_resolution=False, _is_bleb=False, _is_dead_dead=False, _is_live_dead=False, _is_bead=False, _is_metastasis=False ) _tuples = experiments_load.experiments_groups_as_tuples(_experiments) _tuples = experiments_filtering.by_time_frames_amount(_tuples, EXPERIMENTS_TIME_FRAME) _tuples = experiments_filtering.by_main_cell(_tuples) _arguments = [] for _tuple in _tuples: _experiment, _series_id, _group = _tuple for _offset_x, _direction in product(OFFSETS_X, ['left', 'right']): _arguments.append({ 'experiment': _experiment, 'series_id': _series_id, 'group': _group, 'length_x': experiments_config.QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER, 'length_y': experiments_config.QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER, 'length_z': experiments_config.QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER, 'offset_x': _offset_x, 'offset_y': OFFSET_Y, 'offset_z': OFFSET_Z, 'cell_id': 'cell', 'direction': _direction, 'time_point': EXPERIMENTS_TIME_FRAME - 1 }) _windows_dictionary, _windows_to_compute = \ experiments_compute.windows(_arguments, _keys=['experiment', 'series_id', 'group', 'offset_x', 'direction']) _fiber_densities = experiments_compute.fiber_densities(_windows_to_compute) _tuples = experiments_organize.by_single_cell_id(_tuples) _experiments_fiber_densities = [[] for _i in range(len(OFFSETS_X))] for _tuple in tqdm(_tuples, desc='Experiments loop'): _experiment, _series_id, _cell_id = _tuple _normalization = experiments_load.normalization_series_file_data(_experiment, _series_id) for _offset_x_index, _offset_x in enumerate(OFFSETS_X): _cell_fiber_densities = [] for _cell_tuple in _tuples[_tuple]: _, _, _group = _cell_tuple for _direction in ['left', 'right']: _window_tuple = _windows_dictionary[(_experiment, _series_id, _group, _offset_x, _direction)][0] _fiber_density = _fiber_densities[_window_tuple] if not OUT_OF_BOUNDARIES and _fiber_density[1]: continue _normalized_fiber_density = compute_lib.z_score( _x=_fiber_density[0], _average=_normalization['average'], _std=_normalization['std'] ) if not np.isnan(_normalized_fiber_density): _cell_fiber_densities.append(_normalized_fiber_density) if len(_cell_fiber_densities) > 0: _experiments_fiber_densities[_offset_x_index].append(np.mean(_cell_fiber_densities)) return _experiments_fiber_densities
def main(_real_cells=True, _static=False, _band=True, _high_temporal_resolution=False): _experiments = all_experiments() _experiments = filtering.by_categories( _experiments=_experiments, _is_single_cell=False, _is_high_temporal_resolution=_high_temporal_resolution, _is_bleb=False, _is_dead_dead=False, _is_live_dead=False, _is_bead=False, _is_metastasis=False ) _tuples = load.experiments_groups_as_tuples(_experiments) _tuples = filtering.by_pair_distance_range(_tuples, PAIR_DISTANCE_RANGE) _tuples = filtering.by_real_pairs(_tuples, _real_pairs=_real_cells) _tuples = filtering.by_fake_static_pairs(_tuples, _fake_static_pairs=_static) _tuples = filtering.by_band(_tuples, _band=_band) _tuples = filtering.by_time_frames_amount(_tuples, compute.minimum_time_frames_for_correlation(_experiments[0])) print('Total tuples:', len(_tuples)) print('Density') _arguments = [] for _tuple in _tuples: _experiment, _series_id, _group = _tuple _time_frame = compute.minimum_time_frames_for_correlation(_experiment) for _cell_id in ['left_cell', 'right_cell']: _arguments.append({ 'experiment': _experiment, 'series_id': _series_id, 'group': _group, 'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER, 'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER, 'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER, 'offset_x': OFFSET_X, 'offset_y': OFFSET_Y_DENSITY, 'offset_z': OFFSET_Z, 'cell_id': _cell_id, 'direction': 'inside', 'time_point': _time_frame - 1 }) _windows_dictionary, _windows_to_compute = \ compute.windows(_arguments, _keys=['experiment', 'series_id', 'group', 'cell_id']) _densities_fiber_densities = compute.fiber_densities(_windows_to_compute, _subtract_border=False) _densities_experiments_fiber_densities = { _key: [_densities_fiber_densities[_tuple] for _tuple in _windows_dictionary[_key]] for _key in _windows_dictionary } print('Correlations') _arguments = [] for _tuple in _tuples: _experiment, _series_id, _group = _tuple _latest_time_frame = compute.latest_time_frame_before_overlapping(_experiment, _series_id, _group, OFFSET_X) for _cell_id in ['left_cell', 'right_cell']: _arguments.append({ 'experiment': _experiment, 'series_id': _series_id, 'group': _group, 'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER, 'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER, 'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER, 'offset_x': OFFSET_X, 'offset_y': OFFSET_Y_CORRELATION, 'offset_z': OFFSET_Z, 'cell_id': _cell_id, 'direction': 'inside', 'time_points': _latest_time_frame }) _windows_dictionary, _windows_to_compute = \ compute.windows(_arguments, _keys=['experiment', 'series_id', 'group', 'cell_id']) _correlations_fiber_densities = compute.fiber_densities(_windows_to_compute, _subtract_border=True) _correlations_experiments_fiber_densities = { _key: [_correlations_fiber_densities[_tuple] for _tuple in _windows_dictionary[_key]] for _key in _windows_dictionary } _densities = [] _correlations = [] for _tuple in tqdm(_tuples, desc='Main loop'): _experiment, _series_id, _group = _tuple # density _left_cell_fiber_density = \ _densities_experiments_fiber_densities[(_experiment, _series_id, _group, 'left_cell')][0] _right_cell_fiber_density = \ _densities_experiments_fiber_densities[(_experiment, _series_id, _group, 'right_cell')][0] # not relevant if _left_cell_fiber_density[1] or _right_cell_fiber_density[1]: continue _normalization = load.normalization_series_file_data(_experiment, _series_id) _left_cell_fiber_density_normalized = compute_lib.z_score( _x=_left_cell_fiber_density[0], _average=_normalization['average'], _std=_normalization['std'] ) _right_cell_fiber_density_normalized = compute_lib.z_score( _x=_right_cell_fiber_density[0], _average=_normalization['average'], _std=_normalization['std'] ) _density = (_left_cell_fiber_density_normalized + _right_cell_fiber_density_normalized) / 2 # correlation _left_cell_fiber_densities = \ _correlations_experiments_fiber_densities[(_experiment, _series_id, _group, 'left_cell')] _right_cell_fiber_densities = \ _correlations_experiments_fiber_densities[(_experiment, _series_id, _group, 'right_cell')] _properties = load.group_properties(_experiment, _series_id, _group) _left_cell_fiber_densities = compute.remove_blacklist( _experiment, _series_id, _properties['cells_ids']['left_cell'], _left_cell_fiber_densities) _right_cell_fiber_densities = compute.remove_blacklist( _experiment, _series_id, _properties['cells_ids']['right_cell'], _right_cell_fiber_densities) _left_cell_fiber_densities_filtered, _right_cell_fiber_densities_filtered = \ compute.longest_same_indices_shared_in_borders_sub_array( _left_cell_fiber_densities, _right_cell_fiber_densities ) # ignore small arrays if len(_left_cell_fiber_densities_filtered) < compute.minimum_time_frames_for_correlation(_experiment): continue _correlation = compute_lib.correlation( compute_lib.derivative(_left_cell_fiber_densities_filtered, _n=DERIVATIVE), compute_lib.derivative(_right_cell_fiber_densities_filtered, _n=DERIVATIVE) ) _densities.append(_density) _correlations.append(_correlation) print('Total tuples:', len(_densities)) print('Pearson correlation of densities and correlations:') print(compute_lib.correlation(_densities, _correlations, _with_p_value=True)) # plot _fig = go.Figure( data=go.Scatter( x=_densities, y=_correlations, mode='markers', marker={ 'size': 5, 'color': '#ea8500' }, showlegend=False ), layout={ 'xaxis': { 'title': 'Fiber density (z-score)', 'zeroline': False, 'range': [-1.1, 15.2], # 'tickmode': 'array', # 'tickvals': [-1, -0.5, 0, 0.5, 1] }, 'yaxis': { 'title': 'Correlation', 'zeroline': False, 'range': [-1.1, 1.2], 'tickmode': 'array', 'tickvals': [-1, -0.5, 0, 0.5, 1] }, 'shapes': [ { 'type': 'line', 'x0': 0, 'y0': -1, 'x1': 0, 'y1': 1, 'line': { 'color': 'black', 'width': 2 } }, { 'type': 'line', 'x0': 0, 'y0': -1, 'x1': 15, 'y1': -1, 'line': { 'color': 'black', 'width': 2 } } ] } ) save.to_html( _fig=_fig, _path=os.path.join(paths.PLOTS, save.get_module_name()), _filename='plot_real_' + str(_real_cells) + '_static_' + str(_static) + '_band_' + str(_band) + '_high_time_' + str(_high_temporal_resolution) + '_y_density_' + str(OFFSET_Y_DENSITY) + '_y_correlation_' + str(OFFSET_Y_CORRELATION) )
def main(_high_temporal_resolution=False, _pair_distance_range=None): if _pair_distance_range is None: _pair_distance_range = PAIR_DISTANCE_RANGE _experiments = all_experiments() _experiments = filtering.by_categories( _experiments=_experiments, _is_single_cell=False, _is_high_temporal_resolution=_high_temporal_resolution, _is_bleb=False, _is_dead_dead=False, _is_live_dead=False, _is_bead=False, _is_metastasis=False) _experiments = ['SN16'] _tuples = load.experiments_groups_as_tuples(_experiments) _tuples = filtering.by_pair_distance_range(_tuples, _pair_distance_range) _tuples = filtering.by_real_pairs(_tuples) _tuples = filtering.by_band(_tuples) print('Total tuples:', len(_tuples)) _arguments = [] for _tuple in _tuples: _experiment, _series_id, _group = _tuple _latest_time_frame = compute.latest_time_frame_before_overlapping( _experiment, _series_id, _group, OFFSET_X) for _cell_id, _offset_y in product(['left_cell', 'right_cell'], [0, OFFSET_Y]): _arguments.append({ 'experiment': _experiment, 'series_id': _series_id, 'group': _group, 'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER, 'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER, 'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER, 'offset_x': OFFSET_X, 'offset_y': _offset_y, 'offset_z': OFFSET_Z, 'cell_id': _cell_id, 'direction': 'inside', 'time_points': _latest_time_frame }) _windows_dictionary, _windows_to_compute = compute.windows( _arguments, _keys=['experiment', 'series_id', 'group', 'cell_id', 'offset_y']) _fiber_densities = compute.fiber_densities(_windows_to_compute) _experiments_fiber_densities = { _key: [_fiber_densities[_tuple] for _tuple in _windows_dictionary[_key]] for _key in _windows_dictionary } _no_offset_derivatives_array = [] _offset_derivatives_array = [] for _tuple in tqdm(_tuples, desc='Main loop'): _experiment, _series_id, _group = _tuple _no_offset_left_cell_fiber_densities = _experiments_fiber_densities[( _experiment, _series_id, _group, 'left_cell', 0)] _no_offset_right_cell_fiber_densities = _experiments_fiber_densities[( _experiment, _series_id, _group, 'right_cell', 0)] _offset_left_cell_fiber_densities = _experiments_fiber_densities[( _experiment, _series_id, _group, 'left_cell', OFFSET_Y)] _offset_right_cell_fiber_densities = _experiments_fiber_densities[( _experiment, _series_id, _group, 'right_cell', OFFSET_Y)] _properties = \ load.group_properties(_experiment, _series_id, _group) _no_offset_left_cell_fiber_densities = compute.remove_blacklist( _experiment, _series_id, _properties['cells_ids']['left_cell'], _no_offset_left_cell_fiber_densities) _no_offset_right_cell_fiber_densities = compute.remove_blacklist( _experiment, _series_id, _properties['cells_ids']['right_cell'], _no_offset_right_cell_fiber_densities) _offset_left_cell_fiber_densities = compute.remove_blacklist( _experiment, _series_id, _properties['cells_ids']['left_cell'], _offset_left_cell_fiber_densities) _offset_right_cell_fiber_densities = compute.remove_blacklist( _experiment, _series_id, _properties['cells_ids']['right_cell'], _offset_right_cell_fiber_densities) _normalization = load.normalization_series_file_data( _experiment, _series_id) for _cell_pair in [(_no_offset_left_cell_fiber_densities, _offset_left_cell_fiber_densities), (_no_offset_right_cell_fiber_densities, _offset_right_cell_fiber_densities)]: for _time_frame in range( 1, min(len(_cell_pair[0]), len(_cell_pair[1]))): _no_offset_cell_fiber_density_previous = _cell_pair[0][ _time_frame - 1] _no_offset_cell_fiber_density = _cell_pair[0][_time_frame] _offset_cell_fiber_density_previous = _cell_pair[1][_time_frame - 1] _offset_cell_fiber_density = _cell_pair[1][_time_frame] if any([ _no_offset_cell_fiber_density_previous[1], _no_offset_cell_fiber_density[1], _offset_cell_fiber_density_previous[1], _offset_cell_fiber_density[1] ]): continue _no_offset_cell_fiber_density_previous_z_score = compute_lib.z_score( _x=_no_offset_cell_fiber_density_previous[0], _average=_normalization['average'], _std=_normalization['std']) _no_offset_cell_fiber_density_z_score = compute_lib.z_score( _x=_no_offset_cell_fiber_density[0], _average=_normalization['average'], _std=_normalization['std']) _offset_cell_fiber_density_previous_z_score = compute_lib.z_score( _x=_offset_cell_fiber_density_previous[0], _average=_normalization['average'], _std=_normalization['std']) _offset_cell_fiber_density_z_score = compute_lib.z_score( _x=_offset_cell_fiber_density[0], _average=_normalization['average'], _std=_normalization['std']) _no_offset_derivative = _no_offset_cell_fiber_density_z_score - _no_offset_cell_fiber_density_previous_z_score _offset_derivative = _offset_cell_fiber_density_z_score - _offset_cell_fiber_density_previous_z_score _no_offset_derivatives_array.append(_no_offset_derivative) _offset_derivatives_array.append(_offset_derivative) print('Total points:', len(_no_offset_derivatives_array)) _offset_minus_no_offset = \ np.array(_offset_derivatives_array) - np.array(_no_offset_derivatives_array) print('Wilcoxon of offset minus no offset around the zero:') print(wilcoxon(_offset_minus_no_offset)) print('Higher offset amount:', (_offset_minus_no_offset > 0).sum() / len(_offset_minus_no_offset)) # plot _fig = go.Figure( data=go.Scatter(x=_no_offset_derivatives_array, y=_offset_derivatives_array, mode='markers', marker={ 'size': 5, 'color': '#ea8500' }, showlegend=False), layout={ 'xaxis': { 'title': 'No offset derivative', # 'zeroline': False, # 'range': [-1.1, 1.2], # 'tickmode': 'array', # 'tickvals': [-1, -0.5, 0, 0.5, 1] }, 'yaxis': { 'title': 'Offset derivative', # 'zeroline': False, # 'range': [-1.1, 1.2], # 'tickmode': 'array', # 'tickvals': [-1, -0.5, 0, 0.5, 1] }, 'shapes': [ # { # 'type': 'line', # 'x0': -1, # 'y0': -1, # 'x1': -1, # 'y1': 1, # 'line': { # 'color': 'black', # 'width': 2 # } # }, # { # 'type': 'line', # 'x0': -1, # 'y0': -1, # 'x1': 1, # 'y1': -1, # 'line': { # 'color': 'black', # 'width': 2 # } # }, { 'type': 'line', 'x0': -5, 'y0': -5, 'x1': 5, 'y1': 5, 'line': { 'color': 'red', 'width': 2 } } ] }) save.to_html(_fig=_fig, _path=os.path.join(paths.PLOTS, save.get_module_name()), _filename='plot')
def compute_experiments_data(): _experiments = all_experiments() _experiments = experiments_filtering.by_categories( _experiments=_experiments, _is_single_cell=False, _is_high_temporal_resolution=False, _is_bleb=False, _is_dead_dead=False, _is_live_dead=False, _is_bead=False, _is_metastasis=False ) _tuples = experiments_load.experiments_groups_as_tuples(_experiments) _tuples = experiments_filtering.by_time_frames_amount(_tuples, EXPERIMENTS_TIME_FRAMES) _tuples = experiments_filtering.by_real_pairs(_tuples) _tuples = experiments_filtering.by_pair_distance_range(_tuples, PAIR_DISTANCE_RANGE) _tuples = experiments_filtering.by_band(_tuples) print('Total tuples:', len(_tuples)) _arguments = [] for _tuple in _tuples: _experiment, _series_id, _group = _tuple _time_frame = experiments_compute.density_time_frame(_experiment) for _cell_id in ['left_cell', 'right_cell']: _arguments.append({ 'experiment': _experiment, 'series_id': _series_id, 'group': _group, 'length_x': experiments_config.QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER, 'length_y': experiments_config.QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER, 'length_z': experiments_config.QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER, 'offset_x': OFFSET_X, 'offset_y': OFFSET_Y, 'offset_z': OFFSET_Z, 'cell_id': _cell_id, 'direction': 'inside', 'time_points': _time_frame }) _windows_dictionary, _windows_to_compute = \ experiments_compute.windows(_arguments, _keys=['experiment', 'series_id', 'group', 'cell_id']) _fiber_densities = experiments_compute.fiber_densities(_windows_to_compute) _experiments_fiber_densities = [[] for _i in range(EXPERIMENTS_TIME_FRAMES)] for _tuple in tqdm(_tuples, desc='Experiments loop'): _experiment, _series_id, _group = _tuple _normalization = experiments_load.normalization_series_file_data(_experiment, _series_id) for _time_frame in range(EXPERIMENTS_TIME_FRAMES): for _cell_id in ['left_cell', 'right_cell']: _window_tuple = _windows_dictionary[(_experiment, _series_id, _group, _cell_id)][_time_frame] _fiber_density = _fiber_densities[_window_tuple] if not OUT_OF_BOUNDARIES and _fiber_density[1]: continue _normalized_fiber_density = compute_lib.z_score( _x=_fiber_density[0], _average=_normalization['average'], _std=_normalization['std'] ) if not np.isnan(_normalized_fiber_density): _experiments_fiber_densities[_time_frame].append(_normalized_fiber_density) print('Total experiments pairs:', len(_experiments_fiber_densities[0])) return _experiments_fiber_densities
def compute_matched_fiber(_tuples): _tuples = filtering.by_time_frames_amount( _tuples, compute.density_time_frame(_tuples[0])) _tuples = filtering.by_pair_distance_range(_tuples, PAIR_DISTANCE_RANGE) _experiments_matched = organize.by_matched_real_and_fake(_tuples) print('Total matched pairs:', len(_experiments_matched)) _max_offsets_x = [] _arguments = [] for _matched_tuple in _experiments_matched: for _tuple in _matched_tuple: _experiment, _series_id, _group = _tuple _time_frame = compute.density_time_frame(_experiment) _pair_distance = compute.pair_distance_in_cell_size_time_frame( _experiment, _series_id, _group, _time_frame - 1) _offsets_x = \ np.arange(start=0, stop=_pair_distance / 2 - 0.5 - QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER, step=OFFSET_X_STEP) if len(_offsets_x) > len(_max_offsets_x): _max_offsets_x = _offsets_x for _offset_x in _offsets_x: for _cell_id in ['left_cell', 'right_cell']: _arguments.append({ 'experiment': _experiment, 'series_id': _series_id, 'group': _group, 'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER, 'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER, 'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER, 'offset_x': _offset_x, 'offset_y': OFFSET_Y, 'offset_z': OFFSET_Z, 'cell_id': _cell_id, 'direction': 'inside', 'time_point': _time_frame - 1 }) _windows_dictionary, _windows_to_compute = \ compute.windows(_arguments, _keys=['experiment', 'series_id', 'group', 'offset_x', 'cell_id']) _fiber_densities = compute.fiber_densities(_windows_to_compute) _experiments_fiber_densities_real = [[] for _i in range(len(_max_offsets_x))] _experiments_fiber_densities_fake = [[] for _i in range(len(_max_offsets_x))] for _tuple in tqdm(_experiments_matched, desc='Experiments loop'): _tuple_real, _tuple_fake = _tuple _experiment_real, _series_id_real, _group_real = _tuple_real _experiment_fake, _series_id_fake, _group_fake = _tuple_fake for _offset_x_index, _offset_x in enumerate(_max_offsets_x): for _cell_id in ['left_cell', 'right_cell']: if (_experiment_real, _series_id_real, _group_real, _offset_x, _cell_id) and \ (_experiment_fake, _series_id_fake, _group_fake, _offset_x, _cell_id) in _windows_dictionary: _normalization = load.normalization_series_file_data( _experiment_real, _series_id_real) _window_tuple_real = \ _windows_dictionary[(_experiment_real, _series_id_real, _group_real, _offset_x, _cell_id)][0] _fiber_density_real = _fiber_densities[_window_tuple_real] _window_tuple_fake = \ _windows_dictionary[(_experiment_fake, _series_id_fake, _group_fake, _offset_x, _cell_id)][0] _fiber_density_fake = _fiber_densities[_window_tuple_fake] if not OUT_OF_BOUNDARIES and (_fiber_density_real[1] or _fiber_density_fake[1]): continue _normalized_fiber_density_real = compute_lib.z_score( _x=_fiber_density_real[0], _average=_normalization['average'], _std=_normalization['std']) _normalized_fiber_density_fake = compute_lib.z_score( _x=_fiber_density_fake[0], _average=_normalization['average'], _std=_normalization['std']) if not np.isnan( _normalized_fiber_density_real) and not np.isnan( _normalized_fiber_density_fake): _experiments_fiber_densities_real[ _offset_x_index].append( _normalized_fiber_density_real) _experiments_fiber_densities_fake[ _offset_x_index].append( _normalized_fiber_density_fake) return _experiments_fiber_densities_real, _experiments_fiber_densities_fake, _max_offsets_x
def main(): _experiments = all_experiments() _experiments = filtering.by_categories(_experiments=_experiments, _is_single_cell=False, _is_high_temporal_resolution=False, _is_bleb=False, _is_dead_dead=False, _is_live_dead=False, _is_bead=False, _is_metastasis=False) _tuples = load.experiments_groups_as_tuples(_experiments) _tuples = filtering.by_time_frames_amount( _tuples, compute.density_time_frame(_experiments[0])) _tuples = filtering.by_real_pairs(_tuples) _tuples = filtering.by_band(_tuples) _max_offsets_x = [] _arguments = [] for _tuple in _tuples: _experiment, _series_id, _group = _tuple _time_frame = compute.minimum_time_frames_for_correlation(_experiment) _pair_distance = \ compute.pair_distance_in_cell_size_time_frame(_experiment, _series_id, _group, _time_frame=_time_frame - 1) _offsets_x = \ np.arange(start=0, stop=_pair_distance / 2 - 0.5 - QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER, step=OFFSET_X_STEP) if len(_offsets_x) > len(_max_offsets_x): _max_offsets_x = _offsets_x for _offset_x in _offsets_x: for _cell_id in ['left_cell', 'right_cell']: _arguments.append({ 'experiment': _experiment, 'series_id': _series_id, 'group': _group, 'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER, 'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER, 'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER, 'offset_x': _offset_x, 'offset_y': OFFSET_Y, 'offset_z': OFFSET_Z, 'cell_id': _cell_id, 'direction': 'inside', 'time_point': _time_frame - 1 }) _windows_dictionary, _windows_to_compute = \ compute.windows(_arguments, _keys=['experiment', 'series_id', 'group', 'offset_x', 'cell_id']) _fiber_densities = compute.fiber_densities(_windows_to_compute) _x_array = [] _y_array = [] _n_array = [] _p_value_array = [] for _offset_x in _max_offsets_x: _pair_distances = [] _z_scores = [] for _tuple in _tuples: _experiment, _series_id, _group = _tuple for _cell_id in ['left_cell', 'right_cell']: if (_experiment, _series_id, _group, _offset_x, _cell_id) in _windows_dictionary: _normalization = load.normalization_series_file_data( _experiment, _series_id) _window_tuple = _windows_dictionary[(_experiment, _series_id, _group, _offset_x, _cell_id)][0] _fiber_density = _fiber_densities[_window_tuple] if not OUT_OF_BOUNDARIES and _fiber_density[1]: continue _normalized_fiber_density = compute_lib.z_score( _x=_fiber_density[0], _average=_normalization['average'], _std=_normalization['std']) if not np.isnan(_normalized_fiber_density): _pair_distances.append( compute.pair_distance_in_cell_size_time_frame( _experiment, _series_id, _group, _time_frame=0)) _z_scores.append(_normalized_fiber_density) if len(_pair_distances) > 2: _x_array.append(round(_offset_x, 1)) _correlation = compute_lib.correlation(_pair_distances, _z_scores, _with_p_value=True) _y_array.append(round(_correlation[0], 2)) _n_array.append(len(_pair_distances)) _p_value_array.append(round(_correlation[1], 2)) print('Pearson:') print(compute_lib.correlation(_x_array, _y_array, _with_p_value=True)) # plot _significant_x_array = [ _x for _x, _p_value in zip(_x_array, _p_value_array) if _p_value < 0.05 ] _fig = go.Figure(data=[ go.Scatter(x=_x_array, y=_y_array, mode='markers', marker={ 'size': 15, 'color': 'black' }, showlegend=False), go.Scatter(x=_significant_x_array, y=[-0.79] * len(_significant_x_array), mode='text', text='*', textfont={'color': 'red'}, showlegend=False) ], layout={ 'xaxis': { 'title': 'Window distance (cell diameter)', 'zeroline': False }, 'yaxis': { 'title': 'Correlation:<br>pair distance vs. fiber density', 'zeroline': False, 'range': [-0.82, 0.3], 'tickmode': 'array', 'tickvals': [-0.75, -0.25, 0.25] }, 'shapes': [{ 'type': 'line', 'x0': -0.2, 'y0': -0.8, 'x1': 3.2, 'y1': -0.8, 'line': { 'color': 'black', 'width': 2 } }, { 'type': 'line', 'x0': -0.2, 'y0': -0.8, 'x1': -0.2, 'y1': 0.3, 'line': { 'color': 'black', 'width': 2 } }] }) save.to_html(_fig=_fig, _path=os.path.join(paths.PLOTS, save.get_module_name()), _filename='plot') # table print('Window distance (cell diameter)', 'Correlation: pair distance vs. fiber density', 'N', 'P-value', sep='\t') for _x, _y, _n, _p_value in zip(_x_array, _y_array, _n_array, _p_value_array): print(_x, _y, _n, _p_value, sep='\t')
def main(_real_cells=True, _static=False, _band=True, _high_temporal_resolution=False, _pair_distance_range=None, _offset_y=0.5): if _pair_distance_range is None: _pair_distance_range = PAIR_DISTANCE_RANGE _experiments = all_experiments() _experiments = filtering.by_categories( _experiments=_experiments, _is_single_cell=False, _is_high_temporal_resolution=_high_temporal_resolution, _is_bleb=False, _is_dead_dead=False, _is_live_dead=False, _is_bead=False, _is_metastasis=False) _tuples = load.experiments_groups_as_tuples(_experiments) _tuples = filtering.by_pair_distance_range(_tuples, _pair_distance_range) _tuples = filtering.by_real_pairs(_tuples, _real_pairs=_real_cells) _tuples = filtering.by_fake_static_pairs(_tuples, _fake_static_pairs=_static) _tuples = filtering.by_band(_tuples, _band=_band) print('Total tuples:', len(_tuples)) _arguments = [] _longest_time_frame = 0 for _tuple in _tuples: _experiment, _series_id, _group = _tuple _latest_time_frame = compute.latest_time_frame_before_overlapping( _experiment, _series_id, _group, OFFSET_X) # save for later if _latest_time_frame > _longest_time_frame: _longest_time_frame = _latest_time_frame for _cell_id in ['left_cell', 'right_cell']: _arguments.append({ 'experiment': _experiment, 'series_id': _series_id, 'group': _group, 'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER, 'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER, 'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER, 'offset_x': OFFSET_X, 'offset_y': _offset_y, 'offset_z': OFFSET_Z, 'cell_id': _cell_id, 'direction': 'inside', 'time_points': _latest_time_frame }) _windows_dictionary, _windows_to_compute = compute.windows( _arguments, _keys=['experiment', 'series_id', 'group', 'cell_id']) _fiber_densities = compute.fiber_densities(_windows_to_compute) _experiments_fiber_densities = { _key: [_fiber_densities[_tuple] for _tuple in _windows_dictionary[_key]] for _key in _windows_dictionary } _valid_tuples = [] _valid_cells = [] _densities = [[] for _ in range(_longest_time_frame)] for _tuple in tqdm(_tuples, desc='Experiments loop'): _experiment, _series_id, _group = _tuple _normalization = load.normalization_series_file_data( _experiment, _series_id) _properties = load.group_properties(_experiment, _series_id, _group) for _cell_id in ['left_cell', 'right_cell']: _cell_fiber_densities = \ _experiments_fiber_densities[(_experiment, _series_id, _group, _cell_id)] _cell_fiber_densities = compute.remove_blacklist( _experiment, _series_id, _properties['cells_ids'][_cell_id], _cell_fiber_densities) _previous_cell_fiber_density_normalized = None for _time_frame, _cell_fiber_density in enumerate( _cell_fiber_densities): # not out of border if _cell_fiber_density[1]: _previous_cell_fiber_density_normalized = None continue # normalize _cell_fiber_density_normalized = compute_lib.z_score( _x=_cell_fiber_density[0], _average=_normalization['average'], _std=_normalization['std']) # no previous if _previous_cell_fiber_density_normalized is None: _previous_cell_fiber_density_normalized = _cell_fiber_density_normalized continue # change _cell_fiber_density_normalized_change = _cell_fiber_density_normalized - _previous_cell_fiber_density_normalized _previous_cell_fiber_density_normalized = _cell_fiber_density_normalized # save _densities[_time_frame].append( _cell_fiber_density_normalized_change) if _tuple not in _valid_tuples: _valid_tuples.append(_tuple) _cell_tuple = (_experiment, _series_id, _group, _cell_id) if _cell_tuple not in _valid_cells: _valid_cells.append(_cell_tuple) print('Total pairs:', len(_valid_tuples)) print('Total cells:', len(_valid_cells)) # plot _temporal_resolution = compute.temporal_resolution_in_minutes( _experiments[0]) _fig = go.Figure( data=go.Scatter(x=np.array(range(_longest_time_frame)) * _temporal_resolution, y=[np.mean(_array) for _array in _densities], name='Fiber density change (z-score)', error_y={ 'type': 'data', 'array': [np.std(_array) for _array in _densities], 'thickness': 1 }, mode='lines+markers', marker={ 'size': 5, 'color': '#ea8500' }, line={'dash': 'solid'}, showlegend=False), layout={ 'xaxis': { 'title': 'Time (minutes)', # 'zeroline': False }, 'yaxis': { 'title': 'Fiber density change (z-score)', # 'zeroline': False }, # 'shapes': [ # { # 'type': 'line', # 'x0': -_temporal_resolution, # 'y0': -0.5, # 'x1': -_temporal_resolution, # 'y1': 2, # 'line': { # 'color': 'black', # 'width': 2 # } # }, # { # 'type': 'line', # 'x0': -_temporal_resolution, # 'y0': -0.5, # 'x1': 350, # 'y1': -0.5, # 'line': { # 'color': 'black', # 'width': 2 # } # } # ] }) save.to_html( _fig=_fig, _path=os.path.join(paths.PLOTS, save.get_module_name()), _filename='plot_real_' + str(_real_cells) + '_static_' + str(_static) + '_band_' + str(_band) + '_high_time_' + str(_high_temporal_resolution) + '_range_' + '_'.join([str(_distance) for _distance in _pair_distance_range]) + '_y_' + str(_offset_y))
def main(): # simulations print('Simulations') _simulations = simulations_load.structured() _simulations = simulations_filtering.by_time_points_amount( _simulations, _time_points=SIMULATIONS_TIME_POINTS) _simulations = simulations_filtering.by_categories( _simulations, _is_single_cell=True, _is_heterogeneity=False, _is_low_connectivity=False, _is_causality=False, _is_dominant_passive=False, _is_fibrin=False) print('Total simulations:', len(_simulations)) _fiber_densities = compute_simulations_fiber_densities(_simulations) _simulations_fiber_densities = [[] for _i in range(SIMULATIONS_TIME_POINTS)] for _simulation in tqdm(_simulations, desc='Simulations Loop'): _normalization = simulations_load.normalization(_simulation) for _time_frame in range(SIMULATIONS_TIME_POINTS): _direction_fiber_densities = [] for _direction in ['left', 'right', 'up', 'down']: _fiber_density = _fiber_densities[(_simulation, _direction)][_time_frame] _normalized_fiber_density = compute_lib.z_score( _fiber_density, _normalization['average'], _normalization['std']) _direction_fiber_densities.append(_normalized_fiber_density) _simulations_fiber_densities[_time_frame].append( np.mean(_direction_fiber_densities)) print('Total simulations cells:', len(_simulations_fiber_densities[0])) # experiments print('Experiments') _experiments_fiber_densities = compute_experiments_fiber_densities() # plot _fig = go.Figure( data=[ go.Scatter( x=list(range(SIMULATIONS_TIME_POINTS))[::SIMULATIONS_STEP], y=[np.mean(_array) for _array in _simulations_fiber_densities ][::SIMULATIONS_STEP], name='Simulations', error_y={ 'type': 'data', 'array': [ np.std(_array) for _array in _simulations_fiber_densities ][::SIMULATIONS_STEP], 'thickness': 1, 'color': '#005b96' }, mode='markers', marker={ 'size': 15, 'color': '#005b96' }, opacity=0.7), go.Scatter( x=np.array(range(EXPERIMENTS_TIME_FRAMES)) * EXPERIMENTS_TEMPORAL_RESOLUTION, xaxis='x2', y=[np.mean(_array) for _array in _experiments_fiber_densities], name='Experiments', error_y={ 'type': 'data', 'array': [ np.std(_array) for _array in _experiments_fiber_densities ], 'thickness': 1, 'color': '#ea8500' }, mode='markers', marker={ 'size': 15, 'color': '#ea8500' }, opacity=0.7) ], layout={ 'xaxis': { 'title': 'Cell contraction (%)', 'titlefont': { 'color': '#005b96' }, 'tickfont': { 'color': '#005b96' }, 'zeroline': False }, 'xaxis2': { 'title': 'Time (minutes)', 'titlefont': { 'color': '#ea8500' }, 'tickfont': { 'color': '#ea8500' }, # 'anchor': 'free', 'overlaying': 'x', 'side': 'bottom', 'showgrid': False, 'zeroline': False }, 'yaxis': { 'title': 'Fiber density (z-score)', # 'domain': [0.3, 1], 'range': [-1.7, 14], 'zeroline': False, 'tickmode': 'array', 'tickvals': [0, 4, 8, 12] }, 'legend': { 'xanchor': 'left', 'x': 0.1, 'yanchor': 'top', 'bordercolor': 'black', 'borderwidth': 2 }, 'shapes': [{ 'type': 'line', 'x0': -2, 'y0': -1.5, 'x1': 53, 'y1': -1.5, 'line': { 'color': 'black', 'width': 2 } }, { 'type': 'line', 'x0': -2, 'y0': -1.5, 'x1': -2, 'y1': 14, 'line': { 'color': 'black', 'width': 2 } }] }) save.to_html(_fig=_fig, _path=os.path.join(paths_lib.PLOTS, save.get_module_name()), _filename='plot')
def main(): print('Single Cell') _experiments = all_experiments() _experiments = filtering.by_categories(_experiments=_experiments, _is_single_cell=True, _is_high_temporal_resolution=False, _is_bleb=False, _is_dead_dead=False, _is_live_dead=False, _is_bead=False, _is_metastasis=False) _tuples = load.experiments_groups_as_tuples(_experiments) _tuples = filtering.by_time_frames_amount( _tuples, compute.minimum_time_frames_for_correlation(_experiments[0])) _arguments = [] for _tuple in _tuples: _experiment, _series_id, _group = _tuple _time_frame = compute.minimum_time_frames_for_correlation(_experiment) for _offset_x, _direction in product(OFFSETS_X, ['left', 'right']): _arguments.append({ 'experiment': _experiment, 'series_id': _series_id, 'group': _group, 'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER, 'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER, 'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER, 'offset_x': _offset_x, 'offset_y': OFFSET_Y, 'offset_z': OFFSET_Z, 'cell_id': 'cell', 'direction': _direction, 'time_point': _time_frame - 1 }) _windows_dictionary, _windows_to_compute = \ compute.windows(_arguments, _keys=['experiment', 'series_id', 'group', 'offset_x', 'direction']) _fiber_densities = compute.fiber_densities(_windows_to_compute) _tuples = organize.by_single_cell_id(_tuples) _single_cell_fiber_densities = [[] for _i in range(len(OFFSETS_X))] for _tuple in _tuples: _experiment, _series_id, _cell_id = _tuple print('Experiment:', _experiment, 'Series ID:', _series_id, 'Cell ID:', _cell_id, sep='\t') _offset_index = 0 _normalization = load.normalization_series_file_data( _experiment, _series_id) for _offset_x in OFFSETS_X: _cell_fiber_densities = [] for _cell_tuple in _tuples[_tuple]: _, _, _group = _cell_tuple for _direction in ['left', 'right']: _window_tuple = _windows_dictionary[(_experiment, _series_id, _group, _offset_x, _direction)][0] _fiber_density = _fiber_densities[_window_tuple] if not OUT_OF_BOUNDARIES and _fiber_density[1]: continue _normalized_fiber_density = compute_lib.z_score( _x=_fiber_density[0], _average=_normalization['average'], _std=_normalization['std']) _cell_fiber_densities.append(_normalized_fiber_density) if len(_cell_fiber_densities) > 0: _single_cell_fiber_densities[_offset_index].append( np.mean(_cell_fiber_densities)) _offset_index += 1 print('Pairs') _experiments = all_experiments() _experiments = filtering.by_categories(_experiments=_experiments, _is_single_cell=False, _is_high_temporal_resolution=False, _is_bleb=False, _is_dead_dead=False, _is_live_dead=False, _is_bead=False, _is_metastasis=False) _tuples = load.experiments_groups_as_tuples(_experiments) _tuples = filtering.by_time_frames_amount( _tuples, compute.minimum_time_frames_for_correlation(_experiments[0])) _tuples = filtering.by_real_pairs(_tuples) _tuples = filtering.by_pair_distance(_tuples, PAIR_DISTANCE) _tuples = filtering.by_band(_tuples) _arguments = [] for _tuple in _tuples: _experiment, _series_id, _group = _tuple _time_frame = compute.minimum_time_frames_for_correlation(_experiment) for _offset_x in OFFSETS_X: _arguments.append({ 'experiment': _experiment, 'series_id': _series_id, 'group': _group, 'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER, 'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER, 'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER, 'offset_x': _offset_x, 'offset_y': OFFSET_Y, 'offset_z': OFFSET_Z, 'cell_id': 'left_cell', 'direction': 'inside', 'time_point': _time_frame - 1 }) _windows_dictionary, _windows_to_compute = \ compute.windows(_arguments, _keys=['experiment', 'series_id', 'group', 'offset_x']) _fiber_densities = compute.fiber_densities(_windows_to_compute) _pairs_fiber_densities = [[] for _i in range(len(OFFSETS_X))] for _tuple in _tuples: _experiment, _series_id, _group = _tuple print('Experiment:', _experiment, 'Series ID:', _series_id, 'Group:', _group, sep='\t') _offset_index = 0 _normalization = load.normalization_series_file_data( _experiment, _series_id) # take offsets based on pair distance _properties = load.group_properties(_experiment, _series_id, _group) _left_cell_coordinates = [ list(_properties['time_points'][0]['left_cell'] ['coordinates'].values()) ] _right_cell_coordinates = [ list(_properties['time_points'][0]['right_cell'] ['coordinates'].values()) ] _pair_distance = compute.pair_distance_in_cell_size( _experiment, _series_id, _left_cell_coordinates, _right_cell_coordinates) _edges_distance = _pair_distance - 1 _max_x_offset = _edges_distance - QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER for _offset_x in OFFSETS_X: if _offset_x > _max_x_offset: break _fiber_density = _fiber_densities[_windows_dictionary[( _experiment, _series_id, _group, _offset_x)][0]] if not OUT_OF_BOUNDARIES and _fiber_density[1]: continue _normalized_fiber_density = compute_lib.z_score( _x=_fiber_density[0], _average=_normalization['average'], _std=_normalization['std']) _pairs_fiber_densities[_offset_index].append( _normalized_fiber_density) _offset_index += 1 # plot _fig = go.Figure(data=[ go.Scatter(x=OFFSETS_X, y=[np.mean(_array) for _array in _pairs_fiber_densities], name='Pairs', error_y={ 'type': 'data', 'array': [np.std(_array) for _array in _pairs_fiber_densities], 'thickness': 1 }, mode='lines+markers', line={'dash': 'solid'}), go.Scatter( x=OFFSETS_X, y=[np.mean(_array) for _array in _single_cell_fiber_densities], name='Single Cell', error_y={ 'type': 'data', 'array': [np.std(_array) for _array in _single_cell_fiber_densities], 'thickness': 1 }, mode='lines+markers', line={'dash': 'dash'}) ], layout={ 'xaxis_title': 'Distance from left cell (cell size)', 'yaxis_title': 'Fiber density (z-score)' }) save.to_html(_fig=_fig, _path=os.path.join(paths.PLOTS, save.get_module_name()), _filename='plot_distance_' + str(PAIR_DISTANCE))
def main(): _experiments = config.all_experiments() _experiments = filtering.by_categories(_experiments=_experiments, _is_single_cell=False, _is_high_temporal_resolution=None, _is_bleb=False, _is_dead_dead=False, _is_live_dead=False, _is_bead=False, _is_metastasis=False) _tuples = load.experiments_groups_as_tuples(_experiments) _tuples = filtering.by_pair_distance_range(_tuples, PAIR_DISTANCE_RANGE) _tuples = filtering.by_real_pairs(_tuples, _real_pairs=REAL) _tuples = filtering.by_fake_static_pairs(_tuples, _fake_static_pairs=STATIC) _tuples = filtering.by_real_fake_pairs(_tuples, _real_fake_pairs=REAL_FAKE) print('Total tuples:', len(_tuples)) _arguments = [] for _tuple in tqdm(_tuples, desc='Setting windows to compute'): _experiment, _series_id, _group = _tuple _latest_time_frame = compute.latest_time_frame_before_overlapping( _experiment, _series_id, _group, _offset_x=0) for _cell_id in ['left_cell', 'right_cell']: for _offset_y in OFFSETS_Y: _arguments.append({ 'experiment': _experiment, 'series_id': _series_id, 'group': _group, 'length_x': config.QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER, 'length_y': config.QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER, 'length_z': config.QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER, 'offset_x': 0, 'offset_y': _offset_y, 'offset_z': 0, 'cell_id': _cell_id, 'direction': 'inside', 'time_points': _latest_time_frame }) _windows_dictionary, _windows_to_compute = \ compute.windows(_arguments, _keys=['experiment', 'series_id', 'group', 'cell_id', 'offset_y']) _fiber_densities = compute.fiber_densities(_windows_to_compute) _experiments_fiber_densities = { _key: [_fiber_densities[_tuple] for _tuple in _windows_dictionary[_key]] for _key in _windows_dictionary } _headers = [ 'time_frame', 'experiment', 'series_id', 'group', 'left_cell_id', 'right_cell_id', 'band', 'fake_following', 'fake_static', 'pair_distance_in_cell_diameter', 'offset_z', 'left_cell_fiber_density', 'left_cell_fiber_density_z_score', 'left_cell_out_of_boundaries', 'right_cell_fiber_density', 'right_cell_fiber_density_z_score', 'right_cell_out_of_boundaries' ] _csv_path = os.path.join(paths.OUTPUTS, 'experiments_density_cell_pairs.csv') with open(_csv_path, 'w', newline='') as _csv_file: _csv_writer = csv.writer(_csv_file) _csv_writer.writerow(_headers) for _tuple in tqdm(_tuples, desc='Main loop'): _experiment, _series_id, _group = _tuple _group_properties = load.group_properties(_experiment, _series_id, _group) _left_cell_id, _right_cell_id = _group_properties[ 'cells_ids'].values() _band = _group_properties['band'] _fake_following, _fake_static = _group_properties[ 'fake'], _group_properties['static'] _average, _std = load.normalization_series_file_data( _experiment, _series_id).values() for _offset_y in OFFSETS_Y: _left_cell_fiber_densities = \ _experiments_fiber_densities[(_experiment, _series_id, _group, 'left_cell', _offset_y)] _right_cell_fiber_densities = \ _experiments_fiber_densities[(_experiment, _series_id, _group, 'right_cell', _offset_y)] _left_cell_fiber_densities = compute.remove_blacklist( _experiment, _series_id, _left_cell_id, _left_cell_fiber_densities) _right_cell_fiber_densities = compute.remove_blacklist( _experiment, _series_id, _right_cell_id, _right_cell_fiber_densities) for _time_frame, (_left_cell_fiber_density, _right_cell_fiber_density) in \ enumerate(zip(_left_cell_fiber_densities, _right_cell_fiber_densities)): _pair_distance = compute.pair_distance_in_cell_size_time_frame( _experiment, _series_id, _group, _time_frame) _left_cell_fiber_density, _left_cell_out_of_boundaries = _left_cell_fiber_density _right_cell_fiber_density, _right_cell_out_of_boundaries = _right_cell_fiber_density _left_cell_fiber_density_z_score = compute_lib.z_score( _left_cell_fiber_density, _average, _std) _right_cell_fiber_density_z_score = compute_lib.z_score( _right_cell_fiber_density, _average, _std) _csv_writer.writerow([ _time_frame, _experiment, _series_id, _group, _left_cell_id, _right_cell_id, _band, _fake_following, _fake_static, _pair_distance, _offset_y, _left_cell_fiber_density, _left_cell_fiber_density_z_score, _left_cell_out_of_boundaries, _right_cell_fiber_density, _right_cell_fiber_density_z_score, _right_cell_out_of_boundaries ])
def compute_cell_pairs(): _x_array = [] _y_array = [] _names_array = [] for _distances_range in PAIR_DISTANCE_RANGES: print('Pair distance range:', str(_distances_range)) _experiments = all_experiments() _experiments = filtering.by_categories( _experiments=_experiments, _is_single_cell=False, _is_high_temporal_resolution=False, _is_bleb=False, _is_dead_dead=False, _is_live_dead=False, _is_bead=False, _is_metastasis=False) _tuples = load.experiments_groups_as_tuples(_experiments) _tuples = filtering.by_time_frames_amount( _tuples, compute.density_time_frame(_experiments[0])) _tuples = filtering.by_real_pairs(_tuples) _tuples = filtering.by_pair_distance_range(_tuples, _distances_range) _tuples = filtering.by_band(_tuples) print('Total tuples:', len(_tuples)) _max_offsets_x = [] _arguments = [] for _tuple in _tuples: _experiment, _series_id, _group = _tuple _time_frame = compute.minimum_time_frames_for_correlation( _experiment) _pair_distance = \ compute.pair_distance_in_cell_size_time_frame(_experiment, _series_id, _group, _time_frame - 1) _offsets_x = np.arange( start=0, stop=_pair_distance / 2 - 0.5 - QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER, step=OFFSET_X_STEP) if len(_offsets_x) > len(_max_offsets_x): _max_offsets_x = _offsets_x for _offset_x in _offsets_x: for _cell_id in ['left_cell', 'right_cell']: _arguments.append({ 'experiment': _experiment, 'series_id': _series_id, 'group': _group, 'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER, 'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER, 'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER, 'offset_x': _offset_x, 'offset_y': OFFSET_Y, 'offset_z': OFFSET_Z, 'cell_id': _cell_id, 'direction': 'inside', 'time_point': _time_frame - 1 }) _windows_dictionary, _windows_to_compute = \ compute.windows(_arguments, _keys=['experiment', 'series_id', 'group', 'offset_x', 'cell_id']) _fiber_densities = compute.fiber_densities(_windows_to_compute) _pair_distance_fiber_densities = [[] for _i in range(len(_max_offsets_x))] for _tuple in _tuples: _experiment, _series_id, _group = _tuple for _offset_x_index, _offset_x in enumerate(_max_offsets_x): for _cell_id in ['left_cell', 'right_cell']: if (_experiment, _series_id, _group, _offset_x, _cell_id) in _windows_dictionary: _normalization = load.normalization_series_file_data( _experiment, _series_id) _window_tuple = _windows_dictionary[(_experiment, _series_id, _group, _offset_x, _cell_id)][0] _fiber_density = _fiber_densities[_window_tuple] if not OUT_OF_BOUNDARIES and _fiber_density[1]: continue _normalized_fiber_density = compute_lib.z_score( _x=_fiber_density[0], _average=_normalization['average'], _std=_normalization['std']) if not np.isnan(_normalized_fiber_density): _pair_distance_fiber_densities[ _offset_x_index].append( _normalized_fiber_density) _x_array.append(_max_offsets_x) _y_array.append(_pair_distance_fiber_densities) _names_array.append('Pair distance ' + str(_distances_range[0]) + '-' + str(_distances_range[1])) return _names_array, _x_array, _y_array
def main(): _experiments = all_experiments() _experiments = filtering.by_categories(_experiments=_experiments, _is_single_cell=False, _is_high_temporal_resolution=False, _is_bleb=False, _is_dead_dead=False, _is_live_dead=False, _is_bead=False, _is_metastasis=False) _tuples = load.experiments_groups_as_tuples(_experiments) _tuples = filtering.by_time_frames_amount( _tuples, compute.density_time_frame(_experiments[0])) _tuples = filtering.by_real_pairs(_tuples) _tuples = filtering.by_band(_tuples) print('Total tuples:', len(_tuples)) _arguments = [] for _tuple in _tuples: _experiment, _series_id, _group = _tuple _time_frame = compute.density_time_frame(_experiment) _pair_distance = \ compute.pair_distance_in_cell_size_time_frame(_experiment, _series_id, _group, _time_frame=_time_frame - 1) for _offset_x in OFFSETS_X: if _pair_distance / 2 - 0.5 - QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER >= _offset_x: for _cell_id in ['left_cell', 'right_cell']: _arguments.append({ 'experiment': _experiment, 'series_id': _series_id, 'group': _group, 'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER, 'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER, 'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER, 'offset_x': _offset_x, 'offset_y': OFFSET_Y, 'offset_z': OFFSET_Z, 'cell_id': _cell_id, 'direction': 'inside', 'time_point': _time_frame - 1 }) _windows_dictionary, _windows_to_compute = \ compute.windows(_arguments, _keys=['experiment', 'series_id', 'group', 'cell_id', 'offset_x']) _fiber_densities = compute.fiber_densities(_windows_to_compute) for _offset_x in OFFSETS_X: _x_array = [] _y_array = [] for _tuple in _tuples: _experiment, _series_id, _group = _tuple for _cell_id in ['left_cell', 'right_cell']: if (_experiment, _series_id, _group, _cell_id, _offset_x) in _windows_dictionary: _pair_distance = \ compute.pair_distance_in_cell_size_time_frame(_experiment, _series_id, _group, _time_frame=0) _normalization = load.normalization_series_file_data( _experiment, _series_id) _window_tuple = _windows_dictionary[(_experiment, _series_id, _group, _cell_id, _offset_x)][0] _fiber_density = _fiber_densities[_window_tuple] if not OUT_OF_BOUNDARIES and _fiber_density[1]: continue _normalized_fiber_density = compute_lib.z_score( _x=_fiber_density[0], _average=_normalization['average'], _std=_normalization['std']) if not np.isnan(_normalized_fiber_density): _x_array.append(_pair_distance) _y_array.append(_normalized_fiber_density) print('Offset x (cell diameter):', _offset_x) print('Total pairs:', len(_x_array)) print(compute_lib.correlation(_x_array, _y_array, _with_p_value=True)) # plot _fig = go.Figure(data=go.Scatter(x=_x_array, y=_y_array, mode='markers', marker={ 'size': 15, 'color': 'black' }), layout={ 'xaxis': { 'title': 'Pair distance (cell diameter)', 'zeroline': False }, 'yaxis': { 'title': 'Fiber density (z-score)', 'zeroline': False, 'range': [-2.2, 13], 'tickmode': 'array', 'tickvals': [0, 4, 8, 12] }, 'shapes': [{ 'type': 'line', 'x0': 4.5, 'y0': -2, 'x1': 9.5, 'y1': -2, 'line': { 'color': 'black', 'width': 2 } }, { 'type': 'line', 'x0': 4.5, 'y0': -2, 'x1': 4.5, 'y1': 13, 'line': { 'color': 'black', 'width': 2 } }] }) save.to_html(_fig=_fig, _path=os.path.join(paths.PLOTS, save.get_module_name()), _filename='plot_offset_x_' + str(_offset_x))
def main(): _simulations = load.structured() _simulations = filtering.by_categories( _simulations, _is_single_cell=False, _is_heterogeneity=None, _is_low_connectivity=False, _is_causality=CAUSALITY, _is_dominant_passive=DOMINANT_PASSIVE, _is_fibrin=False) _simulations = filtering.by_pair_distances(_simulations, _distances=PAIR_DISTANCE) print('Total simulations:', len(_simulations)) _arguments = [] for _simulation in _simulations: for _cell_id in ['left_cell', 'right_cell']: _arguments.append({ 'simulation': _simulation, 'length_x': config.QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER, 'length_y': config.QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER, 'offset_x': 0, 'offset_y': 0, 'cell_id': _cell_id, 'direction': 'inside', 'time_points': TIME_POINTS }) _fiber_densities = {} with Pool(CPUS_TO_USE) as _p: for _keys, _value in tqdm(_p.imap_unordered( compute.window_fiber_density_by_time, _arguments), total=len(_arguments), desc='Computing windows & fiber densities'): _fiber_densities[(_keys['simulation'], _keys['cell_id'])] = _value _p.close() _p.join() _simulations_by_heterogeneity = organize.by_heterogeneity(_simulations) _headers = [ 'time_point', 'simulation', 'pair_distance_in_cell_diameter', 'heterogeneity_std', 'left_cell_fiber_density', 'left_cell_fiber_density_z_score', 'right_cell_fiber_density', 'right_cell_fiber_density_z_score', ] _csv_path = os.path.join(paths.OUTPUTS, 'simulations_density_cell_pairs.csv') with open(_csv_path, 'w', newline='') as _csv_file: _csv_writer = csv.writer(_csv_file) _csv_writer.writerow(_headers) for _simulation_std in _simulations_by_heterogeneity: print('STD:', _simulation_std) _std_simulations = _simulations_by_heterogeneity[_simulation_std] for _simulation in tqdm(_std_simulations, desc='Simulations loop'): _properties = load.properties(_simulation) _pair_distance = compute.pair_distance(_properties) _average, _std = load.normalization(_simulation).values() _left_cell_fiber_densities = _fiber_densities[(_simulation, 'left_cell')] _right_cell_fiber_densities = _fiber_densities[(_simulation, 'right_cell')] for _time_point, (_left_cell_fiber_density, _right_cell_fiber_density) in \ enumerate(zip(_left_cell_fiber_densities, _right_cell_fiber_densities)): _left_cell_fiber_density_z_score = compute_lib.z_score( _left_cell_fiber_density, _average, _std) _right_cell_fiber_density_z_score = compute_lib.z_score( _right_cell_fiber_density, _average, _std) _csv_writer.writerow([ _time_point, _simulation, _pair_distance, _simulation_std, _left_cell_fiber_density, _left_cell_fiber_density_z_score, _right_cell_fiber_density, _right_cell_fiber_density_z_score ])
def main(_band=None, _high_temporal_resolution=True, _tuples_to_mark=None, _tuples_to_plot=None, _plots=None): if _plots is None: _plots = ['whiteness', 'granger'] _experiments = all_experiments() _experiments = filtering.by_categories( _experiments=_experiments, _is_single_cell=False, _is_high_temporal_resolution=_high_temporal_resolution, _is_bleb=False, _is_dead_dead=False, _is_live_dead=False, _is_bead=False, _is_metastasis=False) _tuples = load.experiments_groups_as_tuples(_experiments) _tuples = filtering.by_time_frames_amount(_tuples, _time_frames=MINIMUM_TIME_FRAMES) _tuples = filtering.by_pair_distance_range( _tuples, _distance_range=PAIR_DISTANCE_RANGE) _tuples = filtering.by_real_pairs(_tuples) _tuples = filtering.by_band(_tuples, _band=_band) print('Total tuples:', len(_tuples)) _arguments = [] for _tuple in _tuples: _experiment, _series_id, _group = _tuple _latest_time_frame = compute.latest_time_frame_before_overlapping( _experiment, _series_id, _group, OFFSET_X) for _cell_id in ['left_cell', 'right_cell']: _arguments.append({ 'experiment': _experiment, 'series_id': _series_id, 'group': _group, 'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER, 'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER, 'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER, 'offset_x': OFFSET_X, 'offset_y': OFFSET_Y, 'offset_z': OFFSET_Z, 'cell_id': _cell_id, 'direction': 'inside', 'time_points': _latest_time_frame }) _windows_dictionary, _windows_to_compute = compute.windows( _arguments, _keys=['experiment', 'series_id', 'group', 'cell_id']) _fiber_densities = compute.fiber_densities(_windows_to_compute, _subtract_border=True) _experiments_fiber_densities = { _key: [_fiber_densities[_tuple] for _tuple in _windows_dictionary[_key]] for _key in _windows_dictionary } _n_pairs = 0 _n_pairs_with_band = 0 _whiteness_p_values = [] _n_passed_whiteness_with_band = 0 _granger_causality_p_values = [] _n_passed_granger_causality_with_band = 0 _correlations = [] _time_lag_correlations = [] _end_fiber_densities = [] for _tuple in _tuples: _experiment, _series_id, _group = _tuple _left_cell_fiber_densities = \ _experiments_fiber_densities[(_experiment, _series_id, _group, 'left_cell')] _right_cell_fiber_densities = \ _experiments_fiber_densities[(_experiment, _series_id, _group, 'right_cell')] _properties = load.group_properties(_experiment, _series_id, _group) _left_cell_fiber_densities = compute.remove_blacklist( _experiment, _series_id, _properties['cells_ids']['left_cell'], _left_cell_fiber_densities) _right_cell_fiber_densities = compute.remove_blacklist( _experiment, _series_id, _properties['cells_ids']['right_cell'], _right_cell_fiber_densities) _left_cell_fiber_densities_filtered, _right_cell_fiber_densities_filtered = \ compute.longest_same_indices_shared_in_borders_sub_array( _left_cell_fiber_densities, _right_cell_fiber_densities) # ignore small arrays if len(_left_cell_fiber_densities_filtered) < MINIMUM_TIME_FRAMES: continue _n_pairs += 1 if _properties['band']: _n_pairs_with_band += 1 _start_time_frame = 0 for _left in _left_cell_fiber_densities: if _left[0] == _left_cell_fiber_densities_filtered[0]: break _start_time_frame += 1 # stationary test with warnings.catch_warnings(): warnings.simplefilter('ignore', category=InterpolationWarning) # find derivative for stationary for _derivative in range(10): _left_cell_fiber_densities_derivative = \ compute_lib.derivative(_left_cell_fiber_densities_filtered, _n=_derivative) _right_cell_fiber_densities_derivative = \ compute_lib.derivative(_right_cell_fiber_densities_filtered, _n=_derivative) if ADF_TEST: _, _left_cell_adf_p_value, _, _, _, _ = adfuller( _left_cell_fiber_densities_derivative) _, _right_cell_adf_p_value, _, _, _, _ = adfuller( _right_cell_fiber_densities_derivative) if _left_cell_adf_p_value > 0.05 or _right_cell_adf_p_value > 0.05: continue if KPSS_TEST: _, _left_cell_kpss_p_value, _, _ = kpss( _left_cell_fiber_densities_derivative, nlags='legacy') _, _right_cell_kpss_p_value, _, _ = kpss( _right_cell_fiber_densities_derivative, nlags='legacy') if _left_cell_kpss_p_value < 0.05 or _right_cell_kpss_p_value < 0.05: continue # stationary break # causality try: _x = pd.DataFrame(data=[[_left_value, _right_value] for _left_value, _right_value in zip( _left_cell_fiber_densities_derivative, _right_cell_fiber_densities_derivative) ], columns=['left', 'right']) # var model to retrieve lag _var_model = VAR(_x) _lag_order_results = _var_model.select_order() _estimators_lags = [ _lag_order_results.aic, _lag_order_results.bic, _lag_order_results.fpe, _lag_order_results.hqic ] _min_estimator_lag = min(_estimators_lags) # found a lag if 0 < _min_estimator_lag <= MAXIMUM_LAG: _var_model_results = _var_model.fit(maxlags=_min_estimator_lag, ic=None) _whiteness = _var_model_results.test_whiteness( nlags=_min_estimator_lag + 1) _whiteness_p_values.append(_whiteness.pvalue) if _tuples_to_mark is not None and _tuple in _tuples_to_mark and _whiteness.pvalue > 0.05: print(_tuple, 'marked whiteness p-value:', _whiteness.pvalue) # no autocorrelation in the residuals if _whiteness.pvalue > 0.05: if _properties['band']: _n_passed_whiteness_with_band += 1 # time lag = 0 _correlation = compute_lib.correlation( _left_cell_fiber_densities_derivative, _right_cell_fiber_densities_derivative) # if _correlation < 0.5: # continue # granger causality for _caused, _causing in zip(['left', 'right'], ['right', 'left']): _granger = _var_model_results.test_causality( caused=_caused, causing=_causing) _granger_causality_p_values.append(_granger.pvalue) # time lag = 0 _correlations.append(_correlation) # time lag = min estimator if _causing == 'left': _left_fiber_densities_time_lag = \ _left_cell_fiber_densities_derivative[:-_min_estimator_lag] _right_fiber_densities_time_lag = \ _right_cell_fiber_densities_derivative[_min_estimator_lag:] else: _left_fiber_densities_time_lag = \ _left_cell_fiber_densities_derivative[_min_estimator_lag:] _right_fiber_densities_time_lag = \ _right_cell_fiber_densities_derivative[:-_min_estimator_lag] _time_lag_correlation = compute_lib.correlation( _left_fiber_densities_time_lag, _right_fiber_densities_time_lag) _time_lag_correlations.append(_time_lag_correlation) # end fiber density _time_frame = compute.density_time_frame(_experiment) if len(_left_cell_fiber_densities_filtered ) > _time_frame: _end_fiber_density = \ (_left_cell_fiber_densities_filtered[_time_frame] + _right_cell_fiber_densities_filtered[_time_frame]) / 2 else: _end_fiber_density = \ (_left_cell_fiber_densities_filtered[-1] + _right_cell_fiber_densities_filtered[-1]) / 2 _normalization = load.normalization_series_file_data( _experiment, _series_id) _normalized_fiber_density = compute_lib.z_score( _end_fiber_density, _normalization['average'], _normalization['std']) _end_fiber_densities.append(_normalized_fiber_density) # marking if _tuples_to_mark is not None and _tuple in _tuples_to_mark and _granger.pvalue < 0.05: print(_tuple, 'causing:', _causing, 'marked granger p-value:', _granger.pvalue) if _granger.pvalue < 0.05: if _properties['band']: _n_passed_granger_causality_with_band += 1 _normality = _var_model_results.test_normality() _inst_granger = _var_model_results.test_inst_causality( causing=_causing) print( _tuple, _causing.capitalize() + ' causes ' + _caused + '!', 'time-points: ' + str(len( _left_cell_fiber_densities_derivative)), 'stationary derivative: ' + str(_derivative), 'band:' + str(_properties['band']), 'p-value: ' + str(round(_granger.pvalue, 4)), 'lag: ' + str(_min_estimator_lag), 'normality p-value: ' + str(round(_normality.pvalue, 4)), 'inst p-value: ' + str(round(_inst_granger.pvalue, 4)), sep='\t') # lag = 0 print('Time lag = 0 correlation:', _correlation) # rest of lags for _lag in range(1, _min_estimator_lag + 1): if _causing == 'left': _left_fiber_densities_time_lag = _left_cell_fiber_densities_derivative[: -_lag] _right_fiber_densities_time_lag = _right_cell_fiber_densities_derivative[ _lag:] else: _left_fiber_densities_time_lag = _left_cell_fiber_densities_derivative[ _lag:] _right_fiber_densities_time_lag = _right_cell_fiber_densities_derivative[: -_lag] _correlation = compute_lib.correlation( _left_fiber_densities_time_lag, _right_fiber_densities_time_lag) print( 'Time lag = ' + str(_lag) + ' correlation:', _correlation) # plots if _tuples_to_plot is not None and _tuple in _tuples_to_plot: _y_arrays = [ _left_cell_fiber_densities_derivative, _right_cell_fiber_densities_derivative ] _names_array = ['Left cell', 'Right cell'] _colors_array = config.colors(2) _temporal_resolution = compute.temporal_resolution_in_minutes( _experiment) _fig = go.Figure(data=[ go.Scatter(x=np.arange( start=_start_time_frame, stop=_start_time_frame + len(_left_cell_fiber_densities_derivative ), step=1) * _temporal_resolution, y=_y, name=_name, mode='lines', line={ 'color': _color, 'width': 1 }) for _y, _name, _color in zip( _y_arrays, _names_array, _colors_array) ], layout={ 'xaxis': { 'title': 'Time (minutes)', 'zeroline': False }, 'yaxis': { 'title': 'Fiber density (z-score)' + '\'' * _derivative, 'zeroline': False }, 'legend': { 'xanchor': 'left', 'x': 0.1, 'yanchor': 'top', 'bordercolor': 'black', 'borderwidth': 2, 'bgcolor': 'white' }, }) _experiment, _series_id, _group = _tuple save.to_html( _fig=_fig, _path=os.path.join(paths.PLOTS, save.get_module_name()), _filename='plot_' + _experiment + '_' + str(_series_id) + '_' + _group) # residuals _y_arrays = \ [_var_model_results.resid.values[:, 0], _var_model_results.resid.values[:, 1]] _fig = go.Figure(data=[ go.Scatter(x=np.arange( start=_start_time_frame, stop=_start_time_frame + len(_y), step=1) * _temporal_resolution, y=_y, name=_name, mode='lines', line={ 'color': _color, 'width': 1 }) for _y, _name, _color in zip( _y_arrays, _names_array, _colors_array) ], layout={ 'xaxis': { 'title': 'Time (minutes)', 'zeroline': False }, 'yaxis': { 'title': 'Residual', 'zeroline': False }, 'legend': { 'xanchor': 'left', 'x': 0.1, 'yanchor': 'top', 'bordercolor': 'black', 'borderwidth': 2, 'bgcolor': 'white' }, }) _experiment, _series_id, _group = _tuple save.to_html( _fig=_fig, _path=os.path.join(paths.PLOTS, save.get_module_name()), _filename='plot_residuals_' + _experiment + '_' + str(_series_id) + '_' + _group) # not enough time points except ValueError: continue print('Total pairs:', _n_pairs) print('Total pairs with band:', _n_pairs_with_band) print('Total pairs passed whiteness:', (np.array(_whiteness_p_values) > 0.05).sum()) print('Total pairs passed whiteness with band:', _n_passed_whiteness_with_band) print('Total cells passed granger causality:', (np.array(_granger_causality_p_values) < 0.05).sum()) print('Total cells passed granger causality with band:', _n_passed_granger_causality_with_band) # p-value correction print('Corrections of GC p-value < 0.05:') _granger_causality_p_values_corrected = multipletests( pvals=_granger_causality_p_values, method='fdr_bh') for _p_value, _p_value_corrected in zip( _granger_causality_p_values, _granger_causality_p_values_corrected[1]): if _p_value < 0.05: print('Original GC p-value:', _p_value, 'corrected:', _p_value_corrected) # plots for _test_name, _y_title, _y_array in \ zip( ['whiteness', 'granger'], ['Whiteness p-value', 'Granger causality p-value'], [_whiteness_p_values, _granger_causality_p_values] ): if _test_name in _plots: _fig = go.Figure(data=go.Box(y=_y_array, boxpoints='all', jitter=1, pointpos=0, line={'width': 1}, fillcolor='white', marker={ 'size': 10, 'color': '#ea8500' }, opacity=0.7, showlegend=False), layout={ 'xaxis': { 'zeroline': False }, 'yaxis': { 'title': _y_title, 'zeroline': False, 'range': [-0.1, 1.1], 'tickmode': 'array', 'tickvals': [0.05, 1] }, 'shapes': [{ 'type': 'line', 'x0': -0.75, 'y0': 0.05, 'x1': 0.75, 'y1': 0.05, 'line': { 'color': 'red', 'width': 2, 'dash': 'dash' } }] }) save.to_html(_fig=_fig, _path=os.path.join(paths.PLOTS, save.get_module_name()), _filename='plot_' + _test_name) # granger versus correlation print( 'GC vs. correlation pearson correlation:', compute_lib.correlation(_granger_causality_p_values, _correlations, _with_p_value=True)) _fig = go.Figure(data=go.Scatter(x=_granger_causality_p_values, y=_correlations, mode='markers', marker={ 'size': 10, 'color': '#ea8500' }, showlegend=False), layout={ 'xaxis': { 'title': 'Granger causality p-value', 'zeroline': False, }, 'yaxis': { 'title': 'Inner correlation', 'zeroline': False, } }) save.to_html(_fig=_fig, _path=os.path.join(paths.PLOTS, save.get_module_name()), _filename='plot_gc_vs_correlation') # granger versus time lag correlation print( 'GC vs. time lag correlation pearson correlation:', compute_lib.correlation(_granger_causality_p_values, _time_lag_correlations, _with_p_value=True)) _fig = go.Figure(data=go.Scatter(x=_granger_causality_p_values, y=_time_lag_correlations, mode='markers', marker={ 'size': 10, 'color': '#ea8500' }, showlegend=False), layout={ 'xaxis': { 'title': 'Granger causality p-value', 'zeroline': False, }, 'yaxis': { 'title': 'GC lag inner correlation', 'zeroline': False, } }) save.to_html(_fig=_fig, _path=os.path.join(paths.PLOTS, save.get_module_name()), _filename='plot_gc_vs_time_lag_correlation') # granger versus end fiber density print( 'GC vs. end fiber density pearson correlation:', compute_lib.correlation(_granger_causality_p_values, _end_fiber_densities, _with_p_value=True)) _fig = go.Figure(data=go.Scatter(x=_granger_causality_p_values, y=_end_fiber_densities, mode='markers', marker={ 'size': 10, 'color': '#ea8500' }, showlegend=False), layout={ 'xaxis': { 'title': 'Granger causality p-value', 'zeroline': False, }, 'yaxis': { 'title': 'End fiber density (z-score)', 'zeroline': False, } }) save.to_html(_fig=_fig, _path=os.path.join(paths.PLOTS, save.get_module_name()), _filename='plot_gc_vs_end_density')