def main(): # single cells print('Single cells') _single_cells = load.experiments_groups_as_tuples(SINGLE_CELLS) _single_cells = filtering.by_main_cell(_single_cells) _single_cells = organize.by_single_cell_id(_single_cells) print('\tTotal single cell experiments:', len(_single_cells)) # cell pairs _cell_pairs = load.experiments_groups_as_tuples(CELL_PAIRS) print('\nTotal cell pairs:', len(_cell_pairs)) for _distance_range in DISTANCE_RANGES: print('Cell pairs distance range:', _distance_range) # total _distance_range_cell_pairs = filtering.by_pair_distance_range(_cell_pairs, _distance_range=_distance_range) print('\tTotal cell pairs:', len(_distance_range_cell_pairs)) for _experiment in CELL_PAIRS: print('\tExperiment:', _experiment) _experiment_cell_pairs = load.experiment_groups_as_tuples(_experiment) _distance_range_experiment_cell_pairs = \ filtering.by_pair_distance_range(_experiment_cell_pairs, _distance_range=_distance_range) print('\t\tTotal cell pairs:', len(_distance_range_experiment_cell_pairs)) # real cell pairs _real_cell_pairs = filtering.by_real_pairs(_distance_range_experiment_cell_pairs) print('\t\tTotal real cell pairs:', len(_real_cell_pairs)) _real_cell_pairs_with_band = filtering.by_band(_real_cell_pairs) print('\t\tTotal real cell pairs with band:', len(_real_cell_pairs_with_band)) # fake cell pairs _fake_cell_pairs = filtering.by_real_pairs(_distance_range_experiment_cell_pairs, _real_pairs=False) print('\t\tTotal fake cell pairs:', len(_fake_cell_pairs)) _fake_static_cell_pairs = filtering.by_fake_static_pairs(_fake_cell_pairs) print('\t\tTotal fake static cell pairs:', len(_fake_static_cell_pairs)) _fake_following_cell_pairs = filtering.by_fake_static_pairs(_fake_cell_pairs, _fake_static_pairs=False) print('\t\tTotal fake following cell pairs:', len(_fake_following_cell_pairs)) _fake_following_cell_pairs_with_band = filtering.by_band(_fake_following_cell_pairs) print('\t\tTotal fake following cell pairs with band:', len(_fake_following_cell_pairs_with_band))
def main(): _experiments = all_experiments() _experiments = filtering.by_categories(_experiments=_experiments, _is_single_cell=True, _is_high_temporal_resolution=False, _is_bleb=False, _is_dead_dead=False, _is_live_dead=False, _is_bead=False, _is_metastasis=False) _tuples = load.experiments_groups_as_tuples(_experiments) _tuples = filtering.by_time_frames_amount( _tuples, compute.density_time_frame(_experiments[0])) _tuples = filtering.by_main_cell(_tuples) _arguments = [] for _tuple in _tuples: _experiment, _series_id, _group = _tuple _time_frame = compute.density_time_frame(_experiment) for _direction in ['left', 'right']: _arguments.append({ 'experiment': _experiment, 'series_id': _series_id, 'group': _group, 'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER, 'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER, 'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER, 'offset_x': OFFSET_X, 'offset_y': OFFSET_Y, 'offset_z': OFFSET_Z, 'cell_id': 'cell', 'direction': _direction, 'time_points': _time_frame }) _windows_dictionary, _windows_to_compute = \ compute.windows(_arguments, _keys=['experiment', 'series_id', 'group', 'direction']) _fiber_densities = compute.fiber_densities(_windows_to_compute, _subtract_border=True) _tuples = organize.by_single_cell_id(_tuples) print('Total experiments:', len(_tuples)) _kpss_y_arrays = [[] for _i in DERIVATIVES] _adf_y_arrays = [[] for _i in DERIVATIVES] for _tuple in tqdm(_tuples, desc='Experiments loop'): _experiment, _series_id, _cell_id = _tuple _cell_fiber_densities = compute_single_cell_mean( _experiment=_experiment, _series_id=_series_id, _cell_tuples=_tuples[_tuple], _windows_dictionary=_windows_dictionary, _fiber_densities=_fiber_densities) for _derivative_index, _derivative in enumerate(DERIVATIVES): _cell_fiber_densities_derivative = compute_lib.derivative( _cell_fiber_densities, _n=_derivative) _, _kpss_p_value, _, _ = kpss(_cell_fiber_densities_derivative, nlags='legacy') _kpss_y_arrays[_derivative_index].append(_kpss_p_value) _, _adf_p_value, _, _, _, _ = adfuller( _cell_fiber_densities_derivative) _adf_y_arrays[_derivative_index].append(_adf_p_value) print('Total cells:', len(_kpss_y_arrays[0])) # print results print('KPSS:') for _derivative_index, _derivative in enumerate(DERIVATIVES): _stationary_count = len([ _value for _value in _kpss_y_arrays[_derivative_index] if _value > 0.05 ]) print( 'Derivative:', _derivative, 'Stationary:', str(_stationary_count / len(_kpss_y_arrays[_derivative_index]) * 100) + '%') print('ADF:') for _derivative_index, _derivative in enumerate(DERIVATIVES): _stationary_count = len([ _value for _value in _adf_y_arrays[_derivative_index] if _value < 0.05 ]) print( 'Derivative:', _derivative, 'Stationary:', str(_stationary_count / len(_adf_y_arrays[_derivative_index]) * 100) + '%') # plot _colors_array = config.colors(3) for _test_name, _y_title, _y_tickvals, _p_value_line, _y_arrays in \ zip( ['kpss', 'adf'], ['KPSS test p-value', 'ADF test p-value'], [[0.05, 0.1], [0.05, 1]], [0.05, 0.05], [_kpss_y_arrays, _adf_y_arrays] ): _fig = go.Figure(data=[ go.Box(y=_y, name=_derivative, boxpoints='all', jitter=1, pointpos=0, line={'width': 1}, fillcolor='white', marker={ 'size': 10, 'color': _color }, opacity=0.7, showlegend=False) for _y, _derivative, _color in zip( _y_arrays, DERIVATIVES_TEXT, _colors_array) ], layout={ 'xaxis': { 'title': 'Fiber density derivative', 'zeroline': False }, 'yaxis': { 'title': _y_title, 'zeroline': False, 'tickmode': 'array', 'tickvals': _y_tickvals }, 'shapes': [{ 'type': 'line', 'x0': DERIVATIVES[0] - 0.75, 'y0': _p_value_line, 'x1': DERIVATIVES[-1] + 0.75, 'y1': _p_value_line, 'line': { 'color': 'red', 'width': 2, 'dash': 'dash' } }] }) save.to_html(_fig=_fig, _path=os.path.join(paths.PLOTS, save.get_module_name()), _filename='plot_' + _test_name)
def main(): print('Single Cell') _experiments = all_experiments() _experiments = filtering.by_categories(_experiments=_experiments, _is_single_cell=True, _is_high_temporal_resolution=False, _is_bleb=False, _is_dead_dead=False, _is_live_dead=False, _is_bead=False, _is_metastasis=False) _tuples = load.experiments_groups_as_tuples(_experiments) _tuples = filtering.by_time_frames_amount( _tuples, compute.minimum_time_frames_for_correlation(_experiments[0])) _arguments = [] for _tuple in _tuples: _experiment, _series_id, _group = _tuple _time_frame = compute.minimum_time_frames_for_correlation(_experiment) for _offset_x, _direction in product(OFFSETS_X, ['left', 'right']): _arguments.append({ 'experiment': _experiment, 'series_id': _series_id, 'group': _group, 'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER, 'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER, 'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER, 'offset_x': _offset_x, 'offset_y': OFFSET_Y, 'offset_z': OFFSET_Z, 'cell_id': 'cell', 'direction': _direction, 'time_point': _time_frame - 1 }) _windows_dictionary, _windows_to_compute = \ compute.windows(_arguments, _keys=['experiment', 'series_id', 'group', 'offset_x', 'direction']) _fiber_densities = compute.fiber_densities(_windows_to_compute) _tuples = organize.by_single_cell_id(_tuples) _single_cell_fiber_densities = [[] for _i in range(len(OFFSETS_X))] for _tuple in _tuples: _experiment, _series_id, _cell_id = _tuple print('Experiment:', _experiment, 'Series ID:', _series_id, 'Cell ID:', _cell_id, sep='\t') _offset_index = 0 _normalization = load.normalization_series_file_data( _experiment, _series_id) for _offset_x in OFFSETS_X: _cell_fiber_densities = [] for _cell_tuple in _tuples[_tuple]: _, _, _group = _cell_tuple for _direction in ['left', 'right']: _window_tuple = _windows_dictionary[(_experiment, _series_id, _group, _offset_x, _direction)][0] _fiber_density = _fiber_densities[_window_tuple] if not OUT_OF_BOUNDARIES and _fiber_density[1]: continue _normalized_fiber_density = compute_lib.z_score( _x=_fiber_density[0], _average=_normalization['average'], _std=_normalization['std']) _cell_fiber_densities.append(_normalized_fiber_density) if len(_cell_fiber_densities) > 0: _single_cell_fiber_densities[_offset_index].append( np.mean(_cell_fiber_densities)) _offset_index += 1 print('Pairs') _experiments = all_experiments() _experiments = filtering.by_categories(_experiments=_experiments, _is_single_cell=False, _is_high_temporal_resolution=False, _is_bleb=False, _is_dead_dead=False, _is_live_dead=False, _is_bead=False, _is_metastasis=False) _tuples = load.experiments_groups_as_tuples(_experiments) _tuples = filtering.by_time_frames_amount( _tuples, compute.minimum_time_frames_for_correlation(_experiments[0])) _tuples = filtering.by_real_pairs(_tuples) _tuples = filtering.by_pair_distance(_tuples, PAIR_DISTANCE) _tuples = filtering.by_band(_tuples) _arguments = [] for _tuple in _tuples: _experiment, _series_id, _group = _tuple _time_frame = compute.minimum_time_frames_for_correlation(_experiment) for _offset_x in OFFSETS_X: _arguments.append({ 'experiment': _experiment, 'series_id': _series_id, 'group': _group, 'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER, 'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER, 'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER, 'offset_x': _offset_x, 'offset_y': OFFSET_Y, 'offset_z': OFFSET_Z, 'cell_id': 'left_cell', 'direction': 'inside', 'time_point': _time_frame - 1 }) _windows_dictionary, _windows_to_compute = \ compute.windows(_arguments, _keys=['experiment', 'series_id', 'group', 'offset_x']) _fiber_densities = compute.fiber_densities(_windows_to_compute) _pairs_fiber_densities = [[] for _i in range(len(OFFSETS_X))] for _tuple in _tuples: _experiment, _series_id, _group = _tuple print('Experiment:', _experiment, 'Series ID:', _series_id, 'Group:', _group, sep='\t') _offset_index = 0 _normalization = load.normalization_series_file_data( _experiment, _series_id) # take offsets based on pair distance _properties = load.group_properties(_experiment, _series_id, _group) _left_cell_coordinates = [ list(_properties['time_points'][0]['left_cell'] ['coordinates'].values()) ] _right_cell_coordinates = [ list(_properties['time_points'][0]['right_cell'] ['coordinates'].values()) ] _pair_distance = compute.pair_distance_in_cell_size( _experiment, _series_id, _left_cell_coordinates, _right_cell_coordinates) _edges_distance = _pair_distance - 1 _max_x_offset = _edges_distance - QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER for _offset_x in OFFSETS_X: if _offset_x > _max_x_offset: break _fiber_density = _fiber_densities[_windows_dictionary[( _experiment, _series_id, _group, _offset_x)][0]] if not OUT_OF_BOUNDARIES and _fiber_density[1]: continue _normalized_fiber_density = compute_lib.z_score( _x=_fiber_density[0], _average=_normalization['average'], _std=_normalization['std']) _pairs_fiber_densities[_offset_index].append( _normalized_fiber_density) _offset_index += 1 # plot _fig = go.Figure(data=[ go.Scatter(x=OFFSETS_X, y=[np.mean(_array) for _array in _pairs_fiber_densities], name='Pairs', error_y={ 'type': 'data', 'array': [np.std(_array) for _array in _pairs_fiber_densities], 'thickness': 1 }, mode='lines+markers', line={'dash': 'solid'}), go.Scatter( x=OFFSETS_X, y=[np.mean(_array) for _array in _single_cell_fiber_densities], name='Single Cell', error_y={ 'type': 'data', 'array': [np.std(_array) for _array in _single_cell_fiber_densities], 'thickness': 1 }, mode='lines+markers', line={'dash': 'dash'}) ], layout={ 'xaxis_title': 'Distance from left cell (cell size)', 'yaxis_title': 'Fiber density (z-score)' }) save.to_html(_fig=_fig, _path=os.path.join(paths.PLOTS, save.get_module_name()), _filename='plot_distance_' + str(PAIR_DISTANCE))
def main(): _experiments = all_experiments() _experiments = filtering.by_categories(_experiments=_experiments, _is_single_cell=True, _is_high_temporal_resolution=False, _is_bleb=False, _is_dead_dead=False, _is_live_dead=False, _is_bead=False, _is_metastasis=False) _tuples = load.experiments_groups_as_tuples(_experiments) _tuples = filtering.by_time_frames_amount( _tuples, compute.density_time_frame(_experiments[0])) _tuples = filtering.by_main_cell(_tuples) _arguments = [] for _tuple in _tuples: _experiment, _series_id, _group = _tuple _time_frame = compute.density_time_frame(_experiment) for _direction in ['left', 'right']: _arguments.append({ 'experiment': _experiment, 'series_id': _series_id, 'group': _group, 'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER, 'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER, 'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER, 'offset_x': OFFSET_X, 'offset_y': OFFSET_Y, 'offset_z': OFFSET_Z, 'cell_id': 'cell', 'direction': _direction, 'time_points': _time_frame }) _windows_dictionary, _windows_to_compute = \ compute.windows(_arguments, _keys=['experiment', 'series_id', 'group', 'direction']) _fiber_densities = compute.fiber_densities(_windows_to_compute, _subtract_border=True) _tuples = organize.by_single_cell_id(_tuples) print('Total tuples:', len(_tuples)) _experiments_ids = list(_tuples.keys()) _y_arrays = [[] for _i in DERIVATIVES] for _index_1 in tqdm(range(len(_experiments_ids)), desc='Main loop'): _tuple_1 = _experiments_ids[_index_1] _experiment_1, _series_id_1, _cell_id_1 = _tuple_1 _fiber_densities_1 = compute_single_cell_mean( _experiment=_experiment_1, _series_id=_series_id_1, _cell_tuples=_tuples[_tuple_1], _windows_dictionary=_windows_dictionary, _fiber_densities=_fiber_densities) for _index_2 in range(_index_1 + 1, len(_experiments_ids)): _tuple_2 = _experiments_ids[_index_2] _experiment_2, _series_id_2, _cell_id_2 = _tuple_2 _fiber_densities_2 = compute_single_cell_mean( _experiment=_experiment_2, _series_id=_series_id_2, _cell_tuples=_tuples[_tuple_2], _windows_dictionary=_windows_dictionary, _fiber_densities=_fiber_densities) for _derivative_index, _derivative in enumerate(DERIVATIVES): _y_arrays[_derivative_index].append( compute_lib.correlation( compute_lib.derivative(_fiber_densities_1, _n=_derivative), compute_lib.derivative(_fiber_densities_2, _n=_derivative))) print('Total points:', len(_y_arrays[0])) print('Wilcoxon around the zero') for _y_array, _derivative in zip(_y_arrays, DERIVATIVES): print('Derivative:', _derivative, wilcoxon(_y_array)) # plot _colors_array = config.colors(3) _fig = go.Figure(data=[ go.Box(y=_y, name=_derivative, boxpoints='all', jitter=1, pointpos=0, line={'width': 1}, fillcolor='white', marker={ 'size': 10, 'color': _color }, opacity=0.7, showlegend=False) for _y, _derivative, _color in zip( _y_arrays, DERIVATIVES_TEXT, _colors_array) ], layout={ 'xaxis': { 'title': 'Fiber density derivative', 'zeroline': False }, 'yaxis': { 'title': 'Correlation', 'range': [-1, 1], 'zeroline': False, 'tickmode': 'array', 'tickvals': [-1, -0.5, 0, 0.5, 1] } }) save.to_html(_fig=_fig, _path=os.path.join(paths.PLOTS, save.get_module_name()), _filename='plot')
def compute_experiments_fiber_densities(): _experiments = all_experiments() _experiments = experiments_filtering.by_categories( _experiments=_experiments, _is_single_cell=True, _is_high_temporal_resolution=False, _is_bleb=False, _is_dead_dead=False, _is_live_dead=False, _is_bead=False, _is_metastasis=False) _tuples = experiments_load.experiments_groups_as_tuples(_experiments) _tuples = experiments_filtering.by_time_frames_amount( _tuples, EXPERIMENTS_TIME_FRAMES) _tuples = experiments_filtering.by_main_cell(_tuples) _arguments = [] for _tuple in _tuples: _experiment, _series_id, _group = _tuple for _direction in ['left', 'right']: _arguments.append({ 'experiment': _experiment, 'series_id': _series_id, 'group': _group, 'length_x': experiments_config. QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER, 'length_y': experiments_config. QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER, 'length_z': experiments_config. QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER, 'offset_x': OFFSET_X, 'offset_y': OFFSET_Y, 'offset_z': OFFSET_Z, 'cell_id': 'cell', 'direction': _direction, 'time_points': EXPERIMENTS_TIME_FRAMES }) _windows_dictionary, _windows_to_compute = \ experiments_compute.windows(_arguments, _keys=['experiment', 'series_id', 'group', 'direction']) _fiber_densities = experiments_compute.fiber_densities(_windows_to_compute) _tuples = experiments_organize.by_single_cell_id(_tuples) print('Total experiments:', len(_tuples)) _experiments_fiber_densities = [[] for _i in range(EXPERIMENTS_TIME_FRAMES)] for _tuple in tqdm(_tuples, desc='Experiments loop'): _experiment, _series_id, _ = _tuple _normalization = experiments_load.normalization_series_file_data( _experiment, _series_id) for _time_frame in range(EXPERIMENTS_TIME_FRAMES): _cell_fiber_densities = [] for _cell_tuple in _tuples[_tuple]: _, _, _group = _cell_tuple for _direction in ['left', 'right']: _window_tuple = _windows_dictionary[( _experiment, _series_id, _group, _direction)][_time_frame] _fiber_density = _fiber_densities[_window_tuple] if not OUT_OF_BOUNDARIES and _fiber_density[1]: continue _normalized_fiber_density = compute_lib.z_score( _x=_fiber_density[0], _average=_normalization['average'], _std=_normalization['std']) if not np.isnan(_normalized_fiber_density): _cell_fiber_densities.append(_normalized_fiber_density) if len(_cell_fiber_densities) > 0: _experiments_fiber_densities[_time_frame].append( np.mean(_cell_fiber_densities)) print('Total experiments cells:', len(_experiments_fiber_densities[0])) return _experiments_fiber_densities