def compute_data(_tuples, _arguments, _padding_y_by, _padding_z_by, _space_y_by, _space_z_by):
_windows_dictionary, _windows_to_compute = \
compute.windows(_arguments, _keys=['experiment', 'series_id', 'group', 'cell_id'])
_fiber_densities = compute.fiber_densities(_windows_to_compute, _subtract_border=True, _padding_y_by=_padding_y_by,
_padding_z_by=_padding_z_by, _space_y_by=_space_y_by,
_space_z_by=_space_z_by)
_experiments_fiber_densities = {
_key: [_fiber_densities[_tuple] for _tuple in _windows_dictionary[_key]]
for _key in _windows_dictionary
}
_correlations = []
for _tuple in _tuples:
_experiment, _series, _group = _tuple
_left_cell_fiber_densities = _experiments_fiber_densities[(_experiment, _series, _group, 'left_cell')]
_right_cell_fiber_densities = _experiments_fiber_densities[(_experiment, _series, _group, 'right_cell')]
_properties = load.group_properties(_experiment, _series, _group)
_left_cell_fiber_densities = compute.remove_blacklist(
_experiment,
_series,
_properties['cells_ids']['left_cell'],
_left_cell_fiber_densities
)
_right_cell_fiber_densities = compute.remove_blacklist(
_experiment,
_series,
_properties['cells_ids']['right_cell'],
_right_cell_fiber_densities
)
_left_cell_fiber_densities_filtered, _right_cell_fiber_densities_filtered = \
compute.longest_same_indices_shared_in_borders_sub_array(
_left_cell_fiber_densities, _right_cell_fiber_densities
)
# ignore small arrays
if len(_left_cell_fiber_densities_filtered) < compute.minimum_time_frames_for_correlation(_experiment):
continue
_correlation = compute_lib.correlation(
compute_lib.derivative(_left_cell_fiber_densities_filtered, _n=DERIVATIVE),
compute_lib.derivative(_right_cell_fiber_densities_filtered, _n=DERIVATIVE)
)
_correlations.append(_correlation)
return np.mean(_correlations)
def main():
_experiments = all_experiments()
_experiments = filtering.by_categories(_experiments=_experiments,
_is_single_cell=False,
_is_high_temporal_resolution=False,
_is_bleb=False,
_is_dead_dead=False,
_is_live_dead=False,
_is_bead=False,
_is_metastasis=False)
_tuples = load.experiments_groups_as_tuples(_experiments)
_tuples = filtering.by_time_frames_amount(
_tuples, compute.density_time_frame(_experiments[0]))
_tuples = filtering.by_real_pairs(_tuples)
_tuples = filtering.by_band(_tuples)
_max_offsets_x = []
_arguments = []
for _tuple in _tuples:
_experiment, _series_id, _group = _tuple
_time_frame = compute.minimum_time_frames_for_correlation(_experiment)
_pair_distance = \
compute.pair_distance_in_cell_size_time_frame(_experiment, _series_id, _group, _time_frame=_time_frame - 1)
_offsets_x = \
np.arange(start=0, stop=_pair_distance / 2 - 0.5 - QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER,
step=OFFSET_X_STEP)
if len(_offsets_x) > len(_max_offsets_x):
_max_offsets_x = _offsets_x
for _offset_x in _offsets_x:
for _cell_id in ['left_cell', 'right_cell']:
_arguments.append({
'experiment': _experiment,
'series_id': _series_id,
'group': _group,
'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER,
'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER,
'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER,
'offset_x': _offset_x,
'offset_y': OFFSET_Y,
'offset_z': OFFSET_Z,
'cell_id': _cell_id,
'direction': 'inside',
'time_point': _time_frame - 1
})
_windows_dictionary, _windows_to_compute = \
compute.windows(_arguments, _keys=['experiment', 'series_id', 'group', 'offset_x', 'cell_id'])
_fiber_densities = compute.fiber_densities(_windows_to_compute)
_x_array = []
_y_array = []
_n_array = []
_p_value_array = []
for _offset_x in _max_offsets_x:
_pair_distances = []
_z_scores = []
for _tuple in _tuples:
_experiment, _series_id, _group = _tuple
for _cell_id in ['left_cell', 'right_cell']:
if (_experiment, _series_id, _group, _offset_x,
_cell_id) in _windows_dictionary:
_normalization = load.normalization_series_file_data(
_experiment, _series_id)
_window_tuple = _windows_dictionary[(_experiment,
_series_id, _group,
_offset_x,
_cell_id)][0]
_fiber_density = _fiber_densities[_window_tuple]
if not OUT_OF_BOUNDARIES and _fiber_density[1]:
continue
_normalized_fiber_density = compute_lib.z_score(
_x=_fiber_density[0],
_average=_normalization['average'],
_std=_normalization['std'])
if not np.isnan(_normalized_fiber_density):
_pair_distances.append(
compute.pair_distance_in_cell_size_time_frame(
_experiment, _series_id, _group,
_time_frame=0))
_z_scores.append(_normalized_fiber_density)
if len(_pair_distances) > 2:
_x_array.append(round(_offset_x, 1))
_correlation = compute_lib.correlation(_pair_distances,
_z_scores,
_with_p_value=True)
_y_array.append(round(_correlation[0], 2))
_n_array.append(len(_pair_distances))
_p_value_array.append(round(_correlation[1], 2))
print('Pearson:')
print(compute_lib.correlation(_x_array, _y_array, _with_p_value=True))
# plot
_significant_x_array = [
_x for _x, _p_value in zip(_x_array, _p_value_array) if _p_value < 0.05
]
_fig = go.Figure(data=[
go.Scatter(x=_x_array,
y=_y_array,
mode='markers',
marker={
'size': 15,
'color': 'black'
},
showlegend=False),
go.Scatter(x=_significant_x_array,
y=[-0.79] * len(_significant_x_array),
mode='text',
text='*',
textfont={'color': 'red'},
showlegend=False)
],
layout={
'xaxis': {
'title': 'Window distance (cell diameter)',
'zeroline': False
},
'yaxis': {
'title':
'Correlation:<br>pair distance vs. fiber density',
'zeroline': False,
'range': [-0.82, 0.3],
'tickmode': 'array',
'tickvals': [-0.75, -0.25, 0.25]
},
'shapes': [{
'type': 'line',
'x0': -0.2,
'y0': -0.8,
'x1': 3.2,
'y1': -0.8,
'line': {
'color': 'black',
'width': 2
}
}, {
'type': 'line',
'x0': -0.2,
'y0': -0.8,
'x1': -0.2,
'y1': 0.3,
'line': {
'color': 'black',
'width': 2
}
}]
})
save.to_html(_fig=_fig,
_path=os.path.join(paths.PLOTS, save.get_module_name()),
_filename='plot')
# table
print('Window distance (cell diameter)',
'Correlation: pair distance vs. fiber density',
'N',
'P-value',
sep='\t')
for _x, _y, _n, _p_value in zip(_x_array, _y_array, _n_array,
_p_value_array):
print(_x, _y, _n, _p_value, sep='\t')
def compute_data(_arguments):
_offset_y_index, _offset_y, _offset_z_index, _offset_z = _arguments
_same_correlations_array = []
_different_correlations_array = []
for _experiment in _tuples_by_experiment:
_experiment_tuples = _tuples_by_experiment[_experiment]
for _same_index in range(len(_experiment_tuples)):
_same_tuple = _experiment_tuples[_same_index]
_same_experiment, _same_series, _same_group = _same_tuple
_same_left_cell_fiber_densities = _experiments_fiber_densities[(
_same_experiment, _same_series, _same_group, _offset_y,
_offset_z, 'left_cell')]
_same_right_cell_fiber_densities = _experiments_fiber_densities[(
_same_experiment, _same_series, _same_group, _offset_y,
_offset_z, 'right_cell')]
_same_properties = \
load.group_properties(_same_experiment, _same_series, _same_group)
_same_left_cell_fiber_densities = compute.remove_blacklist(
_same_experiment, _same_series,
_same_properties['cells_ids']['left_cell'],
_same_left_cell_fiber_densities)
_same_right_cell_fiber_densities = compute.remove_blacklist(
_same_experiment, _same_series,
_same_properties['cells_ids']['right_cell'],
_same_right_cell_fiber_densities)
_same_left_cell_fiber_densities_filtered, _same_right_cell_fiber_densities_filtered = \
compute.longest_same_indices_shared_in_borders_sub_array(
_same_left_cell_fiber_densities, _same_right_cell_fiber_densities
)
# ignore small arrays
if len(_same_left_cell_fiber_densities_filtered
) < compute.minimum_time_frames_for_correlation(
_same_experiment):
continue
_same_correlation = compute_lib.correlation(
compute_lib.derivative(
_same_left_cell_fiber_densities_filtered, _n=DERIVATIVE),
compute_lib.derivative(
_same_right_cell_fiber_densities_filtered, _n=DERIVATIVE))
for _different_index in range(len(_experiment_tuples)):
if _same_index != _different_index:
_different_tuple = _experiment_tuples[_different_index]
_different_experiment, _different_series, _different_group = _different_tuple
for _same_cell_id, _different_cell_id in product(
['left_cell', 'right_cell'],
['left_cell', 'right_cell']):
_same_fiber_densities = _experiments_fiber_densities[(
_same_experiment, _same_series, _same_group,
_offset_y, _offset_z, _same_cell_id)]
_different_fiber_densities = _experiments_fiber_densities[
(_different_experiment, _different_series,
_different_group, _offset_y, _offset_z,
_different_cell_id)]
_different_properties = load.group_properties(
_different_experiment, _different_series,
_different_group)
_same_fiber_densities = compute.remove_blacklist(
_same_experiment, _same_series,
_same_properties['cells_ids'][_same_cell_id],
_same_fiber_densities)
_different_fiber_densities = compute.remove_blacklist(
_different_experiment, _different_series,
_different_properties['cells_ids']
[_different_cell_id], _different_fiber_densities)
_same_fiber_densities_filtered, _different_fiber_densities_filtered = \
compute.longest_same_indices_shared_in_borders_sub_array(
_same_fiber_densities, _different_fiber_densities
)
# ignore small arrays
if len(_same_fiber_densities_filtered
) < compute.minimum_time_frames_for_correlation(
_different_experiment):
continue
_different_correlation = compute_lib.correlation(
compute_lib.derivative(
_same_fiber_densities_filtered, _n=DERIVATIVE),
compute_lib.derivative(
_different_fiber_densities_filtered,
_n=DERIVATIVE))
_same_correlations_array.append(_same_correlation)
_different_correlations_array.append(
_different_correlation)
# compute fraction
_annotation = None
_same_minus_different = np.array(_same_correlations_array) - np.array(
_different_correlations_array)
_same_count = len(_same_minus_different[_same_minus_different > 0])
if len(_same_minus_different) > 0:
_same_fraction = round(_same_count / len(_same_minus_different), 10)
_wilcoxon = wilcoxon(_same_minus_different)
_p_value = _wilcoxon[1]
if _p_value > 0.05:
_annotation = {
'text': 'x',
'showarrow': False,
'x': _offset_z,
'y': _offset_y,
'font': {
'size': 6,
'color': 'red'
}
}
else:
_same_fraction = None
return _offset_y_index, _offset_z_index, _same_fraction, _annotation
def compute_fiber_densities(_band=True, _high_temporal_resolution=True):
_experiments = all_experiments()
_experiments = filtering.by_categories(
_experiments=_experiments,
_is_single_cell=False,
_is_high_temporal_resolution=_high_temporal_resolution,
_is_bleb=False,
_is_dead_dead=False,
_is_live_dead=False,
_is_bead=False,
_is_metastasis=False)
_tuples = load.experiments_groups_as_tuples(_experiments)
_tuples = filtering.by_pair_distance_range(_tuples, PAIR_DISTANCE_RANGE)
_tuples = filtering.by_real_pairs(_tuples)
_tuples = filtering.by_band(_tuples, _band=_band)
print('Total tuples:', len(_tuples))
_arguments = []
for _tuple in _tuples:
_experiment, _series_id, _group = _tuple
_latest_time_frame = compute.latest_time_frame_before_overlapping(
_experiment, _series_id, _group, OFFSET_X)
for _cell_id in ['left_cell', 'right_cell']:
_arguments.append({
'experiment': _experiment,
'series_id': _series_id,
'group': _group,
'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER,
'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER,
'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER,
'offset_x': OFFSET_X,
'offset_y': OFFSET_Y,
'offset_z': OFFSET_Z,
'cell_id': _cell_id,
'direction': 'inside',
'time_points': _latest_time_frame
})
_windows_dictionary, _windows_to_compute = compute.windows(
_arguments, _keys=['experiment', 'series_id', 'group', 'cell_id'])
_fiber_densities = compute.fiber_densities(_windows_to_compute,
_subtract_border=True)
_experiments_fiber_densities = {
_key:
[_fiber_densities[_tuple] for _tuple in _windows_dictionary[_key]]
for _key in _windows_dictionary
}
_same_correlation_vs_time_lag = {}
_same_time_lags_arrays = [[]
for _i in TIME_LAGS[_high_temporal_resolution]]
_different_time_lags_arrays = [
[] for _i in TIME_LAGS[_high_temporal_resolution]
]
_same_time_lags_highest = [
0 for _i in TIME_LAGS[_high_temporal_resolution]
]
_different_time_lags_highest = [
0 for _i in TIME_LAGS[_high_temporal_resolution]
]
_valid_tuples = []
for _same_index in tqdm(range(len(_tuples)), desc='Main loop'):
_same_tuple = _tuples[_same_index]
_same_experiment, _same_series, _same_group = _same_tuple
_same_left_cell_fiber_densities = \
_experiments_fiber_densities[
(_same_experiment, _same_series, _same_group, 'left_cell')
]
_same_right_cell_fiber_densities = \
_experiments_fiber_densities[
(_same_experiment, _same_series, _same_group, 'right_cell')
]
_same_properties = \
load.group_properties(_same_experiment, _same_series, _same_group)
_same_left_cell_fiber_densities = compute.remove_blacklist(
_same_experiment, _same_series,
_same_properties['cells_ids']['left_cell'],
_same_left_cell_fiber_densities)
_same_right_cell_fiber_densities = compute.remove_blacklist(
_same_experiment, _same_series,
_same_properties['cells_ids']['right_cell'],
_same_right_cell_fiber_densities)
# time lag
_same_highest_correlation = -1.1
_same_highest_correlation_time_lag_index = 0
_same_correlation_vs_time_lag[_same_tuple] = []
for _time_lag_index, _time_lag in enumerate(
TIME_LAGS[_high_temporal_resolution]):
# choose either negative or positive lag
for _symbol in [-1, 1]:
# if no time lag consider it only once
if _time_lag == 0 and _symbol == -1:
continue
_time_lag_symbol = _time_lag * _symbol
if _time_lag_symbol > 0:
_same_left_cell_fiber_densities_time_lag = _same_left_cell_fiber_densities[:
-_time_lag_symbol]
_same_right_cell_fiber_densities_time_lag = _same_right_cell_fiber_densities[
_time_lag_symbol:]
elif _time_lag_symbol < 0:
_same_left_cell_fiber_densities_time_lag = _same_left_cell_fiber_densities[
-_time_lag_symbol:]
_same_right_cell_fiber_densities_time_lag = _same_right_cell_fiber_densities[:
_time_lag_symbol]
else:
_same_left_cell_fiber_densities_time_lag = _same_left_cell_fiber_densities
_same_right_cell_fiber_densities_time_lag = _same_right_cell_fiber_densities
_same_left_cell_fiber_densities_filtered, _same_right_cell_fiber_densities_filtered = \
compute.longest_same_indices_shared_in_borders_sub_array(
_same_left_cell_fiber_densities_time_lag, _same_right_cell_fiber_densities_time_lag
)
# ignore small arrays
if len(_same_left_cell_fiber_densities_filtered
) < compute.minimum_time_frames_for_correlation(
_same_experiment):
_same_correlation_vs_time_lag[_same_tuple].append(None)
continue
_same_correlation = compute_lib.correlation(
compute_lib.derivative(
_same_left_cell_fiber_densities_filtered,
_n=DERIVATIVE),
compute_lib.derivative(
_same_right_cell_fiber_densities_filtered,
_n=DERIVATIVE))
_same_time_lags_arrays[_time_lag_index].append(
_same_correlation)
_same_correlation_vs_time_lag[_same_tuple].append(
_same_correlation)
if _same_correlation > _same_highest_correlation:
_same_highest_correlation = _same_correlation
_same_highest_correlation_time_lag_index = _time_lag_index
_same_time_lags_highest[_same_highest_correlation_time_lag_index] += 1
for _different_index in range(len(_tuples)):
if _same_index != _different_index:
_different_tuple = _tuples[_different_index]
_different_experiment, _different_series, _different_group = \
_different_tuple
for _same_cell_id, _different_cell_id in product(
['left_cell', 'right_cell'], ['left_cell', 'right_cell']):
_same_fiber_densities = _experiments_fiber_densities[(
_same_experiment, _same_series, _same_group,
_same_cell_id)]
_different_fiber_densities = _experiments_fiber_densities[(
_different_experiment, _different_series,
_different_group, _different_cell_id)]
_different_properties = load.group_properties(
_different_experiment, _different_series,
_different_group)
_same_fiber_densities = compute.remove_blacklist(
_same_experiment, _same_series,
_same_properties['cells_ids'][_same_cell_id],
_same_fiber_densities)
_different_fiber_densities = compute.remove_blacklist(
_different_experiment, _different_series,
_different_properties['cells_ids'][_different_cell_id],
_different_fiber_densities)
# time lag
_different_highest_correlation = -1.1
_different_highest_correlation_time_lag_index = 0
for _time_lag_index, _time_lag in enumerate(
TIME_LAGS[_high_temporal_resolution]):
# choose either negative or positive lag
for _symbol in [-1, 1]:
# if no time lag consider it only once
if _time_lag == 0 and _symbol == -1:
continue
_time_lag_symbol = _time_lag * _symbol
if _time_lag_symbol > 0:
_same_fiber_densities_time_lag = _same_fiber_densities[:
-_time_lag_symbol]
_different_fiber_densities_time_lag = _different_fiber_densities[
_time_lag_symbol:]
elif _time_lag_symbol < 0:
_same_fiber_densities_time_lag = _same_fiber_densities[
-_time_lag_symbol:]
_different_fiber_densities_time_lag = _different_fiber_densities[:
_time_lag_symbol]
else:
_same_fiber_densities_time_lag = _same_fiber_densities
_different_fiber_densities_time_lag = _different_fiber_densities
_same_fiber_densities_filtered, _different_fiber_densities_filtered = \
compute.longest_same_indices_shared_in_borders_sub_array(
_same_fiber_densities_time_lag, _different_fiber_densities_time_lag
)
# ignore small arrays
if len(
_same_fiber_densities_filtered
) < compute.minimum_time_frames_for_correlation(
_different_experiment):
continue
_different_correlation = compute_lib.correlation(
compute_lib.derivative(
_same_fiber_densities_filtered,
_n=DERIVATIVE),
compute_lib.derivative(
_different_fiber_densities_filtered,
_n=DERIVATIVE))
_different_time_lags_arrays[
_time_lag_index].append(_different_correlation)
if _different_correlation > _different_highest_correlation:
_different_highest_correlation = _different_correlation
_different_highest_correlation_time_lag_index = _time_lag_index
if _same_tuple not in _valid_tuples:
_valid_tuples.append(_same_tuple)
_different_time_lags_highest[
_different_highest_correlation_time_lag_index] += 1
print('Total tuples:', len(_valid_tuples))
return _same_correlation_vs_time_lag, _same_time_lags_arrays, _different_time_lags_arrays, \
_same_time_lags_highest, _different_time_lags_highest
def compute_fiber_densities(_offset_y=0.5, _high_temporal_resolution=False):
_experiments = all_experiments()
_experiments = filtering.by_categories(
_experiments=_experiments,
_is_single_cell=False,
_is_high_temporal_resolution=_high_temporal_resolution,
_is_bleb=False,
_is_dead_dead=False,
_is_live_dead=False,
_is_bead=False,
_is_metastasis=False)
_tuples = load.experiments_groups_as_tuples(_experiments)
_tuples = filtering.by_pair_distance_range(_tuples, PAIR_DISTANCE_RANGE)
_tuples = filtering.by_band(_tuples)
_tuples = filtering.by_real_fake_pairs(_tuples, _real_fake_pairs=False)
_experiments_matched = organize.by_matched_real_and_fake(_tuples)
print('Total matched pairs:', len(_experiments_matched))
_arguments = []
for _matched_tuple in _experiments_matched:
for _tuple in _matched_tuple:
_experiment, _series_id, _group = _tuple
_latest_time_frame = compute.latest_time_frame_before_overlapping(
_experiment, _series_id, _group, OFFSET_X)
for _cell_id in ['left_cell', 'right_cell']:
_arguments.append({
'experiment': _experiment,
'series_id': _series_id,
'group': _group,
'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER,
'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER,
'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER,
'offset_x': OFFSET_X,
'offset_y': _offset_y,
'offset_z': OFFSET_Z,
'cell_id': _cell_id,
'direction': 'inside',
'time_points': _latest_time_frame
})
_windows_dictionary, _windows_to_compute = compute.windows(
_arguments, _keys=['experiment', 'series_id', 'group', 'cell_id'])
_fiber_densities = compute.fiber_densities(_windows_to_compute,
_subtract_border=True)
_experiments_fiber_densities = {
_key:
[_fiber_densities[_tuple] for _tuple in _windows_dictionary[_key]]
for _key in _windows_dictionary
}
_tuples_by_experiment = organize.by_experiment(_tuples)
# same (real, fake), different (real, fake)
_correlations = [[[], []], [[], []]]
_valid_real_tuples = []
for _experiment in _tuples_by_experiment:
print('Experiment:', _experiment)
_experiment_tuples = _tuples_by_experiment[_experiment]
_experiments_matched = organize.by_matched_real_and_fake(
_experiment_tuples)
print('Matched pairs:', len(_experiments_matched))
for _same_index in tqdm(range(len(_experiments_matched)),
desc='Main loop'):
for _group_type_index in [0, 1]:
_same_tuple = _experiments_matched[_same_index][
_group_type_index]
_same_experiment, _same_series, _same_group = _same_tuple
_same_left_cell_fiber_densities = \
_experiments_fiber_densities[
(_same_experiment, _same_series, _same_group, 'left_cell')
]
_same_right_cell_fiber_densities = \
_experiments_fiber_densities[
(_same_experiment, _same_series, _same_group, 'right_cell')
]
_same_properties = \
load.group_properties(_same_experiment, _same_series, _same_group)
_same_left_cell_fiber_densities = compute.remove_blacklist(
_same_experiment, _same_series,
_same_properties['cells_ids']['left_cell'],
_same_left_cell_fiber_densities)
_same_right_cell_fiber_densities = compute.remove_blacklist(
_same_experiment, _same_series,
_same_properties['cells_ids']['right_cell'],
_same_right_cell_fiber_densities)
_same_left_cell_fiber_densities_filtered, _same_right_cell_fiber_densities_filtered = \
compute.longest_same_indices_shared_in_borders_sub_array(
_same_left_cell_fiber_densities, _same_right_cell_fiber_densities
)
# ignore small arrays
if len(_same_left_cell_fiber_densities_filtered
) < compute.minimum_time_frames_for_correlation(
_same_experiment):
for _different_index in range(len(_experiments_matched)):
if _same_index != _different_index:
# for all combinations
for _i in range(4):
_correlations[0][_group_type_index].append(
None)
_correlations[1][_group_type_index].append(
None)
continue
_same_correlation = compute_lib.correlation(
compute_lib.derivative(
_same_left_cell_fiber_densities_filtered,
_n=DERIVATIVE),
compute_lib.derivative(
_same_right_cell_fiber_densities_filtered,
_n=DERIVATIVE))
for _different_index in range(len(_experiments_matched)):
if _same_index != _different_index:
_different_tuple = _experiments_matched[
_different_index][_group_type_index]
_different_experiment, _different_series, _different_group = _different_tuple
for _same_cell_id, _different_cell_id in product(
['left_cell', 'right_cell'],
['left_cell', 'right_cell']):
_same_fiber_densities = _experiments_fiber_densities[
(_same_experiment, _same_series, _same_group,
_same_cell_id)]
_different_fiber_densities = _experiments_fiber_densities[
(_different_experiment, _different_series,
_different_group, _different_cell_id)]
_different_properties = load.group_properties(
_different_experiment, _different_series,
_different_group)
_same_fiber_densities = compute.remove_blacklist(
_same_experiment, _same_series,
_same_properties['cells_ids'][_same_cell_id],
_same_fiber_densities)
_different_fiber_densities = compute.remove_blacklist(
_different_experiment, _different_series,
_different_properties['cells_ids']
[_different_cell_id],
_different_fiber_densities)
_same_fiber_densities_filtered, _different_fiber_densities_filtered = \
compute.longest_same_indices_shared_in_borders_sub_array(
_same_fiber_densities, _different_fiber_densities
)
# ignore small arrays
if len(
_same_fiber_densities_filtered
) < compute.minimum_time_frames_for_correlation(
_different_experiment):
_correlations[0][_group_type_index].append(
None)
_correlations[1][_group_type_index].append(
None)
continue
_different_correlation = compute_lib.correlation(
compute_lib.derivative(
_same_fiber_densities_filtered,
_n=DERIVATIVE),
compute_lib.derivative(
_different_fiber_densities_filtered,
_n=DERIVATIVE))
_correlations[0][_group_type_index].append(
_same_correlation)
_correlations[1][_group_type_index].append(
_different_correlation)
if _group_type_index == 0 and _same_tuple not in _valid_real_tuples:
_valid_real_tuples.append(_same_tuple)
print('Total real tuples:', len(_valid_real_tuples))
_distances_from_y_equal_x = [[], []]
_same_correlations, _different_correlations = _correlations
_same_real_correlations, _same_fake_correlations = _same_correlations
_different_real_correlations, _different_fake_correlations = _different_correlations
for _same_real, _same_fake, _different_real, _different_fake in \
zip(_same_real_correlations, _same_fake_correlations,
_different_real_correlations, _different_fake_correlations):
# one of the correlations is none - not valid
if None in [_same_real, _same_fake, _different_real, _different_fake]:
continue
for _group_type_index, _same, _different in \
zip([0, 1], [_same_real, _same_fake], [_different_real, _different_fake]):
_point_distance = compute_lib.distance_from_a_point_to_a_line(
_line=[-1, -1, 1, 1], _point=[_same, _different])
if _same > _different:
_distances_from_y_equal_x[_group_type_index].append(
_point_distance)
else:
_distances_from_y_equal_x[_group_type_index].append(
-_point_distance)
return _distances_from_y_equal_x
def main(_real_cells=True,
_static=False,
_dead_dead=False,
_live_dead=False,
_dead=False,
_live=False,
_bead=False,
_metastasis=False,
_bleb=False,
_bleb_amount_um=None,
_band=True,
_offset_y=0.5,
_high_temporal_resolution=False):
_experiments = all_experiments()
_experiments = filtering.by_categories(
_experiments=_experiments,
_is_single_cell=False,
_is_high_temporal_resolution=_high_temporal_resolution,
_is_bleb=_bleb,
_is_dead_dead=_dead_dead,
_is_live_dead=_live_dead,
_is_bead=_bead,
_is_metastasis=_metastasis)
_tuples = load.experiments_groups_as_tuples(_experiments)
_tuples = filtering.by_pair_distance_range(_tuples, PAIR_DISTANCE_RANGE)
_tuples = filtering.by_real_pairs(_tuples, _real_pairs=_real_cells)
_tuples = filtering.by_fake_static_pairs(_tuples,
_fake_static_pairs=_static)
if _dead_dead is not False or _live_dead is not False:
_tuples = filtering.by_dead_live(_tuples, _dead=_dead, _live=_live)
_tuples = filtering.by_band(_tuples, _band=_band)
if _bleb:
_tuples = filtering.by_bleb_amount_um(_tuples,
_amount_um=_bleb_amount_um)
_tuples_matched = organize.by_matched_real_real_and_real_fake(_tuples)
print('Total matched pairs:', len(_tuples_matched))
_arguments = []
for _matched_tuple in _tuples_matched:
for _tuple in _matched_tuple:
_experiment, _series_id, _group = _tuple
_latest_time_frame = compute.latest_time_frame_before_overlapping(
_experiment, _series_id, _group, OFFSET_X)
for _cell_id in ['left_cell', 'right_cell']:
_arguments.append({
'experiment': _experiment,
'series_id': _series_id,
'group': _group,
'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER,
'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER,
'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER,
'offset_x': OFFSET_X,
'offset_y': _offset_y,
'offset_z': OFFSET_Z,
'cell_id': _cell_id,
'direction': 'inside',
'time_points': _latest_time_frame
})
_windows_dictionary, _windows_to_compute = compute.windows(
_arguments, _keys=['experiment', 'series_id', 'group', 'cell_id'])
_fiber_densities = compute.fiber_densities(_windows_to_compute,
_subtract_border=True)
_experiments_fiber_densities = {
_key:
[_fiber_densities[_tuple] for _tuple in _windows_dictionary[_key]]
for _key in _windows_dictionary
}
_real_real_correlations_array = []
_real_fake_correlations_array = []
_valid_real_real_tuples = []
for _tuple_matched in tqdm(_tuples_matched, desc='Main loop'):
_real_real_tuple, _real_fake_tuple = _tuple_matched
_real_real_experiment, _real_real_series, _real_real_group = _real_real_tuple
_, _, _real_fake_group = _real_fake_tuple
_real_real_left_cell_fiber_densities = \
_experiments_fiber_densities[
(_real_real_experiment, _real_real_series, _real_real_group, 'left_cell')
]
_real_real_right_cell_fiber_densities = \
_experiments_fiber_densities[
(_real_real_experiment, _real_real_series, _real_real_group, 'right_cell')
]
_real_fake_left_cell_fiber_densities = \
_experiments_fiber_densities[
(_real_real_experiment, _real_real_series, _real_fake_group, 'left_cell')
]
_real_fake_right_cell_fiber_densities = \
_experiments_fiber_densities[
(_real_real_experiment, _real_real_series, _real_fake_group, 'right_cell')
]
_properties = \
load.group_properties(_real_real_experiment, _real_real_series, _real_real_group)
_real_real_left_cell_fiber_densities = compute.remove_blacklist(
_real_real_experiment, _real_real_series,
_properties['cells_ids']['left_cell'],
_real_real_left_cell_fiber_densities)
_real_real_right_cell_fiber_densities = compute.remove_blacklist(
_real_real_experiment, _real_real_series,
_properties['cells_ids']['right_cell'],
_real_real_right_cell_fiber_densities)
_real_fake_left_cell_fiber_densities = compute.remove_blacklist(
_real_real_experiment, _real_real_series,
_properties['cells_ids']['left_cell'],
_real_fake_left_cell_fiber_densities)
_real_fake_right_cell_fiber_densities = compute.remove_blacklist(
_real_real_experiment, _real_real_series,
_properties['cells_ids']['right_cell'],
_real_fake_right_cell_fiber_densities)
_real_real_left_cell_fiber_densities_filtered, _real_real_right_cell_fiber_densities_filtered = \
compute.longest_same_indices_shared_in_borders_sub_array(
_real_real_left_cell_fiber_densities, _real_real_right_cell_fiber_densities
)
# ignore small arrays
_minimum_time_frame_for_correlation = compute.minimum_time_frames_for_correlation(
_real_real_experiment)
if len(_real_real_left_cell_fiber_densities_filtered
) < _minimum_time_frame_for_correlation:
continue
_real_real_correlation = compute_lib.correlation(
compute_lib.derivative(
_real_real_left_cell_fiber_densities_filtered, _n=DERIVATIVE),
compute_lib.derivative(
_real_real_right_cell_fiber_densities_filtered, _n=DERIVATIVE))
_real_fake_left_cell_fiber_densities_filtered, _real_fake_right_cell_fiber_densities_filtered = \
compute.longest_same_indices_shared_in_borders_sub_array(
_real_fake_left_cell_fiber_densities, _real_fake_right_cell_fiber_densities
)
# ignore small arrays
if len(_real_fake_left_cell_fiber_densities_filtered
) < _minimum_time_frame_for_correlation:
continue
_real_fake_correlation = compute_lib.correlation(
compute_lib.derivative(
_real_fake_left_cell_fiber_densities_filtered, _n=DERIVATIVE),
compute_lib.derivative(
_real_fake_right_cell_fiber_densities_filtered, _n=DERIVATIVE))
_real_real_correlations_array.append(_real_real_correlation)
_real_fake_correlations_array.append(_real_fake_correlation)
if _real_real_tuple not in _valid_real_real_tuples:
_valid_real_real_tuples.append(_real_real_tuple)
print('Total real-real pairs:', len(_valid_real_real_tuples))
_real_minus_fake = np.array(_real_real_correlations_array) - np.array(
_real_fake_correlations_array)
print('Wilcoxon of real-real minus real-fake around the zero:')
print(wilcoxon(_real_minus_fake))
print('Higher real-real amount:',
(_real_minus_fake > 0).sum() / len(_real_minus_fake))
# plot
_fig = go.Figure(data=go.Scatter(x=_real_real_correlations_array,
y=_real_fake_correlations_array,
mode='markers',
marker={
'size': 5,
'color': '#ea8500'
},
showlegend=False),
layout={
'xaxis': {
'title': 'Real-real correlation',
'zeroline': False,
'range': [-1.1, 1.2],
'tickmode': 'array',
'tickvals': [-1, -0.5, 0, 0.5, 1]
},
'yaxis': {
'title': 'Real-fake correlation',
'zeroline': False,
'range': [-1.1, 1.2],
'tickmode': 'array',
'tickvals': [-1, -0.5, 0, 0.5, 1]
},
'shapes': [{
'type': 'line',
'x0': -1,
'y0': -1,
'x1': -1,
'y1': 1,
'line': {
'color': 'black',
'width': 2
}
}, {
'type': 'line',
'x0': -1,
'y0': -1,
'x1': 1,
'y1': -1,
'line': {
'color': 'black',
'width': 2
}
}, {
'type': 'line',
'x0': -1,
'y0': -1,
'x1': 1,
'y1': 1,
'line': {
'color': 'red',
'width': 2
}
}]
})
save.to_html(_fig=_fig,
_path=os.path.join(paths.PLOTS, save.get_module_name()),
_filename='plot_real_' + str(_real_cells) + '_static_' +
str(_static) + '_dead_dead_' + str(_dead_dead) +
'_live_dead_' + str(_live_dead) + '_dead_' + str(_dead) +
'_live_' + str(_live) + '_bead_' + str(_bead) +
'_metastasis_' + str(_metastasis) + '_bleb_' + str(_bleb) +
str(_bleb_amount_um) + '_band_' + str(_band) + '_high_time_' +
str(_high_temporal_resolution) + '_y_' + str(_offset_y))
def compute_fiber_densities(_band=True, _high_temporal_resolution=False):
_experiments = all_experiments()
_experiments = filtering.by_categories(
_experiments=_experiments,
_is_single_cell=False,
_is_high_temporal_resolution=_high_temporal_resolution,
_is_bleb=False,
_is_dead_dead=False,
_is_live_dead=False,
_is_bead=False,
_is_metastasis=False)
_tuples = load.experiments_groups_as_tuples(_experiments)
_tuples = filtering.by_pair_distance_range(_tuples, PAIR_DISTANCE_RANGE)
_tuples = filtering.by_real_pairs(_tuples)
_tuples = filtering.by_band(_tuples, _band=_band)
print('Total tuples:', len(_tuples))
_arguments = []
for _tuple in _tuples:
_experiment, _series_id, _group = _tuple
_latest_time_frame = compute.latest_time_frame_before_overlapping(
_experiment, _series_id, _group, OFFSET_X)
for _cell_id in ['left_cell', 'right_cell']:
_arguments.append({
'experiment': _experiment,
'series_id': _series_id,
'group': _group,
'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER,
'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER,
'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER,
'offset_x': OFFSET_X,
'offset_y': OFFSET_Y,
'offset_z': OFFSET_Z,
'cell_id': _cell_id,
'direction': 'inside',
'time_points': _latest_time_frame
})
_windows_dictionary, _windows_to_compute = compute.windows(
_arguments, _keys=['experiment', 'series_id', 'group', 'cell_id'])
_fiber_densities = compute.fiber_densities(_windows_to_compute,
_subtract_border=True)
_experiments_fiber_densities = {
_key:
[_fiber_densities[_tuple] for _tuple in _windows_dictionary[_key]]
for _key in _windows_dictionary
}
_tuples_by_experiment = organize.by_experiment(_tuples)
_distances_from_y_equal_x = []
_z_positions_array = []
for _experiment in _tuples_by_experiment:
print('Experiment:', _experiment)
_experiment_tuples = _tuples_by_experiment[_experiment]
for _same_index in tqdm(range(len(_experiment_tuples)),
desc='Main loop'):
_same_tuple = _experiment_tuples[_same_index]
_same_experiment, _same_series, _same_group = _same_tuple
_same_left_cell_fiber_densities = \
_experiments_fiber_densities[
(_same_experiment, _same_series, _same_group, 'left_cell')
]
_same_right_cell_fiber_densities = \
_experiments_fiber_densities[
(_same_experiment, _same_series, _same_group, 'right_cell')
]
_same_properties = \
load.group_properties(_same_experiment, _same_series, _same_group)
_same_left_cell_fiber_densities = compute.remove_blacklist(
_same_experiment, _same_series,
_same_properties['cells_ids']['left_cell'],
_same_left_cell_fiber_densities)
_same_right_cell_fiber_densities = compute.remove_blacklist(
_same_experiment, _same_series,
_same_properties['cells_ids']['right_cell'],
_same_right_cell_fiber_densities)
_same_left_cell_fiber_densities_filtered, _same_right_cell_fiber_densities_filtered = \
compute.longest_same_indices_shared_in_borders_sub_array(
_same_left_cell_fiber_densities, _same_right_cell_fiber_densities
)
# ignore small arrays
if len(_same_left_cell_fiber_densities_filtered
) < compute.minimum_time_frames_for_correlation(
_same_experiment):
continue
_same_correlation = compute_lib.correlation(
compute_lib.derivative(
_same_left_cell_fiber_densities_filtered, _n=DERIVATIVE),
compute_lib.derivative(
_same_right_cell_fiber_densities_filtered, _n=DERIVATIVE))
_same_group_mean_z_position = \
compute.group_mean_z_position_from_substrate(_same_experiment, _same_series, _same_group)
for _different_index in range(len(_experiment_tuples)):
if _same_index != _different_index:
_different_tuple = _experiment_tuples[_different_index]
_different_experiment, _different_series, _different_group = \
_different_tuple
for _same_cell_id, _different_cell_id in product(
['left_cell', 'right_cell'],
['left_cell', 'right_cell']):
_same_fiber_densities = _experiments_fiber_densities[(
_same_experiment, _same_series, _same_group,
_same_cell_id)]
_different_fiber_densities = _experiments_fiber_densities[
(_different_experiment, _different_series,
_different_group, _different_cell_id)]
_different_properties = load.group_properties(
_different_experiment, _different_series,
_different_group)
_same_fiber_densities = compute.remove_blacklist(
_same_experiment, _same_series,
_same_properties['cells_ids'][_same_cell_id],
_same_fiber_densities)
_different_fiber_densities = compute.remove_blacklist(
_different_experiment, _different_series,
_different_properties['cells_ids']
[_different_cell_id], _different_fiber_densities)
_same_fiber_densities_filtered, _different_fiber_densities_filtered = \
compute.longest_same_indices_shared_in_borders_sub_array(
_same_fiber_densities, _different_fiber_densities
)
# ignore small arrays
if len(_same_fiber_densities_filtered
) < compute.minimum_time_frames_for_correlation(
_different_experiment):
continue
_different_correlation = compute_lib.correlation(
compute_lib.derivative(
_same_fiber_densities_filtered, _n=DERIVATIVE),
compute_lib.derivative(
_different_fiber_densities_filtered,
_n=DERIVATIVE))
_point_distance = compute_lib.distance_from_a_point_to_a_line(
_line=[-1, -1, 1, 1],
_point=[_same_correlation, _different_correlation])
if _same_correlation > _different_correlation:
_distances_from_y_equal_x.append(_point_distance)
else:
_distances_from_y_equal_x.append(-_point_distance)
_z_positions_array.append(_same_group_mean_z_position)
print('Total points:', len(_distances_from_y_equal_x))
print('Wilcoxon of distances from y = x around the zero:')
print(wilcoxon(_distances_from_y_equal_x))
print(
'Pearson correlation of distances from y = x and z position distances:'
)
print(
compute_lib.correlation(_distances_from_y_equal_x,
_z_positions_array,
_with_p_value=True))
return _distances_from_y_equal_x, _z_positions_array
def main(_real_cells=True, _static=False, _band=True, _high_temporal_resolution=False):
_experiments = all_experiments()
_experiments = filtering.by_categories(
_experiments=_experiments,
_is_single_cell=False,
_is_high_temporal_resolution=_high_temporal_resolution,
_is_bleb=False,
_is_dead_dead=False,
_is_live_dead=False,
_is_bead=False,
_is_metastasis=False
)
_tuples = load.experiments_groups_as_tuples(_experiments)
_tuples = filtering.by_pair_distance_range(_tuples, PAIR_DISTANCE_RANGE)
_tuples = filtering.by_real_pairs(_tuples, _real_pairs=_real_cells)
_tuples = filtering.by_fake_static_pairs(_tuples, _fake_static_pairs=_static)
_tuples = filtering.by_band(_tuples, _band=_band)
_tuples = filtering.by_time_frames_amount(_tuples, compute.minimum_time_frames_for_correlation(_experiments[0]))
print('Total tuples:', len(_tuples))
print('Density')
_arguments = []
for _tuple in _tuples:
_experiment, _series_id, _group = _tuple
_time_frame = compute.minimum_time_frames_for_correlation(_experiment)
for _cell_id in ['left_cell', 'right_cell']:
_arguments.append({
'experiment': _experiment,
'series_id': _series_id,
'group': _group,
'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER,
'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER,
'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER,
'offset_x': OFFSET_X,
'offset_y': OFFSET_Y_DENSITY,
'offset_z': OFFSET_Z,
'cell_id': _cell_id,
'direction': 'inside',
'time_point': _time_frame - 1
})
_windows_dictionary, _windows_to_compute = \
compute.windows(_arguments, _keys=['experiment', 'series_id', 'group', 'cell_id'])
_densities_fiber_densities = compute.fiber_densities(_windows_to_compute, _subtract_border=False)
_densities_experiments_fiber_densities = {
_key: [_densities_fiber_densities[_tuple] for _tuple in _windows_dictionary[_key]]
for _key in _windows_dictionary
}
print('Correlations')
_arguments = []
for _tuple in _tuples:
_experiment, _series_id, _group = _tuple
_latest_time_frame = compute.latest_time_frame_before_overlapping(_experiment, _series_id, _group, OFFSET_X)
for _cell_id in ['left_cell', 'right_cell']:
_arguments.append({
'experiment': _experiment,
'series_id': _series_id,
'group': _group,
'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER,
'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER,
'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER,
'offset_x': OFFSET_X,
'offset_y': OFFSET_Y_CORRELATION,
'offset_z': OFFSET_Z,
'cell_id': _cell_id,
'direction': 'inside',
'time_points': _latest_time_frame
})
_windows_dictionary, _windows_to_compute = \
compute.windows(_arguments, _keys=['experiment', 'series_id', 'group', 'cell_id'])
_correlations_fiber_densities = compute.fiber_densities(_windows_to_compute, _subtract_border=True)
_correlations_experiments_fiber_densities = {
_key: [_correlations_fiber_densities[_tuple] for _tuple in _windows_dictionary[_key]]
for _key in _windows_dictionary
}
_densities = []
_correlations = []
for _tuple in tqdm(_tuples, desc='Main loop'):
_experiment, _series_id, _group = _tuple
# density
_left_cell_fiber_density = \
_densities_experiments_fiber_densities[(_experiment, _series_id, _group, 'left_cell')][0]
_right_cell_fiber_density = \
_densities_experiments_fiber_densities[(_experiment, _series_id, _group, 'right_cell')][0]
# not relevant
if _left_cell_fiber_density[1] or _right_cell_fiber_density[1]:
continue
_normalization = load.normalization_series_file_data(_experiment, _series_id)
_left_cell_fiber_density_normalized = compute_lib.z_score(
_x=_left_cell_fiber_density[0],
_average=_normalization['average'],
_std=_normalization['std']
)
_right_cell_fiber_density_normalized = compute_lib.z_score(
_x=_right_cell_fiber_density[0],
_average=_normalization['average'],
_std=_normalization['std']
)
_density = (_left_cell_fiber_density_normalized + _right_cell_fiber_density_normalized) / 2
# correlation
_left_cell_fiber_densities = \
_correlations_experiments_fiber_densities[(_experiment, _series_id, _group, 'left_cell')]
_right_cell_fiber_densities = \
_correlations_experiments_fiber_densities[(_experiment, _series_id, _group, 'right_cell')]
_properties = load.group_properties(_experiment, _series_id, _group)
_left_cell_fiber_densities = compute.remove_blacklist(
_experiment, _series_id, _properties['cells_ids']['left_cell'], _left_cell_fiber_densities)
_right_cell_fiber_densities = compute.remove_blacklist(
_experiment, _series_id, _properties['cells_ids']['right_cell'], _right_cell_fiber_densities)
_left_cell_fiber_densities_filtered, _right_cell_fiber_densities_filtered = \
compute.longest_same_indices_shared_in_borders_sub_array(
_left_cell_fiber_densities, _right_cell_fiber_densities
)
# ignore small arrays
if len(_left_cell_fiber_densities_filtered) < compute.minimum_time_frames_for_correlation(_experiment):
continue
_correlation = compute_lib.correlation(
compute_lib.derivative(_left_cell_fiber_densities_filtered, _n=DERIVATIVE),
compute_lib.derivative(_right_cell_fiber_densities_filtered, _n=DERIVATIVE)
)
_densities.append(_density)
_correlations.append(_correlation)
print('Total tuples:', len(_densities))
print('Pearson correlation of densities and correlations:')
print(compute_lib.correlation(_densities, _correlations, _with_p_value=True))
# plot
_fig = go.Figure(
data=go.Scatter(
x=_densities,
y=_correlations,
mode='markers',
marker={
'size': 5,
'color': '#ea8500'
},
showlegend=False
),
layout={
'xaxis': {
'title': 'Fiber density (z-score)',
'zeroline': False,
'range': [-1.1, 15.2],
# 'tickmode': 'array',
# 'tickvals': [-1, -0.5, 0, 0.5, 1]
},
'yaxis': {
'title': 'Correlation',
'zeroline': False,
'range': [-1.1, 1.2],
'tickmode': 'array',
'tickvals': [-1, -0.5, 0, 0.5, 1]
},
'shapes': [
{
'type': 'line',
'x0': 0,
'y0': -1,
'x1': 0,
'y1': 1,
'line': {
'color': 'black',
'width': 2
}
},
{
'type': 'line',
'x0': 0,
'y0': -1,
'x1': 15,
'y1': -1,
'line': {
'color': 'black',
'width': 2
}
}
]
}
)
save.to_html(
_fig=_fig,
_path=os.path.join(paths.PLOTS, save.get_module_name()),
_filename='plot_real_' + str(_real_cells) + '_static_' + str(_static) + '_band_' + str(_band) +
'_high_time_' + str(_high_temporal_resolution) + '_y_density_' + str(OFFSET_Y_DENSITY) +
'_y_correlation_' + str(OFFSET_Y_CORRELATION)
)
def main(_band=True):
_experiments = all_experiments()
_experiments = filtering.by_categories(_experiments=_experiments,
_is_single_cell=False,
_is_high_temporal_resolution=None,
_is_bleb=True,
_is_dead_dead=False,
_is_live_dead=False,
_is_bead=False,
_is_metastasis=False)
_tuples = load.experiments_groups_as_tuples(_experiments)
_tuples = filtering.by_pair_distance_range(
_tuples, _distance_range=PAIR_DISTANCE_RANGE)
_tuples = filtering.by_real_pairs(_tuples, _real_pairs=REAL_CELLS)
_tuples = filtering.by_fake_static_pairs(_tuples,
_fake_static_pairs=STATIC)
_tuples = filtering.by_band(_tuples, _band=_band)
_tuples = filtering.by_bleb_from_start(_tuples, _from_start=False)
print('Total tuples:', len(_tuples))
_arguments = []
for _tuple in _tuples:
_experiment, _series_id, _group = _tuple
_latest_time_frame = compute.latest_time_frame_before_overlapping(
_experiment, _series_id, _group, OFFSET_X)
for _cell_id in ['left_cell', 'right_cell']:
_arguments.append({
'experiment': _experiment,
'series_id': _series_id,
'group': _group,
'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER,
'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER,
'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER,
'offset_x': OFFSET_X,
'offset_y': OFFSET_Y,
'offset_z': OFFSET_Z,
'cell_id': _cell_id,
'direction': 'inside',
'time_points': _latest_time_frame
})
_windows_dictionary, _windows_to_compute = compute.windows(
_arguments, _keys=['experiment', 'series_id', 'group', 'cell_id'])
_fiber_densities = compute.fiber_densities(_windows_to_compute,
_subtract_border=True)
_experiments_fiber_densities = {
_key:
[_fiber_densities[_tuple] for _tuple in _windows_dictionary[_key]]
for _key in _windows_dictionary
}
_tuples_by_experiment = organize.by_experiment(_tuples)
# same (before, after), different (before, after)
_correlations = [[[], []], [[], []]]
_valid_real_tuples = []
for _experiment in _tuples_by_experiment:
print('Experiment:', _experiment)
_experiment_tuples = _tuples_by_experiment[_experiment]
for _same_index in tqdm(range(len(_experiment_tuples)),
desc='Main loop'):
_same_tuple = _experiment_tuples[_same_index]
_same_experiment, _same_series, _same_group = _same_tuple
_same_left_cell_fiber_densities = \
_experiments_fiber_densities[
(_same_experiment, _same_series, _same_group, 'left_cell')
]
_same_right_cell_fiber_densities = \
_experiments_fiber_densities[
(_same_experiment, _same_series, _same_group, 'right_cell')
]
_same_properties = \
load.group_properties(_same_experiment, _same_series, _same_group)
_same_left_cell_fiber_densities = compute.remove_blacklist(
_same_experiment, _same_series,
_same_properties['cells_ids']['left_cell'],
_same_left_cell_fiber_densities)
_same_right_cell_fiber_densities = compute.remove_blacklist(
_same_experiment, _same_series,
_same_properties['cells_ids']['right_cell'],
_same_right_cell_fiber_densities)
_same_before_left_cell_fiber_densities = \
_same_left_cell_fiber_densities[:AFTER_BLEB_INJECTION_FIRST_TIME_FRAME[_same_experiment]]
_same_before_right_cell_fiber_densities = \
_same_right_cell_fiber_densities[:AFTER_BLEB_INJECTION_FIRST_TIME_FRAME[_same_experiment]]
_same_after_left_cell_fiber_densities = \
_same_left_cell_fiber_densities[AFTER_BLEB_INJECTION_FIRST_TIME_FRAME[_same_experiment]:]
_same_after_right_cell_fiber_densities = \
_same_right_cell_fiber_densities[AFTER_BLEB_INJECTION_FIRST_TIME_FRAME[_same_experiment]:]
_same_before_left_cell_fiber_densities_filtered, _same_before_right_cell_fiber_densities_filtered = \
compute.longest_same_indices_shared_in_borders_sub_array(
_same_before_left_cell_fiber_densities, _same_before_right_cell_fiber_densities
)
_same_after_left_cell_fiber_densities_filtered, _same_after_right_cell_fiber_densities_filtered = \
compute.longest_same_indices_shared_in_borders_sub_array(
_same_after_left_cell_fiber_densities, _same_after_right_cell_fiber_densities
)
# ignore small arrays
_minimum_time_frame_for_correlation = compute.minimum_time_frames_for_correlation(
_same_experiment)
if len(_same_before_left_cell_fiber_densities_filtered) < _minimum_time_frame_for_correlation or \
len(_same_after_left_cell_fiber_densities_filtered) < _minimum_time_frame_for_correlation:
continue
_same_before_correlation = compute_lib.correlation(
compute_lib.derivative(
_same_before_left_cell_fiber_densities_filtered,
_n=DERIVATIVE),
compute_lib.derivative(
_same_before_right_cell_fiber_densities_filtered,
_n=DERIVATIVE))
_same_after_correlation = compute_lib.correlation(
compute_lib.derivative(
_same_after_left_cell_fiber_densities_filtered,
_n=DERIVATIVE),
compute_lib.derivative(
_same_after_right_cell_fiber_densities_filtered,
_n=DERIVATIVE))
for _different_index in range(len(_experiment_tuples)):
if _same_index != _different_index:
_different_tuple = _experiment_tuples[_different_index]
_different_experiment, _different_series, _different_group = _different_tuple
for _same_cell_id, _different_cell_id in product(
['left_cell', 'right_cell'],
['left_cell', 'right_cell']):
_same_fiber_densities = _experiments_fiber_densities[(
_same_experiment, _same_series, _same_group,
_same_cell_id)]
_different_fiber_densities = _experiments_fiber_densities[
(_different_experiment, _different_series,
_different_group, _different_cell_id)]
_different_properties = load.group_properties(
_different_experiment, _different_series,
_different_group)
_same_fiber_densities = compute.remove_blacklist(
_same_experiment, _same_series,
_same_properties['cells_ids'][_same_cell_id],
_same_fiber_densities)
_different_fiber_densities = compute.remove_blacklist(
_different_experiment, _different_series,
_different_properties['cells_ids']
[_different_cell_id], _different_fiber_densities)
_same_before_fiber_densities = \
_same_fiber_densities[:AFTER_BLEB_INJECTION_FIRST_TIME_FRAME[_same_experiment]]
_same_after_fiber_densities = \
_same_fiber_densities[AFTER_BLEB_INJECTION_FIRST_TIME_FRAME[_same_experiment]:]
_different_before_fiber_densities = \
_different_fiber_densities[:AFTER_BLEB_INJECTION_FIRST_TIME_FRAME[_different_experiment]]
_different_after_fiber_densities = \
_different_fiber_densities[AFTER_BLEB_INJECTION_FIRST_TIME_FRAME[_different_experiment]:]
_same_before_fiber_densities_filtered, _different_before_fiber_densities_filtered = \
compute.longest_same_indices_shared_in_borders_sub_array(
_same_before_fiber_densities, _different_before_fiber_densities
)
_same_after_fiber_densities_filtered, _different_after_fiber_densities_filtered = \
compute.longest_same_indices_shared_in_borders_sub_array(
_same_after_fiber_densities, _different_after_fiber_densities
)
# ignore small arrays
if len(_same_before_fiber_densities_filtered) < _minimum_time_frame_for_correlation or \
len(_same_after_fiber_densities_filtered) < _minimum_time_frame_for_correlation:
continue
_different_before_correlation = compute_lib.correlation(
compute_lib.derivative(
_same_before_fiber_densities_filtered,
_n=DERIVATIVE),
compute_lib.derivative(
_different_before_fiber_densities_filtered,
_n=DERIVATIVE))
_different_after_correlation = compute_lib.correlation(
compute_lib.derivative(
_same_after_fiber_densities_filtered,
_n=DERIVATIVE),
compute_lib.derivative(
_different_after_fiber_densities_filtered,
_n=DERIVATIVE))
_correlations[0][0].append(_same_before_correlation)
_correlations[0][1].append(_same_after_correlation)
_correlations[1][0].append(
_different_before_correlation)
_correlations[1][1].append(
_different_after_correlation)
if _same_tuple not in _valid_real_tuples:
_valid_real_tuples.append(_same_tuple)
print('Total tuples:', len(_valid_real_tuples))
_distances_from_y_equal_x = [[], []]
_same_correlations, _different_correlations = _correlations
_same_before_correlations, _same_after_correlations = _same_correlations
_different_before_correlations, _different_after_correlations = _different_correlations
for _same_before, _same_after, _different_before, _different_after in \
zip(_same_before_correlations, _same_after_correlations,
_different_before_correlations, _different_after_correlations):
for _group_type_index, _same, _different in \
zip([0, 1], [_same_before, _same_after], [_different_before, _different_after]):
_point_distance = compute_lib.distance_from_a_point_to_a_line(
_line=[-1, -1, 1, 1], _point=[_same, _different])
if _same > _different:
_distances_from_y_equal_x[_group_type_index].append(
_point_distance)
else:
_distances_from_y_equal_x[_group_type_index].append(
-_point_distance)
print('Total points:', len(_distances_from_y_equal_x[0]))
print('Higher before same amount:',
(np.array(_distances_from_y_equal_x[0]) > 0).sum() /
len(_distances_from_y_equal_x[0]))
print('Wilcoxon of before points:', wilcoxon(_distances_from_y_equal_x[0]))
print('Higher after same amount:',
(np.array(_distances_from_y_equal_x[1]) > 0).sum() /
len(_distances_from_y_equal_x[1]))
print('Wilcoxon of after points:', wilcoxon(_distances_from_y_equal_x[1]))
_before_minus_after = np.array(_distances_from_y_equal_x[0]) - np.array(
_distances_from_y_equal_x[1])
print('Before > after amount:',
(_before_minus_after > 0).sum() / len(_before_minus_after))
print('Wilcoxon before & after:',
wilcoxon(_distances_from_y_equal_x[0], _distances_from_y_equal_x[1]))
# box plot
_colors_array = config.colors(2)
_names_array = ['Before', 'After']
_fig = go.Figure(data=[
go.Box(y=_y_array,
name=_name,
boxpoints=False,
line={'width': 1},
marker={'color': _color},
showlegend=False) for _y_array, _name, _color in zip(
_distances_from_y_equal_x, _names_array, _colors_array)
],
layout={
'xaxis': {
'zeroline': False
},
'yaxis': {
'title': 'Same minus different correlation',
'zeroline': False,
'range': [-1, 1.1],
'tickmode': 'array',
'tickvals': [-1, -0.5, 0, 0.5, 1]
}
})
save.to_html(_fig=_fig,
_path=os.path.join(paths.PLOTS, save.get_module_name()),
_filename='plot_box')
# scatter plot
_fig = go.Figure(data=go.Scatter(x=_distances_from_y_equal_x[0],
y=_distances_from_y_equal_x[1],
mode='markers',
marker={
'size': 5,
'color': '#ea8500'
},
showlegend=False),
layout={
'xaxis': {
'title': 'Before bleb',
'zeroline': False,
'range': [-1.1, 1.2],
'tickmode': 'array',
'tickvals': [-1, -0.5, 0, 0.5, 1]
},
'yaxis': {
'title': 'After bleb',
'zeroline': False,
'range': [-1.1, 1.2],
'tickmode': 'array',
'tickvals': [-1, -0.5, 0, 0.5, 1]
},
'shapes': [{
'type': 'line',
'x0': -1,
'y0': -1,
'x1': -1,
'y1': 1,
'line': {
'color': 'black',
'width': 2
}
}, {
'type': 'line',
'x0': -1,
'y0': -1,
'x1': 1,
'y1': -1,
'line': {
'color': 'black',
'width': 2
}
}, {
'type': 'line',
'x0': -1,
'y0': -1,
'x1': 1,
'y1': 1,
'line': {
'color': 'red',
'width': 2
}
}]
})
save.to_html(_fig=_fig,
_path=os.path.join(paths.PLOTS, save.get_module_name()),
_filename='plot')
def main(_directions=None):
if _directions is None:
_directions = ['inside', 'outside']
_experiments = all_experiments()
_experiments = filtering.by_categories(
_experiments=_experiments,
_is_single_cell=False,
_is_high_temporal_resolution=False,
_is_bleb=False,
_is_dead_dead=False,
_is_live_dead=False,
_is_bead=False,
_is_metastasis=False
)
_tuples = load.experiments_groups_as_tuples(_experiments)
_tuples = filtering.by_real_pairs(_tuples)
_tuples = filtering.by_band(_tuples)
_tuples = filtering.by_pair_distance_range(_tuples, PAIR_DISTANCE_RANGE)
print('Total tuples:', len(_tuples))
_arguments = []
for _tuple in _tuples:
_experiment, _series_id, _group = _tuple
_latest_time_frame = compute.latest_time_frame_before_overlapping(_experiment, _series_id, _group, OFFSET_X)
for _cell_id, _direction in product(['left_cell', 'right_cell'], _directions):
_arguments.append({
'experiment': _experiment,
'series_id': _series_id,
'group': _group,
'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER,
'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER,
'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER,
'offset_x': OFFSET_X,
'offset_y': OFFSET_Y,
'offset_z': OFFSET_Z,
'cell_id': _cell_id,
'direction': _direction,
'time_points': _latest_time_frame
})
_windows_dictionary, _windows_to_compute = \
compute.windows(_arguments, _keys=['experiment', 'series_id', 'group', 'cell_id', 'direction'])
_fiber_densities = compute.fiber_densities(_windows_to_compute, _subtract_border=True)
_experiments_fiber_densities = {
_key: [_fiber_densities[_tuple] for _tuple in _windows_dictionary[_key]]
for _key in _windows_dictionary
}
for _direction in _directions:
_y_arrays = [[] for _i in DERIVATIVES]
for _tuple in tqdm(_tuples, desc='Experiments loop'):
_experiment, _series_id, _group = _tuple
if (_experiment, _series_id, _group, 'left_cell', _direction) not in _windows_dictionary or \
(_experiment, _series_id, _group, 'right_cell', _direction) not in _windows_dictionary:
continue
_properties = load.group_properties(_experiment, _series_id, _group)
_left_cell_fiber_densities = \
_experiments_fiber_densities[(_experiment, _series_id, _group, 'left_cell', _direction)]
_right_cell_fiber_densities = \
_experiments_fiber_densities[(_experiment, _series_id, _group, 'right_cell', _direction)]
_left_cell_fiber_densities = compute.remove_blacklist(
_experiment, _series_id, _properties['cells_ids']['left_cell'], _left_cell_fiber_densities)
_right_cell_fiber_densities = compute.remove_blacklist(
_experiment, _series_id, _properties['cells_ids']['right_cell'], _right_cell_fiber_densities)
if not OUT_OF_BOUNDARIES:
_left_cell_fiber_densities, _right_cell_fiber_densities = \
compute.longest_same_indices_shared_in_borders_sub_array(
_left_cell_fiber_densities, _right_cell_fiber_densities
)
else:
_left_cell_fiber_densities = [_fiber_density[0] for _fiber_density in _left_cell_fiber_densities]
_right_cell_fiber_densities = [_fiber_density[0] for _fiber_density in _right_cell_fiber_densities]
# ignore small arrays
_minimum_time_frame_for_correlation = compute.minimum_time_frames_for_correlation(_experiment)
if len(_left_cell_fiber_densities) < _minimum_time_frame_for_correlation or \
len(_right_cell_fiber_densities) < _minimum_time_frame_for_correlation:
continue
for _derivative_index, _derivative in enumerate(DERIVATIVES):
_y_arrays[_derivative_index].append(compute_lib.correlation(
compute_lib.derivative(_left_cell_fiber_densities, _n=_derivative),
compute_lib.derivative(_right_cell_fiber_densities, _n=_derivative)
))
print('Direction:', _direction)
print('Total pairs:', len(_y_arrays[0]))
print('Wilcoxon around the zero')
for _y_array, _derivative in zip(_y_arrays, DERIVATIVES):
print('Derivative:', _derivative, wilcoxon(_y_array))
# plot
_y_title = 'Inner correlation' if _direction == 'inside' else 'Outer correlation'
_colors_array = config.colors(3)
_fig = go.Figure(
data=[
go.Box(
y=_y,
name=_derivative,
boxpoints='all',
jitter=1,
pointpos=0,
line={
'width': 1
},
fillcolor='white',
marker={
'size': 10,
'color': _color
},
opacity=0.7,
showlegend=False
) for _y, _derivative, _color in zip(_y_arrays, DERIVATIVES_TEXT, _colors_array)
],
layout={
'xaxis': {
'title': 'Fiber density derivative',
'zeroline': False
},
'yaxis': {
'title': _y_title,
'range': [-1, 1],
'zeroline': False,
'tickmode': 'array',
'tickvals': [-1, -0.5, 0, 0.5, 1]
}
}
)
save.to_html(
_fig=_fig,
_path=os.path.join(paths.PLOTS, save.get_module_name()),
_filename='plot_direction_' + _direction
)
def compute_cell_pairs():
_x_array = []
_y_array = []
_names_array = []
for _distances_range in PAIR_DISTANCE_RANGES:
print('Pair distance range:', str(_distances_range))
_experiments = all_experiments()
_experiments = filtering.by_categories(
_experiments=_experiments,
_is_single_cell=False,
_is_high_temporal_resolution=False,
_is_bleb=False,
_is_dead_dead=False,
_is_live_dead=False,
_is_bead=False,
_is_metastasis=False)
_tuples = load.experiments_groups_as_tuples(_experiments)
_tuples = filtering.by_time_frames_amount(
_tuples, compute.density_time_frame(_experiments[0]))
_tuples = filtering.by_real_pairs(_tuples)
_tuples = filtering.by_pair_distance_range(_tuples, _distances_range)
_tuples = filtering.by_band(_tuples)
print('Total tuples:', len(_tuples))
_max_offsets_x = []
_arguments = []
for _tuple in _tuples:
_experiment, _series_id, _group = _tuple
_time_frame = compute.minimum_time_frames_for_correlation(
_experiment)
_pair_distance = \
compute.pair_distance_in_cell_size_time_frame(_experiment, _series_id, _group, _time_frame - 1)
_offsets_x = np.arange(
start=0,
stop=_pair_distance / 2 - 0.5 -
QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER,
step=OFFSET_X_STEP)
if len(_offsets_x) > len(_max_offsets_x):
_max_offsets_x = _offsets_x
for _offset_x in _offsets_x:
for _cell_id in ['left_cell', 'right_cell']:
_arguments.append({
'experiment': _experiment,
'series_id': _series_id,
'group': _group,
'length_x':
QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER,
'length_y':
QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER,
'length_z':
QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER,
'offset_x': _offset_x,
'offset_y': OFFSET_Y,
'offset_z': OFFSET_Z,
'cell_id': _cell_id,
'direction': 'inside',
'time_point': _time_frame - 1
})
_windows_dictionary, _windows_to_compute = \
compute.windows(_arguments, _keys=['experiment', 'series_id', 'group', 'offset_x', 'cell_id'])
_fiber_densities = compute.fiber_densities(_windows_to_compute)
_pair_distance_fiber_densities = [[]
for _i in range(len(_max_offsets_x))]
for _tuple in _tuples:
_experiment, _series_id, _group = _tuple
for _offset_x_index, _offset_x in enumerate(_max_offsets_x):
for _cell_id in ['left_cell', 'right_cell']:
if (_experiment, _series_id, _group, _offset_x,
_cell_id) in _windows_dictionary:
_normalization = load.normalization_series_file_data(
_experiment, _series_id)
_window_tuple = _windows_dictionary[(_experiment,
_series_id,
_group, _offset_x,
_cell_id)][0]
_fiber_density = _fiber_densities[_window_tuple]
if not OUT_OF_BOUNDARIES and _fiber_density[1]:
continue
_normalized_fiber_density = compute_lib.z_score(
_x=_fiber_density[0],
_average=_normalization['average'],
_std=_normalization['std'])
if not np.isnan(_normalized_fiber_density):
_pair_distance_fiber_densities[
_offset_x_index].append(
_normalized_fiber_density)
_x_array.append(_max_offsets_x)
_y_array.append(_pair_distance_fiber_densities)
_names_array.append('Pair distance ' + str(_distances_range[0]) + '-' +
str(_distances_range[1]))
return _names_array, _x_array, _y_array
def main(_high_temporal_resolution=True):
_experiments = all_experiments()
_experiments = filtering.by_categories(
_experiments=_experiments,
_is_single_cell=False,
_is_high_temporal_resolution=_high_temporal_resolution,
_is_bleb=False,
_is_dead_dead=False,
_is_live_dead=False,
_is_bead=False,
_is_metastasis=False
)
_tuples = load.experiments_groups_as_tuples(_experiments)
_tuples = filtering.by_pair_distance_range(_tuples, PAIR_DISTANCE_RANGE)
_tuples = filtering.by_real_pairs(_tuples)
_tuples = filtering.by_band(_tuples)
print('Total tuples:', len(_tuples))
_arguments = []
for _tuple in _tuples:
_experiment, _series_id, _group = _tuple
_latest_time_frame = compute.latest_time_frame_before_overlapping(_experiment, _series_id, _group, OFFSET_X)
for _cell_id in ['left_cell', 'right_cell']:
_arguments.append({
'experiment': _experiment,
'series_id': _series_id,
'group': _group,
'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER,
'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER,
'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER,
'offset_x': OFFSET_X,
'offset_y': OFFSET_Y,
'offset_z': OFFSET_Z,
'cell_id': _cell_id,
'direction': 'inside',
'time_points': _latest_time_frame
})
_windows_dictionary, _windows_to_compute = compute.windows(_arguments,
_keys=['experiment', 'series_id', 'group', 'cell_id'])
_fiber_densities = compute.fiber_densities(_windows_to_compute, _subtract_border=True)
_experiments_fiber_densities = {
_key: [_fiber_densities[_tuple] for _tuple in _windows_dictionary[_key]]
for _key in _windows_dictionary
}
_tuples_by_experiment = organize.by_experiment(_tuples)
_y_arrays = [[] for _i in TIME_FRAMES_STEPS[_high_temporal_resolution]]
_x_array = []
for _time_frame_index, _time_frame_every in enumerate(TIME_FRAMES_STEPS[_high_temporal_resolution]):
print('Temporal resolution (minutes):', _time_frame_every * TEMPORAL_RESOLUTION[_high_temporal_resolution])
_higher_same_counter = 0
_valid_tuples = []
for _time_frame_begin in range(_time_frame_every):
for _experiment in _tuples_by_experiment:
_experiment_tuples = _tuples_by_experiment[_experiment]
for _same_index in range(len(_experiment_tuples)):
_same_tuple = _experiment_tuples[_same_index]
_same_experiment, _same_series, _same_group = _same_tuple
_same_left_cell_fiber_densities = \
_experiments_fiber_densities[
(_same_experiment, _same_series, _same_group, 'left_cell')
]
_same_right_cell_fiber_densities = \
_experiments_fiber_densities[
(_same_experiment, _same_series, _same_group, 'right_cell')
]
_same_properties = \
load.group_properties(_same_experiment, _same_series, _same_group)
_same_left_cell_fiber_densities = compute.remove_blacklist(
_same_experiment,
_same_series,
_same_properties['cells_ids']['left_cell'],
_same_left_cell_fiber_densities
)
_same_right_cell_fiber_densities = compute.remove_blacklist(
_same_experiment,
_same_series,
_same_properties['cells_ids']['right_cell'],
_same_right_cell_fiber_densities
)
_same_left_cell_fiber_densities_filtered, _same_right_cell_fiber_densities_filtered = \
compute.longest_same_indices_shared_in_borders_sub_array(
_same_left_cell_fiber_densities, _same_right_cell_fiber_densities
)
# ignore small arrays
if len(_same_left_cell_fiber_densities_filtered) < compute.minimum_time_frames_for_correlation(_same_experiment):
continue
_same_left_cell_fiber_densities_filtered = \
_same_left_cell_fiber_densities_filtered[_time_frame_begin::_time_frame_every]
_same_right_cell_fiber_densities_filtered = \
_same_right_cell_fiber_densities_filtered[_time_frame_begin::_time_frame_every]
# secondary ignore
if len(_same_left_cell_fiber_densities_filtered) < \
GENERAL_MINIMUM_CORRELATION_TIME_FRAMES[_high_temporal_resolution]:
continue
_same_correlation = compute_lib.correlation(
compute_lib.derivative(_same_left_cell_fiber_densities_filtered, _n=DERIVATIVE),
compute_lib.derivative(_same_right_cell_fiber_densities_filtered, _n=DERIVATIVE)
)
for _different_index in range(len(_experiment_tuples)):
if _same_index != _different_index:
_different_tuple = _experiment_tuples[_different_index]
_different_experiment, _different_series, _different_group = \
_different_tuple
for _same_cell_id, _different_cell_id in product(['left_cell', 'right_cell'],
['left_cell', 'right_cell']):
_same_fiber_densities = _experiments_fiber_densities[(
_same_experiment,
_same_series,
_same_group,
_same_cell_id
)]
_different_fiber_densities = _experiments_fiber_densities[(
_different_experiment,
_different_series,
_different_group,
_different_cell_id
)]
_different_properties = load.group_properties(
_different_experiment, _different_series, _different_group
)
_same_fiber_densities = compute.remove_blacklist(
_same_experiment,
_same_series,
_same_properties['cells_ids'][_same_cell_id],
_same_fiber_densities
)
_different_fiber_densities = compute.remove_blacklist(
_different_experiment,
_different_series,
_different_properties['cells_ids'][_different_cell_id],
_different_fiber_densities
)
_same_fiber_densities_filtered, _different_fiber_densities_filtered = \
compute.longest_same_indices_shared_in_borders_sub_array(
_same_fiber_densities, _different_fiber_densities
)
# ignore small arrays
if len(_same_fiber_densities_filtered) < compute.minimum_time_frames_for_correlation(_different_experiment):
continue
_same_fiber_densities_filtered = \
_same_fiber_densities_filtered[_time_frame_begin::_time_frame_every]
_different_fiber_densities_filtered = \
_different_fiber_densities_filtered[_time_frame_begin::_time_frame_every]
# secondary ignore
if len(_same_fiber_densities_filtered) < \
GENERAL_MINIMUM_CORRELATION_TIME_FRAMES[_high_temporal_resolution]:
continue
_different_correlation = compute_lib.correlation(
compute_lib.derivative(_same_fiber_densities_filtered, _n=DERIVATIVE),
compute_lib.derivative(_different_fiber_densities_filtered, _n=DERIVATIVE)
)
_point_distance = compute_lib.distance_from_a_point_to_a_line(
_line=[-1, -1, 1, 1],
_point=[_same_correlation, _different_correlation]
)
if _same_correlation > _different_correlation:
_y_arrays[_time_frame_index].append(_point_distance)
_higher_same_counter += 1
else:
_y_arrays[_time_frame_index].append(-_point_distance)
if _same_tuple not in _valid_tuples:
_valid_tuples.append(_same_tuple)
print('Total tuples:', len(_valid_tuples))
print('Total points:', len(_y_arrays[_time_frame_index]))
print('Wilcoxon around the zero:')
print(wilcoxon(_y_arrays[_time_frame_index]))
print('Higher same amount:', _higher_same_counter / len(_y_arrays[_time_frame_index]))
_x_array.append(_time_frame_every * TEMPORAL_RESOLUTION[_high_temporal_resolution])
# plot
_fig = go.Figure(
data=go.Scatter(
x=_x_array,
y=[np.mean(_array) for _array in _y_arrays],
error_y={
'type': 'data',
'array': [np.std(_array) for _array in _y_arrays],
'thickness': 1,
'color': '#ea8500'
},
mode='markers',
marker={
'size': 15,
'color': '#ea8500'
},
showlegend=False
),
layout={
'xaxis': {
'title': 'Temporal resolution (minutes)',
'zeroline': False
},
'yaxis': {
'title': 'Same minus different correlation',
'range': [-1, 1.1],
'zeroline': False,
'tickmode': 'array',
'tickvals': [-1, -0.5, 0, 0.5, 1]
}
}
)
save.to_html(
_fig=_fig,
_path=os.path.join(paths.PLOTS, save.get_module_name()),
_filename='plot_high_temporal_res_' + str(_high_temporal_resolution)
)
def main():
print('Single Cell')
_experiments = all_experiments()
_experiments = filtering.by_categories(_experiments=_experiments,
_is_single_cell=True,
_is_high_temporal_resolution=False,
_is_bleb=False,
_is_dead_dead=False,
_is_live_dead=False,
_is_bead=False,
_is_metastasis=False)
_tuples = load.experiments_groups_as_tuples(_experiments)
_tuples = filtering.by_time_frames_amount(
_tuples, compute.minimum_time_frames_for_correlation(_experiments[0]))
_arguments = []
for _tuple in _tuples:
_experiment, _series_id, _group = _tuple
_time_frame = compute.minimum_time_frames_for_correlation(_experiment)
for _offset_x, _direction in product(OFFSETS_X, ['left', 'right']):
_arguments.append({
'experiment': _experiment,
'series_id': _series_id,
'group': _group,
'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER,
'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER,
'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER,
'offset_x': _offset_x,
'offset_y': OFFSET_Y,
'offset_z': OFFSET_Z,
'cell_id': 'cell',
'direction': _direction,
'time_point': _time_frame - 1
})
_windows_dictionary, _windows_to_compute = \
compute.windows(_arguments, _keys=['experiment', 'series_id', 'group', 'offset_x', 'direction'])
_fiber_densities = compute.fiber_densities(_windows_to_compute)
_tuples = organize.by_single_cell_id(_tuples)
_single_cell_fiber_densities = [[] for _i in range(len(OFFSETS_X))]
for _tuple in _tuples:
_experiment, _series_id, _cell_id = _tuple
print('Experiment:',
_experiment,
'Series ID:',
_series_id,
'Cell ID:',
_cell_id,
sep='\t')
_offset_index = 0
_normalization = load.normalization_series_file_data(
_experiment, _series_id)
for _offset_x in OFFSETS_X:
_cell_fiber_densities = []
for _cell_tuple in _tuples[_tuple]:
_, _, _group = _cell_tuple
for _direction in ['left', 'right']:
_window_tuple = _windows_dictionary[(_experiment,
_series_id, _group,
_offset_x,
_direction)][0]
_fiber_density = _fiber_densities[_window_tuple]
if not OUT_OF_BOUNDARIES and _fiber_density[1]:
continue
_normalized_fiber_density = compute_lib.z_score(
_x=_fiber_density[0],
_average=_normalization['average'],
_std=_normalization['std'])
_cell_fiber_densities.append(_normalized_fiber_density)
if len(_cell_fiber_densities) > 0:
_single_cell_fiber_densities[_offset_index].append(
np.mean(_cell_fiber_densities))
_offset_index += 1
print('Pairs')
_experiments = all_experiments()
_experiments = filtering.by_categories(_experiments=_experiments,
_is_single_cell=False,
_is_high_temporal_resolution=False,
_is_bleb=False,
_is_dead_dead=False,
_is_live_dead=False,
_is_bead=False,
_is_metastasis=False)
_tuples = load.experiments_groups_as_tuples(_experiments)
_tuples = filtering.by_time_frames_amount(
_tuples, compute.minimum_time_frames_for_correlation(_experiments[0]))
_tuples = filtering.by_real_pairs(_tuples)
_tuples = filtering.by_pair_distance(_tuples, PAIR_DISTANCE)
_tuples = filtering.by_band(_tuples)
_arguments = []
for _tuple in _tuples:
_experiment, _series_id, _group = _tuple
_time_frame = compute.minimum_time_frames_for_correlation(_experiment)
for _offset_x in OFFSETS_X:
_arguments.append({
'experiment': _experiment,
'series_id': _series_id,
'group': _group,
'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER,
'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER,
'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER,
'offset_x': _offset_x,
'offset_y': OFFSET_Y,
'offset_z': OFFSET_Z,
'cell_id': 'left_cell',
'direction': 'inside',
'time_point': _time_frame - 1
})
_windows_dictionary, _windows_to_compute = \
compute.windows(_arguments, _keys=['experiment', 'series_id', 'group', 'offset_x'])
_fiber_densities = compute.fiber_densities(_windows_to_compute)
_pairs_fiber_densities = [[] for _i in range(len(OFFSETS_X))]
for _tuple in _tuples:
_experiment, _series_id, _group = _tuple
print('Experiment:',
_experiment,
'Series ID:',
_series_id,
'Group:',
_group,
sep='\t')
_offset_index = 0
_normalization = load.normalization_series_file_data(
_experiment, _series_id)
# take offsets based on pair distance
_properties = load.group_properties(_experiment, _series_id, _group)
_left_cell_coordinates = [
list(_properties['time_points'][0]['left_cell']
['coordinates'].values())
]
_right_cell_coordinates = [
list(_properties['time_points'][0]['right_cell']
['coordinates'].values())
]
_pair_distance = compute.pair_distance_in_cell_size(
_experiment, _series_id, _left_cell_coordinates,
_right_cell_coordinates)
_edges_distance = _pair_distance - 1
_max_x_offset = _edges_distance - QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER
for _offset_x in OFFSETS_X:
if _offset_x > _max_x_offset:
break
_fiber_density = _fiber_densities[_windows_dictionary[(
_experiment, _series_id, _group, _offset_x)][0]]
if not OUT_OF_BOUNDARIES and _fiber_density[1]:
continue
_normalized_fiber_density = compute_lib.z_score(
_x=_fiber_density[0],
_average=_normalization['average'],
_std=_normalization['std'])
_pairs_fiber_densities[_offset_index].append(
_normalized_fiber_density)
_offset_index += 1
# plot
_fig = go.Figure(data=[
go.Scatter(x=OFFSETS_X,
y=[np.mean(_array) for _array in _pairs_fiber_densities],
name='Pairs',
error_y={
'type':
'data',
'array':
[np.std(_array) for _array in _pairs_fiber_densities],
'thickness':
1
},
mode='lines+markers',
line={'dash': 'solid'}),
go.Scatter(
x=OFFSETS_X,
y=[np.mean(_array) for _array in _single_cell_fiber_densities],
name='Single Cell',
error_y={
'type':
'data',
'array':
[np.std(_array) for _array in _single_cell_fiber_densities],
'thickness':
1
},
mode='lines+markers',
line={'dash': 'dash'})
],
layout={
'xaxis_title': 'Distance from left cell (cell size)',
'yaxis_title': 'Fiber density (z-score)'
})
save.to_html(_fig=_fig,
_path=os.path.join(paths.PLOTS, save.get_module_name()),
_filename='plot_distance_' + str(PAIR_DISTANCE))
def compute_fiber_densities(_offset_y):
_experiments = all_experiments()
_experiments = filtering.by_categories(_experiments=_experiments,
_is_single_cell=False,
_is_high_temporal_resolution=False,
_is_bleb=False,
_is_dead_dead=False,
_is_live_dead=False,
_is_bead=False,
_is_metastasis=False)
_tuples = load.experiments_groups_as_tuples(_experiments)
_tuples = filtering.by_pair_distance_range(_tuples, PAIR_DISTANCE_RANGE)
_tuples = filtering.by_real_pairs(_tuples)
_tuples = filtering.by_band(_tuples)
print('Total tuples:', len(_tuples))
_arguments = []
for _tuple in _tuples:
_experiment, _series_id, _group = _tuple
_latest_time_frame = compute.latest_time_frame_before_overlapping(
_experiment, _series_id, _group, OFFSET_X)
for _cell_id in ['left_cell', 'right_cell']:
_arguments.append({
'experiment': _experiment,
'series_id': _series_id,
'group': _group,
'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER,
'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER,
'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER,
'offset_x': OFFSET_X,
'offset_y': _offset_y,
'offset_z': OFFSET_Z,
'cell_id': _cell_id,
'direction': 'inside',
'time_points': _latest_time_frame
})
_windows_dictionary, _windows_to_compute = compute.windows(
_arguments, _keys=['experiment', 'series_id', 'group', 'cell_id'])
_fiber_densities = compute.fiber_densities(_windows_to_compute,
_saturation=True,
_subtract_border=True)
_experiments_fiber_densities = {
_key:
[_fiber_densities[_tuple][:2] for _tuple in _windows_dictionary[_key]]
for _key in _windows_dictionary
}
_experiments_saturation = {
_key: [(_fiber_densities[_tuple][2], _fiber_densities[_tuple][1])
for _tuple in _windows_dictionary[_key]]
for _key in _windows_dictionary
}
_tuples_by_experiment = organize.by_experiment(_tuples)
_correlations_array = []
_saturation_array = []
for _experiment in _tuples_by_experiment:
print('Experiment:', _experiment)
_experiment_tuples = _tuples_by_experiment[_experiment]
for _tuple in tqdm(_experiment_tuples, desc='Main loop'):
_, _series, _group = _tuple
_left_cell_fiber_densities = _experiments_fiber_densities[(
_experiment, _series, _group, 'left_cell')]
_right_cell_fiber_densities = _experiments_fiber_densities[(
_experiment, _series, _group, 'right_cell')]
_properties = load.group_properties(_experiment, _series, _group)
_left_cell_fiber_densities = compute.remove_blacklist(
_experiment, _series, _properties['cells_ids']['left_cell'],
_left_cell_fiber_densities)
_right_cell_fiber_densities = compute.remove_blacklist(
_experiment, _series, _properties['cells_ids']['right_cell'],
_right_cell_fiber_densities)
_left_cell_fiber_densities_filtered, _right_cell_fiber_densities_filtered = \
compute.longest_same_indices_shared_in_borders_sub_array(
_left_cell_fiber_densities, _right_cell_fiber_densities
)
# ignore small arrays
if len(_left_cell_fiber_densities_filtered
) < compute.minimum_time_frames_for_correlation(
_experiment):
continue
_correlation = compute_lib.correlation(
compute_lib.derivative(_left_cell_fiber_densities_filtered,
_n=DERIVATIVE),
compute_lib.derivative(_right_cell_fiber_densities_filtered,
_n=DERIVATIVE))
# saturation
_left_cell_saturation = _experiments_saturation[(_experiment,
_series, _group,
'left_cell')]
_right_cell_saturation = _experiments_saturation[(_experiment,
_series, _group,
'right_cell')]
_left_cell_saturation = compute.remove_blacklist(
_experiment, _series, _properties['cells_ids']['left_cell'],
_left_cell_saturation)
_right_cell_saturation = compute.remove_blacklist(
_experiment, _series, _properties['cells_ids']['right_cell'],
_right_cell_saturation)
_left_cell_saturation_filtered, _right_cell_saturation_filtered = \
compute.longest_same_indices_shared_in_borders_sub_array(
_left_cell_saturation, _right_cell_saturation
)
_left_saturation_fraction_last_time_frame = _left_cell_saturation_filtered[
-1]
_right_saturation_fraction_last_time_frame = _right_cell_saturation_filtered[
-1]
_group_saturation_fraction_mean = \
(_left_saturation_fraction_last_time_frame + _right_saturation_fraction_last_time_frame) / 2
_correlations_array.append(_correlation)
_saturation_array.append(_group_saturation_fraction_mean)
print('Total points:', len(_correlations_array))
print('Wilcoxon of correlations around the zero:')
print(wilcoxon(_correlations_array))
print('Pearson correlation of correlations and saturation fraction mean:')
print(
compute_lib.correlation(_correlations_array,
_saturation_array,
_with_p_value=True))
return _correlations_array, _saturation_array
def main(_band=True, _high_temporal_resolution=False):
_y_arrays = [[] for _i in PAIR_DISTANCE_RANGES]
_names_array = []
for _distances_index, _distances_range in enumerate(PAIR_DISTANCE_RANGES):
print('Pair distance range:', str(_distances_range))
_experiments = all_experiments()
_experiments = filtering.by_categories(
_experiments=_experiments,
_is_single_cell=False,
_is_high_temporal_resolution=_high_temporal_resolution,
_is_bleb=False,
_is_dead_dead=False,
_is_live_dead=False,
_is_bead=False,
_is_metastasis=False)
_tuples = load.experiments_groups_as_tuples(_experiments)
_tuples = filtering.by_pair_distance_range(_tuples, _distances_range)
_tuples = filtering.by_real_pairs(_tuples)
_tuples = filtering.by_band(_tuples, _band=_band)
print('Total tuples:', len(_tuples))
_arguments = []
for _tuple in _tuples:
_experiment, _series_id, _group = _tuple
_latest_time_frame = compute.latest_time_frame_before_overlapping(
_experiment, _series_id, _group, OFFSET_X)
for _cell_id in ['left_cell', 'right_cell']:
_arguments.append({
'experiment': _experiment,
'series_id': _series_id,
'group': _group,
'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER,
'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER,
'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER,
'offset_x': OFFSET_X,
'offset_y': OFFSET_Y,
'offset_z': OFFSET_Z,
'cell_id': _cell_id,
'direction': 'inside',
'time_points': _latest_time_frame
})
_windows_dictionary, _windows_to_compute = compute.windows(
_arguments, _keys=['experiment', 'series_id', 'group', 'cell_id'])
_fiber_densities = compute.fiber_densities(_windows_to_compute,
_subtract_border=True)
_experiments_fiber_densities = {
_key:
[_fiber_densities[_tuple] for _tuple in _windows_dictionary[_key]]
for _key in _windows_dictionary
}
_tuples_by_experiment = organize.by_experiment(_tuples)
_higher_same_counter = 0
_valid_tuples = []
for _experiment in _tuples_by_experiment:
print('Experiment:', _experiment)
_experiment_tuples = _tuples_by_experiment[_experiment]
for _same_index in tqdm(range(len(_experiment_tuples)),
desc='Main loop'):
_same_tuple = _experiment_tuples[_same_index]
_same_experiment, _same_series, _same_group = _same_tuple
_same_left_cell_fiber_densities = \
_experiments_fiber_densities[
(_same_experiment, _same_series, _same_group, 'left_cell')
]
_same_right_cell_fiber_densities = \
_experiments_fiber_densities[
(_same_experiment, _same_series, _same_group, 'right_cell')
]
_same_properties = \
load.group_properties(_same_experiment, _same_series, _same_group)
_same_left_cell_fiber_densities = compute.remove_blacklist(
_same_experiment, _same_series,
_same_properties['cells_ids']['left_cell'],
_same_left_cell_fiber_densities)
_same_right_cell_fiber_densities = compute.remove_blacklist(
_same_experiment, _same_series,
_same_properties['cells_ids']['right_cell'],
_same_right_cell_fiber_densities)
_same_left_cell_fiber_densities_filtered, _same_right_cell_fiber_densities_filtered = \
compute.longest_same_indices_shared_in_borders_sub_array(
_same_left_cell_fiber_densities, _same_right_cell_fiber_densities
)
# ignore small arrays
if len(_same_left_cell_fiber_densities_filtered
) < compute.minimum_time_frames_for_correlation(
_same_experiment):
continue
_same_correlation = compute_lib.correlation(
compute_lib.derivative(
_same_left_cell_fiber_densities_filtered,
_n=DERIVATIVE),
compute_lib.derivative(
_same_right_cell_fiber_densities_filtered,
_n=DERIVATIVE))
for _different_index in range(len(_experiment_tuples)):
if _same_index != _different_index:
_different_tuple = _experiment_tuples[_different_index]
_different_experiment, _different_series, _different_group = \
_different_tuple
for _same_cell_id, _different_cell_id in product(
['left_cell', 'right_cell'],
['left_cell', 'right_cell']):
_same_fiber_densities = _experiments_fiber_densities[
(_same_experiment, _same_series, _same_group,
_same_cell_id)]
_different_fiber_densities = _experiments_fiber_densities[
(_different_experiment, _different_series,
_different_group, _different_cell_id)]
_different_properties = load.group_properties(
_different_experiment, _different_series,
_different_group)
_same_fiber_densities = compute.remove_blacklist(
_same_experiment, _same_series,
_same_properties['cells_ids'][_same_cell_id],
_same_fiber_densities)
_different_fiber_densities = compute.remove_blacklist(
_different_experiment, _different_series,
_different_properties['cells_ids']
[_different_cell_id],
_different_fiber_densities)
_same_fiber_densities_filtered, _different_fiber_densities_filtered = \
compute.longest_same_indices_shared_in_borders_sub_array(
_same_fiber_densities, _different_fiber_densities
)
# ignore small arrays
if len(
_same_fiber_densities_filtered
) < compute.minimum_time_frames_for_correlation(
_different_experiment):
continue
_different_correlation = compute_lib.correlation(
compute_lib.derivative(
_same_fiber_densities_filtered,
_n=DERIVATIVE),
compute_lib.derivative(
_different_fiber_densities_filtered,
_n=DERIVATIVE))
_point_distance = compute_lib.distance_from_a_point_to_a_line(
_line=[-1, -1, 1, 1],
_point=[
_same_correlation, _different_correlation
])
if _same_correlation > _different_correlation:
_y_arrays[_distances_index].append(
_point_distance)
_higher_same_counter += 1
else:
_y_arrays[_distances_index].append(
-_point_distance)
if _same_tuple not in _valid_tuples:
_valid_tuples.append(_same_tuple)
print('Total tuples:', len(_valid_tuples))
print('Total points:', len(_y_arrays[_distances_index]))
print('Wilcoxon around the zero:')
print(wilcoxon(_y_arrays[_distances_index]))
print('Higher same amount:',
_higher_same_counter / len(_y_arrays[_distances_index]))
_names_array.append(
str(_distances_range[0]) + '-' + str(_distances_range[1]))
# plot
_colors_array = config.colors(3)
_fig = go.Figure(data=[
go.Box(y=_y,
name=_name,
boxpoints=False,
line={'width': 1},
marker={
'size': 10,
'color': _color
},
showlegend=False)
for _y, _name, _color in zip(_y_arrays, _names_array, _colors_array)
],
layout={
'xaxis': {
'title': 'Pair distance (cell diameter)',
'zeroline': False,
'type': 'category'
},
'yaxis': {
'title': 'Same minus different correlation',
'range': [-1, 1.1],
'zeroline': False,
'tickmode': 'array',
'tickvals': [-1, -0.5, 0, 0.5, 1]
}
})
save.to_html(_fig=_fig,
_path=os.path.join(paths.PLOTS, save.get_module_name()),
_filename='plot_band_' + str(_band) + '_high_temporal_res_' +
str(_high_temporal_resolution))
def main():
_experiments = all_experiments()
_experiments = filtering.by_categories(_experiments=_experiments,
_is_single_cell=False,
_is_high_temporal_resolution=False,
_is_bleb=False,
_is_dead_dead=False,
_is_live_dead=False,
_is_bead=False,
_is_metastasis=False)
_tuples = load.experiments_groups_as_tuples(_experiments)
_tuples = filtering.by_time_frames_amount(
_tuples, compute.density_time_frame(_experiments[0]))
_tuples = filtering.by_real_pairs(_tuples)
_tuples = filtering.by_band(_tuples)
_tuples = filtering.by_pair_distance_range(_tuples, PAIR_DISTANCE_RANGE)
print('Total tuples:', len(_tuples))
_arguments = []
for _tuple in _tuples:
_experiment, _series_id, _group = _tuple
_latest_time_frame = compute.latest_time_frame_before_overlapping(
_experiment, _series_id, _group, OFFSET_X)
for _cell_id in ['left_cell', 'right_cell']:
_arguments.append({
'experiment': _experiment,
'series_id': _series_id,
'group': _group,
'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER,
'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER,
'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER,
'offset_x': OFFSET_X,
'offset_y': OFFSET_Y,
'offset_z': OFFSET_Z,
'cell_id': _cell_id,
'direction': 'inside',
'time_points': _latest_time_frame
})
_windows_dictionary, _windows_to_compute = \
compute.windows(_arguments, _keys=['experiment', 'series_id', 'group', 'cell_id'])
_fiber_densities = compute.fiber_densities(_windows_to_compute,
_subtract_border=True)
_experiments_fiber_densities = {
_key:
[_fiber_densities[_tuple] for _tuple in _windows_dictionary[_key]]
for _key in _windows_dictionary
}
_kpss_y_arrays = [[] for _i in DERIVATIVES]
_adf_y_arrays = [[] for _i in DERIVATIVES]
for _tuple in tqdm(_tuples, desc='Experiments loop'):
_experiment, _series_id, _group = _tuple
_properties = load.group_properties(_experiment, _series_id, _group)
_left_cell_fiber_densities = _experiments_fiber_densities[(
_experiment, _series_id, _group, 'left_cell')]
_left_cell_fiber_densities = compute.remove_blacklist(
_experiment, _series_id, _properties['cells_ids']['left_cell'],
_left_cell_fiber_densities)
_right_cell_fiber_densities = _experiments_fiber_densities[(
_experiment, _series_id, _group, 'right_cell')]
_right_cell_fiber_densities = compute.remove_blacklist(
_experiment, _series_id, _properties['cells_ids']['right_cell'],
_right_cell_fiber_densities)
if not OUT_OF_BOUNDARIES:
_left_cell_fiber_densities = \
compute.longest_fiber_densities_ascending_sequence(_left_cell_fiber_densities)
_right_cell_fiber_densities = \
compute.longest_fiber_densities_ascending_sequence(_right_cell_fiber_densities)
else:
_left_cell_fiber_densities = [
_fiber_density[0]
for _fiber_density in _left_cell_fiber_densities
]
_right_cell_fiber_densities = [
_fiber_density[0]
for _fiber_density in _right_cell_fiber_densities
]
# ignore small arrays
_minimum_time_frames_for_correlation = compute.minimum_time_frames_for_correlation(
_experiment)
if len(_left_cell_fiber_densities) < _minimum_time_frames_for_correlation or \
len(_right_cell_fiber_densities) < _minimum_time_frames_for_correlation:
continue
for _derivative_index, _derivative in enumerate(DERIVATIVES):
for _cell_fiber_densities in [
_left_cell_fiber_densities, _right_cell_fiber_densities
]:
_cell_fiber_densities_derivative = compute_lib.derivative(
_cell_fiber_densities, _n=_derivative)
_, _kpss_p_value, _, _ = kpss(_cell_fiber_densities_derivative,
nlags='legacy')
_kpss_y_arrays[_derivative_index].append(_kpss_p_value)
_, _adf_p_value, _, _, _, _ = adfuller(
_cell_fiber_densities_derivative)
_adf_y_arrays[_derivative_index].append(_adf_p_value)
print('Total pairs:', len(_kpss_y_arrays[0]) / 2)
# print results
print('KPSS:')
for _derivative_index, _derivative in enumerate(DERIVATIVES):
_stationary_count = len([
_value for _value in _kpss_y_arrays[_derivative_index]
if _value > 0.05
])
print(
'Derivative:', _derivative, 'Stationary:',
str(_stationary_count / len(_kpss_y_arrays[_derivative_index]) *
100) + '%')
print('ADF:')
for _derivative_index, _derivative in enumerate(DERIVATIVES):
_stationary_count = len([
_value for _value in _adf_y_arrays[_derivative_index]
if _value < 0.05
])
print(
'Derivative:', _derivative, 'Stationary:',
str(_stationary_count / len(_adf_y_arrays[_derivative_index]) *
100) + '%')
# plot
_colors_array = config.colors(3)
for _test_name, _y_title, _y_tickvals, _p_value_line, _y_arrays in \
zip(
['kpss', 'adf'],
['KPSS test p-value', 'ADF test p-value'],
[[0.05, 0.1], [0.05, 1]],
[0.05, 0.05],
[_kpss_y_arrays, _adf_y_arrays]
):
_fig = go.Figure(data=[
go.Box(y=_y,
name=_derivative,
boxpoints='all',
jitter=1,
pointpos=0,
line={'width': 1},
fillcolor='white',
marker={
'size': 10,
'color': _color
},
opacity=0.7,
showlegend=False) for _y, _derivative, _color in zip(
_y_arrays, DERIVATIVES_TEXT, _colors_array)
],
layout={
'xaxis': {
'title': 'Fiber density derivative',
'zeroline': False
},
'yaxis': {
'title': _y_title,
'zeroline': False,
'tickmode': 'array',
'tickvals': _y_tickvals
},
'shapes': [{
'type': 'line',
'x0': DERIVATIVES[0] - 0.75,
'y0': _p_value_line,
'x1': DERIVATIVES[-1] + 0.75,
'y1': _p_value_line,
'line': {
'color': 'red',
'width': 2,
'dash': 'dash'
}
}]
})
save.to_html(_fig=_fig,
_path=os.path.join(paths.PLOTS, save.get_module_name()),
_filename='plot_' + _test_name)
def main(_real_cells=True,
_static=False,
_dead_dead=False,
_live_dead=False,
_dead=False,
_live=False,
_bead=False,
_metastasis=False,
_bleb=False,
_bleb_amount_um=None,
_band=True,
_high_temporal_resolution=False,
_offset_y=0.5):
_experiments = all_experiments()
_experiments = filtering.by_categories(
_experiments=_experiments,
_is_single_cell=False,
_is_high_temporal_resolution=_high_temporal_resolution,
_is_bleb=_bleb,
_is_dead_dead=_dead_dead,
_is_live_dead=_live_dead,
_is_bead=_bead,
_is_metastasis=_metastasis)
_tuples = load.experiments_groups_as_tuples(_experiments)
_tuples = filtering.by_pair_distance_range(_tuples, PAIR_DISTANCE_RANGE)
_tuples = filtering.by_real_pairs(_tuples, _real_pairs=_real_cells)
_tuples = filtering.by_fake_static_pairs(_tuples,
_fake_static_pairs=_static)
if _dead_dead is not False or _live_dead is not False:
_tuples = filtering.by_dead_live(_tuples, _dead=_dead, _live=_live)
_tuples = filtering.by_band(_tuples, _band=_band)
if _bleb:
_tuples = filtering.by_bleb_amount_um(_tuples,
_amount_um=_bleb_amount_um)
print('Total tuples:', len(_tuples))
_arguments = []
for _tuple in _tuples:
_experiment, _series_id, _group = _tuple
_latest_time_frame = compute.latest_time_frame_before_overlapping(
_experiment, _series_id, _group, OFFSET_X)
for _cell_id in ['left_cell', 'right_cell']:
_arguments.append({
'experiment': _experiment,
'series_id': _series_id,
'group': _group,
'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER,
'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER,
'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER,
'offset_x': OFFSET_X,
'offset_y': _offset_y,
'offset_z': OFFSET_Z,
'cell_id': _cell_id,
'direction': 'inside',
'time_points': _latest_time_frame
})
_windows_dictionary, _windows_to_compute = compute.windows(
_arguments, _keys=['experiment', 'series_id', 'group', 'cell_id'])
_fiber_densities = compute.fiber_densities(_windows_to_compute,
_subtract_border=True)
_experiments_fiber_densities = {
_key:
[_fiber_densities[_tuple] for _tuple in _windows_dictionary[_key]]
for _key in _windows_dictionary
}
_tuples_by_experiment = organize.by_experiment(_tuples)
_n = 0
_cells_ranks = []
for _experiment in _tuples_by_experiment:
print('Experiment:', _experiment)
_experiment_tuples = _tuples_by_experiment[_experiment]
for _pivot_tuple in tqdm(_experiment_tuples, desc='Main loop'):
_pivot_experiment, _pivot_series_id, _pivot_group = _pivot_tuple
_pivot_experiment_properties = load.group_properties(
_pivot_experiment, _pivot_series_id, _pivot_group)
for _pivot_cell_id, _pivot_cell_correct_match_cell_id in \
zip(['left_cell', 'right_cell'], ['right_cell', 'left_cell']):
_pivot_cell = (_pivot_experiment, _pivot_series_id,
_pivot_group, _pivot_cell_id)
_pivot_cell_correct_match_cell = (
_pivot_experiment, _pivot_series_id, _pivot_group,
_pivot_cell_correct_match_cell_id)
_pivot_cell_fiber_densities = _experiments_fiber_densities[
_pivot_cell]
_pivot_cell_fiber_densities = compute.remove_blacklist(
_pivot_experiment, _pivot_series_id,
_pivot_experiment_properties['cells_ids'][_pivot_cell_id],
_pivot_cell_fiber_densities)
_pivot_cell_correlations = []
# correct match
_pivot_cell_correct_match_fiber_densities = _experiments_fiber_densities[
_pivot_cell_correct_match_cell]
_pivot_cell_correct_match_fiber_densities = compute.remove_blacklist(
_pivot_experiment, _pivot_series_id,
_pivot_experiment_properties['cells_ids']
[_pivot_cell_correct_match_cell_id],
_pivot_cell_correct_match_fiber_densities)
_pivot_cell_fiber_densities_filtered, _pivot_cell_correct_match_fiber_densities_filtered = \
compute.longest_same_indices_shared_in_borders_sub_array(
_pivot_cell_fiber_densities, _pivot_cell_correct_match_fiber_densities
)
# ignore small arrays
if len(_pivot_cell_fiber_densities_filtered
) < compute.minimum_time_frames_for_correlation(
_pivot_experiment):
continue
_correlation = compute_lib.correlation(
compute_lib.derivative(
_pivot_cell_fiber_densities_filtered, _n=DERIVATIVE),
compute_lib.derivative(
_pivot_cell_correct_match_fiber_densities_filtered,
_n=DERIVATIVE))
_pivot_cell_correlations.append(_correlation)
_pivot_cell_correct_match_correlation = _correlation
# create list of potential cells
_candidate_tuples = []
for _candidate_tuple in _experiment_tuples:
_candidate_experiment, _candidate_series_id, _candidate_group = _candidate_tuple
for _candidate_cell_id in ['left_cell', 'right_cell']:
_candidate_cell = (_candidate_experiment,
_candidate_series_id,
_candidate_group,
_candidate_cell_id)
# skip if same cell or correct match
if _candidate_cell == _pivot_cell or _candidate_cell == _pivot_cell_correct_match_cell:
continue
_candidate_tuples.append(_candidate_cell)
# compare with each potential candidate, until reached the maximum or nothing to compare with
while len(_pivot_cell_correlations
) < POTENTIAL_MATCHES and len(_candidate_tuples) > 0:
# sample randomly
_candidate_cell = random.choice(_candidate_tuples)
_candidate_experiment, _candidate_series_id, _candidate_group, _candidate_cell_id = _candidate_cell
_candidate_experiment_properties = load.group_properties(
_candidate_experiment, _candidate_series_id,
_candidate_group)
_candidate_cell_fiber_densities = _experiments_fiber_densities[
_candidate_cell]
_candidate_cell_fiber_densities = compute.remove_blacklist(
_candidate_experiment, _candidate_series_id,
_candidate_experiment_properties['cells_ids']
[_candidate_cell_id], _candidate_cell_fiber_densities)
_pivot_cell_fiber_densities_filtered, _candidate_cell_fiber_densities_filtered = \
compute.longest_same_indices_shared_in_borders_sub_array(
_pivot_cell_fiber_densities, _candidate_cell_fiber_densities
)
# ignore small arrays
if len(_pivot_cell_fiber_densities_filtered
) < compute.minimum_time_frames_for_correlation(
_pivot_experiment):
_candidate_tuples.remove(_candidate_cell)
continue
_correlation = compute_lib.correlation(
compute_lib.derivative(
_pivot_cell_fiber_densities_filtered,
_n=DERIVATIVE),
compute_lib.derivative(
_candidate_cell_fiber_densities_filtered,
_n=DERIVATIVE))
_pivot_cell_correlations.append(_correlation)
# nothing to compare with
if len(_pivot_cell_correlations) == 1:
continue
# check matchmaking
_pivot_cell_correct_match_rank = 1
for _potential_match_correlation in sorted(
_pivot_cell_correlations, reverse=True):
if _pivot_cell_correct_match_correlation == _potential_match_correlation:
break
_pivot_cell_correct_match_rank += 1
_n += 1
_cells_ranks.append(_pivot_cell_correct_match_rank)
# results
_correct_match_probability = 1 / POTENTIAL_MATCHES
_first_place_correct_matches = sum(
[1 for _rank in _cells_ranks if _rank == 1])
_first_place_fraction = _first_place_correct_matches / _n
print('Matchmaking results:')
print('Total cells:', _n)
print('Correct match probability:', round(_correct_match_probability, 2))
print('Fraction of first place correct matches:',
round(_first_place_fraction, 2))
# plot
_x = list(range(MAX_RANK))
_x_text = [str(_rank + 1) for _rank in _x[:-1]] + [str(MAX_RANK) + '+']
_ranks_sums = [0 for _rank in _x]
for _rank in _cells_ranks:
if _rank < MAX_RANK:
_ranks_sums[_rank - 1] += 1
else:
_ranks_sums[-1] += 1
_y = np.array(_ranks_sums) / _n
_fig = go.Figure(data=go.Bar(x=_x, y=_y, marker={'color': '#ea8500'}),
layout={
'xaxis': {
'title': 'Correct match correlation rank',
'zeroline': False,
'tickmode': 'array',
'tickvals': _x,
'ticktext': _x_text
},
'yaxis': {
'title': 'Fraction',
'range': [0, 1.1],
'zeroline': False,
'tickmode': 'array',
'tickvals': [0, 0.5, 1]
}
})
save.to_html(_fig=_fig,
_path=os.path.join(paths.PLOTS, save.get_module_name()),
_filename='plot_real_' + str(_real_cells) + '_static_' +
str(_static) + '_dead_dead_' + str(_dead_dead) +
'_live_dead_' + str(_live_dead) + '_dead_' + str(_dead) +
'_live_' + str(_live) + '_bead_' + str(_bead) +
'_metastasis_' + str(_metastasis) + '_bleb_' + str(_bleb) +
str(_bleb_amount_um) + '_band_' + str(_band) + '_high_time_' +
str(_high_temporal_resolution) + '_y_' + str(_offset_y))
# correct match probability plot
_y = [_correct_match_probability] * (MAX_RANK - 1) + [
_correct_match_probability * (POTENTIAL_MATCHES - MAX_RANK + 1)
]
_fig = go.Figure(data=go.Bar(x=_x, y=_y, marker={'color': '#ea8500'}),
layout={
'xaxis': {
'title': 'Correct match correlation rank',
'zeroline': False,
'tickmode': 'array',
'tickvals': _x,
'ticktext': _x_text
},
'yaxis': {
'title': 'Fraction',
'range': [0, 1.1],
'zeroline': False,
'tickmode': 'array',
'tickvals': [0, 0.5, 1]
}
})
save.to_html(_fig=_fig,
_path=os.path.join(paths.PLOTS, save.get_module_name()),
_filename='plot_real_' + str(_real_cells) + '_static_' +
str(_static) + '_dead_dead_' + str(_dead_dead) +
'_live_dead_' + str(_live_dead) + '_dead_' + str(_dead) +
'_live_' + str(_live) + '_bead_' + str(_bead) +
'_metastasis_' + str(_metastasis) + '_bleb_' + str(_bleb) +
str(_bleb_amount_um) + '_band_' + str(_band) + '_high_time_' +
str(_high_temporal_resolution) + '_y_' + str(_offset_y) +
'_correct_match_prob')
def main(_band=True):
_experiments = all_experiments()
_experiments = filtering.by_categories(
_experiments=_experiments,
_is_single_cell=False,
_is_high_temporal_resolution=None,
_is_bleb=True,
_is_dead_dead=False,
_is_live_dead=False,
_is_bead=False,
_is_metastasis=False
)
_tuples = load.experiments_groups_as_tuples(_experiments)
_tuples = filtering.by_pair_distance_range(_tuples, _distance_range=PAIR_DISTANCE_RANGE)
_tuples = filtering.by_real_pairs(_tuples, _real_pairs=REAL_CELLS)
_tuples = filtering.by_fake_static_pairs(_tuples, _fake_static_pairs=STATIC)
_tuples = filtering.by_band(_tuples, _band=_band)
_tuples = filtering.by_bleb_from_start(_tuples, _from_start=False)
print('Total tuples:', len(_tuples))
_arguments = []
for _tuple in _tuples:
_experiment, _series_id, _group = _tuple
_latest_time_frame = compute.latest_time_frame_before_overlapping(_experiment, _series_id, _group, OFFSET_X)
for _cell_id in ['left_cell', 'right_cell']:
_arguments.append({
'experiment': _experiment,
'series_id': _series_id,
'group': _group,
'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER,
'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER,
'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER,
'offset_x': OFFSET_X,
'offset_y': OFFSET_Y,
'offset_z': OFFSET_Z,
'cell_id': _cell_id,
'direction': 'inside',
'time_points': _latest_time_frame
})
_windows_dictionary, _windows_to_compute = compute.windows(_arguments,
_keys=['experiment', 'series_id', 'group', 'cell_id'])
_fiber_densities = compute.fiber_densities(_windows_to_compute, _subtract_border=True)
_experiments_fiber_densities = {
_key: [_fiber_densities[_tuple] for _tuple in _windows_dictionary[_key]]
for _key in _windows_dictionary
}
_n_pairs = 0
_before_correlations = []
_after_correlations = []
for _tuple in tqdm(_tuples, desc='Experiments loop'):
_experiment, _series_id, _group = _tuple
_left_cell_fiber_densities = \
_experiments_fiber_densities[(_experiment, _series_id, _group, 'left_cell')]
_right_cell_fiber_densities = \
_experiments_fiber_densities[(_experiment, _series_id, _group, 'right_cell')]
_properties = load.group_properties(_experiment, _series_id, _group)
_left_cell_fiber_densities = compute.remove_blacklist(
_experiment, _series_id, _properties['cells_ids']['left_cell'], _left_cell_fiber_densities)
_right_cell_fiber_densities = compute.remove_blacklist(
_experiment, _series_id, _properties['cells_ids']['right_cell'], _right_cell_fiber_densities)
_before_left_cell_fiber_densities = \
_left_cell_fiber_densities[:AFTER_BLEB_INJECTION_FIRST_TIME_FRAME[_experiment]]
_before_right_cell_fiber_densities = \
_right_cell_fiber_densities[:AFTER_BLEB_INJECTION_FIRST_TIME_FRAME[_experiment]]
_after_left_cell_fiber_densities = \
_left_cell_fiber_densities[AFTER_BLEB_INJECTION_FIRST_TIME_FRAME[_experiment]:]
_after_right_cell_fiber_densities = \
_right_cell_fiber_densities[AFTER_BLEB_INJECTION_FIRST_TIME_FRAME[_experiment]:]
_before_left_cell_fiber_densities_filtered, _before_right_cell_fiber_densities_filtered = \
compute.longest_same_indices_shared_in_borders_sub_array(
_before_left_cell_fiber_densities, _before_right_cell_fiber_densities)
_after_left_cell_fiber_densities_filtered, _after_right_cell_fiber_densities_filtered = \
compute.longest_same_indices_shared_in_borders_sub_array(
_after_left_cell_fiber_densities, _after_right_cell_fiber_densities)
# ignore small arrays
_minimum_time_frame_for_correlation = compute.minimum_time_frames_for_correlation(_experiment)
if len(_before_left_cell_fiber_densities_filtered) < _minimum_time_frame_for_correlation or \
len(_after_left_cell_fiber_densities_filtered) < _minimum_time_frame_for_correlation:
continue
_n_pairs += 1
_before_correlations.append(compute_lib.correlation(
compute_lib.derivative(_before_left_cell_fiber_densities_filtered, _n=DERIVATIVE),
compute_lib.derivative(_before_right_cell_fiber_densities_filtered, _n=DERIVATIVE)
))
_after_correlations.append(compute_lib.correlation(
compute_lib.derivative(_after_left_cell_fiber_densities_filtered, _n=DERIVATIVE),
compute_lib.derivative(_after_right_cell_fiber_densities_filtered, _n=DERIVATIVE)
))
print('Total pairs:', _n_pairs)
_before_minus_after = np.array(_before_correlations) - np.array(_after_correlations)
print('Wilcoxon of before minus after around the zero:')
print(wilcoxon(_before_minus_after))
print('Higher before amount:', (_before_minus_after > 0).sum() / len(_before_minus_after))
# plot
_fig = go.Figure(
data=go.Scatter(
x=_before_correlations,
y=_after_correlations,
mode='markers',
marker={
'size': 5,
'color': '#ea8500'
},
showlegend=False
),
layout={
'xaxis': {
'title': 'Correlation before bleb',
'zeroline': False,
'range': [-1.1, 1.2],
'tickmode': 'array',
'tickvals': [-1, -0.5, 0, 0.5, 1]
},
'yaxis': {
'title': 'Correlation after bleb',
'zeroline': False,
'range': [-1.1, 1.2],
'tickmode': 'array',
'tickvals': [-1, -0.5, 0, 0.5, 1]
},
'shapes': [
{
'type': 'line',
'x0': -1,
'y0': -1,
'x1': -1,
'y1': 1,
'line': {
'color': 'black',
'width': 2
}
},
{
'type': 'line',
'x0': -1,
'y0': -1,
'x1': 1,
'y1': -1,
'line': {
'color': 'black',
'width': 2
}
},
{
'type': 'line',
'x0': -1,
'y0': -1,
'x1': 1,
'y1': 1,
'line': {
'color': 'red',
'width': 2
}
}
]
}
)
save.to_html(
_fig=_fig,
_path=os.path.join(paths.PLOTS, save.get_module_name()),
_filename='plot'
)
def compute_z_array(_band=True,
_high_temporal_resolution=False,
_offset_x=OFFSET_X,
_offset_y_start=OFFSET_Y_START,
_offset_y_end=OFFSET_Y_END,
_offset_y_step=OFFSET_Y_STEP,
_offset_z_start=OFFSET_Z_START,
_offset_z_end=OFFSET_Z_END,
_offset_z_step=OFFSET_Z_STEP):
global _tuples, _experiments_fiber_densities, _z_array
_experiments = all_experiments()
_experiments = filtering.by_categories(
_experiments=_experiments,
_is_single_cell=False,
_is_high_temporal_resolution=_high_temporal_resolution,
_is_bleb=False,
_is_dead_dead=False,
_is_live_dead=False,
_is_bead=False,
_is_metastasis=False)
_tuples = load.experiments_groups_as_tuples(_experiments)
_tuples = filtering.by_time_frames_amount(
_tuples, compute.minimum_time_frames_for_correlation(_experiments[0]))
_tuples = filtering.by_pair_distance_range(_tuples, PAIR_DISTANCE_RANGE)
_tuples = filtering.by_real_pairs(_tuples)
_tuples = filtering.by_band(_tuples, _band=_band)
print('Total tuples:', len(_tuples))
_offsets_y = np.arange(start=_offset_y_start,
stop=_offset_y_end + _offset_y_step,
step=_offset_y_step)
_offsets_z = np.arange(start=_offset_z_start,
stop=_offset_z_end + _offset_z_step,
step=_offset_z_step)
_arguments = []
for _tuple in _tuples:
_experiment, _series_id, _group = _tuple
_latest_time_frame = compute.latest_time_frame_before_overlapping(
_experiment, _series_id, _group, OFFSET_X)
for _offset_y, _offset_z, _cell_id in product(
_offsets_y, _offsets_z, ['left_cell', 'right_cell']):
_arguments.append({
'experiment': _experiment,
'series_id': _series_id,
'group': _group,
'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER,
'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER,
'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER,
'offset_x': OFFSET_X,
'offset_y': _offset_y,
'offset_z': _offset_z,
'cell_id': _cell_id,
'direction': 'inside',
'time_points': _latest_time_frame
})
_windows_dictionary, _windows_to_compute = \
compute.windows(_arguments, _keys=['experiment', 'series_id', 'group', 'offset_y', 'offset_z', 'cell_id'])
_fiber_densities = compute.fiber_densities(_windows_to_compute)
_experiments_fiber_densities = {
_key:
[_fiber_densities[_tuple] for _tuple in _windows_dictionary[_key]]
for _key in _windows_dictionary
}
# clean
_fiber_densities = None
_windows_dictionary = None
_windows_to_compute = None
_arguments = []
for (_offset_y_index, _offset_y), (_offset_z_index, _offset_z) in \
product(enumerate(_offsets_y), enumerate(_offsets_z)):
_arguments.append(
(_offset_y_index, _offset_y, _offset_z_index, _offset_z))
_z_array = np.zeros(shape=(len(_offsets_y), len(_offsets_z)))
with Pool(CPUS_TO_USE) as _p:
for _answer in tqdm(_p.imap_unordered(compute_data, _arguments),
total=len(_arguments),
desc='Computing heatmap'):
_offset_y_index, _offset_z_index, _mean = _answer
_z_array[_offset_y_index, _offset_z_index] = _mean
_p.close()
_p.join()
return _z_array
def main(_high_temporal_resolution=True):
_experiments = all_experiments()
_experiments = filtering.by_categories(
_experiments=_experiments,
_is_single_cell=False,
_is_high_temporal_resolution=_high_temporal_resolution,
_is_bleb=False,
_is_dead_dead=False,
_is_live_dead=False,
_is_bead=False,
_is_metastasis=False)
_tuples = load.experiments_groups_as_tuples(_experiments)
_tuples = filtering.by_pair_distance_range(_tuples, PAIR_DISTANCE_RANGE)
_tuples = filtering.by_real_pairs(_tuples)
_tuples = filtering.by_band(_tuples)
print('Total tuples:', len(_tuples))
_arguments = []
for _tuple in _tuples:
_experiment, _series_id, _group = _tuple
for _cell_id in ['left_cell', 'right_cell']:
_latest_time_frame = compute.latest_time_frame_before_overlapping(
_experiment, _series_id, _group, OFFSET_X)
_arguments.append({
'experiment': _experiment,
'series_id': _series_id,
'group': _group,
'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER,
'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER,
'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER,
'offset_x': OFFSET_X,
'offset_y': OFFSET_Y,
'offset_z': OFFSET_Z,
'cell_id': _cell_id,
'direction': 'inside',
'time_points': _latest_time_frame
})
if ALIGNMENT_OFFSET_Y != OFFSET_Y:
_arguments.append({
'experiment': _experiment,
'series_id': _series_id,
'group': _group,
'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER,
'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER,
'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER,
'offset_x': OFFSET_X,
'offset_y': ALIGNMENT_OFFSET_Y,
'offset_z': OFFSET_Z,
'cell_id': _cell_id,
'direction': 'inside',
'time_points': _latest_time_frame
})
_windows_dictionary, _windows_to_compute = \
compute.windows(_arguments, _keys=['experiment', 'series_id', 'group', 'cell_id', 'offset_y'])
_fiber_densities = compute.fiber_densities(_windows_to_compute)
_experiments_fiber_densities = {
_key:
[_fiber_densities[_tuple] for _tuple in _windows_dictionary[_key]]
for _key in _windows_dictionary
}
_experiments_fiber_densities_aligned = align_by_z_score(
_tuples, _experiments_fiber_densities)
_tuples_by_experiment = organize.by_experiment(_tuples)
_same_correlations_array = []
_different_correlations_array = []
_valid_tuples = []
for _experiment in _tuples_by_experiment:
print('Experiment:', _experiment)
_experiment_tuples = _tuples_by_experiment[_experiment]
for _same_index in tqdm(range(len(_experiment_tuples)),
desc='Main loop'):
_same_tuple = _experiment_tuples[_same_index]
_same_experiment, _same_series, _same_group = _same_tuple
_same_left_cell_fiber_densities = \
_experiments_fiber_densities_aligned[
(_same_experiment, _same_series, _same_group, 'left_cell')
]
_same_right_cell_fiber_densities = \
_experiments_fiber_densities_aligned[
(_same_experiment, _same_series, _same_group, 'right_cell')
]
_same_properties = \
load.group_properties(_same_experiment, _same_series, _same_group)
_same_left_cell_fiber_densities = compute.remove_blacklist(
_same_experiment, _same_series,
_same_properties['cells_ids']['left_cell'],
_same_left_cell_fiber_densities)
_same_right_cell_fiber_densities = compute.remove_blacklist(
_same_experiment, _same_series,
_same_properties['cells_ids']['right_cell'],
_same_right_cell_fiber_densities)
_same_left_cell_fiber_densities_filtered, _same_right_cell_fiber_densities_filtered = \
compute.longest_same_indices_shared_in_borders_sub_array(
_same_left_cell_fiber_densities, _same_right_cell_fiber_densities
)
# ignore small arrays
if len(_same_left_cell_fiber_densities_filtered
) < compute.minimum_time_frames_for_correlation(
_same_experiment):
continue
_same_correlation = compute_lib.correlation(
compute_lib.derivative(
_same_left_cell_fiber_densities_filtered, _n=DERIVATIVE),
compute_lib.derivative(
_same_right_cell_fiber_densities_filtered, _n=DERIVATIVE))
for _different_index in range(len(_experiment_tuples)):
if _same_index != _different_index:
_different_tuple = _experiment_tuples[_different_index]
_different_experiment, _different_series, _different_group = \
_different_tuple
for _same_cell_id, _different_cell_id in product(
['left_cell', 'right_cell'],
['left_cell', 'right_cell']):
_same_fiber_densities = _experiments_fiber_densities_aligned[
(_same_experiment, _same_series, _same_group,
_same_cell_id)]
_different_fiber_densities = _experiments_fiber_densities_aligned[
(_different_experiment, _different_series,
_different_group, _different_cell_id)]
_different_properties = load.group_properties(
_different_experiment, _different_series,
_different_group)
_same_fiber_densities = compute.remove_blacklist(
_same_experiment, _same_series,
_same_properties['cells_ids'][_same_cell_id],
_same_fiber_densities)
_different_fiber_densities = compute.remove_blacklist(
_different_experiment, _different_series,
_different_properties['cells_ids']
[_different_cell_id], _different_fiber_densities)
_same_fiber_densities_filtered, _different_fiber_densities_filtered = \
compute.longest_same_indices_shared_in_borders_sub_array(
_same_fiber_densities, _different_fiber_densities
)
# ignore small arrays
if len(_same_fiber_densities_filtered
) < compute.minimum_time_frames_for_correlation(
_different_experiment):
continue
_different_correlation = compute_lib.correlation(
compute_lib.derivative(
_same_fiber_densities_filtered, _n=DERIVATIVE),
compute_lib.derivative(
_different_fiber_densities_filtered,
_n=DERIVATIVE))
_same_correlations_array.append(_same_correlation)
_different_correlations_array.append(
_different_correlation)
if _same_tuple not in _valid_tuples:
_valid_tuples.append(_same_tuple)
print('Total tuples:', len(_valid_tuples))
print('Total points:', len(_same_correlations_array))
_same_minus_different = \
np.array(_same_correlations_array) - np.array(_different_correlations_array)
print('Wilcoxon of same minus different around the zero:')
print(wilcoxon(_same_minus_different))
print('Higher same amount:',
(_same_minus_different > 0).sum() / len(_same_minus_different))
# plot
_fig = go.Figure(data=go.Scatter(x=_same_correlations_array,
y=_different_correlations_array,
mode='markers',
marker={
'size': 5,
'color': '#ea8500'
},
showlegend=False),
layout={
'xaxis': {
'title': 'Same network correlation',
'zeroline': False,
'range': [-1.1, 1.2],
'tickmode': 'array',
'tickvals': [-1, -0.5, 0, 0.5, 1]
},
'yaxis': {
'title': 'Different network correlation',
'zeroline': False,
'range': [-1.1, 1.2],
'tickmode': 'array',
'tickvals': [-1, -0.5, 0, 0.5, 1]
},
'shapes': [{
'type': 'line',
'x0': -1,
'y0': -1,
'x1': -1,
'y1': 1,
'line': {
'color': 'black',
'width': 2
}
}, {
'type': 'line',
'x0': -1,
'y0': -1,
'x1': 1,
'y1': -1,
'line': {
'color': 'black',
'width': 2
}
}, {
'type': 'line',
'x0': -1,
'y0': -1,
'x1': 1,
'y1': 1,
'line': {
'color': 'red',
'width': 2
}
}]
})
save.to_html(_fig=_fig,
_path=os.path.join(paths.PLOTS, save.get_module_name()),
_filename='plot_high_time_' + str(_high_temporal_resolution))
def main(_high_temporal_resolution=True, _offset_y=0.5):
_experiments = all_experiments()
_experiments = filtering.by_categories(
_experiments=_experiments,
_is_single_cell=False,
_is_high_temporal_resolution=_high_temporal_resolution,
_is_bleb=False,
_is_dead_dead=False,
_is_live_dead=False,
_is_bead=False,
_is_metastasis=False)
_tuples = load.experiments_groups_as_tuples(_experiments)
_tuples = filtering.by_real_pairs(_tuples)
# _tuples = filtering.by_band(_tuples)
_tuples = filtering.by_pair_distance_range(_tuples, PAIR_DISTANCE_RANGE)
print('Total tuples:', len(_tuples))
_arguments = []
for _tuple in _tuples:
_experiment, _series_id, _group = _tuple
_latest_time_frame = compute.latest_time_frame_before_overlapping(
_experiment, _series_id, _group, OFFSET_X)
for _cell_id, _direction in product(['left_cell', 'right_cell'],
['inside', 'outside']):
_arguments.append({
'experiment': _experiment,
'series_id': _series_id,
'group': _group,
'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER,
'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER,
'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER,
'offset_x': OFFSET_X,
'offset_y': _offset_y,
'offset_z': OFFSET_Z,
'cell_id': _cell_id,
'direction': _direction,
'time_points': _latest_time_frame
})
_windows_dictionary, _windows_to_compute = \
compute.windows(_arguments, _keys=['experiment', 'series_id', 'group', 'cell_id', 'direction'])
_fiber_densities = compute.fiber_densities(_windows_to_compute,
_subtract_border=True)
_experiments_fiber_densities = {
_key:
[_fiber_densities[_tuple] for _tuple in _windows_dictionary[_key]]
for _key in _windows_dictionary
}
_inner_correlations = []
_outer_correlations = []
for _tuple in _tuples:
_experiment, _series_id, _group = _tuple
_left_inner_tuple = (_experiment, _series_id, _group, 'left_cell',
'inside')
_left_outer_tuple = (_experiment, _series_id, _group, 'left_cell',
'outside')
_right_inner_tuple = (_experiment, _series_id, _group, 'right_cell',
'inside')
_right_outer_tuple = (_experiment, _series_id, _group, 'right_cell',
'outside')
if _left_inner_tuple not in _windows_dictionary or _left_outer_tuple not in _windows_dictionary or \
_right_inner_tuple not in _windows_dictionary or _right_outer_tuple not in _windows_dictionary:
continue
_properties = load.group_properties(_experiment, _series_id, _group)
_left_inner_fiber_densities = _experiments_fiber_densities[
_left_inner_tuple]
_left_outer_fiber_densities = _experiments_fiber_densities[
_left_outer_tuple]
_right_inner_fiber_densities = _experiments_fiber_densities[
_right_inner_tuple]
_right_outer_fiber_densities = _experiments_fiber_densities[
_right_outer_tuple]
_left_inner_fiber_densities = compute.remove_blacklist(
_experiment, _series_id, _properties['cells_ids']['left_cell'],
_left_inner_fiber_densities)
_left_outer_fiber_densities = compute.remove_blacklist(
_experiment, _series_id, _properties['cells_ids']['left_cell'],
_left_outer_fiber_densities)
_right_inner_fiber_densities = compute.remove_blacklist(
_experiment, _series_id, _properties['cells_ids']['right_cell'],
_right_inner_fiber_densities)
_right_outer_fiber_densities = compute.remove_blacklist(
_experiment, _series_id, _properties['cells_ids']['right_cell'],
_right_outer_fiber_densities)
_left_inner_fiber_densities, _right_inner_fiber_densities = \
compute.longest_same_indices_shared_in_borders_sub_array(
_left_inner_fiber_densities, _right_inner_fiber_densities)
_left_outer_fiber_densities, _right_outer_fiber_densities = \
compute.longest_same_indices_shared_in_borders_sub_array(
_left_outer_fiber_densities, _right_outer_fiber_densities)
# ignore small arrays
_minimum_time_frame_for_correlation = compute.minimum_time_frames_for_correlation(
_experiment)
if len(_left_inner_fiber_densities) < _minimum_time_frame_for_correlation or \
len(_left_outer_fiber_densities) < _minimum_time_frame_for_correlation:
continue
_inner_correlations.append(
compute_lib.correlation(
compute_lib.derivative(_left_inner_fiber_densities,
_n=DERIVATIVE),
compute_lib.derivative(_right_inner_fiber_densities,
_n=DERIVATIVE)))
_outer_correlations.append(
compute_lib.correlation(
compute_lib.derivative(_left_outer_fiber_densities,
_n=DERIVATIVE),
compute_lib.derivative(_right_outer_fiber_densities,
_n=DERIVATIVE)))
print('Total pairs:', len(_inner_correlations))
print(
'Pearson correlation:',
compute_lib.correlation(_inner_correlations,
_outer_correlations,
_with_p_value=True))
print('Wilcoxon of inner around the zero:', wilcoxon(_inner_correlations))
print('Wilcoxon of outer around the zero:', wilcoxon(_outer_correlations))
_inner_minus_outer = np.array(_inner_correlations) - np.array(
_outer_correlations)
print('Wilcoxon of inner minus outer:', wilcoxon(_inner_minus_outer))
print('Higher inner amount:',
(_inner_minus_outer > 0).sum() / len(_inner_minus_outer))
# plot
_fig = go.Figure(data=go.Scatter(x=_inner_correlations,
y=_outer_correlations,
mode='markers',
marker={
'size': 5,
'color': '#ea8500'
},
showlegend=False),
layout={
'xaxis': {
'title': 'Inner correlation',
'zeroline': False,
'range': [-1.1, 1.2],
'tickmode': 'array',
'tickvals': [-1, -0.5, 0, 0.5, 1]
},
'yaxis': {
'title': 'Outer correlation',
'zeroline': False,
'range': [-1.1, 1.2],
'tickmode': 'array',
'tickvals': [-1, -0.5, 0, 0.5, 1]
},
'shapes': [{
'type': 'line',
'x0': -1,
'y0': -1,
'x1': -1,
'y1': 1,
'line': {
'color': 'black',
'width': 2
}
}, {
'type': 'line',
'x0': -1,
'y0': -1,
'x1': 1,
'y1': -1,
'line': {
'color': 'black',
'width': 2
}
}, {
'type': 'line',
'x0': -1,
'y0': -1,
'x1': 1,
'y1': 1,
'line': {
'color': 'red',
'width': 2
}
}]
})
save.to_html(_fig=_fig,
_path=os.path.join(paths.PLOTS, save.get_module_name()),
_filename='plot_high_' + str(_high_temporal_resolution) +
'_offset_y_' + str(_offset_y))
def compute_fiber_densities(_real_cells=True, _static=False, _dead_dead=False, _live_dead=False, _dead=False,
_live=False, _bead=False, _metastasis=False, _bleb=False, _bleb_amount_um=None, _band=True,
_high_temporal_resolution=False, _pair_distance_range=None, _offset_y=0.5, _offset_z=0,
_padding_y_by=0.5, _padding_z_by=0.0, _space_y_by=0.25, _space_z_by=0.0):
if _pair_distance_range is None:
_pair_distance_range = PAIR_DISTANCE_RANGE
_experiments = all_experiments()
_experiments = filtering.by_categories(
_experiments=_experiments,
_is_single_cell=False,
_is_high_temporal_resolution=_high_temporal_resolution,
_is_bleb=_bleb,
_is_dead_dead=_dead_dead,
_is_live_dead=_live_dead,
_is_bead=_bead,
_is_metastasis=_metastasis
)
_tuples = load.experiments_groups_as_tuples(_experiments)
_tuples = filtering.by_pair_distance_range(_tuples, _pair_distance_range)
_tuples = filtering.by_real_pairs(_tuples, _real_pairs=_real_cells)
_tuples = filtering.by_fake_static_pairs(_tuples, _fake_static_pairs=_static)
if _dead_dead is not False or _live_dead is not False:
_tuples = filtering.by_dead_live(_tuples, _dead=_dead, _live=_live)
_tuples = filtering.by_band(_tuples, _band=_band)
if _bleb:
_tuples = filtering.by_bleb_amount_um(_tuples, _amount_um=_bleb_amount_um)
print('Total tuples:', len(_tuples))
_arguments = []
for _tuple in _tuples:
_experiment, _series_id, _group = _tuple
_latest_time_frame = compute.latest_time_frame_before_overlapping(_experiment, _series_id, _group, OFFSET_X)
for _cell_id in ['left_cell', 'right_cell']:
_arguments.append({
'experiment': _experiment,
'series_id': _series_id,
'group': _group,
'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER,
'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER,
'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER,
'offset_x': OFFSET_X,
'offset_y': _offset_y,
'offset_z': _offset_z,
'cell_id': _cell_id,
'direction': 'inside',
'time_points': _latest_time_frame
})
_windows_dictionary, _windows_to_compute = compute.windows(_arguments,
_keys=['experiment', 'series_id', 'group', 'cell_id'])
_fiber_densities = compute.fiber_densities(_windows_to_compute, _subtract_border=True, _padding_y_by=_padding_y_by,
_padding_z_by=_padding_z_by, _space_y_by=_space_y_by,
_space_z_by=_space_z_by)
_experiments_fiber_densities = {
_key: [_fiber_densities[_tuple] for _tuple in _windows_dictionary[_key]]
for _key in _windows_dictionary
}
_tuples_by_experiment = organize.by_experiment(_tuples)
_same_correlations_array = []
_different_correlations_array = []
_valid_tuples = []
for _experiment in _tuples_by_experiment:
print('Experiment:', _experiment)
_experiment_tuples = _tuples_by_experiment[_experiment]
for _same_index in tqdm(range(len(_experiment_tuples)), desc='Main loop'):
_same_tuple = _experiment_tuples[_same_index]
_same_experiment, _same_series, _same_group = _same_tuple
_same_left_cell_fiber_densities = \
_experiments_fiber_densities[
(_same_experiment, _same_series, _same_group, 'left_cell')
]
_same_right_cell_fiber_densities = \
_experiments_fiber_densities[
(_same_experiment, _same_series, _same_group, 'right_cell')
]
_same_properties = \
load.group_properties(_same_experiment, _same_series, _same_group)
_same_left_cell_fiber_densities = compute.remove_blacklist(
_same_experiment,
_same_series,
_same_properties['cells_ids']['left_cell'],
_same_left_cell_fiber_densities
)
_same_right_cell_fiber_densities = compute.remove_blacklist(
_same_experiment,
_same_series,
_same_properties['cells_ids']['right_cell'],
_same_right_cell_fiber_densities
)
_same_left_cell_fiber_densities_filtered, _same_right_cell_fiber_densities_filtered = \
compute.longest_same_indices_shared_in_borders_sub_array(
_same_left_cell_fiber_densities, _same_right_cell_fiber_densities
)
# ignore small arrays
if len(_same_left_cell_fiber_densities_filtered) < compute.minimum_time_frames_for_correlation(_same_experiment):
continue
_same_correlation = compute_lib.correlation(
compute_lib.derivative(_same_left_cell_fiber_densities_filtered, _n=DERIVATIVE),
compute_lib.derivative(_same_right_cell_fiber_densities_filtered, _n=DERIVATIVE)
)
for _different_index in range(len(_experiment_tuples)):
if _same_index != _different_index:
_different_tuple = _experiment_tuples[_different_index]
_different_experiment, _different_series, _different_group = \
_different_tuple
for _same_cell_id, _different_cell_id in product(['left_cell', 'right_cell'],
['left_cell', 'right_cell']):
_same_fiber_densities = _experiments_fiber_densities[(
_same_experiment,
_same_series,
_same_group,
_same_cell_id
)]
_different_fiber_densities = _experiments_fiber_densities[(
_different_experiment,
_different_series,
_different_group,
_different_cell_id
)]
_different_properties = load.group_properties(
_different_experiment, _different_series, _different_group
)
_same_fiber_densities = compute.remove_blacklist(
_same_experiment,
_same_series,
_same_properties['cells_ids'][_same_cell_id],
_same_fiber_densities
)
_different_fiber_densities = compute.remove_blacklist(
_different_experiment,
_different_series,
_different_properties['cells_ids'][_different_cell_id],
_different_fiber_densities
)
_same_fiber_densities_filtered, _different_fiber_densities_filtered = \
compute.longest_same_indices_shared_in_borders_sub_array(
_same_fiber_densities, _different_fiber_densities
)
# ignore small arrays
if len(_same_fiber_densities_filtered) < compute.minimum_time_frames_for_correlation(_different_experiment):
continue
_different_correlation = compute_lib.correlation(
compute_lib.derivative(_same_fiber_densities_filtered, _n=DERIVATIVE),
compute_lib.derivative(_different_fiber_densities_filtered, _n=DERIVATIVE)
)
_same_correlations_array.append(_same_correlation)
_different_correlations_array.append(_different_correlation)
if _same_tuple not in _valid_tuples:
_valid_tuples.append(_same_tuple)
print('Offset Y:', _offset_y, 'Offset Z:', _offset_z)
print('Total tuples:', len(_valid_tuples))
print('Total points:', len(_same_correlations_array))
_same_minus_different = \
np.array(_same_correlations_array) - np.array(_different_correlations_array)
print('Wilcoxon of same minus different around the zero:')
print(wilcoxon(_same_minus_different))
print('Higher same amount:', (_same_minus_different > 0).sum() /
len(_same_minus_different))
return _same_correlations_array, _different_correlations_array
