def align_by_z_score(_experiments, _experiments_fiber_densities):
_experiments_fiber_densities_aligned = {}
for _tuple in tqdm(_experiments, desc='Temporal aligning experiments'):
_experiment, _series_id, _group = _tuple
_left_cell_alignment_fiber_densities = \
_experiments_fiber_densities[(_experiment, _series_id, _group, 'left_cell', ALIGNMENT_OFFSET_Y)]
_right_cell_alignment_fiber_densities = \
_experiments_fiber_densities[(_experiment, _series_id, _group, 'right_cell', ALIGNMENT_OFFSET_Y)]
_left_cell_fiber_densities = \
_experiments_fiber_densities[(_experiment, _series_id, _group, 'left_cell', OFFSET_Y)]
_right_cell_fiber_densities = \
_experiments_fiber_densities[(_experiment, _series_id, _group, 'right_cell', OFFSET_Y)]
_normalization = load.normalization_series_file_data(
_experiment, _series_id)
_left_cell_value_aligned, _right_cell_value_aligned = None, None
for _time_frame, (_left_cell_fiber_density, _right_cell_fiber_density) in \
enumerate(zip(_left_cell_alignment_fiber_densities, _right_cell_alignment_fiber_densities)):
_normalized_left_cell_fiber_density = compute_lib.z_score(
_x=_left_cell_fiber_density[0],
_average=_normalization['average'],
_std=_normalization['std'])
_normalized_right_cell_fiber_density = compute_lib.z_score(
_x=_right_cell_fiber_density[0],
_average=_normalization['average'],
_std=_normalization['std'])
_mean_z_score = (_normalized_left_cell_fiber_density +
_normalized_right_cell_fiber_density) / 2
if _mean_z_score > START_TIME_FRAME_Z_SCORE:
if _time_frame == 0 and BLOCK_ALREADY_DENSE:
_left_cell_value_aligned = []
_right_cell_value_aligned = []
else:
_left_cell_value_aligned = _left_cell_fiber_densities[
_time_frame:]
_right_cell_value_aligned = _right_cell_fiber_densities[
_time_frame:]
break
# in case z-score not reached
if _left_cell_value_aligned is None:
_left_cell_value_aligned = []
_right_cell_value_aligned = []
_experiments_fiber_densities_aligned[(_experiment, _series_id, _group, 'left_cell')] = \
_left_cell_value_aligned
_experiments_fiber_densities_aligned[(_experiment, _series_id, _group, 'right_cell')] = \
_right_cell_value_aligned
return _experiments_fiber_densities_aligned
def compute_data():
_simulations = load.structured()
_simulations = filtering.by_time_points_amount(_simulations,
_time_points=TIME_POINTS)
_simulations = filtering.by_categories(_simulations,
_is_single_cell=False,
_is_heterogeneity=False,
_is_low_connectivity=False,
_is_causality=False,
_is_dominant_passive=False,
_is_fibrin=False)
_simulations = filtering.by_pair_distance(_simulations,
_distance=PAIR_DISTANCE)
print('Total simulations:', len(_simulations))
_fiber_densities = compute_fiber_densities(_simulations)
_simulations_fiber_densities = [[] for _i in range(TIME_POINTS)]
for _simulation in tqdm(_simulations, desc='Main loop'):
_normalization = load.normalization(_simulation)
for _time_point in range(TIME_POINTS):
for _cell_id in ['left_cell', 'right_cell']:
_fiber_density = _fiber_densities[(_simulation,
_cell_id)][_time_point]
_normalized_fiber_density = compute_lib.z_score(
_fiber_density, _normalization['average'],
_normalization['std'])
_simulations_fiber_densities[_time_point].append(
_normalized_fiber_density)
print('Total pairs:', len(_simulations_fiber_densities[0]))
return _simulations_fiber_densities
def compute_data(_arguments):
_offset_y_index, _offset_y, _offset_z_index, _offset_z = _arguments
_fiber_densities_array = []
for _tuple in _tuples:
_experiment, _series_id, _group = _tuple
_normalization = load.normalization_series_file_data(
_experiment, _series_id)
for _cell_id in ['left_cell', 'right_cell']:
_fiber_density = \
_experiments_fiber_densities[(_experiment, _series_id, _group, _offset_y, _offset_z, _cell_id)]
# remove out of boundaries
if _fiber_density[1]:
continue
_normalized_fiber_density = compute_lib.z_score(
_x=_fiber_density[0],
_average=_normalization['average'],
_std=_normalization['std'])
_fiber_densities_array.append(_normalized_fiber_density)
if len(_fiber_densities_array) > 0:
return _offset_y_index, _offset_z_index, np.mean(
_fiber_densities_array)
else:
return _offset_y_index, _offset_z_index, None
def compute_data(_arguments):
_offset_y_index, _offset_y, _offset_z_index, _offset_z = _arguments
_fiber_density_changes_array = []
for _tuple in _tuples:
_experiment, _series_id, _group = _tuple
_normalization = load.normalization_series_file_data(
_experiment, _series_id)
_properties = load.group_properties(_experiment, _series_id, _group)
for _cell_id in ['left_cell', 'right_cell']:
_cell_fiber_densities = \
_experiments_fiber_densities[(_experiment, _series_id, _group, _offset_y, _offset_z, _cell_id)]
_cell_fiber_densities = compute.remove_blacklist(
_experiment, _series_id, _properties['cells_ids'][_cell_id],
_cell_fiber_densities)
_previous_cell_fiber_density_normalized = None
_cell_values = []
for _time_frame, _cell_fiber_density in enumerate(
_cell_fiber_densities):
# not out of border
if _cell_fiber_density[1]:
_previous_cell_fiber_density_normalized = None
continue
# normalize
_cell_fiber_density_normalized = compute_lib.z_score(
_x=_cell_fiber_density[0],
_average=_normalization['average'],
_std=_normalization['std'])
# no previous
if _previous_cell_fiber_density_normalized is None:
_previous_cell_fiber_density_normalized = _cell_fiber_density_normalized
continue
# change
_cell_fiber_density_normalized_change = abs(
_cell_fiber_density_normalized -
_previous_cell_fiber_density_normalized)
_previous_cell_fiber_density_normalized = _cell_fiber_density_normalized
# save
_cell_values.append(_cell_fiber_density_normalized_change)
# save
if len(_cell_values) > 0:
_fiber_density_changes_array.append(np.mean(_cell_values))
if len(_fiber_density_changes_array) > 0:
return _offset_y_index, _offset_z_index, np.mean(
_fiber_density_changes_array)
else:
return _offset_y_index, _offset_z_index, None
def compute_cell_pairs(_low_connectivity):
_x_array = []
_y_array = []
_names_array = []
_max_offsets_x = []
for _distance in PAIR_DISTANCE:
print('Pair distance ' + str(_distance))
_simulations = load.structured()
_simulations = filtering.by_categories(
_simulations,
_is_single_cell=False,
_is_heterogeneity=False,
_is_low_connectivity=_low_connectivity,
_is_causality=False,
_is_dominant_passive=False,
_is_fibrin=False)
_simulations = filtering.by_pair_distance(_simulations,
_distance=_distance)
_simulations = filtering.by_time_points_amount(
_simulations, _time_points=TIME_POINT[_low_connectivity])
_offsets_x = np.arange(start=0,
stop=_distance / 2 - 0.25 -
QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER,
step=OFFSET_X_STEP)
if len(_offsets_x) > len(_max_offsets_x):
_max_offsets_x = _offsets_x
_fiber_densities = compute_fiber_densities(_simulations, _offsets_x,
_low_connectivity)
_pair_distance_fiber_densities = [[] for _i in range(len(_offsets_x))]
for _simulation in _simulations:
_offset_index = 0
_normalization = load.normalization(_simulation)
for _offset_x in _offsets_x:
for _cell_id in ['left_cell', 'right_cell']:
_fiber_density = _fiber_densities[(_simulation, _offset_x,
_cell_id)]
_normalized_fiber_density = compute_lib.z_score(
_fiber_density, _normalization['average'],
_normalization['std'])
_pair_distance_fiber_densities[_offset_index].append(
_normalized_fiber_density)
_offset_index += 1
_x_array.append(_offsets_x)
_y_array.append(_pair_distance_fiber_densities)
_names_array.append('Pair distance ' + str(_distance))
return _names_array, _x_array, _y_array
def compute_data(_arguments):
_offset_y_index, _offset_y = _arguments
_fiber_densities_array = []
for _simulation in _simulations:
_simulation_normalization = load.normalization(_simulation)
for _cell_id in ['left_cell', 'right_cell']:
_fiber_density = _fiber_densities[(_simulation, _offset_y, _cell_id)]
_normalized_fiber_density = compute_lib.z_score(
_x=_fiber_density,
_average=_simulation_normalization['average'],
_std=_simulation_normalization['std']
)
_fiber_densities_array.append(_normalized_fiber_density)
if len(_fiber_densities_array) > 0:
return _offset_y_index, np.mean(_fiber_densities_array)
else:
return _offset_y_index, None
def compute_simulations_data(_low_connectivity):
_simulations = simulations_load.structured()
_simulations = simulations_filtering.by_time_points_amount(
_simulations, SIMULATIONS_TIME_POINT[_low_connectivity])
_simulations = simulations_filtering.by_categories(
_simulations,
_is_single_cell=False,
_is_heterogeneity=False,
_is_low_connectivity=_low_connectivity,
_is_causality=False,
_is_dominant_passive=False,
_is_fibrin=False)
_simulations = simulations_filtering.by_pair_distance(
_simulations, _distance=PAIR_DISTANCE)
print('Total simulations:', len(_simulations))
_fiber_densities = compute_simulations_fiber_densities(
_simulations, _low_connectivity)
_simulations_fiber_densities = [[]
for _i in range(len(SIMULATIONS_OFFSETS_X))
]
for _simulation in tqdm(_simulations, desc='Simulations Loop'):
_normalization = simulations_load.normalization(_simulation)
for _offset_x_index, _offset_x in enumerate(SIMULATIONS_OFFSETS_X):
for _cell_id in ['left_cell', 'right_cell']:
_fiber_density = _fiber_densities[(_simulation, _offset_x,
_cell_id)]
_normalized_fiber_density = compute_lib.z_score(
_fiber_density, _normalization['average'],
_normalization['std'])
_simulations_fiber_densities[_offset_x_index].append(
_normalized_fiber_density)
return _simulations_fiber_densities
def compute_simulations_data(_low_connectivity):
_simulations = simulations_load.structured()
_simulations = simulations_filtering.by_time_points_amount(
_simulations, _time_points=SIMULATIONS_TIME_POINT[_low_connectivity]
)
_simulations = simulations_filtering.by_categories(
_simulations,
_is_single_cell=True,
_is_heterogeneity=False,
_is_low_connectivity=_low_connectivity,
_is_causality=False,
_is_dominant_passive=False,
_is_fibrin=False
)
_fiber_densities = compute_simulations_fiber_densities(_simulations, _low_connectivity)
_simulations_fiber_densities = [[] for _i in range(len(OFFSETS_X))]
for _simulation in tqdm(_simulations, desc='Simulations Loop'):
_normalization = simulations_load.normalization(_simulation)
for _offset_x_index, _offset_x in enumerate(OFFSETS_X):
_direction_fiber_densities = []
for _direction in ['left', 'right', 'up', 'down']:
_fiber_density = _fiber_densities[(_simulation, _offset_x, _direction)]
_normalized_fiber_density = compute_lib.z_score(
_fiber_density,
_normalization['average'],
_normalization['std']
)
_direction_fiber_densities.append(_normalized_fiber_density)
_simulations_fiber_densities[_offset_x_index].append(np.mean(_direction_fiber_densities))
return _simulations_fiber_densities
def compute_experiments_data():
_experiments = all_experiments()
_experiments = experiments_filtering.by_categories(
_experiments=_experiments,
_is_single_cell=True,
_is_high_temporal_resolution=False,
_is_bleb=False,
_is_dead_dead=False,
_is_live_dead=False,
_is_bead=False,
_is_metastasis=False
)
_tuples = experiments_load.experiments_groups_as_tuples(_experiments)
_tuples = experiments_filtering.by_time_frames_amount(_tuples, EXPERIMENTS_TIME_FRAME)
_tuples = experiments_filtering.by_main_cell(_tuples)
_arguments = []
for _tuple in _tuples:
_experiment, _series_id, _group = _tuple
for _offset_x, _direction in product(OFFSETS_X, ['left', 'right']):
_arguments.append({
'experiment': _experiment,
'series_id': _series_id,
'group': _group,
'length_x': experiments_config.QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER,
'length_y': experiments_config.QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER,
'length_z': experiments_config.QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER,
'offset_x': _offset_x,
'offset_y': OFFSET_Y,
'offset_z': OFFSET_Z,
'cell_id': 'cell',
'direction': _direction,
'time_point': EXPERIMENTS_TIME_FRAME - 1
})
_windows_dictionary, _windows_to_compute = \
experiments_compute.windows(_arguments, _keys=['experiment', 'series_id', 'group', 'offset_x', 'direction'])
_fiber_densities = experiments_compute.fiber_densities(_windows_to_compute)
_tuples = experiments_organize.by_single_cell_id(_tuples)
_experiments_fiber_densities = [[] for _i in range(len(OFFSETS_X))]
for _tuple in tqdm(_tuples, desc='Experiments loop'):
_experiment, _series_id, _cell_id = _tuple
_normalization = experiments_load.normalization_series_file_data(_experiment, _series_id)
for _offset_x_index, _offset_x in enumerate(OFFSETS_X):
_cell_fiber_densities = []
for _cell_tuple in _tuples[_tuple]:
_, _, _group = _cell_tuple
for _direction in ['left', 'right']:
_window_tuple = _windows_dictionary[(_experiment, _series_id, _group, _offset_x, _direction)][0]
_fiber_density = _fiber_densities[_window_tuple]
if not OUT_OF_BOUNDARIES and _fiber_density[1]:
continue
_normalized_fiber_density = compute_lib.z_score(
_x=_fiber_density[0],
_average=_normalization['average'],
_std=_normalization['std']
)
if not np.isnan(_normalized_fiber_density):
_cell_fiber_densities.append(_normalized_fiber_density)
if len(_cell_fiber_densities) > 0:
_experiments_fiber_densities[_offset_x_index].append(np.mean(_cell_fiber_densities))
return _experiments_fiber_densities
def main(_real_cells=True, _static=False, _band=True, _high_temporal_resolution=False):
_experiments = all_experiments()
_experiments = filtering.by_categories(
_experiments=_experiments,
_is_single_cell=False,
_is_high_temporal_resolution=_high_temporal_resolution,
_is_bleb=False,
_is_dead_dead=False,
_is_live_dead=False,
_is_bead=False,
_is_metastasis=False
)
_tuples = load.experiments_groups_as_tuples(_experiments)
_tuples = filtering.by_pair_distance_range(_tuples, PAIR_DISTANCE_RANGE)
_tuples = filtering.by_real_pairs(_tuples, _real_pairs=_real_cells)
_tuples = filtering.by_fake_static_pairs(_tuples, _fake_static_pairs=_static)
_tuples = filtering.by_band(_tuples, _band=_band)
_tuples = filtering.by_time_frames_amount(_tuples, compute.minimum_time_frames_for_correlation(_experiments[0]))
print('Total tuples:', len(_tuples))
print('Density')
_arguments = []
for _tuple in _tuples:
_experiment, _series_id, _group = _tuple
_time_frame = compute.minimum_time_frames_for_correlation(_experiment)
for _cell_id in ['left_cell', 'right_cell']:
_arguments.append({
'experiment': _experiment,
'series_id': _series_id,
'group': _group,
'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER,
'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER,
'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER,
'offset_x': OFFSET_X,
'offset_y': OFFSET_Y_DENSITY,
'offset_z': OFFSET_Z,
'cell_id': _cell_id,
'direction': 'inside',
'time_point': _time_frame - 1
})
_windows_dictionary, _windows_to_compute = \
compute.windows(_arguments, _keys=['experiment', 'series_id', 'group', 'cell_id'])
_densities_fiber_densities = compute.fiber_densities(_windows_to_compute, _subtract_border=False)
_densities_experiments_fiber_densities = {
_key: [_densities_fiber_densities[_tuple] for _tuple in _windows_dictionary[_key]]
for _key in _windows_dictionary
}
print('Correlations')
_arguments = []
for _tuple in _tuples:
_experiment, _series_id, _group = _tuple
_latest_time_frame = compute.latest_time_frame_before_overlapping(_experiment, _series_id, _group, OFFSET_X)
for _cell_id in ['left_cell', 'right_cell']:
_arguments.append({
'experiment': _experiment,
'series_id': _series_id,
'group': _group,
'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER,
'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER,
'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER,
'offset_x': OFFSET_X,
'offset_y': OFFSET_Y_CORRELATION,
'offset_z': OFFSET_Z,
'cell_id': _cell_id,
'direction': 'inside',
'time_points': _latest_time_frame
})
_windows_dictionary, _windows_to_compute = \
compute.windows(_arguments, _keys=['experiment', 'series_id', 'group', 'cell_id'])
_correlations_fiber_densities = compute.fiber_densities(_windows_to_compute, _subtract_border=True)
_correlations_experiments_fiber_densities = {
_key: [_correlations_fiber_densities[_tuple] for _tuple in _windows_dictionary[_key]]
for _key in _windows_dictionary
}
_densities = []
_correlations = []
for _tuple in tqdm(_tuples, desc='Main loop'):
_experiment, _series_id, _group = _tuple
# density
_left_cell_fiber_density = \
_densities_experiments_fiber_densities[(_experiment, _series_id, _group, 'left_cell')][0]
_right_cell_fiber_density = \
_densities_experiments_fiber_densities[(_experiment, _series_id, _group, 'right_cell')][0]
# not relevant
if _left_cell_fiber_density[1] or _right_cell_fiber_density[1]:
continue
_normalization = load.normalization_series_file_data(_experiment, _series_id)
_left_cell_fiber_density_normalized = compute_lib.z_score(
_x=_left_cell_fiber_density[0],
_average=_normalization['average'],
_std=_normalization['std']
)
_right_cell_fiber_density_normalized = compute_lib.z_score(
_x=_right_cell_fiber_density[0],
_average=_normalization['average'],
_std=_normalization['std']
)
_density = (_left_cell_fiber_density_normalized + _right_cell_fiber_density_normalized) / 2
# correlation
_left_cell_fiber_densities = \
_correlations_experiments_fiber_densities[(_experiment, _series_id, _group, 'left_cell')]
_right_cell_fiber_densities = \
_correlations_experiments_fiber_densities[(_experiment, _series_id, _group, 'right_cell')]
_properties = load.group_properties(_experiment, _series_id, _group)
_left_cell_fiber_densities = compute.remove_blacklist(
_experiment, _series_id, _properties['cells_ids']['left_cell'], _left_cell_fiber_densities)
_right_cell_fiber_densities = compute.remove_blacklist(
_experiment, _series_id, _properties['cells_ids']['right_cell'], _right_cell_fiber_densities)
_left_cell_fiber_densities_filtered, _right_cell_fiber_densities_filtered = \
compute.longest_same_indices_shared_in_borders_sub_array(
_left_cell_fiber_densities, _right_cell_fiber_densities
)
# ignore small arrays
if len(_left_cell_fiber_densities_filtered) < compute.minimum_time_frames_for_correlation(_experiment):
continue
_correlation = compute_lib.correlation(
compute_lib.derivative(_left_cell_fiber_densities_filtered, _n=DERIVATIVE),
compute_lib.derivative(_right_cell_fiber_densities_filtered, _n=DERIVATIVE)
)
_densities.append(_density)
_correlations.append(_correlation)
print('Total tuples:', len(_densities))
print('Pearson correlation of densities and correlations:')
print(compute_lib.correlation(_densities, _correlations, _with_p_value=True))
# plot
_fig = go.Figure(
data=go.Scatter(
x=_densities,
y=_correlations,
mode='markers',
marker={
'size': 5,
'color': '#ea8500'
},
showlegend=False
),
layout={
'xaxis': {
'title': 'Fiber density (z-score)',
'zeroline': False,
'range': [-1.1, 15.2],
# 'tickmode': 'array',
# 'tickvals': [-1, -0.5, 0, 0.5, 1]
},
'yaxis': {
'title': 'Correlation',
'zeroline': False,
'range': [-1.1, 1.2],
'tickmode': 'array',
'tickvals': [-1, -0.5, 0, 0.5, 1]
},
'shapes': [
{
'type': 'line',
'x0': 0,
'y0': -1,
'x1': 0,
'y1': 1,
'line': {
'color': 'black',
'width': 2
}
},
{
'type': 'line',
'x0': 0,
'y0': -1,
'x1': 15,
'y1': -1,
'line': {
'color': 'black',
'width': 2
}
}
]
}
)
save.to_html(
_fig=_fig,
_path=os.path.join(paths.PLOTS, save.get_module_name()),
_filename='plot_real_' + str(_real_cells) + '_static_' + str(_static) + '_band_' + str(_band) +
'_high_time_' + str(_high_temporal_resolution) + '_y_density_' + str(OFFSET_Y_DENSITY) +
'_y_correlation_' + str(OFFSET_Y_CORRELATION)
)
def main(_high_temporal_resolution=False, _pair_distance_range=None):
if _pair_distance_range is None:
_pair_distance_range = PAIR_DISTANCE_RANGE
_experiments = all_experiments()
_experiments = filtering.by_categories(
_experiments=_experiments,
_is_single_cell=False,
_is_high_temporal_resolution=_high_temporal_resolution,
_is_bleb=False,
_is_dead_dead=False,
_is_live_dead=False,
_is_bead=False,
_is_metastasis=False)
_experiments = ['SN16']
_tuples = load.experiments_groups_as_tuples(_experiments)
_tuples = filtering.by_pair_distance_range(_tuples, _pair_distance_range)
_tuples = filtering.by_real_pairs(_tuples)
_tuples = filtering.by_band(_tuples)
print('Total tuples:', len(_tuples))
_arguments = []
for _tuple in _tuples:
_experiment, _series_id, _group = _tuple
_latest_time_frame = compute.latest_time_frame_before_overlapping(
_experiment, _series_id, _group, OFFSET_X)
for _cell_id, _offset_y in product(['left_cell', 'right_cell'],
[0, OFFSET_Y]):
_arguments.append({
'experiment': _experiment,
'series_id': _series_id,
'group': _group,
'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER,
'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER,
'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER,
'offset_x': OFFSET_X,
'offset_y': _offset_y,
'offset_z': OFFSET_Z,
'cell_id': _cell_id,
'direction': 'inside',
'time_points': _latest_time_frame
})
_windows_dictionary, _windows_to_compute = compute.windows(
_arguments,
_keys=['experiment', 'series_id', 'group', 'cell_id', 'offset_y'])
_fiber_densities = compute.fiber_densities(_windows_to_compute)
_experiments_fiber_densities = {
_key:
[_fiber_densities[_tuple] for _tuple in _windows_dictionary[_key]]
for _key in _windows_dictionary
}
_no_offset_derivatives_array = []
_offset_derivatives_array = []
for _tuple in tqdm(_tuples, desc='Main loop'):
_experiment, _series_id, _group = _tuple
_no_offset_left_cell_fiber_densities = _experiments_fiber_densities[(
_experiment, _series_id, _group, 'left_cell', 0)]
_no_offset_right_cell_fiber_densities = _experiments_fiber_densities[(
_experiment, _series_id, _group, 'right_cell', 0)]
_offset_left_cell_fiber_densities = _experiments_fiber_densities[(
_experiment, _series_id, _group, 'left_cell', OFFSET_Y)]
_offset_right_cell_fiber_densities = _experiments_fiber_densities[(
_experiment, _series_id, _group, 'right_cell', OFFSET_Y)]
_properties = \
load.group_properties(_experiment, _series_id, _group)
_no_offset_left_cell_fiber_densities = compute.remove_blacklist(
_experiment, _series_id, _properties['cells_ids']['left_cell'],
_no_offset_left_cell_fiber_densities)
_no_offset_right_cell_fiber_densities = compute.remove_blacklist(
_experiment, _series_id, _properties['cells_ids']['right_cell'],
_no_offset_right_cell_fiber_densities)
_offset_left_cell_fiber_densities = compute.remove_blacklist(
_experiment, _series_id, _properties['cells_ids']['left_cell'],
_offset_left_cell_fiber_densities)
_offset_right_cell_fiber_densities = compute.remove_blacklist(
_experiment, _series_id, _properties['cells_ids']['right_cell'],
_offset_right_cell_fiber_densities)
_normalization = load.normalization_series_file_data(
_experiment, _series_id)
for _cell_pair in [(_no_offset_left_cell_fiber_densities,
_offset_left_cell_fiber_densities),
(_no_offset_right_cell_fiber_densities,
_offset_right_cell_fiber_densities)]:
for _time_frame in range(
1, min(len(_cell_pair[0]), len(_cell_pair[1]))):
_no_offset_cell_fiber_density_previous = _cell_pair[0][
_time_frame - 1]
_no_offset_cell_fiber_density = _cell_pair[0][_time_frame]
_offset_cell_fiber_density_previous = _cell_pair[1][_time_frame
- 1]
_offset_cell_fiber_density = _cell_pair[1][_time_frame]
if any([
_no_offset_cell_fiber_density_previous[1],
_no_offset_cell_fiber_density[1],
_offset_cell_fiber_density_previous[1],
_offset_cell_fiber_density[1]
]):
continue
_no_offset_cell_fiber_density_previous_z_score = compute_lib.z_score(
_x=_no_offset_cell_fiber_density_previous[0],
_average=_normalization['average'],
_std=_normalization['std'])
_no_offset_cell_fiber_density_z_score = compute_lib.z_score(
_x=_no_offset_cell_fiber_density[0],
_average=_normalization['average'],
_std=_normalization['std'])
_offset_cell_fiber_density_previous_z_score = compute_lib.z_score(
_x=_offset_cell_fiber_density_previous[0],
_average=_normalization['average'],
_std=_normalization['std'])
_offset_cell_fiber_density_z_score = compute_lib.z_score(
_x=_offset_cell_fiber_density[0],
_average=_normalization['average'],
_std=_normalization['std'])
_no_offset_derivative = _no_offset_cell_fiber_density_z_score - _no_offset_cell_fiber_density_previous_z_score
_offset_derivative = _offset_cell_fiber_density_z_score - _offset_cell_fiber_density_previous_z_score
_no_offset_derivatives_array.append(_no_offset_derivative)
_offset_derivatives_array.append(_offset_derivative)
print('Total points:', len(_no_offset_derivatives_array))
_offset_minus_no_offset = \
np.array(_offset_derivatives_array) - np.array(_no_offset_derivatives_array)
print('Wilcoxon of offset minus no offset around the zero:')
print(wilcoxon(_offset_minus_no_offset))
print('Higher offset amount:',
(_offset_minus_no_offset > 0).sum() / len(_offset_minus_no_offset))
# plot
_fig = go.Figure(
data=go.Scatter(x=_no_offset_derivatives_array,
y=_offset_derivatives_array,
mode='markers',
marker={
'size': 5,
'color': '#ea8500'
},
showlegend=False),
layout={
'xaxis': {
'title': 'No offset derivative',
# 'zeroline': False,
# 'range': [-1.1, 1.2],
# 'tickmode': 'array',
# 'tickvals': [-1, -0.5, 0, 0.5, 1]
},
'yaxis': {
'title': 'Offset derivative',
# 'zeroline': False,
# 'range': [-1.1, 1.2],
# 'tickmode': 'array',
# 'tickvals': [-1, -0.5, 0, 0.5, 1]
},
'shapes': [
# {
# 'type': 'line',
# 'x0': -1,
# 'y0': -1,
# 'x1': -1,
# 'y1': 1,
# 'line': {
# 'color': 'black',
# 'width': 2
# }
# },
# {
# 'type': 'line',
# 'x0': -1,
# 'y0': -1,
# 'x1': 1,
# 'y1': -1,
# 'line': {
# 'color': 'black',
# 'width': 2
# }
# },
{
'type': 'line',
'x0': -5,
'y0': -5,
'x1': 5,
'y1': 5,
'line': {
'color': 'red',
'width': 2
}
}
]
})
save.to_html(_fig=_fig,
_path=os.path.join(paths.PLOTS, save.get_module_name()),
_filename='plot')
def compute_experiments_data():
_experiments = all_experiments()
_experiments = experiments_filtering.by_categories(
_experiments=_experiments,
_is_single_cell=False,
_is_high_temporal_resolution=False,
_is_bleb=False,
_is_dead_dead=False,
_is_live_dead=False,
_is_bead=False,
_is_metastasis=False
)
_tuples = experiments_load.experiments_groups_as_tuples(_experiments)
_tuples = experiments_filtering.by_time_frames_amount(_tuples, EXPERIMENTS_TIME_FRAMES)
_tuples = experiments_filtering.by_real_pairs(_tuples)
_tuples = experiments_filtering.by_pair_distance_range(_tuples, PAIR_DISTANCE_RANGE)
_tuples = experiments_filtering.by_band(_tuples)
print('Total tuples:', len(_tuples))
_arguments = []
for _tuple in _tuples:
_experiment, _series_id, _group = _tuple
_time_frame = experiments_compute.density_time_frame(_experiment)
for _cell_id in ['left_cell', 'right_cell']:
_arguments.append({
'experiment': _experiment,
'series_id': _series_id,
'group': _group,
'length_x': experiments_config.QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER,
'length_y': experiments_config.QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER,
'length_z': experiments_config.QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER,
'offset_x': OFFSET_X,
'offset_y': OFFSET_Y,
'offset_z': OFFSET_Z,
'cell_id': _cell_id,
'direction': 'inside',
'time_points': _time_frame
})
_windows_dictionary, _windows_to_compute = \
experiments_compute.windows(_arguments, _keys=['experiment', 'series_id', 'group', 'cell_id'])
_fiber_densities = experiments_compute.fiber_densities(_windows_to_compute)
_experiments_fiber_densities = [[] for _i in range(EXPERIMENTS_TIME_FRAMES)]
for _tuple in tqdm(_tuples, desc='Experiments loop'):
_experiment, _series_id, _group = _tuple
_normalization = experiments_load.normalization_series_file_data(_experiment, _series_id)
for _time_frame in range(EXPERIMENTS_TIME_FRAMES):
for _cell_id in ['left_cell', 'right_cell']:
_window_tuple = _windows_dictionary[(_experiment, _series_id, _group, _cell_id)][_time_frame]
_fiber_density = _fiber_densities[_window_tuple]
if not OUT_OF_BOUNDARIES and _fiber_density[1]:
continue
_normalized_fiber_density = compute_lib.z_score(
_x=_fiber_density[0],
_average=_normalization['average'],
_std=_normalization['std']
)
if not np.isnan(_normalized_fiber_density):
_experiments_fiber_densities[_time_frame].append(_normalized_fiber_density)
print('Total experiments pairs:', len(_experiments_fiber_densities[0]))
return _experiments_fiber_densities
def compute_matched_fiber(_tuples):
_tuples = filtering.by_time_frames_amount(
_tuples, compute.density_time_frame(_tuples[0]))
_tuples = filtering.by_pair_distance_range(_tuples, PAIR_DISTANCE_RANGE)
_experiments_matched = organize.by_matched_real_and_fake(_tuples)
print('Total matched pairs:', len(_experiments_matched))
_max_offsets_x = []
_arguments = []
for _matched_tuple in _experiments_matched:
for _tuple in _matched_tuple:
_experiment, _series_id, _group = _tuple
_time_frame = compute.density_time_frame(_experiment)
_pair_distance = compute.pair_distance_in_cell_size_time_frame(
_experiment, _series_id, _group, _time_frame - 1)
_offsets_x = \
np.arange(start=0, stop=_pair_distance / 2 - 0.5 - QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER,
step=OFFSET_X_STEP)
if len(_offsets_x) > len(_max_offsets_x):
_max_offsets_x = _offsets_x
for _offset_x in _offsets_x:
for _cell_id in ['left_cell', 'right_cell']:
_arguments.append({
'experiment': _experiment,
'series_id': _series_id,
'group': _group,
'length_x':
QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER,
'length_y':
QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER,
'length_z':
QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER,
'offset_x': _offset_x,
'offset_y': OFFSET_Y,
'offset_z': OFFSET_Z,
'cell_id': _cell_id,
'direction': 'inside',
'time_point': _time_frame - 1
})
_windows_dictionary, _windows_to_compute = \
compute.windows(_arguments, _keys=['experiment', 'series_id', 'group', 'offset_x', 'cell_id'])
_fiber_densities = compute.fiber_densities(_windows_to_compute)
_experiments_fiber_densities_real = [[]
for _i in range(len(_max_offsets_x))]
_experiments_fiber_densities_fake = [[]
for _i in range(len(_max_offsets_x))]
for _tuple in tqdm(_experiments_matched, desc='Experiments loop'):
_tuple_real, _tuple_fake = _tuple
_experiment_real, _series_id_real, _group_real = _tuple_real
_experiment_fake, _series_id_fake, _group_fake = _tuple_fake
for _offset_x_index, _offset_x in enumerate(_max_offsets_x):
for _cell_id in ['left_cell', 'right_cell']:
if (_experiment_real, _series_id_real, _group_real, _offset_x, _cell_id) and \
(_experiment_fake, _series_id_fake, _group_fake, _offset_x, _cell_id) in _windows_dictionary:
_normalization = load.normalization_series_file_data(
_experiment_real, _series_id_real)
_window_tuple_real = \
_windows_dictionary[(_experiment_real, _series_id_real, _group_real, _offset_x, _cell_id)][0]
_fiber_density_real = _fiber_densities[_window_tuple_real]
_window_tuple_fake = \
_windows_dictionary[(_experiment_fake, _series_id_fake, _group_fake, _offset_x, _cell_id)][0]
_fiber_density_fake = _fiber_densities[_window_tuple_fake]
if not OUT_OF_BOUNDARIES and (_fiber_density_real[1]
or _fiber_density_fake[1]):
continue
_normalized_fiber_density_real = compute_lib.z_score(
_x=_fiber_density_real[0],
_average=_normalization['average'],
_std=_normalization['std'])
_normalized_fiber_density_fake = compute_lib.z_score(
_x=_fiber_density_fake[0],
_average=_normalization['average'],
_std=_normalization['std'])
if not np.isnan(
_normalized_fiber_density_real) and not np.isnan(
_normalized_fiber_density_fake):
_experiments_fiber_densities_real[
_offset_x_index].append(
_normalized_fiber_density_real)
_experiments_fiber_densities_fake[
_offset_x_index].append(
_normalized_fiber_density_fake)
return _experiments_fiber_densities_real, _experiments_fiber_densities_fake, _max_offsets_x
def main():
_experiments = all_experiments()
_experiments = filtering.by_categories(_experiments=_experiments,
_is_single_cell=False,
_is_high_temporal_resolution=False,
_is_bleb=False,
_is_dead_dead=False,
_is_live_dead=False,
_is_bead=False,
_is_metastasis=False)
_tuples = load.experiments_groups_as_tuples(_experiments)
_tuples = filtering.by_time_frames_amount(
_tuples, compute.density_time_frame(_experiments[0]))
_tuples = filtering.by_real_pairs(_tuples)
_tuples = filtering.by_band(_tuples)
_max_offsets_x = []
_arguments = []
for _tuple in _tuples:
_experiment, _series_id, _group = _tuple
_time_frame = compute.minimum_time_frames_for_correlation(_experiment)
_pair_distance = \
compute.pair_distance_in_cell_size_time_frame(_experiment, _series_id, _group, _time_frame=_time_frame - 1)
_offsets_x = \
np.arange(start=0, stop=_pair_distance / 2 - 0.5 - QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER,
step=OFFSET_X_STEP)
if len(_offsets_x) > len(_max_offsets_x):
_max_offsets_x = _offsets_x
for _offset_x in _offsets_x:
for _cell_id in ['left_cell', 'right_cell']:
_arguments.append({
'experiment': _experiment,
'series_id': _series_id,
'group': _group,
'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER,
'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER,
'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER,
'offset_x': _offset_x,
'offset_y': OFFSET_Y,
'offset_z': OFFSET_Z,
'cell_id': _cell_id,
'direction': 'inside',
'time_point': _time_frame - 1
})
_windows_dictionary, _windows_to_compute = \
compute.windows(_arguments, _keys=['experiment', 'series_id', 'group', 'offset_x', 'cell_id'])
_fiber_densities = compute.fiber_densities(_windows_to_compute)
_x_array = []
_y_array = []
_n_array = []
_p_value_array = []
for _offset_x in _max_offsets_x:
_pair_distances = []
_z_scores = []
for _tuple in _tuples:
_experiment, _series_id, _group = _tuple
for _cell_id in ['left_cell', 'right_cell']:
if (_experiment, _series_id, _group, _offset_x,
_cell_id) in _windows_dictionary:
_normalization = load.normalization_series_file_data(
_experiment, _series_id)
_window_tuple = _windows_dictionary[(_experiment,
_series_id, _group,
_offset_x,
_cell_id)][0]
_fiber_density = _fiber_densities[_window_tuple]
if not OUT_OF_BOUNDARIES and _fiber_density[1]:
continue
_normalized_fiber_density = compute_lib.z_score(
_x=_fiber_density[0],
_average=_normalization['average'],
_std=_normalization['std'])
if not np.isnan(_normalized_fiber_density):
_pair_distances.append(
compute.pair_distance_in_cell_size_time_frame(
_experiment, _series_id, _group,
_time_frame=0))
_z_scores.append(_normalized_fiber_density)
if len(_pair_distances) > 2:
_x_array.append(round(_offset_x, 1))
_correlation = compute_lib.correlation(_pair_distances,
_z_scores,
_with_p_value=True)
_y_array.append(round(_correlation[0], 2))
_n_array.append(len(_pair_distances))
_p_value_array.append(round(_correlation[1], 2))
print('Pearson:')
print(compute_lib.correlation(_x_array, _y_array, _with_p_value=True))
# plot
_significant_x_array = [
_x for _x, _p_value in zip(_x_array, _p_value_array) if _p_value < 0.05
]
_fig = go.Figure(data=[
go.Scatter(x=_x_array,
y=_y_array,
mode='markers',
marker={
'size': 15,
'color': 'black'
},
showlegend=False),
go.Scatter(x=_significant_x_array,
y=[-0.79] * len(_significant_x_array),
mode='text',
text='*',
textfont={'color': 'red'},
showlegend=False)
],
layout={
'xaxis': {
'title': 'Window distance (cell diameter)',
'zeroline': False
},
'yaxis': {
'title':
'Correlation:<br>pair distance vs. fiber density',
'zeroline': False,
'range': [-0.82, 0.3],
'tickmode': 'array',
'tickvals': [-0.75, -0.25, 0.25]
},
'shapes': [{
'type': 'line',
'x0': -0.2,
'y0': -0.8,
'x1': 3.2,
'y1': -0.8,
'line': {
'color': 'black',
'width': 2
}
}, {
'type': 'line',
'x0': -0.2,
'y0': -0.8,
'x1': -0.2,
'y1': 0.3,
'line': {
'color': 'black',
'width': 2
}
}]
})
save.to_html(_fig=_fig,
_path=os.path.join(paths.PLOTS, save.get_module_name()),
_filename='plot')
# table
print('Window distance (cell diameter)',
'Correlation: pair distance vs. fiber density',
'N',
'P-value',
sep='\t')
for _x, _y, _n, _p_value in zip(_x_array, _y_array, _n_array,
_p_value_array):
print(_x, _y, _n, _p_value, sep='\t')
def main(_real_cells=True,
_static=False,
_band=True,
_high_temporal_resolution=False,
_pair_distance_range=None,
_offset_y=0.5):
if _pair_distance_range is None:
_pair_distance_range = PAIR_DISTANCE_RANGE
_experiments = all_experiments()
_experiments = filtering.by_categories(
_experiments=_experiments,
_is_single_cell=False,
_is_high_temporal_resolution=_high_temporal_resolution,
_is_bleb=False,
_is_dead_dead=False,
_is_live_dead=False,
_is_bead=False,
_is_metastasis=False)
_tuples = load.experiments_groups_as_tuples(_experiments)
_tuples = filtering.by_pair_distance_range(_tuples, _pair_distance_range)
_tuples = filtering.by_real_pairs(_tuples, _real_pairs=_real_cells)
_tuples = filtering.by_fake_static_pairs(_tuples,
_fake_static_pairs=_static)
_tuples = filtering.by_band(_tuples, _band=_band)
print('Total tuples:', len(_tuples))
_arguments = []
_longest_time_frame = 0
for _tuple in _tuples:
_experiment, _series_id, _group = _tuple
_latest_time_frame = compute.latest_time_frame_before_overlapping(
_experiment, _series_id, _group, OFFSET_X)
# save for later
if _latest_time_frame > _longest_time_frame:
_longest_time_frame = _latest_time_frame
for _cell_id in ['left_cell', 'right_cell']:
_arguments.append({
'experiment': _experiment,
'series_id': _series_id,
'group': _group,
'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER,
'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER,
'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER,
'offset_x': OFFSET_X,
'offset_y': _offset_y,
'offset_z': OFFSET_Z,
'cell_id': _cell_id,
'direction': 'inside',
'time_points': _latest_time_frame
})
_windows_dictionary, _windows_to_compute = compute.windows(
_arguments, _keys=['experiment', 'series_id', 'group', 'cell_id'])
_fiber_densities = compute.fiber_densities(_windows_to_compute)
_experiments_fiber_densities = {
_key:
[_fiber_densities[_tuple] for _tuple in _windows_dictionary[_key]]
for _key in _windows_dictionary
}
_valid_tuples = []
_valid_cells = []
_densities = [[] for _ in range(_longest_time_frame)]
for _tuple in tqdm(_tuples, desc='Experiments loop'):
_experiment, _series_id, _group = _tuple
_normalization = load.normalization_series_file_data(
_experiment, _series_id)
_properties = load.group_properties(_experiment, _series_id, _group)
for _cell_id in ['left_cell', 'right_cell']:
_cell_fiber_densities = \
_experiments_fiber_densities[(_experiment, _series_id, _group, _cell_id)]
_cell_fiber_densities = compute.remove_blacklist(
_experiment, _series_id, _properties['cells_ids'][_cell_id],
_cell_fiber_densities)
_previous_cell_fiber_density_normalized = None
for _time_frame, _cell_fiber_density in enumerate(
_cell_fiber_densities):
# not out of border
if _cell_fiber_density[1]:
_previous_cell_fiber_density_normalized = None
continue
# normalize
_cell_fiber_density_normalized = compute_lib.z_score(
_x=_cell_fiber_density[0],
_average=_normalization['average'],
_std=_normalization['std'])
# no previous
if _previous_cell_fiber_density_normalized is None:
_previous_cell_fiber_density_normalized = _cell_fiber_density_normalized
continue
# change
_cell_fiber_density_normalized_change = _cell_fiber_density_normalized - _previous_cell_fiber_density_normalized
_previous_cell_fiber_density_normalized = _cell_fiber_density_normalized
# save
_densities[_time_frame].append(
_cell_fiber_density_normalized_change)
if _tuple not in _valid_tuples:
_valid_tuples.append(_tuple)
_cell_tuple = (_experiment, _series_id, _group, _cell_id)
if _cell_tuple not in _valid_cells:
_valid_cells.append(_cell_tuple)
print('Total pairs:', len(_valid_tuples))
print('Total cells:', len(_valid_cells))
# plot
_temporal_resolution = compute.temporal_resolution_in_minutes(
_experiments[0])
_fig = go.Figure(
data=go.Scatter(x=np.array(range(_longest_time_frame)) *
_temporal_resolution,
y=[np.mean(_array) for _array in _densities],
name='Fiber density change (z-score)',
error_y={
'type': 'data',
'array': [np.std(_array) for _array in _densities],
'thickness': 1
},
mode='lines+markers',
marker={
'size': 5,
'color': '#ea8500'
},
line={'dash': 'solid'},
showlegend=False),
layout={
'xaxis': {
'title': 'Time (minutes)',
# 'zeroline': False
},
'yaxis': {
'title': 'Fiber density change (z-score)',
# 'zeroline': False
},
# 'shapes': [
# {
# 'type': 'line',
# 'x0': -_temporal_resolution,
# 'y0': -0.5,
# 'x1': -_temporal_resolution,
# 'y1': 2,
# 'line': {
# 'color': 'black',
# 'width': 2
# }
# },
# {
# 'type': 'line',
# 'x0': -_temporal_resolution,
# 'y0': -0.5,
# 'x1': 350,
# 'y1': -0.5,
# 'line': {
# 'color': 'black',
# 'width': 2
# }
# }
# ]
})
save.to_html(
_fig=_fig,
_path=os.path.join(paths.PLOTS, save.get_module_name()),
_filename='plot_real_' + str(_real_cells) + '_static_' + str(_static) +
'_band_' + str(_band) + '_high_time_' +
str(_high_temporal_resolution) + '_range_' +
'_'.join([str(_distance) for _distance in _pair_distance_range]) +
'_y_' + str(_offset_y))
def main():
# simulations
print('Simulations')
_simulations = simulations_load.structured()
_simulations = simulations_filtering.by_time_points_amount(
_simulations, _time_points=SIMULATIONS_TIME_POINTS)
_simulations = simulations_filtering.by_categories(
_simulations,
_is_single_cell=True,
_is_heterogeneity=False,
_is_low_connectivity=False,
_is_causality=False,
_is_dominant_passive=False,
_is_fibrin=False)
print('Total simulations:', len(_simulations))
_fiber_densities = compute_simulations_fiber_densities(_simulations)
_simulations_fiber_densities = [[]
for _i in range(SIMULATIONS_TIME_POINTS)]
for _simulation in tqdm(_simulations, desc='Simulations Loop'):
_normalization = simulations_load.normalization(_simulation)
for _time_frame in range(SIMULATIONS_TIME_POINTS):
_direction_fiber_densities = []
for _direction in ['left', 'right', 'up', 'down']:
_fiber_density = _fiber_densities[(_simulation,
_direction)][_time_frame]
_normalized_fiber_density = compute_lib.z_score(
_fiber_density, _normalization['average'],
_normalization['std'])
_direction_fiber_densities.append(_normalized_fiber_density)
_simulations_fiber_densities[_time_frame].append(
np.mean(_direction_fiber_densities))
print('Total simulations cells:', len(_simulations_fiber_densities[0]))
# experiments
print('Experiments')
_experiments_fiber_densities = compute_experiments_fiber_densities()
# plot
_fig = go.Figure(
data=[
go.Scatter(
x=list(range(SIMULATIONS_TIME_POINTS))[::SIMULATIONS_STEP],
y=[np.mean(_array) for _array in _simulations_fiber_densities
][::SIMULATIONS_STEP],
name='Simulations',
error_y={
'type':
'data',
'array': [
np.std(_array)
for _array in _simulations_fiber_densities
][::SIMULATIONS_STEP],
'thickness':
1,
'color':
'#005b96'
},
mode='markers',
marker={
'size': 15,
'color': '#005b96'
},
opacity=0.7),
go.Scatter(
x=np.array(range(EXPERIMENTS_TIME_FRAMES)) *
EXPERIMENTS_TEMPORAL_RESOLUTION,
xaxis='x2',
y=[np.mean(_array) for _array in _experiments_fiber_densities],
name='Experiments',
error_y={
'type':
'data',
'array': [
np.std(_array)
for _array in _experiments_fiber_densities
],
'thickness':
1,
'color':
'#ea8500'
},
mode='markers',
marker={
'size': 15,
'color': '#ea8500'
},
opacity=0.7)
],
layout={
'xaxis': {
'title': 'Cell contraction (%)',
'titlefont': {
'color': '#005b96'
},
'tickfont': {
'color': '#005b96'
},
'zeroline': False
},
'xaxis2': {
'title': 'Time (minutes)',
'titlefont': {
'color': '#ea8500'
},
'tickfont': {
'color': '#ea8500'
},
# 'anchor': 'free',
'overlaying': 'x',
'side': 'bottom',
'showgrid': False,
'zeroline': False
},
'yaxis': {
'title': 'Fiber density (z-score)',
# 'domain': [0.3, 1],
'range': [-1.7, 14],
'zeroline': False,
'tickmode': 'array',
'tickvals': [0, 4, 8, 12]
},
'legend': {
'xanchor': 'left',
'x': 0.1,
'yanchor': 'top',
'bordercolor': 'black',
'borderwidth': 2
},
'shapes': [{
'type': 'line',
'x0': -2,
'y0': -1.5,
'x1': 53,
'y1': -1.5,
'line': {
'color': 'black',
'width': 2
}
}, {
'type': 'line',
'x0': -2,
'y0': -1.5,
'x1': -2,
'y1': 14,
'line': {
'color': 'black',
'width': 2
}
}]
})
save.to_html(_fig=_fig,
_path=os.path.join(paths_lib.PLOTS, save.get_module_name()),
_filename='plot')
def main():
print('Single Cell')
_experiments = all_experiments()
_experiments = filtering.by_categories(_experiments=_experiments,
_is_single_cell=True,
_is_high_temporal_resolution=False,
_is_bleb=False,
_is_dead_dead=False,
_is_live_dead=False,
_is_bead=False,
_is_metastasis=False)
_tuples = load.experiments_groups_as_tuples(_experiments)
_tuples = filtering.by_time_frames_amount(
_tuples, compute.minimum_time_frames_for_correlation(_experiments[0]))
_arguments = []
for _tuple in _tuples:
_experiment, _series_id, _group = _tuple
_time_frame = compute.minimum_time_frames_for_correlation(_experiment)
for _offset_x, _direction in product(OFFSETS_X, ['left', 'right']):
_arguments.append({
'experiment': _experiment,
'series_id': _series_id,
'group': _group,
'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER,
'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER,
'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER,
'offset_x': _offset_x,
'offset_y': OFFSET_Y,
'offset_z': OFFSET_Z,
'cell_id': 'cell',
'direction': _direction,
'time_point': _time_frame - 1
})
_windows_dictionary, _windows_to_compute = \
compute.windows(_arguments, _keys=['experiment', 'series_id', 'group', 'offset_x', 'direction'])
_fiber_densities = compute.fiber_densities(_windows_to_compute)
_tuples = organize.by_single_cell_id(_tuples)
_single_cell_fiber_densities = [[] for _i in range(len(OFFSETS_X))]
for _tuple in _tuples:
_experiment, _series_id, _cell_id = _tuple
print('Experiment:',
_experiment,
'Series ID:',
_series_id,
'Cell ID:',
_cell_id,
sep='\t')
_offset_index = 0
_normalization = load.normalization_series_file_data(
_experiment, _series_id)
for _offset_x in OFFSETS_X:
_cell_fiber_densities = []
for _cell_tuple in _tuples[_tuple]:
_, _, _group = _cell_tuple
for _direction in ['left', 'right']:
_window_tuple = _windows_dictionary[(_experiment,
_series_id, _group,
_offset_x,
_direction)][0]
_fiber_density = _fiber_densities[_window_tuple]
if not OUT_OF_BOUNDARIES and _fiber_density[1]:
continue
_normalized_fiber_density = compute_lib.z_score(
_x=_fiber_density[0],
_average=_normalization['average'],
_std=_normalization['std'])
_cell_fiber_densities.append(_normalized_fiber_density)
if len(_cell_fiber_densities) > 0:
_single_cell_fiber_densities[_offset_index].append(
np.mean(_cell_fiber_densities))
_offset_index += 1
print('Pairs')
_experiments = all_experiments()
_experiments = filtering.by_categories(_experiments=_experiments,
_is_single_cell=False,
_is_high_temporal_resolution=False,
_is_bleb=False,
_is_dead_dead=False,
_is_live_dead=False,
_is_bead=False,
_is_metastasis=False)
_tuples = load.experiments_groups_as_tuples(_experiments)
_tuples = filtering.by_time_frames_amount(
_tuples, compute.minimum_time_frames_for_correlation(_experiments[0]))
_tuples = filtering.by_real_pairs(_tuples)
_tuples = filtering.by_pair_distance(_tuples, PAIR_DISTANCE)
_tuples = filtering.by_band(_tuples)
_arguments = []
for _tuple in _tuples:
_experiment, _series_id, _group = _tuple
_time_frame = compute.minimum_time_frames_for_correlation(_experiment)
for _offset_x in OFFSETS_X:
_arguments.append({
'experiment': _experiment,
'series_id': _series_id,
'group': _group,
'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER,
'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER,
'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER,
'offset_x': _offset_x,
'offset_y': OFFSET_Y,
'offset_z': OFFSET_Z,
'cell_id': 'left_cell',
'direction': 'inside',
'time_point': _time_frame - 1
})
_windows_dictionary, _windows_to_compute = \
compute.windows(_arguments, _keys=['experiment', 'series_id', 'group', 'offset_x'])
_fiber_densities = compute.fiber_densities(_windows_to_compute)
_pairs_fiber_densities = [[] for _i in range(len(OFFSETS_X))]
for _tuple in _tuples:
_experiment, _series_id, _group = _tuple
print('Experiment:',
_experiment,
'Series ID:',
_series_id,
'Group:',
_group,
sep='\t')
_offset_index = 0
_normalization = load.normalization_series_file_data(
_experiment, _series_id)
# take offsets based on pair distance
_properties = load.group_properties(_experiment, _series_id, _group)
_left_cell_coordinates = [
list(_properties['time_points'][0]['left_cell']
['coordinates'].values())
]
_right_cell_coordinates = [
list(_properties['time_points'][0]['right_cell']
['coordinates'].values())
]
_pair_distance = compute.pair_distance_in_cell_size(
_experiment, _series_id, _left_cell_coordinates,
_right_cell_coordinates)
_edges_distance = _pair_distance - 1
_max_x_offset = _edges_distance - QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER
for _offset_x in OFFSETS_X:
if _offset_x > _max_x_offset:
break
_fiber_density = _fiber_densities[_windows_dictionary[(
_experiment, _series_id, _group, _offset_x)][0]]
if not OUT_OF_BOUNDARIES and _fiber_density[1]:
continue
_normalized_fiber_density = compute_lib.z_score(
_x=_fiber_density[0],
_average=_normalization['average'],
_std=_normalization['std'])
_pairs_fiber_densities[_offset_index].append(
_normalized_fiber_density)
_offset_index += 1
# plot
_fig = go.Figure(data=[
go.Scatter(x=OFFSETS_X,
y=[np.mean(_array) for _array in _pairs_fiber_densities],
name='Pairs',
error_y={
'type':
'data',
'array':
[np.std(_array) for _array in _pairs_fiber_densities],
'thickness':
1
},
mode='lines+markers',
line={'dash': 'solid'}),
go.Scatter(
x=OFFSETS_X,
y=[np.mean(_array) for _array in _single_cell_fiber_densities],
name='Single Cell',
error_y={
'type':
'data',
'array':
[np.std(_array) for _array in _single_cell_fiber_densities],
'thickness':
1
},
mode='lines+markers',
line={'dash': 'dash'})
],
layout={
'xaxis_title': 'Distance from left cell (cell size)',
'yaxis_title': 'Fiber density (z-score)'
})
save.to_html(_fig=_fig,
_path=os.path.join(paths.PLOTS, save.get_module_name()),
_filename='plot_distance_' + str(PAIR_DISTANCE))
def main():
_experiments = config.all_experiments()
_experiments = filtering.by_categories(_experiments=_experiments,
_is_single_cell=False,
_is_high_temporal_resolution=None,
_is_bleb=False,
_is_dead_dead=False,
_is_live_dead=False,
_is_bead=False,
_is_metastasis=False)
_tuples = load.experiments_groups_as_tuples(_experiments)
_tuples = filtering.by_pair_distance_range(_tuples, PAIR_DISTANCE_RANGE)
_tuples = filtering.by_real_pairs(_tuples, _real_pairs=REAL)
_tuples = filtering.by_fake_static_pairs(_tuples,
_fake_static_pairs=STATIC)
_tuples = filtering.by_real_fake_pairs(_tuples, _real_fake_pairs=REAL_FAKE)
print('Total tuples:', len(_tuples))
_arguments = []
for _tuple in tqdm(_tuples, desc='Setting windows to compute'):
_experiment, _series_id, _group = _tuple
_latest_time_frame = compute.latest_time_frame_before_overlapping(
_experiment, _series_id, _group, _offset_x=0)
for _cell_id in ['left_cell', 'right_cell']:
for _offset_y in OFFSETS_Y:
_arguments.append({
'experiment': _experiment,
'series_id': _series_id,
'group': _group,
'length_x':
config.QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER,
'length_y':
config.QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER,
'length_z':
config.QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER,
'offset_x': 0,
'offset_y': _offset_y,
'offset_z': 0,
'cell_id': _cell_id,
'direction': 'inside',
'time_points': _latest_time_frame
})
_windows_dictionary, _windows_to_compute = \
compute.windows(_arguments, _keys=['experiment', 'series_id', 'group', 'cell_id', 'offset_y'])
_fiber_densities = compute.fiber_densities(_windows_to_compute)
_experiments_fiber_densities = {
_key:
[_fiber_densities[_tuple] for _tuple in _windows_dictionary[_key]]
for _key in _windows_dictionary
}
_headers = [
'time_frame', 'experiment', 'series_id', 'group', 'left_cell_id',
'right_cell_id', 'band', 'fake_following', 'fake_static',
'pair_distance_in_cell_diameter', 'offset_z',
'left_cell_fiber_density', 'left_cell_fiber_density_z_score',
'left_cell_out_of_boundaries', 'right_cell_fiber_density',
'right_cell_fiber_density_z_score', 'right_cell_out_of_boundaries'
]
_csv_path = os.path.join(paths.OUTPUTS,
'experiments_density_cell_pairs.csv')
with open(_csv_path, 'w', newline='') as _csv_file:
_csv_writer = csv.writer(_csv_file)
_csv_writer.writerow(_headers)
for _tuple in tqdm(_tuples, desc='Main loop'):
_experiment, _series_id, _group = _tuple
_group_properties = load.group_properties(_experiment, _series_id,
_group)
_left_cell_id, _right_cell_id = _group_properties[
'cells_ids'].values()
_band = _group_properties['band']
_fake_following, _fake_static = _group_properties[
'fake'], _group_properties['static']
_average, _std = load.normalization_series_file_data(
_experiment, _series_id).values()
for _offset_y in OFFSETS_Y:
_left_cell_fiber_densities = \
_experiments_fiber_densities[(_experiment, _series_id, _group, 'left_cell', _offset_y)]
_right_cell_fiber_densities = \
_experiments_fiber_densities[(_experiment, _series_id, _group, 'right_cell', _offset_y)]
_left_cell_fiber_densities = compute.remove_blacklist(
_experiment, _series_id, _left_cell_id,
_left_cell_fiber_densities)
_right_cell_fiber_densities = compute.remove_blacklist(
_experiment, _series_id, _right_cell_id,
_right_cell_fiber_densities)
for _time_frame, (_left_cell_fiber_density, _right_cell_fiber_density) in \
enumerate(zip(_left_cell_fiber_densities, _right_cell_fiber_densities)):
_pair_distance = compute.pair_distance_in_cell_size_time_frame(
_experiment, _series_id, _group, _time_frame)
_left_cell_fiber_density, _left_cell_out_of_boundaries = _left_cell_fiber_density
_right_cell_fiber_density, _right_cell_out_of_boundaries = _right_cell_fiber_density
_left_cell_fiber_density_z_score = compute_lib.z_score(
_left_cell_fiber_density, _average, _std)
_right_cell_fiber_density_z_score = compute_lib.z_score(
_right_cell_fiber_density, _average, _std)
_csv_writer.writerow([
_time_frame, _experiment, _series_id, _group,
_left_cell_id, _right_cell_id, _band, _fake_following,
_fake_static, _pair_distance, _offset_y,
_left_cell_fiber_density,
_left_cell_fiber_density_z_score,
_left_cell_out_of_boundaries,
_right_cell_fiber_density,
_right_cell_fiber_density_z_score,
_right_cell_out_of_boundaries
])
def compute_cell_pairs():
_x_array = []
_y_array = []
_names_array = []
for _distances_range in PAIR_DISTANCE_RANGES:
print('Pair distance range:', str(_distances_range))
_experiments = all_experiments()
_experiments = filtering.by_categories(
_experiments=_experiments,
_is_single_cell=False,
_is_high_temporal_resolution=False,
_is_bleb=False,
_is_dead_dead=False,
_is_live_dead=False,
_is_bead=False,
_is_metastasis=False)
_tuples = load.experiments_groups_as_tuples(_experiments)
_tuples = filtering.by_time_frames_amount(
_tuples, compute.density_time_frame(_experiments[0]))
_tuples = filtering.by_real_pairs(_tuples)
_tuples = filtering.by_pair_distance_range(_tuples, _distances_range)
_tuples = filtering.by_band(_tuples)
print('Total tuples:', len(_tuples))
_max_offsets_x = []
_arguments = []
for _tuple in _tuples:
_experiment, _series_id, _group = _tuple
_time_frame = compute.minimum_time_frames_for_correlation(
_experiment)
_pair_distance = \
compute.pair_distance_in_cell_size_time_frame(_experiment, _series_id, _group, _time_frame - 1)
_offsets_x = np.arange(
start=0,
stop=_pair_distance / 2 - 0.5 -
QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER,
step=OFFSET_X_STEP)
if len(_offsets_x) > len(_max_offsets_x):
_max_offsets_x = _offsets_x
for _offset_x in _offsets_x:
for _cell_id in ['left_cell', 'right_cell']:
_arguments.append({
'experiment': _experiment,
'series_id': _series_id,
'group': _group,
'length_x':
QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER,
'length_y':
QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER,
'length_z':
QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER,
'offset_x': _offset_x,
'offset_y': OFFSET_Y,
'offset_z': OFFSET_Z,
'cell_id': _cell_id,
'direction': 'inside',
'time_point': _time_frame - 1
})
_windows_dictionary, _windows_to_compute = \
compute.windows(_arguments, _keys=['experiment', 'series_id', 'group', 'offset_x', 'cell_id'])
_fiber_densities = compute.fiber_densities(_windows_to_compute)
_pair_distance_fiber_densities = [[]
for _i in range(len(_max_offsets_x))]
for _tuple in _tuples:
_experiment, _series_id, _group = _tuple
for _offset_x_index, _offset_x in enumerate(_max_offsets_x):
for _cell_id in ['left_cell', 'right_cell']:
if (_experiment, _series_id, _group, _offset_x,
_cell_id) in _windows_dictionary:
_normalization = load.normalization_series_file_data(
_experiment, _series_id)
_window_tuple = _windows_dictionary[(_experiment,
_series_id,
_group, _offset_x,
_cell_id)][0]
_fiber_density = _fiber_densities[_window_tuple]
if not OUT_OF_BOUNDARIES and _fiber_density[1]:
continue
_normalized_fiber_density = compute_lib.z_score(
_x=_fiber_density[0],
_average=_normalization['average'],
_std=_normalization['std'])
if not np.isnan(_normalized_fiber_density):
_pair_distance_fiber_densities[
_offset_x_index].append(
_normalized_fiber_density)
_x_array.append(_max_offsets_x)
_y_array.append(_pair_distance_fiber_densities)
_names_array.append('Pair distance ' + str(_distances_range[0]) + '-' +
str(_distances_range[1]))
return _names_array, _x_array, _y_array
def main():
_experiments = all_experiments()
_experiments = filtering.by_categories(_experiments=_experiments,
_is_single_cell=False,
_is_high_temporal_resolution=False,
_is_bleb=False,
_is_dead_dead=False,
_is_live_dead=False,
_is_bead=False,
_is_metastasis=False)
_tuples = load.experiments_groups_as_tuples(_experiments)
_tuples = filtering.by_time_frames_amount(
_tuples, compute.density_time_frame(_experiments[0]))
_tuples = filtering.by_real_pairs(_tuples)
_tuples = filtering.by_band(_tuples)
print('Total tuples:', len(_tuples))
_arguments = []
for _tuple in _tuples:
_experiment, _series_id, _group = _tuple
_time_frame = compute.density_time_frame(_experiment)
_pair_distance = \
compute.pair_distance_in_cell_size_time_frame(_experiment, _series_id, _group, _time_frame=_time_frame - 1)
for _offset_x in OFFSETS_X:
if _pair_distance / 2 - 0.5 - QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER >= _offset_x:
for _cell_id in ['left_cell', 'right_cell']:
_arguments.append({
'experiment': _experiment,
'series_id': _series_id,
'group': _group,
'length_x':
QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER,
'length_y':
QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER,
'length_z':
QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER,
'offset_x': _offset_x,
'offset_y': OFFSET_Y,
'offset_z': OFFSET_Z,
'cell_id': _cell_id,
'direction': 'inside',
'time_point': _time_frame - 1
})
_windows_dictionary, _windows_to_compute = \
compute.windows(_arguments, _keys=['experiment', 'series_id', 'group', 'cell_id', 'offset_x'])
_fiber_densities = compute.fiber_densities(_windows_to_compute)
for _offset_x in OFFSETS_X:
_x_array = []
_y_array = []
for _tuple in _tuples:
_experiment, _series_id, _group = _tuple
for _cell_id in ['left_cell', 'right_cell']:
if (_experiment, _series_id, _group, _cell_id,
_offset_x) in _windows_dictionary:
_pair_distance = \
compute.pair_distance_in_cell_size_time_frame(_experiment, _series_id, _group, _time_frame=0)
_normalization = load.normalization_series_file_data(
_experiment, _series_id)
_window_tuple = _windows_dictionary[(_experiment,
_series_id, _group,
_cell_id,
_offset_x)][0]
_fiber_density = _fiber_densities[_window_tuple]
if not OUT_OF_BOUNDARIES and _fiber_density[1]:
continue
_normalized_fiber_density = compute_lib.z_score(
_x=_fiber_density[0],
_average=_normalization['average'],
_std=_normalization['std'])
if not np.isnan(_normalized_fiber_density):
_x_array.append(_pair_distance)
_y_array.append(_normalized_fiber_density)
print('Offset x (cell diameter):', _offset_x)
print('Total pairs:', len(_x_array))
print(compute_lib.correlation(_x_array, _y_array, _with_p_value=True))
# plot
_fig = go.Figure(data=go.Scatter(x=_x_array,
y=_y_array,
mode='markers',
marker={
'size': 15,
'color': 'black'
}),
layout={
'xaxis': {
'title': 'Pair distance (cell diameter)',
'zeroline': False
},
'yaxis': {
'title': 'Fiber density (z-score)',
'zeroline': False,
'range': [-2.2, 13],
'tickmode': 'array',
'tickvals': [0, 4, 8, 12]
},
'shapes': [{
'type': 'line',
'x0': 4.5,
'y0': -2,
'x1': 9.5,
'y1': -2,
'line': {
'color': 'black',
'width': 2
}
}, {
'type': 'line',
'x0': 4.5,
'y0': -2,
'x1': 4.5,
'y1': 13,
'line': {
'color': 'black',
'width': 2
}
}]
})
save.to_html(_fig=_fig,
_path=os.path.join(paths.PLOTS, save.get_module_name()),
_filename='plot_offset_x_' + str(_offset_x))
def main():
_simulations = load.structured()
_simulations = filtering.by_categories(
_simulations,
_is_single_cell=False,
_is_heterogeneity=None,
_is_low_connectivity=False,
_is_causality=CAUSALITY,
_is_dominant_passive=DOMINANT_PASSIVE,
_is_fibrin=False)
_simulations = filtering.by_pair_distances(_simulations,
_distances=PAIR_DISTANCE)
print('Total simulations:', len(_simulations))
_arguments = []
for _simulation in _simulations:
for _cell_id in ['left_cell', 'right_cell']:
_arguments.append({
'simulation': _simulation,
'length_x':
config.QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER,
'length_y':
config.QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER,
'offset_x': 0,
'offset_y': 0,
'cell_id': _cell_id,
'direction': 'inside',
'time_points': TIME_POINTS
})
_fiber_densities = {}
with Pool(CPUS_TO_USE) as _p:
for _keys, _value in tqdm(_p.imap_unordered(
compute.window_fiber_density_by_time, _arguments),
total=len(_arguments),
desc='Computing windows & fiber densities'):
_fiber_densities[(_keys['simulation'], _keys['cell_id'])] = _value
_p.close()
_p.join()
_simulations_by_heterogeneity = organize.by_heterogeneity(_simulations)
_headers = [
'time_point',
'simulation',
'pair_distance_in_cell_diameter',
'heterogeneity_std',
'left_cell_fiber_density',
'left_cell_fiber_density_z_score',
'right_cell_fiber_density',
'right_cell_fiber_density_z_score',
]
_csv_path = os.path.join(paths.OUTPUTS,
'simulations_density_cell_pairs.csv')
with open(_csv_path, 'w', newline='') as _csv_file:
_csv_writer = csv.writer(_csv_file)
_csv_writer.writerow(_headers)
for _simulation_std in _simulations_by_heterogeneity:
print('STD:', _simulation_std)
_std_simulations = _simulations_by_heterogeneity[_simulation_std]
for _simulation in tqdm(_std_simulations, desc='Simulations loop'):
_properties = load.properties(_simulation)
_pair_distance = compute.pair_distance(_properties)
_average, _std = load.normalization(_simulation).values()
_left_cell_fiber_densities = _fiber_densities[(_simulation,
'left_cell')]
_right_cell_fiber_densities = _fiber_densities[(_simulation,
'right_cell')]
for _time_point, (_left_cell_fiber_density, _right_cell_fiber_density) in \
enumerate(zip(_left_cell_fiber_densities, _right_cell_fiber_densities)):
_left_cell_fiber_density_z_score = compute_lib.z_score(
_left_cell_fiber_density, _average, _std)
_right_cell_fiber_density_z_score = compute_lib.z_score(
_right_cell_fiber_density, _average, _std)
_csv_writer.writerow([
_time_point, _simulation, _pair_distance,
_simulation_std, _left_cell_fiber_density,
_left_cell_fiber_density_z_score,
_right_cell_fiber_density,
_right_cell_fiber_density_z_score
])
def main(_band=None,
_high_temporal_resolution=True,
_tuples_to_mark=None,
_tuples_to_plot=None,
_plots=None):
if _plots is None:
_plots = ['whiteness', 'granger']
_experiments = all_experiments()
_experiments = filtering.by_categories(
_experiments=_experiments,
_is_single_cell=False,
_is_high_temporal_resolution=_high_temporal_resolution,
_is_bleb=False,
_is_dead_dead=False,
_is_live_dead=False,
_is_bead=False,
_is_metastasis=False)
_tuples = load.experiments_groups_as_tuples(_experiments)
_tuples = filtering.by_time_frames_amount(_tuples,
_time_frames=MINIMUM_TIME_FRAMES)
_tuples = filtering.by_pair_distance_range(
_tuples, _distance_range=PAIR_DISTANCE_RANGE)
_tuples = filtering.by_real_pairs(_tuples)
_tuples = filtering.by_band(_tuples, _band=_band)
print('Total tuples:', len(_tuples))
_arguments = []
for _tuple in _tuples:
_experiment, _series_id, _group = _tuple
_latest_time_frame = compute.latest_time_frame_before_overlapping(
_experiment, _series_id, _group, OFFSET_X)
for _cell_id in ['left_cell', 'right_cell']:
_arguments.append({
'experiment': _experiment,
'series_id': _series_id,
'group': _group,
'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER,
'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER,
'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER,
'offset_x': OFFSET_X,
'offset_y': OFFSET_Y,
'offset_z': OFFSET_Z,
'cell_id': _cell_id,
'direction': 'inside',
'time_points': _latest_time_frame
})
_windows_dictionary, _windows_to_compute = compute.windows(
_arguments, _keys=['experiment', 'series_id', 'group', 'cell_id'])
_fiber_densities = compute.fiber_densities(_windows_to_compute,
_subtract_border=True)
_experiments_fiber_densities = {
_key:
[_fiber_densities[_tuple] for _tuple in _windows_dictionary[_key]]
for _key in _windows_dictionary
}
_n_pairs = 0
_n_pairs_with_band = 0
_whiteness_p_values = []
_n_passed_whiteness_with_band = 0
_granger_causality_p_values = []
_n_passed_granger_causality_with_band = 0
_correlations = []
_time_lag_correlations = []
_end_fiber_densities = []
for _tuple in _tuples:
_experiment, _series_id, _group = _tuple
_left_cell_fiber_densities = \
_experiments_fiber_densities[(_experiment, _series_id, _group, 'left_cell')]
_right_cell_fiber_densities = \
_experiments_fiber_densities[(_experiment, _series_id, _group, 'right_cell')]
_properties = load.group_properties(_experiment, _series_id, _group)
_left_cell_fiber_densities = compute.remove_blacklist(
_experiment, _series_id, _properties['cells_ids']['left_cell'],
_left_cell_fiber_densities)
_right_cell_fiber_densities = compute.remove_blacklist(
_experiment, _series_id, _properties['cells_ids']['right_cell'],
_right_cell_fiber_densities)
_left_cell_fiber_densities_filtered, _right_cell_fiber_densities_filtered = \
compute.longest_same_indices_shared_in_borders_sub_array(
_left_cell_fiber_densities, _right_cell_fiber_densities)
# ignore small arrays
if len(_left_cell_fiber_densities_filtered) < MINIMUM_TIME_FRAMES:
continue
_n_pairs += 1
if _properties['band']:
_n_pairs_with_band += 1
_start_time_frame = 0
for _left in _left_cell_fiber_densities:
if _left[0] == _left_cell_fiber_densities_filtered[0]:
break
_start_time_frame += 1
# stationary test
with warnings.catch_warnings():
warnings.simplefilter('ignore', category=InterpolationWarning)
# find derivative for stationary
for _derivative in range(10):
_left_cell_fiber_densities_derivative = \
compute_lib.derivative(_left_cell_fiber_densities_filtered, _n=_derivative)
_right_cell_fiber_densities_derivative = \
compute_lib.derivative(_right_cell_fiber_densities_filtered, _n=_derivative)
if ADF_TEST:
_, _left_cell_adf_p_value, _, _, _, _ = adfuller(
_left_cell_fiber_densities_derivative)
_, _right_cell_adf_p_value, _, _, _, _ = adfuller(
_right_cell_fiber_densities_derivative)
if _left_cell_adf_p_value > 0.05 or _right_cell_adf_p_value > 0.05:
continue
if KPSS_TEST:
_, _left_cell_kpss_p_value, _, _ = kpss(
_left_cell_fiber_densities_derivative, nlags='legacy')
_, _right_cell_kpss_p_value, _, _ = kpss(
_right_cell_fiber_densities_derivative, nlags='legacy')
if _left_cell_kpss_p_value < 0.05 or _right_cell_kpss_p_value < 0.05:
continue
# stationary
break
# causality
try:
_x = pd.DataFrame(data=[[_left_value, _right_value]
for _left_value, _right_value in zip(
_left_cell_fiber_densities_derivative,
_right_cell_fiber_densities_derivative)
],
columns=['left', 'right'])
# var model to retrieve lag
_var_model = VAR(_x)
_lag_order_results = _var_model.select_order()
_estimators_lags = [
_lag_order_results.aic, _lag_order_results.bic,
_lag_order_results.fpe, _lag_order_results.hqic
]
_min_estimator_lag = min(_estimators_lags)
# found a lag
if 0 < _min_estimator_lag <= MAXIMUM_LAG:
_var_model_results = _var_model.fit(maxlags=_min_estimator_lag,
ic=None)
_whiteness = _var_model_results.test_whiteness(
nlags=_min_estimator_lag + 1)
_whiteness_p_values.append(_whiteness.pvalue)
if _tuples_to_mark is not None and _tuple in _tuples_to_mark and _whiteness.pvalue > 0.05:
print(_tuple, 'marked whiteness p-value:',
_whiteness.pvalue)
# no autocorrelation in the residuals
if _whiteness.pvalue > 0.05:
if _properties['band']:
_n_passed_whiteness_with_band += 1
# time lag = 0
_correlation = compute_lib.correlation(
_left_cell_fiber_densities_derivative,
_right_cell_fiber_densities_derivative)
# if _correlation < 0.5:
# continue
# granger causality
for _caused, _causing in zip(['left', 'right'],
['right', 'left']):
_granger = _var_model_results.test_causality(
caused=_caused, causing=_causing)
_granger_causality_p_values.append(_granger.pvalue)
# time lag = 0
_correlations.append(_correlation)
# time lag = min estimator
if _causing == 'left':
_left_fiber_densities_time_lag = \
_left_cell_fiber_densities_derivative[:-_min_estimator_lag]
_right_fiber_densities_time_lag = \
_right_cell_fiber_densities_derivative[_min_estimator_lag:]
else:
_left_fiber_densities_time_lag = \
_left_cell_fiber_densities_derivative[_min_estimator_lag:]
_right_fiber_densities_time_lag = \
_right_cell_fiber_densities_derivative[:-_min_estimator_lag]
_time_lag_correlation = compute_lib.correlation(
_left_fiber_densities_time_lag,
_right_fiber_densities_time_lag)
_time_lag_correlations.append(_time_lag_correlation)
# end fiber density
_time_frame = compute.density_time_frame(_experiment)
if len(_left_cell_fiber_densities_filtered
) > _time_frame:
_end_fiber_density = \
(_left_cell_fiber_densities_filtered[_time_frame] +
_right_cell_fiber_densities_filtered[_time_frame]) / 2
else:
_end_fiber_density = \
(_left_cell_fiber_densities_filtered[-1] +
_right_cell_fiber_densities_filtered[-1]) / 2
_normalization = load.normalization_series_file_data(
_experiment, _series_id)
_normalized_fiber_density = compute_lib.z_score(
_end_fiber_density, _normalization['average'],
_normalization['std'])
_end_fiber_densities.append(_normalized_fiber_density)
# marking
if _tuples_to_mark is not None and _tuple in _tuples_to_mark and _granger.pvalue < 0.05:
print(_tuple, 'causing:', _causing,
'marked granger p-value:', _granger.pvalue)
if _granger.pvalue < 0.05:
if _properties['band']:
_n_passed_granger_causality_with_band += 1
_normality = _var_model_results.test_normality()
_inst_granger = _var_model_results.test_inst_causality(
causing=_causing)
print(
_tuple,
_causing.capitalize() + ' causes ' + _caused +
'!',
'time-points: ' +
str(len(
_left_cell_fiber_densities_derivative)),
'stationary derivative: ' + str(_derivative),
'band:' + str(_properties['band']),
'p-value: ' + str(round(_granger.pvalue, 4)),
'lag: ' + str(_min_estimator_lag),
'normality p-value: ' +
str(round(_normality.pvalue, 4)),
'inst p-value: ' +
str(round(_inst_granger.pvalue, 4)),
sep='\t')
# lag = 0
print('Time lag = 0 correlation:', _correlation)
# rest of lags
for _lag in range(1, _min_estimator_lag + 1):
if _causing == 'left':
_left_fiber_densities_time_lag = _left_cell_fiber_densities_derivative[:
-_lag]
_right_fiber_densities_time_lag = _right_cell_fiber_densities_derivative[
_lag:]
else:
_left_fiber_densities_time_lag = _left_cell_fiber_densities_derivative[
_lag:]
_right_fiber_densities_time_lag = _right_cell_fiber_densities_derivative[:
-_lag]
_correlation = compute_lib.correlation(
_left_fiber_densities_time_lag,
_right_fiber_densities_time_lag)
print(
'Time lag = ' + str(_lag) +
' correlation:', _correlation)
# plots
if _tuples_to_plot is not None and _tuple in _tuples_to_plot:
_y_arrays = [
_left_cell_fiber_densities_derivative,
_right_cell_fiber_densities_derivative
]
_names_array = ['Left cell', 'Right cell']
_colors_array = config.colors(2)
_temporal_resolution = compute.temporal_resolution_in_minutes(
_experiment)
_fig = go.Figure(data=[
go.Scatter(x=np.arange(
start=_start_time_frame,
stop=_start_time_frame +
len(_left_cell_fiber_densities_derivative
),
step=1) * _temporal_resolution,
y=_y,
name=_name,
mode='lines',
line={
'color': _color,
'width': 1
})
for _y, _name, _color in zip(
_y_arrays, _names_array, _colors_array)
],
layout={
'xaxis': {
'title':
'Time (minutes)',
'zeroline': False
},
'yaxis': {
'title':
'Fiber density (z-score)'
+ '\'' * _derivative,
'zeroline':
False
},
'legend': {
'xanchor': 'left',
'x': 0.1,
'yanchor': 'top',
'bordercolor':
'black',
'borderwidth': 2,
'bgcolor': 'white'
},
})
_experiment, _series_id, _group = _tuple
save.to_html(
_fig=_fig,
_path=os.path.join(paths.PLOTS,
save.get_module_name()),
_filename='plot_' + _experiment + '_' +
str(_series_id) + '_' + _group)
# residuals
_y_arrays = \
[_var_model_results.resid.values[:, 0], _var_model_results.resid.values[:, 1]]
_fig = go.Figure(data=[
go.Scatter(x=np.arange(
start=_start_time_frame,
stop=_start_time_frame + len(_y),
step=1) * _temporal_resolution,
y=_y,
name=_name,
mode='lines',
line={
'color': _color,
'width': 1
})
for _y, _name, _color in zip(
_y_arrays, _names_array, _colors_array)
],
layout={
'xaxis': {
'title':
'Time (minutes)',
'zeroline': False
},
'yaxis': {
'title': 'Residual',
'zeroline': False
},
'legend': {
'xanchor': 'left',
'x': 0.1,
'yanchor': 'top',
'bordercolor':
'black',
'borderwidth': 2,
'bgcolor': 'white'
},
})
_experiment, _series_id, _group = _tuple
save.to_html(
_fig=_fig,
_path=os.path.join(paths.PLOTS,
save.get_module_name()),
_filename='plot_residuals_' + _experiment +
'_' + str(_series_id) + '_' + _group)
# not enough time points
except ValueError:
continue
print('Total pairs:', _n_pairs)
print('Total pairs with band:', _n_pairs_with_band)
print('Total pairs passed whiteness:',
(np.array(_whiteness_p_values) > 0.05).sum())
print('Total pairs passed whiteness with band:',
_n_passed_whiteness_with_band)
print('Total cells passed granger causality:',
(np.array(_granger_causality_p_values) < 0.05).sum())
print('Total cells passed granger causality with band:',
_n_passed_granger_causality_with_band)
# p-value correction
print('Corrections of GC p-value < 0.05:')
_granger_causality_p_values_corrected = multipletests(
pvals=_granger_causality_p_values, method='fdr_bh')
for _p_value, _p_value_corrected in zip(
_granger_causality_p_values,
_granger_causality_p_values_corrected[1]):
if _p_value < 0.05:
print('Original GC p-value:', _p_value, 'corrected:',
_p_value_corrected)
# plots
for _test_name, _y_title, _y_array in \
zip(
['whiteness', 'granger'],
['Whiteness p-value', 'Granger causality p-value'],
[_whiteness_p_values, _granger_causality_p_values]
):
if _test_name in _plots:
_fig = go.Figure(data=go.Box(y=_y_array,
boxpoints='all',
jitter=1,
pointpos=0,
line={'width': 1},
fillcolor='white',
marker={
'size': 10,
'color': '#ea8500'
},
opacity=0.7,
showlegend=False),
layout={
'xaxis': {
'zeroline': False
},
'yaxis': {
'title': _y_title,
'zeroline': False,
'range': [-0.1, 1.1],
'tickmode': 'array',
'tickvals': [0.05, 1]
},
'shapes': [{
'type': 'line',
'x0': -0.75,
'y0': 0.05,
'x1': 0.75,
'y1': 0.05,
'line': {
'color': 'red',
'width': 2,
'dash': 'dash'
}
}]
})
save.to_html(_fig=_fig,
_path=os.path.join(paths.PLOTS,
save.get_module_name()),
_filename='plot_' + _test_name)
# granger versus correlation
print(
'GC vs. correlation pearson correlation:',
compute_lib.correlation(_granger_causality_p_values,
_correlations,
_with_p_value=True))
_fig = go.Figure(data=go.Scatter(x=_granger_causality_p_values,
y=_correlations,
mode='markers',
marker={
'size': 10,
'color': '#ea8500'
},
showlegend=False),
layout={
'xaxis': {
'title': 'Granger causality p-value',
'zeroline': False,
},
'yaxis': {
'title': 'Inner correlation',
'zeroline': False,
}
})
save.to_html(_fig=_fig,
_path=os.path.join(paths.PLOTS, save.get_module_name()),
_filename='plot_gc_vs_correlation')
# granger versus time lag correlation
print(
'GC vs. time lag correlation pearson correlation:',
compute_lib.correlation(_granger_causality_p_values,
_time_lag_correlations,
_with_p_value=True))
_fig = go.Figure(data=go.Scatter(x=_granger_causality_p_values,
y=_time_lag_correlations,
mode='markers',
marker={
'size': 10,
'color': '#ea8500'
},
showlegend=False),
layout={
'xaxis': {
'title': 'Granger causality p-value',
'zeroline': False,
},
'yaxis': {
'title': 'GC lag inner correlation',
'zeroline': False,
}
})
save.to_html(_fig=_fig,
_path=os.path.join(paths.PLOTS, save.get_module_name()),
_filename='plot_gc_vs_time_lag_correlation')
# granger versus end fiber density
print(
'GC vs. end fiber density pearson correlation:',
compute_lib.correlation(_granger_causality_p_values,
_end_fiber_densities,
_with_p_value=True))
_fig = go.Figure(data=go.Scatter(x=_granger_causality_p_values,
y=_end_fiber_densities,
mode='markers',
marker={
'size': 10,
'color': '#ea8500'
},
showlegend=False),
layout={
'xaxis': {
'title': 'Granger causality p-value',
'zeroline': False,
},
'yaxis': {
'title': 'End fiber density (z-score)',
'zeroline': False,
}
})
save.to_html(_fig=_fig,
_path=os.path.join(paths.PLOTS, save.get_module_name()),
_filename='plot_gc_vs_end_density')
