def main(_real_cells=True,
_static=False,
_band=True,
_high_temporal_resolution=False):
_z_array = np.zeros(shape=(len(BY), len(BY)))
for (_padding_index,
_padding_by), (_space_index,
_space_by) in product(enumerate(BY), enumerate(BY)):
print('Padding by: ', _padding_by, ', space by: ', _space_by)
_same_correlations_array, _different_correlations_array = \
same_inner_correlation_vs_different_inner_correlation.compute_fiber_densities(
_real_cells=_real_cells, _static=_static, _band=_band, _high_temporal_resolution=_high_temporal_resolution,
_padding_y_by=_padding_by, _padding_z_by=_padding_by, _space_y_by=_space_by, _space_z_by=_space_by
)
_same_minus_different = np.array(_same_correlations_array) - np.array(
_different_correlations_array)
_same_greater_than_different = (_same_minus_different >
0).sum() / len(_same_minus_different)
_z_array[_space_index, _padding_index] = _same_greater_than_different
# plot
_colors_array = ['black', 'white', config.colors(1)]
_fig = go.Figure(data=go.Heatmap(
x=BY,
y=BY,
z=_z_array,
colorscale=sns.color_palette(_colors_array).as_hex(),
colorbar={
'tickmode': 'array',
'tickvals': [0.5, 0.75, 1],
'ticktext': ['0.5', 'Same > different', '1'],
'tickangle': -90
},
showscale=True,
zmin=0.5,
zmax=1),
layout={
'xaxis': {
'title': 'Border size (cell diameter)',
'zeroline': False,
'tickmode': 'array',
'tickvals': [0, 0.5, 1, 1.5, 2]
},
'yaxis': {
'title': 'Space from window (cell diameter)',
'zeroline': False,
'tickmode': 'array',
'tickvals': [0, 0.5, 1, 1.5, 2]
}
})
save.to_html(_fig=_fig,
_path=os.path.join(paths.PLOTS, save.get_module_name()),
_filename='plot_real_' + str(_real_cells) + '_static_' +
str(_static) + '_band_' + str(_band) + '_high_time_' +
str(_high_temporal_resolution))
def main(_band=True, _high_temporal_resolution=False):
print('Computing fiber densities vs. offsets in axes:')
_fiber_densities_z_array = inner_density_vs_offsets_in_axes.compute_z_array(
_band=_band, _high_temporal_resolution=_high_temporal_resolution)
print('Computing fiber density changes vs. offsets in axes:')
_fiber_density_changes_z_array = inner_density_change_vs_offsets_in_axes.compute_z_array(
_band=_band, _high_temporal_resolution=_high_temporal_resolution)
_z_array = _fiber_density_changes_z_array / _fiber_densities_z_array
# plot
_offsets_y = np.arange(start=OFFSET_Y_START,
stop=OFFSET_Y_END + OFFSET_Y_STEP,
step=OFFSET_Y_STEP)
_offsets_z = np.arange(start=OFFSET_Z_START,
stop=OFFSET_Z_END + OFFSET_Z_STEP,
step=OFFSET_Z_STEP)
_colors_array = ['white', config.colors(1)]
_fig = go.Figure(data=go.Heatmap(
x=_offsets_z,
y=_offsets_y,
z=_z_array,
colorscale=sns.color_palette(_colors_array).as_hex(),
colorbar={
'tickmode': 'array',
'tickvals': [0, 0.25, 0.5],
'ticktext': ['0.0', 'Z-score change / z-score', '0.5'],
'tickangle': -90
},
showscale=True,
zmin=-0,
zmax=0.5),
layout={
'xaxis': {
'title': 'Offset in XY axis (cell diameter)',
'zeroline': False,
'tickmode': 'array',
'tickvals': [-4, -2, 0, 2, 4]
},
'yaxis': {
'title': 'Offset in Z axis (cell diameter)',
'zeroline': False,
'tickmode': 'array',
'tickvals': [-1, 0, 1, 2]
}
})
save.to_html(_fig=_fig,
_path=os.path.join(paths.PLOTS, save.get_module_name()),
_filename='plot_high_time_' + str(_high_temporal_resolution) +
'_band_' + str(_band))
def main(_high_temporal_resolution=False):
_y_arrays = [[], []]
for _band_index, _band in enumerate([True, False]):
print('Band:', _band)
_same_correlations_array, _different_correlations_array = \
same_inner_correlation_vs_different_inner_correlation.compute_fiber_densities(_band=_band,
_high_temporal_resolution=_high_temporal_resolution)
for _same, _different in zip(_same_correlations_array, _different_correlations_array):
_point_distance = compute_lib.distance_from_a_point_to_a_line(
_line=[-1, -1, 1, 1],
_point=[_same, _different]
)
if _same > _different:
_y_arrays[_band_index].append(_point_distance)
else:
_y_arrays[_band_index].append(-_point_distance)
# plot
_colors_array = config.colors(2)
_names_array = ['Band', 'No Band']
_fig = go.Figure(
data=[
go.Box(
y=_y_array,
name=_name,
boxpoints=False,
line={
'width': 1,
'color': _color
},
showlegend=False
) for _y_array, _name, _color in zip(_y_arrays, _names_array, _colors_array)
],
layout={
'xaxis': {
'zeroline': False
},
'yaxis': {
'title': 'Same minus different correlation',
'zeroline': False
}
}
)
save.to_html(
_fig=_fig,
_path=os.path.join(paths.PLOTS, save.get_module_name()),
_filename='plot_high_temporal_res_' + str(_high_temporal_resolution)
)
def main(_band=True,
_high_temporal_resolution=False,
_offset_x=OFFSET_X,
_offset_y_start=OFFSET_Y_START,
_offset_y_end=OFFSET_Y_END,
_offset_y_step=OFFSET_Y_STEP,
_offset_z_start=OFFSET_Z_START,
_offset_z_end=OFFSET_Z_END,
_offset_z_step=OFFSET_Z_STEP):
global _tuples, _experiments_fiber_densities, _z_array
compute_z_array(_band=_band,
_high_temporal_resolution=_high_temporal_resolution,
_offset_x=OFFSET_X,
_offset_y_start=OFFSET_Y_START,
_offset_y_end=OFFSET_Y_END,
_offset_y_step=OFFSET_Y_STEP,
_offset_z_start=OFFSET_Z_START,
_offset_z_end=OFFSET_Z_END,
_offset_z_step=OFFSET_Z_STEP)
# plot
_offsets_y = np.arange(start=_offset_y_start,
stop=_offset_y_end + _offset_y_step,
step=_offset_y_step)
_offsets_z = np.arange(start=_offset_z_start,
stop=_offset_z_end + _offset_z_step,
step=_offset_z_step)
_colors_array = ['white', config.colors(1)]
_fig = go.Figure(data=go.Heatmap(
x=_offsets_z,
y=_offsets_y,
z=_z_array,
colorscale=sns.color_palette(_colors_array).as_hex(),
colorbar={
'tickmode': 'array',
'tickvals': [0, 0.35, 0.7],
'ticktext': ['0.0', 'Z-score change', '0.7'],
'tickangle': -90
},
showscale=True,
zmin=0,
zmax=0.7),
layout={
'xaxis': {
'title': 'Offset in XY axis (cell diameter)',
'zeroline': False,
'tickmode': 'array',
'tickvals': [-4, -2, 0, 2, 4]
},
'yaxis': {
'title': 'Offset in Z axis (cell diameter)',
'zeroline': False,
'tickmode': 'array',
'tickvals': [-1, 0, 1, 2]
}
})
save.to_html(_fig=_fig,
_path=os.path.join(paths.PLOTS, save.get_module_name()),
_filename='plot_high_time_' + str(_high_temporal_resolution) +
'_band_' + str(_band))
def main():
_experiments = all_experiments()
_experiments = filtering.by_categories(_experiments=_experiments,
_is_single_cell=False,
_is_high_temporal_resolution=False,
_is_bleb=False,
_is_dead_dead=False,
_is_live_dead=False,
_is_bead=False,
_is_metastasis=False)
_tuples = load.experiments_groups_as_tuples(_experiments)
_tuples = filtering.by_time_frames_amount(
_tuples, compute.density_time_frame(_experiments[0]))
_tuples = filtering.by_real_pairs(_tuples)
_tuples = filtering.by_band(_tuples)
_tuples = filtering.by_pair_distance_range(_tuples, PAIR_DISTANCE_RANGE)
print('Total tuples:', len(_tuples))
_arguments = []
for _tuple in _tuples:
_experiment, _series_id, _group = _tuple
_latest_time_frame = compute.latest_time_frame_before_overlapping(
_experiment, _series_id, _group, OFFSET_X)
for _cell_id in ['left_cell', 'right_cell']:
_arguments.append({
'experiment': _experiment,
'series_id': _series_id,
'group': _group,
'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER,
'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER,
'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER,
'offset_x': OFFSET_X,
'offset_y': OFFSET_Y,
'offset_z': OFFSET_Z,
'cell_id': _cell_id,
'direction': 'inside',
'time_points': _latest_time_frame
})
_windows_dictionary, _windows_to_compute = \
compute.windows(_arguments, _keys=['experiment', 'series_id', 'group', 'cell_id'])
_fiber_densities = compute.fiber_densities(_windows_to_compute,
_subtract_border=True)
_experiments_fiber_densities = {
_key:
[_fiber_densities[_tuple] for _tuple in _windows_dictionary[_key]]
for _key in _windows_dictionary
}
_kpss_y_arrays = [[] for _i in DERIVATIVES]
_adf_y_arrays = [[] for _i in DERIVATIVES]
for _tuple in tqdm(_tuples, desc='Experiments loop'):
_experiment, _series_id, _group = _tuple
_properties = load.group_properties(_experiment, _series_id, _group)
_left_cell_fiber_densities = _experiments_fiber_densities[(
_experiment, _series_id, _group, 'left_cell')]
_left_cell_fiber_densities = compute.remove_blacklist(
_experiment, _series_id, _properties['cells_ids']['left_cell'],
_left_cell_fiber_densities)
_right_cell_fiber_densities = _experiments_fiber_densities[(
_experiment, _series_id, _group, 'right_cell')]
_right_cell_fiber_densities = compute.remove_blacklist(
_experiment, _series_id, _properties['cells_ids']['right_cell'],
_right_cell_fiber_densities)
if not OUT_OF_BOUNDARIES:
_left_cell_fiber_densities = \
compute.longest_fiber_densities_ascending_sequence(_left_cell_fiber_densities)
_right_cell_fiber_densities = \
compute.longest_fiber_densities_ascending_sequence(_right_cell_fiber_densities)
else:
_left_cell_fiber_densities = [
_fiber_density[0]
for _fiber_density in _left_cell_fiber_densities
]
_right_cell_fiber_densities = [
_fiber_density[0]
for _fiber_density in _right_cell_fiber_densities
]
# ignore small arrays
_minimum_time_frames_for_correlation = compute.minimum_time_frames_for_correlation(
_experiment)
if len(_left_cell_fiber_densities) < _minimum_time_frames_for_correlation or \
len(_right_cell_fiber_densities) < _minimum_time_frames_for_correlation:
continue
for _derivative_index, _derivative in enumerate(DERIVATIVES):
for _cell_fiber_densities in [
_left_cell_fiber_densities, _right_cell_fiber_densities
]:
_cell_fiber_densities_derivative = compute_lib.derivative(
_cell_fiber_densities, _n=_derivative)
_, _kpss_p_value, _, _ = kpss(_cell_fiber_densities_derivative,
nlags='legacy')
_kpss_y_arrays[_derivative_index].append(_kpss_p_value)
_, _adf_p_value, _, _, _, _ = adfuller(
_cell_fiber_densities_derivative)
_adf_y_arrays[_derivative_index].append(_adf_p_value)
print('Total pairs:', len(_kpss_y_arrays[0]) / 2)
# print results
print('KPSS:')
for _derivative_index, _derivative in enumerate(DERIVATIVES):
_stationary_count = len([
_value for _value in _kpss_y_arrays[_derivative_index]
if _value > 0.05
])
print(
'Derivative:', _derivative, 'Stationary:',
str(_stationary_count / len(_kpss_y_arrays[_derivative_index]) *
100) + '%')
print('ADF:')
for _derivative_index, _derivative in enumerate(DERIVATIVES):
_stationary_count = len([
_value for _value in _adf_y_arrays[_derivative_index]
if _value < 0.05
])
print(
'Derivative:', _derivative, 'Stationary:',
str(_stationary_count / len(_adf_y_arrays[_derivative_index]) *
100) + '%')
# plot
_colors_array = config.colors(3)
for _test_name, _y_title, _y_tickvals, _p_value_line, _y_arrays in \
zip(
['kpss', 'adf'],
['KPSS test p-value', 'ADF test p-value'],
[[0.05, 0.1], [0.05, 1]],
[0.05, 0.05],
[_kpss_y_arrays, _adf_y_arrays]
):
_fig = go.Figure(data=[
go.Box(y=_y,
name=_derivative,
boxpoints='all',
jitter=1,
pointpos=0,
line={'width': 1},
fillcolor='white',
marker={
'size': 10,
'color': _color
},
opacity=0.7,
showlegend=False) for _y, _derivative, _color in zip(
_y_arrays, DERIVATIVES_TEXT, _colors_array)
],
layout={
'xaxis': {
'title': 'Fiber density derivative',
'zeroline': False
},
'yaxis': {
'title': _y_title,
'zeroline': False,
'tickmode': 'array',
'tickvals': _y_tickvals
},
'shapes': [{
'type': 'line',
'x0': DERIVATIVES[0] - 0.75,
'y0': _p_value_line,
'x1': DERIVATIVES[-1] + 0.75,
'y1': _p_value_line,
'line': {
'color': 'red',
'width': 2,
'dash': 'dash'
}
}]
})
save.to_html(_fig=_fig,
_path=os.path.join(paths.PLOTS, save.get_module_name()),
_filename='plot_' + _test_name)
def main():
print('Single cells')
_single_cells_fiber_densities = inner_density_vs_window_distance_single_cells.compute_experiments_data()
print('Cell pairs')
_names_array, _x_array, _y_array = inner_density_vs_window_distance.compute_cell_pairs()
# plot
_colors_array = config.colors(len(_names_array) + 1)
_fig = go.Figure(
data=[
go.Scatter(
x=_x,
y=[np.mean(_array) for _array in _y],
name=_name,
error_y={
'type': 'data',
'array': [np.std(_array) for _array in _y],
'thickness': 1,
'color': _color
},
mode='markers',
marker={
'size': 15,
'color': _color
},
opacity=0.7
) for _x, _y, _name, _color in zip(_x_array, _y_array, _names_array, _colors_array[:-1])
] + [
go.Scatter(
x=inner_density_vs_window_distance_single_cells.OFFSETS_X,
y=[np.mean(_array) for _array in _single_cells_fiber_densities],
name='Single cells',
error_y={
'type': 'data',
'array': [np.std(_array) for _array in _single_cells_fiber_densities],
'thickness': 1,
'color': _colors_array[-1]
},
mode='markers',
marker={
'size': 15,
'color': _colors_array[-1]
},
opacity=0.7
)
],
layout={
'xaxis': {
'title': 'Window distance (cell diameter)',
'zeroline': False
},
'yaxis': {
'title': 'Fiber density (z-score)',
'range': [-1.7, 13],
'zeroline': False,
'tickmode': 'array',
'tickvals': [0, 4, 8, 12]
},
'legend': {
'xanchor': 'right',
'yanchor': 'top',
'bordercolor': 'black',
'borderwidth': 2
},
'shapes': [
{
'type': 'line',
'x0': -0.2,
'y0': -1.5,
'x1': 3.4,
'y1': -1.5,
'line': {
'color': 'black',
'width': 2
}
},
{
'type': 'line',
'x0': -0.2,
'y0': -1.5,
'x1': -0.2,
'y1': 13,
'line': {
'color': 'black',
'width': 2
}
}
]
}
)
save.to_html(
_fig=_fig,
_path=os.path.join(paths.PLOTS, save.get_module_name()),
_filename='plot'
)
def main():
_experiments = all_experiments()
_experiments = filtering.by_categories(
_experiments=_experiments,
_is_single_cell=False,
_is_high_temporal_resolution=False,
_is_bleb=False,
_is_dead_dead=False,
_is_live_dead=False,
_is_bead=False,
_is_metastasis=False
)
_tuples = load.experiments_groups_as_tuples(_experiments)
_tuples = filtering.by_real_pairs(_tuples)
_tuples = filtering.by_band(_tuples)
_tuples = filtering.by_time_frames_amount(_tuples, MINIMUM_TIME_FRAMES)
_tuples = filtering.by_pair_distance_range(_tuples, [MINIMUM_PAIR_DISTANCE, sys.maxsize])
_triplets = filtering.by_triplets(_tuples)
print('Total triplets:', len(_triplets))
_arguments = []
for _triplet in _triplets:
for _tuple in _triplet:
_experiment, _series_id, _group = _tuple
_latest_time_frame = compute.latest_time_frame_before_overlapping(_experiment, _series_id, _group, OFFSET_X)
for _cell_id in ['left_cell', 'right_cell']:
_arguments.append({
'experiment': _experiment,
'series_id': _series_id,
'group': _group,
'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER,
'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER,
'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER,
'offset_x': OFFSET_X,
'offset_y': OFFSET_Y,
'offset_z': OFFSET_Z,
'cell_id': _cell_id,
'direction': 'inside',
'time_points': _latest_time_frame
})
_windows_dictionary, _windows_to_compute = compute.windows(_arguments,
_keys=['experiment', 'series_id', 'group', 'cell_id'])
_fiber_densities = compute.fiber_densities(_windows_to_compute, _subtract_border=True)
_experiments_fiber_densities = {
_key: [_fiber_densities[_tuple] for _tuple in _windows_dictionary[_key]]
for _key in _windows_dictionary
}
_same_correlations_arrays = [[] for _i in _triplets]
_different_correlations_arrays = [[] for _i in _triplets]
_names_array = []
for _triplet_index, _triplet in enumerate(_triplets):
for _same_index in tqdm(range(len(_triplet)), desc='Main loop'):
_same_tuple = _triplet[_same_index]
_same_experiment, _same_series, _same_group = _same_tuple
_same_left_cell_fiber_densities = \
_experiments_fiber_densities[
(_same_experiment, _same_series, _same_group, 'left_cell')
]
_same_right_cell_fiber_densities = \
_experiments_fiber_densities[
(_same_experiment, _same_series, _same_group, 'right_cell')
]
_same_properties = \
load.group_properties(_same_experiment, _same_series, _same_group)
_same_left_cell_fiber_densities = compute.remove_blacklist(
_same_experiment,
_same_series,
_same_properties['cells_ids']['left_cell'],
_same_left_cell_fiber_densities
)
_same_right_cell_fiber_densities = compute.remove_blacklist(
_same_experiment,
_same_series,
_same_properties['cells_ids']['right_cell'],
_same_right_cell_fiber_densities
)
_same_left_cell_fiber_densities_filtered, _same_right_cell_fiber_densities_filtered = \
compute.longest_same_indices_shared_in_borders_sub_array(
_same_left_cell_fiber_densities, _same_right_cell_fiber_densities
)
_same_correlation = compute_lib.correlation(
compute_lib.derivative(_same_left_cell_fiber_densities_filtered, _n=DERIVATIVE),
compute_lib.derivative(_same_right_cell_fiber_densities_filtered, _n=DERIVATIVE)
)
for _different_index in range(len(_triplet)):
if _same_index != _different_index:
_different_tuple = _triplet[_different_index]
_different_experiment, _different_series, _different_group = \
_different_tuple
for _same_cell_id, _different_cell_id in product(['left_cell', 'right_cell'],
['left_cell', 'right_cell']):
_same_fiber_densities = _experiments_fiber_densities[(
_same_experiment,
_same_series,
_same_group,
_same_cell_id
)]
_different_fiber_densities = _experiments_fiber_densities[(
_different_experiment,
_different_series,
_different_group,
_different_cell_id
)]
_different_properties = load.group_properties(
_different_experiment, _different_series, _different_group
)
_same_fiber_densities = compute.remove_blacklist(
_same_experiment,
_same_series,
_same_properties['cells_ids'][_same_cell_id],
_same_fiber_densities
)
_different_fiber_densities = compute.remove_blacklist(
_different_experiment,
_different_series,
_different_properties['cells_ids'][_different_cell_id],
_different_fiber_densities
)
_same_fiber_densities_filtered, _different_fiber_densities_filtered = \
compute.longest_same_indices_shared_in_borders_sub_array(
_same_fiber_densities, _different_fiber_densities
)
_different_correlation = compute_lib.correlation(
compute_lib.derivative(_same_fiber_densities_filtered, _n=DERIVATIVE),
compute_lib.derivative(_different_fiber_densities_filtered, _n=DERIVATIVE)
)
_same_correlations_arrays[_triplet_index].append(_same_correlation)
_different_correlations_arrays[_triplet_index].append(_different_correlation)
_names_array.append('Triplet #' + str(_triplet_index + 1))
print('Total points:', len(np.array(_same_correlations_arrays).flatten()))
_same_minus_different = \
np.array(_same_correlations_arrays).flatten() - np.array(_different_correlations_arrays).flatten()
print('Wilcoxon of same minus different around the zero:')
print(wilcoxon(_same_minus_different))
print('Higher same amount:', (_same_minus_different > 0).sum() /
len(_same_minus_different))
# plot
_colors_array = ['green', 'blue', config.colors(1)]
_fig = go.Figure(
data=[
go.Scatter(
x=_same_correlations_array,
y=_different_correlations_array,
name=_name,
mode='markers',
marker={
'size': 15,
'color': _color
},
opacity=0.7
) for _same_correlations_array, _different_correlations_array, _name, _color in
zip(_same_correlations_arrays, _different_correlations_arrays, _names_array, _colors_array)
],
layout={
'xaxis': {
'title': 'Same network correlation',
'zeroline': False,
'range': [-1.1, 1.2],
'tickmode': 'array',
'tickvals': [-1, -0.5, 0, 0.5, 1]
},
'yaxis': {
'title': 'Different network correlation',
'zeroline': False,
'range': [-1.1, 1.2],
'tickmode': 'array',
'tickvals': [-1, -0.5, 0, 0.5, 1]
},
'legend': {
'xanchor': 'left',
'x': 0.1,
'yanchor': 'top',
'bordercolor': 'black',
'borderwidth': 2,
'bgcolor': 'white'
},
'shapes': [
{
'type': 'line',
'x0': -1,
'y0': -1,
'x1': -1,
'y1': 1,
'line': {
'color': 'black',
'width': 2
}
},
{
'type': 'line',
'x0': -1,
'y0': -1,
'x1': 1,
'y1': -1,
'line': {
'color': 'black',
'width': 2
}
},
{
'type': 'line',
'x0': -1,
'y0': -1,
'x1': 1,
'y1': 1,
'line': {
'color': 'red',
'width': 2
}
}
]
}
)
save.to_html(
_fig=_fig,
_path=os.path.join(paths.PLOTS, save.get_module_name()),
_filename='plot'
)
def main(_offset_y=0.5, _high_temporal_resolution=False):
_distances_from_y_equal_x = compute_fiber_densities(
_offset_y, _high_temporal_resolution)
print('Total points:', len(_distances_from_y_equal_x[0]))
print('Higher real same amount:',
(np.array(_distances_from_y_equal_x[0]) > 0).sum() /
len(_distances_from_y_equal_x[0]))
print('Wilcoxon of real points:', wilcoxon(_distances_from_y_equal_x[0]))
print('Higher fake same amount:',
(np.array(_distances_from_y_equal_x[1]) > 0).sum() /
len(_distances_from_y_equal_x[1]))
print('Wilcoxon of fake points:', wilcoxon(_distances_from_y_equal_x[1]))
_real_minus_fake = np.array(_distances_from_y_equal_x[0]) - np.array(
_distances_from_y_equal_x[1])
print('Real > fake amount:',
(_real_minus_fake > 0).sum() / len(_real_minus_fake))
print('Wilcoxon real & fake:',
wilcoxon(_distances_from_y_equal_x[0], _distances_from_y_equal_x[1]))
# box plot
_colors_array = config.colors(2)
_names_array = ['Real', 'Fake']
_fig = go.Figure(data=[
go.Box(y=_y_array,
name=_name,
boxpoints=False,
line={'width': 1},
marker={'color': _color},
showlegend=False) for _y_array, _name, _color in zip(
_distances_from_y_equal_x, _names_array, _colors_array)
],
layout={
'xaxis': {
'zeroline': False
},
'yaxis': {
'title': 'Same minus different correlation',
'zeroline': False,
'range': [-1, 1.1],
'tickmode': 'array',
'tickvals': [-1, -0.5, 0, 0.5, 1]
}
})
save.to_html(_fig=_fig,
_path=os.path.join(paths.PLOTS, save.get_module_name()),
_filename='plot_box_high_temporal_res_' +
str(_high_temporal_resolution) + '_offset_y_' +
str(_offset_y))
# scatter plot
_fig = go.Figure(data=go.Scatter(x=_distances_from_y_equal_x[0],
y=_distances_from_y_equal_x[1],
mode='markers',
marker={
'size': 5,
'color': '#ea8500'
},
showlegend=False),
layout={
'xaxis': {
'title': 'Real pair',
'zeroline': False,
'range': [-1.1, 1.2],
'tickmode': 'array',
'tickvals': [-1, -0.5, 0, 0.5, 1]
},
'yaxis': {
'title': 'Fake pair',
'zeroline': False,
'range': [-1.1, 1.2],
'tickmode': 'array',
'tickvals': [-1, -0.5, 0, 0.5, 1]
},
'shapes': [{
'type': 'line',
'x0': -1,
'y0': -1,
'x1': -1,
'y1': 1,
'line': {
'color': 'black',
'width': 2
}
}, {
'type': 'line',
'x0': -1,
'y0': -1,
'x1': 1,
'y1': -1,
'line': {
'color': 'black',
'width': 2
}
}, {
'type': 'line',
'x0': -1,
'y0': -1,
'x1': 1,
'y1': 1,
'line': {
'color': 'red',
'width': 2
}
}]
})
save.to_html(_fig=_fig,
_path=os.path.join(paths.PLOTS, save.get_module_name()),
_filename='plot_high_temporal_res_' +
str(_high_temporal_resolution) + '_offset_y_' +
str(_offset_y))
def main(_band=True):
_experiments = all_experiments()
_experiments = filtering.by_categories(_experiments=_experiments,
_is_single_cell=False,
_is_high_temporal_resolution=None,
_is_bleb=True,
_is_dead_dead=False,
_is_live_dead=False,
_is_bead=False,
_is_metastasis=False)
_tuples = load.experiments_groups_as_tuples(_experiments)
_tuples = filtering.by_pair_distance_range(
_tuples, _distance_range=PAIR_DISTANCE_RANGE)
_tuples = filtering.by_real_pairs(_tuples, _real_pairs=REAL_CELLS)
_tuples = filtering.by_fake_static_pairs(_tuples,
_fake_static_pairs=STATIC)
_tuples = filtering.by_band(_tuples, _band=_band)
_tuples = filtering.by_bleb_from_start(_tuples, _from_start=False)
print('Total tuples:', len(_tuples))
_arguments = []
for _tuple in _tuples:
_experiment, _series_id, _group = _tuple
_latest_time_frame = compute.latest_time_frame_before_overlapping(
_experiment, _series_id, _group, OFFSET_X)
for _cell_id in ['left_cell', 'right_cell']:
_arguments.append({
'experiment': _experiment,
'series_id': _series_id,
'group': _group,
'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER,
'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER,
'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER,
'offset_x': OFFSET_X,
'offset_y': OFFSET_Y,
'offset_z': OFFSET_Z,
'cell_id': _cell_id,
'direction': 'inside',
'time_points': _latest_time_frame
})
_windows_dictionary, _windows_to_compute = compute.windows(
_arguments, _keys=['experiment', 'series_id', 'group', 'cell_id'])
_fiber_densities = compute.fiber_densities(_windows_to_compute,
_subtract_border=True)
_experiments_fiber_densities = {
_key:
[_fiber_densities[_tuple] for _tuple in _windows_dictionary[_key]]
for _key in _windows_dictionary
}
_tuples_by_experiment = organize.by_experiment(_tuples)
# same (before, after), different (before, after)
_correlations = [[[], []], [[], []]]
_valid_real_tuples = []
for _experiment in _tuples_by_experiment:
print('Experiment:', _experiment)
_experiment_tuples = _tuples_by_experiment[_experiment]
for _same_index in tqdm(range(len(_experiment_tuples)),
desc='Main loop'):
_same_tuple = _experiment_tuples[_same_index]
_same_experiment, _same_series, _same_group = _same_tuple
_same_left_cell_fiber_densities = \
_experiments_fiber_densities[
(_same_experiment, _same_series, _same_group, 'left_cell')
]
_same_right_cell_fiber_densities = \
_experiments_fiber_densities[
(_same_experiment, _same_series, _same_group, 'right_cell')
]
_same_properties = \
load.group_properties(_same_experiment, _same_series, _same_group)
_same_left_cell_fiber_densities = compute.remove_blacklist(
_same_experiment, _same_series,
_same_properties['cells_ids']['left_cell'],
_same_left_cell_fiber_densities)
_same_right_cell_fiber_densities = compute.remove_blacklist(
_same_experiment, _same_series,
_same_properties['cells_ids']['right_cell'],
_same_right_cell_fiber_densities)
_same_before_left_cell_fiber_densities = \
_same_left_cell_fiber_densities[:AFTER_BLEB_INJECTION_FIRST_TIME_FRAME[_same_experiment]]
_same_before_right_cell_fiber_densities = \
_same_right_cell_fiber_densities[:AFTER_BLEB_INJECTION_FIRST_TIME_FRAME[_same_experiment]]
_same_after_left_cell_fiber_densities = \
_same_left_cell_fiber_densities[AFTER_BLEB_INJECTION_FIRST_TIME_FRAME[_same_experiment]:]
_same_after_right_cell_fiber_densities = \
_same_right_cell_fiber_densities[AFTER_BLEB_INJECTION_FIRST_TIME_FRAME[_same_experiment]:]
_same_before_left_cell_fiber_densities_filtered, _same_before_right_cell_fiber_densities_filtered = \
compute.longest_same_indices_shared_in_borders_sub_array(
_same_before_left_cell_fiber_densities, _same_before_right_cell_fiber_densities
)
_same_after_left_cell_fiber_densities_filtered, _same_after_right_cell_fiber_densities_filtered = \
compute.longest_same_indices_shared_in_borders_sub_array(
_same_after_left_cell_fiber_densities, _same_after_right_cell_fiber_densities
)
# ignore small arrays
_minimum_time_frame_for_correlation = compute.minimum_time_frames_for_correlation(
_same_experiment)
if len(_same_before_left_cell_fiber_densities_filtered) < _minimum_time_frame_for_correlation or \
len(_same_after_left_cell_fiber_densities_filtered) < _minimum_time_frame_for_correlation:
continue
_same_before_correlation = compute_lib.correlation(
compute_lib.derivative(
_same_before_left_cell_fiber_densities_filtered,
_n=DERIVATIVE),
compute_lib.derivative(
_same_before_right_cell_fiber_densities_filtered,
_n=DERIVATIVE))
_same_after_correlation = compute_lib.correlation(
compute_lib.derivative(
_same_after_left_cell_fiber_densities_filtered,
_n=DERIVATIVE),
compute_lib.derivative(
_same_after_right_cell_fiber_densities_filtered,
_n=DERIVATIVE))
for _different_index in range(len(_experiment_tuples)):
if _same_index != _different_index:
_different_tuple = _experiment_tuples[_different_index]
_different_experiment, _different_series, _different_group = _different_tuple
for _same_cell_id, _different_cell_id in product(
['left_cell', 'right_cell'],
['left_cell', 'right_cell']):
_same_fiber_densities = _experiments_fiber_densities[(
_same_experiment, _same_series, _same_group,
_same_cell_id)]
_different_fiber_densities = _experiments_fiber_densities[
(_different_experiment, _different_series,
_different_group, _different_cell_id)]
_different_properties = load.group_properties(
_different_experiment, _different_series,
_different_group)
_same_fiber_densities = compute.remove_blacklist(
_same_experiment, _same_series,
_same_properties['cells_ids'][_same_cell_id],
_same_fiber_densities)
_different_fiber_densities = compute.remove_blacklist(
_different_experiment, _different_series,
_different_properties['cells_ids']
[_different_cell_id], _different_fiber_densities)
_same_before_fiber_densities = \
_same_fiber_densities[:AFTER_BLEB_INJECTION_FIRST_TIME_FRAME[_same_experiment]]
_same_after_fiber_densities = \
_same_fiber_densities[AFTER_BLEB_INJECTION_FIRST_TIME_FRAME[_same_experiment]:]
_different_before_fiber_densities = \
_different_fiber_densities[:AFTER_BLEB_INJECTION_FIRST_TIME_FRAME[_different_experiment]]
_different_after_fiber_densities = \
_different_fiber_densities[AFTER_BLEB_INJECTION_FIRST_TIME_FRAME[_different_experiment]:]
_same_before_fiber_densities_filtered, _different_before_fiber_densities_filtered = \
compute.longest_same_indices_shared_in_borders_sub_array(
_same_before_fiber_densities, _different_before_fiber_densities
)
_same_after_fiber_densities_filtered, _different_after_fiber_densities_filtered = \
compute.longest_same_indices_shared_in_borders_sub_array(
_same_after_fiber_densities, _different_after_fiber_densities
)
# ignore small arrays
if len(_same_before_fiber_densities_filtered) < _minimum_time_frame_for_correlation or \
len(_same_after_fiber_densities_filtered) < _minimum_time_frame_for_correlation:
continue
_different_before_correlation = compute_lib.correlation(
compute_lib.derivative(
_same_before_fiber_densities_filtered,
_n=DERIVATIVE),
compute_lib.derivative(
_different_before_fiber_densities_filtered,
_n=DERIVATIVE))
_different_after_correlation = compute_lib.correlation(
compute_lib.derivative(
_same_after_fiber_densities_filtered,
_n=DERIVATIVE),
compute_lib.derivative(
_different_after_fiber_densities_filtered,
_n=DERIVATIVE))
_correlations[0][0].append(_same_before_correlation)
_correlations[0][1].append(_same_after_correlation)
_correlations[1][0].append(
_different_before_correlation)
_correlations[1][1].append(
_different_after_correlation)
if _same_tuple not in _valid_real_tuples:
_valid_real_tuples.append(_same_tuple)
print('Total tuples:', len(_valid_real_tuples))
_distances_from_y_equal_x = [[], []]
_same_correlations, _different_correlations = _correlations
_same_before_correlations, _same_after_correlations = _same_correlations
_different_before_correlations, _different_after_correlations = _different_correlations
for _same_before, _same_after, _different_before, _different_after in \
zip(_same_before_correlations, _same_after_correlations,
_different_before_correlations, _different_after_correlations):
for _group_type_index, _same, _different in \
zip([0, 1], [_same_before, _same_after], [_different_before, _different_after]):
_point_distance = compute_lib.distance_from_a_point_to_a_line(
_line=[-1, -1, 1, 1], _point=[_same, _different])
if _same > _different:
_distances_from_y_equal_x[_group_type_index].append(
_point_distance)
else:
_distances_from_y_equal_x[_group_type_index].append(
-_point_distance)
print('Total points:', len(_distances_from_y_equal_x[0]))
print('Higher before same amount:',
(np.array(_distances_from_y_equal_x[0]) > 0).sum() /
len(_distances_from_y_equal_x[0]))
print('Wilcoxon of before points:', wilcoxon(_distances_from_y_equal_x[0]))
print('Higher after same amount:',
(np.array(_distances_from_y_equal_x[1]) > 0).sum() /
len(_distances_from_y_equal_x[1]))
print('Wilcoxon of after points:', wilcoxon(_distances_from_y_equal_x[1]))
_before_minus_after = np.array(_distances_from_y_equal_x[0]) - np.array(
_distances_from_y_equal_x[1])
print('Before > after amount:',
(_before_minus_after > 0).sum() / len(_before_minus_after))
print('Wilcoxon before & after:',
wilcoxon(_distances_from_y_equal_x[0], _distances_from_y_equal_x[1]))
# box plot
_colors_array = config.colors(2)
_names_array = ['Before', 'After']
_fig = go.Figure(data=[
go.Box(y=_y_array,
name=_name,
boxpoints=False,
line={'width': 1},
marker={'color': _color},
showlegend=False) for _y_array, _name, _color in zip(
_distances_from_y_equal_x, _names_array, _colors_array)
],
layout={
'xaxis': {
'zeroline': False
},
'yaxis': {
'title': 'Same minus different correlation',
'zeroline': False,
'range': [-1, 1.1],
'tickmode': 'array',
'tickvals': [-1, -0.5, 0, 0.5, 1]
}
})
save.to_html(_fig=_fig,
_path=os.path.join(paths.PLOTS, save.get_module_name()),
_filename='plot_box')
# scatter plot
_fig = go.Figure(data=go.Scatter(x=_distances_from_y_equal_x[0],
y=_distances_from_y_equal_x[1],
mode='markers',
marker={
'size': 5,
'color': '#ea8500'
},
showlegend=False),
layout={
'xaxis': {
'title': 'Before bleb',
'zeroline': False,
'range': [-1.1, 1.2],
'tickmode': 'array',
'tickvals': [-1, -0.5, 0, 0.5, 1]
},
'yaxis': {
'title': 'After bleb',
'zeroline': False,
'range': [-1.1, 1.2],
'tickmode': 'array',
'tickvals': [-1, -0.5, 0, 0.5, 1]
},
'shapes': [{
'type': 'line',
'x0': -1,
'y0': -1,
'x1': -1,
'y1': 1,
'line': {
'color': 'black',
'width': 2
}
}, {
'type': 'line',
'x0': -1,
'y0': -1,
'x1': 1,
'y1': -1,
'line': {
'color': 'black',
'width': 2
}
}, {
'type': 'line',
'x0': -1,
'y0': -1,
'x1': 1,
'y1': 1,
'line': {
'color': 'red',
'width': 2
}
}]
})
save.to_html(_fig=_fig,
_path=os.path.join(paths.PLOTS, save.get_module_name()),
_filename='plot')
def compute_tuples(_tuples):
_arguments = []
for _tuple in _tuples:
_experiment, _series_id, _group = _tuple
# stop when windows are overlapping
_properties = load.group_properties(_experiment, _series_id, _group)
_latest_time_frame = len(_properties['time_points'])
_middle_offsets_x = []
for _time_frame in range(len(_properties['time_points'])):
_pair_distance = \
compute.pair_distance_in_cell_size_time_frame(_experiment, _series_id, _group, _time_frame)
if _time_frame == 0:
print(_tuple, 'Pair distance:', _pair_distance)
_middle_offsets_x.append(
(_time_frame, (_pair_distance - 1) / 2 -
QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER / 2))
if _pair_distance - 1 - OFFSET_X * 2 < QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER * 2:
_latest_time_frame = _time_frame - 1
break
for _cell_id in ['left_cell', 'right_cell']:
_arguments.append({
'experiment': _experiment,
'series_id': _series_id,
'group': _group,
'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER,
'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER,
'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER,
'offset_x': OFFSET_X,
'offset_y': OFFSET_Y,
'offset_z': OFFSET_Z,
'cell_id': _cell_id,
'direction': 'inside',
'time_points': _latest_time_frame,
'middle_time_frame': -1
})
# middle one
for (_time_frame, _offset_x) in _middle_offsets_x:
_arguments.append({
'experiment': _experiment,
'series_id': _series_id,
'group': _group,
'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER,
'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER,
'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER,
'offset_x': _offset_x,
'offset_y': OFFSET_Y,
'offset_z': OFFSET_Z,
'cell_id': 'left_cell',
'direction': 'inside',
'time_point': _time_frame,
'middle_time_frame': _time_frame
})
_windows_dictionary, _windows_to_compute = \
compute.windows(_arguments, _keys=['experiment', 'series_id', 'group', 'cell_id', 'middle_time_frame'])
_fiber_densities = compute.fiber_densities(_windows_to_compute)
_experiments_fiber_densities = {
_key:
[_fiber_densities[_tuple] for _tuple in _windows_dictionary[_key]]
for _key in _windows_dictionary
}
_tuples_data = []
for _tuple in tqdm(_tuples, desc='Main loop'):
_experiment, _series_id, _group = _tuple
_left_cell_fiber_densities = \
_experiments_fiber_densities[(_experiment, _series_id, _group, 'left_cell', -1)]
_right_cell_fiber_densities = \
_experiments_fiber_densities[(_experiment, _series_id, _group, 'right_cell', -1)]
_middle_fiber_densities = \
[_experiments_fiber_densities[(_experiment, _series_id, _group, 'left_cell', _time_frame)][0]
for _time_frame in range(0, len(_left_cell_fiber_densities))]
_properties = load.group_properties(_experiment, _series_id, _group)
_left_cell_fiber_densities = compute.remove_blacklist(
_experiment, _series_id, _properties['cells_ids']['left_cell'],
_left_cell_fiber_densities)
_right_cell_fiber_densities = compute.remove_blacklist(
_experiment, _series_id, _properties['cells_ids']['right_cell'],
_right_cell_fiber_densities)
_left_cell_fiber_densities_filtered, _right_cell_fiber_densities_filtered = \
compute.longest_same_indices_shared_in_borders_sub_array(
_left_cell_fiber_densities, _right_cell_fiber_densities)
if len(_left_cell_fiber_densities_filtered) == 0:
continue
_start_time_frame = 0
for _left in _left_cell_fiber_densities:
if _left[0] == _left_cell_fiber_densities_filtered[0]:
break
_start_time_frame += 1
_middle_fiber_densities_filtered = \
[_fiber_density[0] for _fiber_density in _middle_fiber_densities]
_middle_fiber_densities_filtered = \
_middle_fiber_densities_filtered[_start_time_frame:
_start_time_frame + len(_left_cell_fiber_densities_filtered)]
_normalization = load.normalization_series_file_data(
_experiment, _series_id)
_left_cell_fiber_densities_normalized = compute_lib.z_score_array(
_array=_left_cell_fiber_densities_filtered,
_average=_normalization['average'],
_std=_normalization['std'])
_right_cell_fiber_densities_normalized = compute_lib.z_score_array(
_array=_right_cell_fiber_densities_filtered,
_average=_normalization['average'],
_std=_normalization['std'])
_middle_fiber_densities_normalized = compute_lib.z_score_array(
_array=_middle_fiber_densities_filtered,
_average=_normalization['average'],
_std=_normalization['std'])
_left_cell_fiber_densities_normalized = \
compute_lib.derivative(_left_cell_fiber_densities_normalized, _n=DERIVATIVE)
_right_cell_fiber_densities_normalized = \
compute_lib.derivative(_right_cell_fiber_densities_normalized, _n=DERIVATIVE)
_middle_fiber_densities_normalized = \
compute_lib.derivative(_middle_fiber_densities_normalized, _n=DERIVATIVE)
_correlation = \
compute_lib.correlation(
compute_lib.derivative(_left_cell_fiber_densities_normalized, _n=1),
compute_lib.derivative(_right_cell_fiber_densities_normalized, _n=1)
)
_tuples_data.append([
_tuple, _start_time_frame,
(_left_cell_fiber_densities_normalized,
_right_cell_fiber_densities_normalized,
_middle_fiber_densities_normalized), _correlation
])
# plots
_names_array = ['Left cell', 'Right cell', 'Middle']
_colors_array = config.colors(3)
for _tuple_data in _tuples_data:
_tuple, _start_time_frame, _y_arrays, _correlation = _tuple_data
_fig = go.Figure(data=[
go.Scatter(x=np.arange(
start=_start_time_frame,
stop=_start_time_frame + len(_y_arrays[0]) - DERIVATIVE,
step=1) * TEMPORAL_RESOLUTION,
y=_y,
name=_name,
mode='lines',
line={'color': _color}) for _y, _name, _color in zip(
_y_arrays, _names_array, _colors_array)
],
layout={
'xaxis': {
'title': 'Time (minutes)',
'zeroline': False
},
'yaxis': {
'title': 'Fiber density (z-score)',
'zeroline': False,
'range': [-3, 8],
'tickmode': 'array',
'tickvals': [-3, 0, 3, 6]
},
'legend': {
'xanchor': 'left',
'x': 0.1,
'yanchor': 'top',
'bordercolor': 'black',
'borderwidth': 2,
'bgcolor': 'white'
},
})
_experiment, _series_id, _group = _tuple
save.to_html(_fig=_fig,
_path=os.path.join(paths.PLOTS, save.get_module_name()),
_filename='plot_' + _experiment + '_' + str(_series_id) +
'_' + _group)
print('Tuple:', _tuple, 'Correlation:', _correlation)
def main():
_experiments = all_experiments()
_experiments = filtering.by_categories(_experiments=_experiments,
_is_single_cell=True,
_is_high_temporal_resolution=False,
_is_bleb=False,
_is_dead_dead=False,
_is_live_dead=False,
_is_bead=False,
_is_metastasis=False)
_tuples = load.experiments_groups_as_tuples(_experiments)
_tuples = filtering.by_time_frames_amount(
_tuples, compute.density_time_frame(_experiments[0]))
_tuples = filtering.by_main_cell(_tuples)
_arguments = []
for _tuple in _tuples:
_experiment, _series_id, _group = _tuple
_time_frame = compute.density_time_frame(_experiment)
for _direction in ['left', 'right']:
_arguments.append({
'experiment': _experiment,
'series_id': _series_id,
'group': _group,
'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER,
'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER,
'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER,
'offset_x': OFFSET_X,
'offset_y': OFFSET_Y,
'offset_z': OFFSET_Z,
'cell_id': 'cell',
'direction': _direction,
'time_points': _time_frame
})
_windows_dictionary, _windows_to_compute = \
compute.windows(_arguments, _keys=['experiment', 'series_id', 'group', 'direction'])
_fiber_densities = compute.fiber_densities(_windows_to_compute,
_subtract_border=True)
_tuples = organize.by_single_cell_id(_tuples)
print('Total tuples:', len(_tuples))
_experiments_ids = list(_tuples.keys())
_y_arrays = [[] for _i in DERIVATIVES]
for _index_1 in tqdm(range(len(_experiments_ids)), desc='Main loop'):
_tuple_1 = _experiments_ids[_index_1]
_experiment_1, _series_id_1, _cell_id_1 = _tuple_1
_fiber_densities_1 = compute_single_cell_mean(
_experiment=_experiment_1,
_series_id=_series_id_1,
_cell_tuples=_tuples[_tuple_1],
_windows_dictionary=_windows_dictionary,
_fiber_densities=_fiber_densities)
for _index_2 in range(_index_1 + 1, len(_experiments_ids)):
_tuple_2 = _experiments_ids[_index_2]
_experiment_2, _series_id_2, _cell_id_2 = _tuple_2
_fiber_densities_2 = compute_single_cell_mean(
_experiment=_experiment_2,
_series_id=_series_id_2,
_cell_tuples=_tuples[_tuple_2],
_windows_dictionary=_windows_dictionary,
_fiber_densities=_fiber_densities)
for _derivative_index, _derivative in enumerate(DERIVATIVES):
_y_arrays[_derivative_index].append(
compute_lib.correlation(
compute_lib.derivative(_fiber_densities_1,
_n=_derivative),
compute_lib.derivative(_fiber_densities_2,
_n=_derivative)))
print('Total points:', len(_y_arrays[0]))
print('Wilcoxon around the zero')
for _y_array, _derivative in zip(_y_arrays, DERIVATIVES):
print('Derivative:', _derivative, wilcoxon(_y_array))
# plot
_colors_array = config.colors(3)
_fig = go.Figure(data=[
go.Box(y=_y,
name=_derivative,
boxpoints='all',
jitter=1,
pointpos=0,
line={'width': 1},
fillcolor='white',
marker={
'size': 10,
'color': _color
},
opacity=0.7,
showlegend=False) for _y, _derivative, _color in zip(
_y_arrays, DERIVATIVES_TEXT, _colors_array)
],
layout={
'xaxis': {
'title': 'Fiber density derivative',
'zeroline': False
},
'yaxis': {
'title': 'Correlation',
'range': [-1, 1],
'zeroline': False,
'tickmode': 'array',
'tickvals': [-1, -0.5, 0, 0.5, 1]
}
})
save.to_html(_fig=_fig,
_path=os.path.join(paths.PLOTS, save.get_module_name()),
_filename='plot')
def main(_directions=None):
if _directions is None:
_directions = ['inside', 'outside']
_experiments = all_experiments()
_experiments = filtering.by_categories(
_experiments=_experiments,
_is_single_cell=False,
_is_high_temporal_resolution=False,
_is_bleb=False,
_is_dead_dead=False,
_is_live_dead=False,
_is_bead=False,
_is_metastasis=False
)
_tuples = load.experiments_groups_as_tuples(_experiments)
_tuples = filtering.by_real_pairs(_tuples)
_tuples = filtering.by_band(_tuples)
_tuples = filtering.by_pair_distance_range(_tuples, PAIR_DISTANCE_RANGE)
print('Total tuples:', len(_tuples))
_arguments = []
for _tuple in _tuples:
_experiment, _series_id, _group = _tuple
_latest_time_frame = compute.latest_time_frame_before_overlapping(_experiment, _series_id, _group, OFFSET_X)
for _cell_id, _direction in product(['left_cell', 'right_cell'], _directions):
_arguments.append({
'experiment': _experiment,
'series_id': _series_id,
'group': _group,
'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER,
'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER,
'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER,
'offset_x': OFFSET_X,
'offset_y': OFFSET_Y,
'offset_z': OFFSET_Z,
'cell_id': _cell_id,
'direction': _direction,
'time_points': _latest_time_frame
})
_windows_dictionary, _windows_to_compute = \
compute.windows(_arguments, _keys=['experiment', 'series_id', 'group', 'cell_id', 'direction'])
_fiber_densities = compute.fiber_densities(_windows_to_compute, _subtract_border=True)
_experiments_fiber_densities = {
_key: [_fiber_densities[_tuple] for _tuple in _windows_dictionary[_key]]
for _key in _windows_dictionary
}
for _direction in _directions:
_y_arrays = [[] for _i in DERIVATIVES]
for _tuple in tqdm(_tuples, desc='Experiments loop'):
_experiment, _series_id, _group = _tuple
if (_experiment, _series_id, _group, 'left_cell', _direction) not in _windows_dictionary or \
(_experiment, _series_id, _group, 'right_cell', _direction) not in _windows_dictionary:
continue
_properties = load.group_properties(_experiment, _series_id, _group)
_left_cell_fiber_densities = \
_experiments_fiber_densities[(_experiment, _series_id, _group, 'left_cell', _direction)]
_right_cell_fiber_densities = \
_experiments_fiber_densities[(_experiment, _series_id, _group, 'right_cell', _direction)]
_left_cell_fiber_densities = compute.remove_blacklist(
_experiment, _series_id, _properties['cells_ids']['left_cell'], _left_cell_fiber_densities)
_right_cell_fiber_densities = compute.remove_blacklist(
_experiment, _series_id, _properties['cells_ids']['right_cell'], _right_cell_fiber_densities)
if not OUT_OF_BOUNDARIES:
_left_cell_fiber_densities, _right_cell_fiber_densities = \
compute.longest_same_indices_shared_in_borders_sub_array(
_left_cell_fiber_densities, _right_cell_fiber_densities
)
else:
_left_cell_fiber_densities = [_fiber_density[0] for _fiber_density in _left_cell_fiber_densities]
_right_cell_fiber_densities = [_fiber_density[0] for _fiber_density in _right_cell_fiber_densities]
# ignore small arrays
_minimum_time_frame_for_correlation = compute.minimum_time_frames_for_correlation(_experiment)
if len(_left_cell_fiber_densities) < _minimum_time_frame_for_correlation or \
len(_right_cell_fiber_densities) < _minimum_time_frame_for_correlation:
continue
for _derivative_index, _derivative in enumerate(DERIVATIVES):
_y_arrays[_derivative_index].append(compute_lib.correlation(
compute_lib.derivative(_left_cell_fiber_densities, _n=_derivative),
compute_lib.derivative(_right_cell_fiber_densities, _n=_derivative)
))
print('Direction:', _direction)
print('Total pairs:', len(_y_arrays[0]))
print('Wilcoxon around the zero')
for _y_array, _derivative in zip(_y_arrays, DERIVATIVES):
print('Derivative:', _derivative, wilcoxon(_y_array))
# plot
_y_title = 'Inner correlation' if _direction == 'inside' else 'Outer correlation'
_colors_array = config.colors(3)
_fig = go.Figure(
data=[
go.Box(
y=_y,
name=_derivative,
boxpoints='all',
jitter=1,
pointpos=0,
line={
'width': 1
},
fillcolor='white',
marker={
'size': 10,
'color': _color
},
opacity=0.7,
showlegend=False
) for _y, _derivative, _color in zip(_y_arrays, DERIVATIVES_TEXT, _colors_array)
],
layout={
'xaxis': {
'title': 'Fiber density derivative',
'zeroline': False
},
'yaxis': {
'title': _y_title,
'range': [-1, 1],
'zeroline': False,
'tickmode': 'array',
'tickvals': [-1, -0.5, 0, 0.5, 1]
}
}
)
save.to_html(
_fig=_fig,
_path=os.path.join(paths.PLOTS, save.get_module_name()),
_filename='plot_direction_' + _direction
)
def main(_band=True, _high_temporal_resolution=False):
_y_arrays = [[] for _i in PAIR_DISTANCE_RANGES]
_names_array = []
for _distances_index, _distances_range in enumerate(PAIR_DISTANCE_RANGES):
print('Pair distance range:', str(_distances_range))
_experiments = all_experiments()
_experiments = filtering.by_categories(
_experiments=_experiments,
_is_single_cell=False,
_is_high_temporal_resolution=_high_temporal_resolution,
_is_bleb=False,
_is_dead_dead=False,
_is_live_dead=False,
_is_bead=False,
_is_metastasis=False)
_tuples = load.experiments_groups_as_tuples(_experiments)
_tuples = filtering.by_pair_distance_range(_tuples, _distances_range)
_tuples = filtering.by_real_pairs(_tuples)
_tuples = filtering.by_band(_tuples, _band=_band)
print('Total tuples:', len(_tuples))
_arguments = []
for _tuple in _tuples:
_experiment, _series_id, _group = _tuple
_latest_time_frame = compute.latest_time_frame_before_overlapping(
_experiment, _series_id, _group, OFFSET_X)
for _cell_id in ['left_cell', 'right_cell']:
_arguments.append({
'experiment': _experiment,
'series_id': _series_id,
'group': _group,
'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER,
'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER,
'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER,
'offset_x': OFFSET_X,
'offset_y': OFFSET_Y,
'offset_z': OFFSET_Z,
'cell_id': _cell_id,
'direction': 'inside',
'time_points': _latest_time_frame
})
_windows_dictionary, _windows_to_compute = compute.windows(
_arguments, _keys=['experiment', 'series_id', 'group', 'cell_id'])
_fiber_densities = compute.fiber_densities(_windows_to_compute,
_subtract_border=True)
_experiments_fiber_densities = {
_key:
[_fiber_densities[_tuple] for _tuple in _windows_dictionary[_key]]
for _key in _windows_dictionary
}
_tuples_by_experiment = organize.by_experiment(_tuples)
_higher_same_counter = 0
_valid_tuples = []
for _experiment in _tuples_by_experiment:
print('Experiment:', _experiment)
_experiment_tuples = _tuples_by_experiment[_experiment]
for _same_index in tqdm(range(len(_experiment_tuples)),
desc='Main loop'):
_same_tuple = _experiment_tuples[_same_index]
_same_experiment, _same_series, _same_group = _same_tuple
_same_left_cell_fiber_densities = \
_experiments_fiber_densities[
(_same_experiment, _same_series, _same_group, 'left_cell')
]
_same_right_cell_fiber_densities = \
_experiments_fiber_densities[
(_same_experiment, _same_series, _same_group, 'right_cell')
]
_same_properties = \
load.group_properties(_same_experiment, _same_series, _same_group)
_same_left_cell_fiber_densities = compute.remove_blacklist(
_same_experiment, _same_series,
_same_properties['cells_ids']['left_cell'],
_same_left_cell_fiber_densities)
_same_right_cell_fiber_densities = compute.remove_blacklist(
_same_experiment, _same_series,
_same_properties['cells_ids']['right_cell'],
_same_right_cell_fiber_densities)
_same_left_cell_fiber_densities_filtered, _same_right_cell_fiber_densities_filtered = \
compute.longest_same_indices_shared_in_borders_sub_array(
_same_left_cell_fiber_densities, _same_right_cell_fiber_densities
)
# ignore small arrays
if len(_same_left_cell_fiber_densities_filtered
) < compute.minimum_time_frames_for_correlation(
_same_experiment):
continue
_same_correlation = compute_lib.correlation(
compute_lib.derivative(
_same_left_cell_fiber_densities_filtered,
_n=DERIVATIVE),
compute_lib.derivative(
_same_right_cell_fiber_densities_filtered,
_n=DERIVATIVE))
for _different_index in range(len(_experiment_tuples)):
if _same_index != _different_index:
_different_tuple = _experiment_tuples[_different_index]
_different_experiment, _different_series, _different_group = \
_different_tuple
for _same_cell_id, _different_cell_id in product(
['left_cell', 'right_cell'],
['left_cell', 'right_cell']):
_same_fiber_densities = _experiments_fiber_densities[
(_same_experiment, _same_series, _same_group,
_same_cell_id)]
_different_fiber_densities = _experiments_fiber_densities[
(_different_experiment, _different_series,
_different_group, _different_cell_id)]
_different_properties = load.group_properties(
_different_experiment, _different_series,
_different_group)
_same_fiber_densities = compute.remove_blacklist(
_same_experiment, _same_series,
_same_properties['cells_ids'][_same_cell_id],
_same_fiber_densities)
_different_fiber_densities = compute.remove_blacklist(
_different_experiment, _different_series,
_different_properties['cells_ids']
[_different_cell_id],
_different_fiber_densities)
_same_fiber_densities_filtered, _different_fiber_densities_filtered = \
compute.longest_same_indices_shared_in_borders_sub_array(
_same_fiber_densities, _different_fiber_densities
)
# ignore small arrays
if len(
_same_fiber_densities_filtered
) < compute.minimum_time_frames_for_correlation(
_different_experiment):
continue
_different_correlation = compute_lib.correlation(
compute_lib.derivative(
_same_fiber_densities_filtered,
_n=DERIVATIVE),
compute_lib.derivative(
_different_fiber_densities_filtered,
_n=DERIVATIVE))
_point_distance = compute_lib.distance_from_a_point_to_a_line(
_line=[-1, -1, 1, 1],
_point=[
_same_correlation, _different_correlation
])
if _same_correlation > _different_correlation:
_y_arrays[_distances_index].append(
_point_distance)
_higher_same_counter += 1
else:
_y_arrays[_distances_index].append(
-_point_distance)
if _same_tuple not in _valid_tuples:
_valid_tuples.append(_same_tuple)
print('Total tuples:', len(_valid_tuples))
print('Total points:', len(_y_arrays[_distances_index]))
print('Wilcoxon around the zero:')
print(wilcoxon(_y_arrays[_distances_index]))
print('Higher same amount:',
_higher_same_counter / len(_y_arrays[_distances_index]))
_names_array.append(
str(_distances_range[0]) + '-' + str(_distances_range[1]))
# plot
_colors_array = config.colors(3)
_fig = go.Figure(data=[
go.Box(y=_y,
name=_name,
boxpoints=False,
line={'width': 1},
marker={
'size': 10,
'color': _color
},
showlegend=False)
for _y, _name, _color in zip(_y_arrays, _names_array, _colors_array)
],
layout={
'xaxis': {
'title': 'Pair distance (cell diameter)',
'zeroline': False,
'type': 'category'
},
'yaxis': {
'title': 'Same minus different correlation',
'range': [-1, 1.1],
'zeroline': False,
'tickmode': 'array',
'tickvals': [-1, -0.5, 0, 0.5, 1]
}
})
save.to_html(_fig=_fig,
_path=os.path.join(paths.PLOTS, save.get_module_name()),
_filename='plot_band_' + str(_band) + '_high_temporal_res_' +
str(_high_temporal_resolution))
def main():
_arguments = []
for _tuple in TRIPLET:
_experiment, _series_id, _group = _tuple
_pair_distance = compute.pair_distance_in_cell_size_time_frame(_experiment, _series_id, _group, _time_frame=0)
print(_tuple, 'pairs distance:', round(_pair_distance, 2))
_latest_time_frame = compute.latest_time_frame_before_overlapping(_experiment, _series_id, _group, OFFSET_X)
for _cell_id in ['left_cell', 'right_cell']:
_arguments.append({
'experiment': _experiment,
'series_id': _series_id,
'group': _group,
'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER,
'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER,
'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER,
'offset_x': OFFSET_X,
'offset_y': OFFSET_Y,
'offset_z': OFFSET_Z,
'cell_id': _cell_id,
'direction': 'inside',
'time_points': _latest_time_frame
})
_windows_dictionary, _windows_to_compute = compute.windows(_arguments,
_keys=['experiment', 'series_id', 'group', 'cell_id'])
_fiber_densities = compute.fiber_densities(_windows_to_compute, _subtract_border=True)
_experiments_fiber_densities = {
_key: [_fiber_densities[_tuple] for _tuple in _windows_dictionary[_key]]
for _key in _windows_dictionary
}
_same_correlations_arrays = [[], [], []]
_different_correlations_arrays = [[], [], []]
_names_array = []
for _same_index in tqdm(range(3), desc='Main loop'):
_same_tuple = TRIPLET[_same_index]
_same_experiment, _same_series, _same_group = _same_tuple
_same_left_cell_fiber_densities = \
_experiments_fiber_densities[
(_same_experiment, _same_series, _same_group, 'left_cell')
]
_same_right_cell_fiber_densities = \
_experiments_fiber_densities[
(_same_experiment, _same_series, _same_group, 'right_cell')
]
_same_properties = \
load.group_properties(_same_experiment, _same_series, _same_group)
_same_left_cell_fiber_densities = compute.remove_blacklist(
_same_experiment,
_same_series,
_same_properties['cells_ids']['left_cell'],
_same_left_cell_fiber_densities
)
_same_right_cell_fiber_densities = compute.remove_blacklist(
_same_experiment,
_same_series,
_same_properties['cells_ids']['right_cell'],
_same_right_cell_fiber_densities
)
_same_left_cell_fiber_densities_filtered, _same_right_cell_fiber_densities_filtered = \
compute.longest_same_indices_shared_in_borders_sub_array(
_same_left_cell_fiber_densities, _same_right_cell_fiber_densities
)
_same_correlation = compute_lib.correlation(
compute_lib.derivative(_same_left_cell_fiber_densities_filtered, _n=DERIVATIVE),
compute_lib.derivative(_same_right_cell_fiber_densities_filtered, _n=DERIVATIVE)
)
for _different_index in range(3):
if _same_index != _different_index:
_different_tuple = TRIPLET[_different_index]
_different_experiment, _different_series, _different_group = \
_different_tuple
for _same_cell_id, _different_cell_id in product(['left_cell', 'right_cell'],
['left_cell', 'right_cell']):
_same_fiber_densities = _experiments_fiber_densities[(
_same_experiment,
_same_series,
_same_group,
_same_cell_id
)]
_different_fiber_densities = _experiments_fiber_densities[(
_different_experiment,
_different_series,
_different_group,
_different_cell_id
)]
_different_properties = load.group_properties(
_different_experiment, _different_series, _different_group
)
_same_fiber_densities = compute.remove_blacklist(
_same_experiment,
_same_series,
_same_properties['cells_ids'][_same_cell_id],
_same_fiber_densities
)
_different_fiber_densities = compute.remove_blacklist(
_different_experiment,
_different_series,
_different_properties['cells_ids'][_different_cell_id],
_different_fiber_densities
)
_same_fiber_densities_filtered, _different_fiber_densities_filtered = \
compute.longest_same_indices_shared_in_borders_sub_array(
_same_fiber_densities, _different_fiber_densities
)
_different_correlation = compute_lib.correlation(
compute_lib.derivative(_same_fiber_densities_filtered, _n=DERIVATIVE),
compute_lib.derivative(_different_fiber_densities_filtered, _n=DERIVATIVE)
)
_same_correlations_arrays[_same_index].append(_same_correlation)
_different_correlations_arrays[_same_index].append(_different_correlation)
_names_array.append('Cells ' + _same_group.split('_')[1] + ' & ' + _same_group.split('_')[2])
print('Group:', TRIPLET[_same_index])
print('Points:', len(_same_correlations_arrays[_same_index]))
_same_minus_different = \
np.array(_same_correlations_arrays[_same_index]) - np.array(_different_correlations_arrays[_same_index])
print('Wilcoxon of same minus different around the zero:')
print(wilcoxon(_same_minus_different))
print('Higher same amount:', (_same_minus_different > 0).sum() /
len(_same_minus_different))
print('Total points:', len(np.array(_same_correlations_arrays).flatten()))
_same_minus_different = \
np.array(_same_correlations_arrays).flatten() - np.array(_different_correlations_arrays).flatten()
print('Wilcoxon of same minus different around the zero:')
print(wilcoxon(_same_minus_different))
print('Higher same amount:', (_same_minus_different > 0).sum() /
len(_same_minus_different))
# plot
_colors_array = config.colors(3)
_fig = go.Figure(
data=[
go.Scatter(
x=_same_correlations_array,
y=_different_correlations_array,
name=_name,
mode='markers',
marker={
'size': 15,
'color': _color
},
opacity=0.7
) for _same_correlations_array, _different_correlations_array, _name, _color in
zip(_same_correlations_arrays, _different_correlations_arrays, _names_array, _colors_array)
],
layout={
'xaxis': {
'title': 'Same network correlation',
'zeroline': False,
'range': [-1.1, 1.2],
'tickmode': 'array',
'tickvals': [-1, -0.5, 0, 0.5, 1]
},
'yaxis': {
'title': 'Different network correlation',
'zeroline': False,
'range': [-1.1, 1.2],
'tickmode': 'array',
'tickvals': [-1, -0.5, 0, 0.5, 1]
},
'legend': {
'xanchor': 'left',
'x': 0.1,
'yanchor': 'top',
'bordercolor': 'black',
'borderwidth': 2,
'bgcolor': 'white'
},
'shapes': [
{
'type': 'line',
'x0': -1,
'y0': -1,
'x1': -1,
'y1': 1,
'line': {
'color': 'black',
'width': 2
}
},
{
'type': 'line',
'x0': -1,
'y0': -1,
'x1': 1,
'y1': -1,
'line': {
'color': 'black',
'width': 2
}
},
{
'type': 'line',
'x0': -1,
'y0': -1,
'x1': 1,
'y1': 1,
'line': {
'color': 'red',
'width': 2
}
}
]
}
)
save.to_html(
_fig=_fig,
_path=os.path.join(paths.PLOTS, save.get_module_name()),
_filename='plot'
)
def main():
_experiments = all_experiments()
_experiments = filtering.by_categories(_experiments=_experiments,
_is_single_cell=True,
_is_high_temporal_resolution=False,
_is_bleb=False,
_is_dead_dead=False,
_is_live_dead=False,
_is_bead=False,
_is_metastasis=False)
_tuples = load.experiments_groups_as_tuples(_experiments)
_tuples = filtering.by_time_frames_amount(
_tuples, compute.density_time_frame(_experiments[0]))
_tuples = filtering.by_main_cell(_tuples)
_arguments = []
for _tuple in _tuples:
_experiment, _series_id, _group = _tuple
_time_frame = compute.density_time_frame(_experiment)
for _direction in ['left', 'right']:
_arguments.append({
'experiment': _experiment,
'series_id': _series_id,
'group': _group,
'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER,
'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER,
'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER,
'offset_x': OFFSET_X,
'offset_y': OFFSET_Y,
'offset_z': OFFSET_Z,
'cell_id': 'cell',
'direction': _direction,
'time_points': _time_frame
})
_windows_dictionary, _windows_to_compute = \
compute.windows(_arguments, _keys=['experiment', 'series_id', 'group', 'direction'])
_fiber_densities = compute.fiber_densities(_windows_to_compute,
_subtract_border=True)
_tuples = organize.by_single_cell_id(_tuples)
print('Total experiments:', len(_tuples))
_kpss_y_arrays = [[] for _i in DERIVATIVES]
_adf_y_arrays = [[] for _i in DERIVATIVES]
for _tuple in tqdm(_tuples, desc='Experiments loop'):
_experiment, _series_id, _cell_id = _tuple
_cell_fiber_densities = compute_single_cell_mean(
_experiment=_experiment,
_series_id=_series_id,
_cell_tuples=_tuples[_tuple],
_windows_dictionary=_windows_dictionary,
_fiber_densities=_fiber_densities)
for _derivative_index, _derivative in enumerate(DERIVATIVES):
_cell_fiber_densities_derivative = compute_lib.derivative(
_cell_fiber_densities, _n=_derivative)
_, _kpss_p_value, _, _ = kpss(_cell_fiber_densities_derivative,
nlags='legacy')
_kpss_y_arrays[_derivative_index].append(_kpss_p_value)
_, _adf_p_value, _, _, _, _ = adfuller(
_cell_fiber_densities_derivative)
_adf_y_arrays[_derivative_index].append(_adf_p_value)
print('Total cells:', len(_kpss_y_arrays[0]))
# print results
print('KPSS:')
for _derivative_index, _derivative in enumerate(DERIVATIVES):
_stationary_count = len([
_value for _value in _kpss_y_arrays[_derivative_index]
if _value > 0.05
])
print(
'Derivative:', _derivative, 'Stationary:',
str(_stationary_count / len(_kpss_y_arrays[_derivative_index]) *
100) + '%')
print('ADF:')
for _derivative_index, _derivative in enumerate(DERIVATIVES):
_stationary_count = len([
_value for _value in _adf_y_arrays[_derivative_index]
if _value < 0.05
])
print(
'Derivative:', _derivative, 'Stationary:',
str(_stationary_count / len(_adf_y_arrays[_derivative_index]) *
100) + '%')
# plot
_colors_array = config.colors(3)
for _test_name, _y_title, _y_tickvals, _p_value_line, _y_arrays in \
zip(
['kpss', 'adf'],
['KPSS test p-value', 'ADF test p-value'],
[[0.05, 0.1], [0.05, 1]],
[0.05, 0.05],
[_kpss_y_arrays, _adf_y_arrays]
):
_fig = go.Figure(data=[
go.Box(y=_y,
name=_derivative,
boxpoints='all',
jitter=1,
pointpos=0,
line={'width': 1},
fillcolor='white',
marker={
'size': 10,
'color': _color
},
opacity=0.7,
showlegend=False) for _y, _derivative, _color in zip(
_y_arrays, DERIVATIVES_TEXT, _colors_array)
],
layout={
'xaxis': {
'title': 'Fiber density derivative',
'zeroline': False
},
'yaxis': {
'title': _y_title,
'zeroline': False,
'tickmode': 'array',
'tickvals': _y_tickvals
},
'shapes': [{
'type': 'line',
'x0': DERIVATIVES[0] - 0.75,
'y0': _p_value_line,
'x1': DERIVATIVES[-1] + 0.75,
'y1': _p_value_line,
'line': {
'color': 'red',
'width': 2,
'dash': 'dash'
}
}]
})
save.to_html(_fig=_fig,
_path=os.path.join(paths.PLOTS, save.get_module_name()),
_filename='plot_' + _test_name)
def main(_real_cells=True, _static=False, _band=True, _high_temporal_resolution=False):
_experiments = all_experiments()
_experiments = filtering.by_categories(
_experiments=_experiments,
_is_single_cell=False,
_is_high_temporal_resolution=_high_temporal_resolution,
_is_bleb=False,
_is_dead_dead=False,
_is_live_dead=False,
_is_bead=False,
_is_metastasis=False
)
_tuples = load.experiments_groups_as_tuples(_experiments)
_tuples = filtering.by_pair_distance_range(_tuples, PAIR_DISTANCE_RANGE)
_tuples = filtering.by_real_pairs(_tuples, _real_pairs=_real_cells)
_tuples = filtering.by_fake_static_pairs(_tuples, _fake_static_pairs=_static)
_tuples = filtering.by_band(_tuples, _band=_band)
print('Total tuples:', len(_tuples))
_arguments = []
for _tuple in _tuples:
_experiment, _series_id, _group = _tuple
_latest_time_frame = compute.latest_time_frame_before_overlapping(_experiment, _series_id, _group, OFFSET_X)
for _cell_id in ['left_cell', 'right_cell']:
_arguments.append({
'experiment': _experiment,
'series_id': _series_id,
'group': _group,
'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER,
'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER,
'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER,
'offset_x': OFFSET_X,
'offset_y': OFFSET_Y,
'offset_z': OFFSET_Z,
'cell_id': _cell_id,
'direction': 'inside',
'time_points': _latest_time_frame
})
_z_array = np.zeros(shape=(len(BY), len(BY)))
for (_padding_index, _padding_by), (_space_index, _space_by) in product(enumerate(BY), enumerate(BY)):
print('Padding by: ', _padding_by, ', space by: ', _space_by)
_correlation = compute_data(_tuples, _arguments, _padding_y_by=_padding_by, _padding_z_by=_padding_by,
_space_y_by=_space_by, _space_z_by=_space_by)
_z_array[_space_index, _padding_index] = _correlation
# plot
_colors_array = ['white', config.colors(1)]
_fig = go.Figure(
data=go.Heatmap(
x=BY,
y=BY,
z=_z_array,
colorscale=sns.color_palette(_colors_array).as_hex(),
colorbar={
'tickmode': 'array',
'tickvals': [0, 0.5, 1],
'ticktext': ['0', 'Correlation', '1'],
'tickangle': -90
},
showscale=True,
zmin=0,
zmax=1
),
layout={
'xaxis': {
'title': 'Border size (cell diameter)',
'zeroline': False,
'tickmode': 'array',
'tickvals': [0, 0.5, 1, 1.5, 2]
},
'yaxis': {
'title': 'Space from quantification window (cell diameter)',
'zeroline': False,
'tickmode': 'array',
'tickvals': [0, 0.5, 1, 1.5, 2]
}
}
)
save.to_html(
_fig=_fig,
_path=os.path.join(paths.PLOTS, save.get_module_name()),
_filename='plot_high_time_' + str(_high_temporal_resolution) + '_band_' + str(_band)
)
def main(_band=None,
_high_temporal_resolution=True,
_tuples_to_mark=None,
_tuples_to_plot=None,
_plots=None):
if _plots is None:
_plots = ['whiteness', 'granger']
_experiments = all_experiments()
_experiments = filtering.by_categories(
_experiments=_experiments,
_is_single_cell=False,
_is_high_temporal_resolution=_high_temporal_resolution,
_is_bleb=False,
_is_dead_dead=False,
_is_live_dead=False,
_is_bead=False,
_is_metastasis=False)
_tuples = load.experiments_groups_as_tuples(_experiments)
_tuples = filtering.by_time_frames_amount(_tuples,
_time_frames=MINIMUM_TIME_FRAMES)
_tuples = filtering.by_pair_distance_range(
_tuples, _distance_range=PAIR_DISTANCE_RANGE)
_tuples = filtering.by_real_pairs(_tuples)
_tuples = filtering.by_band(_tuples, _band=_band)
print('Total tuples:', len(_tuples))
_arguments = []
for _tuple in _tuples:
_experiment, _series_id, _group = _tuple
_latest_time_frame = compute.latest_time_frame_before_overlapping(
_experiment, _series_id, _group, OFFSET_X)
for _cell_id in ['left_cell', 'right_cell']:
_arguments.append({
'experiment': _experiment,
'series_id': _series_id,
'group': _group,
'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER,
'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER,
'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER,
'offset_x': OFFSET_X,
'offset_y': OFFSET_Y,
'offset_z': OFFSET_Z,
'cell_id': _cell_id,
'direction': 'inside',
'time_points': _latest_time_frame
})
_windows_dictionary, _windows_to_compute = compute.windows(
_arguments, _keys=['experiment', 'series_id', 'group', 'cell_id'])
_fiber_densities = compute.fiber_densities(_windows_to_compute,
_subtract_border=True)
_experiments_fiber_densities = {
_key:
[_fiber_densities[_tuple] for _tuple in _windows_dictionary[_key]]
for _key in _windows_dictionary
}
_n_pairs = 0
_n_pairs_with_band = 0
_whiteness_p_values = []
_n_passed_whiteness_with_band = 0
_granger_causality_p_values = []
_n_passed_granger_causality_with_band = 0
_correlations = []
_time_lag_correlations = []
_end_fiber_densities = []
for _tuple in _tuples:
_experiment, _series_id, _group = _tuple
_left_cell_fiber_densities = \
_experiments_fiber_densities[(_experiment, _series_id, _group, 'left_cell')]
_right_cell_fiber_densities = \
_experiments_fiber_densities[(_experiment, _series_id, _group, 'right_cell')]
_properties = load.group_properties(_experiment, _series_id, _group)
_left_cell_fiber_densities = compute.remove_blacklist(
_experiment, _series_id, _properties['cells_ids']['left_cell'],
_left_cell_fiber_densities)
_right_cell_fiber_densities = compute.remove_blacklist(
_experiment, _series_id, _properties['cells_ids']['right_cell'],
_right_cell_fiber_densities)
_left_cell_fiber_densities_filtered, _right_cell_fiber_densities_filtered = \
compute.longest_same_indices_shared_in_borders_sub_array(
_left_cell_fiber_densities, _right_cell_fiber_densities)
# ignore small arrays
if len(_left_cell_fiber_densities_filtered) < MINIMUM_TIME_FRAMES:
continue
_n_pairs += 1
if _properties['band']:
_n_pairs_with_band += 1
_start_time_frame = 0
for _left in _left_cell_fiber_densities:
if _left[0] == _left_cell_fiber_densities_filtered[0]:
break
_start_time_frame += 1
# stationary test
with warnings.catch_warnings():
warnings.simplefilter('ignore', category=InterpolationWarning)
# find derivative for stationary
for _derivative in range(10):
_left_cell_fiber_densities_derivative = \
compute_lib.derivative(_left_cell_fiber_densities_filtered, _n=_derivative)
_right_cell_fiber_densities_derivative = \
compute_lib.derivative(_right_cell_fiber_densities_filtered, _n=_derivative)
if ADF_TEST:
_, _left_cell_adf_p_value, _, _, _, _ = adfuller(
_left_cell_fiber_densities_derivative)
_, _right_cell_adf_p_value, _, _, _, _ = adfuller(
_right_cell_fiber_densities_derivative)
if _left_cell_adf_p_value > 0.05 or _right_cell_adf_p_value > 0.05:
continue
if KPSS_TEST:
_, _left_cell_kpss_p_value, _, _ = kpss(
_left_cell_fiber_densities_derivative, nlags='legacy')
_, _right_cell_kpss_p_value, _, _ = kpss(
_right_cell_fiber_densities_derivative, nlags='legacy')
if _left_cell_kpss_p_value < 0.05 or _right_cell_kpss_p_value < 0.05:
continue
# stationary
break
# causality
try:
_x = pd.DataFrame(data=[[_left_value, _right_value]
for _left_value, _right_value in zip(
_left_cell_fiber_densities_derivative,
_right_cell_fiber_densities_derivative)
],
columns=['left', 'right'])
# var model to retrieve lag
_var_model = VAR(_x)
_lag_order_results = _var_model.select_order()
_estimators_lags = [
_lag_order_results.aic, _lag_order_results.bic,
_lag_order_results.fpe, _lag_order_results.hqic
]
_min_estimator_lag = min(_estimators_lags)
# found a lag
if 0 < _min_estimator_lag <= MAXIMUM_LAG:
_var_model_results = _var_model.fit(maxlags=_min_estimator_lag,
ic=None)
_whiteness = _var_model_results.test_whiteness(
nlags=_min_estimator_lag + 1)
_whiteness_p_values.append(_whiteness.pvalue)
if _tuples_to_mark is not None and _tuple in _tuples_to_mark and _whiteness.pvalue > 0.05:
print(_tuple, 'marked whiteness p-value:',
_whiteness.pvalue)
# no autocorrelation in the residuals
if _whiteness.pvalue > 0.05:
if _properties['band']:
_n_passed_whiteness_with_band += 1
# time lag = 0
_correlation = compute_lib.correlation(
_left_cell_fiber_densities_derivative,
_right_cell_fiber_densities_derivative)
# if _correlation < 0.5:
# continue
# granger causality
for _caused, _causing in zip(['left', 'right'],
['right', 'left']):
_granger = _var_model_results.test_causality(
caused=_caused, causing=_causing)
_granger_causality_p_values.append(_granger.pvalue)
# time lag = 0
_correlations.append(_correlation)
# time lag = min estimator
if _causing == 'left':
_left_fiber_densities_time_lag = \
_left_cell_fiber_densities_derivative[:-_min_estimator_lag]
_right_fiber_densities_time_lag = \
_right_cell_fiber_densities_derivative[_min_estimator_lag:]
else:
_left_fiber_densities_time_lag = \
_left_cell_fiber_densities_derivative[_min_estimator_lag:]
_right_fiber_densities_time_lag = \
_right_cell_fiber_densities_derivative[:-_min_estimator_lag]
_time_lag_correlation = compute_lib.correlation(
_left_fiber_densities_time_lag,
_right_fiber_densities_time_lag)
_time_lag_correlations.append(_time_lag_correlation)
# end fiber density
_time_frame = compute.density_time_frame(_experiment)
if len(_left_cell_fiber_densities_filtered
) > _time_frame:
_end_fiber_density = \
(_left_cell_fiber_densities_filtered[_time_frame] +
_right_cell_fiber_densities_filtered[_time_frame]) / 2
else:
_end_fiber_density = \
(_left_cell_fiber_densities_filtered[-1] +
_right_cell_fiber_densities_filtered[-1]) / 2
_normalization = load.normalization_series_file_data(
_experiment, _series_id)
_normalized_fiber_density = compute_lib.z_score(
_end_fiber_density, _normalization['average'],
_normalization['std'])
_end_fiber_densities.append(_normalized_fiber_density)
# marking
if _tuples_to_mark is not None and _tuple in _tuples_to_mark and _granger.pvalue < 0.05:
print(_tuple, 'causing:', _causing,
'marked granger p-value:', _granger.pvalue)
if _granger.pvalue < 0.05:
if _properties['band']:
_n_passed_granger_causality_with_band += 1
_normality = _var_model_results.test_normality()
_inst_granger = _var_model_results.test_inst_causality(
causing=_causing)
print(
_tuple,
_causing.capitalize() + ' causes ' + _caused +
'!',
'time-points: ' +
str(len(
_left_cell_fiber_densities_derivative)),
'stationary derivative: ' + str(_derivative),
'band:' + str(_properties['band']),
'p-value: ' + str(round(_granger.pvalue, 4)),
'lag: ' + str(_min_estimator_lag),
'normality p-value: ' +
str(round(_normality.pvalue, 4)),
'inst p-value: ' +
str(round(_inst_granger.pvalue, 4)),
sep='\t')
# lag = 0
print('Time lag = 0 correlation:', _correlation)
# rest of lags
for _lag in range(1, _min_estimator_lag + 1):
if _causing == 'left':
_left_fiber_densities_time_lag = _left_cell_fiber_densities_derivative[:
-_lag]
_right_fiber_densities_time_lag = _right_cell_fiber_densities_derivative[
_lag:]
else:
_left_fiber_densities_time_lag = _left_cell_fiber_densities_derivative[
_lag:]
_right_fiber_densities_time_lag = _right_cell_fiber_densities_derivative[:
-_lag]
_correlation = compute_lib.correlation(
_left_fiber_densities_time_lag,
_right_fiber_densities_time_lag)
print(
'Time lag = ' + str(_lag) +
' correlation:', _correlation)
# plots
if _tuples_to_plot is not None and _tuple in _tuples_to_plot:
_y_arrays = [
_left_cell_fiber_densities_derivative,
_right_cell_fiber_densities_derivative
]
_names_array = ['Left cell', 'Right cell']
_colors_array = config.colors(2)
_temporal_resolution = compute.temporal_resolution_in_minutes(
_experiment)
_fig = go.Figure(data=[
go.Scatter(x=np.arange(
start=_start_time_frame,
stop=_start_time_frame +
len(_left_cell_fiber_densities_derivative
),
step=1) * _temporal_resolution,
y=_y,
name=_name,
mode='lines',
line={
'color': _color,
'width': 1
})
for _y, _name, _color in zip(
_y_arrays, _names_array, _colors_array)
],
layout={
'xaxis': {
'title':
'Time (minutes)',
'zeroline': False
},
'yaxis': {
'title':
'Fiber density (z-score)'
+ '\'' * _derivative,
'zeroline':
False
},
'legend': {
'xanchor': 'left',
'x': 0.1,
'yanchor': 'top',
'bordercolor':
'black',
'borderwidth': 2,
'bgcolor': 'white'
},
})
_experiment, _series_id, _group = _tuple
save.to_html(
_fig=_fig,
_path=os.path.join(paths.PLOTS,
save.get_module_name()),
_filename='plot_' + _experiment + '_' +
str(_series_id) + '_' + _group)
# residuals
_y_arrays = \
[_var_model_results.resid.values[:, 0], _var_model_results.resid.values[:, 1]]
_fig = go.Figure(data=[
go.Scatter(x=np.arange(
start=_start_time_frame,
stop=_start_time_frame + len(_y),
step=1) * _temporal_resolution,
y=_y,
name=_name,
mode='lines',
line={
'color': _color,
'width': 1
})
for _y, _name, _color in zip(
_y_arrays, _names_array, _colors_array)
],
layout={
'xaxis': {
'title':
'Time (minutes)',
'zeroline': False
},
'yaxis': {
'title': 'Residual',
'zeroline': False
},
'legend': {
'xanchor': 'left',
'x': 0.1,
'yanchor': 'top',
'bordercolor':
'black',
'borderwidth': 2,
'bgcolor': 'white'
},
})
_experiment, _series_id, _group = _tuple
save.to_html(
_fig=_fig,
_path=os.path.join(paths.PLOTS,
save.get_module_name()),
_filename='plot_residuals_' + _experiment +
'_' + str(_series_id) + '_' + _group)
# not enough time points
except ValueError:
continue
print('Total pairs:', _n_pairs)
print('Total pairs with band:', _n_pairs_with_band)
print('Total pairs passed whiteness:',
(np.array(_whiteness_p_values) > 0.05).sum())
print('Total pairs passed whiteness with band:',
_n_passed_whiteness_with_band)
print('Total cells passed granger causality:',
(np.array(_granger_causality_p_values) < 0.05).sum())
print('Total cells passed granger causality with band:',
_n_passed_granger_causality_with_band)
# p-value correction
print('Corrections of GC p-value < 0.05:')
_granger_causality_p_values_corrected = multipletests(
pvals=_granger_causality_p_values, method='fdr_bh')
for _p_value, _p_value_corrected in zip(
_granger_causality_p_values,
_granger_causality_p_values_corrected[1]):
if _p_value < 0.05:
print('Original GC p-value:', _p_value, 'corrected:',
_p_value_corrected)
# plots
for _test_name, _y_title, _y_array in \
zip(
['whiteness', 'granger'],
['Whiteness p-value', 'Granger causality p-value'],
[_whiteness_p_values, _granger_causality_p_values]
):
if _test_name in _plots:
_fig = go.Figure(data=go.Box(y=_y_array,
boxpoints='all',
jitter=1,
pointpos=0,
line={'width': 1},
fillcolor='white',
marker={
'size': 10,
'color': '#ea8500'
},
opacity=0.7,
showlegend=False),
layout={
'xaxis': {
'zeroline': False
},
'yaxis': {
'title': _y_title,
'zeroline': False,
'range': [-0.1, 1.1],
'tickmode': 'array',
'tickvals': [0.05, 1]
},
'shapes': [{
'type': 'line',
'x0': -0.75,
'y0': 0.05,
'x1': 0.75,
'y1': 0.05,
'line': {
'color': 'red',
'width': 2,
'dash': 'dash'
}
}]
})
save.to_html(_fig=_fig,
_path=os.path.join(paths.PLOTS,
save.get_module_name()),
_filename='plot_' + _test_name)
# granger versus correlation
print(
'GC vs. correlation pearson correlation:',
compute_lib.correlation(_granger_causality_p_values,
_correlations,
_with_p_value=True))
_fig = go.Figure(data=go.Scatter(x=_granger_causality_p_values,
y=_correlations,
mode='markers',
marker={
'size': 10,
'color': '#ea8500'
},
showlegend=False),
layout={
'xaxis': {
'title': 'Granger causality p-value',
'zeroline': False,
},
'yaxis': {
'title': 'Inner correlation',
'zeroline': False,
}
})
save.to_html(_fig=_fig,
_path=os.path.join(paths.PLOTS, save.get_module_name()),
_filename='plot_gc_vs_correlation')
# granger versus time lag correlation
print(
'GC vs. time lag correlation pearson correlation:',
compute_lib.correlation(_granger_causality_p_values,
_time_lag_correlations,
_with_p_value=True))
_fig = go.Figure(data=go.Scatter(x=_granger_causality_p_values,
y=_time_lag_correlations,
mode='markers',
marker={
'size': 10,
'color': '#ea8500'
},
showlegend=False),
layout={
'xaxis': {
'title': 'Granger causality p-value',
'zeroline': False,
},
'yaxis': {
'title': 'GC lag inner correlation',
'zeroline': False,
}
})
save.to_html(_fig=_fig,
_path=os.path.join(paths.PLOTS, save.get_module_name()),
_filename='plot_gc_vs_time_lag_correlation')
# granger versus end fiber density
print(
'GC vs. end fiber density pearson correlation:',
compute_lib.correlation(_granger_causality_p_values,
_end_fiber_densities,
_with_p_value=True))
_fig = go.Figure(data=go.Scatter(x=_granger_causality_p_values,
y=_end_fiber_densities,
mode='markers',
marker={
'size': 10,
'color': '#ea8500'
},
showlegend=False),
layout={
'xaxis': {
'title': 'Granger causality p-value',
'zeroline': False,
},
'yaxis': {
'title': 'End fiber density (z-score)',
'zeroline': False,
}
})
save.to_html(_fig=_fig,
_path=os.path.join(paths.PLOTS, save.get_module_name()),
_filename='plot_gc_vs_end_density')
