def main():
# single cells
print('Single cells')
_single_cells = load.experiments_groups_as_tuples(SINGLE_CELLS)
_single_cells = filtering.by_main_cell(_single_cells)
_single_cells = organize.by_single_cell_id(_single_cells)
print('\tTotal single cell experiments:', len(_single_cells))
# cell pairs
_cell_pairs = load.experiments_groups_as_tuples(CELL_PAIRS)
print('\nTotal cell pairs:', len(_cell_pairs))
for _distance_range in DISTANCE_RANGES:
print('Cell pairs distance range:', _distance_range)
# total
_distance_range_cell_pairs = filtering.by_pair_distance_range(_cell_pairs, _distance_range=_distance_range)
print('\tTotal cell pairs:', len(_distance_range_cell_pairs))
for _experiment in CELL_PAIRS:
print('\tExperiment:', _experiment)
_experiment_cell_pairs = load.experiment_groups_as_tuples(_experiment)
_distance_range_experiment_cell_pairs = \
filtering.by_pair_distance_range(_experiment_cell_pairs, _distance_range=_distance_range)
print('\t\tTotal cell pairs:', len(_distance_range_experiment_cell_pairs))
# real cell pairs
_real_cell_pairs = filtering.by_real_pairs(_distance_range_experiment_cell_pairs)
print('\t\tTotal real cell pairs:', len(_real_cell_pairs))
_real_cell_pairs_with_band = filtering.by_band(_real_cell_pairs)
print('\t\tTotal real cell pairs with band:', len(_real_cell_pairs_with_band))
# fake cell pairs
_fake_cell_pairs = filtering.by_real_pairs(_distance_range_experiment_cell_pairs, _real_pairs=False)
print('\t\tTotal fake cell pairs:', len(_fake_cell_pairs))
_fake_static_cell_pairs = filtering.by_fake_static_pairs(_fake_cell_pairs)
print('\t\tTotal fake static cell pairs:', len(_fake_static_cell_pairs))
_fake_following_cell_pairs = filtering.by_fake_static_pairs(_fake_cell_pairs, _fake_static_pairs=False)
print('\t\tTotal fake following cell pairs:', len(_fake_following_cell_pairs))
_fake_following_cell_pairs_with_band = filtering.by_band(_fake_following_cell_pairs)
print('\t\tTotal fake following cell pairs with band:', len(_fake_following_cell_pairs_with_band))
def main():
_experiments = all_experiments()
_experiments = filtering.by_categories(_experiments=_experiments,
_is_single_cell=True,
_is_high_temporal_resolution=False,
_is_bleb=False,
_is_dead_dead=False,
_is_live_dead=False,
_is_bead=False,
_is_metastasis=False)
_tuples = load.experiments_groups_as_tuples(_experiments)
_tuples = filtering.by_time_frames_amount(
_tuples, compute.density_time_frame(_experiments[0]))
_tuples = filtering.by_main_cell(_tuples)
_arguments = []
for _tuple in _tuples:
_experiment, _series_id, _group = _tuple
_time_frame = compute.density_time_frame(_experiment)
for _direction in ['left', 'right']:
_arguments.append({
'experiment': _experiment,
'series_id': _series_id,
'group': _group,
'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER,
'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER,
'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER,
'offset_x': OFFSET_X,
'offset_y': OFFSET_Y,
'offset_z': OFFSET_Z,
'cell_id': 'cell',
'direction': _direction,
'time_points': _time_frame
})
_windows_dictionary, _windows_to_compute = \
compute.windows(_arguments, _keys=['experiment', 'series_id', 'group', 'direction'])
_fiber_densities = compute.fiber_densities(_windows_to_compute,
_subtract_border=True)
_tuples = organize.by_single_cell_id(_tuples)
print('Total experiments:', len(_tuples))
_kpss_y_arrays = [[] for _i in DERIVATIVES]
_adf_y_arrays = [[] for _i in DERIVATIVES]
for _tuple in tqdm(_tuples, desc='Experiments loop'):
_experiment, _series_id, _cell_id = _tuple
_cell_fiber_densities = compute_single_cell_mean(
_experiment=_experiment,
_series_id=_series_id,
_cell_tuples=_tuples[_tuple],
_windows_dictionary=_windows_dictionary,
_fiber_densities=_fiber_densities)
for _derivative_index, _derivative in enumerate(DERIVATIVES):
_cell_fiber_densities_derivative = compute_lib.derivative(
_cell_fiber_densities, _n=_derivative)
_, _kpss_p_value, _, _ = kpss(_cell_fiber_densities_derivative,
nlags='legacy')
_kpss_y_arrays[_derivative_index].append(_kpss_p_value)
_, _adf_p_value, _, _, _, _ = adfuller(
_cell_fiber_densities_derivative)
_adf_y_arrays[_derivative_index].append(_adf_p_value)
print('Total cells:', len(_kpss_y_arrays[0]))
# print results
print('KPSS:')
for _derivative_index, _derivative in enumerate(DERIVATIVES):
_stationary_count = len([
_value for _value in _kpss_y_arrays[_derivative_index]
if _value > 0.05
])
print(
'Derivative:', _derivative, 'Stationary:',
str(_stationary_count / len(_kpss_y_arrays[_derivative_index]) *
100) + '%')
print('ADF:')
for _derivative_index, _derivative in enumerate(DERIVATIVES):
_stationary_count = len([
_value for _value in _adf_y_arrays[_derivative_index]
if _value < 0.05
])
print(
'Derivative:', _derivative, 'Stationary:',
str(_stationary_count / len(_adf_y_arrays[_derivative_index]) *
100) + '%')
# plot
_colors_array = config.colors(3)
for _test_name, _y_title, _y_tickvals, _p_value_line, _y_arrays in \
zip(
['kpss', 'adf'],
['KPSS test p-value', 'ADF test p-value'],
[[0.05, 0.1], [0.05, 1]],
[0.05, 0.05],
[_kpss_y_arrays, _adf_y_arrays]
):
_fig = go.Figure(data=[
go.Box(y=_y,
name=_derivative,
boxpoints='all',
jitter=1,
pointpos=0,
line={'width': 1},
fillcolor='white',
marker={
'size': 10,
'color': _color
},
opacity=0.7,
showlegend=False) for _y, _derivative, _color in zip(
_y_arrays, DERIVATIVES_TEXT, _colors_array)
],
layout={
'xaxis': {
'title': 'Fiber density derivative',
'zeroline': False
},
'yaxis': {
'title': _y_title,
'zeroline': False,
'tickmode': 'array',
'tickvals': _y_tickvals
},
'shapes': [{
'type': 'line',
'x0': DERIVATIVES[0] - 0.75,
'y0': _p_value_line,
'x1': DERIVATIVES[-1] + 0.75,
'y1': _p_value_line,
'line': {
'color': 'red',
'width': 2,
'dash': 'dash'
}
}]
})
save.to_html(_fig=_fig,
_path=os.path.join(paths.PLOTS, save.get_module_name()),
_filename='plot_' + _test_name)
def main():
print('Single Cell')
_experiments = all_experiments()
_experiments = filtering.by_categories(_experiments=_experiments,
_is_single_cell=True,
_is_high_temporal_resolution=False,
_is_bleb=False,
_is_dead_dead=False,
_is_live_dead=False,
_is_bead=False,
_is_metastasis=False)
_tuples = load.experiments_groups_as_tuples(_experiments)
_tuples = filtering.by_time_frames_amount(
_tuples, compute.minimum_time_frames_for_correlation(_experiments[0]))
_arguments = []
for _tuple in _tuples:
_experiment, _series_id, _group = _tuple
_time_frame = compute.minimum_time_frames_for_correlation(_experiment)
for _offset_x, _direction in product(OFFSETS_X, ['left', 'right']):
_arguments.append({
'experiment': _experiment,
'series_id': _series_id,
'group': _group,
'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER,
'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER,
'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER,
'offset_x': _offset_x,
'offset_y': OFFSET_Y,
'offset_z': OFFSET_Z,
'cell_id': 'cell',
'direction': _direction,
'time_point': _time_frame - 1
})
_windows_dictionary, _windows_to_compute = \
compute.windows(_arguments, _keys=['experiment', 'series_id', 'group', 'offset_x', 'direction'])
_fiber_densities = compute.fiber_densities(_windows_to_compute)
_tuples = organize.by_single_cell_id(_tuples)
_single_cell_fiber_densities = [[] for _i in range(len(OFFSETS_X))]
for _tuple in _tuples:
_experiment, _series_id, _cell_id = _tuple
print('Experiment:',
_experiment,
'Series ID:',
_series_id,
'Cell ID:',
_cell_id,
sep='\t')
_offset_index = 0
_normalization = load.normalization_series_file_data(
_experiment, _series_id)
for _offset_x in OFFSETS_X:
_cell_fiber_densities = []
for _cell_tuple in _tuples[_tuple]:
_, _, _group = _cell_tuple
for _direction in ['left', 'right']:
_window_tuple = _windows_dictionary[(_experiment,
_series_id, _group,
_offset_x,
_direction)][0]
_fiber_density = _fiber_densities[_window_tuple]
if not OUT_OF_BOUNDARIES and _fiber_density[1]:
continue
_normalized_fiber_density = compute_lib.z_score(
_x=_fiber_density[0],
_average=_normalization['average'],
_std=_normalization['std'])
_cell_fiber_densities.append(_normalized_fiber_density)
if len(_cell_fiber_densities) > 0:
_single_cell_fiber_densities[_offset_index].append(
np.mean(_cell_fiber_densities))
_offset_index += 1
print('Pairs')
_experiments = all_experiments()
_experiments = filtering.by_categories(_experiments=_experiments,
_is_single_cell=False,
_is_high_temporal_resolution=False,
_is_bleb=False,
_is_dead_dead=False,
_is_live_dead=False,
_is_bead=False,
_is_metastasis=False)
_tuples = load.experiments_groups_as_tuples(_experiments)
_tuples = filtering.by_time_frames_amount(
_tuples, compute.minimum_time_frames_for_correlation(_experiments[0]))
_tuples = filtering.by_real_pairs(_tuples)
_tuples = filtering.by_pair_distance(_tuples, PAIR_DISTANCE)
_tuples = filtering.by_band(_tuples)
_arguments = []
for _tuple in _tuples:
_experiment, _series_id, _group = _tuple
_time_frame = compute.minimum_time_frames_for_correlation(_experiment)
for _offset_x in OFFSETS_X:
_arguments.append({
'experiment': _experiment,
'series_id': _series_id,
'group': _group,
'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER,
'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER,
'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER,
'offset_x': _offset_x,
'offset_y': OFFSET_Y,
'offset_z': OFFSET_Z,
'cell_id': 'left_cell',
'direction': 'inside',
'time_point': _time_frame - 1
})
_windows_dictionary, _windows_to_compute = \
compute.windows(_arguments, _keys=['experiment', 'series_id', 'group', 'offset_x'])
_fiber_densities = compute.fiber_densities(_windows_to_compute)
_pairs_fiber_densities = [[] for _i in range(len(OFFSETS_X))]
for _tuple in _tuples:
_experiment, _series_id, _group = _tuple
print('Experiment:',
_experiment,
'Series ID:',
_series_id,
'Group:',
_group,
sep='\t')
_offset_index = 0
_normalization = load.normalization_series_file_data(
_experiment, _series_id)
# take offsets based on pair distance
_properties = load.group_properties(_experiment, _series_id, _group)
_left_cell_coordinates = [
list(_properties['time_points'][0]['left_cell']
['coordinates'].values())
]
_right_cell_coordinates = [
list(_properties['time_points'][0]['right_cell']
['coordinates'].values())
]
_pair_distance = compute.pair_distance_in_cell_size(
_experiment, _series_id, _left_cell_coordinates,
_right_cell_coordinates)
_edges_distance = _pair_distance - 1
_max_x_offset = _edges_distance - QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER
for _offset_x in OFFSETS_X:
if _offset_x > _max_x_offset:
break
_fiber_density = _fiber_densities[_windows_dictionary[(
_experiment, _series_id, _group, _offset_x)][0]]
if not OUT_OF_BOUNDARIES and _fiber_density[1]:
continue
_normalized_fiber_density = compute_lib.z_score(
_x=_fiber_density[0],
_average=_normalization['average'],
_std=_normalization['std'])
_pairs_fiber_densities[_offset_index].append(
_normalized_fiber_density)
_offset_index += 1
# plot
_fig = go.Figure(data=[
go.Scatter(x=OFFSETS_X,
y=[np.mean(_array) for _array in _pairs_fiber_densities],
name='Pairs',
error_y={
'type':
'data',
'array':
[np.std(_array) for _array in _pairs_fiber_densities],
'thickness':
1
},
mode='lines+markers',
line={'dash': 'solid'}),
go.Scatter(
x=OFFSETS_X,
y=[np.mean(_array) for _array in _single_cell_fiber_densities],
name='Single Cell',
error_y={
'type':
'data',
'array':
[np.std(_array) for _array in _single_cell_fiber_densities],
'thickness':
1
},
mode='lines+markers',
line={'dash': 'dash'})
],
layout={
'xaxis_title': 'Distance from left cell (cell size)',
'yaxis_title': 'Fiber density (z-score)'
})
save.to_html(_fig=_fig,
_path=os.path.join(paths.PLOTS, save.get_module_name()),
_filename='plot_distance_' + str(PAIR_DISTANCE))
def main():
_experiments = all_experiments()
_experiments = filtering.by_categories(_experiments=_experiments,
_is_single_cell=True,
_is_high_temporal_resolution=False,
_is_bleb=False,
_is_dead_dead=False,
_is_live_dead=False,
_is_bead=False,
_is_metastasis=False)
_tuples = load.experiments_groups_as_tuples(_experiments)
_tuples = filtering.by_time_frames_amount(
_tuples, compute.density_time_frame(_experiments[0]))
_tuples = filtering.by_main_cell(_tuples)
_arguments = []
for _tuple in _tuples:
_experiment, _series_id, _group = _tuple
_time_frame = compute.density_time_frame(_experiment)
for _direction in ['left', 'right']:
_arguments.append({
'experiment': _experiment,
'series_id': _series_id,
'group': _group,
'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER,
'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER,
'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER,
'offset_x': OFFSET_X,
'offset_y': OFFSET_Y,
'offset_z': OFFSET_Z,
'cell_id': 'cell',
'direction': _direction,
'time_points': _time_frame
})
_windows_dictionary, _windows_to_compute = \
compute.windows(_arguments, _keys=['experiment', 'series_id', 'group', 'direction'])
_fiber_densities = compute.fiber_densities(_windows_to_compute,
_subtract_border=True)
_tuples = organize.by_single_cell_id(_tuples)
print('Total tuples:', len(_tuples))
_experiments_ids = list(_tuples.keys())
_y_arrays = [[] for _i in DERIVATIVES]
for _index_1 in tqdm(range(len(_experiments_ids)), desc='Main loop'):
_tuple_1 = _experiments_ids[_index_1]
_experiment_1, _series_id_1, _cell_id_1 = _tuple_1
_fiber_densities_1 = compute_single_cell_mean(
_experiment=_experiment_1,
_series_id=_series_id_1,
_cell_tuples=_tuples[_tuple_1],
_windows_dictionary=_windows_dictionary,
_fiber_densities=_fiber_densities)
for _index_2 in range(_index_1 + 1, len(_experiments_ids)):
_tuple_2 = _experiments_ids[_index_2]
_experiment_2, _series_id_2, _cell_id_2 = _tuple_2
_fiber_densities_2 = compute_single_cell_mean(
_experiment=_experiment_2,
_series_id=_series_id_2,
_cell_tuples=_tuples[_tuple_2],
_windows_dictionary=_windows_dictionary,
_fiber_densities=_fiber_densities)
for _derivative_index, _derivative in enumerate(DERIVATIVES):
_y_arrays[_derivative_index].append(
compute_lib.correlation(
compute_lib.derivative(_fiber_densities_1,
_n=_derivative),
compute_lib.derivative(_fiber_densities_2,
_n=_derivative)))
print('Total points:', len(_y_arrays[0]))
print('Wilcoxon around the zero')
for _y_array, _derivative in zip(_y_arrays, DERIVATIVES):
print('Derivative:', _derivative, wilcoxon(_y_array))
# plot
_colors_array = config.colors(3)
_fig = go.Figure(data=[
go.Box(y=_y,
name=_derivative,
boxpoints='all',
jitter=1,
pointpos=0,
line={'width': 1},
fillcolor='white',
marker={
'size': 10,
'color': _color
},
opacity=0.7,
showlegend=False) for _y, _derivative, _color in zip(
_y_arrays, DERIVATIVES_TEXT, _colors_array)
],
layout={
'xaxis': {
'title': 'Fiber density derivative',
'zeroline': False
},
'yaxis': {
'title': 'Correlation',
'range': [-1, 1],
'zeroline': False,
'tickmode': 'array',
'tickvals': [-1, -0.5, 0, 0.5, 1]
}
})
save.to_html(_fig=_fig,
_path=os.path.join(paths.PLOTS, save.get_module_name()),
_filename='plot')
def compute_experiments_fiber_densities():
_experiments = all_experiments()
_experiments = experiments_filtering.by_categories(
_experiments=_experiments,
_is_single_cell=True,
_is_high_temporal_resolution=False,
_is_bleb=False,
_is_dead_dead=False,
_is_live_dead=False,
_is_bead=False,
_is_metastasis=False)
_tuples = experiments_load.experiments_groups_as_tuples(_experiments)
_tuples = experiments_filtering.by_time_frames_amount(
_tuples, EXPERIMENTS_TIME_FRAMES)
_tuples = experiments_filtering.by_main_cell(_tuples)
_arguments = []
for _tuple in _tuples:
_experiment, _series_id, _group = _tuple
for _direction in ['left', 'right']:
_arguments.append({
'experiment': _experiment,
'series_id': _series_id,
'group': _group,
'length_x': experiments_config.
QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER,
'length_y': experiments_config.
QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER,
'length_z': experiments_config.
QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER,
'offset_x': OFFSET_X,
'offset_y': OFFSET_Y,
'offset_z': OFFSET_Z,
'cell_id': 'cell',
'direction': _direction,
'time_points': EXPERIMENTS_TIME_FRAMES
})
_windows_dictionary, _windows_to_compute = \
experiments_compute.windows(_arguments, _keys=['experiment', 'series_id', 'group', 'direction'])
_fiber_densities = experiments_compute.fiber_densities(_windows_to_compute)
_tuples = experiments_organize.by_single_cell_id(_tuples)
print('Total experiments:', len(_tuples))
_experiments_fiber_densities = [[]
for _i in range(EXPERIMENTS_TIME_FRAMES)]
for _tuple in tqdm(_tuples, desc='Experiments loop'):
_experiment, _series_id, _ = _tuple
_normalization = experiments_load.normalization_series_file_data(
_experiment, _series_id)
for _time_frame in range(EXPERIMENTS_TIME_FRAMES):
_cell_fiber_densities = []
for _cell_tuple in _tuples[_tuple]:
_, _, _group = _cell_tuple
for _direction in ['left', 'right']:
_window_tuple = _windows_dictionary[(
_experiment, _series_id, _group,
_direction)][_time_frame]
_fiber_density = _fiber_densities[_window_tuple]
if not OUT_OF_BOUNDARIES and _fiber_density[1]:
continue
_normalized_fiber_density = compute_lib.z_score(
_x=_fiber_density[0],
_average=_normalization['average'],
_std=_normalization['std'])
if not np.isnan(_normalized_fiber_density):
_cell_fiber_densities.append(_normalized_fiber_density)
if len(_cell_fiber_densities) > 0:
_experiments_fiber_densities[_time_frame].append(
np.mean(_cell_fiber_densities))
print('Total experiments cells:', len(_experiments_fiber_densities[0]))
return _experiments_fiber_densities