def main():
_tuples = [('Metastasis_Airyscan_210401_sgAXL', 2, 'cells_1_2', True),
('Metastasis_Airyscan_210401_sgAXL', 3, 'cells_1_3', True),
('Metastasis_Airyscan_210401_sgAXL', 5, 'cells_0_1', True),
('Metastasis_Airyscan_210401_sgAXL', 9, 'cells_0_5', True),
('Metastasis_Airyscan_210401_sgAXL', 11, 'cells_1_2', True),
('Metastasis_Airyscan_210401_sgAXL', 14, 'cells_0_1', True),
('Metastasis_Airyscan_210401_sgAXL', 15, 'cells_1_2', True)]
for _tuple in _tuples:
_e, _s, _g, _b = _tuple
_p = load.group_properties(_e, _s, _g)
_p['band'] = _b
_group_structured_path = paths.structured(_e, _s, _g)
_properties_json_path = os.path.join(_group_structured_path,
'properties.json')
save_lib.to_json(_p, _properties_json_path)
print(_tuple)
_g_fake = 'fake_' + _g.split('cells_')[1]
_group_structured_path = paths.structured(_e, _s, _g_fake)
_properties_json_path = os.path.join(_group_structured_path,
'properties.json')
if os.path.isfile(_properties_json_path):
print(_g_fake)
_p = load.group_properties(_e, _s, _g_fake)
_p['band'] = _b
save_lib.to_json(_p, _properties_json_path)
def process_group(_experiment, _series_id, _group, _overwrite=False):
_group_properties = load.group_properties(_experiment, _series_id, _group)
if not _overwrite and _group_properties['band'] is not None:
return
if 'static' in _group:
return
elif 'fake' in _group:
_group_real = 'cells_' + _group.split('fake_')[1]
else:
_group_real = _group
_group_real_properties = load.group_properties(_experiment, _series_id,
_group_real)
_time_frames_amount = len(_group_real_properties['time_points'])
_fiber_density = []
for _time_frame in [
0, int(round(_time_frames_amount / 2)), _time_frames_amount - 1
]:
_left_cell_coordinates = _group_real_properties['time_points'][
_time_frame]['left_cell']['coordinates']
_right_cell_coordinates = _group_real_properties['time_points'][
_time_frame]['right_cell']['coordinates']
_cell_diameter_x = \
AVERAGE_CELL_DIAMETER_IN_MICRONS / _group_real_properties['time_points'][_time_frame]['resolutions']['x']
_cell_diameter_y = \
AVERAGE_CELL_DIAMETER_IN_MICRONS / _group_real_properties['time_points'][_time_frame]['resolutions']['y']
_cell_diameter_z = \
AVERAGE_CELL_DIAMETER_IN_MICRONS / _group_real_properties['time_points'][_time_frame]['resolutions']['z']
_x1 = (_left_cell_coordinates['x'] +
_right_cell_coordinates['x']) / 2 - _cell_diameter_x / 2
_x2 = _x1 + _cell_diameter_x
_y1 = (_left_cell_coordinates['y'] +
_right_cell_coordinates['y']) / 2 - _cell_diameter_y / 2
_y2 = _y1 + _cell_diameter_y
_z1 = (_left_cell_coordinates['z'] +
_right_cell_coordinates['z']) / 2 - _cell_diameter_z / 2
_z2 = _z1 + _cell_diameter_z
_window = [_x1, _y1, _z1, _x2, _y2, _z2]
_fiber_density = compute.window_fiber_density(
_experiment=_experiment,
_series_id=_series_id,
_group=_group_real,
_time_frame=_time_frame,
_window=[int(round(_value)) for _value in _window])[:2]
_fiber_density.append(_fiber_density)
if _fiber_density[0] < _fiber_density[1] < _fiber_density[2]:
_band = True
else:
_band = False
print(_experiment, _series_id, _group, _band, sep='\t')
_group_properties['band'] = _band
_group_structured_path = paths.structured(_experiment, _series_id, _group)
_properties_json_path = os.path.join(_group_structured_path,
'properties.json')
save_lib.to_json(_group_properties, _properties_json_path)
def by_pair_distance_range(_experiments_tuples,
_distance_range,
_time_frame=0):
_experiments_tuples_filtered = []
for _tuple in _experiments_tuples:
_experiment, _series_id, _group = _tuple
_group_properties = load.group_properties(_experiment, _series_id,
_group)
if len(_group_properties['time_points']) <= _time_frame:
continue
_left_cell_coordinates = _group_properties['time_points'][_time_frame][
'left_cell']['coordinates']
_right_cell_coordinates = _group_properties['time_points'][
_time_frame]['right_cell']['coordinates']
_pair_distance = compute.pair_distance_in_cell_size(
_experiment=_experiment,
_series_id=_series_id,
_cell_1_coordinates=[
(_left_cell_coordinates['x'], _left_cell_coordinates['y'],
_left_cell_coordinates['z'])
],
_cell_2_coordinates=[
(_right_cell_coordinates['x'], _right_cell_coordinates['y'],
_right_cell_coordinates['z'])
])
if _distance_range[0] <= _pair_distance <= _distance_range[1]:
_experiments_tuples_filtered.append(_tuple)
return _experiments_tuples_filtered
def by_pair_distance(_experiments_tuples):
_tuples_by_distance = {}
for _tuple in _experiments_tuples:
_experiment, _series_id, _group = _tuple
_group_properties = load.group_properties(_experiment, _series_id,
_group)
_left_cell_coordinates = _group_properties['time_points'][0][
'left_cell']['coordinates']
_right_cell_coordinates = _group_properties['time_points'][0][
'right_cell']['coordinates']
_pair_distance = int(
round(
compute.pair_distance_in_cell_size(
_experiment=_experiment,
_series_id=_series_id,
_cell_1_coordinates=[(_left_cell_coordinates['x'],
_left_cell_coordinates['y'],
_left_cell_coordinates['z'])],
_cell_2_coordinates=[(_right_cell_coordinates['x'],
_right_cell_coordinates['y'],
_right_cell_coordinates['z'])])))
if _pair_distance in _tuples_by_distance:
_tuples_by_distance[_pair_distance].append(_tuple)
else:
_tuples_by_distance[_pair_distance] = [_tuple]
return {
_distance: _tuples_by_distance[_distance]
for _distance in sorted(_tuples_by_distance.keys())
}
def by_fake_static_pairs(_experiments_tuples, _fake_static_pairs=True):
_experiments_tuples_filtered = []
for _tuple in _experiments_tuples:
_experiment, _series_id, _group = _tuple
_group_properties = load.group_properties(_experiment, _series_id,
_group)
if _group_properties['static'] == _fake_static_pairs:
_experiments_tuples_filtered.append(_tuple)
return _experiments_tuples_filtered
def compute_data(_arguments):
_offset_y_index, _offset_y, _offset_z_index, _offset_z = _arguments
_fiber_density_changes_array = []
for _tuple in _tuples:
_experiment, _series_id, _group = _tuple
_normalization = load.normalization_series_file_data(
_experiment, _series_id)
_properties = load.group_properties(_experiment, _series_id, _group)
for _cell_id in ['left_cell', 'right_cell']:
_cell_fiber_densities = \
_experiments_fiber_densities[(_experiment, _series_id, _group, _offset_y, _offset_z, _cell_id)]
_cell_fiber_densities = compute.remove_blacklist(
_experiment, _series_id, _properties['cells_ids'][_cell_id],
_cell_fiber_densities)
_previous_cell_fiber_density_normalized = None
_cell_values = []
for _time_frame, _cell_fiber_density in enumerate(
_cell_fiber_densities):
# not out of border
if _cell_fiber_density[1]:
_previous_cell_fiber_density_normalized = None
continue
# normalize
_cell_fiber_density_normalized = compute_lib.z_score(
_x=_cell_fiber_density[0],
_average=_normalization['average'],
_std=_normalization['std'])
# no previous
if _previous_cell_fiber_density_normalized is None:
_previous_cell_fiber_density_normalized = _cell_fiber_density_normalized
continue
# change
_cell_fiber_density_normalized_change = abs(
_cell_fiber_density_normalized -
_previous_cell_fiber_density_normalized)
_previous_cell_fiber_density_normalized = _cell_fiber_density_normalized
# save
_cell_values.append(_cell_fiber_density_normalized_change)
# save
if len(_cell_values) > 0:
_fiber_density_changes_array.append(np.mean(_cell_values))
if len(_fiber_density_changes_array) > 0:
return _offset_y_index, _offset_z_index, np.mean(
_fiber_density_changes_array)
else:
return _offset_y_index, _offset_z_index, None
def by_real_pairs(_experiments_tuples, _real_pairs=True):
_experiments_tuples_filtered = []
for _tuple in _experiments_tuples:
_experiment, _series_id, _group = _tuple
_group_properties = load.group_properties(_experiment, _series_id,
_group)
if _group_properties['fake'] != _real_pairs and \
('real_fake' not in _group_properties or _group_properties['real_fake'] != _real_pairs):
_experiments_tuples_filtered.append(_tuple)
return _experiments_tuples_filtered
def pair_distance_in_cell_size_time_frame(_experiment, _series_id, _group,
_time_frame):
_group_properties = load.group_properties(_experiment, _series_id, _group)
_left_cell_coordinates = \
[list(_group_properties['time_points'][_time_frame]['left_cell']['coordinates'].values())]
_right_cell_coordinates = \
[list(_group_properties['time_points'][_time_frame]['right_cell']['coordinates'].values())]
return pair_distance_in_cell_size(_experiment, _series_id,
_left_cell_coordinates,
_right_cell_coordinates)
def by_time_frames_amount(_experiments_tuples, _time_frames, _exactly=False):
_experiments_tuples_filtered = []
for _tuple in _experiments_tuples:
_experiment, _series_id, _group = _tuple
_group_properties = load.group_properties(_experiment, _series_id,
_group)
_time_frames_amount = len(_group_properties['time_points'])
if (_exactly and _time_frames_amount == _time_frames) or \
(not _exactly and _time_frames_amount >= _time_frames):
_experiments_tuples_filtered.append(_tuple)
return _experiments_tuples_filtered
def latest_time_frame_before_overlapping(_experiment, _series_id, _group,
_offset_x):
_properties = load.group_properties(_experiment, _series_id, _group)
_latest_time_frame = len(_properties['time_points'])
for _time_frame in range(len(_properties['time_points'])):
_pair_distance = \
pair_distance_in_cell_size_time_frame(_experiment, _series_id, _group, _time_frame)
if _pair_distance - 1 - _offset_x * 2 < QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER * 2:
_latest_time_frame = _time_frame - 1
break
return _latest_time_frame
def by_band(_experiments_tuples, _band=True):
if _band is None:
return _experiments_tuples
_experiments_tuples_filtered = []
for _tuple in _experiments_tuples:
_experiment, _series_id, _group = _tuple
_group_properties = load.group_properties(_experiment, _series_id,
_group)
_experiments_tuples_filtered.append(
_tuple) if _group_properties['band'] == _band else None
return _experiments_tuples_filtered
def process_group_pairs(_experiment,
_series_id,
_group,
_mark_cells=True,
_draw_borders=True):
_group_properties = load.group_properties(_experiment, _series_id, _group)
_group_path = paths.images(_experiment + ' - All TPs', _series_id, _group)
os.makedirs(_group_path, exist_ok=True)
for _time_frame in range(0, len(_group_properties['time_points']), 1):
print(_experiment, _series_id, _group, _time_frame, sep='\t')
_time_frame_image = load.structured_image(_experiment, _series_id,
_group, _time_frame)
_left_cell_coordinates = _group_properties['time_points'][_time_frame][
'left_cell']['coordinates']
_right_cell_coordinates = _group_properties['time_points'][
_time_frame]['right_cell']['coordinates']
_left_cell_id = _group_properties['cells_ids']['left_cell']
_right_cell_id = _group_properties['cells_ids']['right_cell']
_cell_diameter_in_microns = AVERAGE_CELL_DIAMETER_IN_MICRONS
_z_cell_diameter = _cell_diameter_in_microns / _group_properties[
'time_points'][_time_frame]['resolutions']['z']
_z_image = _time_frame_image[
int(round(_left_cell_coordinates['z'] - _z_cell_diameter / 2)
):int(round(_left_cell_coordinates['z'] +
_z_cell_diameter / 2))]
_average_across_z = np.rint(np.mean(_z_image, axis=0))
if _mark_cells:
_average_across_z = mark_cells(_average_across_z,
_group_properties, _time_frame,
_left_cell_coordinates,
AVERAGE_CELL_DIAMETER_IN_MICRONS)
_average_across_z = mark_cells(_average_across_z,
_group_properties, _time_frame,
_right_cell_coordinates,
AVERAGE_CELL_DIAMETER_IN_MICRONS)
if _draw_borders:
_left_window = get_window(_experiment, _series_id,
_group_properties, _time_frame,
'left_cell', DIRECTION)
_right_window = get_window(_experiment, _series_id,
_group_properties, _time_frame,
'right_cell', DIRECTION)
_average_across_z = draw_borders(_average_across_z, _left_window)
_average_across_z = draw_borders(_average_across_z, _right_window)
_img = Image.fromarray(_average_across_z).convert('L')
_img_path = os.path.join(_group_path, str(_time_frame) + '.png')
_img.save(_img_path)
def compute_data(_tuples, _arguments, _padding_y_by, _padding_z_by, _space_y_by, _space_z_by):
_windows_dictionary, _windows_to_compute = \
compute.windows(_arguments, _keys=['experiment', 'series_id', 'group', 'cell_id'])
_fiber_densities = compute.fiber_densities(_windows_to_compute, _subtract_border=True, _padding_y_by=_padding_y_by,
_padding_z_by=_padding_z_by, _space_y_by=_space_y_by,
_space_z_by=_space_z_by)
_experiments_fiber_densities = {
_key: [_fiber_densities[_tuple] for _tuple in _windows_dictionary[_key]]
for _key in _windows_dictionary
}
_correlations = []
for _tuple in _tuples:
_experiment, _series, _group = _tuple
_left_cell_fiber_densities = _experiments_fiber_densities[(_experiment, _series, _group, 'left_cell')]
_right_cell_fiber_densities = _experiments_fiber_densities[(_experiment, _series, _group, 'right_cell')]
_properties = load.group_properties(_experiment, _series, _group)
_left_cell_fiber_densities = compute.remove_blacklist(
_experiment,
_series,
_properties['cells_ids']['left_cell'],
_left_cell_fiber_densities
)
_right_cell_fiber_densities = compute.remove_blacklist(
_experiment,
_series,
_properties['cells_ids']['right_cell'],
_right_cell_fiber_densities
)
_left_cell_fiber_densities_filtered, _right_cell_fiber_densities_filtered = \
compute.longest_same_indices_shared_in_borders_sub_array(
_left_cell_fiber_densities, _right_cell_fiber_densities
)
# ignore small arrays
if len(_left_cell_fiber_densities_filtered) < compute.minimum_time_frames_for_correlation(_experiment):
continue
_correlation = compute_lib.correlation(
compute_lib.derivative(_left_cell_fiber_densities_filtered, _n=DERIVATIVE),
compute_lib.derivative(_right_cell_fiber_densities_filtered, _n=DERIVATIVE)
)
_correlations.append(_correlation)
return np.mean(_correlations)
def process_group(_experiment, _series_id, _group):
_properties = load.group_properties(_experiment, _series_id, _group)
_cell_1_coordinates = _properties['time_points'][0]['left_cell'][
'coordinates'].values()
_cell_2_coordinates = _properties['time_points'][0]['right_cell'][
'coordinates'].values()
_pair_distance = compute.pair_distance_in_cell_size(
_experiment, _series_id, [_cell_1_coordinates], [_cell_2_coordinates])
print(_experiment,
_series_id,
_group,
_pair_distance,
_properties['band'],
_properties['fake'],
_properties['static'],
len(_properties['time_points']),
sep='\t')
def main():
_experiment = 'LiveDead_201220'
_band = False
_tuples = load.experiment_groups_as_tuples(_experiment)
print('Total tuples:', len(_tuples))
for _tuple in _tuples:
print(_tuple)
_experiment, _series_id, _group = _tuple
_properties = load.group_properties(_experiment, _series_id, _group)
_properties['band'] = _band
_group_structured_path = paths.structured(_experiment, _series_id,
_group)
_properties_json_path = os.path.join(_group_structured_path,
'properties.json')
save_lib.to_json(_properties, _properties_json_path)
def windows_by_time(_arguments):
_group_properties = load.group_properties(_arguments['experiment'],
_arguments['series_id'],
_arguments['group'])
# single time point
if 'time_point' in _arguments:
return _arguments, [window_time_frame(_arguments)]
# multiple time points
if 'time_points' not in _arguments:
_arguments['time_points'] = sys.maxsize
_windows = []
for _time_frame in range(
min(_arguments['time_points'],
len(_group_properties['time_points']))):
_arguments['time_point'] = _time_frame
_windows.append(window_time_frame(_arguments))
return _arguments, _windows
def process_group_single_cells(_experiment,
_series_id,
_group,
_mark_cells=True,
_draw_borders=True):
_group_properties = load.group_properties(_experiment, _series_id, _group)
_group_path = paths.images(_experiment + ' - First TP - Start Not Average',
_series_id, _group)
os.makedirs(_group_path, exist_ok=True)
for _time_frame in [0]:
print(_experiment, _series_id, _group, _time_frame, sep='\t')
_time_frame_image = load.structured_image(_experiment, _series_id,
_group, _time_frame)
_cell_coordinates = _group_properties['time_points'][_time_frame][
'cell']['coordinates']
_cell_diameter_in_microns = AVERAGE_CELL_DIAMETER_IN_MICRONS
_z_cell_diameter = _cell_diameter_in_microns / _group_properties[
'time_points'][_time_frame]['resolutions']['z']
_z_image = _time_frame_image[int(
round(_cell_coordinates['z'] - _z_cell_diameter /
2)):int(round(_cell_coordinates['z'] +
_z_cell_diameter / 2))]
_average_across_z = np.rint(np.mean(_z_image, axis=0))
if _mark_cells:
_average_across_z = mark_cells(_average_across_z,
_group_properties, _time_frame,
_cell_coordinates,
AVERAGE_CELL_DIAMETER_IN_MICRONS)
if _draw_borders:
for _direction in ['left', 'right', 'up', 'down']:
_window = get_window(_experiment, _series_id,
_group_properties, _time_frame, 'cell',
_direction)
_average_across_z = draw_borders(_average_across_z, _window)
_img = Image.fromarray(_average_across_z).convert('L')
_img_path = os.path.join(_group_path, str(_time_frame) + '.png')
_img.save(_img_path)
def window_time_frame(_arguments):
if 'group_properties' not in _arguments:
_arguments['group_properties'] = \
load.group_properties(_arguments['experiment'], _arguments['series_id'], _arguments['group'])
_time_frame_properties = _arguments['group_properties']['time_points'][
_arguments['time_point']]
if QUANTIFICATION_WINDOW_START_BY_AVERAGE_CELL_DIAMETER:
_cell_diameter_in_microns = AVERAGE_CELL_DIAMETER_IN_MICRONS
else:
_cell_diameter_in_microns = load.mean_distance_to_surface_in_microns(
_experiment=_arguments['experiment'],
_series_id=_arguments['series_id'],
_cell_id=_arguments['group_properties']['cells_ids'][_arguments['cell_id']]) * 2 \
if _arguments['cell_id'] != 'cell' else _arguments['group_properties']['cell_id'] * 2
_time_frame_window = window_by_microns(
_resolution_x=_arguments['group_properties']['time_points'][
_arguments['time_point']]['resolutions']['x'],
_resolution_y=_arguments['group_properties']['time_points'][
_arguments['time_point']]['resolutions']['y'],
_resolution_z=_arguments['group_properties']['time_points'][
_arguments['time_point']]['resolutions']['z'],
_length_x=_arguments['length_x'],
_length_y=_arguments['length_y'],
_length_z=_arguments['length_z'],
_offset_x=_arguments['offset_x'],
_offset_y=_arguments['offset_y'],
_offset_z=_arguments['offset_z'],
_cell_coordinates=_arguments['group_properties']['time_points'][
_arguments['time_point']][_arguments['cell_id']]['coordinates'],
_cell_diameter_in_microns=_cell_diameter_in_microns,
_direction='right' if
(_arguments['cell_id'], _arguments['direction']) == ('left_cell',
'inside') or
(_arguments['cell_id'], _arguments['direction']) == ('right_cell',
'outside') or
(_arguments['cell_id'],
_arguments['direction']) == ('cell', 'right') else 'left')
return _arguments['experiment'], _arguments['series_id'], _arguments[
'group'], _arguments['time_point'], _time_frame_window
def process_group(_experiment, _series_id, _cells_coordinates, _cell_1_id, _cell_2_id, _series_image_by_time_frames,
_resolutions, _real_cells=True, _fake_cell_1_id=None, _fake_cell_2_id=None,
_x_change=0, _y_change=0, _z_change=0, _overwrite=False):
_time_frames_data = []
_left_cell_id = None
_right_cell_id = None
_time_frames_amount = min(
len([_value for _value in _cells_coordinates[_cell_1_id] if _value is not None]),
len([_value for _value in _cells_coordinates[_cell_2_id] if _value is not None])
)
# smooth coordinates
_cells_coordinates_cell_1_smoothed = compute.smooth_coordinates_in_time(
[_value for _value in _cells_coordinates[_cell_1_id] if _value is not None], _n=SMOOTH_AMOUNT
)
_cells_coordinates_cell_2_smoothed = compute.smooth_coordinates_in_time(
[_value for _value in _cells_coordinates[_cell_2_id] if _value is not None], _n=SMOOTH_AMOUNT
)
if _real_cells:
_group = 'cells_' + str(_cell_1_id) + '_' + str(_cell_2_id)
elif _fake_cell_1_id is None:
_group = 'fake_' + str(_cell_1_id) + '_' + str(_cell_2_id)
else:
_group = 'static_' + str(_fake_cell_1_id) + '_' + str(_fake_cell_2_id)
# check if needed (missing time-point / properties file)
if not _overwrite:
_missing = False
for _time_frame in range(_time_frames_amount):
_time_frame_pickle_path = paths.structured(_experiment, _series_id, _group, _time_frame)
if not os.path.isfile(_time_frame_pickle_path):
_missing = True
break
_group_structured_path = paths.structured(_experiment, _series_id, _group)
_properties_json_path = os.path.join(_group_structured_path, 'properties.json')
if not os.path.isfile(_properties_json_path):
_missing = True
if not _missing:
return
# running for each time point
for _time_frame in range(_time_frames_amount):
_time_frame_image = _series_image_by_time_frames[_time_frame]
_cell_1_coordinates = [_value for _value in _cells_coordinates_cell_1_smoothed[_time_frame]]
_cell_2_coordinates = [_value for _value in _cells_coordinates_cell_2_smoothed[_time_frame]]
# update coordinates if needed
if any([_x_change != 0, _y_change != 0, _z_change != 0]):
_cell_1_coordinates = [
_cell_1_coordinates[0] + _x_change,
_cell_1_coordinates[1] + _y_change,
_cell_1_coordinates[2] + _z_change
]
_cell_2_coordinates = [
_cell_2_coordinates[0] + _x_change,
_cell_2_coordinates[1] + _y_change,
_cell_2_coordinates[2] + _z_change
]
# choose left and right cells
if _time_frame == 0:
if _cell_1_coordinates[0] <= _cell_2_coordinates[0]:
_left_cell_id, _right_cell_id = _cell_1_id, _cell_2_id
else:
_right_cell_id, _left_cell_id = _cell_1_id, _cell_2_id
print(_experiment, 'Series ' + str(_series_id), 'Cell 1 #:', _cell_1_id, 'Cell 2 #:', _cell_2_id, 'Time point:',
_time_frame, sep='\t')
# set coordinates
if _left_cell_id == _cell_1_id:
_left_cell_coordinates, _right_cell_coordinates = _cell_1_coordinates, _cell_2_coordinates
else:
_right_cell_coordinates, _left_cell_coordinates = _cell_1_coordinates, _cell_2_coordinates
# compute padding
_helper_coordinates = (_left_cell_coordinates[0] + 1, _left_cell_coordinates[1])
_angle = compute.angle_between_three_points(
_right_cell_coordinates, _left_cell_coordinates, _helper_coordinates
)
_padding_x, _padding_y = compute.axes_padding(_2d_image_shape=_time_frame_image[0].shape, _angle=_angle)
_left_cell_coordinates[0] += _padding_x
_left_cell_coordinates[1] += _padding_y
_right_cell_coordinates[0] += _padding_x
_right_cell_coordinates[1] += _padding_y
# rotate image and change axes
_time_frame_image_rotated = np.array([rotate(_z, _angle) for _z in _time_frame_image])
_time_frame_image_swapped = np.swapaxes(_time_frame_image_rotated, 0, 1)
if SHOW_PLOTS:
plt.imshow(_time_frame_image_rotated[int(round(_left_cell_coordinates[2]))])
plt.show()
plt.imshow(_time_frame_image_rotated[int(round(_right_cell_coordinates[2]))])
plt.show()
# update coordinates
_image_center = compute.image_center_coordinates(_image_shape=reversed(_time_frame_image_rotated[0].shape))
_left_cell_coordinates = compute.rotate_point_around_another_point(
_point=_left_cell_coordinates,
_angle_in_radians=math.radians(_angle),
_around_point=_image_center
)
_right_cell_coordinates = compute.rotate_point_around_another_point(
_point=_right_cell_coordinates,
_angle_in_radians=math.radians(_angle),
_around_point=_image_center
)
_fixed_y = (_left_cell_coordinates[1] + _right_cell_coordinates[1]) / 2
# y is now z
_left_cell_coordinates[1] = _left_cell_coordinates[2]
_right_cell_coordinates[1] = _right_cell_coordinates[2]
# z is now y
_left_cell_coordinates[2] = _fixed_y
_right_cell_coordinates[2] = _fixed_y
if SHOW_PLOTS:
plt.imshow(_time_frame_image_swapped[int(round(_left_cell_coordinates[2]))])
plt.show()
# swap resolutions
_new_resolutions = {
'x': _resolutions['x'],
'y': _resolutions['z'],
'z': _resolutions['y']
}
# second rotate, compute padding
_helper_coordinates = (_left_cell_coordinates[0] + 1, _left_cell_coordinates[1])
_angle = compute.angle_between_three_points(
_right_cell_coordinates, _left_cell_coordinates, _helper_coordinates
)
_padding_x, _padding_y = compute.axes_padding(_2d_image_shape=_time_frame_image_swapped[0].shape, _angle=_angle)
_left_cell_coordinates[0] += _padding_x
_left_cell_coordinates[1] += _padding_y
_right_cell_coordinates[0] += _padding_x
_right_cell_coordinates[1] += _padding_y
# rotate image
_time_frame_image_swapped_rotated = np.array([rotate(_z, _angle) for _z in _time_frame_image_swapped])
# convert to 8 bit color depth
if CONVERT_TO_COLOR_DEPTH_8_BIT:
_time_frame_image_swapped_rotated = \
np.rint(_time_frame_image_swapped_rotated / (math.pow(2, 16) - 1) * (math.pow(2, 8) - 1)).astype(np.uint8)
# update coordinates
_image_center = compute.image_center_coordinates(
_image_shape=reversed(_time_frame_image_swapped_rotated[0].shape))
_left_cell_coordinates = compute.rotate_point_around_another_point(
_point=_left_cell_coordinates,
_angle_in_radians=math.radians(_angle),
_around_point=_image_center
)
_right_cell_coordinates = compute.rotate_point_around_another_point(
_point=_right_cell_coordinates,
_angle_in_radians=math.radians(_angle),
_around_point=_image_center
)
_fixed_y = (_left_cell_coordinates[1] + _right_cell_coordinates[1]) / 2
_left_cell_coordinates[1] = _fixed_y
_right_cell_coordinates[1] = _fixed_y
if SHOW_PLOTS:
if _time_frame == 0 or _time_frame == 50 or _time_frame == 150:
plt.imshow(_time_frame_image_swapped_rotated[int(round(_left_cell_coordinates[2]))])
plt.show()
# update resolutions
_angle = abs(_angle)
_new_resolution_x = (_angle / 90) * _new_resolutions['y'] + ((90 - _angle) / 90) * _new_resolutions['x']
_new_resolution_y = (_angle / 90) * _new_resolutions['x'] + ((90 - _angle) / 90) * _new_resolutions['y']
_new_resolutions['x'] = _new_resolution_x
_new_resolutions['y'] = _new_resolution_y
_image_z, _image_y, _image_x = _time_frame_image_swapped_rotated.shape
if not 0 <= _left_cell_coordinates[0] < _image_x or not \
0 <= _left_cell_coordinates[1] < _image_y or not \
0 <= _left_cell_coordinates[2] < _image_z:
break
if not 0 <= _right_cell_coordinates[0] < _image_x or not \
0 <= _right_cell_coordinates[1] < _image_y or not \
0 <= _right_cell_coordinates[2] < _image_z:
break
# add to array
_time_frames_data.append({
'left_cell': {
'coordinates': {
'x': _left_cell_coordinates[0],
'y': _left_cell_coordinates[1],
'z': _left_cell_coordinates[2]
}
},
'right_cell': {
'coordinates': {
'x': _right_cell_coordinates[0],
'y': _right_cell_coordinates[1],
'z': _right_cell_coordinates[2]
}
},
'resolutions': _new_resolutions
})
# save to pickle
_time_frame_pickle_path = paths.structured(_experiment, _series_id, _group, _time_frame)
save_lib.to_pickle(_time_frame_image_swapped_rotated, _time_frame_pickle_path)
# save properties
if _real_cells:
_band = None
_fake = False
_static = False
elif _fake_cell_1_id is None:
_based_on_properties = load.group_properties(_experiment, _series_id, 'cells_' + _group.split('fake_')[1])
_band = _based_on_properties['band']
_fake = True
_static = False
else:
_band = False
_fake = True
_static = True
_properties_data = {
'experiment': _experiment,
'series_id': _series_id,
'cells_ids': {
'left_cell': _left_cell_id,
'right_cell': _right_cell_id
},
'time_points': _time_frames_data,
'band': _band,
'fake': _fake,
'static': _static
}
_group_structured_path = paths.structured(_experiment, _series_id, _group)
_properties_json_path = os.path.join(_group_structured_path, 'properties.json')
save_lib.to_json(_properties_data, _properties_json_path)
def compute_fiber_densities(_band=True, _high_temporal_resolution=False):
_experiments = all_experiments()
_experiments = filtering.by_categories(
_experiments=_experiments,
_is_single_cell=False,
_is_high_temporal_resolution=_high_temporal_resolution,
_is_bleb=False,
_is_dead_dead=False,
_is_live_dead=False,
_is_bead=False,
_is_metastasis=False)
_tuples = load.experiments_groups_as_tuples(_experiments)
_tuples = filtering.by_pair_distance_range(_tuples, PAIR_DISTANCE_RANGE)
_tuples = filtering.by_real_pairs(_tuples)
_tuples = filtering.by_band(_tuples, _band=_band)
print('Total tuples:', len(_tuples))
_arguments = []
for _tuple in _tuples:
_experiment, _series_id, _group = _tuple
_latest_time_frame = compute.latest_time_frame_before_overlapping(
_experiment, _series_id, _group, OFFSET_X)
for _cell_id in ['left_cell', 'right_cell']:
_arguments.append({
'experiment': _experiment,
'series_id': _series_id,
'group': _group,
'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER,
'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER,
'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER,
'offset_x': OFFSET_X,
'offset_y': OFFSET_Y,
'offset_z': OFFSET_Z,
'cell_id': _cell_id,
'direction': 'inside',
'time_points': _latest_time_frame
})
_windows_dictionary, _windows_to_compute = compute.windows(
_arguments, _keys=['experiment', 'series_id', 'group', 'cell_id'])
_fiber_densities = compute.fiber_densities(_windows_to_compute,
_subtract_border=True)
_experiments_fiber_densities = {
_key:
[_fiber_densities[_tuple] for _tuple in _windows_dictionary[_key]]
for _key in _windows_dictionary
}
_tuples_by_experiment = organize.by_experiment(_tuples)
_distances_from_y_equal_x = []
_z_positions_array = []
for _experiment in _tuples_by_experiment:
print('Experiment:', _experiment)
_experiment_tuples = _tuples_by_experiment[_experiment]
for _same_index in tqdm(range(len(_experiment_tuples)),
desc='Main loop'):
_same_tuple = _experiment_tuples[_same_index]
_same_experiment, _same_series, _same_group = _same_tuple
_same_left_cell_fiber_densities = \
_experiments_fiber_densities[
(_same_experiment, _same_series, _same_group, 'left_cell')
]
_same_right_cell_fiber_densities = \
_experiments_fiber_densities[
(_same_experiment, _same_series, _same_group, 'right_cell')
]
_same_properties = \
load.group_properties(_same_experiment, _same_series, _same_group)
_same_left_cell_fiber_densities = compute.remove_blacklist(
_same_experiment, _same_series,
_same_properties['cells_ids']['left_cell'],
_same_left_cell_fiber_densities)
_same_right_cell_fiber_densities = compute.remove_blacklist(
_same_experiment, _same_series,
_same_properties['cells_ids']['right_cell'],
_same_right_cell_fiber_densities)
_same_left_cell_fiber_densities_filtered, _same_right_cell_fiber_densities_filtered = \
compute.longest_same_indices_shared_in_borders_sub_array(
_same_left_cell_fiber_densities, _same_right_cell_fiber_densities
)
# ignore small arrays
if len(_same_left_cell_fiber_densities_filtered
) < compute.minimum_time_frames_for_correlation(
_same_experiment):
continue
_same_correlation = compute_lib.correlation(
compute_lib.derivative(
_same_left_cell_fiber_densities_filtered, _n=DERIVATIVE),
compute_lib.derivative(
_same_right_cell_fiber_densities_filtered, _n=DERIVATIVE))
_same_group_mean_z_position = \
compute.group_mean_z_position_from_substrate(_same_experiment, _same_series, _same_group)
for _different_index in range(len(_experiment_tuples)):
if _same_index != _different_index:
_different_tuple = _experiment_tuples[_different_index]
_different_experiment, _different_series, _different_group = \
_different_tuple
for _same_cell_id, _different_cell_id in product(
['left_cell', 'right_cell'],
['left_cell', 'right_cell']):
_same_fiber_densities = _experiments_fiber_densities[(
_same_experiment, _same_series, _same_group,
_same_cell_id)]
_different_fiber_densities = _experiments_fiber_densities[
(_different_experiment, _different_series,
_different_group, _different_cell_id)]
_different_properties = load.group_properties(
_different_experiment, _different_series,
_different_group)
_same_fiber_densities = compute.remove_blacklist(
_same_experiment, _same_series,
_same_properties['cells_ids'][_same_cell_id],
_same_fiber_densities)
_different_fiber_densities = compute.remove_blacklist(
_different_experiment, _different_series,
_different_properties['cells_ids']
[_different_cell_id], _different_fiber_densities)
_same_fiber_densities_filtered, _different_fiber_densities_filtered = \
compute.longest_same_indices_shared_in_borders_sub_array(
_same_fiber_densities, _different_fiber_densities
)
# ignore small arrays
if len(_same_fiber_densities_filtered
) < compute.minimum_time_frames_for_correlation(
_different_experiment):
continue
_different_correlation = compute_lib.correlation(
compute_lib.derivative(
_same_fiber_densities_filtered, _n=DERIVATIVE),
compute_lib.derivative(
_different_fiber_densities_filtered,
_n=DERIVATIVE))
_point_distance = compute_lib.distance_from_a_point_to_a_line(
_line=[-1, -1, 1, 1],
_point=[_same_correlation, _different_correlation])
if _same_correlation > _different_correlation:
_distances_from_y_equal_x.append(_point_distance)
else:
_distances_from_y_equal_x.append(-_point_distance)
_z_positions_array.append(_same_group_mean_z_position)
print('Total points:', len(_distances_from_y_equal_x))
print('Wilcoxon of distances from y = x around the zero:')
print(wilcoxon(_distances_from_y_equal_x))
print(
'Pearson correlation of distances from y = x and z position distances:'
)
print(
compute_lib.correlation(_distances_from_y_equal_x,
_z_positions_array,
_with_p_value=True))
return _distances_from_y_equal_x, _z_positions_array
def main():
_arguments = []
for _tuple in TRIPLET:
_experiment, _series_id, _group = _tuple
_pair_distance = compute.pair_distance_in_cell_size_time_frame(_experiment, _series_id, _group, _time_frame=0)
print(_tuple, 'pairs distance:', round(_pair_distance, 2))
_latest_time_frame = compute.latest_time_frame_before_overlapping(_experiment, _series_id, _group, OFFSET_X)
for _cell_id in ['left_cell', 'right_cell']:
_arguments.append({
'experiment': _experiment,
'series_id': _series_id,
'group': _group,
'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER,
'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER,
'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER,
'offset_x': OFFSET_X,
'offset_y': OFFSET_Y,
'offset_z': OFFSET_Z,
'cell_id': _cell_id,
'direction': 'inside',
'time_points': _latest_time_frame
})
_windows_dictionary, _windows_to_compute = compute.windows(_arguments,
_keys=['experiment', 'series_id', 'group', 'cell_id'])
_fiber_densities = compute.fiber_densities(_windows_to_compute, _subtract_border=True)
_experiments_fiber_densities = {
_key: [_fiber_densities[_tuple] for _tuple in _windows_dictionary[_key]]
for _key in _windows_dictionary
}
_same_correlations_arrays = [[], [], []]
_different_correlations_arrays = [[], [], []]
_names_array = []
for _same_index in tqdm(range(3), desc='Main loop'):
_same_tuple = TRIPLET[_same_index]
_same_experiment, _same_series, _same_group = _same_tuple
_same_left_cell_fiber_densities = \
_experiments_fiber_densities[
(_same_experiment, _same_series, _same_group, 'left_cell')
]
_same_right_cell_fiber_densities = \
_experiments_fiber_densities[
(_same_experiment, _same_series, _same_group, 'right_cell')
]
_same_properties = \
load.group_properties(_same_experiment, _same_series, _same_group)
_same_left_cell_fiber_densities = compute.remove_blacklist(
_same_experiment,
_same_series,
_same_properties['cells_ids']['left_cell'],
_same_left_cell_fiber_densities
)
_same_right_cell_fiber_densities = compute.remove_blacklist(
_same_experiment,
_same_series,
_same_properties['cells_ids']['right_cell'],
_same_right_cell_fiber_densities
)
_same_left_cell_fiber_densities_filtered, _same_right_cell_fiber_densities_filtered = \
compute.longest_same_indices_shared_in_borders_sub_array(
_same_left_cell_fiber_densities, _same_right_cell_fiber_densities
)
_same_correlation = compute_lib.correlation(
compute_lib.derivative(_same_left_cell_fiber_densities_filtered, _n=DERIVATIVE),
compute_lib.derivative(_same_right_cell_fiber_densities_filtered, _n=DERIVATIVE)
)
for _different_index in range(3):
if _same_index != _different_index:
_different_tuple = TRIPLET[_different_index]
_different_experiment, _different_series, _different_group = \
_different_tuple
for _same_cell_id, _different_cell_id in product(['left_cell', 'right_cell'],
['left_cell', 'right_cell']):
_same_fiber_densities = _experiments_fiber_densities[(
_same_experiment,
_same_series,
_same_group,
_same_cell_id
)]
_different_fiber_densities = _experiments_fiber_densities[(
_different_experiment,
_different_series,
_different_group,
_different_cell_id
)]
_different_properties = load.group_properties(
_different_experiment, _different_series, _different_group
)
_same_fiber_densities = compute.remove_blacklist(
_same_experiment,
_same_series,
_same_properties['cells_ids'][_same_cell_id],
_same_fiber_densities
)
_different_fiber_densities = compute.remove_blacklist(
_different_experiment,
_different_series,
_different_properties['cells_ids'][_different_cell_id],
_different_fiber_densities
)
_same_fiber_densities_filtered, _different_fiber_densities_filtered = \
compute.longest_same_indices_shared_in_borders_sub_array(
_same_fiber_densities, _different_fiber_densities
)
_different_correlation = compute_lib.correlation(
compute_lib.derivative(_same_fiber_densities_filtered, _n=DERIVATIVE),
compute_lib.derivative(_different_fiber_densities_filtered, _n=DERIVATIVE)
)
_same_correlations_arrays[_same_index].append(_same_correlation)
_different_correlations_arrays[_same_index].append(_different_correlation)
_names_array.append('Cells ' + _same_group.split('_')[1] + ' & ' + _same_group.split('_')[2])
print('Group:', TRIPLET[_same_index])
print('Points:', len(_same_correlations_arrays[_same_index]))
_same_minus_different = \
np.array(_same_correlations_arrays[_same_index]) - np.array(_different_correlations_arrays[_same_index])
print('Wilcoxon of same minus different around the zero:')
print(wilcoxon(_same_minus_different))
print('Higher same amount:', (_same_minus_different > 0).sum() /
len(_same_minus_different))
print('Total points:', len(np.array(_same_correlations_arrays).flatten()))
_same_minus_different = \
np.array(_same_correlations_arrays).flatten() - np.array(_different_correlations_arrays).flatten()
print('Wilcoxon of same minus different around the zero:')
print(wilcoxon(_same_minus_different))
print('Higher same amount:', (_same_minus_different > 0).sum() /
len(_same_minus_different))
# plot
_colors_array = config.colors(3)
_fig = go.Figure(
data=[
go.Scatter(
x=_same_correlations_array,
y=_different_correlations_array,
name=_name,
mode='markers',
marker={
'size': 15,
'color': _color
},
opacity=0.7
) for _same_correlations_array, _different_correlations_array, _name, _color in
zip(_same_correlations_arrays, _different_correlations_arrays, _names_array, _colors_array)
],
layout={
'xaxis': {
'title': 'Same network correlation',
'zeroline': False,
'range': [-1.1, 1.2],
'tickmode': 'array',
'tickvals': [-1, -0.5, 0, 0.5, 1]
},
'yaxis': {
'title': 'Different network correlation',
'zeroline': False,
'range': [-1.1, 1.2],
'tickmode': 'array',
'tickvals': [-1, -0.5, 0, 0.5, 1]
},
'legend': {
'xanchor': 'left',
'x': 0.1,
'yanchor': 'top',
'bordercolor': 'black',
'borderwidth': 2,
'bgcolor': 'white'
},
'shapes': [
{
'type': 'line',
'x0': -1,
'y0': -1,
'x1': -1,
'y1': 1,
'line': {
'color': 'black',
'width': 2
}
},
{
'type': 'line',
'x0': -1,
'y0': -1,
'x1': 1,
'y1': -1,
'line': {
'color': 'black',
'width': 2
}
},
{
'type': 'line',
'x0': -1,
'y0': -1,
'x1': 1,
'y1': 1,
'line': {
'color': 'red',
'width': 2
}
}
]
}
)
save.to_html(
_fig=_fig,
_path=os.path.join(paths.PLOTS, save.get_module_name()),
_filename='plot'
)
def main(_real_cells=True, _static=False, _band=True, _high_temporal_resolution=False):
_experiments = all_experiments()
_experiments = filtering.by_categories(
_experiments=_experiments,
_is_single_cell=False,
_is_high_temporal_resolution=_high_temporal_resolution,
_is_bleb=False,
_is_dead_dead=False,
_is_live_dead=False,
_is_bead=False,
_is_metastasis=False
)
_tuples = load.experiments_groups_as_tuples(_experiments)
_tuples = filtering.by_pair_distance_range(_tuples, PAIR_DISTANCE_RANGE)
_tuples = filtering.by_real_pairs(_tuples, _real_pairs=_real_cells)
_tuples = filtering.by_fake_static_pairs(_tuples, _fake_static_pairs=_static)
_tuples = filtering.by_band(_tuples, _band=_band)
_tuples = filtering.by_time_frames_amount(_tuples, compute.minimum_time_frames_for_correlation(_experiments[0]))
print('Total tuples:', len(_tuples))
print('Density')
_arguments = []
for _tuple in _tuples:
_experiment, _series_id, _group = _tuple
_time_frame = compute.minimum_time_frames_for_correlation(_experiment)
for _cell_id in ['left_cell', 'right_cell']:
_arguments.append({
'experiment': _experiment,
'series_id': _series_id,
'group': _group,
'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER,
'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER,
'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER,
'offset_x': OFFSET_X,
'offset_y': OFFSET_Y_DENSITY,
'offset_z': OFFSET_Z,
'cell_id': _cell_id,
'direction': 'inside',
'time_point': _time_frame - 1
})
_windows_dictionary, _windows_to_compute = \
compute.windows(_arguments, _keys=['experiment', 'series_id', 'group', 'cell_id'])
_densities_fiber_densities = compute.fiber_densities(_windows_to_compute, _subtract_border=False)
_densities_experiments_fiber_densities = {
_key: [_densities_fiber_densities[_tuple] for _tuple in _windows_dictionary[_key]]
for _key in _windows_dictionary
}
print('Correlations')
_arguments = []
for _tuple in _tuples:
_experiment, _series_id, _group = _tuple
_latest_time_frame = compute.latest_time_frame_before_overlapping(_experiment, _series_id, _group, OFFSET_X)
for _cell_id in ['left_cell', 'right_cell']:
_arguments.append({
'experiment': _experiment,
'series_id': _series_id,
'group': _group,
'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER,
'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER,
'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER,
'offset_x': OFFSET_X,
'offset_y': OFFSET_Y_CORRELATION,
'offset_z': OFFSET_Z,
'cell_id': _cell_id,
'direction': 'inside',
'time_points': _latest_time_frame
})
_windows_dictionary, _windows_to_compute = \
compute.windows(_arguments, _keys=['experiment', 'series_id', 'group', 'cell_id'])
_correlations_fiber_densities = compute.fiber_densities(_windows_to_compute, _subtract_border=True)
_correlations_experiments_fiber_densities = {
_key: [_correlations_fiber_densities[_tuple] for _tuple in _windows_dictionary[_key]]
for _key in _windows_dictionary
}
_densities = []
_correlations = []
for _tuple in tqdm(_tuples, desc='Main loop'):
_experiment, _series_id, _group = _tuple
# density
_left_cell_fiber_density = \
_densities_experiments_fiber_densities[(_experiment, _series_id, _group, 'left_cell')][0]
_right_cell_fiber_density = \
_densities_experiments_fiber_densities[(_experiment, _series_id, _group, 'right_cell')][0]
# not relevant
if _left_cell_fiber_density[1] or _right_cell_fiber_density[1]:
continue
_normalization = load.normalization_series_file_data(_experiment, _series_id)
_left_cell_fiber_density_normalized = compute_lib.z_score(
_x=_left_cell_fiber_density[0],
_average=_normalization['average'],
_std=_normalization['std']
)
_right_cell_fiber_density_normalized = compute_lib.z_score(
_x=_right_cell_fiber_density[0],
_average=_normalization['average'],
_std=_normalization['std']
)
_density = (_left_cell_fiber_density_normalized + _right_cell_fiber_density_normalized) / 2
# correlation
_left_cell_fiber_densities = \
_correlations_experiments_fiber_densities[(_experiment, _series_id, _group, 'left_cell')]
_right_cell_fiber_densities = \
_correlations_experiments_fiber_densities[(_experiment, _series_id, _group, 'right_cell')]
_properties = load.group_properties(_experiment, _series_id, _group)
_left_cell_fiber_densities = compute.remove_blacklist(
_experiment, _series_id, _properties['cells_ids']['left_cell'], _left_cell_fiber_densities)
_right_cell_fiber_densities = compute.remove_blacklist(
_experiment, _series_id, _properties['cells_ids']['right_cell'], _right_cell_fiber_densities)
_left_cell_fiber_densities_filtered, _right_cell_fiber_densities_filtered = \
compute.longest_same_indices_shared_in_borders_sub_array(
_left_cell_fiber_densities, _right_cell_fiber_densities
)
# ignore small arrays
if len(_left_cell_fiber_densities_filtered) < compute.minimum_time_frames_for_correlation(_experiment):
continue
_correlation = compute_lib.correlation(
compute_lib.derivative(_left_cell_fiber_densities_filtered, _n=DERIVATIVE),
compute_lib.derivative(_right_cell_fiber_densities_filtered, _n=DERIVATIVE)
)
_densities.append(_density)
_correlations.append(_correlation)
print('Total tuples:', len(_densities))
print('Pearson correlation of densities and correlations:')
print(compute_lib.correlation(_densities, _correlations, _with_p_value=True))
# plot
_fig = go.Figure(
data=go.Scatter(
x=_densities,
y=_correlations,
mode='markers',
marker={
'size': 5,
'color': '#ea8500'
},
showlegend=False
),
layout={
'xaxis': {
'title': 'Fiber density (z-score)',
'zeroline': False,
'range': [-1.1, 15.2],
# 'tickmode': 'array',
# 'tickvals': [-1, -0.5, 0, 0.5, 1]
},
'yaxis': {
'title': 'Correlation',
'zeroline': False,
'range': [-1.1, 1.2],
'tickmode': 'array',
'tickvals': [-1, -0.5, 0, 0.5, 1]
},
'shapes': [
{
'type': 'line',
'x0': 0,
'y0': -1,
'x1': 0,
'y1': 1,
'line': {
'color': 'black',
'width': 2
}
},
{
'type': 'line',
'x0': 0,
'y0': -1,
'x1': 15,
'y1': -1,
'line': {
'color': 'black',
'width': 2
}
}
]
}
)
save.to_html(
_fig=_fig,
_path=os.path.join(paths.PLOTS, save.get_module_name()),
_filename='plot_real_' + str(_real_cells) + '_static_' + str(_static) + '_band_' + str(_band) +
'_high_time_' + str(_high_temporal_resolution) + '_y_density_' + str(OFFSET_Y_DENSITY) +
'_y_correlation_' + str(OFFSET_Y_CORRELATION)
)
def main(_early_time_frames=True):
_experiments = all_experiments()
_experiments = filtering.by_categories(_experiments=_experiments,
_is_single_cell=False,
_is_high_temporal_resolution=False,
_is_bleb=False,
_is_dead_dead=False,
_is_live_dead=False,
_is_bead=False,
_is_metastasis=False)
_tuples = load.experiments_groups_as_tuples(_experiments)
_tuples = filtering.by_pair_distance_range(_tuples, PAIR_DISTANCE_RANGE)
_tuples = filtering.by_real_pairs(_tuples)
_tuples = filtering.by_band(_tuples)
print('Total tuples:', len(_tuples))
_arguments = []
for _tuple in _tuples:
_experiment, _series_id, _group = _tuple
for _cell_id in ['left_cell', 'right_cell']:
_arguments.append({
'experiment': _experiment,
'series_id': _series_id,
'group': _group,
'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER,
'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER,
'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER,
'offset_x': OFFSET_X,
'offset_y': OFFSET_Y,
'offset_z': OFFSET_Z,
'cell_id': _cell_id,
'direction': 'inside'
})
_windows_dictionary, _windows_to_compute = compute.windows(
_arguments, _keys=['experiment', 'series_id', 'group', 'cell_id'])
_fiber_densities = compute.fiber_densities(_windows_to_compute)
_heatmap_fiber = []
_heatmap_fiber_change = []
for _tuple in _tuples:
_experiment, _series_id, _group = _tuple
_series_normalization = load.normalization_series_file_data(
_experiment, _series_id)
for _cell_id in ['left_cell', 'right_cell']:
_fiber_densities_by_time = [
_fiber_densities[_tuple]
for _tuple in _windows_dictionary[(_experiment, _series_id,
_group, _cell_id)]
]
_cell_fiber_densities = \
_fiber_densities_by_time[TIME_FRAMES[OFFSET_Y]['early'][0] if _early_time_frames else TIME_FRAMES[OFFSET_Y]['late'][0]:
TIME_FRAMES[OFFSET_Y]['early'][1] if _early_time_frames else TIME_FRAMES[OFFSET_Y]['late'][1]]
_properties = load.group_properties(_experiment, _series_id,
_group)
_cell_fiber_densities = compute.remove_blacklist(
_experiment, _series_id, _properties['cells_ids'][_cell_id],
_cell_fiber_densities)
_cell_fiber_densities = compute.longest_fiber_densities_ascending_sequence(
_cell_fiber_densities)
# fix if found nan
if True in np.isnan(_cell_fiber_densities):
_cell_fiber_densities = _cell_fiber_densities[:np.where(
np.isnan(_cell_fiber_densities))[0][0]]
# not enough data
if len(_cell_fiber_densities) < DERIVATIVE + 1:
continue
_z_score_fiber_density = libs.compute_lib.z_score_array(
_array=_cell_fiber_densities,
_average=_series_normalization['average'],
_std=_series_normalization['std'])
if _experiment in ['SN41', 'SN44']:
for _start_index in [0, 1, 2]:
_heatmap_fiber += _z_score_fiber_density[_start_index::3][
DERIVATIVE:]
_heatmap_fiber_change += compute_lib.derivative(
_z_score_fiber_density[_start_index::3], _n=DERIVATIVE)
else:
_heatmap_fiber += _z_score_fiber_density[DERIVATIVE:]
_heatmap_fiber_change += compute_lib.derivative(
_z_score_fiber_density, _n=DERIVATIVE)
print(
compute_lib.correlation(_heatmap_fiber,
_heatmap_fiber_change,
_with_p_value=True))
if PLOT:
_y_shape = int(round((Y_LABELS_END - Y_LABELS_START) * Y_BINS))
_x_shape = int(round((X_LABELS_END - X_LABELS_START) * X_BINS))
_total_points = 0
_z_array = np.zeros(shape=(_y_shape, _x_shape))
for _y, _x in zip(_heatmap_fiber_change, _heatmap_fiber):
_y_rounded, _x_rounded = int(round(_y * Y_BINS)), int(
round(_x * X_BINS))
_y_index, _x_index = int(_y_rounded - Y_LABELS_START *
Y_BINS), int(_x_rounded -
X_LABELS_START * X_BINS)
if 0 <= _y_index < _z_array.shape[
0] and 0 <= _x_index < _z_array.shape[1]:
_z_array[_y_index][_x_index] += 1
_total_points += 1
_z_array = _z_array / _total_points
if not CONDITIONAL_NORMALIZATION:
_z_array[_z_array == 0] = None
else:
_z_array_plot = np.zeros(shape=np.array(_z_array).shape)
for _fiber_index, _fiber_density_z_score in enumerate(_z_array):
_sum = np.sum(_fiber_density_z_score)
for _change_index, _change_z_score in enumerate(
_fiber_density_z_score):
_z_array_plot[_fiber_index][_change_index] = (
_change_z_score / _sum) if _sum != 0 else 0
_z_array_plot[_z_array_plot == 0] = None
_fig = go.Figure(
data=go.Heatmap(x=np.arange(start=X_LABELS_START,
stop=X_LABELS_END,
step=1 / X_BINS),
y=np.arange(start=Y_LABELS_START,
stop=Y_LABELS_END,
step=1 / Y_BINS),
z=_z_array,
colorscale='Viridis',
colorbar={
'tickmode': 'array',
'tickvals': [0, 0.025, 0.05],
'ticktext': ['0', 'Fraction', '0.05'],
'tickangle': -90
},
zmin=Z_MIN,
zmax=Z_MAX[CONDITIONAL_NORMALIZATION]),
layout={
'xaxis': {
'title': 'Fiber densities z-score',
'zeroline': False
},
'yaxis': {
'title': 'Change in fiber<br>density (z-score)',
'zeroline': False
},
'shapes': [{
'type': 'line',
'x0': X_LABELS_START,
'y0': Y_LABELS_START,
'x1': X_LABELS_END,
'y1': Y_LABELS_START,
'line': {
'color': 'black',
'width': 2
}
}, {
'type': 'line',
'x0': X_LABELS_START,
'y0': Y_LABELS_START,
'x1': X_LABELS_START,
'y1': Y_LABELS_END,
'line': {
'color': 'black',
'width': 2
}
}]
})
save.to_html(_fig=_fig,
_path=os.path.join(paths.PLOTS, save.get_module_name()),
_filename='plot_early_' + str(_early_time_frames))
def main(_real_cells=True,
_static=False,
_dead_dead=False,
_live_dead=False,
_dead=False,
_live=False,
_bead=False,
_metastasis=False,
_bleb=False,
_bleb_amount_um=None,
_band=True,
_high_temporal_resolution=False,
_offset_y=0.5):
_experiments = all_experiments()
_experiments = filtering.by_categories(
_experiments=_experiments,
_is_single_cell=False,
_is_high_temporal_resolution=_high_temporal_resolution,
_is_bleb=_bleb,
_is_dead_dead=_dead_dead,
_is_live_dead=_live_dead,
_is_bead=_bead,
_is_metastasis=_metastasis)
_tuples = load.experiments_groups_as_tuples(_experiments)
_tuples = filtering.by_pair_distance_range(_tuples, PAIR_DISTANCE_RANGE)
_tuples = filtering.by_real_pairs(_tuples, _real_pairs=_real_cells)
_tuples = filtering.by_fake_static_pairs(_tuples,
_fake_static_pairs=_static)
if _dead_dead is not False or _live_dead is not False:
_tuples = filtering.by_dead_live(_tuples, _dead=_dead, _live=_live)
_tuples = filtering.by_band(_tuples, _band=_band)
if _bleb:
_tuples = filtering.by_bleb_amount_um(_tuples,
_amount_um=_bleb_amount_um)
print('Total tuples:', len(_tuples))
_arguments = []
for _tuple in _tuples:
_experiment, _series_id, _group = _tuple
_latest_time_frame = compute.latest_time_frame_before_overlapping(
_experiment, _series_id, _group, OFFSET_X)
for _cell_id in ['left_cell', 'right_cell']:
_arguments.append({
'experiment': _experiment,
'series_id': _series_id,
'group': _group,
'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER,
'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER,
'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER,
'offset_x': OFFSET_X,
'offset_y': _offset_y,
'offset_z': OFFSET_Z,
'cell_id': _cell_id,
'direction': 'inside',
'time_points': _latest_time_frame
})
_windows_dictionary, _windows_to_compute = compute.windows(
_arguments, _keys=['experiment', 'series_id', 'group', 'cell_id'])
_fiber_densities = compute.fiber_densities(_windows_to_compute,
_subtract_border=True)
_experiments_fiber_densities = {
_key:
[_fiber_densities[_tuple] for _tuple in _windows_dictionary[_key]]
for _key in _windows_dictionary
}
_tuples_by_experiment = organize.by_experiment(_tuples)
_n = 0
_cells_ranks = []
for _experiment in _tuples_by_experiment:
print('Experiment:', _experiment)
_experiment_tuples = _tuples_by_experiment[_experiment]
for _pivot_tuple in tqdm(_experiment_tuples, desc='Main loop'):
_pivot_experiment, _pivot_series_id, _pivot_group = _pivot_tuple
_pivot_experiment_properties = load.group_properties(
_pivot_experiment, _pivot_series_id, _pivot_group)
for _pivot_cell_id, _pivot_cell_correct_match_cell_id in \
zip(['left_cell', 'right_cell'], ['right_cell', 'left_cell']):
_pivot_cell = (_pivot_experiment, _pivot_series_id,
_pivot_group, _pivot_cell_id)
_pivot_cell_correct_match_cell = (
_pivot_experiment, _pivot_series_id, _pivot_group,
_pivot_cell_correct_match_cell_id)
_pivot_cell_fiber_densities = _experiments_fiber_densities[
_pivot_cell]
_pivot_cell_fiber_densities = compute.remove_blacklist(
_pivot_experiment, _pivot_series_id,
_pivot_experiment_properties['cells_ids'][_pivot_cell_id],
_pivot_cell_fiber_densities)
_pivot_cell_correlations = []
# correct match
_pivot_cell_correct_match_fiber_densities = _experiments_fiber_densities[
_pivot_cell_correct_match_cell]
_pivot_cell_correct_match_fiber_densities = compute.remove_blacklist(
_pivot_experiment, _pivot_series_id,
_pivot_experiment_properties['cells_ids']
[_pivot_cell_correct_match_cell_id],
_pivot_cell_correct_match_fiber_densities)
_pivot_cell_fiber_densities_filtered, _pivot_cell_correct_match_fiber_densities_filtered = \
compute.longest_same_indices_shared_in_borders_sub_array(
_pivot_cell_fiber_densities, _pivot_cell_correct_match_fiber_densities
)
# ignore small arrays
if len(_pivot_cell_fiber_densities_filtered
) < compute.minimum_time_frames_for_correlation(
_pivot_experiment):
continue
_correlation = compute_lib.correlation(
compute_lib.derivative(
_pivot_cell_fiber_densities_filtered, _n=DERIVATIVE),
compute_lib.derivative(
_pivot_cell_correct_match_fiber_densities_filtered,
_n=DERIVATIVE))
_pivot_cell_correlations.append(_correlation)
_pivot_cell_correct_match_correlation = _correlation
# create list of potential cells
_candidate_tuples = []
for _candidate_tuple in _experiment_tuples:
_candidate_experiment, _candidate_series_id, _candidate_group = _candidate_tuple
for _candidate_cell_id in ['left_cell', 'right_cell']:
_candidate_cell = (_candidate_experiment,
_candidate_series_id,
_candidate_group,
_candidate_cell_id)
# skip if same cell or correct match
if _candidate_cell == _pivot_cell or _candidate_cell == _pivot_cell_correct_match_cell:
continue
_candidate_tuples.append(_candidate_cell)
# compare with each potential candidate, until reached the maximum or nothing to compare with
while len(_pivot_cell_correlations
) < POTENTIAL_MATCHES and len(_candidate_tuples) > 0:
# sample randomly
_candidate_cell = random.choice(_candidate_tuples)
_candidate_experiment, _candidate_series_id, _candidate_group, _candidate_cell_id = _candidate_cell
_candidate_experiment_properties = load.group_properties(
_candidate_experiment, _candidate_series_id,
_candidate_group)
_candidate_cell_fiber_densities = _experiments_fiber_densities[
_candidate_cell]
_candidate_cell_fiber_densities = compute.remove_blacklist(
_candidate_experiment, _candidate_series_id,
_candidate_experiment_properties['cells_ids']
[_candidate_cell_id], _candidate_cell_fiber_densities)
_pivot_cell_fiber_densities_filtered, _candidate_cell_fiber_densities_filtered = \
compute.longest_same_indices_shared_in_borders_sub_array(
_pivot_cell_fiber_densities, _candidate_cell_fiber_densities
)
# ignore small arrays
if len(_pivot_cell_fiber_densities_filtered
) < compute.minimum_time_frames_for_correlation(
_pivot_experiment):
_candidate_tuples.remove(_candidate_cell)
continue
_correlation = compute_lib.correlation(
compute_lib.derivative(
_pivot_cell_fiber_densities_filtered,
_n=DERIVATIVE),
compute_lib.derivative(
_candidate_cell_fiber_densities_filtered,
_n=DERIVATIVE))
_pivot_cell_correlations.append(_correlation)
# nothing to compare with
if len(_pivot_cell_correlations) == 1:
continue
# check matchmaking
_pivot_cell_correct_match_rank = 1
for _potential_match_correlation in sorted(
_pivot_cell_correlations, reverse=True):
if _pivot_cell_correct_match_correlation == _potential_match_correlation:
break
_pivot_cell_correct_match_rank += 1
_n += 1
_cells_ranks.append(_pivot_cell_correct_match_rank)
# results
_correct_match_probability = 1 / POTENTIAL_MATCHES
_first_place_correct_matches = sum(
[1 for _rank in _cells_ranks if _rank == 1])
_first_place_fraction = _first_place_correct_matches / _n
print('Matchmaking results:')
print('Total cells:', _n)
print('Correct match probability:', round(_correct_match_probability, 2))
print('Fraction of first place correct matches:',
round(_first_place_fraction, 2))
# plot
_x = list(range(MAX_RANK))
_x_text = [str(_rank + 1) for _rank in _x[:-1]] + [str(MAX_RANK) + '+']
_ranks_sums = [0 for _rank in _x]
for _rank in _cells_ranks:
if _rank < MAX_RANK:
_ranks_sums[_rank - 1] += 1
else:
_ranks_sums[-1] += 1
_y = np.array(_ranks_sums) / _n
_fig = go.Figure(data=go.Bar(x=_x, y=_y, marker={'color': '#ea8500'}),
layout={
'xaxis': {
'title': 'Correct match correlation rank',
'zeroline': False,
'tickmode': 'array',
'tickvals': _x,
'ticktext': _x_text
},
'yaxis': {
'title': 'Fraction',
'range': [0, 1.1],
'zeroline': False,
'tickmode': 'array',
'tickvals': [0, 0.5, 1]
}
})
save.to_html(_fig=_fig,
_path=os.path.join(paths.PLOTS, save.get_module_name()),
_filename='plot_real_' + str(_real_cells) + '_static_' +
str(_static) + '_dead_dead_' + str(_dead_dead) +
'_live_dead_' + str(_live_dead) + '_dead_' + str(_dead) +
'_live_' + str(_live) + '_bead_' + str(_bead) +
'_metastasis_' + str(_metastasis) + '_bleb_' + str(_bleb) +
str(_bleb_amount_um) + '_band_' + str(_band) + '_high_time_' +
str(_high_temporal_resolution) + '_y_' + str(_offset_y))
# correct match probability plot
_y = [_correct_match_probability] * (MAX_RANK - 1) + [
_correct_match_probability * (POTENTIAL_MATCHES - MAX_RANK + 1)
]
_fig = go.Figure(data=go.Bar(x=_x, y=_y, marker={'color': '#ea8500'}),
layout={
'xaxis': {
'title': 'Correct match correlation rank',
'zeroline': False,
'tickmode': 'array',
'tickvals': _x,
'ticktext': _x_text
},
'yaxis': {
'title': 'Fraction',
'range': [0, 1.1],
'zeroline': False,
'tickmode': 'array',
'tickvals': [0, 0.5, 1]
}
})
save.to_html(_fig=_fig,
_path=os.path.join(paths.PLOTS, save.get_module_name()),
_filename='plot_real_' + str(_real_cells) + '_static_' +
str(_static) + '_dead_dead_' + str(_dead_dead) +
'_live_dead_' + str(_live_dead) + '_dead_' + str(_dead) +
'_live_' + str(_live) + '_bead_' + str(_bead) +
'_metastasis_' + str(_metastasis) + '_bleb_' + str(_bleb) +
str(_bleb_amount_um) + '_band_' + str(_band) + '_high_time_' +
str(_high_temporal_resolution) + '_y_' + str(_offset_y) +
'_correct_match_prob')
def main():
_experiments = all_experiments()
_experiments = filtering.by_categories(_experiments=_experiments,
_is_single_cell=False,
_is_high_temporal_resolution=False,
_is_bleb=False,
_is_dead_dead=False,
_is_live_dead=False,
_is_bead=False,
_is_metastasis=False)
_tuples = load.experiments_groups_as_tuples(_experiments)
_tuples = filtering.by_time_frames_amount(
_tuples, compute.density_time_frame(_experiments[0]))
_tuples = filtering.by_real_pairs(_tuples)
_tuples = filtering.by_band(_tuples)
_tuples = filtering.by_pair_distance_range(_tuples, PAIR_DISTANCE_RANGE)
print('Total tuples:', len(_tuples))
_arguments = []
for _tuple in _tuples:
_experiment, _series_id, _group = _tuple
_latest_time_frame = compute.latest_time_frame_before_overlapping(
_experiment, _series_id, _group, OFFSET_X)
for _cell_id in ['left_cell', 'right_cell']:
_arguments.append({
'experiment': _experiment,
'series_id': _series_id,
'group': _group,
'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER,
'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER,
'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER,
'offset_x': OFFSET_X,
'offset_y': OFFSET_Y,
'offset_z': OFFSET_Z,
'cell_id': _cell_id,
'direction': 'inside',
'time_points': _latest_time_frame
})
_windows_dictionary, _windows_to_compute = \
compute.windows(_arguments, _keys=['experiment', 'series_id', 'group', 'cell_id'])
_fiber_densities = compute.fiber_densities(_windows_to_compute,
_subtract_border=True)
_experiments_fiber_densities = {
_key:
[_fiber_densities[_tuple] for _tuple in _windows_dictionary[_key]]
for _key in _windows_dictionary
}
_kpss_y_arrays = [[] for _i in DERIVATIVES]
_adf_y_arrays = [[] for _i in DERIVATIVES]
for _tuple in tqdm(_tuples, desc='Experiments loop'):
_experiment, _series_id, _group = _tuple
_properties = load.group_properties(_experiment, _series_id, _group)
_left_cell_fiber_densities = _experiments_fiber_densities[(
_experiment, _series_id, _group, 'left_cell')]
_left_cell_fiber_densities = compute.remove_blacklist(
_experiment, _series_id, _properties['cells_ids']['left_cell'],
_left_cell_fiber_densities)
_right_cell_fiber_densities = _experiments_fiber_densities[(
_experiment, _series_id, _group, 'right_cell')]
_right_cell_fiber_densities = compute.remove_blacklist(
_experiment, _series_id, _properties['cells_ids']['right_cell'],
_right_cell_fiber_densities)
if not OUT_OF_BOUNDARIES:
_left_cell_fiber_densities = \
compute.longest_fiber_densities_ascending_sequence(_left_cell_fiber_densities)
_right_cell_fiber_densities = \
compute.longest_fiber_densities_ascending_sequence(_right_cell_fiber_densities)
else:
_left_cell_fiber_densities = [
_fiber_density[0]
for _fiber_density in _left_cell_fiber_densities
]
_right_cell_fiber_densities = [
_fiber_density[0]
for _fiber_density in _right_cell_fiber_densities
]
# ignore small arrays
_minimum_time_frames_for_correlation = compute.minimum_time_frames_for_correlation(
_experiment)
if len(_left_cell_fiber_densities) < _minimum_time_frames_for_correlation or \
len(_right_cell_fiber_densities) < _minimum_time_frames_for_correlation:
continue
for _derivative_index, _derivative in enumerate(DERIVATIVES):
for _cell_fiber_densities in [
_left_cell_fiber_densities, _right_cell_fiber_densities
]:
_cell_fiber_densities_derivative = compute_lib.derivative(
_cell_fiber_densities, _n=_derivative)
_, _kpss_p_value, _, _ = kpss(_cell_fiber_densities_derivative,
nlags='legacy')
_kpss_y_arrays[_derivative_index].append(_kpss_p_value)
_, _adf_p_value, _, _, _, _ = adfuller(
_cell_fiber_densities_derivative)
_adf_y_arrays[_derivative_index].append(_adf_p_value)
print('Total pairs:', len(_kpss_y_arrays[0]) / 2)
# print results
print('KPSS:')
for _derivative_index, _derivative in enumerate(DERIVATIVES):
_stationary_count = len([
_value for _value in _kpss_y_arrays[_derivative_index]
if _value > 0.05
])
print(
'Derivative:', _derivative, 'Stationary:',
str(_stationary_count / len(_kpss_y_arrays[_derivative_index]) *
100) + '%')
print('ADF:')
for _derivative_index, _derivative in enumerate(DERIVATIVES):
_stationary_count = len([
_value for _value in _adf_y_arrays[_derivative_index]
if _value < 0.05
])
print(
'Derivative:', _derivative, 'Stationary:',
str(_stationary_count / len(_adf_y_arrays[_derivative_index]) *
100) + '%')
# plot
_colors_array = config.colors(3)
for _test_name, _y_title, _y_tickvals, _p_value_line, _y_arrays in \
zip(
['kpss', 'adf'],
['KPSS test p-value', 'ADF test p-value'],
[[0.05, 0.1], [0.05, 1]],
[0.05, 0.05],
[_kpss_y_arrays, _adf_y_arrays]
):
_fig = go.Figure(data=[
go.Box(y=_y,
name=_derivative,
boxpoints='all',
jitter=1,
pointpos=0,
line={'width': 1},
fillcolor='white',
marker={
'size': 10,
'color': _color
},
opacity=0.7,
showlegend=False) for _y, _derivative, _color in zip(
_y_arrays, DERIVATIVES_TEXT, _colors_array)
],
layout={
'xaxis': {
'title': 'Fiber density derivative',
'zeroline': False
},
'yaxis': {
'title': _y_title,
'zeroline': False,
'tickmode': 'array',
'tickvals': _y_tickvals
},
'shapes': [{
'type': 'line',
'x0': DERIVATIVES[0] - 0.75,
'y0': _p_value_line,
'x1': DERIVATIVES[-1] + 0.75,
'y1': _p_value_line,
'line': {
'color': 'red',
'width': 2,
'dash': 'dash'
}
}]
})
save.to_html(_fig=_fig,
_path=os.path.join(paths.PLOTS, save.get_module_name()),
_filename='plot_' + _test_name)
def main(_real_cells=True,
_static=False,
_band=True,
_high_temporal_resolution=False,
_pair_distance_range=None,
_offset_y=0.5):
if _pair_distance_range is None:
_pair_distance_range = PAIR_DISTANCE_RANGE
_experiments = all_experiments()
_experiments = filtering.by_categories(
_experiments=_experiments,
_is_single_cell=False,
_is_high_temporal_resolution=_high_temporal_resolution,
_is_bleb=False,
_is_dead_dead=False,
_is_live_dead=False,
_is_bead=False,
_is_metastasis=False)
_tuples = load.experiments_groups_as_tuples(_experiments)
_tuples = filtering.by_pair_distance_range(_tuples, _pair_distance_range)
_tuples = filtering.by_real_pairs(_tuples, _real_pairs=_real_cells)
_tuples = filtering.by_fake_static_pairs(_tuples,
_fake_static_pairs=_static)
_tuples = filtering.by_band(_tuples, _band=_band)
print('Total tuples:', len(_tuples))
_arguments = []
_longest_time_frame = 0
for _tuple in _tuples:
_experiment, _series_id, _group = _tuple
_latest_time_frame = compute.latest_time_frame_before_overlapping(
_experiment, _series_id, _group, OFFSET_X)
# save for later
if _latest_time_frame > _longest_time_frame:
_longest_time_frame = _latest_time_frame
for _cell_id in ['left_cell', 'right_cell']:
_arguments.append({
'experiment': _experiment,
'series_id': _series_id,
'group': _group,
'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER,
'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER,
'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER,
'offset_x': OFFSET_X,
'offset_y': _offset_y,
'offset_z': OFFSET_Z,
'cell_id': _cell_id,
'direction': 'inside',
'time_points': _latest_time_frame
})
_windows_dictionary, _windows_to_compute = compute.windows(
_arguments, _keys=['experiment', 'series_id', 'group', 'cell_id'])
_fiber_densities = compute.fiber_densities(_windows_to_compute)
_experiments_fiber_densities = {
_key:
[_fiber_densities[_tuple] for _tuple in _windows_dictionary[_key]]
for _key in _windows_dictionary
}
_valid_tuples = []
_valid_cells = []
_densities = [[] for _ in range(_longest_time_frame)]
for _tuple in tqdm(_tuples, desc='Experiments loop'):
_experiment, _series_id, _group = _tuple
_normalization = load.normalization_series_file_data(
_experiment, _series_id)
_properties = load.group_properties(_experiment, _series_id, _group)
for _cell_id in ['left_cell', 'right_cell']:
_cell_fiber_densities = \
_experiments_fiber_densities[(_experiment, _series_id, _group, _cell_id)]
_cell_fiber_densities = compute.remove_blacklist(
_experiment, _series_id, _properties['cells_ids'][_cell_id],
_cell_fiber_densities)
_previous_cell_fiber_density_normalized = None
for _time_frame, _cell_fiber_density in enumerate(
_cell_fiber_densities):
# not out of border
if _cell_fiber_density[1]:
_previous_cell_fiber_density_normalized = None
continue
# normalize
_cell_fiber_density_normalized = compute_lib.z_score(
_x=_cell_fiber_density[0],
_average=_normalization['average'],
_std=_normalization['std'])
# no previous
if _previous_cell_fiber_density_normalized is None:
_previous_cell_fiber_density_normalized = _cell_fiber_density_normalized
continue
# change
_cell_fiber_density_normalized_change = _cell_fiber_density_normalized - _previous_cell_fiber_density_normalized
_previous_cell_fiber_density_normalized = _cell_fiber_density_normalized
# save
_densities[_time_frame].append(
_cell_fiber_density_normalized_change)
if _tuple not in _valid_tuples:
_valid_tuples.append(_tuple)
_cell_tuple = (_experiment, _series_id, _group, _cell_id)
if _cell_tuple not in _valid_cells:
_valid_cells.append(_cell_tuple)
print('Total pairs:', len(_valid_tuples))
print('Total cells:', len(_valid_cells))
# plot
_temporal_resolution = compute.temporal_resolution_in_minutes(
_experiments[0])
_fig = go.Figure(
data=go.Scatter(x=np.array(range(_longest_time_frame)) *
_temporal_resolution,
y=[np.mean(_array) for _array in _densities],
name='Fiber density change (z-score)',
error_y={
'type': 'data',
'array': [np.std(_array) for _array in _densities],
'thickness': 1
},
mode='lines+markers',
marker={
'size': 5,
'color': '#ea8500'
},
line={'dash': 'solid'},
showlegend=False),
layout={
'xaxis': {
'title': 'Time (minutes)',
# 'zeroline': False
},
'yaxis': {
'title': 'Fiber density change (z-score)',
# 'zeroline': False
},
# 'shapes': [
# {
# 'type': 'line',
# 'x0': -_temporal_resolution,
# 'y0': -0.5,
# 'x1': -_temporal_resolution,
# 'y1': 2,
# 'line': {
# 'color': 'black',
# 'width': 2
# }
# },
# {
# 'type': 'line',
# 'x0': -_temporal_resolution,
# 'y0': -0.5,
# 'x1': 350,
# 'y1': -0.5,
# 'line': {
# 'color': 'black',
# 'width': 2
# }
# }
# ]
})
save.to_html(
_fig=_fig,
_path=os.path.join(paths.PLOTS, save.get_module_name()),
_filename='plot_real_' + str(_real_cells) + '_static_' + str(_static) +
'_band_' + str(_band) + '_high_time_' +
str(_high_temporal_resolution) + '_range_' +
'_'.join([str(_distance) for _distance in _pair_distance_range]) +
'_y_' + str(_offset_y))
def process_group(_arguments):
_time_frames_data = []
_time_frames_amount = \
len([_value for _value in _arguments['cell_coordinates'][_arguments['cell_id']] if _value is not None])
# smooth coordinates
_cells_coordinates_cell_smoothed = compute.smooth_coordinates_in_time(
[
_value
for _value in _arguments['cell_coordinates'][_arguments['cell_id']]
if _value is not None
],
_n=SMOOTH_AMOUNT)
if _arguments['cell_type'] == 'real':
_group = 'cell_' + str(_arguments['cell_id']) + '_' + str(_arguments['degrees_xy']) + '_' + \
str(_arguments['degrees_z'])
elif _arguments['cell_type'] == 'fake':
_group = 'fake_' + str(_arguments['cell_id']) + '_' + str(_arguments['degrees_xy']) + '_' + \
str(_arguments['degrees_z'])
elif _arguments['cell_type'] == 'static':
_group = 'static_' + str(_arguments['cell_id']) + '_' + str(_arguments['degrees_xy']) + '_' + \
str(_arguments['degrees_z'])
else:
raise Exception('No such cell type')
# check if needed (missing time-point / properties file)
if not _arguments['overwrite']:
_missing = False
for _time_frame in range(_time_frames_amount):
_time_frame_pickle_path = \
paths.structured(_arguments['experiment'], _arguments['series_id'], _group, _time_frame)
if not os.path.isfile(_time_frame_pickle_path):
_missing = True
break
_group_structured_path = paths.structured(_arguments['experiment'],
_arguments['series_id'],
_group)
_properties_json_path = os.path.join(_group_structured_path,
'properties.json')
if not os.path.isfile(_properties_json_path):
_missing = True
if not _missing:
return
# load image if needed
if 'series_image_by_time_frames' not in _arguments:
_series_image_path = \
paths.serieses(_arguments['experiment'], _arguments['series_id'])
_series_image = tifffile.imread(_series_image_path)
_arguments['series_image_by_time_frames'] = [
np.array([
_z[IMAGE_FIBER_CHANNEL_INDEX]
for _z in _series_image[_time_frame]
]) for _time_frame in range(_series_image.shape[0])
]
# running for each time point
for _time_frame in range(_time_frames_amount):
_time_frame_image = _arguments['series_image_by_time_frames'][
_time_frame]
_cell_coordinates = [
_value for _value in _cells_coordinates_cell_smoothed[_time_frame]
]
# update coordinates if needed
if 'x_change' in _arguments:
_cell_coordinates[0] += _arguments['x_change']
if 'y_change' in _arguments:
_cell_coordinates[1] += _arguments['y_change']
if 'z_change' in _arguments:
_cell_coordinates[2] += _arguments['z_change']
# compute padding xy
_padding_x, _padding_y = \
compute.axes_padding(_2d_image_shape=_time_frame_image[0].shape, _angle=_arguments['degrees_xy'])
_cell_coordinates[0] += _padding_x
_cell_coordinates[1] += _padding_y
# rotate image and change axes
_time_frame_image_rotated = np.array(
[rotate(_z, _arguments['degrees_xy']) for _z in _time_frame_image])
_time_frame_image_swapped = np.swapaxes(_time_frame_image_rotated, 0,
1)
if SHOW_PLOTS:
plt.imshow(_time_frame_image_rotated[int(
round(_cell_coordinates[2]))])
plt.show()
# update coordinates
_image_center = compute.image_center_coordinates(
_image_shape=reversed(_time_frame_image_rotated[0].shape))
_cell_coordinates = compute.rotate_point_around_another_point(
_point=_cell_coordinates,
_angle_in_radians=math.radians(_arguments['degrees_xy']),
_around_point=_image_center)
# y is now z, z is now y
_cell_coordinates[1], _cell_coordinates[2] = _cell_coordinates[
2], _cell_coordinates[1]
if SHOW_PLOTS:
plt.imshow(_time_frame_image_swapped[int(
round(_cell_coordinates[2]))])
plt.show()
# swap resolutions
_new_resolutions = {
'x': _arguments['resolutions']['x'],
'y': _arguments['resolutions']['z'],
'z': _arguments['resolutions']['y']
}
# second rotate, compute padding z
_padding_x, _padding_y = \
compute.axes_padding(_2d_image_shape=_time_frame_image_swapped[0].shape, _angle=_arguments['degrees_z'])
_cell_coordinates[0] += _padding_x
_cell_coordinates[1] += _padding_y
# rotate image
_time_frame_image_swapped_rotated = \
np.array([rotate(_z, _arguments['degrees_z']) for _z in _time_frame_image_swapped])
# update coordinates
_image_center = compute.image_center_coordinates(
_image_shape=reversed(_time_frame_image_swapped_rotated[0].shape))
_cell_coordinates = compute.rotate_point_around_another_point(
_point=_cell_coordinates,
_angle_in_radians=math.radians(_arguments['degrees_z']),
_around_point=_image_center)
if SHOW_PLOTS:
if _time_frame == 0 or _time_frame == 50 or _time_frame == 150:
plt.imshow(_time_frame_image_swapped_rotated[int(
round(_cell_coordinates[2]))])
plt.show()
# update resolutions
_angle = abs(_arguments['degrees_z'])
_new_resolution_x = (_angle / 90) * _new_resolutions['y'] + (
(90 - _angle) / 90) * _new_resolutions['x']
_new_resolution_y = (_angle / 90) * _new_resolutions['x'] + (
(90 - _angle) / 90) * _new_resolutions['y']
_new_resolutions['x'] = _new_resolution_x
_new_resolutions['y'] = _new_resolution_y
_image_z, _image_y, _image_x = _time_frame_image_swapped_rotated.shape
if not 0 <= _cell_coordinates[0] < _image_x or not \
0 <= _cell_coordinates[1] < _image_y or not \
0 <= _cell_coordinates[2] < _image_z:
break
# add to array
_time_frames_data.append({
'cell': {
'coordinates': {
'x': _cell_coordinates[0],
'y': _cell_coordinates[1],
'z': _cell_coordinates[2]
}
},
'resolutions': _new_resolutions
})
# save to pickle
_time_frame_pickle_path = \
paths.structured(_arguments['experiment'], _arguments['series_id'], _group, _time_frame)
save_lib.to_pickle(_time_frame_image_swapped_rotated,
_time_frame_pickle_path)
# save properties
if _arguments['cell_type'] == 'real':
_fake = False
_static = False
elif _arguments['cell_type'] == 'fake':
_based_on_properties = \
load.group_properties(_arguments['experiment'], _arguments['series_id'], 'cell_' + _group.split('fake_')[1])
_fake = True
_static = False
elif _arguments['cell_type'] == 'static':
_fake = True
_static = True
else:
raise Exception('No such cell type')
_properties_data = {
'experiment': _arguments['experiment'],
'series_id': _arguments['series_id'],
'cell_id': _arguments['cell_id'],
'time_points': _time_frames_data,
'fake': _fake,
'static': _static
}
_group_structured_path = paths.structured(_arguments['experiment'],
_arguments['series_id'], _group)
_properties_json_path = os.path.join(_group_structured_path,
'properties.json')
save_lib.to_json(_properties_data, _properties_json_path)
def main(_high_temporal_resolution=True):
_experiments = all_experiments()
_experiments = filtering.by_categories(
_experiments=_experiments,
_is_single_cell=False,
_is_high_temporal_resolution=_high_temporal_resolution,
_is_bleb=False,
_is_dead_dead=False,
_is_live_dead=False,
_is_bead=False,
_is_metastasis=False)
_tuples = load.experiments_groups_as_tuples(_experiments)
_tuples = filtering.by_time_frames_amount(
_tuples, _time_frames=MOVING_WINDOW_LENGTH[_high_temporal_resolution])
_tuples = filtering.by_pair_distance_range(
_tuples, _distance_range=PAIR_DISTANCE_RANGE)
_tuples = filtering.by_real_pairs(_tuples)
_tuples = filtering.by_band(_tuples)
print('Total tuples:', len(_tuples))
_arguments = []
for _tuple in _tuples:
_experiment, _series_id, _group = _tuple
_latest_time_frame = compute.latest_time_frame_before_overlapping(
_experiment, _series_id, _group, OFFSET_X)
for _cell_id in ['left_cell', 'right_cell']:
_arguments.append({
'experiment': _experiment,
'series_id': _series_id,
'group': _group,
'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER,
'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER,
'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER,
'offset_x': OFFSET_X,
'offset_y': OFFSET_Y,
'offset_z': OFFSET_Z,
'cell_id': _cell_id,
'direction': 'inside',
'time_points': _latest_time_frame
})
_windows_dictionary, _windows_to_compute = compute.windows(
_arguments, _keys=['experiment', 'series_id', 'group', 'cell_id'])
_fiber_densities = compute.fiber_densities(_windows_to_compute,
_subtract_border=True)
_experiments_fiber_densities = {
_key:
[_fiber_densities[_tuple] for _tuple in _windows_dictionary[_key]]
for _key in _windows_dictionary
}
for _tuple in _tuples:
_correlations = []
_experiment, _series_id, _group = _tuple
_left_cell_fiber_densities = \
_experiments_fiber_densities[(_experiment, _series_id, _group, 'left_cell')]
_right_cell_fiber_densities = \
_experiments_fiber_densities[(_experiment, _series_id, _group, 'right_cell')]
_properties = load.group_properties(_experiment, _series_id, _group)
_left_cell_fiber_densities = compute.remove_blacklist(
_experiment, _series_id, _properties['cells_ids']['left_cell'],
_left_cell_fiber_densities)
_right_cell_fiber_densities = compute.remove_blacklist(
_experiment, _series_id, _properties['cells_ids']['right_cell'],
_right_cell_fiber_densities)
for _start_time_frame in \
range(0, END_TIME_FRAME[_high_temporal_resolution], TIME_FRAME_STEP[_high_temporal_resolution]):
_left_cell_fiber_densities_window = _left_cell_fiber_densities[
_start_time_frame:_start_time_frame +
MOVING_WINDOW_LENGTH[_high_temporal_resolution]]
_right_cell_fiber_densities_window = _right_cell_fiber_densities[
_start_time_frame:_start_time_frame +
MOVING_WINDOW_LENGTH[_high_temporal_resolution]]
_left_cell_fiber_densities_filtered, _right_cell_fiber_densities_filtered = \
compute.longest_same_indices_shared_in_borders_sub_array(
_left_cell_fiber_densities_window, _right_cell_fiber_densities_window)
# ignore small arrays
if len(_left_cell_fiber_densities_filtered
) < MOVING_WINDOW_LENGTH[_high_temporal_resolution]:
_correlations.append(None)
continue
_correlations.append(
compute_lib.correlation(
compute_lib.derivative(_left_cell_fiber_densities_filtered,
_n=DERIVATIVE),
compute_lib.derivative(
_right_cell_fiber_densities_filtered, _n=DERIVATIVE)))
# plot
_temporal_resolution = compute.temporal_resolution_in_minutes(
_experiment)
_fig = go.Figure(data=go.Scatter(
x=np.arange(start=0, stop=len(_correlations), step=1) *
_temporal_resolution * TIME_FRAME_STEP[_high_temporal_resolution],
y=_correlations,
mode='lines+markers',
line={'dash': 'solid'}),
layout={
'xaxis': {
'title': 'Window start time (minutes)',
'zeroline': False
},
'yaxis': {
'title': 'Inner correlation',
'zeroline': False
}
})
save.to_html(_fig=_fig,
_path=os.path.join(paths.PLOTS, save.get_module_name()),
_filename='plot_' + str(_experiment) + '_' +
str(_series_id) + '_' + str(_group))
def main():
_experiments = config.all_experiments()
_experiments = filtering.by_categories(_experiments=_experiments,
_is_single_cell=False,
_is_high_temporal_resolution=None,
_is_bleb=False,
_is_dead_dead=False,
_is_live_dead=False,
_is_bead=False,
_is_metastasis=False)
_tuples = load.experiments_groups_as_tuples(_experiments)
_tuples = filtering.by_pair_distance_range(_tuples, PAIR_DISTANCE_RANGE)
_tuples = filtering.by_real_pairs(_tuples, _real_pairs=REAL)
_tuples = filtering.by_fake_static_pairs(_tuples,
_fake_static_pairs=STATIC)
_tuples = filtering.by_real_fake_pairs(_tuples, _real_fake_pairs=REAL_FAKE)
print('Total tuples:', len(_tuples))
_arguments = []
for _tuple in tqdm(_tuples, desc='Setting windows to compute'):
_experiment, _series_id, _group = _tuple
_latest_time_frame = compute.latest_time_frame_before_overlapping(
_experiment, _series_id, _group, _offset_x=0)
for _cell_id in ['left_cell', 'right_cell']:
for _offset_y in OFFSETS_Y:
_arguments.append({
'experiment': _experiment,
'series_id': _series_id,
'group': _group,
'length_x':
config.QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER,
'length_y':
config.QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER,
'length_z':
config.QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER,
'offset_x': 0,
'offset_y': _offset_y,
'offset_z': 0,
'cell_id': _cell_id,
'direction': 'inside',
'time_points': _latest_time_frame
})
_windows_dictionary, _windows_to_compute = \
compute.windows(_arguments, _keys=['experiment', 'series_id', 'group', 'cell_id', 'offset_y'])
_fiber_densities = compute.fiber_densities(_windows_to_compute)
_experiments_fiber_densities = {
_key:
[_fiber_densities[_tuple] for _tuple in _windows_dictionary[_key]]
for _key in _windows_dictionary
}
_headers = [
'time_frame', 'experiment', 'series_id', 'group', 'left_cell_id',
'right_cell_id', 'band', 'fake_following', 'fake_static',
'pair_distance_in_cell_diameter', 'offset_z',
'left_cell_fiber_density', 'left_cell_fiber_density_z_score',
'left_cell_out_of_boundaries', 'right_cell_fiber_density',
'right_cell_fiber_density_z_score', 'right_cell_out_of_boundaries'
]
_csv_path = os.path.join(paths.OUTPUTS,
'experiments_density_cell_pairs.csv')
with open(_csv_path, 'w', newline='') as _csv_file:
_csv_writer = csv.writer(_csv_file)
_csv_writer.writerow(_headers)
for _tuple in tqdm(_tuples, desc='Main loop'):
_experiment, _series_id, _group = _tuple
_group_properties = load.group_properties(_experiment, _series_id,
_group)
_left_cell_id, _right_cell_id = _group_properties[
'cells_ids'].values()
_band = _group_properties['band']
_fake_following, _fake_static = _group_properties[
'fake'], _group_properties['static']
_average, _std = load.normalization_series_file_data(
_experiment, _series_id).values()
for _offset_y in OFFSETS_Y:
_left_cell_fiber_densities = \
_experiments_fiber_densities[(_experiment, _series_id, _group, 'left_cell', _offset_y)]
_right_cell_fiber_densities = \
_experiments_fiber_densities[(_experiment, _series_id, _group, 'right_cell', _offset_y)]
_left_cell_fiber_densities = compute.remove_blacklist(
_experiment, _series_id, _left_cell_id,
_left_cell_fiber_densities)
_right_cell_fiber_densities = compute.remove_blacklist(
_experiment, _series_id, _right_cell_id,
_right_cell_fiber_densities)
for _time_frame, (_left_cell_fiber_density, _right_cell_fiber_density) in \
enumerate(zip(_left_cell_fiber_densities, _right_cell_fiber_densities)):
_pair_distance = compute.pair_distance_in_cell_size_time_frame(
_experiment, _series_id, _group, _time_frame)
_left_cell_fiber_density, _left_cell_out_of_boundaries = _left_cell_fiber_density
_right_cell_fiber_density, _right_cell_out_of_boundaries = _right_cell_fiber_density
_left_cell_fiber_density_z_score = compute_lib.z_score(
_left_cell_fiber_density, _average, _std)
_right_cell_fiber_density_z_score = compute_lib.z_score(
_right_cell_fiber_density, _average, _std)
_csv_writer.writerow([
_time_frame, _experiment, _series_id, _group,
_left_cell_id, _right_cell_id, _band, _fake_following,
_fake_static, _pair_distance, _offset_y,
_left_cell_fiber_density,
_left_cell_fiber_density_z_score,
_left_cell_out_of_boundaries,
_right_cell_fiber_density,
_right_cell_fiber_density_z_score,
_right_cell_out_of_boundaries
])
def compute_data(_arguments):
_offset_y_index, _offset_y, _offset_z_index, _offset_z = _arguments
_same_correlations_array = []
_different_correlations_array = []
for _experiment in _tuples_by_experiment:
_experiment_tuples = _tuples_by_experiment[_experiment]
for _same_index in range(len(_experiment_tuples)):
_same_tuple = _experiment_tuples[_same_index]
_same_experiment, _same_series, _same_group = _same_tuple
_same_left_cell_fiber_densities = _experiments_fiber_densities[(
_same_experiment, _same_series, _same_group, _offset_y,
_offset_z, 'left_cell')]
_same_right_cell_fiber_densities = _experiments_fiber_densities[(
_same_experiment, _same_series, _same_group, _offset_y,
_offset_z, 'right_cell')]
_same_properties = \
load.group_properties(_same_experiment, _same_series, _same_group)
_same_left_cell_fiber_densities = compute.remove_blacklist(
_same_experiment, _same_series,
_same_properties['cells_ids']['left_cell'],
_same_left_cell_fiber_densities)
_same_right_cell_fiber_densities = compute.remove_blacklist(
_same_experiment, _same_series,
_same_properties['cells_ids']['right_cell'],
_same_right_cell_fiber_densities)
_same_left_cell_fiber_densities_filtered, _same_right_cell_fiber_densities_filtered = \
compute.longest_same_indices_shared_in_borders_sub_array(
_same_left_cell_fiber_densities, _same_right_cell_fiber_densities
)
# ignore small arrays
if len(_same_left_cell_fiber_densities_filtered
) < compute.minimum_time_frames_for_correlation(
_same_experiment):
continue
_same_correlation = compute_lib.correlation(
compute_lib.derivative(
_same_left_cell_fiber_densities_filtered, _n=DERIVATIVE),
compute_lib.derivative(
_same_right_cell_fiber_densities_filtered, _n=DERIVATIVE))
for _different_index in range(len(_experiment_tuples)):
if _same_index != _different_index:
_different_tuple = _experiment_tuples[_different_index]
_different_experiment, _different_series, _different_group = _different_tuple
for _same_cell_id, _different_cell_id in product(
['left_cell', 'right_cell'],
['left_cell', 'right_cell']):
_same_fiber_densities = _experiments_fiber_densities[(
_same_experiment, _same_series, _same_group,
_offset_y, _offset_z, _same_cell_id)]
_different_fiber_densities = _experiments_fiber_densities[
(_different_experiment, _different_series,
_different_group, _offset_y, _offset_z,
_different_cell_id)]
_different_properties = load.group_properties(
_different_experiment, _different_series,
_different_group)
_same_fiber_densities = compute.remove_blacklist(
_same_experiment, _same_series,
_same_properties['cells_ids'][_same_cell_id],
_same_fiber_densities)
_different_fiber_densities = compute.remove_blacklist(
_different_experiment, _different_series,
_different_properties['cells_ids']
[_different_cell_id], _different_fiber_densities)
_same_fiber_densities_filtered, _different_fiber_densities_filtered = \
compute.longest_same_indices_shared_in_borders_sub_array(
_same_fiber_densities, _different_fiber_densities
)
# ignore small arrays
if len(_same_fiber_densities_filtered
) < compute.minimum_time_frames_for_correlation(
_different_experiment):
continue
_different_correlation = compute_lib.correlation(
compute_lib.derivative(
_same_fiber_densities_filtered, _n=DERIVATIVE),
compute_lib.derivative(
_different_fiber_densities_filtered,
_n=DERIVATIVE))
_same_correlations_array.append(_same_correlation)
_different_correlations_array.append(
_different_correlation)
# compute fraction
_annotation = None
_same_minus_different = np.array(_same_correlations_array) - np.array(
_different_correlations_array)
_same_count = len(_same_minus_different[_same_minus_different > 0])
if len(_same_minus_different) > 0:
_same_fraction = round(_same_count / len(_same_minus_different), 10)
_wilcoxon = wilcoxon(_same_minus_different)
_p_value = _wilcoxon[1]
if _p_value > 0.05:
_annotation = {
'text': 'x',
'showarrow': False,
'x': _offset_z,
'y': _offset_y,
'font': {
'size': 6,
'color': 'red'
}
}
else:
_same_fraction = None
return _offset_y_index, _offset_z_index, _same_fraction, _annotation
