def main(): _tuples = [('Metastasis_Airyscan_210401_sgAXL', 2, 'cells_1_2', True), ('Metastasis_Airyscan_210401_sgAXL', 3, 'cells_1_3', True), ('Metastasis_Airyscan_210401_sgAXL', 5, 'cells_0_1', True), ('Metastasis_Airyscan_210401_sgAXL', 9, 'cells_0_5', True), ('Metastasis_Airyscan_210401_sgAXL', 11, 'cells_1_2', True), ('Metastasis_Airyscan_210401_sgAXL', 14, 'cells_0_1', True), ('Metastasis_Airyscan_210401_sgAXL', 15, 'cells_1_2', True)] for _tuple in _tuples: _e, _s, _g, _b = _tuple _p = load.group_properties(_e, _s, _g) _p['band'] = _b _group_structured_path = paths.structured(_e, _s, _g) _properties_json_path = os.path.join(_group_structured_path, 'properties.json') save_lib.to_json(_p, _properties_json_path) print(_tuple) _g_fake = 'fake_' + _g.split('cells_')[1] _group_structured_path = paths.structured(_e, _s, _g_fake) _properties_json_path = os.path.join(_group_structured_path, 'properties.json') if os.path.isfile(_properties_json_path): print(_g_fake) _p = load.group_properties(_e, _s, _g_fake) _p['band'] = _b save_lib.to_json(_p, _properties_json_path)
def process_group(_experiment, _series_id, _group, _overwrite=False): _group_properties = load.group_properties(_experiment, _series_id, _group) if not _overwrite and _group_properties['band'] is not None: return if 'static' in _group: return elif 'fake' in _group: _group_real = 'cells_' + _group.split('fake_')[1] else: _group_real = _group _group_real_properties = load.group_properties(_experiment, _series_id, _group_real) _time_frames_amount = len(_group_real_properties['time_points']) _fiber_density = [] for _time_frame in [ 0, int(round(_time_frames_amount / 2)), _time_frames_amount - 1 ]: _left_cell_coordinates = _group_real_properties['time_points'][ _time_frame]['left_cell']['coordinates'] _right_cell_coordinates = _group_real_properties['time_points'][ _time_frame]['right_cell']['coordinates'] _cell_diameter_x = \ AVERAGE_CELL_DIAMETER_IN_MICRONS / _group_real_properties['time_points'][_time_frame]['resolutions']['x'] _cell_diameter_y = \ AVERAGE_CELL_DIAMETER_IN_MICRONS / _group_real_properties['time_points'][_time_frame]['resolutions']['y'] _cell_diameter_z = \ AVERAGE_CELL_DIAMETER_IN_MICRONS / _group_real_properties['time_points'][_time_frame]['resolutions']['z'] _x1 = (_left_cell_coordinates['x'] + _right_cell_coordinates['x']) / 2 - _cell_diameter_x / 2 _x2 = _x1 + _cell_diameter_x _y1 = (_left_cell_coordinates['y'] + _right_cell_coordinates['y']) / 2 - _cell_diameter_y / 2 _y2 = _y1 + _cell_diameter_y _z1 = (_left_cell_coordinates['z'] + _right_cell_coordinates['z']) / 2 - _cell_diameter_z / 2 _z2 = _z1 + _cell_diameter_z _window = [_x1, _y1, _z1, _x2, _y2, _z2] _fiber_density = compute.window_fiber_density( _experiment=_experiment, _series_id=_series_id, _group=_group_real, _time_frame=_time_frame, _window=[int(round(_value)) for _value in _window])[:2] _fiber_density.append(_fiber_density) if _fiber_density[0] < _fiber_density[1] < _fiber_density[2]: _band = True else: _band = False print(_experiment, _series_id, _group, _band, sep='\t') _group_properties['band'] = _band _group_structured_path = paths.structured(_experiment, _series_id, _group) _properties_json_path = os.path.join(_group_structured_path, 'properties.json') save_lib.to_json(_group_properties, _properties_json_path)
def by_pair_distance_range(_experiments_tuples, _distance_range, _time_frame=0): _experiments_tuples_filtered = [] for _tuple in _experiments_tuples: _experiment, _series_id, _group = _tuple _group_properties = load.group_properties(_experiment, _series_id, _group) if len(_group_properties['time_points']) <= _time_frame: continue _left_cell_coordinates = _group_properties['time_points'][_time_frame][ 'left_cell']['coordinates'] _right_cell_coordinates = _group_properties['time_points'][ _time_frame]['right_cell']['coordinates'] _pair_distance = compute.pair_distance_in_cell_size( _experiment=_experiment, _series_id=_series_id, _cell_1_coordinates=[ (_left_cell_coordinates['x'], _left_cell_coordinates['y'], _left_cell_coordinates['z']) ], _cell_2_coordinates=[ (_right_cell_coordinates['x'], _right_cell_coordinates['y'], _right_cell_coordinates['z']) ]) if _distance_range[0] <= _pair_distance <= _distance_range[1]: _experiments_tuples_filtered.append(_tuple) return _experiments_tuples_filtered
def by_pair_distance(_experiments_tuples): _tuples_by_distance = {} for _tuple in _experiments_tuples: _experiment, _series_id, _group = _tuple _group_properties = load.group_properties(_experiment, _series_id, _group) _left_cell_coordinates = _group_properties['time_points'][0][ 'left_cell']['coordinates'] _right_cell_coordinates = _group_properties['time_points'][0][ 'right_cell']['coordinates'] _pair_distance = int( round( compute.pair_distance_in_cell_size( _experiment=_experiment, _series_id=_series_id, _cell_1_coordinates=[(_left_cell_coordinates['x'], _left_cell_coordinates['y'], _left_cell_coordinates['z'])], _cell_2_coordinates=[(_right_cell_coordinates['x'], _right_cell_coordinates['y'], _right_cell_coordinates['z'])]))) if _pair_distance in _tuples_by_distance: _tuples_by_distance[_pair_distance].append(_tuple) else: _tuples_by_distance[_pair_distance] = [_tuple] return { _distance: _tuples_by_distance[_distance] for _distance in sorted(_tuples_by_distance.keys()) }
def by_fake_static_pairs(_experiments_tuples, _fake_static_pairs=True): _experiments_tuples_filtered = [] for _tuple in _experiments_tuples: _experiment, _series_id, _group = _tuple _group_properties = load.group_properties(_experiment, _series_id, _group) if _group_properties['static'] == _fake_static_pairs: _experiments_tuples_filtered.append(_tuple) return _experiments_tuples_filtered
def compute_data(_arguments): _offset_y_index, _offset_y, _offset_z_index, _offset_z = _arguments _fiber_density_changes_array = [] for _tuple in _tuples: _experiment, _series_id, _group = _tuple _normalization = load.normalization_series_file_data( _experiment, _series_id) _properties = load.group_properties(_experiment, _series_id, _group) for _cell_id in ['left_cell', 'right_cell']: _cell_fiber_densities = \ _experiments_fiber_densities[(_experiment, _series_id, _group, _offset_y, _offset_z, _cell_id)] _cell_fiber_densities = compute.remove_blacklist( _experiment, _series_id, _properties['cells_ids'][_cell_id], _cell_fiber_densities) _previous_cell_fiber_density_normalized = None _cell_values = [] for _time_frame, _cell_fiber_density in enumerate( _cell_fiber_densities): # not out of border if _cell_fiber_density[1]: _previous_cell_fiber_density_normalized = None continue # normalize _cell_fiber_density_normalized = compute_lib.z_score( _x=_cell_fiber_density[0], _average=_normalization['average'], _std=_normalization['std']) # no previous if _previous_cell_fiber_density_normalized is None: _previous_cell_fiber_density_normalized = _cell_fiber_density_normalized continue # change _cell_fiber_density_normalized_change = abs( _cell_fiber_density_normalized - _previous_cell_fiber_density_normalized) _previous_cell_fiber_density_normalized = _cell_fiber_density_normalized # save _cell_values.append(_cell_fiber_density_normalized_change) # save if len(_cell_values) > 0: _fiber_density_changes_array.append(np.mean(_cell_values)) if len(_fiber_density_changes_array) > 0: return _offset_y_index, _offset_z_index, np.mean( _fiber_density_changes_array) else: return _offset_y_index, _offset_z_index, None
def by_real_pairs(_experiments_tuples, _real_pairs=True): _experiments_tuples_filtered = [] for _tuple in _experiments_tuples: _experiment, _series_id, _group = _tuple _group_properties = load.group_properties(_experiment, _series_id, _group) if _group_properties['fake'] != _real_pairs and \ ('real_fake' not in _group_properties or _group_properties['real_fake'] != _real_pairs): _experiments_tuples_filtered.append(_tuple) return _experiments_tuples_filtered
def pair_distance_in_cell_size_time_frame(_experiment, _series_id, _group, _time_frame): _group_properties = load.group_properties(_experiment, _series_id, _group) _left_cell_coordinates = \ [list(_group_properties['time_points'][_time_frame]['left_cell']['coordinates'].values())] _right_cell_coordinates = \ [list(_group_properties['time_points'][_time_frame]['right_cell']['coordinates'].values())] return pair_distance_in_cell_size(_experiment, _series_id, _left_cell_coordinates, _right_cell_coordinates)
def by_time_frames_amount(_experiments_tuples, _time_frames, _exactly=False): _experiments_tuples_filtered = [] for _tuple in _experiments_tuples: _experiment, _series_id, _group = _tuple _group_properties = load.group_properties(_experiment, _series_id, _group) _time_frames_amount = len(_group_properties['time_points']) if (_exactly and _time_frames_amount == _time_frames) or \ (not _exactly and _time_frames_amount >= _time_frames): _experiments_tuples_filtered.append(_tuple) return _experiments_tuples_filtered
def latest_time_frame_before_overlapping(_experiment, _series_id, _group, _offset_x): _properties = load.group_properties(_experiment, _series_id, _group) _latest_time_frame = len(_properties['time_points']) for _time_frame in range(len(_properties['time_points'])): _pair_distance = \ pair_distance_in_cell_size_time_frame(_experiment, _series_id, _group, _time_frame) if _pair_distance - 1 - _offset_x * 2 < QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER * 2: _latest_time_frame = _time_frame - 1 break return _latest_time_frame
def by_band(_experiments_tuples, _band=True): if _band is None: return _experiments_tuples _experiments_tuples_filtered = [] for _tuple in _experiments_tuples: _experiment, _series_id, _group = _tuple _group_properties = load.group_properties(_experiment, _series_id, _group) _experiments_tuples_filtered.append( _tuple) if _group_properties['band'] == _band else None return _experiments_tuples_filtered
def process_group_pairs(_experiment, _series_id, _group, _mark_cells=True, _draw_borders=True): _group_properties = load.group_properties(_experiment, _series_id, _group) _group_path = paths.images(_experiment + ' - All TPs', _series_id, _group) os.makedirs(_group_path, exist_ok=True) for _time_frame in range(0, len(_group_properties['time_points']), 1): print(_experiment, _series_id, _group, _time_frame, sep='\t') _time_frame_image = load.structured_image(_experiment, _series_id, _group, _time_frame) _left_cell_coordinates = _group_properties['time_points'][_time_frame][ 'left_cell']['coordinates'] _right_cell_coordinates = _group_properties['time_points'][ _time_frame]['right_cell']['coordinates'] _left_cell_id = _group_properties['cells_ids']['left_cell'] _right_cell_id = _group_properties['cells_ids']['right_cell'] _cell_diameter_in_microns = AVERAGE_CELL_DIAMETER_IN_MICRONS _z_cell_diameter = _cell_diameter_in_microns / _group_properties[ 'time_points'][_time_frame]['resolutions']['z'] _z_image = _time_frame_image[ int(round(_left_cell_coordinates['z'] - _z_cell_diameter / 2) ):int(round(_left_cell_coordinates['z'] + _z_cell_diameter / 2))] _average_across_z = np.rint(np.mean(_z_image, axis=0)) if _mark_cells: _average_across_z = mark_cells(_average_across_z, _group_properties, _time_frame, _left_cell_coordinates, AVERAGE_CELL_DIAMETER_IN_MICRONS) _average_across_z = mark_cells(_average_across_z, _group_properties, _time_frame, _right_cell_coordinates, AVERAGE_CELL_DIAMETER_IN_MICRONS) if _draw_borders: _left_window = get_window(_experiment, _series_id, _group_properties, _time_frame, 'left_cell', DIRECTION) _right_window = get_window(_experiment, _series_id, _group_properties, _time_frame, 'right_cell', DIRECTION) _average_across_z = draw_borders(_average_across_z, _left_window) _average_across_z = draw_borders(_average_across_z, _right_window) _img = Image.fromarray(_average_across_z).convert('L') _img_path = os.path.join(_group_path, str(_time_frame) + '.png') _img.save(_img_path)
def compute_data(_tuples, _arguments, _padding_y_by, _padding_z_by, _space_y_by, _space_z_by): _windows_dictionary, _windows_to_compute = \ compute.windows(_arguments, _keys=['experiment', 'series_id', 'group', 'cell_id']) _fiber_densities = compute.fiber_densities(_windows_to_compute, _subtract_border=True, _padding_y_by=_padding_y_by, _padding_z_by=_padding_z_by, _space_y_by=_space_y_by, _space_z_by=_space_z_by) _experiments_fiber_densities = { _key: [_fiber_densities[_tuple] for _tuple in _windows_dictionary[_key]] for _key in _windows_dictionary } _correlations = [] for _tuple in _tuples: _experiment, _series, _group = _tuple _left_cell_fiber_densities = _experiments_fiber_densities[(_experiment, _series, _group, 'left_cell')] _right_cell_fiber_densities = _experiments_fiber_densities[(_experiment, _series, _group, 'right_cell')] _properties = load.group_properties(_experiment, _series, _group) _left_cell_fiber_densities = compute.remove_blacklist( _experiment, _series, _properties['cells_ids']['left_cell'], _left_cell_fiber_densities ) _right_cell_fiber_densities = compute.remove_blacklist( _experiment, _series, _properties['cells_ids']['right_cell'], _right_cell_fiber_densities ) _left_cell_fiber_densities_filtered, _right_cell_fiber_densities_filtered = \ compute.longest_same_indices_shared_in_borders_sub_array( _left_cell_fiber_densities, _right_cell_fiber_densities ) # ignore small arrays if len(_left_cell_fiber_densities_filtered) < compute.minimum_time_frames_for_correlation(_experiment): continue _correlation = compute_lib.correlation( compute_lib.derivative(_left_cell_fiber_densities_filtered, _n=DERIVATIVE), compute_lib.derivative(_right_cell_fiber_densities_filtered, _n=DERIVATIVE) ) _correlations.append(_correlation) return np.mean(_correlations)
def process_group(_experiment, _series_id, _group): _properties = load.group_properties(_experiment, _series_id, _group) _cell_1_coordinates = _properties['time_points'][0]['left_cell'][ 'coordinates'].values() _cell_2_coordinates = _properties['time_points'][0]['right_cell'][ 'coordinates'].values() _pair_distance = compute.pair_distance_in_cell_size( _experiment, _series_id, [_cell_1_coordinates], [_cell_2_coordinates]) print(_experiment, _series_id, _group, _pair_distance, _properties['band'], _properties['fake'], _properties['static'], len(_properties['time_points']), sep='\t')
def main(): _experiment = 'LiveDead_201220' _band = False _tuples = load.experiment_groups_as_tuples(_experiment) print('Total tuples:', len(_tuples)) for _tuple in _tuples: print(_tuple) _experiment, _series_id, _group = _tuple _properties = load.group_properties(_experiment, _series_id, _group) _properties['band'] = _band _group_structured_path = paths.structured(_experiment, _series_id, _group) _properties_json_path = os.path.join(_group_structured_path, 'properties.json') save_lib.to_json(_properties, _properties_json_path)
def windows_by_time(_arguments): _group_properties = load.group_properties(_arguments['experiment'], _arguments['series_id'], _arguments['group']) # single time point if 'time_point' in _arguments: return _arguments, [window_time_frame(_arguments)] # multiple time points if 'time_points' not in _arguments: _arguments['time_points'] = sys.maxsize _windows = [] for _time_frame in range( min(_arguments['time_points'], len(_group_properties['time_points']))): _arguments['time_point'] = _time_frame _windows.append(window_time_frame(_arguments)) return _arguments, _windows
def process_group_single_cells(_experiment, _series_id, _group, _mark_cells=True, _draw_borders=True): _group_properties = load.group_properties(_experiment, _series_id, _group) _group_path = paths.images(_experiment + ' - First TP - Start Not Average', _series_id, _group) os.makedirs(_group_path, exist_ok=True) for _time_frame in [0]: print(_experiment, _series_id, _group, _time_frame, sep='\t') _time_frame_image = load.structured_image(_experiment, _series_id, _group, _time_frame) _cell_coordinates = _group_properties['time_points'][_time_frame][ 'cell']['coordinates'] _cell_diameter_in_microns = AVERAGE_CELL_DIAMETER_IN_MICRONS _z_cell_diameter = _cell_diameter_in_microns / _group_properties[ 'time_points'][_time_frame]['resolutions']['z'] _z_image = _time_frame_image[int( round(_cell_coordinates['z'] - _z_cell_diameter / 2)):int(round(_cell_coordinates['z'] + _z_cell_diameter / 2))] _average_across_z = np.rint(np.mean(_z_image, axis=0)) if _mark_cells: _average_across_z = mark_cells(_average_across_z, _group_properties, _time_frame, _cell_coordinates, AVERAGE_CELL_DIAMETER_IN_MICRONS) if _draw_borders: for _direction in ['left', 'right', 'up', 'down']: _window = get_window(_experiment, _series_id, _group_properties, _time_frame, 'cell', _direction) _average_across_z = draw_borders(_average_across_z, _window) _img = Image.fromarray(_average_across_z).convert('L') _img_path = os.path.join(_group_path, str(_time_frame) + '.png') _img.save(_img_path)
def window_time_frame(_arguments): if 'group_properties' not in _arguments: _arguments['group_properties'] = \ load.group_properties(_arguments['experiment'], _arguments['series_id'], _arguments['group']) _time_frame_properties = _arguments['group_properties']['time_points'][ _arguments['time_point']] if QUANTIFICATION_WINDOW_START_BY_AVERAGE_CELL_DIAMETER: _cell_diameter_in_microns = AVERAGE_CELL_DIAMETER_IN_MICRONS else: _cell_diameter_in_microns = load.mean_distance_to_surface_in_microns( _experiment=_arguments['experiment'], _series_id=_arguments['series_id'], _cell_id=_arguments['group_properties']['cells_ids'][_arguments['cell_id']]) * 2 \ if _arguments['cell_id'] != 'cell' else _arguments['group_properties']['cell_id'] * 2 _time_frame_window = window_by_microns( _resolution_x=_arguments['group_properties']['time_points'][ _arguments['time_point']]['resolutions']['x'], _resolution_y=_arguments['group_properties']['time_points'][ _arguments['time_point']]['resolutions']['y'], _resolution_z=_arguments['group_properties']['time_points'][ _arguments['time_point']]['resolutions']['z'], _length_x=_arguments['length_x'], _length_y=_arguments['length_y'], _length_z=_arguments['length_z'], _offset_x=_arguments['offset_x'], _offset_y=_arguments['offset_y'], _offset_z=_arguments['offset_z'], _cell_coordinates=_arguments['group_properties']['time_points'][ _arguments['time_point']][_arguments['cell_id']]['coordinates'], _cell_diameter_in_microns=_cell_diameter_in_microns, _direction='right' if (_arguments['cell_id'], _arguments['direction']) == ('left_cell', 'inside') or (_arguments['cell_id'], _arguments['direction']) == ('right_cell', 'outside') or (_arguments['cell_id'], _arguments['direction']) == ('cell', 'right') else 'left') return _arguments['experiment'], _arguments['series_id'], _arguments[ 'group'], _arguments['time_point'], _time_frame_window
def process_group(_experiment, _series_id, _cells_coordinates, _cell_1_id, _cell_2_id, _series_image_by_time_frames, _resolutions, _real_cells=True, _fake_cell_1_id=None, _fake_cell_2_id=None, _x_change=0, _y_change=0, _z_change=0, _overwrite=False): _time_frames_data = [] _left_cell_id = None _right_cell_id = None _time_frames_amount = min( len([_value for _value in _cells_coordinates[_cell_1_id] if _value is not None]), len([_value for _value in _cells_coordinates[_cell_2_id] if _value is not None]) ) # smooth coordinates _cells_coordinates_cell_1_smoothed = compute.smooth_coordinates_in_time( [_value for _value in _cells_coordinates[_cell_1_id] if _value is not None], _n=SMOOTH_AMOUNT ) _cells_coordinates_cell_2_smoothed = compute.smooth_coordinates_in_time( [_value for _value in _cells_coordinates[_cell_2_id] if _value is not None], _n=SMOOTH_AMOUNT ) if _real_cells: _group = 'cells_' + str(_cell_1_id) + '_' + str(_cell_2_id) elif _fake_cell_1_id is None: _group = 'fake_' + str(_cell_1_id) + '_' + str(_cell_2_id) else: _group = 'static_' + str(_fake_cell_1_id) + '_' + str(_fake_cell_2_id) # check if needed (missing time-point / properties file) if not _overwrite: _missing = False for _time_frame in range(_time_frames_amount): _time_frame_pickle_path = paths.structured(_experiment, _series_id, _group, _time_frame) if not os.path.isfile(_time_frame_pickle_path): _missing = True break _group_structured_path = paths.structured(_experiment, _series_id, _group) _properties_json_path = os.path.join(_group_structured_path, 'properties.json') if not os.path.isfile(_properties_json_path): _missing = True if not _missing: return # running for each time point for _time_frame in range(_time_frames_amount): _time_frame_image = _series_image_by_time_frames[_time_frame] _cell_1_coordinates = [_value for _value in _cells_coordinates_cell_1_smoothed[_time_frame]] _cell_2_coordinates = [_value for _value in _cells_coordinates_cell_2_smoothed[_time_frame]] # update coordinates if needed if any([_x_change != 0, _y_change != 0, _z_change != 0]): _cell_1_coordinates = [ _cell_1_coordinates[0] + _x_change, _cell_1_coordinates[1] + _y_change, _cell_1_coordinates[2] + _z_change ] _cell_2_coordinates = [ _cell_2_coordinates[0] + _x_change, _cell_2_coordinates[1] + _y_change, _cell_2_coordinates[2] + _z_change ] # choose left and right cells if _time_frame == 0: if _cell_1_coordinates[0] <= _cell_2_coordinates[0]: _left_cell_id, _right_cell_id = _cell_1_id, _cell_2_id else: _right_cell_id, _left_cell_id = _cell_1_id, _cell_2_id print(_experiment, 'Series ' + str(_series_id), 'Cell 1 #:', _cell_1_id, 'Cell 2 #:', _cell_2_id, 'Time point:', _time_frame, sep='\t') # set coordinates if _left_cell_id == _cell_1_id: _left_cell_coordinates, _right_cell_coordinates = _cell_1_coordinates, _cell_2_coordinates else: _right_cell_coordinates, _left_cell_coordinates = _cell_1_coordinates, _cell_2_coordinates # compute padding _helper_coordinates = (_left_cell_coordinates[0] + 1, _left_cell_coordinates[1]) _angle = compute.angle_between_three_points( _right_cell_coordinates, _left_cell_coordinates, _helper_coordinates ) _padding_x, _padding_y = compute.axes_padding(_2d_image_shape=_time_frame_image[0].shape, _angle=_angle) _left_cell_coordinates[0] += _padding_x _left_cell_coordinates[1] += _padding_y _right_cell_coordinates[0] += _padding_x _right_cell_coordinates[1] += _padding_y # rotate image and change axes _time_frame_image_rotated = np.array([rotate(_z, _angle) for _z in _time_frame_image]) _time_frame_image_swapped = np.swapaxes(_time_frame_image_rotated, 0, 1) if SHOW_PLOTS: plt.imshow(_time_frame_image_rotated[int(round(_left_cell_coordinates[2]))]) plt.show() plt.imshow(_time_frame_image_rotated[int(round(_right_cell_coordinates[2]))]) plt.show() # update coordinates _image_center = compute.image_center_coordinates(_image_shape=reversed(_time_frame_image_rotated[0].shape)) _left_cell_coordinates = compute.rotate_point_around_another_point( _point=_left_cell_coordinates, _angle_in_radians=math.radians(_angle), _around_point=_image_center ) _right_cell_coordinates = compute.rotate_point_around_another_point( _point=_right_cell_coordinates, _angle_in_radians=math.radians(_angle), _around_point=_image_center ) _fixed_y = (_left_cell_coordinates[1] + _right_cell_coordinates[1]) / 2 # y is now z _left_cell_coordinates[1] = _left_cell_coordinates[2] _right_cell_coordinates[1] = _right_cell_coordinates[2] # z is now y _left_cell_coordinates[2] = _fixed_y _right_cell_coordinates[2] = _fixed_y if SHOW_PLOTS: plt.imshow(_time_frame_image_swapped[int(round(_left_cell_coordinates[2]))]) plt.show() # swap resolutions _new_resolutions = { 'x': _resolutions['x'], 'y': _resolutions['z'], 'z': _resolutions['y'] } # second rotate, compute padding _helper_coordinates = (_left_cell_coordinates[0] + 1, _left_cell_coordinates[1]) _angle = compute.angle_between_three_points( _right_cell_coordinates, _left_cell_coordinates, _helper_coordinates ) _padding_x, _padding_y = compute.axes_padding(_2d_image_shape=_time_frame_image_swapped[0].shape, _angle=_angle) _left_cell_coordinates[0] += _padding_x _left_cell_coordinates[1] += _padding_y _right_cell_coordinates[0] += _padding_x _right_cell_coordinates[1] += _padding_y # rotate image _time_frame_image_swapped_rotated = np.array([rotate(_z, _angle) for _z in _time_frame_image_swapped]) # convert to 8 bit color depth if CONVERT_TO_COLOR_DEPTH_8_BIT: _time_frame_image_swapped_rotated = \ np.rint(_time_frame_image_swapped_rotated / (math.pow(2, 16) - 1) * (math.pow(2, 8) - 1)).astype(np.uint8) # update coordinates _image_center = compute.image_center_coordinates( _image_shape=reversed(_time_frame_image_swapped_rotated[0].shape)) _left_cell_coordinates = compute.rotate_point_around_another_point( _point=_left_cell_coordinates, _angle_in_radians=math.radians(_angle), _around_point=_image_center ) _right_cell_coordinates = compute.rotate_point_around_another_point( _point=_right_cell_coordinates, _angle_in_radians=math.radians(_angle), _around_point=_image_center ) _fixed_y = (_left_cell_coordinates[1] + _right_cell_coordinates[1]) / 2 _left_cell_coordinates[1] = _fixed_y _right_cell_coordinates[1] = _fixed_y if SHOW_PLOTS: if _time_frame == 0 or _time_frame == 50 or _time_frame == 150: plt.imshow(_time_frame_image_swapped_rotated[int(round(_left_cell_coordinates[2]))]) plt.show() # update resolutions _angle = abs(_angle) _new_resolution_x = (_angle / 90) * _new_resolutions['y'] + ((90 - _angle) / 90) * _new_resolutions['x'] _new_resolution_y = (_angle / 90) * _new_resolutions['x'] + ((90 - _angle) / 90) * _new_resolutions['y'] _new_resolutions['x'] = _new_resolution_x _new_resolutions['y'] = _new_resolution_y _image_z, _image_y, _image_x = _time_frame_image_swapped_rotated.shape if not 0 <= _left_cell_coordinates[0] < _image_x or not \ 0 <= _left_cell_coordinates[1] < _image_y or not \ 0 <= _left_cell_coordinates[2] < _image_z: break if not 0 <= _right_cell_coordinates[0] < _image_x or not \ 0 <= _right_cell_coordinates[1] < _image_y or not \ 0 <= _right_cell_coordinates[2] < _image_z: break # add to array _time_frames_data.append({ 'left_cell': { 'coordinates': { 'x': _left_cell_coordinates[0], 'y': _left_cell_coordinates[1], 'z': _left_cell_coordinates[2] } }, 'right_cell': { 'coordinates': { 'x': _right_cell_coordinates[0], 'y': _right_cell_coordinates[1], 'z': _right_cell_coordinates[2] } }, 'resolutions': _new_resolutions }) # save to pickle _time_frame_pickle_path = paths.structured(_experiment, _series_id, _group, _time_frame) save_lib.to_pickle(_time_frame_image_swapped_rotated, _time_frame_pickle_path) # save properties if _real_cells: _band = None _fake = False _static = False elif _fake_cell_1_id is None: _based_on_properties = load.group_properties(_experiment, _series_id, 'cells_' + _group.split('fake_')[1]) _band = _based_on_properties['band'] _fake = True _static = False else: _band = False _fake = True _static = True _properties_data = { 'experiment': _experiment, 'series_id': _series_id, 'cells_ids': { 'left_cell': _left_cell_id, 'right_cell': _right_cell_id }, 'time_points': _time_frames_data, 'band': _band, 'fake': _fake, 'static': _static } _group_structured_path = paths.structured(_experiment, _series_id, _group) _properties_json_path = os.path.join(_group_structured_path, 'properties.json') save_lib.to_json(_properties_data, _properties_json_path)
def compute_fiber_densities(_band=True, _high_temporal_resolution=False): _experiments = all_experiments() _experiments = filtering.by_categories( _experiments=_experiments, _is_single_cell=False, _is_high_temporal_resolution=_high_temporal_resolution, _is_bleb=False, _is_dead_dead=False, _is_live_dead=False, _is_bead=False, _is_metastasis=False) _tuples = load.experiments_groups_as_tuples(_experiments) _tuples = filtering.by_pair_distance_range(_tuples, PAIR_DISTANCE_RANGE) _tuples = filtering.by_real_pairs(_tuples) _tuples = filtering.by_band(_tuples, _band=_band) print('Total tuples:', len(_tuples)) _arguments = [] for _tuple in _tuples: _experiment, _series_id, _group = _tuple _latest_time_frame = compute.latest_time_frame_before_overlapping( _experiment, _series_id, _group, OFFSET_X) for _cell_id in ['left_cell', 'right_cell']: _arguments.append({ 'experiment': _experiment, 'series_id': _series_id, 'group': _group, 'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER, 'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER, 'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER, 'offset_x': OFFSET_X, 'offset_y': OFFSET_Y, 'offset_z': OFFSET_Z, 'cell_id': _cell_id, 'direction': 'inside', 'time_points': _latest_time_frame }) _windows_dictionary, _windows_to_compute = compute.windows( _arguments, _keys=['experiment', 'series_id', 'group', 'cell_id']) _fiber_densities = compute.fiber_densities(_windows_to_compute, _subtract_border=True) _experiments_fiber_densities = { _key: [_fiber_densities[_tuple] for _tuple in _windows_dictionary[_key]] for _key in _windows_dictionary } _tuples_by_experiment = organize.by_experiment(_tuples) _distances_from_y_equal_x = [] _z_positions_array = [] for _experiment in _tuples_by_experiment: print('Experiment:', _experiment) _experiment_tuples = _tuples_by_experiment[_experiment] for _same_index in tqdm(range(len(_experiment_tuples)), desc='Main loop'): _same_tuple = _experiment_tuples[_same_index] _same_experiment, _same_series, _same_group = _same_tuple _same_left_cell_fiber_densities = \ _experiments_fiber_densities[ (_same_experiment, _same_series, _same_group, 'left_cell') ] _same_right_cell_fiber_densities = \ _experiments_fiber_densities[ (_same_experiment, _same_series, _same_group, 'right_cell') ] _same_properties = \ load.group_properties(_same_experiment, _same_series, _same_group) _same_left_cell_fiber_densities = compute.remove_blacklist( _same_experiment, _same_series, _same_properties['cells_ids']['left_cell'], _same_left_cell_fiber_densities) _same_right_cell_fiber_densities = compute.remove_blacklist( _same_experiment, _same_series, _same_properties['cells_ids']['right_cell'], _same_right_cell_fiber_densities) _same_left_cell_fiber_densities_filtered, _same_right_cell_fiber_densities_filtered = \ compute.longest_same_indices_shared_in_borders_sub_array( _same_left_cell_fiber_densities, _same_right_cell_fiber_densities ) # ignore small arrays if len(_same_left_cell_fiber_densities_filtered ) < compute.minimum_time_frames_for_correlation( _same_experiment): continue _same_correlation = compute_lib.correlation( compute_lib.derivative( _same_left_cell_fiber_densities_filtered, _n=DERIVATIVE), compute_lib.derivative( _same_right_cell_fiber_densities_filtered, _n=DERIVATIVE)) _same_group_mean_z_position = \ compute.group_mean_z_position_from_substrate(_same_experiment, _same_series, _same_group) for _different_index in range(len(_experiment_tuples)): if _same_index != _different_index: _different_tuple = _experiment_tuples[_different_index] _different_experiment, _different_series, _different_group = \ _different_tuple for _same_cell_id, _different_cell_id in product( ['left_cell', 'right_cell'], ['left_cell', 'right_cell']): _same_fiber_densities = _experiments_fiber_densities[( _same_experiment, _same_series, _same_group, _same_cell_id)] _different_fiber_densities = _experiments_fiber_densities[ (_different_experiment, _different_series, _different_group, _different_cell_id)] _different_properties = load.group_properties( _different_experiment, _different_series, _different_group) _same_fiber_densities = compute.remove_blacklist( _same_experiment, _same_series, _same_properties['cells_ids'][_same_cell_id], _same_fiber_densities) _different_fiber_densities = compute.remove_blacklist( _different_experiment, _different_series, _different_properties['cells_ids'] [_different_cell_id], _different_fiber_densities) _same_fiber_densities_filtered, _different_fiber_densities_filtered = \ compute.longest_same_indices_shared_in_borders_sub_array( _same_fiber_densities, _different_fiber_densities ) # ignore small arrays if len(_same_fiber_densities_filtered ) < compute.minimum_time_frames_for_correlation( _different_experiment): continue _different_correlation = compute_lib.correlation( compute_lib.derivative( _same_fiber_densities_filtered, _n=DERIVATIVE), compute_lib.derivative( _different_fiber_densities_filtered, _n=DERIVATIVE)) _point_distance = compute_lib.distance_from_a_point_to_a_line( _line=[-1, -1, 1, 1], _point=[_same_correlation, _different_correlation]) if _same_correlation > _different_correlation: _distances_from_y_equal_x.append(_point_distance) else: _distances_from_y_equal_x.append(-_point_distance) _z_positions_array.append(_same_group_mean_z_position) print('Total points:', len(_distances_from_y_equal_x)) print('Wilcoxon of distances from y = x around the zero:') print(wilcoxon(_distances_from_y_equal_x)) print( 'Pearson correlation of distances from y = x and z position distances:' ) print( compute_lib.correlation(_distances_from_y_equal_x, _z_positions_array, _with_p_value=True)) return _distances_from_y_equal_x, _z_positions_array
def main(): _arguments = [] for _tuple in TRIPLET: _experiment, _series_id, _group = _tuple _pair_distance = compute.pair_distance_in_cell_size_time_frame(_experiment, _series_id, _group, _time_frame=0) print(_tuple, 'pairs distance:', round(_pair_distance, 2)) _latest_time_frame = compute.latest_time_frame_before_overlapping(_experiment, _series_id, _group, OFFSET_X) for _cell_id in ['left_cell', 'right_cell']: _arguments.append({ 'experiment': _experiment, 'series_id': _series_id, 'group': _group, 'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER, 'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER, 'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER, 'offset_x': OFFSET_X, 'offset_y': OFFSET_Y, 'offset_z': OFFSET_Z, 'cell_id': _cell_id, 'direction': 'inside', 'time_points': _latest_time_frame }) _windows_dictionary, _windows_to_compute = compute.windows(_arguments, _keys=['experiment', 'series_id', 'group', 'cell_id']) _fiber_densities = compute.fiber_densities(_windows_to_compute, _subtract_border=True) _experiments_fiber_densities = { _key: [_fiber_densities[_tuple] for _tuple in _windows_dictionary[_key]] for _key in _windows_dictionary } _same_correlations_arrays = [[], [], []] _different_correlations_arrays = [[], [], []] _names_array = [] for _same_index in tqdm(range(3), desc='Main loop'): _same_tuple = TRIPLET[_same_index] _same_experiment, _same_series, _same_group = _same_tuple _same_left_cell_fiber_densities = \ _experiments_fiber_densities[ (_same_experiment, _same_series, _same_group, 'left_cell') ] _same_right_cell_fiber_densities = \ _experiments_fiber_densities[ (_same_experiment, _same_series, _same_group, 'right_cell') ] _same_properties = \ load.group_properties(_same_experiment, _same_series, _same_group) _same_left_cell_fiber_densities = compute.remove_blacklist( _same_experiment, _same_series, _same_properties['cells_ids']['left_cell'], _same_left_cell_fiber_densities ) _same_right_cell_fiber_densities = compute.remove_blacklist( _same_experiment, _same_series, _same_properties['cells_ids']['right_cell'], _same_right_cell_fiber_densities ) _same_left_cell_fiber_densities_filtered, _same_right_cell_fiber_densities_filtered = \ compute.longest_same_indices_shared_in_borders_sub_array( _same_left_cell_fiber_densities, _same_right_cell_fiber_densities ) _same_correlation = compute_lib.correlation( compute_lib.derivative(_same_left_cell_fiber_densities_filtered, _n=DERIVATIVE), compute_lib.derivative(_same_right_cell_fiber_densities_filtered, _n=DERIVATIVE) ) for _different_index in range(3): if _same_index != _different_index: _different_tuple = TRIPLET[_different_index] _different_experiment, _different_series, _different_group = \ _different_tuple for _same_cell_id, _different_cell_id in product(['left_cell', 'right_cell'], ['left_cell', 'right_cell']): _same_fiber_densities = _experiments_fiber_densities[( _same_experiment, _same_series, _same_group, _same_cell_id )] _different_fiber_densities = _experiments_fiber_densities[( _different_experiment, _different_series, _different_group, _different_cell_id )] _different_properties = load.group_properties( _different_experiment, _different_series, _different_group ) _same_fiber_densities = compute.remove_blacklist( _same_experiment, _same_series, _same_properties['cells_ids'][_same_cell_id], _same_fiber_densities ) _different_fiber_densities = compute.remove_blacklist( _different_experiment, _different_series, _different_properties['cells_ids'][_different_cell_id], _different_fiber_densities ) _same_fiber_densities_filtered, _different_fiber_densities_filtered = \ compute.longest_same_indices_shared_in_borders_sub_array( _same_fiber_densities, _different_fiber_densities ) _different_correlation = compute_lib.correlation( compute_lib.derivative(_same_fiber_densities_filtered, _n=DERIVATIVE), compute_lib.derivative(_different_fiber_densities_filtered, _n=DERIVATIVE) ) _same_correlations_arrays[_same_index].append(_same_correlation) _different_correlations_arrays[_same_index].append(_different_correlation) _names_array.append('Cells ' + _same_group.split('_')[1] + ' & ' + _same_group.split('_')[2]) print('Group:', TRIPLET[_same_index]) print('Points:', len(_same_correlations_arrays[_same_index])) _same_minus_different = \ np.array(_same_correlations_arrays[_same_index]) - np.array(_different_correlations_arrays[_same_index]) print('Wilcoxon of same minus different around the zero:') print(wilcoxon(_same_minus_different)) print('Higher same amount:', (_same_minus_different > 0).sum() / len(_same_minus_different)) print('Total points:', len(np.array(_same_correlations_arrays).flatten())) _same_minus_different = \ np.array(_same_correlations_arrays).flatten() - np.array(_different_correlations_arrays).flatten() print('Wilcoxon of same minus different around the zero:') print(wilcoxon(_same_minus_different)) print('Higher same amount:', (_same_minus_different > 0).sum() / len(_same_minus_different)) # plot _colors_array = config.colors(3) _fig = go.Figure( data=[ go.Scatter( x=_same_correlations_array, y=_different_correlations_array, name=_name, mode='markers', marker={ 'size': 15, 'color': _color }, opacity=0.7 ) for _same_correlations_array, _different_correlations_array, _name, _color in zip(_same_correlations_arrays, _different_correlations_arrays, _names_array, _colors_array) ], layout={ 'xaxis': { 'title': 'Same network correlation', 'zeroline': False, 'range': [-1.1, 1.2], 'tickmode': 'array', 'tickvals': [-1, -0.5, 0, 0.5, 1] }, 'yaxis': { 'title': 'Different network correlation', 'zeroline': False, 'range': [-1.1, 1.2], 'tickmode': 'array', 'tickvals': [-1, -0.5, 0, 0.5, 1] }, 'legend': { 'xanchor': 'left', 'x': 0.1, 'yanchor': 'top', 'bordercolor': 'black', 'borderwidth': 2, 'bgcolor': 'white' }, 'shapes': [ { 'type': 'line', 'x0': -1, 'y0': -1, 'x1': -1, 'y1': 1, 'line': { 'color': 'black', 'width': 2 } }, { 'type': 'line', 'x0': -1, 'y0': -1, 'x1': 1, 'y1': -1, 'line': { 'color': 'black', 'width': 2 } }, { 'type': 'line', 'x0': -1, 'y0': -1, 'x1': 1, 'y1': 1, 'line': { 'color': 'red', 'width': 2 } } ] } ) save.to_html( _fig=_fig, _path=os.path.join(paths.PLOTS, save.get_module_name()), _filename='plot' )
def main(_real_cells=True, _static=False, _band=True, _high_temporal_resolution=False): _experiments = all_experiments() _experiments = filtering.by_categories( _experiments=_experiments, _is_single_cell=False, _is_high_temporal_resolution=_high_temporal_resolution, _is_bleb=False, _is_dead_dead=False, _is_live_dead=False, _is_bead=False, _is_metastasis=False ) _tuples = load.experiments_groups_as_tuples(_experiments) _tuples = filtering.by_pair_distance_range(_tuples, PAIR_DISTANCE_RANGE) _tuples = filtering.by_real_pairs(_tuples, _real_pairs=_real_cells) _tuples = filtering.by_fake_static_pairs(_tuples, _fake_static_pairs=_static) _tuples = filtering.by_band(_tuples, _band=_band) _tuples = filtering.by_time_frames_amount(_tuples, compute.minimum_time_frames_for_correlation(_experiments[0])) print('Total tuples:', len(_tuples)) print('Density') _arguments = [] for _tuple in _tuples: _experiment, _series_id, _group = _tuple _time_frame = compute.minimum_time_frames_for_correlation(_experiment) for _cell_id in ['left_cell', 'right_cell']: _arguments.append({ 'experiment': _experiment, 'series_id': _series_id, 'group': _group, 'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER, 'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER, 'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER, 'offset_x': OFFSET_X, 'offset_y': OFFSET_Y_DENSITY, 'offset_z': OFFSET_Z, 'cell_id': _cell_id, 'direction': 'inside', 'time_point': _time_frame - 1 }) _windows_dictionary, _windows_to_compute = \ compute.windows(_arguments, _keys=['experiment', 'series_id', 'group', 'cell_id']) _densities_fiber_densities = compute.fiber_densities(_windows_to_compute, _subtract_border=False) _densities_experiments_fiber_densities = { _key: [_densities_fiber_densities[_tuple] for _tuple in _windows_dictionary[_key]] for _key in _windows_dictionary } print('Correlations') _arguments = [] for _tuple in _tuples: _experiment, _series_id, _group = _tuple _latest_time_frame = compute.latest_time_frame_before_overlapping(_experiment, _series_id, _group, OFFSET_X) for _cell_id in ['left_cell', 'right_cell']: _arguments.append({ 'experiment': _experiment, 'series_id': _series_id, 'group': _group, 'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER, 'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER, 'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER, 'offset_x': OFFSET_X, 'offset_y': OFFSET_Y_CORRELATION, 'offset_z': OFFSET_Z, 'cell_id': _cell_id, 'direction': 'inside', 'time_points': _latest_time_frame }) _windows_dictionary, _windows_to_compute = \ compute.windows(_arguments, _keys=['experiment', 'series_id', 'group', 'cell_id']) _correlations_fiber_densities = compute.fiber_densities(_windows_to_compute, _subtract_border=True) _correlations_experiments_fiber_densities = { _key: [_correlations_fiber_densities[_tuple] for _tuple in _windows_dictionary[_key]] for _key in _windows_dictionary } _densities = [] _correlations = [] for _tuple in tqdm(_tuples, desc='Main loop'): _experiment, _series_id, _group = _tuple # density _left_cell_fiber_density = \ _densities_experiments_fiber_densities[(_experiment, _series_id, _group, 'left_cell')][0] _right_cell_fiber_density = \ _densities_experiments_fiber_densities[(_experiment, _series_id, _group, 'right_cell')][0] # not relevant if _left_cell_fiber_density[1] or _right_cell_fiber_density[1]: continue _normalization = load.normalization_series_file_data(_experiment, _series_id) _left_cell_fiber_density_normalized = compute_lib.z_score( _x=_left_cell_fiber_density[0], _average=_normalization['average'], _std=_normalization['std'] ) _right_cell_fiber_density_normalized = compute_lib.z_score( _x=_right_cell_fiber_density[0], _average=_normalization['average'], _std=_normalization['std'] ) _density = (_left_cell_fiber_density_normalized + _right_cell_fiber_density_normalized) / 2 # correlation _left_cell_fiber_densities = \ _correlations_experiments_fiber_densities[(_experiment, _series_id, _group, 'left_cell')] _right_cell_fiber_densities = \ _correlations_experiments_fiber_densities[(_experiment, _series_id, _group, 'right_cell')] _properties = load.group_properties(_experiment, _series_id, _group) _left_cell_fiber_densities = compute.remove_blacklist( _experiment, _series_id, _properties['cells_ids']['left_cell'], _left_cell_fiber_densities) _right_cell_fiber_densities = compute.remove_blacklist( _experiment, _series_id, _properties['cells_ids']['right_cell'], _right_cell_fiber_densities) _left_cell_fiber_densities_filtered, _right_cell_fiber_densities_filtered = \ compute.longest_same_indices_shared_in_borders_sub_array( _left_cell_fiber_densities, _right_cell_fiber_densities ) # ignore small arrays if len(_left_cell_fiber_densities_filtered) < compute.minimum_time_frames_for_correlation(_experiment): continue _correlation = compute_lib.correlation( compute_lib.derivative(_left_cell_fiber_densities_filtered, _n=DERIVATIVE), compute_lib.derivative(_right_cell_fiber_densities_filtered, _n=DERIVATIVE) ) _densities.append(_density) _correlations.append(_correlation) print('Total tuples:', len(_densities)) print('Pearson correlation of densities and correlations:') print(compute_lib.correlation(_densities, _correlations, _with_p_value=True)) # plot _fig = go.Figure( data=go.Scatter( x=_densities, y=_correlations, mode='markers', marker={ 'size': 5, 'color': '#ea8500' }, showlegend=False ), layout={ 'xaxis': { 'title': 'Fiber density (z-score)', 'zeroline': False, 'range': [-1.1, 15.2], # 'tickmode': 'array', # 'tickvals': [-1, -0.5, 0, 0.5, 1] }, 'yaxis': { 'title': 'Correlation', 'zeroline': False, 'range': [-1.1, 1.2], 'tickmode': 'array', 'tickvals': [-1, -0.5, 0, 0.5, 1] }, 'shapes': [ { 'type': 'line', 'x0': 0, 'y0': -1, 'x1': 0, 'y1': 1, 'line': { 'color': 'black', 'width': 2 } }, { 'type': 'line', 'x0': 0, 'y0': -1, 'x1': 15, 'y1': -1, 'line': { 'color': 'black', 'width': 2 } } ] } ) save.to_html( _fig=_fig, _path=os.path.join(paths.PLOTS, save.get_module_name()), _filename='plot_real_' + str(_real_cells) + '_static_' + str(_static) + '_band_' + str(_band) + '_high_time_' + str(_high_temporal_resolution) + '_y_density_' + str(OFFSET_Y_DENSITY) + '_y_correlation_' + str(OFFSET_Y_CORRELATION) )
def main(_early_time_frames=True): _experiments = all_experiments() _experiments = filtering.by_categories(_experiments=_experiments, _is_single_cell=False, _is_high_temporal_resolution=False, _is_bleb=False, _is_dead_dead=False, _is_live_dead=False, _is_bead=False, _is_metastasis=False) _tuples = load.experiments_groups_as_tuples(_experiments) _tuples = filtering.by_pair_distance_range(_tuples, PAIR_DISTANCE_RANGE) _tuples = filtering.by_real_pairs(_tuples) _tuples = filtering.by_band(_tuples) print('Total tuples:', len(_tuples)) _arguments = [] for _tuple in _tuples: _experiment, _series_id, _group = _tuple for _cell_id in ['left_cell', 'right_cell']: _arguments.append({ 'experiment': _experiment, 'series_id': _series_id, 'group': _group, 'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER, 'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER, 'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER, 'offset_x': OFFSET_X, 'offset_y': OFFSET_Y, 'offset_z': OFFSET_Z, 'cell_id': _cell_id, 'direction': 'inside' }) _windows_dictionary, _windows_to_compute = compute.windows( _arguments, _keys=['experiment', 'series_id', 'group', 'cell_id']) _fiber_densities = compute.fiber_densities(_windows_to_compute) _heatmap_fiber = [] _heatmap_fiber_change = [] for _tuple in _tuples: _experiment, _series_id, _group = _tuple _series_normalization = load.normalization_series_file_data( _experiment, _series_id) for _cell_id in ['left_cell', 'right_cell']: _fiber_densities_by_time = [ _fiber_densities[_tuple] for _tuple in _windows_dictionary[(_experiment, _series_id, _group, _cell_id)] ] _cell_fiber_densities = \ _fiber_densities_by_time[TIME_FRAMES[OFFSET_Y]['early'][0] if _early_time_frames else TIME_FRAMES[OFFSET_Y]['late'][0]: TIME_FRAMES[OFFSET_Y]['early'][1] if _early_time_frames else TIME_FRAMES[OFFSET_Y]['late'][1]] _properties = load.group_properties(_experiment, _series_id, _group) _cell_fiber_densities = compute.remove_blacklist( _experiment, _series_id, _properties['cells_ids'][_cell_id], _cell_fiber_densities) _cell_fiber_densities = compute.longest_fiber_densities_ascending_sequence( _cell_fiber_densities) # fix if found nan if True in np.isnan(_cell_fiber_densities): _cell_fiber_densities = _cell_fiber_densities[:np.where( np.isnan(_cell_fiber_densities))[0][0]] # not enough data if len(_cell_fiber_densities) < DERIVATIVE + 1: continue _z_score_fiber_density = libs.compute_lib.z_score_array( _array=_cell_fiber_densities, _average=_series_normalization['average'], _std=_series_normalization['std']) if _experiment in ['SN41', 'SN44']: for _start_index in [0, 1, 2]: _heatmap_fiber += _z_score_fiber_density[_start_index::3][ DERIVATIVE:] _heatmap_fiber_change += compute_lib.derivative( _z_score_fiber_density[_start_index::3], _n=DERIVATIVE) else: _heatmap_fiber += _z_score_fiber_density[DERIVATIVE:] _heatmap_fiber_change += compute_lib.derivative( _z_score_fiber_density, _n=DERIVATIVE) print( compute_lib.correlation(_heatmap_fiber, _heatmap_fiber_change, _with_p_value=True)) if PLOT: _y_shape = int(round((Y_LABELS_END - Y_LABELS_START) * Y_BINS)) _x_shape = int(round((X_LABELS_END - X_LABELS_START) * X_BINS)) _total_points = 0 _z_array = np.zeros(shape=(_y_shape, _x_shape)) for _y, _x in zip(_heatmap_fiber_change, _heatmap_fiber): _y_rounded, _x_rounded = int(round(_y * Y_BINS)), int( round(_x * X_BINS)) _y_index, _x_index = int(_y_rounded - Y_LABELS_START * Y_BINS), int(_x_rounded - X_LABELS_START * X_BINS) if 0 <= _y_index < _z_array.shape[ 0] and 0 <= _x_index < _z_array.shape[1]: _z_array[_y_index][_x_index] += 1 _total_points += 1 _z_array = _z_array / _total_points if not CONDITIONAL_NORMALIZATION: _z_array[_z_array == 0] = None else: _z_array_plot = np.zeros(shape=np.array(_z_array).shape) for _fiber_index, _fiber_density_z_score in enumerate(_z_array): _sum = np.sum(_fiber_density_z_score) for _change_index, _change_z_score in enumerate( _fiber_density_z_score): _z_array_plot[_fiber_index][_change_index] = ( _change_z_score / _sum) if _sum != 0 else 0 _z_array_plot[_z_array_plot == 0] = None _fig = go.Figure( data=go.Heatmap(x=np.arange(start=X_LABELS_START, stop=X_LABELS_END, step=1 / X_BINS), y=np.arange(start=Y_LABELS_START, stop=Y_LABELS_END, step=1 / Y_BINS), z=_z_array, colorscale='Viridis', colorbar={ 'tickmode': 'array', 'tickvals': [0, 0.025, 0.05], 'ticktext': ['0', 'Fraction', '0.05'], 'tickangle': -90 }, zmin=Z_MIN, zmax=Z_MAX[CONDITIONAL_NORMALIZATION]), layout={ 'xaxis': { 'title': 'Fiber densities z-score', 'zeroline': False }, 'yaxis': { 'title': 'Change in fiber<br>density (z-score)', 'zeroline': False }, 'shapes': [{ 'type': 'line', 'x0': X_LABELS_START, 'y0': Y_LABELS_START, 'x1': X_LABELS_END, 'y1': Y_LABELS_START, 'line': { 'color': 'black', 'width': 2 } }, { 'type': 'line', 'x0': X_LABELS_START, 'y0': Y_LABELS_START, 'x1': X_LABELS_START, 'y1': Y_LABELS_END, 'line': { 'color': 'black', 'width': 2 } }] }) save.to_html(_fig=_fig, _path=os.path.join(paths.PLOTS, save.get_module_name()), _filename='plot_early_' + str(_early_time_frames))
def main(_real_cells=True, _static=False, _dead_dead=False, _live_dead=False, _dead=False, _live=False, _bead=False, _metastasis=False, _bleb=False, _bleb_amount_um=None, _band=True, _high_temporal_resolution=False, _offset_y=0.5): _experiments = all_experiments() _experiments = filtering.by_categories( _experiments=_experiments, _is_single_cell=False, _is_high_temporal_resolution=_high_temporal_resolution, _is_bleb=_bleb, _is_dead_dead=_dead_dead, _is_live_dead=_live_dead, _is_bead=_bead, _is_metastasis=_metastasis) _tuples = load.experiments_groups_as_tuples(_experiments) _tuples = filtering.by_pair_distance_range(_tuples, PAIR_DISTANCE_RANGE) _tuples = filtering.by_real_pairs(_tuples, _real_pairs=_real_cells) _tuples = filtering.by_fake_static_pairs(_tuples, _fake_static_pairs=_static) if _dead_dead is not False or _live_dead is not False: _tuples = filtering.by_dead_live(_tuples, _dead=_dead, _live=_live) _tuples = filtering.by_band(_tuples, _band=_band) if _bleb: _tuples = filtering.by_bleb_amount_um(_tuples, _amount_um=_bleb_amount_um) print('Total tuples:', len(_tuples)) _arguments = [] for _tuple in _tuples: _experiment, _series_id, _group = _tuple _latest_time_frame = compute.latest_time_frame_before_overlapping( _experiment, _series_id, _group, OFFSET_X) for _cell_id in ['left_cell', 'right_cell']: _arguments.append({ 'experiment': _experiment, 'series_id': _series_id, 'group': _group, 'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER, 'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER, 'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER, 'offset_x': OFFSET_X, 'offset_y': _offset_y, 'offset_z': OFFSET_Z, 'cell_id': _cell_id, 'direction': 'inside', 'time_points': _latest_time_frame }) _windows_dictionary, _windows_to_compute = compute.windows( _arguments, _keys=['experiment', 'series_id', 'group', 'cell_id']) _fiber_densities = compute.fiber_densities(_windows_to_compute, _subtract_border=True) _experiments_fiber_densities = { _key: [_fiber_densities[_tuple] for _tuple in _windows_dictionary[_key]] for _key in _windows_dictionary } _tuples_by_experiment = organize.by_experiment(_tuples) _n = 0 _cells_ranks = [] for _experiment in _tuples_by_experiment: print('Experiment:', _experiment) _experiment_tuples = _tuples_by_experiment[_experiment] for _pivot_tuple in tqdm(_experiment_tuples, desc='Main loop'): _pivot_experiment, _pivot_series_id, _pivot_group = _pivot_tuple _pivot_experiment_properties = load.group_properties( _pivot_experiment, _pivot_series_id, _pivot_group) for _pivot_cell_id, _pivot_cell_correct_match_cell_id in \ zip(['left_cell', 'right_cell'], ['right_cell', 'left_cell']): _pivot_cell = (_pivot_experiment, _pivot_series_id, _pivot_group, _pivot_cell_id) _pivot_cell_correct_match_cell = ( _pivot_experiment, _pivot_series_id, _pivot_group, _pivot_cell_correct_match_cell_id) _pivot_cell_fiber_densities = _experiments_fiber_densities[ _pivot_cell] _pivot_cell_fiber_densities = compute.remove_blacklist( _pivot_experiment, _pivot_series_id, _pivot_experiment_properties['cells_ids'][_pivot_cell_id], _pivot_cell_fiber_densities) _pivot_cell_correlations = [] # correct match _pivot_cell_correct_match_fiber_densities = _experiments_fiber_densities[ _pivot_cell_correct_match_cell] _pivot_cell_correct_match_fiber_densities = compute.remove_blacklist( _pivot_experiment, _pivot_series_id, _pivot_experiment_properties['cells_ids'] [_pivot_cell_correct_match_cell_id], _pivot_cell_correct_match_fiber_densities) _pivot_cell_fiber_densities_filtered, _pivot_cell_correct_match_fiber_densities_filtered = \ compute.longest_same_indices_shared_in_borders_sub_array( _pivot_cell_fiber_densities, _pivot_cell_correct_match_fiber_densities ) # ignore small arrays if len(_pivot_cell_fiber_densities_filtered ) < compute.minimum_time_frames_for_correlation( _pivot_experiment): continue _correlation = compute_lib.correlation( compute_lib.derivative( _pivot_cell_fiber_densities_filtered, _n=DERIVATIVE), compute_lib.derivative( _pivot_cell_correct_match_fiber_densities_filtered, _n=DERIVATIVE)) _pivot_cell_correlations.append(_correlation) _pivot_cell_correct_match_correlation = _correlation # create list of potential cells _candidate_tuples = [] for _candidate_tuple in _experiment_tuples: _candidate_experiment, _candidate_series_id, _candidate_group = _candidate_tuple for _candidate_cell_id in ['left_cell', 'right_cell']: _candidate_cell = (_candidate_experiment, _candidate_series_id, _candidate_group, _candidate_cell_id) # skip if same cell or correct match if _candidate_cell == _pivot_cell or _candidate_cell == _pivot_cell_correct_match_cell: continue _candidate_tuples.append(_candidate_cell) # compare with each potential candidate, until reached the maximum or nothing to compare with while len(_pivot_cell_correlations ) < POTENTIAL_MATCHES and len(_candidate_tuples) > 0: # sample randomly _candidate_cell = random.choice(_candidate_tuples) _candidate_experiment, _candidate_series_id, _candidate_group, _candidate_cell_id = _candidate_cell _candidate_experiment_properties = load.group_properties( _candidate_experiment, _candidate_series_id, _candidate_group) _candidate_cell_fiber_densities = _experiments_fiber_densities[ _candidate_cell] _candidate_cell_fiber_densities = compute.remove_blacklist( _candidate_experiment, _candidate_series_id, _candidate_experiment_properties['cells_ids'] [_candidate_cell_id], _candidate_cell_fiber_densities) _pivot_cell_fiber_densities_filtered, _candidate_cell_fiber_densities_filtered = \ compute.longest_same_indices_shared_in_borders_sub_array( _pivot_cell_fiber_densities, _candidate_cell_fiber_densities ) # ignore small arrays if len(_pivot_cell_fiber_densities_filtered ) < compute.minimum_time_frames_for_correlation( _pivot_experiment): _candidate_tuples.remove(_candidate_cell) continue _correlation = compute_lib.correlation( compute_lib.derivative( _pivot_cell_fiber_densities_filtered, _n=DERIVATIVE), compute_lib.derivative( _candidate_cell_fiber_densities_filtered, _n=DERIVATIVE)) _pivot_cell_correlations.append(_correlation) # nothing to compare with if len(_pivot_cell_correlations) == 1: continue # check matchmaking _pivot_cell_correct_match_rank = 1 for _potential_match_correlation in sorted( _pivot_cell_correlations, reverse=True): if _pivot_cell_correct_match_correlation == _potential_match_correlation: break _pivot_cell_correct_match_rank += 1 _n += 1 _cells_ranks.append(_pivot_cell_correct_match_rank) # results _correct_match_probability = 1 / POTENTIAL_MATCHES _first_place_correct_matches = sum( [1 for _rank in _cells_ranks if _rank == 1]) _first_place_fraction = _first_place_correct_matches / _n print('Matchmaking results:') print('Total cells:', _n) print('Correct match probability:', round(_correct_match_probability, 2)) print('Fraction of first place correct matches:', round(_first_place_fraction, 2)) # plot _x = list(range(MAX_RANK)) _x_text = [str(_rank + 1) for _rank in _x[:-1]] + [str(MAX_RANK) + '+'] _ranks_sums = [0 for _rank in _x] for _rank in _cells_ranks: if _rank < MAX_RANK: _ranks_sums[_rank - 1] += 1 else: _ranks_sums[-1] += 1 _y = np.array(_ranks_sums) / _n _fig = go.Figure(data=go.Bar(x=_x, y=_y, marker={'color': '#ea8500'}), layout={ 'xaxis': { 'title': 'Correct match correlation rank', 'zeroline': False, 'tickmode': 'array', 'tickvals': _x, 'ticktext': _x_text }, 'yaxis': { 'title': 'Fraction', 'range': [0, 1.1], 'zeroline': False, 'tickmode': 'array', 'tickvals': [0, 0.5, 1] } }) save.to_html(_fig=_fig, _path=os.path.join(paths.PLOTS, save.get_module_name()), _filename='plot_real_' + str(_real_cells) + '_static_' + str(_static) + '_dead_dead_' + str(_dead_dead) + '_live_dead_' + str(_live_dead) + '_dead_' + str(_dead) + '_live_' + str(_live) + '_bead_' + str(_bead) + '_metastasis_' + str(_metastasis) + '_bleb_' + str(_bleb) + str(_bleb_amount_um) + '_band_' + str(_band) + '_high_time_' + str(_high_temporal_resolution) + '_y_' + str(_offset_y)) # correct match probability plot _y = [_correct_match_probability] * (MAX_RANK - 1) + [ _correct_match_probability * (POTENTIAL_MATCHES - MAX_RANK + 1) ] _fig = go.Figure(data=go.Bar(x=_x, y=_y, marker={'color': '#ea8500'}), layout={ 'xaxis': { 'title': 'Correct match correlation rank', 'zeroline': False, 'tickmode': 'array', 'tickvals': _x, 'ticktext': _x_text }, 'yaxis': { 'title': 'Fraction', 'range': [0, 1.1], 'zeroline': False, 'tickmode': 'array', 'tickvals': [0, 0.5, 1] } }) save.to_html(_fig=_fig, _path=os.path.join(paths.PLOTS, save.get_module_name()), _filename='plot_real_' + str(_real_cells) + '_static_' + str(_static) + '_dead_dead_' + str(_dead_dead) + '_live_dead_' + str(_live_dead) + '_dead_' + str(_dead) + '_live_' + str(_live) + '_bead_' + str(_bead) + '_metastasis_' + str(_metastasis) + '_bleb_' + str(_bleb) + str(_bleb_amount_um) + '_band_' + str(_band) + '_high_time_' + str(_high_temporal_resolution) + '_y_' + str(_offset_y) + '_correct_match_prob')
def main(): _experiments = all_experiments() _experiments = filtering.by_categories(_experiments=_experiments, _is_single_cell=False, _is_high_temporal_resolution=False, _is_bleb=False, _is_dead_dead=False, _is_live_dead=False, _is_bead=False, _is_metastasis=False) _tuples = load.experiments_groups_as_tuples(_experiments) _tuples = filtering.by_time_frames_amount( _tuples, compute.density_time_frame(_experiments[0])) _tuples = filtering.by_real_pairs(_tuples) _tuples = filtering.by_band(_tuples) _tuples = filtering.by_pair_distance_range(_tuples, PAIR_DISTANCE_RANGE) print('Total tuples:', len(_tuples)) _arguments = [] for _tuple in _tuples: _experiment, _series_id, _group = _tuple _latest_time_frame = compute.latest_time_frame_before_overlapping( _experiment, _series_id, _group, OFFSET_X) for _cell_id in ['left_cell', 'right_cell']: _arguments.append({ 'experiment': _experiment, 'series_id': _series_id, 'group': _group, 'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER, 'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER, 'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER, 'offset_x': OFFSET_X, 'offset_y': OFFSET_Y, 'offset_z': OFFSET_Z, 'cell_id': _cell_id, 'direction': 'inside', 'time_points': _latest_time_frame }) _windows_dictionary, _windows_to_compute = \ compute.windows(_arguments, _keys=['experiment', 'series_id', 'group', 'cell_id']) _fiber_densities = compute.fiber_densities(_windows_to_compute, _subtract_border=True) _experiments_fiber_densities = { _key: [_fiber_densities[_tuple] for _tuple in _windows_dictionary[_key]] for _key in _windows_dictionary } _kpss_y_arrays = [[] for _i in DERIVATIVES] _adf_y_arrays = [[] for _i in DERIVATIVES] for _tuple in tqdm(_tuples, desc='Experiments loop'): _experiment, _series_id, _group = _tuple _properties = load.group_properties(_experiment, _series_id, _group) _left_cell_fiber_densities = _experiments_fiber_densities[( _experiment, _series_id, _group, 'left_cell')] _left_cell_fiber_densities = compute.remove_blacklist( _experiment, _series_id, _properties['cells_ids']['left_cell'], _left_cell_fiber_densities) _right_cell_fiber_densities = _experiments_fiber_densities[( _experiment, _series_id, _group, 'right_cell')] _right_cell_fiber_densities = compute.remove_blacklist( _experiment, _series_id, _properties['cells_ids']['right_cell'], _right_cell_fiber_densities) if not OUT_OF_BOUNDARIES: _left_cell_fiber_densities = \ compute.longest_fiber_densities_ascending_sequence(_left_cell_fiber_densities) _right_cell_fiber_densities = \ compute.longest_fiber_densities_ascending_sequence(_right_cell_fiber_densities) else: _left_cell_fiber_densities = [ _fiber_density[0] for _fiber_density in _left_cell_fiber_densities ] _right_cell_fiber_densities = [ _fiber_density[0] for _fiber_density in _right_cell_fiber_densities ] # ignore small arrays _minimum_time_frames_for_correlation = compute.minimum_time_frames_for_correlation( _experiment) if len(_left_cell_fiber_densities) < _minimum_time_frames_for_correlation or \ len(_right_cell_fiber_densities) < _minimum_time_frames_for_correlation: continue for _derivative_index, _derivative in enumerate(DERIVATIVES): for _cell_fiber_densities in [ _left_cell_fiber_densities, _right_cell_fiber_densities ]: _cell_fiber_densities_derivative = compute_lib.derivative( _cell_fiber_densities, _n=_derivative) _, _kpss_p_value, _, _ = kpss(_cell_fiber_densities_derivative, nlags='legacy') _kpss_y_arrays[_derivative_index].append(_kpss_p_value) _, _adf_p_value, _, _, _, _ = adfuller( _cell_fiber_densities_derivative) _adf_y_arrays[_derivative_index].append(_adf_p_value) print('Total pairs:', len(_kpss_y_arrays[0]) / 2) # print results print('KPSS:') for _derivative_index, _derivative in enumerate(DERIVATIVES): _stationary_count = len([ _value for _value in _kpss_y_arrays[_derivative_index] if _value > 0.05 ]) print( 'Derivative:', _derivative, 'Stationary:', str(_stationary_count / len(_kpss_y_arrays[_derivative_index]) * 100) + '%') print('ADF:') for _derivative_index, _derivative in enumerate(DERIVATIVES): _stationary_count = len([ _value for _value in _adf_y_arrays[_derivative_index] if _value < 0.05 ]) print( 'Derivative:', _derivative, 'Stationary:', str(_stationary_count / len(_adf_y_arrays[_derivative_index]) * 100) + '%') # plot _colors_array = config.colors(3) for _test_name, _y_title, _y_tickvals, _p_value_line, _y_arrays in \ zip( ['kpss', 'adf'], ['KPSS test p-value', 'ADF test p-value'], [[0.05, 0.1], [0.05, 1]], [0.05, 0.05], [_kpss_y_arrays, _adf_y_arrays] ): _fig = go.Figure(data=[ go.Box(y=_y, name=_derivative, boxpoints='all', jitter=1, pointpos=0, line={'width': 1}, fillcolor='white', marker={ 'size': 10, 'color': _color }, opacity=0.7, showlegend=False) for _y, _derivative, _color in zip( _y_arrays, DERIVATIVES_TEXT, _colors_array) ], layout={ 'xaxis': { 'title': 'Fiber density derivative', 'zeroline': False }, 'yaxis': { 'title': _y_title, 'zeroline': False, 'tickmode': 'array', 'tickvals': _y_tickvals }, 'shapes': [{ 'type': 'line', 'x0': DERIVATIVES[0] - 0.75, 'y0': _p_value_line, 'x1': DERIVATIVES[-1] + 0.75, 'y1': _p_value_line, 'line': { 'color': 'red', 'width': 2, 'dash': 'dash' } }] }) save.to_html(_fig=_fig, _path=os.path.join(paths.PLOTS, save.get_module_name()), _filename='plot_' + _test_name)
def main(_real_cells=True, _static=False, _band=True, _high_temporal_resolution=False, _pair_distance_range=None, _offset_y=0.5): if _pair_distance_range is None: _pair_distance_range = PAIR_DISTANCE_RANGE _experiments = all_experiments() _experiments = filtering.by_categories( _experiments=_experiments, _is_single_cell=False, _is_high_temporal_resolution=_high_temporal_resolution, _is_bleb=False, _is_dead_dead=False, _is_live_dead=False, _is_bead=False, _is_metastasis=False) _tuples = load.experiments_groups_as_tuples(_experiments) _tuples = filtering.by_pair_distance_range(_tuples, _pair_distance_range) _tuples = filtering.by_real_pairs(_tuples, _real_pairs=_real_cells) _tuples = filtering.by_fake_static_pairs(_tuples, _fake_static_pairs=_static) _tuples = filtering.by_band(_tuples, _band=_band) print('Total tuples:', len(_tuples)) _arguments = [] _longest_time_frame = 0 for _tuple in _tuples: _experiment, _series_id, _group = _tuple _latest_time_frame = compute.latest_time_frame_before_overlapping( _experiment, _series_id, _group, OFFSET_X) # save for later if _latest_time_frame > _longest_time_frame: _longest_time_frame = _latest_time_frame for _cell_id in ['left_cell', 'right_cell']: _arguments.append({ 'experiment': _experiment, 'series_id': _series_id, 'group': _group, 'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER, 'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER, 'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER, 'offset_x': OFFSET_X, 'offset_y': _offset_y, 'offset_z': OFFSET_Z, 'cell_id': _cell_id, 'direction': 'inside', 'time_points': _latest_time_frame }) _windows_dictionary, _windows_to_compute = compute.windows( _arguments, _keys=['experiment', 'series_id', 'group', 'cell_id']) _fiber_densities = compute.fiber_densities(_windows_to_compute) _experiments_fiber_densities = { _key: [_fiber_densities[_tuple] for _tuple in _windows_dictionary[_key]] for _key in _windows_dictionary } _valid_tuples = [] _valid_cells = [] _densities = [[] for _ in range(_longest_time_frame)] for _tuple in tqdm(_tuples, desc='Experiments loop'): _experiment, _series_id, _group = _tuple _normalization = load.normalization_series_file_data( _experiment, _series_id) _properties = load.group_properties(_experiment, _series_id, _group) for _cell_id in ['left_cell', 'right_cell']: _cell_fiber_densities = \ _experiments_fiber_densities[(_experiment, _series_id, _group, _cell_id)] _cell_fiber_densities = compute.remove_blacklist( _experiment, _series_id, _properties['cells_ids'][_cell_id], _cell_fiber_densities) _previous_cell_fiber_density_normalized = None for _time_frame, _cell_fiber_density in enumerate( _cell_fiber_densities): # not out of border if _cell_fiber_density[1]: _previous_cell_fiber_density_normalized = None continue # normalize _cell_fiber_density_normalized = compute_lib.z_score( _x=_cell_fiber_density[0], _average=_normalization['average'], _std=_normalization['std']) # no previous if _previous_cell_fiber_density_normalized is None: _previous_cell_fiber_density_normalized = _cell_fiber_density_normalized continue # change _cell_fiber_density_normalized_change = _cell_fiber_density_normalized - _previous_cell_fiber_density_normalized _previous_cell_fiber_density_normalized = _cell_fiber_density_normalized # save _densities[_time_frame].append( _cell_fiber_density_normalized_change) if _tuple not in _valid_tuples: _valid_tuples.append(_tuple) _cell_tuple = (_experiment, _series_id, _group, _cell_id) if _cell_tuple not in _valid_cells: _valid_cells.append(_cell_tuple) print('Total pairs:', len(_valid_tuples)) print('Total cells:', len(_valid_cells)) # plot _temporal_resolution = compute.temporal_resolution_in_minutes( _experiments[0]) _fig = go.Figure( data=go.Scatter(x=np.array(range(_longest_time_frame)) * _temporal_resolution, y=[np.mean(_array) for _array in _densities], name='Fiber density change (z-score)', error_y={ 'type': 'data', 'array': [np.std(_array) for _array in _densities], 'thickness': 1 }, mode='lines+markers', marker={ 'size': 5, 'color': '#ea8500' }, line={'dash': 'solid'}, showlegend=False), layout={ 'xaxis': { 'title': 'Time (minutes)', # 'zeroline': False }, 'yaxis': { 'title': 'Fiber density change (z-score)', # 'zeroline': False }, # 'shapes': [ # { # 'type': 'line', # 'x0': -_temporal_resolution, # 'y0': -0.5, # 'x1': -_temporal_resolution, # 'y1': 2, # 'line': { # 'color': 'black', # 'width': 2 # } # }, # { # 'type': 'line', # 'x0': -_temporal_resolution, # 'y0': -0.5, # 'x1': 350, # 'y1': -0.5, # 'line': { # 'color': 'black', # 'width': 2 # } # } # ] }) save.to_html( _fig=_fig, _path=os.path.join(paths.PLOTS, save.get_module_name()), _filename='plot_real_' + str(_real_cells) + '_static_' + str(_static) + '_band_' + str(_band) + '_high_time_' + str(_high_temporal_resolution) + '_range_' + '_'.join([str(_distance) for _distance in _pair_distance_range]) + '_y_' + str(_offset_y))
def process_group(_arguments): _time_frames_data = [] _time_frames_amount = \ len([_value for _value in _arguments['cell_coordinates'][_arguments['cell_id']] if _value is not None]) # smooth coordinates _cells_coordinates_cell_smoothed = compute.smooth_coordinates_in_time( [ _value for _value in _arguments['cell_coordinates'][_arguments['cell_id']] if _value is not None ], _n=SMOOTH_AMOUNT) if _arguments['cell_type'] == 'real': _group = 'cell_' + str(_arguments['cell_id']) + '_' + str(_arguments['degrees_xy']) + '_' + \ str(_arguments['degrees_z']) elif _arguments['cell_type'] == 'fake': _group = 'fake_' + str(_arguments['cell_id']) + '_' + str(_arguments['degrees_xy']) + '_' + \ str(_arguments['degrees_z']) elif _arguments['cell_type'] == 'static': _group = 'static_' + str(_arguments['cell_id']) + '_' + str(_arguments['degrees_xy']) + '_' + \ str(_arguments['degrees_z']) else: raise Exception('No such cell type') # check if needed (missing time-point / properties file) if not _arguments['overwrite']: _missing = False for _time_frame in range(_time_frames_amount): _time_frame_pickle_path = \ paths.structured(_arguments['experiment'], _arguments['series_id'], _group, _time_frame) if not os.path.isfile(_time_frame_pickle_path): _missing = True break _group_structured_path = paths.structured(_arguments['experiment'], _arguments['series_id'], _group) _properties_json_path = os.path.join(_group_structured_path, 'properties.json') if not os.path.isfile(_properties_json_path): _missing = True if not _missing: return # load image if needed if 'series_image_by_time_frames' not in _arguments: _series_image_path = \ paths.serieses(_arguments['experiment'], _arguments['series_id']) _series_image = tifffile.imread(_series_image_path) _arguments['series_image_by_time_frames'] = [ np.array([ _z[IMAGE_FIBER_CHANNEL_INDEX] for _z in _series_image[_time_frame] ]) for _time_frame in range(_series_image.shape[0]) ] # running for each time point for _time_frame in range(_time_frames_amount): _time_frame_image = _arguments['series_image_by_time_frames'][ _time_frame] _cell_coordinates = [ _value for _value in _cells_coordinates_cell_smoothed[_time_frame] ] # update coordinates if needed if 'x_change' in _arguments: _cell_coordinates[0] += _arguments['x_change'] if 'y_change' in _arguments: _cell_coordinates[1] += _arguments['y_change'] if 'z_change' in _arguments: _cell_coordinates[2] += _arguments['z_change'] # compute padding xy _padding_x, _padding_y = \ compute.axes_padding(_2d_image_shape=_time_frame_image[0].shape, _angle=_arguments['degrees_xy']) _cell_coordinates[0] += _padding_x _cell_coordinates[1] += _padding_y # rotate image and change axes _time_frame_image_rotated = np.array( [rotate(_z, _arguments['degrees_xy']) for _z in _time_frame_image]) _time_frame_image_swapped = np.swapaxes(_time_frame_image_rotated, 0, 1) if SHOW_PLOTS: plt.imshow(_time_frame_image_rotated[int( round(_cell_coordinates[2]))]) plt.show() # update coordinates _image_center = compute.image_center_coordinates( _image_shape=reversed(_time_frame_image_rotated[0].shape)) _cell_coordinates = compute.rotate_point_around_another_point( _point=_cell_coordinates, _angle_in_radians=math.radians(_arguments['degrees_xy']), _around_point=_image_center) # y is now z, z is now y _cell_coordinates[1], _cell_coordinates[2] = _cell_coordinates[ 2], _cell_coordinates[1] if SHOW_PLOTS: plt.imshow(_time_frame_image_swapped[int( round(_cell_coordinates[2]))]) plt.show() # swap resolutions _new_resolutions = { 'x': _arguments['resolutions']['x'], 'y': _arguments['resolutions']['z'], 'z': _arguments['resolutions']['y'] } # second rotate, compute padding z _padding_x, _padding_y = \ compute.axes_padding(_2d_image_shape=_time_frame_image_swapped[0].shape, _angle=_arguments['degrees_z']) _cell_coordinates[0] += _padding_x _cell_coordinates[1] += _padding_y # rotate image _time_frame_image_swapped_rotated = \ np.array([rotate(_z, _arguments['degrees_z']) for _z in _time_frame_image_swapped]) # update coordinates _image_center = compute.image_center_coordinates( _image_shape=reversed(_time_frame_image_swapped_rotated[0].shape)) _cell_coordinates = compute.rotate_point_around_another_point( _point=_cell_coordinates, _angle_in_radians=math.radians(_arguments['degrees_z']), _around_point=_image_center) if SHOW_PLOTS: if _time_frame == 0 or _time_frame == 50 or _time_frame == 150: plt.imshow(_time_frame_image_swapped_rotated[int( round(_cell_coordinates[2]))]) plt.show() # update resolutions _angle = abs(_arguments['degrees_z']) _new_resolution_x = (_angle / 90) * _new_resolutions['y'] + ( (90 - _angle) / 90) * _new_resolutions['x'] _new_resolution_y = (_angle / 90) * _new_resolutions['x'] + ( (90 - _angle) / 90) * _new_resolutions['y'] _new_resolutions['x'] = _new_resolution_x _new_resolutions['y'] = _new_resolution_y _image_z, _image_y, _image_x = _time_frame_image_swapped_rotated.shape if not 0 <= _cell_coordinates[0] < _image_x or not \ 0 <= _cell_coordinates[1] < _image_y or not \ 0 <= _cell_coordinates[2] < _image_z: break # add to array _time_frames_data.append({ 'cell': { 'coordinates': { 'x': _cell_coordinates[0], 'y': _cell_coordinates[1], 'z': _cell_coordinates[2] } }, 'resolutions': _new_resolutions }) # save to pickle _time_frame_pickle_path = \ paths.structured(_arguments['experiment'], _arguments['series_id'], _group, _time_frame) save_lib.to_pickle(_time_frame_image_swapped_rotated, _time_frame_pickle_path) # save properties if _arguments['cell_type'] == 'real': _fake = False _static = False elif _arguments['cell_type'] == 'fake': _based_on_properties = \ load.group_properties(_arguments['experiment'], _arguments['series_id'], 'cell_' + _group.split('fake_')[1]) _fake = True _static = False elif _arguments['cell_type'] == 'static': _fake = True _static = True else: raise Exception('No such cell type') _properties_data = { 'experiment': _arguments['experiment'], 'series_id': _arguments['series_id'], 'cell_id': _arguments['cell_id'], 'time_points': _time_frames_data, 'fake': _fake, 'static': _static } _group_structured_path = paths.structured(_arguments['experiment'], _arguments['series_id'], _group) _properties_json_path = os.path.join(_group_structured_path, 'properties.json') save_lib.to_json(_properties_data, _properties_json_path)
def main(_high_temporal_resolution=True): _experiments = all_experiments() _experiments = filtering.by_categories( _experiments=_experiments, _is_single_cell=False, _is_high_temporal_resolution=_high_temporal_resolution, _is_bleb=False, _is_dead_dead=False, _is_live_dead=False, _is_bead=False, _is_metastasis=False) _tuples = load.experiments_groups_as_tuples(_experiments) _tuples = filtering.by_time_frames_amount( _tuples, _time_frames=MOVING_WINDOW_LENGTH[_high_temporal_resolution]) _tuples = filtering.by_pair_distance_range( _tuples, _distance_range=PAIR_DISTANCE_RANGE) _tuples = filtering.by_real_pairs(_tuples) _tuples = filtering.by_band(_tuples) print('Total tuples:', len(_tuples)) _arguments = [] for _tuple in _tuples: _experiment, _series_id, _group = _tuple _latest_time_frame = compute.latest_time_frame_before_overlapping( _experiment, _series_id, _group, OFFSET_X) for _cell_id in ['left_cell', 'right_cell']: _arguments.append({ 'experiment': _experiment, 'series_id': _series_id, 'group': _group, 'length_x': QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER, 'length_y': QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER, 'length_z': QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER, 'offset_x': OFFSET_X, 'offset_y': OFFSET_Y, 'offset_z': OFFSET_Z, 'cell_id': _cell_id, 'direction': 'inside', 'time_points': _latest_time_frame }) _windows_dictionary, _windows_to_compute = compute.windows( _arguments, _keys=['experiment', 'series_id', 'group', 'cell_id']) _fiber_densities = compute.fiber_densities(_windows_to_compute, _subtract_border=True) _experiments_fiber_densities = { _key: [_fiber_densities[_tuple] for _tuple in _windows_dictionary[_key]] for _key in _windows_dictionary } for _tuple in _tuples: _correlations = [] _experiment, _series_id, _group = _tuple _left_cell_fiber_densities = \ _experiments_fiber_densities[(_experiment, _series_id, _group, 'left_cell')] _right_cell_fiber_densities = \ _experiments_fiber_densities[(_experiment, _series_id, _group, 'right_cell')] _properties = load.group_properties(_experiment, _series_id, _group) _left_cell_fiber_densities = compute.remove_blacklist( _experiment, _series_id, _properties['cells_ids']['left_cell'], _left_cell_fiber_densities) _right_cell_fiber_densities = compute.remove_blacklist( _experiment, _series_id, _properties['cells_ids']['right_cell'], _right_cell_fiber_densities) for _start_time_frame in \ range(0, END_TIME_FRAME[_high_temporal_resolution], TIME_FRAME_STEP[_high_temporal_resolution]): _left_cell_fiber_densities_window = _left_cell_fiber_densities[ _start_time_frame:_start_time_frame + MOVING_WINDOW_LENGTH[_high_temporal_resolution]] _right_cell_fiber_densities_window = _right_cell_fiber_densities[ _start_time_frame:_start_time_frame + MOVING_WINDOW_LENGTH[_high_temporal_resolution]] _left_cell_fiber_densities_filtered, _right_cell_fiber_densities_filtered = \ compute.longest_same_indices_shared_in_borders_sub_array( _left_cell_fiber_densities_window, _right_cell_fiber_densities_window) # ignore small arrays if len(_left_cell_fiber_densities_filtered ) < MOVING_WINDOW_LENGTH[_high_temporal_resolution]: _correlations.append(None) continue _correlations.append( compute_lib.correlation( compute_lib.derivative(_left_cell_fiber_densities_filtered, _n=DERIVATIVE), compute_lib.derivative( _right_cell_fiber_densities_filtered, _n=DERIVATIVE))) # plot _temporal_resolution = compute.temporal_resolution_in_minutes( _experiment) _fig = go.Figure(data=go.Scatter( x=np.arange(start=0, stop=len(_correlations), step=1) * _temporal_resolution * TIME_FRAME_STEP[_high_temporal_resolution], y=_correlations, mode='lines+markers', line={'dash': 'solid'}), layout={ 'xaxis': { 'title': 'Window start time (minutes)', 'zeroline': False }, 'yaxis': { 'title': 'Inner correlation', 'zeroline': False } }) save.to_html(_fig=_fig, _path=os.path.join(paths.PLOTS, save.get_module_name()), _filename='plot_' + str(_experiment) + '_' + str(_series_id) + '_' + str(_group))
def main(): _experiments = config.all_experiments() _experiments = filtering.by_categories(_experiments=_experiments, _is_single_cell=False, _is_high_temporal_resolution=None, _is_bleb=False, _is_dead_dead=False, _is_live_dead=False, _is_bead=False, _is_metastasis=False) _tuples = load.experiments_groups_as_tuples(_experiments) _tuples = filtering.by_pair_distance_range(_tuples, PAIR_DISTANCE_RANGE) _tuples = filtering.by_real_pairs(_tuples, _real_pairs=REAL) _tuples = filtering.by_fake_static_pairs(_tuples, _fake_static_pairs=STATIC) _tuples = filtering.by_real_fake_pairs(_tuples, _real_fake_pairs=REAL_FAKE) print('Total tuples:', len(_tuples)) _arguments = [] for _tuple in tqdm(_tuples, desc='Setting windows to compute'): _experiment, _series_id, _group = _tuple _latest_time_frame = compute.latest_time_frame_before_overlapping( _experiment, _series_id, _group, _offset_x=0) for _cell_id in ['left_cell', 'right_cell']: for _offset_y in OFFSETS_Y: _arguments.append({ 'experiment': _experiment, 'series_id': _series_id, 'group': _group, 'length_x': config.QUANTIFICATION_WINDOW_LENGTH_IN_CELL_DIAMETER, 'length_y': config.QUANTIFICATION_WINDOW_HEIGHT_IN_CELL_DIAMETER, 'length_z': config.QUANTIFICATION_WINDOW_WIDTH_IN_CELL_DIAMETER, 'offset_x': 0, 'offset_y': _offset_y, 'offset_z': 0, 'cell_id': _cell_id, 'direction': 'inside', 'time_points': _latest_time_frame }) _windows_dictionary, _windows_to_compute = \ compute.windows(_arguments, _keys=['experiment', 'series_id', 'group', 'cell_id', 'offset_y']) _fiber_densities = compute.fiber_densities(_windows_to_compute) _experiments_fiber_densities = { _key: [_fiber_densities[_tuple] for _tuple in _windows_dictionary[_key]] for _key in _windows_dictionary } _headers = [ 'time_frame', 'experiment', 'series_id', 'group', 'left_cell_id', 'right_cell_id', 'band', 'fake_following', 'fake_static', 'pair_distance_in_cell_diameter', 'offset_z', 'left_cell_fiber_density', 'left_cell_fiber_density_z_score', 'left_cell_out_of_boundaries', 'right_cell_fiber_density', 'right_cell_fiber_density_z_score', 'right_cell_out_of_boundaries' ] _csv_path = os.path.join(paths.OUTPUTS, 'experiments_density_cell_pairs.csv') with open(_csv_path, 'w', newline='') as _csv_file: _csv_writer = csv.writer(_csv_file) _csv_writer.writerow(_headers) for _tuple in tqdm(_tuples, desc='Main loop'): _experiment, _series_id, _group = _tuple _group_properties = load.group_properties(_experiment, _series_id, _group) _left_cell_id, _right_cell_id = _group_properties[ 'cells_ids'].values() _band = _group_properties['band'] _fake_following, _fake_static = _group_properties[ 'fake'], _group_properties['static'] _average, _std = load.normalization_series_file_data( _experiment, _series_id).values() for _offset_y in OFFSETS_Y: _left_cell_fiber_densities = \ _experiments_fiber_densities[(_experiment, _series_id, _group, 'left_cell', _offset_y)] _right_cell_fiber_densities = \ _experiments_fiber_densities[(_experiment, _series_id, _group, 'right_cell', _offset_y)] _left_cell_fiber_densities = compute.remove_blacklist( _experiment, _series_id, _left_cell_id, _left_cell_fiber_densities) _right_cell_fiber_densities = compute.remove_blacklist( _experiment, _series_id, _right_cell_id, _right_cell_fiber_densities) for _time_frame, (_left_cell_fiber_density, _right_cell_fiber_density) in \ enumerate(zip(_left_cell_fiber_densities, _right_cell_fiber_densities)): _pair_distance = compute.pair_distance_in_cell_size_time_frame( _experiment, _series_id, _group, _time_frame) _left_cell_fiber_density, _left_cell_out_of_boundaries = _left_cell_fiber_density _right_cell_fiber_density, _right_cell_out_of_boundaries = _right_cell_fiber_density _left_cell_fiber_density_z_score = compute_lib.z_score( _left_cell_fiber_density, _average, _std) _right_cell_fiber_density_z_score = compute_lib.z_score( _right_cell_fiber_density, _average, _std) _csv_writer.writerow([ _time_frame, _experiment, _series_id, _group, _left_cell_id, _right_cell_id, _band, _fake_following, _fake_static, _pair_distance, _offset_y, _left_cell_fiber_density, _left_cell_fiber_density_z_score, _left_cell_out_of_boundaries, _right_cell_fiber_density, _right_cell_fiber_density_z_score, _right_cell_out_of_boundaries ])
def compute_data(_arguments): _offset_y_index, _offset_y, _offset_z_index, _offset_z = _arguments _same_correlations_array = [] _different_correlations_array = [] for _experiment in _tuples_by_experiment: _experiment_tuples = _tuples_by_experiment[_experiment] for _same_index in range(len(_experiment_tuples)): _same_tuple = _experiment_tuples[_same_index] _same_experiment, _same_series, _same_group = _same_tuple _same_left_cell_fiber_densities = _experiments_fiber_densities[( _same_experiment, _same_series, _same_group, _offset_y, _offset_z, 'left_cell')] _same_right_cell_fiber_densities = _experiments_fiber_densities[( _same_experiment, _same_series, _same_group, _offset_y, _offset_z, 'right_cell')] _same_properties = \ load.group_properties(_same_experiment, _same_series, _same_group) _same_left_cell_fiber_densities = compute.remove_blacklist( _same_experiment, _same_series, _same_properties['cells_ids']['left_cell'], _same_left_cell_fiber_densities) _same_right_cell_fiber_densities = compute.remove_blacklist( _same_experiment, _same_series, _same_properties['cells_ids']['right_cell'], _same_right_cell_fiber_densities) _same_left_cell_fiber_densities_filtered, _same_right_cell_fiber_densities_filtered = \ compute.longest_same_indices_shared_in_borders_sub_array( _same_left_cell_fiber_densities, _same_right_cell_fiber_densities ) # ignore small arrays if len(_same_left_cell_fiber_densities_filtered ) < compute.minimum_time_frames_for_correlation( _same_experiment): continue _same_correlation = compute_lib.correlation( compute_lib.derivative( _same_left_cell_fiber_densities_filtered, _n=DERIVATIVE), compute_lib.derivative( _same_right_cell_fiber_densities_filtered, _n=DERIVATIVE)) for _different_index in range(len(_experiment_tuples)): if _same_index != _different_index: _different_tuple = _experiment_tuples[_different_index] _different_experiment, _different_series, _different_group = _different_tuple for _same_cell_id, _different_cell_id in product( ['left_cell', 'right_cell'], ['left_cell', 'right_cell']): _same_fiber_densities = _experiments_fiber_densities[( _same_experiment, _same_series, _same_group, _offset_y, _offset_z, _same_cell_id)] _different_fiber_densities = _experiments_fiber_densities[ (_different_experiment, _different_series, _different_group, _offset_y, _offset_z, _different_cell_id)] _different_properties = load.group_properties( _different_experiment, _different_series, _different_group) _same_fiber_densities = compute.remove_blacklist( _same_experiment, _same_series, _same_properties['cells_ids'][_same_cell_id], _same_fiber_densities) _different_fiber_densities = compute.remove_blacklist( _different_experiment, _different_series, _different_properties['cells_ids'] [_different_cell_id], _different_fiber_densities) _same_fiber_densities_filtered, _different_fiber_densities_filtered = \ compute.longest_same_indices_shared_in_borders_sub_array( _same_fiber_densities, _different_fiber_densities ) # ignore small arrays if len(_same_fiber_densities_filtered ) < compute.minimum_time_frames_for_correlation( _different_experiment): continue _different_correlation = compute_lib.correlation( compute_lib.derivative( _same_fiber_densities_filtered, _n=DERIVATIVE), compute_lib.derivative( _different_fiber_densities_filtered, _n=DERIVATIVE)) _same_correlations_array.append(_same_correlation) _different_correlations_array.append( _different_correlation) # compute fraction _annotation = None _same_minus_different = np.array(_same_correlations_array) - np.array( _different_correlations_array) _same_count = len(_same_minus_different[_same_minus_different > 0]) if len(_same_minus_different) > 0: _same_fraction = round(_same_count / len(_same_minus_different), 10) _wilcoxon = wilcoxon(_same_minus_different) _p_value = _wilcoxon[1] if _p_value > 0.05: _annotation = { 'text': 'x', 'showarrow': False, 'x': _offset_z, 'y': _offset_y, 'font': { 'size': 6, 'color': 'red' } } else: _same_fraction = None return _offset_y_index, _offset_z_index, _same_fraction, _annotation