def compile_day_metadata(aux_dir, day):
""" Compile experiment day metadata from wormsorter and hydra rig metadata for a given day in
'AuxiliaryFiles' directory
Parameters
----------
aux_dir : str
Path to "AuxiliaryFiles" containing metadata
day : str, None
Experiment day folder in format 'YYYYMMDD'
Returns
-------
compiled_day_metadata : pandas.DataFrame
Compiled metadata dataframe for given imaging date
"""
day_dir = Path(aux_dir) / str(day)
wormsorter_meta = day_dir / (str(day) + '_wormsorter.csv')
hydra_meta = day_dir / (str(day) + '_manual_metadata.csv')
# expand wormsorter metadata to have separate row for each well
plate_metadata = populate_96WPs(wormsorter_meta)
day_metadata = get_day_metadata(complete_plate_metadata=plate_metadata,
hydra_metadata_file=hydra_meta,
merge_on=['imaging_plate_id'],
n_wells=96,
run_number_regex='run\\d+_',
saveto=None,
del_if_exists=False,
include_imgstore_name=True,
raw_day_dir=None)
return day_metadata
bad_wells_df = convert_bad_wells_lut(bad_wells_file)
plate_metadata = pd.merge(plate_metadata,
bad_wells_df,
on=['imaging_plate_id', 'well_name'],
how='outer')
plate_metadata['is_bad_well'].fillna(False, inplace=True)
metadata_file = day / '{}_day_metadata_test.csv'.format(exp_date)
print('Generating day metadata: {}'.format(metadata_file))
try:
day_metadata = get_day_metadata(plate_metadata,
manualmetadata_file,
saveto=metadata_file,
del_if_exists=True,
include_imgstore_name=True)
except ValueError:
print('imgstore error')
day_metadata = get_day_metadata(plate_metadata,
manualmetadata_file,
saveto=metadata_file,
del_if_exists=True,
include_imgstore_name=False)
day_metadata = pd.merge(
day_metadata,
drug_plates,
left_on=['source_plate_id', 'well_name'],
right_on=['shuffled_plate_id', 'destination_well'],
assert (exp_date in str(manualmetadata_file))
wormsorter_file = list(day.rglob('*_wormsorter.csv'))[0]
assert (exp_date in str(wormsorter_file))
print('Collating manual metadata files: {}'.format(wormsorter_file))
plate_metadata = populate_96WPs(wormsorter_file,
del_if_exists=True,
saveto='default')
metadata_file = day / '{}_day_metadata.csv'.format(exp_date)
print('Generating day metadata: {}'.format(metadata_file))
day_metadata = get_day_metadata(plate_metadata,
manualmetadata_file,
saveto=metadata_file,
del_if_exists=True)
files_to_check = day_metadata_check(day_metadata, day, plate_size=48)
number_wells_per_plate(day_metadata, day)
# %%
import datetime
# combine all the metadata files
concat_meta = concatenate_days_metadata(PROJECT_DIRECTORY /
'AuxiliaryFiles',
list_days=None,
saveto=None)
concat_meta_grouped = concat_meta.groupby('worm_gene')
def compile_day_metadata(aux_dir,
day,
n_wells=96,
from_source_plate=False,
from_robot_runlog=False):
""" Compile experiment day metadata from wormsorter and hydra rig metadata for a given day in
'AuxiliaryFiles' directory
Parameters
----------
aux_dir : str
Path to "AuxiliaryFiles" containing metadata
day : str, None
Experiment day folder in format 'YYYYMMDD'
from_source_plate : bool
Compile day metadata using source plate metadata ('source_plate.csv') and mapping for
each imaging plate ID ('source2imaging.csv')
from_robot_runlog : bool
Compile plate metadata from OpenTrons run log files (eg. for randomised plates)
Returns
-------
day_metadata : pandas.DataFrame
Compiled day metadata
"""
from tierpsytools.hydra.compile_metadata import (
populate_6WPs,
populate_24WPs,
populate_96WPs,
get_day_metadata,
get_source_metadata,
merge_basic_and_source_meta,
#merge_robot_metadata,
#merge_robot_wormsorter,
number_wells_per_plate,
day_metadata_check)
day_dir = Path(aux_dir) / str(day)
wormsorter_meta = day_dir / (str(day) + '_wormsorter.csv')
hydra_meta = day_dir / (str(day) + '_manual_metadata.csv')
# Expand wormsorter metadata to have separate row for each well (96-well or 6-well plate format)
if n_wells == 6:
plate_metadata = populate_6WPs(wormsorter_meta)
elif n_wells == 24:
plate_metadata = populate_24WPs(wormsorter_meta)
elif n_wells == 96:
plate_metadata = populate_96WPs(wormsorter_meta)
else:
raise ValueError("Please choose n_wells from: [6, 24, 96]")
from_source_plate = True if from_robot_runlog else from_source_plate
if from_source_plate:
source_plate_meta = day_dir / 'source_plates.csv'
plate_mapping_meta = day_dir / 'source2plate.csv'
source_metadata = get_source_metadata(
sourceplates_file=source_plate_meta,
imaging2source_file=plate_mapping_meta)
plate_metadata = merge_basic_and_source_meta(
plate_metadata,
source_metadata,
merge_on=['imaging_plate_id', 'well_name'],
saveto=None,
del_if_exists=False)
# =============================================================================
# if from_robot_runlog:
# robot_runlog_meta = day_dir / 'robot_runlog.csv'
#
# drug_metadata = merge_robot_metadata(robot_runlog_meta,
# randomized_by='column',
# saveto=None,
# drug_by_column=True,
# compact_drug_plate=False,
# del_if_exists=False)
#
# plate_metadata = merge_robot_wormsorter(day_dir,
# drug_metadata,
# plate_metadata,
# bad_wells_csv=None,
# merge_on=['imaging_plate_id', 'well_name'],
# saveto=None,
# del_if_exists=False)
# =============================================================================
day_metadata = get_day_metadata(complete_plate_metadata=plate_metadata,
hydra_metadata_file=hydra_meta,
merge_on=['imaging_plate_id'],
n_wells=n_wells,
run_number_regex='run\\d+_',
saveto=None,
del_if_exists=False,
include_imgstore_name=True,
raw_day_dir=None)
day_metadata_check(day_metadata,
day_dir) # check day metadata is correct dimension
print(number_wells_per_plate(day_metadata, day_dir))
return day_metadata
Created on Fri Nov 15 15:50:39 2019
@author: em812
"""
from tierpsytools.hydra.compile_metadata import merge_robot_metadata
from tierpsytools.hydra.compile_metadata import get_day_metadata
from tierpsytools import EXAMPLES_DIR
from pathlib import Path
if __name__=="__main__":
# Input
data_dir = Path(EXAMPLES_DIR) / 'hydra_metadata' / 'data'
day_root_dir = data_dir / 'AuxiliaryFiles' / 'day1'
sourceplate_file = day_root_dir / '20191107_sourceplates.csv'
manual_meta_file = day_root_dir / '20191108_manual_metadata.csv'
# Save to
#robot_metadata_file = day_root_dir.joinpath('20191107_robot_metadata.csv')
metadata_file = day_root_dir / '{}_day_metadata.csv'.format(
day_root_dir.stem)
# Run
robot_metadata = merge_robot_metadata(sourceplate_file, saveto=False)
day_metadata = get_day_metadata(
robot_metadata, manual_meta_file, saveto=metadata_file,
del_if_exists=True, include_imgstore_name = True,
raw_day_dir=data_dir / 'RawVideos' / 'day1')
sourceplate_files = [[file for file in d.glob('*_sourceplates.csv')]
for d in day_root_dirs]
manual_meta_files = [[file for file in d.glob('*_manual_metadata.csv')]
for d in day_root_dirs]
# Saveto
metadata_files = [
d.joinpath('{}_day_metadata.csv'.format(d.stem)) for d in day_root_dirs
]
# Run compilation of day metadata for all the days
for day, source, manual_meta, saveto in zip(day_root_dirs,
sourceplate_files,
manual_meta_files,
metadata_files):
if len(source) != 1:
print('There is not a unique sourceplates file ' +
'in day {}. Metadata cannot be compiled'.format(day))
continue
if len(manual_meta) != 1:
print('There is not a unique manual_metadata file ' +
'in day {}. Metadata cannot be compiled'.format(day))
continue
robot_metadata = merge_robot_metadata(source[0], saveto=False)
day_metadata = get_day_metadata(robot_metadata,
manual_meta[0],
saveto=saveto)
# Conactenate all metadata
concatenate_days_metadata(aux_root_dir)