def distribute_seg_dataset(
test=False,
csv_loc="../input_segs_and_tiffs/raw_seg_013_014_images.csv",
col_name_map={
"fov_path": "original_fov_location",
"FOVId": "fov_id",
"seg_file_name": "2D_fov_tiff_path",
},
dataset_name="2d_segmented_fields",
package_owner="rorydm",
s3_bucket="s3://allencell-internal-quilt",
):
# read in original csv
df = pd.read_csv(csv_loc)
# rename some cols
df = df.rename(col_name_map, axis="columns")
# drop any cols with missing data
vds = validate(df, drop_on_error=True)
df = vds.data.reset_index(drop=True)
# subsample df for eg a test dataset
if test:
df = df.sample(2, random_state=0)
dataset_name = f"{dataset_name}_test"
# create the dataset
ds = Dataset(
dataset=df,
name=dataset_name,
package_owner=package_owner,
readme_path="README.md",
)
# structure scores as auxilary file
score_files = [
Path(f"../structure_scores/structure_score_55000000{p}.csv")
for p in (13, 14)
]
score_dfs = [
pd.read_csv(f).rename({"mh Score": "mh score"}, axis="columns")
for f in score_files
]
df_score = pd.concat(score_dfs, axis="rows", ignore_index=True, sort=False)
df_score.to_csv(Path("../structure_scores/structure_scores.csv"))
# set data path cols, metadata cols, and extra files
ds.set_metadata_columns(["fov_id", "original_fov_location"])
ds.set_path_columns(["2D_fov_tiff_path"])
ds.set_extra_files(
["../channel_defs.json", "../structure_scores/structure_scores.csv"])
# tag with commit hash
label = (subprocess.check_output(["git", "rev-parse",
"HEAD"]).strip().decode("utf-8"))
ds.distribute(s3_bucket,
message=f"git commit hash of fish_morphology_code = {label}")
def distribute_autocontrasted_dataset(
test=False,
csv_loc="/allen/aics/gene-editing/FISH/2019/chaos/data/normalized_2D_tiffs/5500000075_B3/image_manifest_final.csv",
col_name_map={},
dataset_name="2d_autocontrasted_fields_and_single_cells_actn2",
package_owner="rorydm",
s3_bucket="s3://allencell-internal-quilt",
):
# read in original csv
df = pd.read_csv(csv_loc)
# rename some cols
df = df.rename(col_name_map, axis="columns")
df = df.drop(["2D_fov_tiff_path"], axis="columns").rename(col_name_map,
axis="columns")
# drop any cols with missing data
vds = validate(df, drop_on_error=True)
df = vds.data.reset_index(drop=True)
# subsample df for eg a test dataset
if test:
df = df.sample(2, random_state=0)
dataset_name = f"{dataset_name}_test"
# create the dataset
ds = Dataset(
dataset=df,
name=dataset_name,
package_owner=package_owner,
readme_path=
"/allen/aics/gene-editing/FISH/2019/chaos/data/normalized_2D_tiffs/5500000075_B3/README.md",
)
# set data path cols, metadata cols, and extra files
ds.set_metadata_columns(["fov_id", "original_fov_location"])
ds.set_path_columns(
["rescaled_2D_fov_tiff_path", "rescaled_2D_single_cell_tiff_path"])
ds.set_extra_files([
"/allen/aics/gene-editing/FISH/2019/chaos/data/normalized_2D_tiffs/5500000075_B3/channel_defs.json",
"/allen/aics/gene-editing/FISH/2019/chaos/data/normalized_2D_tiffs/5500000075_B3/parameters.json",
])
# tag with commit hash
label = (subprocess.check_output(["git", "rev-parse",
"HEAD"]).strip().decode("utf-8"))
ds.distribute(s3_bucket,
message=f"git commit hash of fish_morphology_code = {label}")
def distribute_nonstructure_dataset(
test=False,
csv_loc="nonstructure_fov_manifest_for_quilt.csv",
col_name_map={
"FOVId": "fov_id",
"fov_path": "original_fov_location"
},
dataset_name="2d_nonstructure_fields",
package_owner="tanyasg",
s3_bucket="s3://allencell-internal-quilt",
):
# read in original csv
df = pd.read_csv(csv_loc)
# rename some cols
df = df.rename(col_name_map, axis="columns")
# drop any cols with missing data
vds = validate(df, drop_on_error=True)
df = vds.data.reset_index(drop=True)
# subsample df for eg a test dataset
if test:
df = df.sample(2, random_state=0)
dataset_name = f"{dataset_name}_test"
# create the dataset
ds = Dataset(
dataset=df,
name=dataset_name,
package_owner=package_owner,
readme_path="README.md",
)
# set data path cols, metadata cols, and extra files
ds.set_metadata_columns(["fov_id", "original_fov_location"])
ds.set_path_columns(["merged_2D_fov_tiff_path"])
ds.set_extra_files(["channel_defs.json"])
# tag with commit hash
label = (subprocess.check_output(["git", "rev-parse",
"HEAD"]).strip().decode("utf-8"))
ds.distribute(s3_bucket,
message=f"git commit hash of fish_morphology_code = {label}")
def distribute_nuclear_masks(
test=False,
csv_loc=Path(
"/allen/aics/microscopy/Calysta/test/fish_struc_seg/sarc_classification_for_Rory.csv"
),
dataset_name="2d_nuclear_masks",
package_owner="calystay",
s3_bucket="s3://allencell-internal-quilt",
readme_path="README.md",
):
# read in original csv
df_in = pd.read_csv(csv_loc)
# extract original_fov_location and nuc_mask_path from dataframe
df = df_in[["original_fov_location", "nuc_mask_path"]]
df = df.drop_duplicates()
# drop any cols with missing data
vds = validate(df, drop_on_error=True)
df = vds.data.reset_index(drop=True)
# subsample df for eg a test dataset
if test:
df = df.sample(2, random_state=0)
dataset_name = f"{dataset_name}_test"
# create the dataset
ds = Dataset(
dataset=df,
name=dataset_name,
package_owner=package_owner,
readme_path=readme_path,
)
# set data path cols, metadata cols, and extra files
ds.set_metadata_columns(["fov_id", "original_fov_location"])
ds.set_path_columns(["nuclear_mask_path"])
# tag with commit hash
label = (subprocess.check_output(["git", "rev-parse",
"HEAD"]).strip().decode("utf-8"))
ds.distribute(s3_bucket,
message=f"git commit hash of fish_morphology_code = {label}")
def test_validate(data, drop_on_error, expected_len):
results = validate(data, drop_on_error=drop_on_error)
if drop_on_error:
assert len(results.data) == expected_len
"FeatureExplorerURL": fe_link
})
# Create dataframe from rows
cell_id_to_fov_id_fe_link = pd.DataFrame(cell_id_to_fov_id_fe_link)
# Merge the dataframes into one
raw = raw.merge(cell_id_to_fov_id_fe_link,
left_on="CellId",
right_on="CellId",
suffixes=("_raw", "_fe_link"))
# Step 3:
# Validate and prune the raw data
# We shouldn't lose any rows here but we are doing this as a safety measure
cleaned = validate(raw, drop_on_error=True)
print(f"Dropped {len(raw) - len(cleaned.data)} rows during validation.")
# Step 4:
# Send to dataset object for package construction
ds = Dataset(cleaned.data, "Pipeline Integrated Cell", "aics", "readme.md")
# Step 5:
# Add a license
ds.add_license("https://www.allencell.org/terms-of-use.html")
# Indicate column values to use for file metadata
ds.set_metadata_columns([
"CellId", "CellIndex", "CellLine", "NucMembSegmentationAlgorithm",
"NucMembSegmentationAlgorithmVersion", "FOVId", "Gene", "PlateId",
"WellId", "ProteinDisplayName", "StructureDisplayName", "Workflow",