def test_update_df_col_absent(self, helpers):
synapse_manifest = helpers.get_data_frame("mock_manifests", "synapse_manifest.csv")
local_manifest = helpers.get_data_frame("mock_manifests", "local_manifest.csv")
with pytest.raises(AssertionError):
df_utils.update_df(local_manifest, synapse_manifest, "Col_Not_In_Dfs")
def test_update_dataframe(self):
input_df = pd.DataFrame(
{
"numCol": [1, 2],
"entityId": ["syn01", "syn02"],
"strCol": ["foo", "bar"],
},
columns=["numCol", "entityId", "strCol"],
)
updates_df = pd.DataFrame(
{
"strCol": ["___", np.nan],
"numCol": [np.nan, 4],
"entityId": ["syn01", "syn02"],
},
columns=["strCol", "numCol", "entityId"],
)
expected_df = pd.DataFrame(
{
"numCol": [int(1), int(4)],
"entityId": ["syn01", "syn02"],
"strCol": ["___", "bar"],
},
columns=["numCol", "entityId", "strCol"],
)
actual_df = df_utils.update_df(input_df, updates_df, "entityId")
pd.testing.assert_frame_equal(expected_df, actual_df)
def test_update_df_col_present(self, helpers):
synapse_manifest = helpers.get_data_frame("mock_manifests", "synapse_manifest.csv")
local_manifest = helpers.get_data_frame("mock_manifests", "local_manifest.csv")
col_pres_res = df_utils.update_df(local_manifest, synapse_manifest, "entityId")
assert_frame_equal(col_pres_res, synapse_manifest)
def get_manifest_with_annotations(
self,
annotations: pd.DataFrame) -> Tuple[ps.Spreadsheet, pd.DataFrame]:
"""Generate manifest, optionally with annotations (if requested).
Args:
annotations (pd.DataFrame): Annotations table (can be empty).
Returns:
Tuple[ps.Spreadsheet, pd.DataFrame]: Both the Google Sheet
URL and the corresponding data frame is returned.
"""
# Map annotation labels to display names to match manifest columns
annotations = self.map_annotation_names_to_display_names(annotations)
# Convert annotations table into dictionary, but maintain order
annotations_dict_raw = annotations.to_dict(into=OrderedDict)
annotations_dict = OrderedDict(
(k, list(v.values())) for k, v in annotations_dict_raw.items())
# Needs to happen before get_empty_manifest() gets called
self.additional_metadata = annotations_dict
# Generate empty manifest using `additional_metadata`
manifest_url = self.get_empty_manifest()
manifest_df = self.get_dataframe_by_url(manifest_url)
# Annotations clashing with manifest attributes are skipped
# during empty manifest generation. For more info, search
# for `additional_metadata` in `self.get_empty_manifest`.
# Hence, the shared columns need to be updated separately.
if self.is_file_based and self.use_annotations:
# This approach assumes that `update_df` returns
# a data frame whose columns are in the same order
manifest_df = update_df(manifest_df, annotations)
manifest_sh = self.set_dataframe_by_url(manifest_url, manifest_df)
manifest_url = manifest_sh.url
return manifest_url, manifest_df