def test_build_mismatched_common_meta_report(self): # rhd3 header needs to be removed meta1 = pd.DataFrame( [["r1_1", "r1_2", "r1_3"], ["r2_1", "r2_2", "r2_3"], ["r3_1", "r3_2", "r3_3"]], index=["r1", "r2", "r3"], columns=["rhd1", "rhd2", "rhd3"]) meta2 = pd.DataFrame( [["r1_1", "r1_2", "r1_3"], ["r2_1", "r2_2", "r2_3"], ["r3_1", "r3_2", "r3_33"]], index=["r1", "r2", "r3"], columns=["rhd1", "rhd2", "rhd3"]) meta3 = pd.DataFrame( [["r3_1", "r3_3", "r3_2"], ["r1_1", "r1_3", "r1_2"], ["r2_1", "r2_3", "r2_2"]], index=["r3", "r1", "r2"], columns=["rhd1", "rhd3", "rhd2"]) logger.debug("meta1:\n{}".format(meta1)) logger.debug("meta2:\n{}".format(meta2)) logger.debug("meta3:\n{}".format(meta3)) common_meta_dfs = [meta1, meta2, meta3] all_meta_df, all_meta_df_with_dups = cg.build_common_all_meta_df( common_meta_dfs, [], False) common_meta_df_shapes = [x.shape for x in common_meta_dfs] sources = ["my_src1", "my_src2", "my_src3"] self.assertFalse(all_meta_df.index.is_unique, "during setup expected the index to not be unique") r = cg.build_mismatched_common_meta_report(common_meta_df_shapes, sources, all_meta_df, all_meta_df_with_dups) logger.debug("r:\n{}".format(r)) self.assertEqual((3, 5), r.shape) self.assertIn("source_file", r.columns) self.assertIn("orig_rid", r.columns) self.assertTrue(set(meta1.columns) < set(r.columns)) self.assertEqual({"r3"}, set(r.orig_rid))
def test_build_common_all_meta_df(self): # rhd3 header needs to be removed meta1 = pd.DataFrame( [["r1_1", "r1_2", "r1_3"], ["r2_1", "r2_2", "r2_3"], ["r3_1", "r3_2", "r3_3"]], index=["r1", "r2", "r3"], columns=["rhd1", "rhd2", "rhd3"]) meta2 = pd.DataFrame( [["r1_1", "r1_2", "r1_3"], ["r2_1", "r2_2", "r2_3"], ["r3_1", "r3_2", "r3_33"]], index=["r1", "r2", "r3"], columns=["rhd1", "rhd2", "rhd3"]) e_meta1 = pd.DataFrame( [["r1_1", "r1_2"], ["r2_1", "r2_2"], ["r3_1", "r3_2"]], index=["r1", "r2", "r3"], columns=["rhd1", "rhd2"]) r_all, r_all_w_dups = cg.build_common_all_meta_df([meta1, meta2], ["rhd3"], False) logger.debug( "rhd3 header needs to be removed - r_all:\n{}".format(r_all)) logger.debug("r_all_w_dups:\n{}".format(r_all_w_dups)) self.assertEqual((3, 2), r_all.shape) self.assertEqual((6, 2), r_all_w_dups.shape) pd.util.testing.assert_frame_equal(e_meta1, r_all) #remove all metadata fields r_all, r_all_w_dups = cg.build_common_all_meta_df([meta1, meta2], [], True) logger.debug("remove all metadata fields - r_all\n{}".format(r_all)) logger.debug("r_all_w_dups:\n{}".format(r_all_w_dups)) self.assertEqual((3, 0), r_all.shape) self.assertTrue((e_meta1.index == r_all.index).all()) meta4 = pd.DataFrame( [["r1_1", "r1_22", "r1_5"], ["r4_1", "r4_22", "r4_5"], ["r3_1", "r3_22", "r3_5"]], index=["r1", "r4", "r3"], columns=["rhd1", "rhd2", "rhd5"]) e_meta3 = pd.DataFrame([["r1_1"], ["r2_1"], ["r3_1"], ["r4_1"]], index=["r1", "r2", "r3", "r4"], columns=["rhd1"]) logger.debug("meta4:\n{}".format(meta4)) logger.debug("e_meta3:\n{}".format(e_meta3)) # rhd5 not in meta4, so it should be dropped even without being # explicitly provided out_meta3, _ = cg.build_common_all_meta_df([meta1, meta4], ["rhd2"], False) logger.debug( """rhd5 not in meta4 so it should be automatically dropped without being explictly listed in fields_to_remove - out_meta3: {}""".format(out_meta3)) pd.util.testing.assert_frame_equal(out_meta3, e_meta3) # Empty metadata empty_meta = pd.DataFrame([], index=["a", "b", "c"]) logger.debug("empty metadata provided - empty_meta.empty: {}".format( empty_meta.empty)) out_meta4, _ = cg.build_common_all_meta_df([empty_meta, empty_meta], [], False) logger.debug( "empty metadata provided - out_meta4:\n{}".format(out_meta4)) pd.util.testing.assert_frame_equal(out_meta4, empty_meta) #metadata has duplicates but index is unique meta5 = pd.DataFrame({"rhd1": [0, 0, 1]}, index=range(3)) meta6 = pd.DataFrame({"rhd1": [0, 0, 1]}, index=range(3)) out_meta5, _ = cg.build_common_all_meta_df([meta5, meta6], [], False) logger.debug( "metadata has duplicates but index is unique - out_meta5:\n{}". format(out_meta5)) self.assertEqual( (3, 1), out_meta5.shape, "metadata contains duplicates but index is unique - should have been kept" )