def test_table_grouped_tail_real_data(self): with temporary_directory() as workdir: for table_path in (SRC / "test" / "data").glob("*.csv"): table = read_table(table_path, schema=SCHEMA) test_output = workdir / f"latest_{table_path.name}" pandas_output = workdir / f"latest_pandas_{table_path.name}" # Create the latest slice of the given table table_grouped_tail(table_path, test_output, ["key"]) # Create a latest slice using pandas grouping table = table.groupby("key").aggregate( agg_last_not_null).reset_index() export_csv(table, path=pandas_output, schema=SCHEMA) # Converting to a CSV in memory sometimes produces out-of-order values with open_file_like(test_output) as fd1, open_file_like( pandas_output) as fd2: test_result_lines = list(sorted(fd1)) pandas_result_lines = list(sorted(fd2)) self.assertEqual(len(test_result_lines), len(pandas_result_lines)) for line1, line2 in zip(test_result_lines, pandas_result_lines): self.assertEqual(line1, line2)
def test_open_file_like_handle(self): with temporary_directory() as workdir: temp_file_path = workdir / "temp.txt" fd1 = open(temp_file_path, "w") fd1.write("hello") with open_file_like(fd1, "w") as fd2: fd2.seek(0, SEEK_END) fd2.write(" ") with open_file_like(fd1, "w") as fd2: fd2.seek(0, SEEK_END) fd2.write("world") fd1.close() self._assert_file_contents_equal(temp_file_path, "hello world")
def test_open_file_like_file(self): with temporary_directory() as workdir: temp_file_path = workdir / "temp.txt" with open_file_like(temp_file_path, "w") as fd: fd.write("hello world") self._assert_file_contents_equal(temp_file_path, "hello world")
def _compare_tables_equal(test_case: ProfiledTestCase, table1: Path, table2: Path) -> None: cols1 = get_table_columns(table1) cols2 = get_table_columns(table2) test_case.assertEqual(set(cols1), set(cols2)) # Converting to a CSV in memory sometimes produces out-of-order values with open_file_like(table1) as fd1, open_file_like(table2) as fd2: records1 = list(line_reader(fd1, skip_empty=True)) records2 = list(line_reader(fd2, skip_empty=True)) test_case.assertEqual(len(records1), len(records2)) reader1 = csv.reader(records1) reader2 = csv.reader(records2) for record1, record2 in zip(reader1, reader2): record1 = {col: val for col, val in zip(cols1, record1)} record2 = {col: val for col, val in zip(cols2, record2)} test_case.assertEqual(record1, record2)
def parse(self, sources: Dict[str, str], aux: Dict[str, DataFrame], **parse_opts) -> DataFrame: dataframes = {} for name, fname in sources.items(): with open_file_like(fname, "r") as fd: data = json.load(fd)["integrated_county_timeseries_external_data"] dataframes[name] = DataFrame.from_records(data) return self.parse_dataframes(dataframes, aux=aux, **parse_opts)