def test_dataframes(): # setup test data wipe_db_tables() # WARNING: kills existing database data labelled_rows = raw_to_labelled_rows(raw_rows=doc_as_iterable(INVESTMENT_DOC), spec_dicts=(REF_HEADER_DICT, REF_UNIT_DICT)) assert labelled_rows == INVESTMENT_PARSED flat_rows = list(stream_flat_data(labelled_rows)) assert flat_rows == INVESTMENT_FLAT_ROW stream_to_database(flat_rows) # check user-end output check_final_dataframes()
def test_dataframes(self): # setup test data wipe_db_tables() # WARNING: kills existing database data labelled_rows = raw_to_labelled_rows(raw_rows=self.doc_as_iterable(self.INVESTMENT_DOC), spec_dicts=(self.REF_HEADER_DICT, self.REF_UNIT_DICT)) self.assertEqual(labelled_rows, self.INVESTMENT_PARSED) flat_rows = list(stream_flat_data(labelled_rows)) self.assertEqual(flat_rows, self.INVESTMENT_FLAT_ROW) stream_to_database(flat_rows) # check user-end output self.check_final_dataframes()
def test_dataframes(self): # setup test data wipe_db_tables() # WARNING: kills existing database data labelled_rows = raw_to_labelled_rows( raw_rows=self.doc_as_iterable(self.INVESTMENT_DOC), spec_dicts=(self.REF_HEADER_DICT, self.REF_UNIT_DICT)) self.assertEqual(labelled_rows, self.INVESTMENT_PARSED) flat_rows = list(stream_flat_data(labelled_rows)) self.assertEqual(flat_rows, self.INVESTMENT_FLAT_ROW) stream_to_database(flat_rows) # check user-end output self.check_final_dataframes()
def test_dataframes(): # setup test data wipe_db_tables() # WARNING: kills existing database data labelled_rows = raw_to_labelled_rows( raw_rows=doc_as_iterable(INVESTMENT_DOC), spec_dicts=(REF_HEADER_DICT, REF_UNIT_DICT)) assert labelled_rows == INVESTMENT_PARSED flat_rows = list(stream_flat_data(labelled_rows)) assert flat_rows == INVESTMENT_FLAT_ROW stream_to_database(flat_rows) # check user-end output check_final_dataframes()
def test_database(labelled_rows): wipe_db_tables() gen = list(stream_flat_data(labelled_rows)) stream_to_database(gen) dfa, dfq, dfm = get_reshaped_dfs() assert dfa.loc[2014, 'I_yoy'] == 97.3
def import_csv(data_folder, kill_existing_data=True): csv, spec, cfg = get_filenames(data_folder) if kill_existing_data: wipe_db_tables() to_database(csv, spec, cfg)
def import_csv(data_folder, kill_existing_data = True): csv, spec, cfg = get_filenames(data_folder) if kill_existing_data: wipe_db_tables() to_database(csv, spec, cfg)