def test_flat_emitter(): lab_rows = [['I', 'bln_rub', '2014', '13527,7', '1863,8', '2942,0', '3447,6', '5274,3', '492,2', '643,2', '728,4', '770,4', '991,1', '1180,5', '1075,1', '1168,5', '1204,0', '1468,5', '1372,5', '2433,3'] , ['PROD_TRANS', 'rog', '2015', '31,1', '126,3', '139,8', '83,8', '94,6', '115,8', '', '', '', '', '', '']] SAFE_NONE = -1 flat_db_rows = [('a', 'I_bln_rub', 2014, SAFE_NONE, SAFE_NONE, 13527.7), ('q', 'I_bln_rub', 2014, 1, SAFE_NONE, 1863.8), ('q', 'I_bln_rub', 2014, 2, SAFE_NONE, 2942.0), ('q', 'I_bln_rub', 2014, 3, SAFE_NONE, 3447.6), ('q', 'I_bln_rub', 2014, 4, SAFE_NONE, 5274.3), ('m', 'I_bln_rub', 2014, SAFE_NONE, 1, 492.2), ('m', 'I_bln_rub', 2014, SAFE_NONE, 2, 643.2), ('m', 'I_bln_rub', 2014, SAFE_NONE, 3, 728.4), ('m', 'I_bln_rub', 2014, SAFE_NONE, 4, 770.4), ('m', 'I_bln_rub', 2014, SAFE_NONE, 5, 991.1), ('m', 'I_bln_rub', 2014, SAFE_NONE, 6, 1180.5), ('m', 'I_bln_rub', 2014, SAFE_NONE, 7, 1075.1), ('m', 'I_bln_rub', 2014, SAFE_NONE, 8, 1168.5), ('m', 'I_bln_rub', 2014, SAFE_NONE, 9, 1204.0), ('m', 'I_bln_rub', 2014, SAFE_NONE, 10, 1468.5), ('m', 'I_bln_rub', 2014, SAFE_NONE, 11, 1372.5), ('m', 'I_bln_rub', 2014, SAFE_NONE, 12, 2433.3), ('m', 'PROD_TRANS_rog', 2015, SAFE_NONE, 1, 31.1), ('m', 'PROD_TRANS_rog', 2015, SAFE_NONE, 2, 126.3), ('m', 'PROD_TRANS_rog', 2015, SAFE_NONE, 3, 139.8), ('m', 'PROD_TRANS_rog', 2015, SAFE_NONE, 4, 83.8), ('m', 'PROD_TRANS_rog', 2015, SAFE_NONE, 5, 94.6), ('m', 'PROD_TRANS_rog', 2015, SAFE_NONE, 6, 115.8)] assert list(stream_flat_data(lab_rows)) == flat_db_rows
def test_database(self): gen = list( stream_flat_data( get_labelled_rows(raw_data_file(), spec_file(), cfg_file()))) stream_to_database(gen) dfa, dfq, dfm = get_reshaped_dfs() self.assertEqual(dfa.loc[2014, 'I_yoy'], 97.3)
def test_flat_emitter(self): lab_rows = [['I', 'bln_rub', '2014', '13527,7', '1863,8', '2942,0', '3447,6', '5274,3', '492,2', '643,2', '728,4', '770,4', '991,1', '1180,5', '1075,1', '1168,5', '1204,0', '1468,5', '1372,5', '2433,3'] , ['PROD_TRANS', 'rog', '2015', '31,1', '126,3', '139,8', '83,8', '94,6', '115,8', '', '', '', '', '', '']] SAFE_NONE = -1 flat_db_rows = [('a', 'I_bln_rub', 2014, SAFE_NONE, SAFE_NONE, 13527.7), ('q', 'I_bln_rub', 2014, 1, SAFE_NONE, 1863.8), ('q', 'I_bln_rub', 2014, 2, SAFE_NONE, 2942.0), ('q', 'I_bln_rub', 2014, 3, SAFE_NONE, 3447.6), ('q', 'I_bln_rub', 2014, 4, SAFE_NONE, 5274.3), ('m', 'I_bln_rub', 2014, SAFE_NONE, 1, 492.2), ('m', 'I_bln_rub', 2014, SAFE_NONE, 2, 643.2), ('m', 'I_bln_rub', 2014, SAFE_NONE, 3, 728.4), ('m', 'I_bln_rub', 2014, SAFE_NONE, 4, 770.4), ('m', 'I_bln_rub', 2014, SAFE_NONE, 5, 991.1), ('m', 'I_bln_rub', 2014, SAFE_NONE, 6, 1180.5), ('m', 'I_bln_rub', 2014, SAFE_NONE, 7, 1075.1), ('m', 'I_bln_rub', 2014, SAFE_NONE, 8, 1168.5), ('m', 'I_bln_rub', 2014, SAFE_NONE, 9, 1204.0), ('m', 'I_bln_rub', 2014, SAFE_NONE, 10, 1468.5), ('m', 'I_bln_rub', 2014, SAFE_NONE, 11, 1372.5), ('m', 'I_bln_rub', 2014, SAFE_NONE, 12, 2433.3), ('m', 'PROD_TRANS_rog', 2015, SAFE_NONE, 1, 31.1), ('m', 'PROD_TRANS_rog', 2015, SAFE_NONE, 2, 126.3), ('m', 'PROD_TRANS_rog', 2015, SAFE_NONE, 3, 139.8), ('m', 'PROD_TRANS_rog', 2015, SAFE_NONE, 4, 83.8), ('m', 'PROD_TRANS_rog', 2015, SAFE_NONE, 5, 94.6), ('m', 'PROD_TRANS_rog', 2015, SAFE_NONE, 6, 115.8)] self.assertEqual(list(stream_flat_data(lab_rows)), flat_db_rows)
def test_dataframes(): # setup test data wipe_db_tables() # WARNING: kills existing database data labelled_rows = raw_to_labelled_rows(raw_rows=doc_as_iterable(INVESTMENT_DOC), spec_dicts=(REF_HEADER_DICT, REF_UNIT_DICT)) assert labelled_rows == INVESTMENT_PARSED flat_rows = list(stream_flat_data(labelled_rows)) assert flat_rows == INVESTMENT_FLAT_ROW stream_to_database(flat_rows) # check user-end output check_final_dataframes()
def test_dataframes(self): # setup test data wipe_db_tables() # WARNING: kills existing database data labelled_rows = raw_to_labelled_rows(raw_rows=self.doc_as_iterable(self.INVESTMENT_DOC), spec_dicts=(self.REF_HEADER_DICT, self.REF_UNIT_DICT)) self.assertEqual(labelled_rows, self.INVESTMENT_PARSED) flat_rows = list(stream_flat_data(labelled_rows)) self.assertEqual(flat_rows, self.INVESTMENT_FLAT_ROW) stream_to_database(flat_rows) # check user-end output self.check_final_dataframes()
def test_dataframes(self): # setup test data wipe_db_tables() # WARNING: kills existing database data labelled_rows = raw_to_labelled_rows( raw_rows=self.doc_as_iterable(self.INVESTMENT_DOC), spec_dicts=(self.REF_HEADER_DICT, self.REF_UNIT_DICT)) self.assertEqual(labelled_rows, self.INVESTMENT_PARSED) flat_rows = list(stream_flat_data(labelled_rows)) self.assertEqual(flat_rows, self.INVESTMENT_FLAT_ROW) stream_to_database(flat_rows) # check user-end output self.check_final_dataframes()
def test_dataframes(): # setup test data wipe_db_tables() # WARNING: kills existing database data labelled_rows = raw_to_labelled_rows( raw_rows=doc_as_iterable(INVESTMENT_DOC), spec_dicts=(REF_HEADER_DICT, REF_UNIT_DICT)) assert labelled_rows == INVESTMENT_PARSED flat_rows = list(stream_flat_data(labelled_rows)) assert flat_rows == INVESTMENT_FLAT_ROW stream_to_database(flat_rows) # check user-end output check_final_dataframes()
def test_database(labelled_rows): wipe_db_tables() gen = list(stream_flat_data(labelled_rows)) stream_to_database(gen) dfa, dfq, dfm = get_reshaped_dfs() assert dfa.loc[2014, 'I_yoy'] == 97.3
def to_database(raw_data_file, spec_file, cfg_file=None): lab_rows = get_labelled_rows(raw_data_file, spec_file, cfg_file) db_rows = stream_flat_data(lab_rows) stream_to_database(db_rows)
def to_database(raw_data_file, spec_file, cfg_file = None): lab_rows = get_labelled_rows(raw_data_file, spec_file, cfg_file) db_rows = stream_flat_data(lab_rows) stream_to_database(db_rows)
def test_database(self): gen = list(stream_flat_data(get_labelled_rows(raw_data_file(), spec_file(), cfg_file()))) stream_to_database(gen) dfa, dfq, dfm = get_reshaped_dfs() self.assertEqual(dfa.loc[2014, 'I_yoy'], 97.3)