Exemplos de stream_flat_data em Python

Exemplo n.º 1

Arquivo: test_stream.py Projeto: Oriundi/rosstat-kep-data

def test_flat_emitter():
    lab_rows = [['I', 'bln_rub', '2014', '13527,7', '1863,8', '2942,0', '3447,6', '5274,3', '492,2', '643,2', '728,4', '770,4', '991,1', '1180,5', '1075,1', '1168,5', '1204,0', '1468,5', '1372,5', '2433,3']   
              , ['PROD_TRANS', 'rog', '2015', '31,1', '126,3', '139,8', '83,8', '94,6', '115,8', '', '', '', '', '', '']]

    SAFE_NONE = -1
     
    flat_db_rows = [('a', 'I_bln_rub', 2014, SAFE_NONE, SAFE_NONE, 13527.7),
                    ('q', 'I_bln_rub', 2014, 1, SAFE_NONE, 1863.8),
                    ('q', 'I_bln_rub', 2014, 2, SAFE_NONE, 2942.0),
                    ('q', 'I_bln_rub', 2014, 3, SAFE_NONE, 3447.6),
                    ('q', 'I_bln_rub', 2014, 4, SAFE_NONE, 5274.3),
                    ('m', 'I_bln_rub', 2014, SAFE_NONE, 1, 492.2),
                    ('m', 'I_bln_rub', 2014, SAFE_NONE, 2, 643.2),
                    ('m', 'I_bln_rub', 2014, SAFE_NONE, 3, 728.4),
                    ('m', 'I_bln_rub', 2014, SAFE_NONE, 4, 770.4),
                    ('m', 'I_bln_rub', 2014, SAFE_NONE, 5, 991.1),
                    ('m', 'I_bln_rub', 2014, SAFE_NONE, 6, 1180.5),
                    ('m', 'I_bln_rub', 2014, SAFE_NONE, 7, 1075.1),
                    ('m', 'I_bln_rub', 2014, SAFE_NONE, 8, 1168.5),
                    ('m', 'I_bln_rub', 2014, SAFE_NONE, 9, 1204.0),
                    ('m', 'I_bln_rub', 2014, SAFE_NONE, 10, 1468.5),
                    ('m', 'I_bln_rub', 2014, SAFE_NONE, 11, 1372.5),
                    ('m', 'I_bln_rub', 2014, SAFE_NONE, 12, 2433.3),
                    ('m', 'PROD_TRANS_rog', 2015, SAFE_NONE, 1, 31.1),
                    ('m', 'PROD_TRANS_rog', 2015, SAFE_NONE, 2, 126.3),
                    ('m', 'PROD_TRANS_rog', 2015, SAFE_NONE, 3, 139.8),
                    ('m', 'PROD_TRANS_rog', 2015, SAFE_NONE, 4, 83.8),
                    ('m', 'PROD_TRANS_rog', 2015, SAFE_NONE, 5, 94.6),
                    ('m', 'PROD_TRANS_rog', 2015, SAFE_NONE, 6, 115.8)]    
    
    assert list(stream_flat_data(lab_rows)) == flat_db_rows

Exemplo n.º 2

Arquivo: test_database.py Projeto: Oriundi/rosstat-kep-data

 def test_database(self):
     gen = list(
         stream_flat_data(
             get_labelled_rows(raw_data_file(), spec_file(), cfg_file())))
     stream_to_database(gen)
     dfa, dfq, dfm = get_reshaped_dfs()
     self.assertEqual(dfa.loc[2014, 'I_yoy'], 97.3)

Exemplo n.º 3

    def test_flat_emitter(self):
        lab_rows = [['I', 'bln_rub', '2014', '13527,7', '1863,8', '2942,0', '3447,6', '5274,3', '492,2', '643,2', '728,4', '770,4', '991,1', '1180,5', '1075,1', '1168,5', '1204,0', '1468,5', '1372,5', '2433,3']
                  , ['PROD_TRANS', 'rog', '2015', '31,1', '126,3', '139,8', '83,8', '94,6', '115,8', '', '', '', '', '', '']]

        SAFE_NONE = -1

        flat_db_rows = [('a', 'I_bln_rub', 2014, SAFE_NONE, SAFE_NONE, 13527.7),
                        ('q', 'I_bln_rub', 2014, 1, SAFE_NONE, 1863.8),
                        ('q', 'I_bln_rub', 2014, 2, SAFE_NONE, 2942.0),
                        ('q', 'I_bln_rub', 2014, 3, SAFE_NONE, 3447.6),
                        ('q', 'I_bln_rub', 2014, 4, SAFE_NONE, 5274.3),
                        ('m', 'I_bln_rub', 2014, SAFE_NONE, 1, 492.2),
                        ('m', 'I_bln_rub', 2014, SAFE_NONE, 2, 643.2),
                        ('m', 'I_bln_rub', 2014, SAFE_NONE, 3, 728.4),
                        ('m', 'I_bln_rub', 2014, SAFE_NONE, 4, 770.4),
                        ('m', 'I_bln_rub', 2014, SAFE_NONE, 5, 991.1),
                        ('m', 'I_bln_rub', 2014, SAFE_NONE, 6, 1180.5),
                        ('m', 'I_bln_rub', 2014, SAFE_NONE, 7, 1075.1),
                        ('m', 'I_bln_rub', 2014, SAFE_NONE, 8, 1168.5),
                        ('m', 'I_bln_rub', 2014, SAFE_NONE, 9, 1204.0),
                        ('m', 'I_bln_rub', 2014, SAFE_NONE, 10, 1468.5),
                        ('m', 'I_bln_rub', 2014, SAFE_NONE, 11, 1372.5),
                        ('m', 'I_bln_rub', 2014, SAFE_NONE, 12, 2433.3),
                        ('m', 'PROD_TRANS_rog', 2015, SAFE_NONE, 1, 31.1),
                        ('m', 'PROD_TRANS_rog', 2015, SAFE_NONE, 2, 126.3),
                        ('m', 'PROD_TRANS_rog', 2015, SAFE_NONE, 3, 139.8),
                        ('m', 'PROD_TRANS_rog', 2015, SAFE_NONE, 4, 83.8),
                        ('m', 'PROD_TRANS_rog', 2015, SAFE_NONE, 5, 94.6),
                        ('m', 'PROD_TRANS_rog', 2015, SAFE_NONE, 6, 115.8)]

        self.assertEqual(list(stream_flat_data(lab_rows)), flat_db_rows)

Exemplo n.º 4

Arquivo: test_mwe.py Projeto: Oriundi/rosstat-kep-data

def test_dataframes():
    # setup test data
    wipe_db_tables()  # WARNING: kills existing database data
    labelled_rows = raw_to_labelled_rows(raw_rows=doc_as_iterable(INVESTMENT_DOC), spec_dicts=(REF_HEADER_DICT, REF_UNIT_DICT))
    assert labelled_rows == INVESTMENT_PARSED
    flat_rows = list(stream_flat_data(labelled_rows))
    assert flat_rows == INVESTMENT_FLAT_ROW
    stream_to_database(flat_rows)
    # check user-end output
    check_final_dataframes()

Exemplo n.º 5

Arquivo: test_mwe.py Projeto: Oriundi/rosstat-kep-data

 def test_dataframes(self):
     # setup test data
     wipe_db_tables()  # WARNING: kills existing database data
     labelled_rows = raw_to_labelled_rows(raw_rows=self.doc_as_iterable(self.INVESTMENT_DOC), spec_dicts=(self.REF_HEADER_DICT, self.REF_UNIT_DICT))
     self.assertEqual(labelled_rows, self.INVESTMENT_PARSED)
     flat_rows = list(stream_flat_data(labelled_rows))
     self.assertEqual(flat_rows, self.INVESTMENT_FLAT_ROW)
     stream_to_database(flat_rows)
     # check user-end output
     self.check_final_dataframes()

Exemplo n.º 6

 def test_dataframes(self):
     # setup test data
     wipe_db_tables()  # WARNING: kills existing database data
     labelled_rows = raw_to_labelled_rows(
         raw_rows=self.doc_as_iterable(self.INVESTMENT_DOC),
         spec_dicts=(self.REF_HEADER_DICT, self.REF_UNIT_DICT))
     self.assertEqual(labelled_rows, self.INVESTMENT_PARSED)
     flat_rows = list(stream_flat_data(labelled_rows))
     self.assertEqual(flat_rows, self.INVESTMENT_FLAT_ROW)
     stream_to_database(flat_rows)
     # check user-end output
     self.check_final_dataframes()

Exemplo n.º 7

def test_dataframes():
    # setup test data
    wipe_db_tables()  # WARNING: kills existing database data
    labelled_rows = raw_to_labelled_rows(
        raw_rows=doc_as_iterable(INVESTMENT_DOC),
        spec_dicts=(REF_HEADER_DICT, REF_UNIT_DICT))
    assert labelled_rows == INVESTMENT_PARSED
    flat_rows = list(stream_flat_data(labelled_rows))
    assert flat_rows == INVESTMENT_FLAT_ROW
    stream_to_database(flat_rows)
    # check user-end output
    check_final_dataframes()

Exemplo n.º 8

Arquivo: test_database.py Projeto: Oriundi/rosstat-kep-data

def test_database(labelled_rows):
    wipe_db_tables()
    gen = list(stream_flat_data(labelled_rows))
    stream_to_database(gen)
    dfa, dfq, dfm = get_reshaped_dfs()
    assert dfa.loc[2014, 'I_yoy'] == 97.3

Exemplo n.º 9

def to_database(raw_data_file, spec_file, cfg_file=None):
    lab_rows = get_labelled_rows(raw_data_file, spec_file, cfg_file)
    db_rows = stream_flat_data(lab_rows)
    stream_to_database(db_rows)

Exemplo n.º 10

Arquivo: csv2db.py Projeto: Oriundi/rosstat-kep-data

def to_database(raw_data_file, spec_file, cfg_file = None):
    lab_rows = get_labelled_rows(raw_data_file, spec_file, cfg_file)
    db_rows = stream_flat_data(lab_rows)
    stream_to_database(db_rows)

Exemplo n.º 11

Arquivo: test_database.py Projeto: Oriundi/rosstat-kep-data

def test_database(labelled_rows):
    wipe_db_tables()
    gen = list(stream_flat_data(labelled_rows))
    stream_to_database(gen)
    dfa, dfq, dfm = get_reshaped_dfs()
    assert dfa.loc[2014, 'I_yoy'] == 97.3

Exemplo n.º 12

Arquivo: test_database.py Projeto: Oriundi/rosstat-kep-data

 def test_database(self):
     gen = list(stream_flat_data(get_labelled_rows(raw_data_file(), spec_file(), cfg_file())))
     stream_to_database(gen)
     dfa, dfq, dfm = get_reshaped_dfs()
     self.assertEqual(dfa.loc[2014, 'I_yoy'], 97.3)