Пример #1
0
    def test_date_parser(self):
        result = date_parser('2010_01').strftime('%Y_%m')
        expected = '2010_01'
        assert result == expected

        result = date_parser('2010-01').strftime('%Y_%m')
        assert result == expected

        result = date_parser('/m2010_01').strftime('%Y_%m')
        assert result == expected

        result = date_parser('m2010_01').strftime('%Y_%m')
        assert result == expected
Пример #2
0
    def test_date_parser(self):
        result = date_parser('2010_01').strftime('%Y_%m')
        expected = '2010_01'
        assert result == expected

        result = date_parser('2010-01').strftime('%Y_%m')
        assert result == expected

        result = date_parser('/m2010_01').strftime('%Y_%m')
        assert result == expected

        result = date_parser('m2010_01').strftime('%Y_%m')
        assert result == expected
Пример #3
0
 def _sanitize_key(self, key):
     if self.kind == 'long':
         return key
     else:
         return self.pre + date_parser(key).strftime('%Y_%m')
Пример #4
0
def make_to_long(panel_h, settings, start=None, stop=None):
    """
    Let's chunk by quarters.
    """

    # need compensation for real wage
    with open('../panel_construction/settings.txt', 'rt') as f:
        settings = json.load(f)

    analyzed = pd.HDFStore(settings['analyzed_path'])
    comp = analyzed.select('bls_productivity_compensation')['compensation']
    prod = analyzed.select('bls_productivity_compensation')['productivity']

    keys = sorted(panel_h.stores.keys())

    m0 = start or keys[0]
    m0 = date_parser(m0)

    mn = stop or keys[-1]
    mn = date_parser(mn)

    months = [
        x.strftime('%Y_%m') for x in arrow.Arrow.range('month', m0, mn)
        if x.strftime('m%Y_%m') in keys
    ]

    # Getting some memory pressure. break into chunks, write each out.
    # read proccessed chucnks.
    # Chunking by quarter

    month_chunks = chunk_quarters(months, 3)
    month_chunks = [x for x in month_chunks if len(x) > 0]
    p = pathlib.Path(str(settings['base_path']))
    out_store = HDFHandler(str(p),
                           kind='long',
                           months=month_chunks,
                           frequency='Q')
    earn_store = HDFHandler(str(p),
                            kind='earn',
                            months=month_chunks,
                            frequency='Q')

    for chunk in month_chunks:
        # need the three month chunks... maybe zip up with out_stoure.
        # may need another dict.
        df = read_to_long(panel_h, chunk)
        name = make_chunk_name(chunk)

        # out_store.write(df, name, format='table', append=False)
        s = out_store.stores[name]

        # add in real hourly wage
        c = comp.reindex(df.index, level='stamp').fillna(method='ffill') / 100

        # adjust weight decimals
        df.loc[:, 'og_weight'] = df['og_weight'] / 10000

        # CPS reports earnings in cents
        df.loc[:, 'earnings'] = df['earnings'] / 100

        df['real_hr_earns'] = (df['earnings'] / df['hours']) / c
        df['real_hr_earns'] = df['real_hr_earns'].replace(np.inf,
                                                          np.nan)  # div by 0

        df = replace_categorical(df, kind='flow', inverse=True)
        with pd.get_store(s.filename) as store:
            df.to_hdf(store, name, format='table', append=False)

        #----------------------------------------------------------------
        # Also write out just earnings (nan issues so can't select later)
        # need to make real hrs fisrt.
        earn = df[~pd.isnull(df.real_hr_earns)]
        earn = earn[(earn.hours > 0) & (earn.earnings > 0)]

        s = earn_store.stores[name]
        with pd.get_store(s.filename) as store:
            earn.to_hdf(store,
                        name,
                        format='table',
                        append=False,
                        data_columns=True)
        print("Finished " + str(chunk))

    # finally, chunk by quarter and write out.
    df = earn_store.select_all().drop(['occupation', 'actual_hours'], axis=1)
    df = df.dropna(how='any', subset=['edu', 'age', 'flow', 'expr'])

    df = quarterize(df)

    df['productivity'] = prod.reindex(df.index, level='qmonth')
    df['real_hr_earns'] = df.real_hr_earns.replace(np.inf, np.nan)
    # df = add_demo_dummies(df)
    # model, res = construct_wage_index(df)
    # df.loc[:, 'wage_index_res'] = res.resid

    cln_path = '/Volumes/HDD/Users/tom/DataStorage/CPS/analyzed/clean.h5'

    with pd.get_store(cln_path) as store:
        df.to_hdf(store, 'cleaned', format='f', append=False)

    out_store.close()
    analyzed.close()
    earn_store.close()
Пример #5
0
 def _sanitize_key(self, key):
     if self.kind == 'long':
         return key
     else:
         return self.pre + date_parser(key).strftime('%Y_%m')
Пример #6
0
def make_to_long(panel_h, settings, start=None, stop=None):
    """
    Let's chunk by quarters.
    """

    # need compensation for real wage
    with open('../panel_construction/settings.txt', 'rt') as f:
        settings = json.load(f)

    analyzed = pd.HDFStore(settings['analyzed_path'])
    comp = analyzed.select('bls_productivity_compensation')['compensation']
    prod = analyzed.select('bls_productivity_compensation')['productivity']

    keys = sorted(panel_h.stores.keys())

    m0 = start or keys[0]
    m0 = date_parser(m0)

    mn = stop or keys[-1]
    mn = date_parser(mn)

    months = [x.strftime('%Y_%m') for x in arrow.Arrow.range('month', m0, mn)
              if x.strftime('m%Y_%m') in keys]

    # Getting some memory pressure. break into chunks, write each out.
    # read proccessed chucnks.
    # Chunking by quarter

    month_chunks = chunk_quarters(months, 3)
    month_chunks = [x for x in month_chunks if len(x) > 0]
    p = pathlib.Path(str(settings['base_path']))
    out_store = HDFHandler(str(p), kind='long', months=month_chunks,
                           frequency='Q')
    earn_store = HDFHandler(str(p), kind='earn', months=month_chunks,
                            frequency='Q')

    for chunk in month_chunks:
        # need the three month chunks... maybe zip up with out_stoure.
        # may need another dict.
        df = read_to_long(panel_h, chunk)
        name = make_chunk_name(chunk)

        # out_store.write(df, name, format='table', append=False)
        s = out_store.stores[name]

        # add in real hourly wage
        c = comp.reindex(df.index, level='stamp').fillna(method='ffill') / 100

        # adjust weight decimals
        df.loc[:, 'og_weight'] = df['og_weight'] / 10000

        # CPS reports earnings in cents
        df.loc[:, 'earnings'] = df['earnings'] / 100

        df['real_hr_earns'] = (df['earnings'] / df['hours']) / c
        df['real_hr_earns'] = df['real_hr_earns'].replace(np.inf, np.nan)  # div by 0

        df = replace_categorical(df, kind='flow', inverse=True)
        with pd.get_store(s.filename) as store:
            df.to_hdf(store, name, format='table', append=False)

        #----------------------------------------------------------------
        # Also write out just earnings (nan issues so can't select later)
        # need to make real hrs fisrt.
        earn = df[~pd.isnull(df.real_hr_earns)]
        earn = earn[(earn.hours > 0) & (earn.earnings > 0)]

        s = earn_store.stores[name]
        with pd.get_store(s.filename) as store:
            earn.to_hdf(store, name, format='table', append=False,
                        data_columns=True)
        print("Finished " + str(chunk))

    # finally, chunk by quarter and write out.
    df = earn_store.select_all().drop(['occupation', 'actual_hours'], axis=1)
    df = df.dropna(how='any', subset=['edu', 'age', 'flow', 'expr'])

    df = quarterize(df)

    df['productivity'] = prod.reindex(df.index, level='qmonth')
    df['real_hr_earns'] = df.real_hr_earns.replace(np.inf, np.nan)
    # df = add_demo_dummies(df)
    # model, res = construct_wage_index(df)
    # df.loc[:, 'wage_index_res'] = res.resid

    cln_path = '/Volumes/HDD/Users/tom/DataStorage/CPS/analyzed/clean.h5'

    with pd.get_store(cln_path) as store:
        df.to_hdf(store, 'cleaned', format='f', append=False)

    out_store.close()
    analyzed.close()
    earn_store.close()