def parse_format_assignments(txt): assignments = thread_last( txt.split(';'), filter(lambda x: x.strip().lower().startswith('format')), mapcat(lambda x: x.lower().split('.')), map(lambda x: x.split()), # break out vars and format (mapcat, lambda y: [(k, y[-1]) for k in y]), # tuple of var, fmt dict ) return assignments
def parse_questions(txt): rqt = re.compile(r'[\"\']') # match quote chars assignments = thread_last( txt.split(';'), filter(lambda x: x.strip().lower().startswith('label')), mapcat(lambda x: x.lower().split('\n')), map(lambda x: x.split('=')), # break out vars and format (map, lambda y: (y[0].strip().lower(), rqt.sub('', y[1].strip()))), # tuple of var, fmt dict ) return assignments
def varlabels2df(vlbls, yr=None): return thread_last( vlbls.items(), map(lambda k, v: pd.DataFrame({'code': list(v.keys()), 'label': list(v.values()), 'var': k})), map(lambda df: df.assign(year=yr) if yr else df), pd.concat, lambda df: (df.set_index(['var', 'year', 'code']) if yr else df.set_index(['var', 'code'])) )
def parse_variable_labels(txt, repl, lbls_to_lower=True): b2d = curry(block2dict)(repl=repl, to_lower=lbls_to_lower) labels = thread_last( txt.split(';'), filter(lambda x: x.strip().lower().startswith('value')), map(lambda x: x.strip().split('\n')), map(lambda x: (x[0].split()[1].lower(), b2d(x[1:]))), dict ) logger.info('parsed varlabels from format txt', nlabeled=len(labels.keys()), nrepl=len(repl.keys())) return labels
def get_metadata_socrata_denovo(soc_cfg): g = soc_cfg revmap = {v: k for k, v in g.mapcols.items()} url = '{api_url}?' + \ '$select={cols}' + \ '&$order={ocols}' meta_diff = set(g.qn_meta).difference(g.computed) meta_diff = list(meta_diff) qncols = ','.join([(revmap[k] if k in revmap else k) for k in meta_diff]) ocols = ','.join([revmap['qid'], 'year']) logger.info('loading SODA meta data') res = thread_last( g.soda_api, map(lambda x: url.format(api_url=x, cols=qncols, ocols=ocols)), map(dl.df_from_socrata_url), curry(pd.concat)(ignore_index=True)) ''' lambda xf: xf.applymap(lambda x: (re.sub('\xa0', '', x)).strip()), lambda xf: xf.rename(index=str, columns={x: x.lower() for x in xf.columns}), lambda xf: xf if not g.mapcols else xf.rename(index=str, columns=g.mapcols), curry(apply_fn2vals)(fns=g.apply_fn), lambda xf: xf if not g.mapvals else xf.replace(g.mapvals), lambda xf: xf if not g.mapvals else xf.applymap(lambda x: g.mapvals[x.lower().strip()] if x.lower().strip() in g.mapvals else x), lambda xf: xf[g.qn_meta]) ''' logger.info('finished transformations', res=res.head()) # pull out question -> response breakouts qns = res[['qid', 'year', 'topic', 'subtopic', 'question', 'response']].drop_duplicates().reset_index(drop=True) # since facets are questions as well # update the dict with response value from fc_res # overriding the original var (N.B.) yrvec = (res[['year']] .drop_duplicates() .assign(facet='year') .rename(index=str, columns={'year': 'facet_level'})) stvec = (res[['sitecode']] .drop_duplicates() .assign(facet='sitecode') .rename(index=str, columns={'sitecode':'facet_level'})) facs = pd.concat( [res[['facet', 'facet_level']].drop_duplicates(), yrvec, stvec], axis=0).reset_index(drop=True) logger.info('created qn and facs', qn=qns.head(), fac=facs.head()) return (qns, facs)
def get_qids_by_year(soc_cfg): g = soc_cfg revmap = {v: k for k, v in g.mapcols.items()} url = '{api_url}?' + \ '$select=year,{qnkey},count(year)' + \ '&$group=year,{qnkey}' + \ '&$order={qnkey},year' qid = revmap['qid'] df = thread_last(g.soda_api, map(lambda x: url.format(api_url=x, qnkey=qid)), map(dl.df_from_socrata_url), curry(pd.concat)(ignore_index=True)) df.to_csv(sys.stdout)
def load_variable_labels(format_f, formas_f, repl, year=None): logger.info("loading format labels", file=format_f) labels = thread_last( format_f, dl.fetch_data_from_url, lambda x: x.read(), lambda t: (t.decode('utf-8', errors='ignore') if type(t) is bytes else t), curry(parse_variable_labels)(repl=repl) ) logger.info("loaded format labels", lbls=labels) logger.info("loading format assignments", file=formas_f) assignments = thread_last( formas_f, dl.fetch_data_from_url, lambda x: x.read(), lambda t: (t.decode('utf-8', errors='ignore') if type(t) is bytes else t), parse_format_assignments ) logger.info("loaded format assns", ass=assignments) return {k: labels[v] for k, v in assignments.items() if v in labels}
def block2dict(lines, repl, to_lower=False): f_lwr = str.lower if to_lower else identity f_repl = curry(lambda k, r: r[k] if k in r else k)(r=repl) rqt = re.compile(r'[\"\']') # match quote chars rws = re.compile(r'\s') # match whitespace # keep only alnum and a few unreserved symbols ruri = re.compile(r'(?![\w\s\-\_\.\'\$\-\+\(\)\/]|\.).') d = thread_last( lines, map(lambda x: x.replace('\x92', "'")), map(lambda x: rqt.sub('', x.strip()).split('=')), map(lambda x: (rws.sub('', x[0].strip()), ruri.sub('', x[1].strip()))), filter(lambda x: x[0].find('-') == -1), # no support for ranges (mapcat, lambda x: map(lambda y: (y, x[1]), x[0].split(','))), filter(lambda x: x[0].isnumeric()), # remove non-numeric codes map(lambda x: (int(x[0]), # cat codes are ints pipe(x[1], f_lwr, f_repl))), dict ) # d[-1] = np.nan #use NA as a marker for unmapped vals return d
def test_thread_last(): assert list(thread_last([1, 2, 3], (map, inc), (filter, iseven))) == [2, 4] assert list(thread_last([1, 2, 3], (map, inc), (filter, isodd))) == [3] assert thread_last(2, (add, 5), double) == 14
def test_thread_last(): assert list(thread_last([1, 2, 3], (map, inc), (filter, iseven))) == [2, 4] assert list(thread_last([1, 2, 3], (map, inc), (filter, isodd))) == [3] assert thread_last(2, (add, 5), double) == 14
import us import pandas as pd import numpy as np from cytoolz.itertoolz import unique from cytoolz.functoolz import thread_last, identity from cytoolz.curried import map, filter, curry from survey_stats import pdutil # import sys # import traceback as tb from survey_stats import log logger = log.getLogger(__name__) US_STATES_FIPS_INTS = thread_last(us.STATES_AND_TERRITORIES, map(lambda x: x.fips), filter(lambda x: x is not None), map(lambda x: int(x)), list) SITECODE_TRANSLATORS = { 'fips': lambda x: (us.states.lookup('%.2d' % x).abbr if int(x) in US_STATES_FIPS_INTS else 'NA'), 'codes': identity } SVYDESIGN_COLS = ['sitecode', 'strata', 'psu', 'weight'] def convert_cat_codes(s, fmt): unq_lvls = list(unique([fmt[k] for k in sorted(fmt.keys())]))