def table_info(): """ Return a list of data sources """ columns = OrderedDict() for mancer in MANCERS: m = import_class(mancer)() col_info = m.column_info() for col in col_info: columns[col['table_id']] = { 'table_id': col['table_id'], 'human_name': col['human_name'], 'mancer': m.name, 'columns': col['columns'], 'source_url': col['source_url'], } response = [] if request.args.get('table_id'): table_id = request.args['table_id'] try: response.append(columns[table_id]) except KeyError: response.append({ 'status': 'error', 'message': 'table_id %s not found' % table_id }) else: response.extend(columns.values()) resp = make_response(json.dumps(response)) resp.headers['Content-Type'] = 'application/json' return resp
def do_the_work(file_contents, field_defs, filename): """ field_defs looks like: { 10: { 'type': 'city_state', 'append_columns': ['total_population', 'median_age'] } } file_contents is a string containing the contents of the uploaded file. """ contents = StringIO(file_contents) reader = UnicodeCSVReader(contents) header = reader.next() result = None geo_ids = set() mancer_mapper = {} for mancer in MANCERS: m = import_class(mancer)() mancer_cols = [k['table_id'] for k in m.column_info()] for k, v in field_defs.items(): field_cols = v['append_columns'] for f in field_cols: if f in mancer_cols: mancer_mapper[f] = { 'mancer': m, 'geo_id_map': {}, 'geo_ids': set(), 'geo_type': v['type'] } for row_idx, row in enumerate(reader): col_idxs = [int(k) for k in field_defs.keys()] for idx in col_idxs: val = row[idx] geo_type = field_defs[idx]['type'] for column in field_defs[idx]['append_columns']: mancer = mancer_mapper[column]['mancer'] try: if val: geoid_search = mancer.geo_lookup(val, geo_type=geo_type) else: continue except MancerError, e: return 'Error message: %s, Body: %s' % (e.message, e.body) row_geoid = geoid_search['geoid'] if row_geoid: mancer_mapper[column]['geo_ids'].add(row_geoid) try: mancer_mapper[column]['geo_id_map'][row_geoid].append( row_idx) except KeyError: mancer_mapper[column]['geo_id_map'][row_geoid] = [ row_idx ]
def do_the_work(file_contents, field_defs, filename): """ field_defs looks like: { 10: { 'type': 'city_state', 'append_columns': ['total_population', 'median_age'] } } or like this: { 10;2: { 'type': 'city;state', 'append_columns': ['total_population', 'median_age'] } } where the semicolon separated values represent a multicolumn geography file_contents is a string containing the contents of the uploaded file. """ contents = StringIO(file_contents) reader = UnicodeCSVReader(contents) header = reader.next() result = None geo_ids = set() mancer_mapper = {} fields_key = field_defs.keys()[0] errors = [] geo_type, col_idxs, val_fmt = find_geo_type(field_defs[fields_key]['type'], fields_key) geo_name = get_geo_types(geo_type=geo_type)[0][0]['info'].human_name for mancer in MANCERS: m = import_class(mancer) api_key = MANCER_KEYS.get(m.machine_name) try: m = m(api_key=api_key) except ImportError, e: errors.append(e.message) continue mancer_cols = [c['table_id'] for c in m.get_metadata()] for k, v in field_defs.items(): field_cols = v['append_columns'] for f in field_cols: if f in mancer_cols: mancer_mapper[f] = { 'mancer': m, 'geo_id_map': {}, 'geo_ids': set(), 'geo_type': geo_type, }
def do_the_work(file_contents, field_defs, filename): """ field_defs looks like: { 10: { 'type': 'city_state', 'append_columns': ['total_population', 'median_age'] } } file_contents is a string containing the contents of the uploaded file. """ contents = StringIO(file_contents) reader = UnicodeCSVReader(contents) header = reader.next() result = None geo_ids = set() mancer_mapper = {} for mancer in MANCERS: m = import_class(mancer)() mancer_cols = [k['table_id'] for k in m.column_info()] for k, v in field_defs.items(): field_cols = v['append_columns'] for f in field_cols: if f in mancer_cols: mancer_mapper[f] = { 'mancer': m, 'geo_id_map': {}, 'geo_ids': set(), 'geo_type': v['type'] } for row_idx, row in enumerate(reader): col_idxs = [int(k) for k in field_defs.keys()] for idx in col_idxs: val = row[idx] geo_type = field_defs[idx]['type'] for column in field_defs[idx]['append_columns']: mancer = mancer_mapper[column]['mancer'] try: if val: geoid_search = mancer.geo_lookup(val, geo_type=geo_type) else: continue except MancerError, e: return 'Error message: %s, Body: %s' % (e.message, e.body) row_geoid = geoid_search['geoid'] if row_geoid: mancer_mapper[column]['geo_ids'].add(row_geoid) try: mancer_mapper[column]['geo_id_map'][row_geoid].append(row_idx) except KeyError: mancer_mapper[column]['geo_id_map'][row_geoid] = [row_idx]
def table_info(): """ Return a list of data sources """ columns = OrderedDict() for mancer in MANCERS: m = import_class(mancer) api_key = MANCER_KEYS.get(m.machine_name) try: m = m(api_key=api_key) except ImportError, e: continue col_info = m.get_metadata() for col in col_info: columns[col['table_id']] = { 'table_id': col['table_id'], 'human_name': col['human_name'], 'mancer': m.name, 'columns': col['columns'], 'source_url': col['source_url'], }