def find_field(fields, ftype='who', default=None, **kwargs): possibilities = { 'who': ['org', 'partner'], 'what': ['sector', 'cluster'], 'where': ['code', 'state', 'region', 'province', 'township'], } field = kwargs.get(ftype) or default return field or tup.find(fields, possibilities[ftype], method='fuzzy')
def find_ids(self, packages, **kwargs): default = {'rid': '', 'pname': ''} kwargs.update({'method': self.query, 'default': default}) return pr.find(packages, **kwargs)
def customize(org_id, **kwargs): """Introspects custom organization values""" verbose = not kwargs['quiet'] ckan_kwargs = {k: v for k, v in kwargs.items() if k in api.CKAN_KEYS} image_sq = kwargs.get('image_sq') image_rect = kwargs.get('image_rect') sanitize = kwargs.get('sanitize') three_dub_id = kwargs.get('3w') geojson_id = kwargs.get('geojson') topline_id = kwargs.get('topline') ckan = CKAN(**ckan_kwargs) organization = ckan.organization_show(id=org_id, include_datasets=True) org_packages = organization['packages'] hdx = ckan.organization_show(id='hdx', include_datasets=True) extras = {e['key']: e['value'] for e in organization['extras']} if three_dub_id: three_dub_set_id = ckan.get_package_id(three_dub_id) else: ids = ckan.find_ids(org_packages, pnamed='3w', ptagged='3w') three_dub_set_id = ids['pname'] three_dub_id = ids['rid'] if not three_dub_id: sys.exit(1) if not topline_id: topline_id = ckan.find_ids(org_packages, pnamed='topline')['rid'] if geojson_id: geojson_set_id = ckan.get_package_id(geojson_id) else: country = org_id.split('-')[1] hkwargs = {'pnamed': 'json-repository', 'rnamed': country} ids = ckan.find_ids(hdx['packages'], **hkwargs) geojson_set_id = ids['pname'] geojson_id = ids['rid'] viz_url = '%s/dataset/%s' % (kwargs['remote'], three_dub_set_id) three_dub_r = ckan.fetch_resource(three_dub_id) _fields = three_dub_r.iter_lines().next().split(',') three_dub_fields = tup.underscorify(_fields) if sanitize else _fields if geojson_id: geojson_r = ckan.fetch_resource(geojson_id) geojson_fields = geojson_r.json()['features'][0]['properties'].keys() else: geojson_fields = [] if verbose: print('3w fields:') pprint(three_dub_fields) print('geojson fields:') pprint(geojson_fields) def_where = tup.find(three_dub_fields, geojson_fields) or '' who_column = find_field(three_dub_fields, 'who', **kwargs) what_column = find_field(three_dub_fields, 'what', **kwargs) where_column = find_field(three_dub_fields, 'where', def_where, **kwargs) where_column_2 = find_field(geojson_fields, 'where', def_where, **kwargs) name_column = kwargs.get('where') or def_where if 'http' not in image_sq: gdocs = 'https://docs.google.com' image_sq = '%s/uc?id=%s&export=download' % (gdocs, image_sq) if 'http' not in image_rect: gdocs = 'https://docs.google.com' image_rect = '%s/uc?id=%s&export=download' % (gdocs, image_rect) data = { 'name': org_id, 'resource_id_1': three_dub_id, 'resource_id_2': geojson_id, 'topline_resource': topline_id, 'datatype_1': kwargs.get('datatype_1') or 'datastore', 'datatype_2': kwargs.get('datatype_2') or 'filestore', 'org_url': extras['org_url'], 'description': organization['description'], 'title': organization['title'], 'image_sq': image_sq, 'image_rect': image_rect, 'highlight_color': kwargs.get('color'), 'dataset_id_1': three_dub_set_id, 'dataset_id_2': geojson_set_id, 'who_column': deref_field(three_dub_fields, who_column), 'what_column': deref_field(three_dub_fields, what_column), 'where_column': deref_field(three_dub_fields, where_column), 'where_column_2': deref_field(geojson_fields, where_column_2), 'map_district_name_column': deref_field(geojson_fields, name_column), 'viz_data_link_url': viz_url, 'visualization_select': kwargs.get('viz_type', '3W-dashboard'), 'viz_title': kwargs.get('viz_title', "Who's doing what and where?"), 'colors': [ '#c6d5ed', '#95b5df', '#659ad2', '#026bb5', '#659ad2', '#213b68', '#101d4e', '#000035'], 'use_org_color': True, 'modified_at': int(time()), } control_sheet_data = [data[k] for k in control_sheet_keys] if verbose: print('\nCustom pages control sheet data:') print(control_sheet_data) return control_sheet_data
def customize(org_id, **kwargs): """Introspects custom organization values""" verbose = not kwargs['quiet'] ckan_kwargs = {k: v for k, v in kwargs.items() if k in api.CKAN_KEYS} image_sq = kwargs.get('image_sq') image_rect = kwargs.get('image_rect') sanitize = kwargs.get('sanitize') three_dub_id = kwargs.get('3w') geojson_id = kwargs.get('geojson') topline_id = kwargs.get('topline') ckan = CKAN(**ckan_kwargs) organization = ckan.organization_show(id=org_id, include_datasets=True) org_packages = organization['packages'] hdx = ckan.organization_show(id='hdx', include_datasets=True) extras = {e['key']: e['value'] for e in organization['extras']} if three_dub_id: three_dub_set_id = ckan.get_package_id(three_dub_id) else: ids = ckan.find_ids(org_packages, pnamed='3w', ptagged='3w') three_dub_set_id = ids['pname'] three_dub_id = ids['rid'] if not three_dub_id: sys.exit(1) if not topline_id: topline_id = ckan.find_ids(org_packages, pnamed='topline')['rid'] if geojson_id: geojson_set_id = ckan.get_package_id(geojson_id) else: country = org_id.split('-')[1] hkwargs = {'pnamed': 'json-repository', 'rnamed': country} ids = ckan.find_ids(hdx['packages'], **hkwargs) geojson_set_id = ids['pname'] geojson_id = ids['rid'] viz_url = '%s/dataset/%s' % (kwargs['remote'], three_dub_set_id) three_dub_r = ckan.fetch_resource(three_dub_id) _fields = three_dub_r.iter_lines().next().split(',') three_dub_fields = tup.underscorify(_fields) if sanitize else _fields if geojson_id: geojson_r = ckan.fetch_resource(geojson_id) geojson_fields = geojson_r.json()['features'][0]['properties'].keys() else: geojson_fields = [] if verbose: print('3w fields:') pprint(three_dub_fields) print('geojson fields:') pprint(geojson_fields) def_where = tup.find(three_dub_fields, geojson_fields) or '' who_column = find_field(three_dub_fields, 'who', **kwargs) what_column = find_field(three_dub_fields, 'what', **kwargs) where_column = find_field(three_dub_fields, 'where', def_where, **kwargs) where_column_2 = find_field(geojson_fields, 'where', def_where, **kwargs) name_column = kwargs.get('where') or def_where if 'http' not in image_sq: gdocs = 'https://docs.google.com' image_sq = '%s/uc?id=%s&export=download' % (gdocs, image_sq) if 'http' not in image_rect: gdocs = 'https://docs.google.com' image_rect = '%s/uc?id=%s&export=download' % (gdocs, image_rect) data = { 'name': org_id, 'resource_id_1': three_dub_id, 'resource_id_2': geojson_id, 'topline_resource': topline_id, 'datatype_1': kwargs.get('datatype_1') or 'datastore', 'datatype_2': kwargs.get('datatype_2') or 'filestore', 'org_url': extras['org_url'], 'description': organization['description'], 'title': organization['title'], 'image_sq': image_sq, 'image_rect': image_rect, 'highlight_color': kwargs.get('color'), 'dataset_id_1': three_dub_set_id, 'dataset_id_2': geojson_set_id, 'who_column': deref_field(three_dub_fields, who_column), 'what_column': deref_field(three_dub_fields, what_column), 'where_column': deref_field(three_dub_fields, where_column), 'where_column_2': deref_field(geojson_fields, where_column_2), 'map_district_name_column': deref_field(geojson_fields, name_column), 'viz_data_link_url': viz_url, 'visualization_select': kwargs.get('viz_type', '3W-dashboard'), 'viz_title': kwargs.get('viz_title', "Who's doing what and where?"), 'colors': [ '#c6d5ed', '#95b5df', '#659ad2', '#026bb5', '#659ad2', '#213b68', '#101d4e', '#000035' ], 'use_org_color': True, 'modified_at': int(time()), } control_sheet_data = [data[k] for k in control_sheet_keys] if verbose: print('\nCustom pages control sheet data:') print(control_sheet_data) return control_sheet_data