def create(org_id, **kwargs): """Creates a package (aka dataset)""" kw = ft.Objectify(kwargs, type='dataset') verbose = not kw.quiet ckan_kwargs = {k: v for k, v in kwargs.items() if k in api.CKAN_KEYS} ckan = CKAN(**ckan_kwargs) licenses = it.imap(itemgetter('id'), ckan.license_list()) orgs = ckan.organization_list() org_ids = it.imap(itemgetter('id'), orgs) org_names = it.imap(itemgetter('name'), orgs) groups = ckan.group_list() name = kw.name or slugify(kw.title) raw_tags = filter(None, kw.tags.split(',')) tags = [{'state': 'active', 'name': t} for t in raw_tags] if kw.start: start = parse(str(kw.start)).strftime('%m/%d/%Y') else: date = None if kw.start and kw.end: date = '%s-%s' % (start, parse(str(kw.end)).strftime('%m/%d/%Y')) elif kw.start: date = start if kw.location in set(groups): group_list = [{'name': kw.location}] elif kw.location: sys.exit('group name: %s not found!' % kw.location) else: group_list = [] if org_id not in set(it.chain(org_ids, org_names)): sys.exit('organization id: %s not found!' % org_id) if kw.license_id not in set(licenses): sys.exit('license id: %s not found!' % kw.license_id) files = filter(None, kw.files.split(',')) names = filter(None, kw.names.split(',')) resource_list = list(it.starmap(make_rkwargs, zip(files, names))) or [] package_kwargs = { 'title': kw.title, 'name': name, 'license_id': kw.license_id, 'owner_org': org_id, 'dataset_source': kw.source, 'notes': kw.description or kw.title, 'type': kw.type, 'tags': tags, 'resources': resource_list, 'package_creator': ckan.user['name'], 'groups': group_list, 'dataset_date': date, 'caveats': kw.caveats, 'methodology': methods.get(kw.methodology, 'Other'), 'methodology_other': methods.get(kw.methodology) or kw.methodology, } if verbose: print('Submitting your package request.') pprint(package_kwargs) print('\n') try: package = ckan.package_create(**package_kwargs) except api.ValidationError as e: exit(e) if kw.private: org = package['organization'] ckan.package_privatize(org_id=org['id'], datasets=[package['id']]) if verbose: print('Your package response.') pprint(package) print('\n') print(package['id']) print('\n')
def update(resource_id, force=None, **kwargs): """Updates a datastore table based on the current filestore resource""" verbose = not kwargs.get('quiet') chunk_bytes = kwargs.get('chunk_bytes', api.CHUNKSIZE_BYTES) ckan_kwargs = {k: v for k, v in kwargs.items() if k in api.CKAN_KEYS} hash_kwargs = {'chunksize': chunk_bytes, 'verbose': verbose} ckan = CKAN(**ckan_kwargs) try: r = ckan.fetch_resource(resource_id) except (api.NotFound, api.NotAuthorized) as err: sys.exit('ERROR: %s\n' % str(err)) else: f = SpooledTemporaryFile(suffix='.xlsx', mode='r+b') write_kwargs = { 'length': r.headers.get('content-length'), 'chunksize': chunk_bytes } tio.write(f, r.iter_content, **write_kwargs) try: old_hash = ckan.get_hash(resource_id) except api.NotFound as err: item = err.args[0]['item'] if item == 'package': orgs = ckan.organization_list(permission='admin_group') owner_org = ( o['id'] for o in orgs if o['display_name'] == kwargs['hash_group']).next() package_kwargs = { 'name': kwargs['hash_table'], 'owner_org': owner_org, 'package_creator': 'Hash Table', 'dataset_source': 'Multiple sources', 'notes': 'Datastore resource hash table' } ckan.hash_table_pack = ckan.package_create(**package_kwargs) if item in {'package', 'resource'}: fileobj = StringIO('datastore_id,hash\n') create_kwargs = {'fileobj': fileobj, 'name': api.DEF_HASH_RES} table = kwargs['hash_table'] resource = ckan.create_resource(table, **create_kwargs) ckan.hash_table_id = resource['id'] ckan.create_hash_table(verbose) old_hash = ckan.get_hash(resource_id) new_hash = tio.hash_file(f, **hash_kwargs) changed = new_hash != old_hash if old_hash else True if verbose: print(get_message(changed, force)) if not (changed or force): sys.exit(0) kwargs['encoding'] = r.encoding kwargs['content_type'] = r.headers['content-type'] updated = ckan.update_datastore(resource_id, f, **kwargs) if updated and verbose: print('Success! Resource %s updated.' % resource_id) if updated and changed: ckan.update_hash_table(resource_id, new_hash, verbose) elif not updated: sys.exit('ERROR: resource %s not updated.' % resource_id)