def upload(source, resource_id=None, package_id=None, **kwargs):
"""Updates the filestore of an existing resource or creates a new one"""
verbose = not kwargs['quiet']
resource_id = resource_id or p.splitext(p.basename(source))[0]
ckan_kwargs = {k: v for k, v in kwargs.items() if k in api.CKAN_KEYS}
if package_id and verbose:
print(
'Creating filestore resource %s in dataset %s...' %
(source, package_id))
elif verbose:
print(
'Uploading %s to filestore resource %s...' % (source, resource_id))
ckan = CKAN(**ckan_kwargs)
resource_kwargs = {
'url' if 'http' in source else 'filepath': source,
'name': kwargs.get('name')
}
if package_id:
resource = ckan.create_resource(package_id, **resource_kwargs)
else:
resource = ckan.update_filestore(resource_id, **resource_kwargs)
if package_id and resource and verbose:
infix = '%s ' % resource['id'] if resource.get('id') else ''
print('Success! Resource %screated.' % infix)
elif resource and verbose:
print('Success! Resource %s updated.' % resource_id)
elif not resource:
sys.exit('Error uploading file!')
def update(resource_id, force=None, **kwargs):
"""Updates a datastore table based on the current filestore resource"""
verbose = not kwargs.get('quiet')
chunk_bytes = kwargs.get('chunk_bytes', api.CHUNKSIZE_BYTES)
ckan_kwargs = {k: v for k, v in kwargs.items() if k in api.CKAN_KEYS}
hash_kwargs = {'chunksize': chunk_bytes, 'verbose': verbose}
ckan = CKAN(**ckan_kwargs)
try:
r = ckan.fetch_resource(resource_id)
except (api.NotFound, api.NotAuthorized) as err:
sys.exit('ERROR: %s\n' % str(err))
else:
f = SpooledTemporaryFile(suffix='.xlsx', mode='r+b')
write_kwargs = {
'length': r.headers.get('content-length'),
'chunksize': chunk_bytes
}
tio.write(f, r.iter_content, **write_kwargs)
try:
old_hash = ckan.get_hash(resource_id)
except api.NotFound as err:
item = err.args[0]['item']
if item == 'package':
orgs = ckan.organization_list(permission='admin_group')
owner_org = (
o['id'] for o in orgs
if o['display_name'] == kwargs['hash_group']).next()
package_kwargs = {
'name': kwargs['hash_table'],
'owner_org': owner_org,
'package_creator': 'Hash Table',
'dataset_source': 'Multiple sources',
'notes': 'Datastore resource hash table'
}
ckan.hash_table_pack = ckan.package_create(**package_kwargs)
if item in {'package', 'resource'}:
fileobj = StringIO('datastore_id,hash\n')
create_kwargs = {'fileobj': fileobj, 'name': api.DEF_HASH_RES}
table = kwargs['hash_table']
resource = ckan.create_resource(table, **create_kwargs)
ckan.hash_table_id = resource['id']
ckan.create_hash_table(verbose)
old_hash = ckan.get_hash(resource_id)
new_hash = tio.hash_file(f, **hash_kwargs)
changed = new_hash != old_hash if old_hash else True
if verbose:
print(get_message(changed, force))
if not (changed or force):
sys.exit(0)
kwargs['encoding'] = r.encoding
kwargs['content_type'] = r.headers['content-type']
updated = ckan.update_datastore(resource_id, f, **kwargs)
if updated and verbose:
print('Success! Resource %s updated.' % resource_id)
if updated and changed:
ckan.update_hash_table(resource_id, new_hash, verbose)
elif not updated:
sys.exit('ERROR: resource %s not updated.' % resource_id)
def update(resource_id, force=None, **kwargs):
"""Updates a datastore table based on the current filestore resource"""
verbose = not kwargs.get('quiet')
chunk_bytes = kwargs.get('chunk_bytes', api.CHUNKSIZE_BYTES)
ckan_kwargs = {k: v for k, v in kwargs.items() if k in api.CKAN_KEYS}
hash_kwargs = {'chunksize': chunk_bytes, 'verbose': verbose}
ckan = CKAN(**ckan_kwargs)
try:
r = ckan.fetch_resource(resource_id)
except (api.NotFound, api.NotAuthorized) as err:
sys.exit('ERROR: %s\n' % str(err))
else:
f = SpooledTemporaryFile(suffix='.xlsx', mode='r+b')
write_kwargs = {
'length': r.headers.get('content-length'),
'chunksize': chunk_bytes
}
tio.write(f, r.iter_content, **write_kwargs)
try:
old_hash = ckan.get_hash(resource_id)
except api.NotFound as err:
item = err.args[0]['item']
if item == 'package':
orgs = ckan.organization_list(permission='admin_group')
owner_org = (
o['id'] for o in orgs
if o['display_name'] == kwargs['hash_group']).next()
package_kwargs = {
'name': kwargs['hash_table'],
'owner_org': owner_org,
'package_creator': 'Hash Table',
'dataset_source': 'Multiple sources',
'notes': 'Datastore resource hash table'
}
ckan.hash_table_pack = ckan.package_create(**package_kwargs)
if item in {'package', 'resource'}:
fileobj = StringIO('datastore_id,hash\n')
create_kwargs = {'fileobj': fileobj, 'name': api.DEF_HASH_RES}
table = kwargs['hash_table']
resource = ckan.create_resource(table, **create_kwargs)
ckan.hash_table_id = resource['id']
ckan.create_hash_table(verbose)
old_hash = ckan.get_hash(resource_id)
new_hash = tio.hash_file(f, **hash_kwargs)
changed = new_hash != old_hash if old_hash else True
if verbose:
print(get_message(changed, force))
if not (changed or force):
sys.exit(0)
kwargs['encoding'] = r.encoding
kwargs['content_type'] = r.headers['content-type']
updated = ckan.update_datastore(resource_id, f, **kwargs)
if updated and verbose:
print('Success! Resource %s updated.' % resource_id)
if updated and changed:
ckan.update_hash_table(resource_id, new_hash, verbose)
elif not updated:
sys.exit('ERROR: resource %s not updated.' % resource_id)
