def upload(source, resource_id=None, **kwargs):
"""Uploads a file to a datastore table"""
verbose = not kwargs['quiet']
resource_id = resource_id or p.splitext(p.basename(source))[0]
if '.' in resource_id:
resource_id = resource_id.split('.')[0]
ckan_kwargs = {k: v for k, v in kwargs.items() if k in api.CKAN_KEYS}
if verbose:
print('Uploading %s to datastore resource %s...' %
(source, resource_id))
# read encoding from extended attributes
x = xattr(source)
try:
kwargs['encoding'] = x.get('com.ckanny.encoding')
except IOError:
pass
if verbose and kwargs['encoding']:
print('Using encoding %s' % kwargs['encoding'])
ckan = CKAN(**ckan_kwargs)
if ckan.update_datastore(resource_id, source, **kwargs):
print('Success! Resource %s uploaded.' % resource_id)
else:
sys.exit('ERROR: resource %s not uploaded.' % resource_id)
def upload(source, resource_id=None, **kwargs):
"""Uploads a file to a datastore table"""
verbose = not kwargs['quiet']
resource_id = resource_id or p.splitext(p.basename(source))[0]
if '.' in resource_id:
resource_id = resource_id.split('.')[0]
ckan_kwargs = {k: v for k, v in kwargs.items() if k in api.CKAN_KEYS}
if verbose:
print(
'Uploading %s to datastore resource %s...' % (source, resource_id))
# read encoding from extended attributes
x = xattr(source)
try:
kwargs['encoding'] = x.get('com.ckanny.encoding')
except IOError:
pass
if verbose and kwargs['encoding']:
print('Using encoding %s' % kwargs['encoding'])
ckan = CKAN(**ckan_kwargs)
if ckan.update_datastore(resource_id, source, **kwargs):
print('Success! Resource %s uploaded.' % resource_id)
else:
sys.exit('ERROR: resource %s not uploaded.' % resource_id)
def update(resource_id, force=None, **kwargs):
"""Updates a datastore table based on the current filestore resource"""
verbose = not kwargs.get('quiet')
chunk_bytes = kwargs.get('chunk_bytes', api.CHUNKSIZE_BYTES)
ckan_kwargs = {k: v for k, v in kwargs.items() if k in api.CKAN_KEYS}
hash_kwargs = {'chunksize': chunk_bytes, 'verbose': verbose}
ckan = CKAN(**ckan_kwargs)
try:
r = ckan.fetch_resource(resource_id)
except (api.NotFound, api.NotAuthorized) as err:
sys.exit('ERROR: %s\n' % str(err))
else:
f = SpooledTemporaryFile(suffix='.xlsx', mode='r+b')
write_kwargs = {
'length': r.headers.get('content-length'),
'chunksize': chunk_bytes
}
tio.write(f, r.iter_content, **write_kwargs)
try:
old_hash = ckan.get_hash(resource_id)
except api.NotFound as err:
item = err.args[0]['item']
if item == 'package':
orgs = ckan.organization_list(permission='admin_group')
owner_org = (
o['id'] for o in orgs
if o['display_name'] == kwargs['hash_group']).next()
package_kwargs = {
'name': kwargs['hash_table'],
'owner_org': owner_org,
'package_creator': 'Hash Table',
'dataset_source': 'Multiple sources',
'notes': 'Datastore resource hash table'
}
ckan.hash_table_pack = ckan.package_create(**package_kwargs)
if item in {'package', 'resource'}:
fileobj = StringIO('datastore_id,hash\n')
create_kwargs = {'fileobj': fileobj, 'name': api.DEF_HASH_RES}
table = kwargs['hash_table']
resource = ckan.create_resource(table, **create_kwargs)
ckan.hash_table_id = resource['id']
ckan.create_hash_table(verbose)
old_hash = ckan.get_hash(resource_id)
new_hash = tio.hash_file(f, **hash_kwargs)
changed = new_hash != old_hash if old_hash else True
if verbose:
print(get_message(changed, force))
if not (changed or force):
sys.exit(0)
kwargs['encoding'] = r.encoding
kwargs['content_type'] = r.headers['content-type']
updated = ckan.update_datastore(resource_id, f, **kwargs)
if updated and verbose:
print('Success! Resource %s updated.' % resource_id)
if updated and changed:
ckan.update_hash_table(resource_id, new_hash, verbose)
elif not updated:
sys.exit('ERROR: resource %s not updated.' % resource_id)
def update(resource_id, force=None, **kwargs):
"""Updates a datastore table based on the current filestore resource"""
verbose = not kwargs.get('quiet')
chunk_bytes = kwargs.get('chunk_bytes', api.CHUNKSIZE_BYTES)
ckan_kwargs = {k: v for k, v in kwargs.items() if k in api.CKAN_KEYS}
hash_kwargs = {'chunksize': chunk_bytes, 'verbose': verbose}
ckan = CKAN(**ckan_kwargs)
try:
r = ckan.fetch_resource(resource_id)
except (api.NotFound, api.NotAuthorized) as err:
sys.exit('ERROR: %s\n' % str(err))
else:
f = SpooledTemporaryFile(suffix='.xlsx', mode='r+b')
write_kwargs = {
'length': r.headers.get('content-length'),
'chunksize': chunk_bytes
}
tio.write(f, r.iter_content, **write_kwargs)
try:
old_hash = ckan.get_hash(resource_id)
except api.NotFound as err:
item = err.args[0]['item']
if item == 'package':
orgs = ckan.organization_list(permission='admin_group')
owner_org = (
o['id'] for o in orgs
if o['display_name'] == kwargs['hash_group']).next()
package_kwargs = {
'name': kwargs['hash_table'],
'owner_org': owner_org,
'package_creator': 'Hash Table',
'dataset_source': 'Multiple sources',
'notes': 'Datastore resource hash table'
}
ckan.hash_table_pack = ckan.package_create(**package_kwargs)
if item in {'package', 'resource'}:
fileobj = StringIO('datastore_id,hash\n')
create_kwargs = {'fileobj': fileobj, 'name': api.DEF_HASH_RES}
table = kwargs['hash_table']
resource = ckan.create_resource(table, **create_kwargs)
ckan.hash_table_id = resource['id']
ckan.create_hash_table(verbose)
old_hash = ckan.get_hash(resource_id)
new_hash = tio.hash_file(f, **hash_kwargs)
changed = new_hash != old_hash if old_hash else True
if verbose:
print(get_message(changed, force))
if not (changed or force):
sys.exit(0)
kwargs['encoding'] = r.encoding
kwargs['content_type'] = r.headers['content-type']
updated = ckan.update_datastore(resource_id, f, **kwargs)
if updated and verbose:
print('Success! Resource %s updated.' % resource_id)
if updated and changed:
ckan.update_hash_table(resource_id, new_hash, verbose)
elif not updated:
sys.exit('ERROR: resource %s not updated.' % resource_id)
