def api(app):
'''
CKAN API fixture.
Returns a ``ckanapi.LocalCKAN`` instance.
'''
return ckanapi.LocalCKAN()
def get_cube(context, data_dict):
"""
Return a dict representation of a cube, given a cubeId, if it exists.
:param cubeId: ID of the cube to retrieve. (i.e. 1310001)
:type cubeId: str
:return: requested cube
:rtype: dict
:raises: ValidationError, ObjectObjectNotFound
"""
cube_id = _get_or_bust(data_dict, 'cubeId')
lc = ckanapi.LocalCKAN(context=context)
result = lc.action.package_search(
q=('type:cube AND '
'product_id_new:{cube_id}').format(cube_id=cube_id),
rows=1)
if not result['count']:
raise ObjectNotFound('Cube not found')
elif result['count'] > 1:
raise ValidationError('More than one cube with given cubeid found')
else:
return result['results'][-1]
def set_related_id(product_id, related_product_ids):
"""
Add product_id to related_products field for each related_product_id
:param product_id: ID of product to add
:type product_id: str
:param related_product_ids: IDs of products to update
:type related_product_ids: list
:return:
"""
if not related_product_ids or not isinstance(related_product_ids, list):
return
q = u'product_id_new:' + u' OR product_id_new:'.join(related_product_ids)
lc = ckanapi.LocalCKAN()
search_result = lc.action.package_search(q=q)
results = search_result.get('results', [])
for result in results:
related_products = result.get(u'related_products', [])
if product_id not in related_products:
related_products.append(product_id)
result.update({u'related_products': related_products})
try:
lc.action.package_update(**result)
except ValidationError:
pass # fail quietly if best effort unsuccessful
def _update_product(product_id,
product_type,
release_date,
publishing_status):
new_values = {'last_release_date': release_date,
'publishing_status': publishing_status,
'status_code': status
}
lc = ckanapi.LocalCKAN(context=context)
result = lc.action.package_search(
q=(
'type:{product_type} AND '
'product_id_new:{product_id}'
).format(
product_type=business_logic[product_type]['type'],
product_id=product_id),
rows=1
)
if not result['count']:
raise _NotFound('Product not found')
elif result['count'] > 1:
raise _ValidationError(
'More than one product with given productid found'
)
product = result['results'][0]
product.update(new_values)
lc.action.package_update(**product)
updated_products.append(product['product_id_new'])
def _update_children(product_id,
release_date,
publishing_status):
new_values = {'last_release_date': release_date,
'publishing_status': publishing_status,
'status_code': status
}
lc = ckanapi.LocalCKAN(context=context)
response = lc.action.package_search(
q=(
'(type:view OR '
'type:indicator OR '
'type:chart OR '
'type:map) AND '
'top_parent_id:{top_parent_id}'
).format(top_parent_id=product_id),
rows=1000
)
if response['count'] > 0:
for product in response['results']:
product.update(new_values)
lc.action.package_update(**product)
updated_products.append(product['product_id_new'])
def get_organization_data(org_id):
try:
lc = ckanapi.LocalCKAN()
org_data = lc.action.organization_show(id=org_id)
except:
org_data = []
return org_data
def generate_datapackage_json(package_id):
'''Generates the datapackage - metadata that would be saved as
datapackage.json.
'''
context = {'model': model, 'session': model.Session}
dataset = get_action('package_show')(context, {'id': package_id})
# filter out resources that are not suitable for inclusion in the data
# package
local_ckan = ckanapi.LocalCKAN()
dataset, resources_to_include, existing_zip_resource = \
remove_resources_that_should_not_be_included_in_the_datapackage(
dataset)
# get the datapackage (metadata)
datapackage = ckanapi.datapackage.dataset_to_datapackage(dataset)
# populate datapackage with the schema from the Datastore data
# dictionary
ckan_and_datapackage_resources = zip(resources_to_include,
datapackage.get('resources', []))
for res, datapackage_res in ckan_and_datapackage_resources:
ckanapi.datapackage.populate_datastore_res_fields(ckan=local_ckan,
res=res)
ckanapi.datapackage.populate_schema_from_datastore(
cres=res, dres=datapackage_res)
# add in any other dataset fields, if configured
fields_to_include = config.get(
u'ckanext.downloadall.dataset_fields_to_add_to_datapackage',
u'').split()
for key in fields_to_include:
datapackage[key] = dataset.get(key)
return (datapackage, ckan_and_datapackage_resources, existing_zip_resource)
def get_products_by_survey(context, data_dict):
# noinspection PyUnresolvedReferences
"""
Find published products which have the given survey as a source
:param surveyID: ID of the survey for which to find related products
:type surveyID: str
:return: list of related products
"""
survey_id = get_or_bust(data_dict, 'surveyID')
lc = ckanapi.LocalCKAN(context=context)
results = lc.action.package_search(
q='survey_source_codes:{survey_id} AND '
'last_publish_status_code:12'.format(survey_id=survey_id),
rows=1000)
products = []
results = results.get('results', [])
for result in results:
title = result.get(u'title')
title = title if title else {u'en': u'', u'fr': u''}
product_id = result.get(u'product_id_new')
try:
url = get_product_url(context, {u'productId': product_id})
except NotFound:
url = {u'en': u'', u'fr': u''}
products.append({
u'product_id': product_id,
u'title': title,
u'url': url
})
return products
def preview_table(self, resource_name, owner_org, errors=None):
lc = ckanapi.LocalCKAN(username=c.user)
try:
chromo = get_chromo(resource_name)
except RecombinantException:
abort(404, _('Recombinant resource_name not found'))
try:
dataset = lc.action.recombinant_show(
dataset_type=chromo['dataset_type'], owner_org=owner_org)
except ckanapi.NotFound:
abort(404, _('Table for this organization not found'))
org = lc.action.organization_show(id=owner_org)
for r in dataset['resources']:
if r['name'] == resource_name:
break
else:
abort(404, _('Resource not found'))
return render('recombinant/resource_edit.html',
extra_vars={
'dataset': dataset,
'resource': r,
'organization': org,
'errors': errors,
})
def archive_children_of_cube(key, data, errors, context):
"""
Apply archive date and status to children of the cube for which ID
is provided
"""
if errors[key]:
return
if key != (u'archive_status_code', ):
return
dataset_type = _data_lookup((u'type', ), data)
if dataset_type != 'cube':
return
cube_archive_status_code = _data_lookup((u'archive_status_code', ), data)
if cube_archive_status_code is missing or not cube_archive_status_code:
return
cube_id = _data_lookup((u'product_id_new', ), data)
child_list = h.get_child_datasets(cube_id)
lc = ckanapi.LocalCKAN(context=context)
for child in child_list:
update = False
child_archive_status_code = child.get(u'archive_status_code')
if child_archive_status_code != cube_archive_status_code:
child[u'archive_status_code'] = cube_archive_status_code
update = True
cube_archive_date = _data_lookup((u'archive_date', ), data)
if cube_archive_date and cube_archive_date is not missing:
child_archive_date = child.get(u'archive_date')
if child_archive_date is missing or not child_archive_date:
child[u'archive_date'] = cube_archive_date
update = True
if update:
lc.action.package_update(**child)
def get_format(context, data_dict):
# noinspection PyUnresolvedReferences
"""
Return a dict representation of a format, given a formatName, if it exists.
:param formatName: slug of the format to retrieve. (i.e. format-10230123_17)
:type releaseName: str
:return: requested format
:rtype: dict
:raises: ValidationError, ObjectObjectNotFound
"""
format_name = _get_or_bust(data_dict, 'formatName')
lc = ckanapi.LocalCKAN(context=context)
result = lc.action.package_search(
q=('dataset_type:format AND '
'name:{format_name}').format(release_name=format_name),
rows=1)
if not result['count']:
raise _NotFound('Format not found')
elif result['count'] > 1:
raise _ValidationError(
'More than one format with given formatName found')
else:
return result['results'][-1]
def clone(self, ds_id):
context = {
'model': base.model,
'session': base.model.Session,
'user': base.c.user or base.c.author
}
context['__cloning'] = True
lc = ckanapi.LocalCKAN(context=context)
pkg = lc.action.package_show(id=ds_id)
# Remove any fields that need to be reset between
# clones, such as unique IDs.
for field in self.PURGE_FIELDS:
if field in pkg:
del pkg[field]
now = datetime.now()
pkg.update({
'name': '{0}-clone'.format(pkg['name']),
'metadata_created': now,
'metadata_modified': now,
'notes': {
u'en': (pkg.get('notes') or {}).get('en') or '',
u'fr': (pkg.get('notes') or {}).get('fr') or ''
}
})
new_pkg = lc.action.package_create(**pkg)
toolkit.redirect_to(
controller='package',
action='edit',
id=new_pkg['id']
)
def __init__(self, id, api=None, default_owner_org=None):
self.id = unicode(id)
self._api = api or ckanapi.LocalCKAN()
self.default_owner_org = default_owner_org
self._synced_child_eids = set()
self._log = Importer._PrefixLoggerAdapter(
logging.getLogger(__name__), 'Importer {!r}: '.format(self.id))
def get_product_issue_articles(context, data_dict):
# noinspection PyUnresolvedReferences
"""
Returns a list of the articles for the specific product/issue number
:param: productId: A non-data product ID.
:param: issueNo: The issue number
:return: A dictionary containing the articles for the specified product and
issue
:rtype: dict
"""
product_id = _get_or_bust(data_dict, 'productId')
issue_number = _get_or_bust(data_dict, 'issueNo')
lc = ckanapi.LocalCKAN(context=context)
results = lc.action.package_search(
q=('top_parent_id:{pid} AND '
'issue_number_int:{issue_number} AND '
'type:article').format(pid=product_id, issue_number=issue_number),
# FIXME: We need to actually paginate on this, but the daily
# team will not accept it (yet).
rows='2000000',
fl=['title', 'product_id_new'])
return [{
'title': result['title'],
'article_id': result['product_id_new'],
'release_date': result['last_release_date']
} for result in results['results']]
def delete_product(context, data_dict):
# noinspection PyUnresolvedReferences
"""
Set the status of a record to 'Deleted' and remove all metadata associated
with that record. This will make the productid available for reuse.
This web service is presently being mocked (i.e. it will return a success
if a valid productid is passed in, but the exact implementation is still
being discussed.)
:param productId: product id of record to be deleted
:type productId: str
:param issueno:
:return: success or failure
:rtype: dict
"""
product_id = _get_or_bust(data_dict, 'productId')
lc = ckanapi.LocalCKAN(context=context)
response = lc.action.package_search(
q='product_id_new:{pid}'.format(pid=product_id),
rows=1,
fl=['id', 'type']
)
if response['count']:
result = response['results'][0]
lc.action.PurgeDataset(id=result['id'])
return {
'message': 'Product successfully deleted',
'product_id_new': product_id
}
def get_datapreview_recombinant(resource_name, res_id):
from ckanext.recombinant.tables import get_chromo
chromo = get_chromo(resource_name)
default_preview_args = {}
lc = ckanapi.LocalCKAN(username=c.user)
results = lc.action.datastore_search(
resource_id=res_id,
limit=0,
)
priority = len(chromo['datastore_primary_key'])
pk_priority = 0
fields = []
for f in chromo['fields']:
out = {
'type': f['datastore_type'],
'id': f['datastore_id'],
'label': h.recombinant_language_text(f['label'])}
if out['id'] in chromo['datastore_primary_key']:
out['priority'] = pk_priority
pk_priority += 1
else:
out['priority'] = priority
priority += 1
fields.append(out)
return h.snippet('package/wet_datatable.html',
resource_name=resource_name,
resource_id=res_id,
ds_fields=fields)
def get_upcoming_releases(context, data_dict):
# noinspection PyUnresolvedReferences
"""
Return all records with a publish_status_code of Verified (08) and a
release date between the two parameters.
:param startDate: Beginning of date range
:param endDate: End of date range
:returns: All matching results.
:rtype: list of dicts
"""
# TODO: date validation? anything else?
context['ignore_capacity_check'] = True
lc = ckanapi.LocalCKAN(context=context)
start_date = to_utc(_get_or_bust(data_dict, 'startDate'),
def_date=default_release_date)
end_date = to_utc(_get_or_bust(data_dict, 'endDate'),
def_date=default_release_date)
result = lc.action.package_search(
q=(
'last_release_date:[{start_date}Z TO {end_date}Z] '
'AND publish_status_code:8'
).format(
start_date=start_date,
end_date=end_date
),
rows=500
)
# Per JIRA #5173, return an empty list instead of the standard
# NotFound Exception (404).
results = result.get('results', [])
# Per JIRA #5206, resolve parent product types and URLs.
for release_result in results:
if not release_result.get('parent_id'):
continue
parent_result = lc.action.package_search(
q='product_id_new:{pid}'.format(pid=release_result['parent_id']),
rows=1
)
if parent_result['results']:
parent_result = parent_result['results'][0]
else:
continue
release_result.update({
'title': parent_result.get('title'),
'url': parent_result.get('url'),
'product_type': parent_result.get('type')
})
return {'count': result['count'], 'results': results}
def recombinant_show_package(pkg):
"""
return recombinant_show results for pkg
"""
lc = ckanapi.LocalCKAN(username=c.user)
return lc.action.recombinant_show(
dataset_type=pkg['type'],
owner_org=pkg['organization']['name'])
def test_local_fail(self):
try:
import ckan
except ImportError:
raise unittest.SkipTest('ckan not importable')
self.assertRaises(ckanapi.CKANAPIError,
ckanapi.LocalCKAN('fake').call_action, 'fake', {},
{}, 'apikey not allowed')
def get_datapreview_ati(res_id):
lc = ckanapi.LocalCKAN(username=c.user)
results = lc.action.datastore_search(resource_id=res_id,
sort='year,month desc',
limit=3000)
return h.snippet('package/wet_datatable.html',
ds_fields=results['fields'],
ds_records=results['records'])
def _home_groups():
gps = []
ckan = ckanapi.LocalCKAN()
groups = ckan.action.group_list()
for g in groups:
gps.append(ckan.action.group_show(id=g))
return gps
def next_article_id(top_parent_id, issue_number):
"""
Get next available product ID
:param top_parent_id:
:type top_parent_id: 8 digit str
:param issue_number
:type issue_number: 7 digit str
:return: 19 or 20 digit str
"""
if not isinstance(top_parent_id, basestring) or len(top_parent_id) != 8:
raise ValidationError(
(_('Invalid top parent ID. Expected 8 digit string'), ))
if not isinstance(issue_number, basestring) or len(issue_number) != 7:
raise ValidationError(
(_('Invalid issue number. Expected 7 digit string'), ))
lc = ckanapi.LocalCKAN()
# check whether issue dataset exists
result = lc.action.package_search(q='{pid}{issue_number}'.format(
pid=top_parent_id, issue_number=issue_number))['results']
if not result:
raise ValidationError({
'{pid}{issue_number}'.format(pid=top_parent_id,
issue_number=issue_number):
'Cannot create article, parent issue missing'
})
i = 0
n = 1
article_sequence_number = 1
while i < n:
results = lc.action.package_search(
q=('type:article AND '
'product_id_new:{top_parent_id}{issue_number}?*').format(
top_parent_id=top_parent_id, issue_number=issue_number),
sort='product_id_new ASC',
rows=1000,
start=i * 1000)
if results['count'] == 0:
return u'{top_parent_id}{issue_number}{sequence_number}'.format(
top_parent_id=top_parent_id,
issue_number=issue_number,
sequence_number=unicode(article_sequence_number).zfill(5))
n = results['count'] / 1000.0
i += 1
for result in results['results']:
old_id = int(result['product_id_new'][15:])
article_sequence_number = max(article_sequence_number, old_id)
return (u'{top_parent_id}'
'{issue_number}'
'{sequence_number}').format(
top_parent_id=top_parent_id,
issue_number=issue_number,
sequence_number=unicode(article_sequence_number + 1).zfill(5))
def get_product_url(context, data_dict):
# noinspection PyUnresolvedReferences
"""
Return the fluent URL of the given format of the product.
If no format is specified, return the url of the preferred (primary) format
:param productId:
:type productId: str
:param formatCode:
:type formatCode: str
:return: dict
:raises: NotFound
"""
product_id = _get_or_bust(data_dict, 'productId')
format_code = data_dict.get('formatCode')
lc = ckanapi.LocalCKAN(context=context)
if format_code:
results = lc.action.package_search(
q='name:format-{product_id}_{format_code}'.format(
product_id=product_id,
format_code=format_code
).lower()
).get('results')
if results:
return results[0].get(u'url', {u'en': u'', u'fr': u''})
else:
raise _NotFound('{product_id}: no format {format_code} '
'found for product'.format(
product_id=product_id,
format_code=format_code
))
else:
results = lc.action.package_search(
q='name:format-{product_id}_*'.format(
product_id=product_id
).lower()
).get('results')
if not results:
raise _NotFound(
'{product_id}: no formats found for product'.format(
product_id=product_id
)
)
choices = scheming_helpers.scheming_get_preset(
'ndm_format'
).get('choices')
sorted_choices = sorted(choices, key=lambda k: k.get('weight', '99'))
for choice in sorted_choices:
for result in results:
if result.get(u'format_code') == choice['value']:
return result.get(u'url', {u'en': u'', u'fr': u''})
return {u'en': u'', u'fr': u''}
def get_product(context, data_dict):
"""
Returns a product given its `productId`, if it exists.
:param productId: product id (i.e. 2112002604)
:type productId: str
:return: product or 404 if not found.
:rtype: dict
:raises: ObjectNotFound, ValidationError
"""
product_id = _get_or_bust(data_dict, 'productId')
lc = ckanapi.LocalCKAN(context=context)
result = lc.action.package_search(
q='product_id_new:{product_id}'.format(
product_id=product_id
),
rows=1
)
if not result['count']:
raise _NotFound(
('product {product_id} not found'.format
(product_id=product_id),
)
)
# If we're getting more than one result for a given product_id
# something has gone terribly wrong with the database.
assert(not result['count'] > 1)
product = result['results'][0]
# As part of JIRA-5048 we need to resolve _code fields and return
# the labels.
codes_to_lookup = (
('frequency_codes', 'frequency', 'codeset'),
('geolevel_codes', 'geolevel', 'codeset'),
('subject_codes', 'subjects', 'subject'),
('survey_source_codes', 'surveys', 'survey'),
('archive_status_code', 'archive_status', 'preset')
)
for code_field, code_id, lookup_type in codes_to_lookup:
cfv = product.get(code_field)
if not cfv:
continue
codes = cfv if isinstance(cfv, list) else [cfv]
product[code_field + '_resolved'] = results = {}
for code in codes:
value = stcndm_helpers.lookup_label(code_id, code, lookup_type)
results[code] = value
return product
def get_dguid_from_pkg_id(pkg_id):
"""
Given the package id, return the dguid of the package
:param pkg_id:
:return:
"""
lc = ckanapi.LocalCKAN()
result = lc.action.package_show(**{u'id': pkg_id})
return result.get(u'geodescriptor_code')
def test_webhook_actions(self):
ckan = ckanapi.LocalCKAN()
callback = 'http://example.com/webhook_callback'
hook = ckan.action.webhook_create(topic='dataset/create', address=callback)
assert hook
show = ckan.action.webhook_show(id=hook)
assert show['id'] == hook
delete = ckan.action.webhook_delete(id=hook)
assert delete == hook
def insert_or_update_pkg(self, pkg_dict, upload=None):
registry = ckanapi.LocalCKAN(username=self.username)
allow_duplicates = tk.asbool(
tk.config.get('ckanext.ddi.allow_duplicates', False)
)
override_datasets = tk.asbool(
tk.config.get('ckanext.ddi.override_datasets', False)
)
visibility = pkg_dict.pop('visibility', 'restricted')
try:
existing_pkg = registry.call_action('package_show', pkg_dict)
if not allow_duplicates and not override_datasets:
raise ContentDuplicateError(
'Dataset already exists and duplicates are not allowed.'
)
if override_datasets:
pkg_dict.pop('id', None)
pkg_dict.pop('name', None)
existing_pkg.update(pkg_dict)
pkg_dict = existing_pkg
registry.call_action('package_update', pkg_dict)
else:
raise ckanapi.NotFound()
except ckanapi.NotFound:
pkg_dict.pop('id', None)
pkg_dict['name'] = self._gen_new_name(pkg_dict['name'])
registry.call_action('package_create', pkg_dict)
if upload is not None:
try:
registry.call_action(
'resource_create',
{
'package_id': pkg_dict['name'],
'upload': upload,
'name': 'DDI XML',
'format': 'xml',
'url': '',
'type': 'attachment',
'file_type': 'other',
'visibility': visibility
}
)
except Exception as e:
raise UploadError(
'Could not upload file: %s' % str(e)
)
log.debug(pkg_dict['name'])
return pkg_dict['name']
def sync_ckan(target, since_ts=None, before_ts=None):
"""
Sync this CKAN instance with a remote CKAN instance.
:param target: A RemoteCKAN instance.
"""
since_ts = since_ts or datetime.utcnow() - timedelta(days=1)
before_ts = before_ts or datetime.utcnow()
lc = ckanapi.LocalCKAN()
for activity in changes_since(since_ts=since_ts, before_ts=before_ts):
sync_dataset(lc, target, activity['object_id'])
def next_non_data_product_id(subject_code, product_type_code):
"""
Get next available product ID
:param subject_code:
:type subject_code: 2 digit str
:param product_type_code:
:type product_type_code: 2 digit str
:return:
"""
valid_product_codes = ['20', '21', '22', '23', '25', '26']
if not isinstance(subject_code, basestring) or \
not re.match('^\d\d$', subject_code):
raise ValidationError((_('Invalid subject code.'), ))
if isinstance(product_type_code, basestring):
if product_type_code not in valid_product_codes:
error_message = 'Invalid product type code. ' \
'Expected one of {codes!r}'.format(
codes=valid_product_codes,
)
raise ValidationError((_(error_message), ))
i = 0
n = 1
product_sequence_number = 1
while i < n:
lc = ckanapi.LocalCKAN()
results = lc.action.package_search(
q='product_id_new:{subject_code}{product_type_code}????'.format(
subject_code=subject_code,
product_type_code=product_type_code),
sort='product_id_new ASC',
rows=1000,
start=i * 1000)
n = results['count'] / 1000.0
i += 1
for result in results['results']:
if product_sequence_number < int(result['product_id_new'][5:8]):
return (
u'{subject_code}{product_type_code}{sequence_number}'
).format(
subject_code=subject_code,
product_type_code=product_type_code,
sequence_number=unicode(product_sequence_number).zfill(4))
else:
product_sequence_number += 1
return u'{subject_code}{product_type_code}{sequence_number}'.format(
subject_code=subject_code,
product_type_code=product_type_code,
sequence_number=unicode(product_sequence_number).zfill(4))
def patch_datasets(self, patch_filepath):
import ckanapi
registry = ckanapi.LocalCKAN()
dataset_names = []
with open(patch_filepath) as f:
for line in f.readlines():
if not line.strip():
continue
print line
dataset = json.loads(line)
dataset_names.append(dataset.get('name') or dataset.get('id'))
registry.action.package_patch(**dataset)
print 'Patched: ' + ' '.join(dataset_names)
