def api(app): ''' CKAN API fixture. Returns a ``ckanapi.LocalCKAN`` instance. ''' return ckanapi.LocalCKAN()
def get_cube(context, data_dict): """ Return a dict representation of a cube, given a cubeId, if it exists. :param cubeId: ID of the cube to retrieve. (i.e. 1310001) :type cubeId: str :return: requested cube :rtype: dict :raises: ValidationError, ObjectObjectNotFound """ cube_id = _get_or_bust(data_dict, 'cubeId') lc = ckanapi.LocalCKAN(context=context) result = lc.action.package_search( q=('type:cube AND ' 'product_id_new:{cube_id}').format(cube_id=cube_id), rows=1) if not result['count']: raise ObjectNotFound('Cube not found') elif result['count'] > 1: raise ValidationError('More than one cube with given cubeid found') else: return result['results'][-1]
def set_related_id(product_id, related_product_ids): """ Add product_id to related_products field for each related_product_id :param product_id: ID of product to add :type product_id: str :param related_product_ids: IDs of products to update :type related_product_ids: list :return: """ if not related_product_ids or not isinstance(related_product_ids, list): return q = u'product_id_new:' + u' OR product_id_new:'.join(related_product_ids) lc = ckanapi.LocalCKAN() search_result = lc.action.package_search(q=q) results = search_result.get('results', []) for result in results: related_products = result.get(u'related_products', []) if product_id not in related_products: related_products.append(product_id) result.update({u'related_products': related_products}) try: lc.action.package_update(**result) except ValidationError: pass # fail quietly if best effort unsuccessful
def _update_product(product_id, product_type, release_date, publishing_status): new_values = {'last_release_date': release_date, 'publishing_status': publishing_status, 'status_code': status } lc = ckanapi.LocalCKAN(context=context) result = lc.action.package_search( q=( 'type:{product_type} AND ' 'product_id_new:{product_id}' ).format( product_type=business_logic[product_type]['type'], product_id=product_id), rows=1 ) if not result['count']: raise _NotFound('Product not found') elif result['count'] > 1: raise _ValidationError( 'More than one product with given productid found' ) product = result['results'][0] product.update(new_values) lc.action.package_update(**product) updated_products.append(product['product_id_new'])
def _update_children(product_id, release_date, publishing_status): new_values = {'last_release_date': release_date, 'publishing_status': publishing_status, 'status_code': status } lc = ckanapi.LocalCKAN(context=context) response = lc.action.package_search( q=( '(type:view OR ' 'type:indicator OR ' 'type:chart OR ' 'type:map) AND ' 'top_parent_id:{top_parent_id}' ).format(top_parent_id=product_id), rows=1000 ) if response['count'] > 0: for product in response['results']: product.update(new_values) lc.action.package_update(**product) updated_products.append(product['product_id_new'])
def get_organization_data(org_id): try: lc = ckanapi.LocalCKAN() org_data = lc.action.organization_show(id=org_id) except: org_data = [] return org_data
def generate_datapackage_json(package_id): '''Generates the datapackage - metadata that would be saved as datapackage.json. ''' context = {'model': model, 'session': model.Session} dataset = get_action('package_show')(context, {'id': package_id}) # filter out resources that are not suitable for inclusion in the data # package local_ckan = ckanapi.LocalCKAN() dataset, resources_to_include, existing_zip_resource = \ remove_resources_that_should_not_be_included_in_the_datapackage( dataset) # get the datapackage (metadata) datapackage = ckanapi.datapackage.dataset_to_datapackage(dataset) # populate datapackage with the schema from the Datastore data # dictionary ckan_and_datapackage_resources = zip(resources_to_include, datapackage.get('resources', [])) for res, datapackage_res in ckan_and_datapackage_resources: ckanapi.datapackage.populate_datastore_res_fields(ckan=local_ckan, res=res) ckanapi.datapackage.populate_schema_from_datastore( cres=res, dres=datapackage_res) # add in any other dataset fields, if configured fields_to_include = config.get( u'ckanext.downloadall.dataset_fields_to_add_to_datapackage', u'').split() for key in fields_to_include: datapackage[key] = dataset.get(key) return (datapackage, ckan_and_datapackage_resources, existing_zip_resource)
def get_products_by_survey(context, data_dict): # noinspection PyUnresolvedReferences """ Find published products which have the given survey as a source :param surveyID: ID of the survey for which to find related products :type surveyID: str :return: list of related products """ survey_id = get_or_bust(data_dict, 'surveyID') lc = ckanapi.LocalCKAN(context=context) results = lc.action.package_search( q='survey_source_codes:{survey_id} AND ' 'last_publish_status_code:12'.format(survey_id=survey_id), rows=1000) products = [] results = results.get('results', []) for result in results: title = result.get(u'title') title = title if title else {u'en': u'', u'fr': u''} product_id = result.get(u'product_id_new') try: url = get_product_url(context, {u'productId': product_id}) except NotFound: url = {u'en': u'', u'fr': u''} products.append({ u'product_id': product_id, u'title': title, u'url': url }) return products
def preview_table(self, resource_name, owner_org, errors=None): lc = ckanapi.LocalCKAN(username=c.user) try: chromo = get_chromo(resource_name) except RecombinantException: abort(404, _('Recombinant resource_name not found')) try: dataset = lc.action.recombinant_show( dataset_type=chromo['dataset_type'], owner_org=owner_org) except ckanapi.NotFound: abort(404, _('Table for this organization not found')) org = lc.action.organization_show(id=owner_org) for r in dataset['resources']: if r['name'] == resource_name: break else: abort(404, _('Resource not found')) return render('recombinant/resource_edit.html', extra_vars={ 'dataset': dataset, 'resource': r, 'organization': org, 'errors': errors, })
def archive_children_of_cube(key, data, errors, context): """ Apply archive date and status to children of the cube for which ID is provided """ if errors[key]: return if key != (u'archive_status_code', ): return dataset_type = _data_lookup((u'type', ), data) if dataset_type != 'cube': return cube_archive_status_code = _data_lookup((u'archive_status_code', ), data) if cube_archive_status_code is missing or not cube_archive_status_code: return cube_id = _data_lookup((u'product_id_new', ), data) child_list = h.get_child_datasets(cube_id) lc = ckanapi.LocalCKAN(context=context) for child in child_list: update = False child_archive_status_code = child.get(u'archive_status_code') if child_archive_status_code != cube_archive_status_code: child[u'archive_status_code'] = cube_archive_status_code update = True cube_archive_date = _data_lookup((u'archive_date', ), data) if cube_archive_date and cube_archive_date is not missing: child_archive_date = child.get(u'archive_date') if child_archive_date is missing or not child_archive_date: child[u'archive_date'] = cube_archive_date update = True if update: lc.action.package_update(**child)
def get_format(context, data_dict): # noinspection PyUnresolvedReferences """ Return a dict representation of a format, given a formatName, if it exists. :param formatName: slug of the format to retrieve. (i.e. format-10230123_17) :type releaseName: str :return: requested format :rtype: dict :raises: ValidationError, ObjectObjectNotFound """ format_name = _get_or_bust(data_dict, 'formatName') lc = ckanapi.LocalCKAN(context=context) result = lc.action.package_search( q=('dataset_type:format AND ' 'name:{format_name}').format(release_name=format_name), rows=1) if not result['count']: raise _NotFound('Format not found') elif result['count'] > 1: raise _ValidationError( 'More than one format with given formatName found') else: return result['results'][-1]
def clone(self, ds_id): context = { 'model': base.model, 'session': base.model.Session, 'user': base.c.user or base.c.author } context['__cloning'] = True lc = ckanapi.LocalCKAN(context=context) pkg = lc.action.package_show(id=ds_id) # Remove any fields that need to be reset between # clones, such as unique IDs. for field in self.PURGE_FIELDS: if field in pkg: del pkg[field] now = datetime.now() pkg.update({ 'name': '{0}-clone'.format(pkg['name']), 'metadata_created': now, 'metadata_modified': now, 'notes': { u'en': (pkg.get('notes') or {}).get('en') or '', u'fr': (pkg.get('notes') or {}).get('fr') or '' } }) new_pkg = lc.action.package_create(**pkg) toolkit.redirect_to( controller='package', action='edit', id=new_pkg['id'] )
def __init__(self, id, api=None, default_owner_org=None): self.id = unicode(id) self._api = api or ckanapi.LocalCKAN() self.default_owner_org = default_owner_org self._synced_child_eids = set() self._log = Importer._PrefixLoggerAdapter( logging.getLogger(__name__), 'Importer {!r}: '.format(self.id))
def get_product_issue_articles(context, data_dict): # noinspection PyUnresolvedReferences """ Returns a list of the articles for the specific product/issue number :param: productId: A non-data product ID. :param: issueNo: The issue number :return: A dictionary containing the articles for the specified product and issue :rtype: dict """ product_id = _get_or_bust(data_dict, 'productId') issue_number = _get_or_bust(data_dict, 'issueNo') lc = ckanapi.LocalCKAN(context=context) results = lc.action.package_search( q=('top_parent_id:{pid} AND ' 'issue_number_int:{issue_number} AND ' 'type:article').format(pid=product_id, issue_number=issue_number), # FIXME: We need to actually paginate on this, but the daily # team will not accept it (yet). rows='2000000', fl=['title', 'product_id_new']) return [{ 'title': result['title'], 'article_id': result['product_id_new'], 'release_date': result['last_release_date'] } for result in results['results']]
def delete_product(context, data_dict): # noinspection PyUnresolvedReferences """ Set the status of a record to 'Deleted' and remove all metadata associated with that record. This will make the productid available for reuse. This web service is presently being mocked (i.e. it will return a success if a valid productid is passed in, but the exact implementation is still being discussed.) :param productId: product id of record to be deleted :type productId: str :param issueno: :return: success or failure :rtype: dict """ product_id = _get_or_bust(data_dict, 'productId') lc = ckanapi.LocalCKAN(context=context) response = lc.action.package_search( q='product_id_new:{pid}'.format(pid=product_id), rows=1, fl=['id', 'type'] ) if response['count']: result = response['results'][0] lc.action.PurgeDataset(id=result['id']) return { 'message': 'Product successfully deleted', 'product_id_new': product_id }
def get_datapreview_recombinant(resource_name, res_id): from ckanext.recombinant.tables import get_chromo chromo = get_chromo(resource_name) default_preview_args = {} lc = ckanapi.LocalCKAN(username=c.user) results = lc.action.datastore_search( resource_id=res_id, limit=0, ) priority = len(chromo['datastore_primary_key']) pk_priority = 0 fields = [] for f in chromo['fields']: out = { 'type': f['datastore_type'], 'id': f['datastore_id'], 'label': h.recombinant_language_text(f['label'])} if out['id'] in chromo['datastore_primary_key']: out['priority'] = pk_priority pk_priority += 1 else: out['priority'] = priority priority += 1 fields.append(out) return h.snippet('package/wet_datatable.html', resource_name=resource_name, resource_id=res_id, ds_fields=fields)
def get_upcoming_releases(context, data_dict): # noinspection PyUnresolvedReferences """ Return all records with a publish_status_code of Verified (08) and a release date between the two parameters. :param startDate: Beginning of date range :param endDate: End of date range :returns: All matching results. :rtype: list of dicts """ # TODO: date validation? anything else? context['ignore_capacity_check'] = True lc = ckanapi.LocalCKAN(context=context) start_date = to_utc(_get_or_bust(data_dict, 'startDate'), def_date=default_release_date) end_date = to_utc(_get_or_bust(data_dict, 'endDate'), def_date=default_release_date) result = lc.action.package_search( q=( 'last_release_date:[{start_date}Z TO {end_date}Z] ' 'AND publish_status_code:8' ).format( start_date=start_date, end_date=end_date ), rows=500 ) # Per JIRA #5173, return an empty list instead of the standard # NotFound Exception (404). results = result.get('results', []) # Per JIRA #5206, resolve parent product types and URLs. for release_result in results: if not release_result.get('parent_id'): continue parent_result = lc.action.package_search( q='product_id_new:{pid}'.format(pid=release_result['parent_id']), rows=1 ) if parent_result['results']: parent_result = parent_result['results'][0] else: continue release_result.update({ 'title': parent_result.get('title'), 'url': parent_result.get('url'), 'product_type': parent_result.get('type') }) return {'count': result['count'], 'results': results}
def recombinant_show_package(pkg): """ return recombinant_show results for pkg """ lc = ckanapi.LocalCKAN(username=c.user) return lc.action.recombinant_show( dataset_type=pkg['type'], owner_org=pkg['organization']['name'])
def test_local_fail(self): try: import ckan except ImportError: raise unittest.SkipTest('ckan not importable') self.assertRaises(ckanapi.CKANAPIError, ckanapi.LocalCKAN('fake').call_action, 'fake', {}, {}, 'apikey not allowed')
def get_datapreview_ati(res_id): lc = ckanapi.LocalCKAN(username=c.user) results = lc.action.datastore_search(resource_id=res_id, sort='year,month desc', limit=3000) return h.snippet('package/wet_datatable.html', ds_fields=results['fields'], ds_records=results['records'])
def _home_groups(): gps = [] ckan = ckanapi.LocalCKAN() groups = ckan.action.group_list() for g in groups: gps.append(ckan.action.group_show(id=g)) return gps
def next_article_id(top_parent_id, issue_number): """ Get next available product ID :param top_parent_id: :type top_parent_id: 8 digit str :param issue_number :type issue_number: 7 digit str :return: 19 or 20 digit str """ if not isinstance(top_parent_id, basestring) or len(top_parent_id) != 8: raise ValidationError( (_('Invalid top parent ID. Expected 8 digit string'), )) if not isinstance(issue_number, basestring) or len(issue_number) != 7: raise ValidationError( (_('Invalid issue number. Expected 7 digit string'), )) lc = ckanapi.LocalCKAN() # check whether issue dataset exists result = lc.action.package_search(q='{pid}{issue_number}'.format( pid=top_parent_id, issue_number=issue_number))['results'] if not result: raise ValidationError({ '{pid}{issue_number}'.format(pid=top_parent_id, issue_number=issue_number): 'Cannot create article, parent issue missing' }) i = 0 n = 1 article_sequence_number = 1 while i < n: results = lc.action.package_search( q=('type:article AND ' 'product_id_new:{top_parent_id}{issue_number}?*').format( top_parent_id=top_parent_id, issue_number=issue_number), sort='product_id_new ASC', rows=1000, start=i * 1000) if results['count'] == 0: return u'{top_parent_id}{issue_number}{sequence_number}'.format( top_parent_id=top_parent_id, issue_number=issue_number, sequence_number=unicode(article_sequence_number).zfill(5)) n = results['count'] / 1000.0 i += 1 for result in results['results']: old_id = int(result['product_id_new'][15:]) article_sequence_number = max(article_sequence_number, old_id) return (u'{top_parent_id}' '{issue_number}' '{sequence_number}').format( top_parent_id=top_parent_id, issue_number=issue_number, sequence_number=unicode(article_sequence_number + 1).zfill(5))
def get_product_url(context, data_dict): # noinspection PyUnresolvedReferences """ Return the fluent URL of the given format of the product. If no format is specified, return the url of the preferred (primary) format :param productId: :type productId: str :param formatCode: :type formatCode: str :return: dict :raises: NotFound """ product_id = _get_or_bust(data_dict, 'productId') format_code = data_dict.get('formatCode') lc = ckanapi.LocalCKAN(context=context) if format_code: results = lc.action.package_search( q='name:format-{product_id}_{format_code}'.format( product_id=product_id, format_code=format_code ).lower() ).get('results') if results: return results[0].get(u'url', {u'en': u'', u'fr': u''}) else: raise _NotFound('{product_id}: no format {format_code} ' 'found for product'.format( product_id=product_id, format_code=format_code )) else: results = lc.action.package_search( q='name:format-{product_id}_*'.format( product_id=product_id ).lower() ).get('results') if not results: raise _NotFound( '{product_id}: no formats found for product'.format( product_id=product_id ) ) choices = scheming_helpers.scheming_get_preset( 'ndm_format' ).get('choices') sorted_choices = sorted(choices, key=lambda k: k.get('weight', '99')) for choice in sorted_choices: for result in results: if result.get(u'format_code') == choice['value']: return result.get(u'url', {u'en': u'', u'fr': u''}) return {u'en': u'', u'fr': u''}
def get_product(context, data_dict): """ Returns a product given its `productId`, if it exists. :param productId: product id (i.e. 2112002604) :type productId: str :return: product or 404 if not found. :rtype: dict :raises: ObjectNotFound, ValidationError """ product_id = _get_or_bust(data_dict, 'productId') lc = ckanapi.LocalCKAN(context=context) result = lc.action.package_search( q='product_id_new:{product_id}'.format( product_id=product_id ), rows=1 ) if not result['count']: raise _NotFound( ('product {product_id} not found'.format (product_id=product_id), ) ) # If we're getting more than one result for a given product_id # something has gone terribly wrong with the database. assert(not result['count'] > 1) product = result['results'][0] # As part of JIRA-5048 we need to resolve _code fields and return # the labels. codes_to_lookup = ( ('frequency_codes', 'frequency', 'codeset'), ('geolevel_codes', 'geolevel', 'codeset'), ('subject_codes', 'subjects', 'subject'), ('survey_source_codes', 'surveys', 'survey'), ('archive_status_code', 'archive_status', 'preset') ) for code_field, code_id, lookup_type in codes_to_lookup: cfv = product.get(code_field) if not cfv: continue codes = cfv if isinstance(cfv, list) else [cfv] product[code_field + '_resolved'] = results = {} for code in codes: value = stcndm_helpers.lookup_label(code_id, code, lookup_type) results[code] = value return product
def get_dguid_from_pkg_id(pkg_id): """ Given the package id, return the dguid of the package :param pkg_id: :return: """ lc = ckanapi.LocalCKAN() result = lc.action.package_show(**{u'id': pkg_id}) return result.get(u'geodescriptor_code')
def test_webhook_actions(self): ckan = ckanapi.LocalCKAN() callback = 'http://example.com/webhook_callback' hook = ckan.action.webhook_create(topic='dataset/create', address=callback) assert hook show = ckan.action.webhook_show(id=hook) assert show['id'] == hook delete = ckan.action.webhook_delete(id=hook) assert delete == hook
def insert_or_update_pkg(self, pkg_dict, upload=None): registry = ckanapi.LocalCKAN(username=self.username) allow_duplicates = tk.asbool( tk.config.get('ckanext.ddi.allow_duplicates', False) ) override_datasets = tk.asbool( tk.config.get('ckanext.ddi.override_datasets', False) ) visibility = pkg_dict.pop('visibility', 'restricted') try: existing_pkg = registry.call_action('package_show', pkg_dict) if not allow_duplicates and not override_datasets: raise ContentDuplicateError( 'Dataset already exists and duplicates are not allowed.' ) if override_datasets: pkg_dict.pop('id', None) pkg_dict.pop('name', None) existing_pkg.update(pkg_dict) pkg_dict = existing_pkg registry.call_action('package_update', pkg_dict) else: raise ckanapi.NotFound() except ckanapi.NotFound: pkg_dict.pop('id', None) pkg_dict['name'] = self._gen_new_name(pkg_dict['name']) registry.call_action('package_create', pkg_dict) if upload is not None: try: registry.call_action( 'resource_create', { 'package_id': pkg_dict['name'], 'upload': upload, 'name': 'DDI XML', 'format': 'xml', 'url': '', 'type': 'attachment', 'file_type': 'other', 'visibility': visibility } ) except Exception as e: raise UploadError( 'Could not upload file: %s' % str(e) ) log.debug(pkg_dict['name']) return pkg_dict['name']
def sync_ckan(target, since_ts=None, before_ts=None): """ Sync this CKAN instance with a remote CKAN instance. :param target: A RemoteCKAN instance. """ since_ts = since_ts or datetime.utcnow() - timedelta(days=1) before_ts = before_ts or datetime.utcnow() lc = ckanapi.LocalCKAN() for activity in changes_since(since_ts=since_ts, before_ts=before_ts): sync_dataset(lc, target, activity['object_id'])
def next_non_data_product_id(subject_code, product_type_code): """ Get next available product ID :param subject_code: :type subject_code: 2 digit str :param product_type_code: :type product_type_code: 2 digit str :return: """ valid_product_codes = ['20', '21', '22', '23', '25', '26'] if not isinstance(subject_code, basestring) or \ not re.match('^\d\d$', subject_code): raise ValidationError((_('Invalid subject code.'), )) if isinstance(product_type_code, basestring): if product_type_code not in valid_product_codes: error_message = 'Invalid product type code. ' \ 'Expected one of {codes!r}'.format( codes=valid_product_codes, ) raise ValidationError((_(error_message), )) i = 0 n = 1 product_sequence_number = 1 while i < n: lc = ckanapi.LocalCKAN() results = lc.action.package_search( q='product_id_new:{subject_code}{product_type_code}????'.format( subject_code=subject_code, product_type_code=product_type_code), sort='product_id_new ASC', rows=1000, start=i * 1000) n = results['count'] / 1000.0 i += 1 for result in results['results']: if product_sequence_number < int(result['product_id_new'][5:8]): return ( u'{subject_code}{product_type_code}{sequence_number}' ).format( subject_code=subject_code, product_type_code=product_type_code, sequence_number=unicode(product_sequence_number).zfill(4)) else: product_sequence_number += 1 return u'{subject_code}{product_type_code}{sequence_number}'.format( subject_code=subject_code, product_type_code=product_type_code, sequence_number=unicode(product_sequence_number).zfill(4))
def patch_datasets(self, patch_filepath): import ckanapi registry = ckanapi.LocalCKAN() dataset_names = [] with open(patch_filepath) as f: for line in f.readlines(): if not line.strip(): continue print line dataset = json.loads(line) dataset_names.append(dataset.get('name') or dataset.get('id')) registry.action.package_patch(**dataset) print 'Patched: ' + ' '.join(dataset_names)