def test_successful_reimport(self): '''If all is good and the FIS-Broker service returns a record, return an HTTP 200.''' fb_dataset_dict, source, job = self._harvester_setup( FISBROKER_HARVESTER_CONFIG) job.status = u'Finished' job.save() package_update(self.context, fb_dataset_dict) package_id = fb_dataset_dict['id'] package = Package.get(package_id) old_title = package.title response = self.app.get( url="/api/harvest/reimport?id={}".format(package_id), headers={'Accept': 'application/json'}, extra_environ={ 'REMOTE_USER': self.context['user'].encode('ascii') }) # assert successful HTTP response _assert_equal(response.status_int, 200) content = json.loads(response.body) # assert success marker in resonse JSON assert content['success'] # assert that title has changed to the correct value (i.e., the reimport has actually happened) _assert_equal( package.title, u"NÀhrstoffversorgung des Oberbodens 2015 (Umweltatlas) - [WFS]") _assert_not_equal(package.title, old_title)
def test_reimport_invalid_dataset_triggers_deletion(self): """If a previously harvested dataset is reimported, and the reimport results in an ERROR_DURING_IMPORT, the package should have its state changed to deleted.""" fb_dataset_dict, source, job = self._harvester_setup( FISBROKER_HARVESTER_CONFIG, fb_guid=INVALID_GUID) job.status = u'Finished' job.save() package_update(self.context, fb_dataset_dict) package_id = fb_dataset_dict['id'] response = self.app.get( url="/api/harvest/reimport?id={}".format(package_id), headers={'Accept': 'application/json'}, extra_environ={ 'REMOTE_USER': self.context['user'].encode('ascii') }) # assert successful HTTP response _assert_equal(response.status_int, 200) content = json.loads(response.body) # assert failure marker in response JSON assert not content['success'] _assert_equal(content['error']['code'], ERROR_DURING_IMPORT) package = Package.get(package_id) _assert_equal(package.state, 'deleted')
def test_reimport_browser_triggers_redirect(self): '''If the reimport is triggered via the Browser (HTML is requested), the response should be a 302 redirect to a specific URL.''' # unsuccessful request, /dataset/dunk does not exist: response = self.app.get(url='/dataset/dunk/reimport', headers={'Accept': 'text/html'}, expect_errors=True, extra_environ={ 'REMOTE_USER': self.context['user'].encode('ascii') }) url = urlparse(response.location) _assert_equal(response.status_int, 302) _assert_equal(url.path, "/dataset/dunk") # successful request: fb_dataset_dict, source, job = self._harvester_setup( FISBROKER_HARVESTER_CONFIG) job.status = u'Finished' job.save() package_update(self.context, fb_dataset_dict) package_id = fb_dataset_dict['id'] response = self.app.get(url='/dataset/{}/reimport'.format(package_id), headers={'Accept': 'text/html'}, expect_errors=True, extra_environ={ 'REMOTE_USER': self.context['user'].encode('ascii') }) url = urlparse(response.location) _assert_equal(response.status_int, 302) _assert_equal(url.path, "/dataset/{}".format(package_id))
def package_update_minimal(context, data_dict): '''Update a dataset (package). You must be authorized to edit the dataset and the groups that it belongs to. It is recommended to call :py:func:`ckan.logic.action.get.package_show`, make the desired changes to the result, and then call ``package_update()`` with it. Plugins may change the parameters of this function depending on the value of the dataset's ``type`` attribute, see the :py:class:`~ckan.plugins.interfaces.IDatasetForm` plugin interface. For further parameters see :py:func:`~ckan.logic.action.create.package_create`. :param id: the name or id of the dataset to update :type id: string :returns: the updated dataset (if ``'return_package_dict'`` is ``True`` in the context, which is the default. Otherwise returns just the dataset id) :rtype: dictionary ''' setup() package= '' fulltext = '' old_fulltext = '' if data_dict.has_key('extras'): contains = _contains_key(data_dict['extras'], 'full_text_search') if(contains): fulltext = contains data_dict = _del_extra_field_from_dict(data_dict, 'full_text_search') package = update.package_update(context, data_dict) old_fulltext = None if package.has_key('id'): old_fulltext = Session.query(PackageFulltext) \ .filter(PackageFulltext.package_id==package['id']) \ .first() fulltext_dict_save(fulltext, old_fulltext, package, context) else: package = update.package_update(context, data_dict) else: package = update.package_update(context, data_dict) if check_logged_in(context): fulltext = _get_fulltext(package['id']) if fulltext: fulltext_dict = { 'key': 'full_text_search', 'value': fulltext.text } package['extras'].append(fulltext_dict) return package minimal_package = _del_extra_field_from_dict(package) minimal_package = _del_main_field_from_dict(minimal_package) return minimal_package
def package_update_minimal(context, data_dict): '''Update a dataset (package). You must be authorized to edit the dataset and the groups that it belongs to. It is recommended to call :py:func:`ckan.logic.action.get.package_show`, make the desired changes to the result, and then call ``package_update()`` with it. Plugins may change the parameters of this function depending on the value of the dataset's ``type`` attribute, see the :py:class:`~ckan.plugins.interfaces.IDatasetForm` plugin interface. For further parameters see :py:func:`~ckan.logic.action.create.package_create`. :param id: the name or id of the dataset to update :type id: string :returns: the updated dataset (if ``'return_package_dict'`` is ``True`` in the context, which is the default. Otherwise returns just the dataset id) :rtype: dictionary ''' setup() package = '' fulltext = '' old_fulltext = '' if data_dict.has_key('extras'): contains = _contains_key(data_dict['extras'], 'full_text_search') if (contains): fulltext = contains data_dict = _del_extra_field_from_dict(data_dict, 'full_text_search') package = update.package_update(context, data_dict) old_fulltext = None if package.has_key('id'): old_fulltext = Session.query(PackageFulltext) \ .filter(PackageFulltext.package_id==package['id']) \ .first() fulltext_dict_save(fulltext, old_fulltext, package, context) else: package = update.package_update(context, data_dict) else: package = update.package_update(context, data_dict) if check_logged_in(context): fulltext = _get_fulltext(package['id']) if fulltext: fulltext_dict = {'key': 'full_text_search', 'value': fulltext.text} package['extras'].append(fulltext_dict) return package minimal_package = _del_extra_field_from_dict(package) minimal_package = _del_main_field_from_dict(minimal_package) return minimal_package
def pkg_update(context, data_dict): # Here we are doing custom update for only 2 dataset types ('model-package' and 'model-configuration') # These 2 dataset types have metadata elements that are populated by the system only. That's why this custom # update is needed in order to be able to edit these 2 types of datasets as some of the metadata elements are not # exposed to the UI for user to update since there are supposed to be populated by the system. log.debug('my very own package_update() called') # get the already saved package orig_pkg = p.toolkit.get_action('package_show')(context, data_dict) # find the type pf package/dataset pkg_type = orig_pkg.get('type', None) if pkg_type is None: return package_update(context, data_dict) if pkg_type == 'model-package': if data_dict.get('package_run_status', None) is None: if orig_pkg.get('package_run_status', None): data_dict['package_run_status'] = orig_pkg[ 'package_run_status'] else: data_dict['package_run_status'] = 'Not yet submitted' if data_dict.get('package_run_job_id', None) is None: if orig_pkg.get('package_run_job_id', None): data_dict['package_run_job_id'] = orig_pkg[ 'package_run_job_id'] else: data_dict['package_run_job_id'] = ' ' elif pkg_type == 'model-configuration': if data_dict.get('model_name', None) is None: data_dict['model_name'] = orig_pkg['model_name'] if data_dict.get('processing_status', None) is None: data_dict['processing_status'] = orig_pkg['processing_status'] if data_dict.get('package_build_request_job_id', None) is None: if orig_pkg.get('package_build_request_job_id', None): data_dict['package_build_request_job_id'] = orig_pkg[ 'package_build_request_job_id'] else: data_dict['package_build_request_job_id'] = '' if data_dict.get('package_availability', None) is None: if orig_pkg.get('package_availability', None): data_dict['package_availability'] = orig_pkg[ 'package_availability'] else: data_dict['package_availability'] = 'Not available' return package_update(context, data_dict)
def test_raise_error_for_failed_getrecordbyid_request(self): '''A reimport should fail with a NoConnectionError when the getrecordbyid-Request failed with a timeout.''' fb_controller = controller.FISBrokerController() fb_dataset_dict, source, job = self._harvester_setup( FISBROKER_HARVESTER_CONFIG, fb_guid='cannot_connect') package_update(self.context, fb_dataset_dict) package_id = fb_dataset_dict['id'] package_ids = [package_id] with assert_raises(NoConnectionError): fb_controller.reimport_batch(package_ids, self.context)
def pkg_update(context, data_dict): # Here we are doing custom update for only 2 dataset types ('model-package' and 'model-configuration') # These 2 dataset types have metadata elements that are populated by the system only. That's why this custom # update is needed in order to be able to edit these 2 types of datasets as some of the metadata elements are not # exposed to the UI for user to update since there are supposed to be populated by the system. log.debug('my very own package_update() called') # get the already saved package orig_pkg = p.toolkit.get_action('package_show')(context, data_dict) # find the type pf package/dataset pkg_type = orig_pkg.get('type', None) if pkg_type is None: return package_update(context, data_dict) if pkg_type == 'model-package': if data_dict.get('package_run_status', None) is None: if orig_pkg.get('package_run_status', None): data_dict['package_run_status'] = orig_pkg['package_run_status'] else: data_dict['package_run_status'] = 'Not yet submitted' if data_dict.get('package_run_job_id', None) is None: if orig_pkg.get('package_run_job_id', None): data_dict['package_run_job_id'] = orig_pkg['package_run_job_id'] else: data_dict['package_run_job_id'] = ' ' elif pkg_type == 'model-configuration': if data_dict.get('model_name', None) is None: data_dict['model_name'] = orig_pkg['model_name'] if data_dict.get('processing_status', None) is None: data_dict['processing_status'] = orig_pkg['processing_status'] if data_dict.get('package_build_request_job_id', None) is None: if orig_pkg.get('package_build_request_job_id', None): data_dict['package_build_request_job_id'] = orig_pkg['package_build_request_job_id'] else: data_dict['package_build_request_job_id'] = '' if data_dict.get('package_availability', None) is None: if orig_pkg.get('package_availability', None): data_dict['package_availability'] = orig_pkg['package_availability'] else: data_dict['package_availability'] = 'Not available' return package_update(context, data_dict)
def test_reimport_anonymously_fails(self): '''Only a logged in user can initiate a successful reimport, so anonymous access should raise an authorization error.''' fb_dataset_dict, source, job = self._harvester_setup( FISBROKER_HARVESTER_CONFIG) job.status = u'Finished' job.save() package_update(self.context, fb_dataset_dict) package_id = fb_dataset_dict['id'] with assert_raises(NotAuthorized): self.app.get(url='/dataset/{}/reimport'.format(package_id), headers={'Accept': 'text/html'}, expect_errors=True)
def package_patch(context, data_dict): '''Patch a dataset (package). :param id: the id or name of the dataset :type id: string The difference between the update and patch methods is that the patch will perform an update of the provided parameters, while leaving all other parameters unchanged, whereas the update methods deletes all parameters not explicitly provided in the data_dict You must be authorized to edit the dataset and the groups that it belongs to. ''' _check_access('package_patch', context, data_dict) show_context = { 'model': context['model'], 'session': context['session'], 'user': context['user'], 'auth_user_obj': context['auth_user_obj'], } package_dict = _get_action('package_show')( show_context, {'id': _get_or_bust(data_dict, 'id')}) patched = dict(package_dict) patched.update(data_dict) patched['id'] = package_dict['id'] return _update.package_update(context, patched)
def package_update(context, data_dict): # The only thing we do here is remove some extras that are always # inherited from the dataset publisher, to avoid duplicating them _remove_extras_from_data_dict(data_dict) return update_core.package_update(context, data_dict)
def package_update(context, data_dict): userobj = toolkit.c.userobj # Decide if we need notification # - deposited-datset AND # - not a test env AND # - just published notify = False if data_dict.get('type') == 'deposited-dataset' and hasattr(userobj, 'id'): dataset = toolkit.get_action('package_show')(context, { 'id': data_dict['id'] }) if dataset.get('state') == 'draft' and data_dict.get( 'state') == 'active': notify = True # Update dataset dataset = update_core.package_update(context, data_dict) # Send notification if needed if notify: dataset['url'] = toolkit.url_for('dataset_read', id=dataset.get('name'), qualified=True) curation = helpers.get_deposited_dataset_user_curation_status( dataset, userobj.id) subj = mailer.compose_curation_email_subj(dataset) body = mailer.compose_curation_email_body(dataset, curation, userobj.display_name, 'deposit') mailer.mail_user_by_id(userobj.id, subj, body) return dataset
def package_update_rest_minimal(context, data_dict): setup() package = '' fulltext = '' old_fulltext = '' if data_dict.has_key('extras'): if 'full_text_search' in data_dict['extras'].keys(): fulltext = data_dict['extras']['full_text_search'] data_dict = _del_extra_field_from_list(data_dict, 'full_text_search') package = update.package_update_rest(context, data_dict) old_fulltext = None if package.has_key('id'): old_fulltext = Session.query(PackageFulltext) \ .filter(PackageFulltext.package_id==package['id']) \ .first() fulltext_dict_save(fulltext, old_fulltext, package, context) else: package = update.package_update(context, data_dict) else: package = update.package_update_rest(context, data_dict) if check_logged_in(context): fulltext = _get_fulltext(package['id']) if fulltext: package['extras']['full_text_search'] = fulltext.text return package minimal_package = _del_extra_field_from_list(package) minimal_package = _del_main_field_from_dict(minimal_package) return minimal_package
def test_resource_list(self): # TODO restore this test. It doesn't make much sense with the # present resource list design. name = 'annakarenina' cache_url = 'http://thedatahub.org/test_cache_url.csv' # add a cache_url to the first resource in the package context = {'model': model, 'session': model.Session, 'user': '******'} data = {'id': 'annakarenina'} pkg = get.package_show(context, data) pkg['resources'][0]['cache_url'] = cache_url # FIXME need to pretend to be called by the api context['api_version'] = 3 update.package_update(context, pkg) # check that the cache url is included on the dataset view page offset = url_for(controller='package', action='read', id=name) res = self.app.get(offset)
def package_patch(context, data_dict): '''Patch a dataset (package). :param id: the id or name of the dataset :type id: string The difference between the update and patch methods is that the patch will perform an update of the provided parameters, while leaving all other parameters unchanged, whereas the update methods deletes all parameters not explicitly provided in the data_dict You must be authorized to edit the dataset and the groups that it belongs to. ''' _check_access('package_patch', context, data_dict) show_context = { 'model': context['model'], 'session': context['session'], 'user': context['user'], 'auth_user_obj': context['auth_user_obj'], } package_dict = _get_action('package_show')( show_context, { 'id': _get_or_bust(data_dict, 'id') }) patched = dict(package_dict) patched.update(data_dict) patched['id'] = package_dict['id'] return _update.package_update(context, patched)
def package_update_rest_minimal(context, data_dict): setup() package= '' fulltext = '' old_fulltext = '' if data_dict.has_key('extras'): if 'full_text_search' in data_dict['extras'].keys(): fulltext = data_dict['extras']['full_text_search'] data_dict = _del_extra_field_from_list(data_dict, 'full_text_search') package = update.package_update_rest(context, data_dict) old_fulltext = None if package.has_key('id'): old_fulltext = Session.query(PackageFulltext) \ .filter(PackageFulltext.package_id==package['id']) \ .first() fulltext_dict_save(fulltext, old_fulltext, package, context) else: package = update.package_update(context, data_dict) else: package = update.package_update_rest(context, data_dict) if check_logged_in(context): fulltext = _get_fulltext(package['id']) if fulltext: package['extras']['full_text_search'] = fulltext.text return package minimal_package = _del_extra_field_from_list(package) minimal_package = _del_main_field_from_dict(minimal_package) return minimal_package
def drupal_package_update(self, context, data_dict): session = context['model'].Session context['nid'] = data_dict.pop('nid') context['extras_as_string'] = True package_update = update.package_update(context, data_dict) package_update['nid'] = context['nid'] package_update['revision_message'] = '%s-%s'%(session.revision.id,session.revision.message) return package_update
def dataset_update(context, data_dict=None): ret_val = package_update(context, data_dict) if not context.get('defer_commit'): log.debug("package_update sync dataset") queue.put(ret_val) return ret_val
def drupal_package_update(self, context, data_dict): session = context['model'].Session context['nid'] = data_dict.pop('nid') context['extras_as_string'] = True package_update = update.package_update(context, data_dict) package_update['nid'] = context['nid'] package_update['revision_message'] = '%s-%s' % ( session.revision.id, session.revision.message) return package_update
def drupal_package_update(context, data_dict): session = context['model'].Session context['nid'] = data_dict.pop('nid') context['vid'] = data_dict.pop('vid') package_create = update.package_update(context, data_dict) package_create['nid'] = context['nid'] package_create['vid'] = context['vid'] package_create['revision_message'] = '%s-%s'%(session.revision.id,session.revision.message) return package_create
def test_handle_not_found_fisbroker(self): '''If FIS-Broker service replies that no record with the given guid exisits, return an HTTP 404 with internal error code 9.''' fb_dataset_dict, source, job = self._harvester_setup( FISBROKER_HARVESTER_CONFIG, fb_guid='invalid_guid') package_update(self.context, fb_dataset_dict) package_id = fb_dataset_dict['id'] response = self.app.get( "/api/harvest/reimport?id={}".format(package_id), headers={'Accept': 'application/json'}, expect_errors=True, extra_environ={ 'REMOTE_USER': self.context['user'].encode('ascii') }) _assert_equal(response.status_int, 404) content = json.loads(response.body) _assert_equal(content['error']['code'], controller.ERROR_NOT_FOUND_IN_FISBROKER)
def test_reimport_batch_raises_no_connection_error(self): '''A batch reimport show trigger a NoConnectionError if a connection to FIS-Broker cannot be established.''' fb_controller = controller.FISBrokerController() unreachable_config = { 'title': 'Unreachable FIS-Broker Harvest Source', 'name': 'unreachable-fis-broker-harvest-source', 'source_type': HARVESTER_ID, 'url': "http://somewhere.over.the.ra.invalid/csw" } fb_dataset_dict, source, job = self._harvester_setup( unreachable_config) package_update(self.context, fb_dataset_dict) package_id = fb_dataset_dict['id'] package_ids = [package_id] with assert_raises(NoConnectionError): fb_controller.reimport_batch(package_ids, self.context)
def test_resource_list(self, app): # TODO restore this test. It doesn't make much sense with the # present resource list design. name = "annakarenina" cache_url = "http://thedatahub.org/test_cache_url.csv" # add a cache_url to the first resource in the package context = { "model": model, "session": model.Session, "user": "******", } data = {"id": "annakarenina"} pkg = get.package_show(context, data) pkg["resources"][0]["cache_url"] = cache_url # FIXME need to pretend to be called by the api context["api_version"] = 3 update.package_update(context, pkg) # check that the cache url is included on the dataset view page offset = url_for("dataset.read", id=name) res = app.get(offset)
def package_update(context, data_dict): mapped_resources = [] if 'resources' in data_dict: for resource in data_dict['resources']: mapped_resource = generate_mapping(context, resource) mapped_resources.append(mapped_resource) data_dict['resources'] = mapped_resources package_dict = update.package_update(context, data_dict) if package_dict['type'] == 'dataset': send_dataset_log(context, package_dict, 'Dataset updated', 'DatasetMetadataUpdated') update_de(package_dict) return package_dict
def test_handle_no_connection_to_fisbroker(self): '''If the FIS-Broker service cannot be reached, return an HTTP 500 with internal error code 8.''' unreachable_config = { 'title': 'Unreachable FIS-Broker Harvest Source', 'name': 'unreachable-fis-broker-harvest-source', 'source_type': HARVESTER_ID, 'url': "http://somewhere.over.the.ra.invalid/csw" } fb_dataset_dict, source, job = self._harvester_setup( unreachable_config) package_update(self.context, fb_dataset_dict) package_id = fb_dataset_dict['id'] response = self.app.get( "/api/harvest/reimport?id={}".format(package_id), headers={'Accept': 'application/json'}, expect_errors=True, extra_environ={ 'REMOTE_USER': self.context['user'].encode('ascii') }) _assert_equal(response.status_int, 500) content = json.loads(response.body) _assert_equal(content['error']['code'], controller.ERROR_NO_CONNECTION)
def package_update(self, context, data_dict): preview = context.get('preview', False) schema = context.get('schema') or default_update_package_schema() if preview: return if 'id' not in data_dict: raise NotFound result = self.engine.execute( select( [self.package_table.c.nid], or_(self.package_table.c.id == data_dict['id'], self.package_table.c.name == data_dict['id']) ) ).fetchone() if not result: raise NotFound nid = result['nid'] data_dict['body'] = data_dict['notes'] ## run through validate to make sure tags are in correct place data, errors = validate(data_dict, schema, context) terms = {} for num, tag in enumerate(data.get('tags', [])): terms[str(num)] = tag['name'] data_dict['terms'] = terms if data_dict.get('state', 'active') == 'active': data_dict['status'] = '1' else: data_dict['status'] = '0' url = urlparse.urljoin(self.base_url, 'services/package/%s.json' % (nid)) data = json.dumps({'data': data_dict}) req = urllib2.Request(url, data, {'Content-type': 'application/json'}) req.get_method = lambda: 'PUT' ##XXX think about error conditions a bit more f = urllib2.urlopen(req, None, 3) try: drupal_info = json.loads(f.read()) finally: f.close() session = context['model'].Session context['nid'] = result['nid'] package_update = update.package_update(context, data_dict) package_update['nid'] = result['nid'] package_update['revision_message'] = '%s-%s'%(session.revision.id,session.revision.message) return package_update
def package_update(self, context, data_dict): preview = context.get('preview', False) schema = context.get('schema') or default_update_package_schema() if preview: return if 'id' not in data_dict: raise NotFound result = self.engine.execute( select( [self.package_table.c.nid], or_(self.package_table.c.id == data_dict['id'], self.package_table.c.name == data_dict['id']))).fetchone() if not result: raise NotFound nid = result['nid'] data_dict['body'] = data_dict['notes'] ## run through validate to make sure tags are in correct place data, errors = validate(data_dict, schema, context) terms = {} for num, tag in enumerate(data.get('tags', [])): terms[str(num)] = tag['name'] data_dict['terms'] = terms if data_dict.get('state', 'active') == 'active': data_dict['status'] = '1' else: data_dict['status'] = '0' url = urlparse.urljoin(self.base_url, 'services/package/%s.json' % (nid)) data = json.dumps({'data': data_dict}) req = urllib2.Request(url, data, {'Content-type': 'application/json'}) req.get_method = lambda: 'PUT' ##XXX think about error conditions a bit more f = urllib2.urlopen(req, None, 3) try: drupal_info = json.loads(f.read()) finally: f.close() session = context['model'].Session context['nid'] = result['nid'] package_update = update.package_update(context, data_dict) package_update['nid'] = result['nid'] package_update['revision_message'] = '%s-%s' % ( session.revision.id, session.revision.message) return package_update
def showcase_update(context, data_dict): # If get_uploader is available (introduced for IUploader in CKAN 2.5), use # it, otherwise use the default uploader. # https://github.com/ckan/ckan/pull/2510 try: upload = uploader.get_uploader('showcase', data_dict['image_url']) except AttributeError: upload = uploader.Upload('showcase', data_dict['image_url']) upload.update_data_dict(data_dict, 'image_url', 'image_upload', 'clear_upload') upload.upload(uploader.get_max_image_size()) pkg = package_update(context, data_dict) #pkg = toolkit.get_action('package_update')(context, data_dict) return pkg
def package_patch(context: Context, data_dict: DataDict) -> ActionResult.PackagePatch: '''Patch a dataset (package). :param id: the id or name of the dataset :type id: string The difference between the update and patch methods is that the patch will perform an update of the provided parameters, while leaving all other parameters unchanged, whereas the update methods deletes all parameters not explicitly provided in the data_dict. You are able to partially update and/or create resources with package_patch. If you are updating existing resources be sure to provide the resource id. Existing resources excluded from the package_patch data_dict will be removed. Resources in the package data_dict without an id will be treated as new resources and will be added. New resources added with the patch method do not create the default views. You must be authorized to edit the dataset and the groups that it belongs to. ''' _check_access('package_patch', context, data_dict) show_context: Context = { 'model': context['model'], 'session': context['session'], 'user': context['user'], 'auth_user_obj': context['auth_user_obj'], 'ignore_auth': context.get('ignore_auth', False), 'for_update': True } package_dict = _get_action('package_show')( show_context, { 'id': _get_or_bust(data_dict, 'id') }) patched = dict(package_dict) patched.update(data_dict) patched['id'] = package_dict['id'] return _update.package_update(context, patched)
def package_update(context, data_dict): # The only thing we do here is remove some extras that are always # inherited from the dataset publisher, to avoid duplicating them _remove_extras_from_data_dict(data_dict) old_package_dict = get_core.package_show( context, {"id": data_dict.get('id') or data_dict.get('name')}) updated_package = update_core.package_update(context, data_dict) # Part of first publisher date patch if 'owner_org' in data_dict: hlp.first_published_date_patch(updated_package.get('owner_org')) if old_package_dict.get('private') and not updated_package.get('private') and \ updated_package.get('satte') != "deleted": # Data is published send an email package_title = updated_package.get('title') or updated_package.get( 'name') send_data_published_notification(context, updated_package.get('owner_org', ''), package_title) return updated_package
def package_patch(context, data_dict): '''Patch a dataset (package). :param id: the id or name of the dataset :type id: string The difference between the update and patch methods is that the patch will perform an update of the provided parameters, while leaving all other parameters unchanged, whereas the update methods deletes all parameters not explicitly provided in the data_dict. You are able to partially update and/or create resources with package_patch. If you are updating existing resources be sure to provide the resource id. Existing resources excluded from the package_patch data_dict will be removed. Resources in the package data_dict without an id will be treated as new resources and will be added. New resources added with the patch method do not create the default views. You must be authorized to edit the dataset and the groups that it belongs to. ''' _check_access('package_patch', context, data_dict) show_context = { 'model': context['model'], 'session': context['session'], 'user': context['user'], 'auth_user_obj': context['auth_user_obj'], } package_dict = _get_action('package_show')( show_context, {'id': _get_or_bust(data_dict, 'id')}) patched = dict(package_dict) patched.update(data_dict) patched['id'] = package_dict['id'] return _update.package_update(context, patched)
def update_groups_for_all_datasets(): # This job re-calculate all harvested dataset groups from scratch # It makes actual update call only on changed packages time.sleep(3) # Get groups groups = helpers.get_groups_with_extras() # Get packages offset = 0 limit = 1000 packages = [] while True: page = toolkit.get_action('package_search')( { 'model': model }, { 'start': offset, 'rows': limit, 'fq': 'dataset_type:dataset', })['results'] if not page: break packages.extend(page) offset += limit # Update packages for package in packages: old_group_names = _extract_group_names(package) package = tagging.recalculate_dataset_groups(package, groups=groups) new_group_names = _extract_group_names(package) if old_group_names != new_group_names: # We don't want to recalculate groups twice # (see ckanext.lacounts.logic.actions.package_create/udpate) context = {'model': model, 'user': toolkit.c.user} package = update_core.package_update(context, package) log.debug('Updated groups for package: %s' % package['name'])
def test_user_activity(self): """Test user activity streams HTML rendering.""" # Register a new user. user_dict = {'name': 'billybeane', 'fullname': 'Billy Beane', 'about': 'General Manager, Oakland Athletics', 'email': '*****@*****.**', 'password': '******'} context = { 'model': ckan.model, 'session': ckan.model.Session, 'user': self.sysadmin_user.name, 'allow_partial_update': True, } user = user_create(context, user_dict) offset = url_for('user.activity', id=user['id']) result = self.app.get(offset, status=200) stripped = self.strip_tags(result) assert '%s signed up' % user['fullname'] in stripped, stripped # Create a new package. package = { 'name' : 'baseball_stats', 'title' : "Billy's Stats about Baseball Players", } context['user'] = user['name'] # FIXME This test use an old way to get at the schema to # recreate this we need to pretend to be using the api. We # should not be calling package_create like this we should be # going via the api or package controllers context['api_version'] = 3 context['ignore_auth'] = True package = package_create(context, package) result = self.app.get(offset, status=200) stripped = self.strip_tags(result) assert '%s created the dataset %s ' % ( user['fullname'], package['title']) in stripped, stripped # Add a resource to the package. resource = { 'url': 'http://www.example.com', 'description': "Chad Bradford's OBP Stats`", 'format': 'cvs', 'name': 'Chad Bradford Stats', } package['resources'].append(resource) request_data = { 'id': package['id'], 'resources': package['resources'], } package = package_update(context, request_data) result = self.app.get(offset, status=200) stripped = self.strip_tags(result) assert '%s added the resource %s to the dataset %s' % \ (user['fullname'], resource['name'], package['title']) \ in stripped, stripped # Update the package. package['title'] = "Billy's Updated Stats about Baseball Players" package = package_update(context, package) result = self.app.get(offset, status=200) stripped = self.strip_tags(result) assert '%s updated the dataset %s' \ % (user['fullname'], package['title']) \ in stripped, stripped # Update the resource. resource = package['resources'][0] resource['name'] = 'Chad Bradford Updated Stats' resource = resource_update(context, resource) result = self.app.get(offset, status=200) stripped = self.strip_tags(result) assert '%s updated the resource %s in the dataset %s' \ % (user['fullname'], resource['name'], package['title']) \ in stripped, stripped # Delete the resource. context['allow_partial_update'] = False package['resources'] = [] package_update(context, package) result = self.app.get(offset, status=200) stripped = self.strip_tags(result) assert '%s deleted the resource %s from the dataset %s' % \ (user['fullname'], resource['name'], package['title']) \ in stripped, stripped # Follow the package. follow_dataset(context, {'id': package['id']}) result = self.app.get(offset, status=200) stripped = self.strip_tags(result) assert '%s started following %s' % (user['fullname'], package['title']) not in stripped, stripped # Follow another user. follow_user(context, {'id': 'joeadmin'}) result = self.app.get(offset, status=200) stripped = self.strip_tags(result) assert '%s started following %s' % (user['fullname'], 'joeadmin') not in stripped, stripped # Create a new group. group = { 'name': 'baseball-stats-group', 'title': 'A Group for Datasets about Baseball' } context['allow_partial_update'] = True group = group_create(context, group) result = self.app.get(offset, status=200) stripped = self.strip_tags(result) assert '%s created the group %s' % (user['fullname'], group['title']) \ in stripped, stripped # Update the group. group['title'] = 'updated' group = group_update(context, group) result = self.app.get(offset, status=200) stripped = self.strip_tags(result) assert '%s updated the group %s' % (user['fullname'], group['title']) \ in stripped, stripped # Delete the group. group['state'] = 'deleted' group_update(context, group) result = self.app.get(offset, status=200) stripped = self.strip_tags(result) assert '%s deleted the group %s' % (user['fullname'], group['title']) \ in stripped, stripped # Add a new tag to the package. tag = {'name': 'baseball'} package['tags'].append(tag) package = package_update(context, package) result = self.app.get(offset, status=200) stripped = self.strip_tags(result) assert '%s added the tag %s to the dataset %s' % \ (user['fullname'], tag['name'], package['title']) \ in stripped, stripped # Remove the tag from the package. package['tags'] = [] context['allow_partial_update'] = False package_update(context, package) result = self.app.get(offset, status=200) stripped = self.strip_tags(result) assert '%s removed the tag %s from the dataset %s' % \ (user['fullname'], tag['name'], package['title']) \ in stripped, stripped # Add an extra to the package. package['extras'].append({'key': 'quality', 'value': '10000'}) package = package_update(context, package) result = self.app.get(offset, status=200) stripped = self.strip_tags(result) assert '%s added the extra "%s" to the dataset %s' % \ (user['fullname'], 'quality', package['title']) \ in stripped, stripped # Update the extra. package['extras'][0]['value'] = 'updated' package = package_update(context, package) result = self.app.get(offset, status=200) stripped = self.strip_tags(result) assert '%s changed the extra "%s" of the dataset %s' % \ (user['fullname'], 'quality', package['title']) \ in stripped, stripped # Delete the extra. del package['extras'][0] package = package_update(context, package) result = self.app.get(offset, status=200) stripped = self.strip_tags(result) assert '%s deleted the extra "%s" from the dataset %s' % \ (user['fullname'], 'quality', package['title']) \ in stripped, stripped # Delete the package. # we need to get round the delete permission context['ignore_auth'] = True package_delete(context, package) del context['ignore_auth'] result = self.app.get(offset, status=200) stripped = self.strip_tags(result) assert '%s deleted the dataset %s' % \ (user['fullname'], package['title']) \ in stripped, stripped # Update the user's profile. user['about'] = '' user_update(context, user) result = self.app.get(offset, status=200) stripped = self.strip_tags(result) assert '%s updated their profile' % user['fullname'] \ in stripped, stripped # By now we've created >15 activities, but only the latest 15 should # appear on the page. result = self.app.get(offset, status=200) assert result.body.count('<span class="actor">') \ == 15, result.body.count('<span class="actor">') # The user's dashboard page should load successfully and have the # latest 15 activities on it. offset = url_for('dashboard.index') extra_environ = {'Authorization': str(ckan.model.User.get('billybeane').apikey)} result = self.app.get(offset, extra_environ=extra_environ, status=200) assert result.body.count('<span class="actor">') == 15, \ result.body.count('<span class="actor">')
def package_update(context, data_dict): if data_dict.get('type') == 'dataset': data_dict = _package_create_or_update(data_dict) return update_core.package_update(context, data_dict)
def pkg_update(context, data_dict): log.debug('my very own package_update() called') # turning context 'validate' key on/off to allow schema changes to work with existing dataset context['validate'] = False origpkg = p.toolkit.get_action('package_show')(context, data_dict) context['validate'] = True # this is needed when adding a resource to an existing dataset if context.get('save', None) is None: for extra in origpkg['extras']: if data_dict.get(extra['key'], None) is None: data_dict[extra['key']] = extra['value'] #get name of the author to use in citation author = origpkg.get('author', None) # get the name of the submitter to use in citation if author is not available sub_name = None sub_email = None submitter_dict = [extra for extra in origpkg['extras'] if extra['key'] == 'sub_name' or extra['key'] == 'sub_email'] for extra in submitter_dict: if extra['key'] == 'sub_name': sub_name = extra['value'] if extra['key'] == 'sub_email': sub_email = extra['value'] if not sub_name: context['return_minimal'] = True # turning context 'validate' key on/off to allow schema changes to work with existing dataset context['validate'] = False user = p.toolkit.get_action('user_show')(context, {'id': context['user']}) context['validate'] = True data_dict['sub_name'] = user['fullname'] data_dict['sub_email'] = user['email'] else: data_dict['sub_name'] = sub_name data_dict['sub_email'] = sub_email # TODO: may be we do not need the original CKAN author information if not author: data_dict['author'] = data_dict['sub_name'] data_dict['author_email'] = data_dict['sub_email'] data_dict['version'] = u'1.0' data_dict['license_id'] = u'cc-by' dateval = origpkg['metadata_created'] year = dateval.split("-")[0] if origpkg['state'] != 'active': if data_dict.get('author', None): data_dict['citation'] = createcitation(context, data_dict, year) else: data_dict['citation'] = u'' else: data_dict['citation'] = createcitation(context, data_dict, year) context['validate'] = False # This was added to allow creation metadata only dataset (dataset without resources) # Here we are deleting our dummy resource if it exists if origpkg['state'] == 'draft' or origpkg['state'] == 'draft-complete': if data_dict.get('resources', None): if len(data_dict['resources']) > 0: dummy_resource = data_dict['resources'][0] if dummy_resource.get('dummy_resource', None): del data_dict['resources'][0] elif origpkg.get('resources', None): if len(origpkg['resources']) > 0: dummy_resource = origpkg['resources'][0] if dummy_resource.get('dummy_resource', None): del origpkg['resources'][0] iutahorg = p.toolkit.get_action('organization_show')(context, {'id': 'iutah'}) if not data_dict.get('owner_org', None): data_dict['owner_org'] = origpkg['owner_org'] data_dict['private'] = origpkg['private'] else: if data_dict['owner_org'] == iutahorg['id']: data_dict['private'] = origpkg['private'] # remove if there any deleted repeatable elements from the data_dict _remove_deleted_repeatable_elements(data_dict, 'creators') _remove_deleted_repeatable_elements(data_dict, 'contributors') _remove_deleted_repeatable_elements(data_dict, 'variables') # remove any invalid variables _remove_invalid_variables(data_dict) # add tag names to the tag_string element if 'tag_string' is missing from the data_dict # needed to make the entry of one tag (keyword) as required if not 'tag_string' in data_dict.keys(): tags = ','.join(tag['name'] for tag in data_dict['tags']) data_dict['tag_string'] = tags return package_update(context, data_dict)
def test_activity(self): """Test activity streams HTML rendering.""" # Register a new user. user_dict = {'name': 'billybeane', 'fullname': 'Billy Beane', 'about': 'General Manager, Oakland Athletics', 'email': '*****@*****.**', 'password': '******'} context = { 'model': ckan.model, 'session': ckan.model.Session, 'user': self.sysadmin_user.name, 'allow_partial_update': True, 'extras_as_string': True, } user = user_create(context, user_dict) offset = url_for(controller='user', action='read', id=user['id']) result = self.app.get(offset, status=200) stripped = self.strip_tags(result) assert '%s signed up.' % user['fullname'] in stripped, stripped # Create a new package. package = { 'name' : 'baseball_stats', 'title' : "Billy's Stats about Baseball Players", } context['user'] = user['name'] package = package_create(context, package) result = self.app.get(offset, status=200) stripped = self.strip_tags(result) assert '%s created the dataset %s ' % ( user['fullname'], package['title']) in stripped, stripped # Add a resource to the package. resource = { 'url': 'http://www.example.com', 'description': "Chad Bradford's OBP Stats`", 'format': 'cvs', 'name': 'Chad Bradford Stats', } package['resources'].append(resource) request_data = { 'id': package['id'], 'resources': package['resources'], } package = package_update(context, request_data) result = self.app.get(offset, status=200) stripped = self.strip_tags(result) assert '%s added the resource %s to the dataset %s' % \ (user['fullname'], resource['name'], package['title']) \ in stripped, stripped # Update the package. package['title'] = "Billy's Updated Stats about Baseball Players" package = package_update(context, package) result = self.app.get(offset, status=200) stripped = self.strip_tags(result) assert '%s updated the dataset %s' \ % (user['fullname'], package['title']) \ in stripped, stripped # Update the resource. resource = package['resources'][0] resource['name'] = 'Chad Bradford Updated Stats' resource = resource_update(context, resource) result = self.app.get(offset, status=200) stripped = self.strip_tags(result) assert '%s updated the resource %s in the dataset %s' \ % (user['fullname'], resource['name'], package['title']) \ in stripped, stripped # Delete the resource. context['allow_partial_update'] = False package['resources'] = [] package_update(context, package) result = self.app.get(offset, status=200) stripped = self.strip_tags(result) assert '%s deleted the resource %s from the dataset %s' % \ (user['fullname'], resource['name'], package['title']) \ in stripped, stripped # Create a new group. group = { 'name': 'baseball-stats-group', 'title': 'A Group for Datasets about Baseball' } context['allow_partial_update'] = True group = group_create(context, group) result = self.app.get(offset, status=200) stripped = self.strip_tags(result) assert '%s created the group %s' % (user['fullname'], group['name']) \ in stripped, stripped # Update the group. group['title'] = 'updated' group = group_update(context, group) result = self.app.get(offset, status=200) stripped = self.strip_tags(result) assert '%s updated the group %s' % (user['fullname'], group['name']) \ in stripped, stripped # Delete the group. group['state'] = 'deleted' group_update(context, group) result = self.app.get(offset, status=200) stripped = self.strip_tags(result) assert '%s deleted the group %s' % (user['fullname'], group['name']) \ in stripped, stripped # Add a new tag to the package. tag = {'name': 'baseball'} package['tags'].append(tag) package = package_update(context, package) result = self.app.get(offset, status=200) stripped = self.strip_tags(result) assert '%s added the tag %s to the dataset %s' % \ (user['fullname'], tag['name'], package['title']) \ in stripped, stripped # Remove the tag from the package. package['tags'] = [] context['allow_partial_update'] = False package_update(context, package) result = self.app.get(offset, status=200) stripped = self.strip_tags(result) assert '%s removed the tag %s from the dataset %s' % \ (user['fullname'], tag['name'], package['title']) \ in stripped, stripped # Add an extra to the package. package['extras'].append({'key': 'quality', 'value': '10000'}) package = package_update(context, package) result = self.app.get(offset, status=200) stripped = self.strip_tags(result) assert '%s added the extra "%s" to the dataset %s' % \ (user['fullname'], 'quality', package['title']) \ in stripped, stripped # Update the extra. package['extras'][0]['value'] = 'updated' package = package_update(context, package) result = self.app.get(offset, status=200) stripped = self.strip_tags(result) assert '%s changed the extra "%s" of the dataset %s' % \ (user['fullname'], 'quality', package['title']) \ in stripped, stripped # Delete the extra. del package['extras'][0] package = package_update(context, package) result = self.app.get(offset, status=200) stripped = self.strip_tags(result) assert '%s deleted the extra "%s" from the dataset %s' % \ (user['fullname'], 'quality', package['title']) \ in stripped, stripped # Delete the package. package_delete(context, package) result = self.app.get(offset, status=200) stripped = self.strip_tags(result) assert '%s deleted the dataset %s' % \ (user['fullname'], package['title']) \ in stripped, stripped # Update the user's profile. user['about'] = '' user_update(context, user) result = self.app.get(offset, status=200) stripped = self.strip_tags(result) assert '%s updated their profile.' % user['fullname'] \ in stripped, stripped # By now we've created >15 activities, but only the latest 15 should # appear on the page. result = self.app.get(offset, status=200) assert result.body.count('<div class="activity">') \ == 15
def SaerischemaPlugin_package_update(context, data_dict): data_dict = SaerischemaPlugin_add_group_to_package_based_on_topic_category( context, data_dict) return package_update(context, data_dict)
def test_user_activity(self): """Test user activity streams HTML rendering.""" # Register a new user. user_dict = { "name": "billybeane", "fullname": "Billy Beane", "about": "General Manager, Oakland Athletics", "email": "*****@*****.**", "password": "******", } context = { "model": ckan.model, "session": ckan.model.Session, "user": self.sysadmin_user.name, "allow_partial_update": True, "extras_as_string": True, } user = user_create(context, user_dict) offset = url_for(controller="user", action="read", id=user["id"]) result = self.app.get(offset, status=200) stripped = self.strip_tags(result) assert "%s signed up" % user["fullname"] in stripped, stripped # Create a new package. package = {"name": "baseball_stats", "title": "Billy's Stats about Baseball Players"} context["user"] = user["name"] # FIXME This test use an old way to get at the schema to # recreate this we need to pretend to be using the api. We # should not be calling package_create like this we should be # going via the api or package controllers context["api_version"] = 3 context["ignore_auth"] = True package = package_create(context, package) result = self.app.get(offset, status=200) stripped = self.strip_tags(result) assert "%s created the dataset %s " % (user["fullname"], package["title"]) in stripped, stripped # Add a resource to the package. resource = { "url": "http://www.example.com", "description": "Chad Bradford's OBP Stats`", "format": "cvs", "name": "Chad Bradford Stats", } package["resources"].append(resource) request_data = {"id": package["id"], "resources": package["resources"]} package = package_update(context, request_data) result = self.app.get(offset, status=200) stripped = self.strip_tags(result) assert ( "%s added the resource %s to the dataset %s" % (user["fullname"], resource["name"], package["title"]) in stripped ), stripped # Update the package. package["title"] = "Billy's Updated Stats about Baseball Players" package = package_update(context, package) result = self.app.get(offset, status=200) stripped = self.strip_tags(result) assert "%s updated the dataset %s" % (user["fullname"], package["title"]) in stripped, stripped # Update the resource. resource = package["resources"][0] resource["name"] = "Chad Bradford Updated Stats" resource = resource_update(context, resource) result = self.app.get(offset, status=200) stripped = self.strip_tags(result) assert ( "%s updated the resource %s in the dataset %s" % (user["fullname"], resource["name"], package["title"]) in stripped ), stripped # Delete the resource. context["allow_partial_update"] = False package["resources"] = [] package_update(context, package) result = self.app.get(offset, status=200) stripped = self.strip_tags(result) assert ( "%s deleted the resource %s from the dataset %s" % (user["fullname"], resource["name"], package["title"]) in stripped ), stripped # Follow the package. follow_dataset(context, {"id": package["id"]}) result = self.app.get(offset, status=200) stripped = self.strip_tags(result) assert "%s started following %s" % (user["fullname"], package["title"]) not in stripped, stripped # Follow another user. follow_user(context, {"id": "joeadmin"}) result = self.app.get(offset, status=200) stripped = self.strip_tags(result) assert "%s started following %s" % (user["fullname"], "joeadmin") not in stripped, stripped # Create a new group. group = {"name": "baseball-stats-group", "title": "A Group for Datasets about Baseball"} context["allow_partial_update"] = True group = group_create(context, group) result = self.app.get(offset, status=200) stripped = self.strip_tags(result) assert "%s created the group %s" % (user["fullname"], group["title"]) in stripped, stripped # Update the group. group["title"] = "updated" group = group_update(context, group) result = self.app.get(offset, status=200) stripped = self.strip_tags(result) assert "%s updated the group %s" % (user["fullname"], group["title"]) in stripped, stripped # Delete the group. group["state"] = "deleted" group_update(context, group) result = self.app.get(offset, status=200) stripped = self.strip_tags(result) assert "%s deleted the group %s" % (user["fullname"], group["title"]) in stripped, stripped # Add a new tag to the package. tag = {"name": "baseball"} package["tags"].append(tag) package = package_update(context, package) result = self.app.get(offset, status=200) stripped = self.strip_tags(result) assert ( "%s added the tag %s to the dataset %s" % (user["fullname"], tag["name"], package["title"]) in stripped ), stripped # Remove the tag from the package. package["tags"] = [] context["allow_partial_update"] = False package_update(context, package) result = self.app.get(offset, status=200) stripped = self.strip_tags(result) assert ( "%s removed the tag %s from the dataset %s" % (user["fullname"], tag["name"], package["title"]) in stripped ), stripped # Add an extra to the package. package["extras"].append({"key": "quality", "value": "10000"}) package = package_update(context, package) result = self.app.get(offset, status=200) stripped = self.strip_tags(result) assert ( '%s added the extra "%s" to the dataset %s' % (user["fullname"], "quality", package["title"]) in stripped ), stripped # Update the extra. package["extras"][0]["value"] = "updated" package = package_update(context, package) result = self.app.get(offset, status=200) stripped = self.strip_tags(result) assert ( '%s changed the extra "%s" of the dataset %s' % (user["fullname"], "quality", package["title"]) in stripped ), stripped # Delete the extra. del package["extras"][0] package = package_update(context, package) result = self.app.get(offset, status=200) stripped = self.strip_tags(result) assert ( '%s deleted the extra "%s" from the dataset %s' % (user["fullname"], "quality", package["title"]) in stripped ), stripped # Delete the package. # we need to get round the delete permission context["ignore_auth"] = True package_delete(context, package) del context["ignore_auth"] result = self.app.get(offset, status=200) stripped = self.strip_tags(result) assert "%s deleted the dataset %s" % (user["fullname"], package["title"]) in stripped, stripped # Update the user's profile. user["about"] = "" user_update(context, user) result = self.app.get(offset, status=200) stripped = self.strip_tags(result) assert "%s updated their profile" % user["fullname"] in stripped, stripped # By now we've created >15 activities, but only the latest 15 should # appear on the page. result = self.app.get(offset, status=200) assert result.body.count('<div class="activity">') == 15 # The user's dashboard page should load successfully and have the # latest 15 activities on it. offset = url_for(controller="user", action="dashboard") extra_environ = {"Authorization": str(ckan.model.User.get("billybeane").apikey)} result = self.app.post(offset, extra_environ=extra_environ, status=200) assert result.body.count('<div class="activity">') == 15