def test_metadata_created_and_modified(self): # create a new package name = "test_metadata" rev = model.repo.new_revision() package = model.Package(name=name) model.Session.add(package) model.Session.flush() revision_id = model.Session().revision.id created_timestamp = model.Session().revision.timestamp model.repo.commit_and_remove() package = model.Package.by_name(name) assert package.metadata_created == created_timestamp,\ (package.metadata_created, created_timestamp) assert package.metadata_modified == created_timestamp,\ (package.metadata_modified, created_timestamp) # update the package it rev = model.repo.new_revision() package = model.Package.by_name(name) package.title = "test_metadata_new_title" modified_timestamp = model.Session().revision.timestamp model.repo.commit_and_remove() package = model.Package.by_name(name) assert package.metadata_created == created_timestamp assert package.metadata_modified == modified_timestamp
def test_basic_revisioning(self): # create a package with package_fixture_data name = "frob" rev = model.repo.new_revision() package = model.Package(name=name) model.Session.add(package) model.Session.flush() revision_id = model.Session().revision.id timestamp = model.Session().revision.timestamp model.repo.commit_and_remove() package = model.Package.by_name(name) assert len(package.all_revisions) == 1 assert package.all_revisions[0].revision_id == revision_id assert package.all_revisions[0].revision_timestamp == timestamp assert package.all_revisions[0].expired_id is None # change it rev = model.repo.new_revision() package = model.Package.by_name(name) package.title = "wobsnasm" revision_id2 = model.Session().revision.id timestamp2 = model.Session().revision.timestamp model.repo.commit_and_remove() package = model.Package.by_name(name) assert len(package.all_revisions) == 2 assert package.all_revisions[0].revision_id == revision_id2 assert package.all_revisions[0].revision_timestamp == timestamp2 assert package.all_revisions[0].expired_id is None assert package.all_revisions[1].revision_id == revision_id assert package.all_revisions[1].revision_timestamp == timestamp assert package.all_revisions[1].expired_id == revision_id2
def delete(cls): '''Purges packages etc. that were created by this class.''' for pkg_name in cls.pkg_names: model.Session().autoflush = False pkg = model.Package.by_name(unicode(pkg_name)) if pkg: pkg.purge() for tag_name in cls.tag_names: tag = model.Tag.by_name(unicode(tag_name)) if tag: tag.purge() for group_name in cls.group_names: group = model.Group.by_name(unicode(group_name)) if group: model.Session.delete(group) revs = model.Session.query(model.Revision).filter_by(author=cls.author) for rev in revs: for pkg in rev.packages: pkg.purge() for grp in rev.groups: grp.purge() model.Session.commit() model.Session.delete(rev) for user_name in cls.user_refs: user = model.User.get(unicode(user_name)) if user: user.purge() model.Session.commit() model.Session.remove() cls.reset()
def action(self, logic_function, ver=None): try: function = get_action(logic_function) except KeyError: log.error('Can\'t find logic function: %s' % logic_function) return self._finish_bad_request( _('Action name not known: %s') % logic_function) context = { 'model': model, 'session': model.Session, 'user': c.user, 'api_version': ver, 'auth_user_obj': c.userobj } model.Session()._context = context return_dict = { 'help': h.url_for( controller='api', action='action', logic_function='help_show', ver=ver, name=logic_function, qualified=True, ) } try: side_effect_free = getattr(function, 'side_effect_free', False) request_data = self._get_request_data( try_url_params=side_effect_free) except ValueError, inst: log.error('Bad request data: %s' % inst) return self._finish_bad_request(_('JSON Error: %s') % inst)
def action(self, logic_function): function = get_action(logic_function) context = {'model': model, 'session': model.Session, 'user': c.user} model.Session()._context = context return_dict = {'help': function.__doc__} try: request_data = self._get_request_data() except ValueError, inst: return self._finish_bad_request( gettext('JSON Error: %s') % str(inst))
def action(logic_function, ver=API_DEFAULT_VERSION): u'''Main endpoint for the action API (v3) Creates a dict with the incoming request data and calls the appropiate logic function. Returns a JSON response with the following keys: * ``help``: A URL to the docstring for the specified action * ``success``: A boolean indicating if the request was successful or an exception was raised * ``result``: The output of the action, generally an Object or an Array ''' # Check if action exists try: function = get_action(logic_function) except KeyError: msg = u'Action name not known: {0}'.format(logic_function) log.info(msg) return _finish_bad_request(msg) context = { u'model': model, u'session': model.Session, u'user': g.user, u'api_version': ver, u'auth_user_obj': g.userobj } model.Session()._context = context return_dict = { u'help': url_for( u'api.action', logic_function=u'help_show', ver=ver, name=logic_function, _external=True, ) } # Get the request data try: side_effect_free = getattr(function, u'side_effect_free', False) request_data = _get_request_data(try_url_params=side_effect_free) except ValueError, inst: log.info(u'Bad Action API request data: %s', inst) return _finish_bad_request(_(u'JSON Error: %s') % inst)
def action(self, logic_function): function = get_action(logic_function) if not function: log.error('Can\'t find logic function: %s' % logic_function) return self._finish_bad_request( gettext('Action name not known: %s') % str(logic_function)) context = {'model': model, 'session': model.Session, 'user': c.user} model.Session()._context = context return_dict = {'help': function.__doc__} try: request_data = self._get_request_data() except ValueError, inst: log.error('Bad request data: %s' % str(inst)) return self._finish_bad_request( gettext('JSON Error: %s') % str(inst))
def action(self, logic_function, ver=None): function = get_action(logic_function) if not function: log.error('Can\'t find logic function: %s' % logic_function) return self._finish_bad_request( gettext('Action name not known: %s') % str(logic_function)) context = {'model': model, 'session': model.Session, 'user': c.user, 'api_version':ver} model.Session()._context = context return_dict = {'help': function.__doc__} try: side_effect_free = getattr(function, 'side_effect_free', False) request_data = self._get_request_data(try_url_params=side_effect_free) except ValueError, inst: log.error('Bad request data: %s' % str(inst)) return self._finish_bad_request( gettext('JSON Error: %s') % str(inst))
def delete(cls): '''Purges packages etc. that were created by this class.''' for pkg_name in cls.pkg_names: model.Session().autoflush = False pkg = model.Package.by_name(str(pkg_name)) if pkg: pkg.purge() for tag_name in cls.tag_names: tag = model.Tag.by_name(str(tag_name)) if tag: tag.purge() for group_name in cls.group_names: group = model.Group.by_name(str(group_name)) if group: model.Session.delete(group) for user_name in cls.user_refs: user = model.User.get(str(user_name)) if user: user.purge() model.Session.commit() model.Session.remove() cls.reset()
def delete(cls): '''Purges packages etc. that were created by this class.''' import ckan.model as model for pkg_name in cls.pkg_names: pkg = model.Package.by_name(unicode(pkg_name)) if pkg: sql = "DELETE FROM package_search WHERE package_id='%s'" % pkg.id model.Session.execute(sql) model.repo.commit_and_remove() for pkg_name in cls.pkg_names: model.Session().autoflush = False pkg = model.Package.by_name(unicode(pkg_name)) if pkg: pkg.purge() for tag_name in cls.tag_names: tag = model.Tag.by_name(unicode(tag_name)) if tag: tag.purge() for group_name in cls.group_names: group = model.Group.by_name(unicode(group_name)) if group: model.Session.delete(group) revs = model.Session.query(model.Revision).filter_by(author=cls.author) for rev in revs: for pkg in rev.packages: pkg.purge() for grp in rev.groups: grp.purge() model.Session.commit() model.Session.delete(rev) for user_name in cls.user_refs: user = model.User.get(unicode(user_name)) if user: user.purge() model.Session.commit() model.Session.remove() cls.reset()
def action(logic_function, ver=API_DEFAULT_VERSION): u'''Main endpoint for the action API (v3) Creates a dict with the incoming request data and calls the appropiate logic function. Returns a JSON response with the following keys: * ``help``: A URL to the docstring for the specified action * ``success``: A boolean indicating if the request was successful or an exception was raised * ``result``: The output of the action, generally an Object or an Array ''' # Check if action exists try: function = get_action(logic_function) except KeyError: msg = u'Action name not known: {0}'.format(logic_function) log.info(msg) return _finish_bad_request(msg) context = { u'model': model, u'session': model.Session, u'user': g.user, u'api_version': ver, u'auth_user_obj': g.userobj } model.Session()._context = context return_dict = { u'help': url_for( u'api.action', logic_function=u'help_show', ver=ver, name=logic_function, _external=True, ) } # Get the request data try: side_effect_free = getattr(function, u'side_effect_free', False) request_data = _get_request_data(try_url_params=side_effect_free) except ValueError as inst: log.info(u'Bad Action API request data: %s', inst) return _finish_bad_request(_(u'JSON Error: %s') % inst) if not isinstance(request_data, dict): # this occurs if request_data is blank log.info(u'Bad Action API request data - not dict: %r', request_data) return _finish_bad_request( _(u'Bad request data: %s') % u'Request data JSON decoded to %r but ' u'it needs to be a dictionary.' % request_data) if u'callback' in request_data: del request_data[u'callback'] g.user = None g.userobj = None context[u'user'] = None context[u'auth_user_obj'] = None # Call the action function, catch any exception try: result = function(context, request_data) return_dict[u'success'] = True return_dict[u'result'] = result except DataError as e: log.info(u'Format incorrect (Action API): %s - %s', e.error, request_data) return_dict[u'error'] = { u'__type': u'Integrity Error', u'message': e.error, u'data': request_data } return_dict[u'success'] = False return _finish(400, return_dict, content_type=u'json') except NotAuthorized as e: return_dict[u'error'] = { u'__type': u'Authorization Error', u'message': _(u'Access denied') } return_dict[u'success'] = False if unicode(e): return_dict[u'error'][u'message'] += u': %s' % e return _finish(403, return_dict, content_type=u'json') except NotFound as e: return_dict[u'error'] = { u'__type': u'Not Found Error', u'message': _(u'Not found') } if unicode(e): return_dict[u'error'][u'message'] += u': %s' % e return_dict[u'success'] = False return _finish(404, return_dict, content_type=u'json') except ValidationError as e: error_dict = e.error_dict error_dict[u'__type'] = u'Validation Error' return_dict[u'error'] = error_dict return_dict[u'success'] = False # CS nasty_string ignore log.info(u'Validation error (Action API): %r', str(e.error_dict)) return _finish(409, return_dict, content_type=u'json') except SearchQueryError as e: return_dict[u'error'] = { u'__type': u'Search Query Error', u'message': u'Search Query is invalid: %r' % e.args } return_dict[u'success'] = False return _finish(400, return_dict, content_type=u'json') except SearchError as e: return_dict[u'error'] = { u'__type': u'Search Error', u'message': u'Search error: %r' % e.args } return_dict[u'success'] = False return _finish(409, return_dict, content_type=u'json') except SearchIndexError as e: return_dict[u'error'] = { u'__type': u'Search Index Error', u'message': u'Unable to add package to search index: %s' % str(e) } return_dict[u'success'] = False return _finish(500, return_dict, content_type=u'json') return _finish_ok(return_dict)
def action(self, logic_function, ver=None): try: function = get_action(logic_function) except KeyError: log.info('Can\'t find logic function: %s', logic_function) return self._finish_bad_request( _('Action name not known: %s') % logic_function) context = { 'model': model, 'session': model.Session, 'user': c.user, 'api_version': ver, 'auth_user_obj': c.userobj } model.Session()._context = context return_dict = { 'help': h.url_for( controller='api', action='action', logic_function='help_show', ver=ver, name=logic_function, qualified=True, ) } try: side_effect_free = getattr(function, 'side_effect_free', False) request_data = self._get_request_data( try_url_params=side_effect_free) except ValueError as inst: log.info('Bad Action API request data: %s', inst) return self._finish_bad_request(_('JSON Error: %s') % inst) if not isinstance(request_data, dict): # this occurs if request_data is blank log.info('Bad Action API request data - not dict: %r', request_data) return self._finish_bad_request( _('Bad request data: %s') % 'Request data JSON decoded to %r but ' 'it needs to be a dictionary.' % request_data) # if callback is specified we do not want to send that to the search if 'callback' in request_data: del request_data['callback'] c.user = None c.userobj = None context['user'] = None context['auth_user_obj'] = None try: result = function(context, request_data) return_dict['success'] = True return_dict['result'] = result except DataError as e: log.info('Format incorrect (Action API): %s - %s', e.error, request_data) return_dict['error'] = { '__type': 'Integrity Error', 'message': e.error, 'data': request_data } return_dict['success'] = False return self._finish(400, return_dict, content_type='json') except NotAuthorized as e: return_dict['error'] = { '__type': 'Authorization Error', 'message': _('Access denied') } return_dict['success'] = False if unicode(e): return_dict['error']['message'] += u': %s' % e return self._finish(403, return_dict, content_type='json') except NotFound as e: return_dict['error'] = { '__type': 'Not Found Error', 'message': _('Not found') } if unicode(e): return_dict['error']['message'] += u': %s' % e return_dict['success'] = False return self._finish(404, return_dict, content_type='json') except ValidationError as e: error_dict = e.error_dict error_dict['__type'] = 'Validation Error' return_dict['error'] = error_dict return_dict['success'] = False # CS nasty_string ignore log.info('Validation error (Action API): %r', str(e.error_dict)) return self._finish(409, return_dict, content_type='json') except search.SearchQueryError as e: return_dict['error'] = { '__type': 'Search Query Error', 'message': 'Search Query is invalid: %r' % e.args } return_dict['success'] = False return self._finish(400, return_dict, content_type='json') except search.SearchError as e: return_dict['error'] = { '__type': 'Search Error', 'message': 'Search error: %r' % e.args } return_dict['success'] = False return self._finish(409, return_dict, content_type='json') except search.SearchIndexError as e: return_dict['error'] = { '__type': 'Search Index Error', 'message': 'Unable to add package to search index: %s' % str(e) } return_dict['success'] = False return self._finish(500, return_dict, content_type='json') return self._finish_ok(return_dict)
def test_metadata_created_and_modified(self): # create a new package name = "test_metadata" rev = model.repo.new_revision() package = model.Package(name=name) model.Session.add(package) model.Session.flush() revision_id = model.Session().revision.id created_timestamp = model.Session().revision.timestamp model.repo.commit_and_remove() package = model.Package.by_name(name) assert package.metadata_created == created_timestamp,\ (package.metadata_created, created_timestamp) assert package.metadata_modified == created_timestamp,\ (package.metadata_modified, created_timestamp) # update the package rev = model.repo.new_revision() package = model.Package.by_name(name) package.title = "test_metadata_new_title" modified_timestamp = model.Session().revision.timestamp model.repo.commit_and_remove() package = model.Package.by_name(name) assert package.metadata_created == created_timestamp assert package.metadata_modified == modified_timestamp last_modified_timestamp = modified_timestamp # update a package's tag rev = model.repo.new_revision() package = model.Package.by_name(name) package.add_tag_by_name('new-tag') modified_timestamp = model.Session().revision.timestamp assert modified_timestamp != last_modified_timestamp model.repo.commit_and_remove() package = model.Package.by_name(name) assert package.metadata_created == created_timestamp assert package.metadata_modified == modified_timestamp last_modified_timestamp = modified_timestamp # update a package's extra rev = model.repo.new_revision() package = model.Package.by_name(name) package.extras['new-key'] = 'value' modified_timestamp = model.Session().revision.timestamp assert modified_timestamp != last_modified_timestamp model.repo.commit_and_remove() package = model.Package.by_name(name) assert package.metadata_created == created_timestamp assert package.metadata_modified == modified_timestamp last_modified_timestamp = modified_timestamp # update a package's relationship rev = model.repo.new_revision() package = model.Package.by_name(name) anna = model.Package.by_name(u'annakarenina') package.add_relationship(u'child_of', anna) modified_timestamp = model.Session().revision.timestamp assert modified_timestamp != last_modified_timestamp model.repo.commit_and_remove() package = model.Package.by_name(name) assert package.metadata_created == created_timestamp assert package.metadata_modified == modified_timestamp last_modified_timestamp = modified_timestamp # update a package's group - NB no change this time rev = model.repo.new_revision() group = model.Group.by_name('roger') group.add_package_by_name(name) modified_timestamp = model.Session().revision.timestamp assert modified_timestamp != last_modified_timestamp model.repo.commit_and_remove() package = model.Package.by_name(name) assert package.metadata_created == created_timestamp assert package.metadata_modified == last_modified_timestamp # no change
def migrate_data(self, context, data_dict): if not authz.Authorizer().is_sysadmin(unicode(context['user'])): raise NotAuthorized model.Session.remove() model.Session().connection conn_ckan = model.Session.connection() conn_drupal = self.engine.connect() #package packages = conn_ckan.execute( select([model.package_revision_table], and_( model.package_revision_table.c.current == True, model.package_revision_table.c.state.in_( ['active', 'deleted'])))).fetchall() package_inserts = [] for package in packages: insert = {} for column in self.package_table.c: if column in package: insert[column.name] = package[column.name] package_inserts.append(insert) conn_drupal.execute(self.package_table.insert(), package_inserts) print 'packages done' #resources resources = conn_ckan.execute( select([ model.resource_revision_table, model.resource_group_table.c.package_id ], and_( model.resource_revision_table.c.current == True, model.resource_revision_table.c.state.in_( ['active', 'deleted']), model.resource_group_table.c.id == model.resource_revision_table.c.resource_group_id, ))).fetchall() resource_inserts = [] for resource in resources: insert = {} for column in self.resource_table.c: if column in resource: insert[column.name] = resource[column.name] insert['extras'] = json.dumps(insert['extras']) resource_inserts.append(insert) conn_drupal.execute(self.resource_table.insert(), resource_inserts) print 'resources done' #extras extras = conn_ckan.execute( select([model.extra_revision_table], and_( model.extra_revision_table.c.current == True, model.extra_revision_table.c.state.in_( ['active', 'deleted'])))).fetchall() package_extra_inserts = [] for extra in extras: insert = {} for column in self.package_extra_table.c: if column in extra: insert[column.name] = extra[column.name] insert['value'] = json.dumps(insert['value']) package_extra_inserts.append(insert) conn_drupal.execute(self.package_extra_table.insert(), package_extra_inserts) print 'extras done' #tags tags = conn_ckan.execute( select([model.package_tag_revision_table, model.tag_table.c.name], and_( model.package_tag_revision_table.c.current == True, model.package_tag_revision_table.c.state.in_( ['active', 'deleted']), model.tag_table.c.id == model.package_tag_revision_table.c.tag_id))).fetchall() tag_inserts = [] for tag in tags: tag_inserts.append({ 'name': tag['name'], 'id': tag['id'], 'package_id': tag['package_id'] }) conn_drupal.execute(self.tag_table.insert(), tag_inserts) print 'tags done' ## get nodes packages = conn_drupal.execute(select( [self.package_table], )).fetchall() for num, package in enumerate(packages): print num data_dict = {} for column in self.package_table.c: data_dict[column.name] = package[column.name] url = urlparse.urljoin(self.base_url, 'services/package.json') data_dict['body'] = data_dict.get('notes', '') if not data_dict['title']: data_dict['title'] = data_dict['name'] tags = conn_drupal.execute( select( [self.tag_table.c.name], self.tag_table.c.package_id == package['id'])).fetchall() data_dict['tags'] = [{'name': tag[0]} for tag in tags] data = json.dumps({'data': data_dict}) req = urllib2.Request(url, data, {'Content-type': 'application/json'}) f = urllib2.urlopen(req, None, 3) try: drupal_info = json.loads(f.read()) finally: f.close() nid = drupal_info['nid'] update = {'nid': nid} conn_drupal.execute(self.package_table.update().where( self.package_table.c.id == data_dict['id']).values(nid=nid)) conn_drupal.execute( 'update ckan_resource set nid = (select nid from ckan_package where ckan_resource.package_id = ckan_package.id);' ) conn_drupal.execute( 'update ckan_package_extra set nid = (select nid from ckan_package where ckan_package_extra.package_id = ckan_package.id);' ) conn_drupal.execute( 'update ckan_tag set nid = (select nid from ckan_package where ckan_tag.package_id = ckan_package.id);' ) print 'finished migration'