def test_metadata_created_and_modified(self):
# create a new package
name = "test_metadata"
rev = model.repo.new_revision()
package = model.Package(name=name)
model.Session.add(package)
model.Session.flush()
revision_id = model.Session().revision.id
created_timestamp = model.Session().revision.timestamp
model.repo.commit_and_remove()
package = model.Package.by_name(name)
assert package.metadata_created == created_timestamp,\
(package.metadata_created, created_timestamp)
assert package.metadata_modified == created_timestamp,\
(package.metadata_modified, created_timestamp)
# update the package it
rev = model.repo.new_revision()
package = model.Package.by_name(name)
package.title = "test_metadata_new_title"
modified_timestamp = model.Session().revision.timestamp
model.repo.commit_and_remove()
package = model.Package.by_name(name)
assert package.metadata_created == created_timestamp
assert package.metadata_modified == modified_timestamp
def test_basic_revisioning(self):
# create a package with package_fixture_data
name = "frob"
rev = model.repo.new_revision()
package = model.Package(name=name)
model.Session.add(package)
model.Session.flush()
revision_id = model.Session().revision.id
timestamp = model.Session().revision.timestamp
model.repo.commit_and_remove()
package = model.Package.by_name(name)
assert len(package.all_revisions) == 1
assert package.all_revisions[0].revision_id == revision_id
assert package.all_revisions[0].revision_timestamp == timestamp
assert package.all_revisions[0].expired_id is None
# change it
rev = model.repo.new_revision()
package = model.Package.by_name(name)
package.title = "wobsnasm"
revision_id2 = model.Session().revision.id
timestamp2 = model.Session().revision.timestamp
model.repo.commit_and_remove()
package = model.Package.by_name(name)
assert len(package.all_revisions) == 2
assert package.all_revisions[0].revision_id == revision_id2
assert package.all_revisions[0].revision_timestamp == timestamp2
assert package.all_revisions[0].expired_id is None
assert package.all_revisions[1].revision_id == revision_id
assert package.all_revisions[1].revision_timestamp == timestamp
assert package.all_revisions[1].expired_id == revision_id2
def delete(cls):
'''Purges packages etc. that were created by this class.'''
for pkg_name in cls.pkg_names:
model.Session().autoflush = False
pkg = model.Package.by_name(unicode(pkg_name))
if pkg:
pkg.purge()
for tag_name in cls.tag_names:
tag = model.Tag.by_name(unicode(tag_name))
if tag:
tag.purge()
for group_name in cls.group_names:
group = model.Group.by_name(unicode(group_name))
if group:
model.Session.delete(group)
revs = model.Session.query(model.Revision).filter_by(author=cls.author)
for rev in revs:
for pkg in rev.packages:
pkg.purge()
for grp in rev.groups:
grp.purge()
model.Session.commit()
model.Session.delete(rev)
for user_name in cls.user_refs:
user = model.User.get(unicode(user_name))
if user:
user.purge()
model.Session.commit()
model.Session.remove()
cls.reset()
def action(self, logic_function, ver=None):
try:
function = get_action(logic_function)
except KeyError:
log.error('Can\'t find logic function: %s' % logic_function)
return self._finish_bad_request(
_('Action name not known: %s') % logic_function)
context = {
'model': model,
'session': model.Session,
'user': c.user,
'api_version': ver,
'auth_user_obj': c.userobj
}
model.Session()._context = context
return_dict = {
'help':
h.url_for(
controller='api',
action='action',
logic_function='help_show',
ver=ver,
name=logic_function,
qualified=True,
)
}
try:
side_effect_free = getattr(function, 'side_effect_free', False)
request_data = self._get_request_data(
try_url_params=side_effect_free)
except ValueError, inst:
log.error('Bad request data: %s' % inst)
return self._finish_bad_request(_('JSON Error: %s') % inst)
def action(self, logic_function):
function = get_action(logic_function)
context = {'model': model, 'session': model.Session, 'user': c.user}
model.Session()._context = context
return_dict = {'help': function.__doc__}
try:
request_data = self._get_request_data()
except ValueError, inst:
return self._finish_bad_request(
gettext('JSON Error: %s') % str(inst))
def action(logic_function, ver=API_DEFAULT_VERSION):
u'''Main endpoint for the action API (v3)
Creates a dict with the incoming request data and calls the appropiate
logic function. Returns a JSON response with the following keys:
* ``help``: A URL to the docstring for the specified action
* ``success``: A boolean indicating if the request was successful or
an exception was raised
* ``result``: The output of the action, generally an Object or an Array
'''
# Check if action exists
try:
function = get_action(logic_function)
except KeyError:
msg = u'Action name not known: {0}'.format(logic_function)
log.info(msg)
return _finish_bad_request(msg)
context = {
u'model': model,
u'session': model.Session,
u'user': g.user,
u'api_version': ver,
u'auth_user_obj': g.userobj
}
model.Session()._context = context
return_dict = {
u'help':
url_for(
u'api.action',
logic_function=u'help_show',
ver=ver,
name=logic_function,
_external=True,
)
}
# Get the request data
try:
side_effect_free = getattr(function, u'side_effect_free', False)
request_data = _get_request_data(try_url_params=side_effect_free)
except ValueError, inst:
log.info(u'Bad Action API request data: %s', inst)
return _finish_bad_request(_(u'JSON Error: %s') % inst)
def action(self, logic_function):
function = get_action(logic_function)
if not function:
log.error('Can\'t find logic function: %s' % logic_function)
return self._finish_bad_request(
gettext('Action name not known: %s') % str(logic_function))
context = {'model': model, 'session': model.Session, 'user': c.user}
model.Session()._context = context
return_dict = {'help': function.__doc__}
try:
request_data = self._get_request_data()
except ValueError, inst:
log.error('Bad request data: %s' % str(inst))
return self._finish_bad_request(
gettext('JSON Error: %s') % str(inst))
def action(self, logic_function, ver=None):
function = get_action(logic_function)
if not function:
log.error('Can\'t find logic function: %s' % logic_function)
return self._finish_bad_request(
gettext('Action name not known: %s') % str(logic_function))
context = {'model': model, 'session': model.Session, 'user': c.user,
'api_version':ver}
model.Session()._context = context
return_dict = {'help': function.__doc__}
try:
side_effect_free = getattr(function, 'side_effect_free', False)
request_data = self._get_request_data(try_url_params=side_effect_free)
except ValueError, inst:
log.error('Bad request data: %s' % str(inst))
return self._finish_bad_request(
gettext('JSON Error: %s') % str(inst))
def delete(cls):
'''Purges packages etc. that were created by this class.'''
for pkg_name in cls.pkg_names:
model.Session().autoflush = False
pkg = model.Package.by_name(str(pkg_name))
if pkg:
pkg.purge()
for tag_name in cls.tag_names:
tag = model.Tag.by_name(str(tag_name))
if tag:
tag.purge()
for group_name in cls.group_names:
group = model.Group.by_name(str(group_name))
if group:
model.Session.delete(group)
for user_name in cls.user_refs:
user = model.User.get(str(user_name))
if user:
user.purge()
model.Session.commit()
model.Session.remove()
cls.reset()
def delete(cls):
'''Purges packages etc. that were created by this class.'''
import ckan.model as model
for pkg_name in cls.pkg_names:
pkg = model.Package.by_name(unicode(pkg_name))
if pkg:
sql = "DELETE FROM package_search WHERE package_id='%s'" % pkg.id
model.Session.execute(sql)
model.repo.commit_and_remove()
for pkg_name in cls.pkg_names:
model.Session().autoflush = False
pkg = model.Package.by_name(unicode(pkg_name))
if pkg:
pkg.purge()
for tag_name in cls.tag_names:
tag = model.Tag.by_name(unicode(tag_name))
if tag:
tag.purge()
for group_name in cls.group_names:
group = model.Group.by_name(unicode(group_name))
if group:
model.Session.delete(group)
revs = model.Session.query(model.Revision).filter_by(author=cls.author)
for rev in revs:
for pkg in rev.packages:
pkg.purge()
for grp in rev.groups:
grp.purge()
model.Session.commit()
model.Session.delete(rev)
for user_name in cls.user_refs:
user = model.User.get(unicode(user_name))
if user:
user.purge()
model.Session.commit()
model.Session.remove()
cls.reset()
def action(logic_function, ver=API_DEFAULT_VERSION):
u'''Main endpoint for the action API (v3)
Creates a dict with the incoming request data and calls the appropiate
logic function. Returns a JSON response with the following keys:
* ``help``: A URL to the docstring for the specified action
* ``success``: A boolean indicating if the request was successful or
an exception was raised
* ``result``: The output of the action, generally an Object or an Array
'''
# Check if action exists
try:
function = get_action(logic_function)
except KeyError:
msg = u'Action name not known: {0}'.format(logic_function)
log.info(msg)
return _finish_bad_request(msg)
context = {
u'model': model,
u'session': model.Session,
u'user': g.user,
u'api_version': ver,
u'auth_user_obj': g.userobj
}
model.Session()._context = context
return_dict = {
u'help':
url_for(
u'api.action',
logic_function=u'help_show',
ver=ver,
name=logic_function,
_external=True,
)
}
# Get the request data
try:
side_effect_free = getattr(function, u'side_effect_free', False)
request_data = _get_request_data(try_url_params=side_effect_free)
except ValueError as inst:
log.info(u'Bad Action API request data: %s', inst)
return _finish_bad_request(_(u'JSON Error: %s') % inst)
if not isinstance(request_data, dict):
# this occurs if request_data is blank
log.info(u'Bad Action API request data - not dict: %r', request_data)
return _finish_bad_request(
_(u'Bad request data: %s') %
u'Request data JSON decoded to %r but '
u'it needs to be a dictionary.' % request_data)
if u'callback' in request_data:
del request_data[u'callback']
g.user = None
g.userobj = None
context[u'user'] = None
context[u'auth_user_obj'] = None
# Call the action function, catch any exception
try:
result = function(context, request_data)
return_dict[u'success'] = True
return_dict[u'result'] = result
except DataError as e:
log.info(u'Format incorrect (Action API): %s - %s', e.error,
request_data)
return_dict[u'error'] = {
u'__type': u'Integrity Error',
u'message': e.error,
u'data': request_data
}
return_dict[u'success'] = False
return _finish(400, return_dict, content_type=u'json')
except NotAuthorized as e:
return_dict[u'error'] = {
u'__type': u'Authorization Error',
u'message': _(u'Access denied')
}
return_dict[u'success'] = False
if unicode(e):
return_dict[u'error'][u'message'] += u': %s' % e
return _finish(403, return_dict, content_type=u'json')
except NotFound as e:
return_dict[u'error'] = {
u'__type': u'Not Found Error',
u'message': _(u'Not found')
}
if unicode(e):
return_dict[u'error'][u'message'] += u': %s' % e
return_dict[u'success'] = False
return _finish(404, return_dict, content_type=u'json')
except ValidationError as e:
error_dict = e.error_dict
error_dict[u'__type'] = u'Validation Error'
return_dict[u'error'] = error_dict
return_dict[u'success'] = False
# CS nasty_string ignore
log.info(u'Validation error (Action API): %r', str(e.error_dict))
return _finish(409, return_dict, content_type=u'json')
except SearchQueryError as e:
return_dict[u'error'] = {
u'__type': u'Search Query Error',
u'message': u'Search Query is invalid: %r' % e.args
}
return_dict[u'success'] = False
return _finish(400, return_dict, content_type=u'json')
except SearchError as e:
return_dict[u'error'] = {
u'__type': u'Search Error',
u'message': u'Search error: %r' % e.args
}
return_dict[u'success'] = False
return _finish(409, return_dict, content_type=u'json')
except SearchIndexError as e:
return_dict[u'error'] = {
u'__type': u'Search Index Error',
u'message': u'Unable to add package to search index: %s' % str(e)
}
return_dict[u'success'] = False
return _finish(500, return_dict, content_type=u'json')
return _finish_ok(return_dict)
def action(self, logic_function, ver=None):
try:
function = get_action(logic_function)
except KeyError:
log.info('Can\'t find logic function: %s', logic_function)
return self._finish_bad_request(
_('Action name not known: %s') % logic_function)
context = {
'model': model,
'session': model.Session,
'user': c.user,
'api_version': ver,
'auth_user_obj': c.userobj
}
model.Session()._context = context
return_dict = {
'help':
h.url_for(
controller='api',
action='action',
logic_function='help_show',
ver=ver,
name=logic_function,
qualified=True,
)
}
try:
side_effect_free = getattr(function, 'side_effect_free', False)
request_data = self._get_request_data(
try_url_params=side_effect_free)
except ValueError as inst:
log.info('Bad Action API request data: %s', inst)
return self._finish_bad_request(_('JSON Error: %s') % inst)
if not isinstance(request_data, dict):
# this occurs if request_data is blank
log.info('Bad Action API request data - not dict: %r',
request_data)
return self._finish_bad_request(
_('Bad request data: %s') %
'Request data JSON decoded to %r but '
'it needs to be a dictionary.' % request_data)
# if callback is specified we do not want to send that to the search
if 'callback' in request_data:
del request_data['callback']
c.user = None
c.userobj = None
context['user'] = None
context['auth_user_obj'] = None
try:
result = function(context, request_data)
return_dict['success'] = True
return_dict['result'] = result
except DataError as e:
log.info('Format incorrect (Action API): %s - %s', e.error,
request_data)
return_dict['error'] = {
'__type': 'Integrity Error',
'message': e.error,
'data': request_data
}
return_dict['success'] = False
return self._finish(400, return_dict, content_type='json')
except NotAuthorized as e:
return_dict['error'] = {
'__type': 'Authorization Error',
'message': _('Access denied')
}
return_dict['success'] = False
if unicode(e):
return_dict['error']['message'] += u': %s' % e
return self._finish(403, return_dict, content_type='json')
except NotFound as e:
return_dict['error'] = {
'__type': 'Not Found Error',
'message': _('Not found')
}
if unicode(e):
return_dict['error']['message'] += u': %s' % e
return_dict['success'] = False
return self._finish(404, return_dict, content_type='json')
except ValidationError as e:
error_dict = e.error_dict
error_dict['__type'] = 'Validation Error'
return_dict['error'] = error_dict
return_dict['success'] = False
# CS nasty_string ignore
log.info('Validation error (Action API): %r', str(e.error_dict))
return self._finish(409, return_dict, content_type='json')
except search.SearchQueryError as e:
return_dict['error'] = {
'__type': 'Search Query Error',
'message': 'Search Query is invalid: %r' % e.args
}
return_dict['success'] = False
return self._finish(400, return_dict, content_type='json')
except search.SearchError as e:
return_dict['error'] = {
'__type': 'Search Error',
'message': 'Search error: %r' % e.args
}
return_dict['success'] = False
return self._finish(409, return_dict, content_type='json')
except search.SearchIndexError as e:
return_dict['error'] = {
'__type': 'Search Index Error',
'message': 'Unable to add package to search index: %s' % str(e)
}
return_dict['success'] = False
return self._finish(500, return_dict, content_type='json')
return self._finish_ok(return_dict)
def test_metadata_created_and_modified(self):
# create a new package
name = "test_metadata"
rev = model.repo.new_revision()
package = model.Package(name=name)
model.Session.add(package)
model.Session.flush()
revision_id = model.Session().revision.id
created_timestamp = model.Session().revision.timestamp
model.repo.commit_and_remove()
package = model.Package.by_name(name)
assert package.metadata_created == created_timestamp,\
(package.metadata_created, created_timestamp)
assert package.metadata_modified == created_timestamp,\
(package.metadata_modified, created_timestamp)
# update the package
rev = model.repo.new_revision()
package = model.Package.by_name(name)
package.title = "test_metadata_new_title"
modified_timestamp = model.Session().revision.timestamp
model.repo.commit_and_remove()
package = model.Package.by_name(name)
assert package.metadata_created == created_timestamp
assert package.metadata_modified == modified_timestamp
last_modified_timestamp = modified_timestamp
# update a package's tag
rev = model.repo.new_revision()
package = model.Package.by_name(name)
package.add_tag_by_name('new-tag')
modified_timestamp = model.Session().revision.timestamp
assert modified_timestamp != last_modified_timestamp
model.repo.commit_and_remove()
package = model.Package.by_name(name)
assert package.metadata_created == created_timestamp
assert package.metadata_modified == modified_timestamp
last_modified_timestamp = modified_timestamp
# update a package's extra
rev = model.repo.new_revision()
package = model.Package.by_name(name)
package.extras['new-key'] = 'value'
modified_timestamp = model.Session().revision.timestamp
assert modified_timestamp != last_modified_timestamp
model.repo.commit_and_remove()
package = model.Package.by_name(name)
assert package.metadata_created == created_timestamp
assert package.metadata_modified == modified_timestamp
last_modified_timestamp = modified_timestamp
# update a package's relationship
rev = model.repo.new_revision()
package = model.Package.by_name(name)
anna = model.Package.by_name(u'annakarenina')
package.add_relationship(u'child_of', anna)
modified_timestamp = model.Session().revision.timestamp
assert modified_timestamp != last_modified_timestamp
model.repo.commit_and_remove()
package = model.Package.by_name(name)
assert package.metadata_created == created_timestamp
assert package.metadata_modified == modified_timestamp
last_modified_timestamp = modified_timestamp
# update a package's group - NB no change this time
rev = model.repo.new_revision()
group = model.Group.by_name('roger')
group.add_package_by_name(name)
modified_timestamp = model.Session().revision.timestamp
assert modified_timestamp != last_modified_timestamp
model.repo.commit_and_remove()
package = model.Package.by_name(name)
assert package.metadata_created == created_timestamp
assert package.metadata_modified == last_modified_timestamp # no change
def migrate_data(self, context, data_dict):
if not authz.Authorizer().is_sysadmin(unicode(context['user'])):
raise NotAuthorized
model.Session.remove()
model.Session().connection
conn_ckan = model.Session.connection()
conn_drupal = self.engine.connect()
#package
packages = conn_ckan.execute(
select([model.package_revision_table],
and_(
model.package_revision_table.c.current == True,
model.package_revision_table.c.state.in_(
['active', 'deleted'])))).fetchall()
package_inserts = []
for package in packages:
insert = {}
for column in self.package_table.c:
if column in package:
insert[column.name] = package[column.name]
package_inserts.append(insert)
conn_drupal.execute(self.package_table.insert(), package_inserts)
print 'packages done'
#resources
resources = conn_ckan.execute(
select([
model.resource_revision_table,
model.resource_group_table.c.package_id
],
and_(
model.resource_revision_table.c.current == True,
model.resource_revision_table.c.state.in_(
['active', 'deleted']),
model.resource_group_table.c.id ==
model.resource_revision_table.c.resource_group_id,
))).fetchall()
resource_inserts = []
for resource in resources:
insert = {}
for column in self.resource_table.c:
if column in resource:
insert[column.name] = resource[column.name]
insert['extras'] = json.dumps(insert['extras'])
resource_inserts.append(insert)
conn_drupal.execute(self.resource_table.insert(), resource_inserts)
print 'resources done'
#extras
extras = conn_ckan.execute(
select([model.extra_revision_table],
and_(
model.extra_revision_table.c.current == True,
model.extra_revision_table.c.state.in_(
['active', 'deleted'])))).fetchall()
package_extra_inserts = []
for extra in extras:
insert = {}
for column in self.package_extra_table.c:
if column in extra:
insert[column.name] = extra[column.name]
insert['value'] = json.dumps(insert['value'])
package_extra_inserts.append(insert)
conn_drupal.execute(self.package_extra_table.insert(),
package_extra_inserts)
print 'extras done'
#tags
tags = conn_ckan.execute(
select([model.package_tag_revision_table, model.tag_table.c.name],
and_(
model.package_tag_revision_table.c.current == True,
model.package_tag_revision_table.c.state.in_(
['active', 'deleted']), model.tag_table.c.id ==
model.package_tag_revision_table.c.tag_id))).fetchall()
tag_inserts = []
for tag in tags:
tag_inserts.append({
'name': tag['name'],
'id': tag['id'],
'package_id': tag['package_id']
})
conn_drupal.execute(self.tag_table.insert(), tag_inserts)
print 'tags done'
## get nodes
packages = conn_drupal.execute(select(
[self.package_table], )).fetchall()
for num, package in enumerate(packages):
print num
data_dict = {}
for column in self.package_table.c:
data_dict[column.name] = package[column.name]
url = urlparse.urljoin(self.base_url, 'services/package.json')
data_dict['body'] = data_dict.get('notes', '')
if not data_dict['title']:
data_dict['title'] = data_dict['name']
tags = conn_drupal.execute(
select(
[self.tag_table.c.name],
self.tag_table.c.package_id == package['id'])).fetchall()
data_dict['tags'] = [{'name': tag[0]} for tag in tags]
data = json.dumps({'data': data_dict})
req = urllib2.Request(url, data,
{'Content-type': 'application/json'})
f = urllib2.urlopen(req, None, 3)
try:
drupal_info = json.loads(f.read())
finally:
f.close()
nid = drupal_info['nid']
update = {'nid': nid}
conn_drupal.execute(self.package_table.update().where(
self.package_table.c.id == data_dict['id']).values(nid=nid))
conn_drupal.execute(
'update ckan_resource set nid = (select nid from ckan_package where ckan_resource.package_id = ckan_package.id);'
)
conn_drupal.execute(
'update ckan_package_extra set nid = (select nid from ckan_package where ckan_package_extra.package_id = ckan_package.id);'
)
conn_drupal.execute(
'update ckan_tag set nid = (select nid from ckan_package where ckan_tag.package_id = ckan_package.id);'
)
print 'finished migration'
