def merge(self):
if len(self.args) == 3:
closing_id = unicode(self.args[1])
continuing_id = unicode(self.args[2])
elif len(self.args) == 2:
working_changeset = self.get_working_changeset()
if not working_changeset:
print "There is no working changeset to merge into '%s'." % continuing_id
sys.exit(1)
closing_id = working_changeset.id
continuing_id = unicode(self.args[1])
else:
print "Need a target changeset to merge into."
sys.exit(1)
from ckan.model.changeset import ChangesetRegister
from ckan.model.changeset import ConflictException
from ckan.model.changeset import Heads
from ckan.model.changeset import Resolve, CliResolve
changeset_register = ChangesetRegister()
if not len(changeset_register):
print "There are zero changesets in the changeset register."
sys.exit(1)
try:
resolve_class = self.options.is_interactive and CliResolve or Resolve
mergeset = changeset_register.merge(
closing_id=closing_id,
continuing_id=continuing_id,
resolve_class=resolve_class,
)
# Todo: Update repository before commiting changeset?
self.update_repository(mergeset.id)
print ""
except ConflictException, inst:
print inst
sys.exit(1)
def pull(self):
if len(self.args) > 1:
sources = [unicode(self.args[1])]
else:
from pylons import config
sources = config.get('changeset.sources', '').strip().split(',')
sources = [s.strip() for s in sources if s.strip()]
if not sources:
print "No changes source to pull (set 'changeset.sources' in config)."
return
from ckan.model.changeset import ChangesetRegister
from ckan.model.changeset import ChangesSourceException
changeset_register = ChangesetRegister()
is_error = False
for source in sources:
try:
changesets = changeset_register.pull(source)
except ChangesSourceException, inst:
print "%s" % inst
else:
print "Pulled %s changeset%s from '%s'." % (
len(changesets),
(len(changesets) == 1 and "" or "s"),
source
)
def heads(self):
from ckan.model.changeset import Heads
from ckan.model.changeset import ChangesetRegister
print "Most recent changeset for each active line:"
ids = Heads().ids()
ids.reverse() # Ordered by timestamp.
changeset_register = ChangesetRegister()
for id in ids:
head = changeset_register.get(id)
print ""
self.log_changeset(head)
def log(self):
if len(self.args) > 1:
changeset_id = unicode(self.args[1])
else:
changeset_id = None
from ckan.model.changeset import ChangesetRegister
changeset_register = ChangesetRegister()
if changeset_id:
changeset = changeset_register[changeset_id]
self.log_changeset(changeset)
self.print_changes(changeset.changes)
else:
changesets = changeset_register.values()
changesets.reverse() # Ordered by timestamp.
for changeset in changesets:
self.log_changeset(changeset)
print ""
def test_add_unseen(self):
changeset_data = {
"id":
"8772c628-435e-4896-ad04-514aab3d0d10",
"meta": {},
"changes": [{
"ref": "/package/4662c628-435e-4896-ad04-514aab3d0e66",
"diff": {
"new": {
"name": "coat",
"title": "Annie Get Your Coat",
"license_id": "abcd3"
},
"old": {}
},
}]
}
changeset_id = self.changesets.add_unseen(changeset_data)
changeset = ChangesetRegister()[changeset_id]
def diff(self):
if len(self.args) > 2:
changeset_id1 = unicode(self.args[1])
changeset_id2 = unicode(self.args[2])
elif len(self.args) > 1:
working_changeset = self.get_working_changeset()
if not working_changeset:
print "There is no working changeset."
sys.exit(1)
print "Displaying changes from the working changeset (without any outstanding revisions)..."
changeset_id1 = working_changeset.id
changeset_id2 = unicode(self.args[1])
else:
# Todo: Calc changes for outstanding revisions.
print "Sorry, displaying changes of uncommitted revisions is not yet supported."
print ""
print "Providing one target changeset will display the changes from the working changeset. Providing two target changesets will display the sum of changes between the first and the second target."
sys.exit(1)
from ckan.model.changeset import NoIntersectionException
from ckan.model.changeset import ChangesetRegister, Route, Reduce
register = ChangesetRegister()
changeset1 = register.get(changeset_id1, None)
if not changeset1:
print "Changeset '%s' not found." % changeset_id1
sys.exit(1)
changeset2 = register.get(changeset_id2, None)
if not changeset2:
print "Changeset '%s' not found." % changeset_id2
sys.exit(1)
route = Route(changeset1, changeset2)
try:
changes = route.calc_changes()
# Todo: Calc and sum with changes for outstanding revisions.
changes = Reduce(changes).calc_changes()
except NoIntersectionException:
print "The changesets '%s' and '%s' are not on intersecting lines." % (
changeset_id1, changeset_id2
)
else:
print "diff %s %s" % (changeset_id1, changeset_id2)
self.print_changes(changes)
def update_repository(self, changeset_id):
from ckan.model.changeset import ChangesetRegister
from ckan.model.changeset import EmptyChangesetRegisterException
from ckan.model.changeset import UncommittedChangesException
from ckan.model.changeset import WorkingAtHeadException
from ckan.model.changeset import ConflictException
changeset_register = ChangesetRegister()
report = {
'created': [],
'updated': [],
'deleted': [],
}
try:
changeset_register.update(
target_id=changeset_id,
report=report,
moderator=self.options.is_interactive and self or None,
)
except ConflictException, inst:
print "Update aborted due to conflict with the working model."
print inst
sys.exit(1)
def create_cs(self, **kwds):
cs = ChangesetRegister().create_entity(**kwds)
model.Session.commit()
return cs
def setup(self):
super(TestChangesetRegister, self).setup()
self.changesets = ChangesetRegister()
self.revisions = RevisionRegister()
self.packages = PackageRegister()
self.changes = ChangeRegister()
def create_arbitrary(cls,
package_dicts,
relationships=[],
extra_user_names=[],
extra_group_names=[],
commit_changesets=False,
admins=[]):
'''Creates packages and a few extra objects as well at the
same time if required.
@param package_dicts - a list of dictionaries with the package
properties.
Extra keys allowed:
"admins" - list of user names to make admin
for this package.
@param extra_group_names - a list of group names to create. No
properties get set though.
@param admins - a list of user names to make admins of all the
packages created.
'''
assert isinstance(relationships, (list, tuple))
assert isinstance(extra_user_names, (list, tuple))
assert isinstance(extra_group_names, (list, tuple))
import ckan.model as model
model.Session.remove()
new_user_names = extra_user_names
new_group_names = set()
rev = model.repo.new_revision()
rev.author = cls.author
rev.message = u'Creating test packages.'
admins_list = defaultdict(list) # package_name: admin_names
if package_dicts:
if isinstance(package_dicts, dict):
package_dicts = [package_dicts]
for item in package_dicts:
pkg_dict = {}
for field in cls.pkg_core_fields:
if item.has_key(field):
pkg_dict[field] = unicode(item[field])
pkg = model.Package(**pkg_dict)
model.Session.add(pkg)
for attr, val in item.items():
if isinstance(val, str):
val = unicode(val)
if attr == 'name':
continue
if attr in cls.pkg_core_fields:
pass
elif attr == 'download_url':
pkg.add_resource(unicode(val))
elif attr == 'resources':
assert isinstance(val, (list, tuple))
for res_dict in val:
non_extras = dict([(str(k), unicode(v))
for k, v in res_dict.items()
if k != 'extras'])
extras = dict([
(str(k), unicode(v))
for k, v in res_dict.get('extras', {}).items()
])
pkg.add_resource(extras=extras, **non_extras)
elif attr == 'tags':
if isinstance(val, (str, unicode)):
tags = val.split()
elif isinstance(val, list):
tags = val
else:
raise NotImplementedError
for tag_name in tags:
tag_name = unicode(tag_name)
tag = model.Tag.by_name(tag_name)
if not tag:
tag = model.Tag(name=tag_name)
cls.tag_names.append(tag_name)
model.Session.add(tag)
pkg.tags.append(tag)
model.Session.flush()
elif attr == 'groups':
if isinstance(val, (str, unicode)):
group_names = val.split()
elif isinstance(val, list):
group_names = val
else:
raise NotImplementedError
for group_name in group_names:
group = model.Group.by_name(unicode(group_name))
if not group:
group = model.Group(name=unicode(group_name))
model.Session.add(group)
new_group_names.add(group_name)
pkg.groups.append(group)
elif attr == 'license':
pkg.license_id = val
elif attr == 'license_id':
pkg.license_id = val
elif attr == 'extras':
pkg.extras = val
elif attr == 'admins':
assert isinstance(val, list)
admins_list[item['name']].extend(val)
for user_name in val:
if user_name not in new_user_names:
new_user_names.append(user_name)
else:
raise NotImplementedError(attr)
cls.pkg_names.append(item['name'])
model.setup_default_user_roles(pkg, admins=[])
for admin in admins:
admins_list[item['name']].append(admin)
model.repo.commit_and_remove()
needs_commit = False
rev = model.repo.new_revision()
for group_name in extra_group_names:
group = model.Group(name=unicode(group_name))
model.Session.add(group)
new_group_names.add(group_name)
needs_commit = True
if needs_commit:
model.repo.commit_and_remove()
needs_commit = False
# create users that have been identified as being needed
for user_name in new_user_names:
if not model.User.by_name(unicode(user_name)):
user = model.User(name=unicode(user_name))
model.Session.add(user)
cls.user_refs.append(user_name)
needs_commit = True
if needs_commit:
model.repo.commit_and_remove()
needs_commit = False
# setup authz for admins
for pkg_name, admins in admins_list.items():
pkg = model.Package.by_name(unicode(pkg_name))
admins_obj_list = []
for admin in admins:
if isinstance(admin, model.User):
admin_obj = admin
else:
admin_obj = model.User.by_name(unicode(admin))
assert admin_obj, admin
admins_obj_list.append(admin_obj)
model.setup_default_user_roles(pkg, admins_obj_list)
needs_commit = True
# setup authz for groups just created
for group_name in new_group_names:
group = model.Group.by_name(unicode(group_name))
model.setup_default_user_roles(group)
cls.group_names.add(group_name)
needs_commit = True
if needs_commit:
model.repo.commit_and_remove()
needs_commit = False
if relationships:
rev = model.repo.new_revision()
rev.author = cls.author
rev.message = u'Creating package relationships.'
def pkg(pkg_name):
return model.Package.by_name(unicode(pkg_name))
for subject_name, relationship, object_name in relationships:
pkg(subject_name).add_relationship(unicode(relationship),
pkg(object_name))
needs_commit = True
model.repo.commit_and_remove()
if commit_changesets:
from ckan.model.changeset import ChangesetRegister
changeset_ids = ChangesetRegister().commit()
def create(cls, commit_changesets=False):
import ckan.model as model
model.Session.remove()
rev = model.repo.new_revision()
# same name as user we create below
rev.author = cls.author
rev.message = u'''Creating test data.
* Package: annakarenina
* Package: warandpeace
* Associated tags, etc etc
'''
cls.pkg_names = [u'annakarenina', u'warandpeace']
pkg1 = model.Package(name=cls.pkg_names[0])
model.Session.add(pkg1)
pkg1.title = u'A Novel By Tolstoy'
pkg1.version = u'0.7a'
pkg1.url = u'http://www.annakarenina.com'
# put an & in the url string to test escaping
if 'alt_url' in model.Resource.get_extra_columns():
configured_extras = ({
'alt_url': u'alt123'
}, {
'alt_url': u'alt345'
})
else:
configured_extras = ({}, {})
pr1 = model.Resource(
url=u'http://www.annakarenina.com/download/x=1&y=2',
format=u'plain text',
description=u'Full text. Needs escaping: " Umlaut: \xfc',
hash=u'abc123',
extras={'size': u'123'},
**configured_extras[0])
pr2 = model.Resource(url=u'http://www.annakarenina.com/index.json',
format=u'json',
description=u'Index of the novel',
hash=u'def456',
extras={'size': u'345'},
**configured_extras[1])
model.Session.add(pr1)
model.Session.add(pr2)
pkg1.resources.append(pr1)
pkg1.resources.append(pr2)
pkg1.notes = u'''Some test notes
### A 3rd level heading
**Some bolded text.**
*Some italicized text.*
Foreign characters:
u with umlaut \xfc
66-style quote \u201c
foreign word: th\xfcmb
Needs escaping:
left arrow <
<http://ckan.net/>
'''
pkg2 = model.Package(name=cls.pkg_names[1])
tag1 = model.Tag(name=u'russian')
tag2 = model.Tag(name=u'tolstoy')
for obj in [pkg2, tag1, tag2]:
model.Session.add(obj)
pkg1.tags = [tag1, tag2]
pkg2.tags = [tag1]
cls.tag_names = [u'russian', u'tolstoy']
pkg1.license_id = u'other-open'
pkg2.license_id = u'cc-nc' # closed license
pkg2.title = u'A Wonderful Story'
pkg1.extras = {u'genre': 'romantic novel', u'original media': 'book'}
# group
david = model.Group(name=u'david',
title=u'Dave\'s books',
description=u'These are books that David likes.')
roger = model.Group(name=u'roger',
title=u'Roger\'s books',
description=u'Roger likes these books.')
for obj in [david, roger]:
model.Session.add(obj)
cls.group_names.add(u'david')
cls.group_names.add(u'roger')
model.Session.add(model.PackageGroup(package=pkg1, group=david))
model.Session.add(model.PackageGroup(package=pkg2, group=david))
model.Session.add(model.PackageGroup(package=pkg1, group=roger))
# authz
model.Session.add_all([
model.User(name=u'tester', apikey=u'tester', password=u'tester'),
model.User(name=u'joeadmin', password=u'joeadmin'),
model.User(
name=u'annafan',
about=
u'I love reading Annakarenina. My site: <a href="http://anna.com">anna.com</a>',
password=u'annafan'),
model.User(name=u'russianfan', password=u'russianfan'),
model.User(name=u'testsysadmin', password=u'testsysadmin'),
])
cls.user_refs.extend([
u'tester', u'joeadmin', u'annafan', u'russianfan', u'testsysadmin'
])
model.repo.commit_and_remove()
visitor = model.User.by_name(model.PSEUDO_USER__VISITOR)
anna = model.Package.by_name(u'annakarenina')
war = model.Package.by_name(u'warandpeace')
annafan = model.User.by_name(u'annafan')
russianfan = model.User.by_name(u'russianfan')
model.setup_default_user_roles(anna, [annafan])
model.setup_default_user_roles(war, [russianfan])
model.add_user_to_role(visitor, model.Role.ADMIN, war)
david = model.Group.by_name(u'david')
roger = model.Group.by_name(u'roger')
model.setup_default_user_roles(david, [russianfan])
model.setup_default_user_roles(roger, [russianfan])
model.add_user_to_role(visitor, model.Role.ADMIN, roger)
testsysadmin = model.User.by_name(u'testsysadmin')
model.add_user_to_role(testsysadmin, model.Role.ADMIN, model.System())
model.repo.commit_and_remove()
if commit_changesets:
from ckan.model.changeset import ChangesetRegister
changeset_ids = ChangesetRegister().commit()
# Create a couple of authorization groups
for ag_name in [u'anauthzgroup', u'anotherauthzgroup']:
ag = model.AuthorizationGroup.by_name(ag_name)
if not ag: #may already exist, if not create
ag = model.AuthorizationGroup(name=ag_name)
model.Session.add(ag)
model.repo.commit_and_remove()
# and give them a range of roles on various things
ag = model.AuthorizationGroup.by_name(u'anauthzgroup')
aag = model.AuthorizationGroup.by_name(u'anotherauthzgroup')
pkg = model.Package.by_name(u'warandpeace')
g = model.Group.by_name('david')
model.add_authorization_group_to_role(ag, u'editor', model.System())
model.add_authorization_group_to_role(ag, u'reader', pkg)
model.add_authorization_group_to_role(ag, u'admin', aag)
model.add_authorization_group_to_role(aag, u'editor', ag)
model.add_authorization_group_to_role(ag, u'editor', g)
model.repo.commit_and_remove()
def get_working_changeset(self):
from ckan.model.changeset import ChangesetRegister
changeset_register = ChangesetRegister()
return changeset_register.get_working()
def commit(self):
from ckan.model.changeset import ChangesetRegister
changeset_register = ChangesetRegister()
changeset_ids = changeset_register.commit()
print "Committed %s revision%s." % (len(changeset_ids), (len(changeset_ids) != 1) and "s" or "")
fs.sync()
except Exception, inst:
model.Session.rollback()
raise
else:
# only commit if desired and it validates ok
if commit and validates:
model.Session.commit()
elif validates:
# i.e. preview
pkg = fs.model
return pkg
if rev and 'true' == config.get('changeset.auto_commit', '').strip():
try:
from ckan.model.changeset import ChangesetRegister
changeset_ids = ChangesetRegister().commit()
for id in changeset_ids:
msg = "PackageSaver auto-committed changeset '%s'." % id
logging.info(msg)
except Exception, inst:
msg = "PackageSaver failed to auto-commit revision '%s': %s" % (
rev.id, inst)
logging.error(msg)
@classmethod
def _revision_validation(cls, log_message):
errors = []
if log_message and 'http:' in log_message:
errors.append(_('No links are allowed in the log_message.'))
return errors
