def am_migratecollection(annroot, userhome, options):
"""
Apply migrations for a specified collection
annalist_manager migratecollection coll
Reads and writes every entity in a collection, thereby applying data
migrations and saving them in the stored data.
annroot is the root directory for the Annalist software installation.
userhome is the home directory for the host system user issuing the command.
options contains options parsed from the command line.
returns 0 if all is well, or a non-zero status code.
This value is intended to be used as an exit status code
for the calling program.
"""
status, settings, site = get_settings_site(annroot, userhome, options)
if status != am_errors.AM_SUCCESS:
return status
coll_id = getargvalue(getarg(options.args, 0), "Collection Id: ")
coll = Collection.load(site, coll_id)
if not (coll and coll.get_values()):
print("Collection not found: %s" % (coll_id), file=sys.stderr)
return am_errors.AM_NOCOLLECTION
status = am_errors.AM_SUCCESS
print("Apply data migrations in collection '%s'" % (coll_id, ))
msgs = migrate_coll_data(coll)
if msgs:
for msg in msgs:
print(msg)
status = am_errors.AM_MIGRATECOLLFAIL
return status
def get_coll_info(self, coll_id):
"""
Check collection identifier, and get reference to collection object.
"""
assert (self.site is not None)
if not self.http_response:
if not Collection.exists(self.site, coll_id):
self.http_response = self.view.error(
dict(self.view.error404values(),
message=message.COLLECTION_NOT_EXISTS %
{'id': coll_id}))
else:
self.coll_id = coll_id
#@@TODO: try with altscope="site"?
self.collection = Collection.load(self.site,
coll_id,
altscope="all")
self.coll_perms = self.collection
ver = self.collection.get(ANNAL.CURIE.software_version,
None) or "0.0.0"
if LooseVersion(ver) > LooseVersion(annalist.__version__):
self.http_response = self.view.error(
dict(self.view.error500values(),
message=message.COLLECTION_NEWER_VERSION % {
'id': coll_id,
'ver': ver
}))
return self.http_response
def am_migratecollection(annroot, userhome, options):
"""
Apply migrations for a specified collection
annalist_manager migratecollection coll
Reads and writes every entity in a collection, thereby applying data
migrations and saving them in the stored data.
annroot is the root directory for the Annalist software installation.
userhome is the home directory for the host system user issuing the command.
options contains options parsed from the command line.
returns 0 if all is well, or a non-zero status code.
This value is intended to be used as an exit status code
for the calling program.
"""
status, settings, site = get_settings_site(annroot, userhome, options)
if status != am_errors.AM_SUCCESS:
return status
coll_id = getargvalue(getarg(options.args, 0), "Collection Id: ")
coll = Collection.load(site, coll_id)
if not (coll and coll.get_values()):
print("Collection not found: %s"%(coll_id), file=sys.stderr)
return am_errors.AM_NOCOLLECTION
status = am_errors.AM_SUCCESS
print("Apply data migrations in collection '%s'"%(coll_id,))
msgs = migrate_coll_data(coll)
if msgs:
for msg in msgs:
print(msg)
status = am_errors.AM_MIGRATECOLLFAIL
return status
def create_test_coll_inheriting(base_coll_id=None,
coll_id="testcoll",
type_id="testtype"):
"""
Similar to init_annalist_test_coll, but collection also
inherits from named collection.
"""
testsite = Site(TestBaseUri, TestBaseDir)
basecoll = Collection.load(testsite, base_coll_id)
if not basecoll:
msg = "Base collection %s not found" % base_coll_id
log.warning(msg)
assert False, msg
testcoll = Collection.create(testsite, coll_id,
collection_create_values(coll_id))
testcoll.set_alt_entities(basecoll)
testcoll._save()
testtype = RecordType.create(testcoll, type_id,
recordtype_create_values(coll_id, type_id))
testdata = RecordTypeData.create(testcoll, type_id, {})
teste = EntityData.create(
testdata, "entity1",
entitydata_create_values(testcoll, testtype, "entity1"))
testcoll.generate_coll_jsonld_context()
return testcoll
def get_coll_info(self, coll_id):
"""
Check collection identifier, and get reference to collection object.
"""
assert (self.site is not None)
if not self.http_response:
if not Collection.exists(self.site, coll_id):
self.http_response = self.view.error(
dict(self.view.error404values(),
message=message.COLLECTION_NOT_EXISTS%{'id': coll_id}
)
)
else:
self.coll_id = coll_id
#@@TODO: try with altscope="site"?
self.collection = Collection.load(self.site, coll_id, altscope="all")
self.coll_perms = self.collection
ver = self.collection.get(ANNAL.CURIE.software_version, None) or "0.0.0"
if LooseVersion(ver) > LooseVersion(annalist.__version__):
self.http_response = self.view.error(
dict(self.view.error500values(),
message=message.COLLECTION_NEWER_VERSION%{'id': coll_id, 'ver': ver}
)
)
return self.http_response
def am_installcollection(annroot, userhome, options):
"""
Install software-defined collection data
annalist_manager installcollection coll_id
Copies data from an existing collection to a new collection.
annroot is the root directory for the Annalist software installation.
userhome is the home directory for the host system user issuing the command.
options contains options parsed from the command line.
returns 0 if all is well, or a non-zero status code.
This value is intended to be used as an exit status code
for the calling program.
"""
status, settings, site = get_settings_site(annroot, userhome, options)
if status != am_errors.AM_SUCCESS:
return status
if len(options.args) > 1:
print(
"Unexpected arguments for %s: (%s)"%
(options.command, " ".join(options.args)),
file=sys.stderr
)
return am_errors.AM_UNEXPECTEDARGS
# Check collection Id
coll_id = getargvalue(getarg(options.args, 0), "Collection Id to install: ")
if coll_id in installable_collections:
src_dir_name = installable_collections[coll_id]['data_dir']
else:
print("Collection name to install not known: %s"%(coll_id), file=sys.stderr)
print("Available collection Ids are: %s"%(",".join(installable_collections.keys())))
return am_errors.AM_NOCOLLECTION
# Check if ciollection already exists
coll = Collection.load(site, coll_id)
if (coll and coll.get_values()):
if options.force:
print("Existing collection %s will be removed ('--force' specified)"%(coll_id), file=sys.stderr)
Collection.remove(site, coll_id)
else:
print("Collection already exists: %s"%(coll_id), file=sys.stderr)
return am_errors.AM_COLLECTIONEXISTS
# Install collection now
src_dir = os.path.join(annroot, "annalist/data", src_dir_name)
print("Installing collection '%s' from data directory '%s'"%(coll_id, src_dir))
coll_metadata = installable_collections[coll_id]['coll_meta']
date_time_now = datetime.datetime.now().replace(microsecond=0)
coll_metadata[ANNAL.CURIE.comment] = (
"Initialized at %s by `annalist-manager installcollection`"%
date_time_now.isoformat()
)
coll = site.add_collection(coll_id, coll_metadata)
msgs = initialize_coll_data(src_dir, coll)
if msgs:
for msg in msgs:
print(msg)
status = am_errors.AM_INSTALLCOLLFAIL
return status
def am_copycollection(annroot, userhome, options):
"""
Copy collection data
annalist_manager copycollection old_coll_id new_coll_id
Copies data from an existing collection to a new collection.
annroot is the root directory for the Annalist software installation.
userhome is the home directory for the host system user issuing the command.
options contains options parsed from the command line.
returns 0 if all is well, or a non-zero status code.
This value is intended to be used as an exit status code
for the calling program.
"""
status, settings, site = get_settings_site(annroot, userhome, options)
if status != am_errors.AM_SUCCESS:
return status
if len(options.args) > 2:
print(
"Unexpected arguments for %s: (%s)"%
(options.command, " ".join(options.args)),
file=sys.stderr
)
return am_errors.AM_UNEXPECTEDARGS
old_coll_id = getargvalue(getarg(options.args, 0), "Old collection Id: ")
old_coll = Collection.load(site, old_coll_id)
if not (old_coll and old_coll.get_values()):
print("Old collection not found: %s"%(old_coll_id), file=sys.stderr)
return am_errors.AM_NOCOLLECTION
new_coll_id = getargvalue(getarg(options.args, 1), "New collection Id: ")
new_coll = Collection.load(site, new_coll_id)
if (new_coll and new_coll.get_values()):
print("New collection already exists: %s"%(new_coll_id), file=sys.stderr)
return am_errors.AM_COLLECTIONEXISTS
# Copy collection now
print("Copying collection '%s' to '%s'"%(old_coll_id, new_coll_id))
new_coll = site.add_collection(new_coll_id, old_coll.get_values())
msgs = copy_coll_data(old_coll, new_coll)
if msgs:
for msg in msgs:
print(msg)
status = am_errors.AM_COPYCOLLFAIL
print("")
return status
def am_copycollection(annroot, userhome, options):
"""
Copy collection data
annalist_manager copycollection old_coll_id new_coll_id
Copies data from an existing collection to a new collection.
annroot is the root directory for the Annalist software installation.
userhome is the home directory for the host system user issuing the command.
options contains options parsed from the command line.
returns 0 if all is well, or a non-zero status code.
This value is intended to be used as an exit status code
for the calling program.
"""
status, settings, site = get_settings_site(annroot, userhome, options)
if status != am_errors.AM_SUCCESS:
return status
if len(options.args) > 2:
print("Unexpected arguments for %s: (%s)" %
(options.command, " ".join(options.args)),
file=sys.stderr)
return am_errors.AM_UNEXPECTEDARGS
old_coll_id = getargvalue(getarg(options.args, 0), "Old collection Id: ")
old_coll = Collection.load(site, old_coll_id)
if not (old_coll and old_coll.get_values()):
print("Old collection not found: %s" % (old_coll_id), file=sys.stderr)
return am_errors.AM_NOCOLLECTION
new_coll_id = getargvalue(getarg(options.args, 1), "New collection Id: ")
new_coll = Collection.load(site, new_coll_id)
if (new_coll and new_coll.get_values()):
print("New collection already exists: %s" % (new_coll_id),
file=sys.stderr)
return am_errors.AM_COLLECTIONEXISTS
# Copy collection now
print("Copying collection '%s' to '%s'" % (old_coll_id, new_coll_id))
new_coll = site.add_collection(new_coll_id, old_coll.get_values())
msgs = copy_coll_data(old_coll, new_coll)
if msgs:
for msg in msgs:
print(msg)
status = am_errors.AM_COPYCOLLFAIL
print("")
return status
def am_installcollection(annroot, userhome, options):
"""
Install software-defined collection data
annalist_manager installcollection coll_id
Copies data from an existing collection to a new collection.
annroot is the root directory for the Annalist software installation.
userhome is the home directory for the host system user issuing the command.
options contains options parsed from the command line.
returns 0 if all is well, or a non-zero status code.
This value is intended to be used as an exit status code
for the calling program.
"""
status, settings, site = get_settings_site(annroot, userhome, options)
if status != am_errors.AM_SUCCESS:
return status
if len(options.args) > 1:
print("Unexpected arguments for %s: (%s)" %
(options.command, " ".join(options.args)),
file=sys.stderr)
return am_errors.AM_UNEXPECTEDARGS
coll_id = getargvalue(getarg(options.args, 0),
"Collection Id to install: ")
coll = Collection.load(site, coll_id)
if (coll and coll.get_values()):
print("Collection already exists: %s" % (coll_id), file=sys.stderr)
return am_errors.AM_COLLECTIONEXISTS
# Check collection Id
if coll_id in installable_collections:
src_dir_name = installable_collections[coll_id]['data_dir']
else:
print("Collection name to install not known: %s" % (coll_id),
file=sys.stderr)
print("Available collection Ids are: %s" %
(",".join(installable_collections.keys())))
return am_errors.AM_NOCOLLECTION
# Install collection now
src_dir = os.path.join(annroot, "annalist/data", src_dir_name)
print("Installing collection '%s' from data directory '%s'" %
(coll_id, src_dir))
coll_metadata = installable_collections[coll_id]['coll_meta']
date_time_now = datetime.datetime.now().replace(microsecond=0)
coll_metadata[ANNAL.CURIE.comment] = (
"Initialized at %s by `annalist-manager installcollection`" %
date_time_now.isoformat())
coll = site.add_collection(coll_id, coll_metadata)
msgs = initialize_coll_data(src_dir, coll)
if msgs:
for msg in msgs:
print(msg)
status = am_errors.AM_INSTALLCOLLFAIL
return status
def setUp(self):
init_annalist_test_site()
self.testsite = Site(TestBaseUri, TestBaseDir)
self.testcoll = Collection.create(
self.testsite, "testcoll", collection_create_values("testcoll")
)
self.sitecoll = Collection.load(self.testsite, layout.SITEDATA_ID)
self.client = Client(HTTP_HOST=TestHost)
self.uri = reverse("AnnalistServerLogView")
self.homeuri = reverse("AnnalistHomeView")
return
def _check_collection_data_values(self, coll_id=None):
"""
Helper function checks content of annalist collection data
"""
self.assertTrue(Collection.exists(self.testsite, coll_id))
t = Collection.load(self.testsite, coll_id)
self.assertEqual(t.get_id(), coll_id)
self.assertEqual(t.get_view_url_path(), collection_view_url(coll_id="testcoll"))
v = collectiondata_values(coll_id=coll_id)
self.assertDictionaryMatch(t.get_values(), v)
return t
def _check_collection_data_values(self, coll_id=None):
"""
Helper function checks content of annalist collection data
"""
self.assertTrue(Collection.exists(self.testsite, coll_id))
t = Collection.load(self.testsite, coll_id)
self.assertEqual(t.get_id(), coll_id)
self.assertEqual(t.get_view_url_path(),
collection_view_url(coll_id="testcoll"))
v = collectiondata_values(coll_id=coll_id)
self.assertDictionaryMatch(t.get_values(), v)
return t
def setUp(self):
init_annalist_test_site()
self.testsite = Site(TestBaseUri, TestBaseDir)
self.coll1 = Collection.load(self.testsite, "coll1")
self.view_url = collection_view_url(coll_id="coll1")
self.edit_url = collection_edit_url(coll_id="coll1")
self.continuation = "?" + continuation_url_param(self.edit_url)
# Login and permissions
create_test_user(self.coll1, "testuser", "testpassword")
self.client = Client(HTTP_HOST=TestHost)
loggedin = self.client.login(username="******", password="******")
self.assertTrue(loggedin)
return
def collections(self):
"""
Generator enumerates and returns collection descriptions that are part of a site.
Yielded values are collection objects.
"""
log.debug("site.collections: basedir: %s"%(self._entitydir))
for f in self._base_children(Collection):
c = Collection.load(self, f)
# log.info("Site.colections: Collection.load %s %r"%(f, c.get_values()))
if c:
yield c
return
def collections(self):
"""
Generator enumerates and returns collection descriptions that are part of a site.
Yielded values are collection objects.
"""
log.debug("site.collections: basedir: %s" % (self._entitydir))
for f in self._base_children(Collection):
c = Collection.load(self, f)
# log.info("Site.colections: Collection.load %s %r"%(f, c.get_values()))
if c:
yield c
return
def test_edit_collection_metadata(self):
# This test performs a GET to retrieve values used in a form,
# then a POST to save updated collection metadata.
# This test is intended to test a problem encountered with updated
# entity copying logic that needs to take special account of collection
# entities being presented as offspring of the site while being stored
# as part of a collection.
#
coll_id = "testcoll"
self.assertTrue(Collection.exists(self.testsite, coll_id))
c = Collection.load(self.testsite, coll_id)
self.assertEqual(c.get_id(), coll_id)
self.assertEqual(c.get_view_url_path(), collection_view_url(coll_id="testcoll"))
# GET collection metadata form data, and test values
u = collectiondata_url(coll_id="testcoll")
r = self.client.get(u)
self.assertEqual(r.status_code, 200)
self.assertEqual(r.reason_phrase, "OK")
self.assertEqual(r.context['coll_id'], layout.SITEDATA_ID)
self.assertEqual(r.context['type_id'], layout.COLL_TYPEID)
self.assertEqual(r.context['entity_id'], "testcoll")
self.assertEqual(r.context['orig_id'], "testcoll")
self.assertEqual(r.context['orig_coll'], layout.SITEDATA_ID)
self.assertEqual(r.context['action'], "view")
self.assertEqual(r.context['continuation_url'], "")
orig_coll = r.context['orig_coll']
# Assemble and POST form data to =updated collection metadata
new_label = "Updated collection metadata"
f = coll_view_form_data(
coll_id="testcoll",
action="edit",
coll_label=new_label,
# orig_coll="None"
orig_coll=layout.SITEDATA_ID
)
u = collectiondata_view_url(coll_id="testcoll", action="edit")
r = self.client.post(u, f)
self.assertEqual(r.status_code, 302)
self.assertEqual(r.reason_phrase, "FOUND")
self.assertEqual(r.content, "")
self.assertEqual(r['location'], TestBaseUri+"/c/_annalist_site/d/_coll/")
# Check updated collection data
self._check_collection_data_values(coll_id="testcoll", coll_label=new_label)
return
def setUp(self):
init_annalist_test_site()
self.testsite = Site(TestBaseUri, TestBaseDir)
self.testcoll = Collection.create(
self.testsite, "testcoll", collection_create_values("testcoll")
)
self.annalcoll = Collection.load(self.testsite, layout.SITEDATA_ID)
self.no_options = [ FieldChoice('', label="(no options)") ]
# For checking Location: header values...
self.continuation_url = (
entitydata_list_type_url(coll_id="testcoll", type_id=layout.VOCAB_TYPEID)
)
# Login and permissions
create_test_user(self.testcoll, "testuser", "testpassword")
create_user_permissions(self.annalcoll, "testuser", user_permissions=["VIEW"])
self.client = Client(HTTP_HOST=TestHost)
loggedin = self.client.login(username="******", password="******")
self.assertTrue(loggedin)
return
def setUp(self):
init_annalist_test_site()
self.testsite = Site(TestBaseUri, TestBaseDir)
self.testcoll = Collection.create(self.testsite, "testcoll",
collection_create_values("testcoll"))
self.annalcoll = Collection.load(self.testsite, layout.SITEDATA_ID)
self.no_options = [FieldChoice('', label="(no options)")]
# For checking Location: header values...
self.continuation_url = (TestHostUri + entitydata_list_type_url(
coll_id="testcoll", type_id=layout.VOCAB_TYPEID))
# Login and permissions
create_test_user(self.testcoll, "testuser", "testpassword")
create_user_permissions(self.annalcoll,
"testuser",
user_permissions=["VIEW"])
self.client = Client(HTTP_HOST=TestHost)
loggedin = self.client.login(username="******",
password="******")
self.assertTrue(loggedin)
return
def get_entity_info(self, action, entity_id):
"""
Set up entity id and info to use for display
Also handles some special case permissions settings if the entity is a Collection.
"""
if not self.http_response:
assert self.entitytypeinfo is not None
self.src_entity_id = entity_id
if action in ["new", "copy"]:
self.use_entity_id = self.entitytypeinfo.entityclass.allocate_new_id(
self.entitytypeinfo.entityparent, base_id=entity_id)
else:
self.use_entity_id = entity_id
# Special case permissions...
if self.type_id == "_coll":
# log.info("DisplayInfo.get_entity_info: access collection data for %s"%entity_id)
c = Collection.load(self.site, entity_id, altscope="all")
if c:
self.coll_perms = c
return self.http_response
def get_entity_info(self, action, entity_id):
"""
Set up entity id and info to use for display
Also handles some special case permissions settings if the entity is a Collection.
"""
if not self.http_response:
assert self.entitytypeinfo is not None
self.src_entity_id = entity_id
if action in ["new", "copy"]:
self.use_entity_id = self.entitytypeinfo.entityclass.allocate_new_id(
self.entitytypeinfo.entityparent, base_id=entity_id
)
else:
self.use_entity_id = entity_id
# Special case permissions...
if self.type_id == "_coll":
# log.info("DisplayInfo.get_entity_info: access collection data for %s"%entity_id)
c = Collection.load(self.site, entity_id, altscope="all")
if c:
self.coll_perms = c
return self.http_response
def create_test_coll_inheriting(
base_coll_id=None, coll_id="testcoll", type_id="testtype"):
"""
Similar to init_annalist_test_coll, but collection also
inherits from named collection.
"""
testsite = Site(TestBaseUri, TestBaseDir)
basecoll = Collection.load(testsite, base_coll_id)
if not basecoll:
msg = "Base collection %s not found"%base_coll_id
log.warning(msg)
assert False, msg
testcoll = Collection.create(testsite, coll_id, collection_create_values(coll_id))
testcoll.set_alt_entities(basecoll)
testcoll._save()
testtype = RecordType.create(testcoll, type_id, recordtype_create_values(coll_id, type_id))
testdata = RecordTypeData.create(testcoll, type_id, {})
teste = EntityData.create(
testdata, "entity1",
entitydata_create_values(testcoll, testtype, "entity1")
)
testcoll.generate_coll_jsonld_context()
return testcoll
def am_migrationreport(annroot, userhome, options):
"""
Collection migration report helper
annalist_manager migrationreport old_coll new_coll
Generates a report of changes to data needed to match type and property
URI changes moving from old_coll to new_coll.
annroot is the root directory for the Annalist software installation.
userhome is the home directory for the host system user issuing the command.
options contains options parsed from the command line.
returns 0 if all is well, or a non-zero status code.
This value is intended to be used as an exit status code
for the calling program.
"""
status, settings, site = get_settings_site(annroot, userhome, options)
if status != am_errors.AM_SUCCESS:
return status
if len(options.args) > 2:
print("Unexpected arguments for %s: (%s)" %
(options.command, " ".join(options.args)),
file=sys.stderr)
return am_errors.AM_UNEXPECTEDARGS
old_coll_id = getargvalue(getarg(options.args, 0), "Old collection Id: ")
old_coll = Collection.load(site, old_coll_id)
if not (old_coll and old_coll.get_values()):
print("Old collection not found: %s" % (old_coll_id), file=sys.stderr)
return am_errors.AM_NOCOLLECTION
new_coll_id = getargvalue(getarg(options.args, 1), "New collection Id: ")
new_coll = Collection.load(site, new_coll_id)
if not (new_coll and new_coll.get_values()):
print("New collection not found: %s" % (new_coll_id), file=sys.stderr)
return am_errors.AM_NOCOLLECTION
status = am_errors.AM_SUCCESS
print("# Migration report from collection '%s' to '%s' #" %
(old_coll_id, new_coll_id))
print("")
# Scan and report on type URI changes
for new_type in coll_types(new_coll):
type_id = new_type.get_id()
old_type = old_coll.get_type(type_id)
if old_type:
old_uri = old_type[ANNAL.CURIE.uri]
new_uri = new_type[ANNAL.CURIE.uri]
if old_uri != new_uri:
print("* Type %s, URI changed from '%s' to '%s'" %
(type_id, old_uri, new_uri))
supertype_uris = [
u[ANNAL.CURIE.supertype_uri]
for u in new_type.get(ANNAL.CURIE.supertype_uris, [])
]
if old_uri not in supertype_uris:
print(
" Consider adding supertype '%s' to type '%s' in collection '%s'"
% (old_uri, type_id, new_coll_id))
report_type_references(new_coll, old_uri,
" URI '%s'" % (old_uri))
# Scan and report on property URI changes in field definitions
for new_field in coll_fields(new_coll):
field_id = new_field.get_id()
old_field = coll_field(old_coll, field_id)
if old_field:
old_uri = old_field[ANNAL.CURIE.property_uri]
new_uri = new_field[ANNAL.CURIE.property_uri]
if old_uri != new_uri:
print("* Field %s, property URI changed from '%s' to '%s'" %
(field_id, old_uri, new_uri))
type_ids = types_using_field(new_coll, field_id, old_uri)
for tid in type_ids:
print(
" Consider adding property alias for '%s' to type %s in collection '%s'"
% (old_uri, tid, new_coll_id))
# Scan and report on property URI changes in group definitions
for new_group in coll_groups(new_coll):
group_id = new_group.get_id()
old_group = coll_group(old_coll, group_id)
if old_group:
compare_field_list(old_coll, new_coll,
old_group[ANNAL.CURIE.group_fields],
new_group[ANNAL.CURIE.group_fields],
"Group %s" % group_id)
# Scan and report on property URI changes in view definitions
for new_view in coll_views(new_coll):
view_id = new_view.get_id()
old_view = coll_view(old_coll, view_id)
if old_view:
compare_field_list(old_coll, new_coll,
old_view[ANNAL.CURIE.view_fields],
new_view[ANNAL.CURIE.view_fields],
"View %s" % view_id)
# Scan and report on property URI changes in list definitions
for new_list in coll_lists(new_coll):
list_id = new_list.get_id()
old_list = coll_list(old_coll, list_id)
if old_list:
compare_field_list(old_coll, new_coll,
old_list[ANNAL.CURIE.list_fields],
new_list[ANNAL.CURIE.list_fields],
"List %s" % list_id)
print("")
return status
def am_migrationreport(annroot, userhome, options):
"""
Collection migration report helper
annalist_manager migrationreport old_coll new_coll
Generates a report of changes to data needed to match type and property
URI changes moving from old_coll to new_coll.
annroot is the root directory for the Annalist software installation.
userhome is the home directory for the host system user issuing the command.
options contains options parsed from the command line.
returns 0 if all is well, or a non-zero status code.
This value is intended to be used as an exit status code
for the calling program.
"""
status, settings, site = get_settings_site(annroot, userhome, options)
if status != am_errors.AM_SUCCESS:
return status
if len(options.args) > 2:
print("Unexpected arguments for %s: (%s)"%(options.command, " ".join(options.args)), file=sys.stderr)
return am_errors.AM_UNEXPECTEDARGS
old_coll_id = getargvalue(getarg(options.args, 0), "Old collection Id: ")
old_coll = Collection.load(site, old_coll_id)
if not (old_coll and old_coll.get_values()):
print("Old collection not found: %s"%(old_coll_id), file=sys.stderr)
return am_errors.AM_NOCOLLECTION
new_coll_id = getargvalue(getarg(options.args, 1), "New collection Id: ")
new_coll = Collection.load(site, new_coll_id)
if not (new_coll and new_coll.get_values()):
print("New collection not found: %s"%(new_coll_id), file=sys.stderr)
return am_errors.AM_NOCOLLECTION
status = am_errors.AM_SUCCESS
print("# Migration report from collection '%s' to '%s' #"%(old_coll_id, new_coll_id))
print("")
# Scan and report on type URI changes
for new_type in coll_types(new_coll):
type_id = new_type.get_id()
old_type = old_coll.get_type(type_id)
if old_type:
old_uri = old_type[ANNAL.CURIE.uri]
new_uri = new_type[ANNAL.CURIE.uri]
if old_uri != new_uri:
print("* Type %s, URI changed from '%s' to '%s'"%(type_id, old_uri, new_uri))
supertype_uris = [ u[ANNAL.CURIE.supertype_uri] for u in new_type.get(ANNAL.CURIE.supertype_uris,[]) ]
if old_uri not in supertype_uris:
print(
" Consider adding supertype '%s' to type '%s' in collection '%s'"%
(old_uri, type_id, new_coll_id)
)
report_type_references(new_coll, old_uri, " URI '%s'"%(old_uri))
# Scan and report on property URI changes in field definitions
for new_field in coll_fields(new_coll):
field_id = new_field.get_id()
old_field = coll_field(old_coll, field_id)
if old_field:
old_uri = old_field[ANNAL.CURIE.property_uri]
new_uri = new_field[ANNAL.CURIE.property_uri]
if old_uri != new_uri:
print("* Field %s, property URI changed from '%s' to '%s'"%(field_id, old_uri, new_uri))
type_ids = types_using_field(new_coll, field_id, old_uri)
for tid in type_ids:
print(
" Consider adding property alias for '%s' to type %s in collection '%s'"%
(old_uri, tid, new_coll_id)
)
# Scan and report on property URI changes in group definitions
for new_group in coll_groups(new_coll):
group_id = new_group.get_id()
old_group = coll_group(old_coll, group_id)
if old_group:
compare_field_list(
old_coll, new_coll,
old_group[ANNAL.CURIE.group_fields],
new_group[ANNAL.CURIE.group_fields],
"Group %s"%group_id)
# Scan and report on property URI changes in view definitions
for new_view in coll_views(new_coll):
view_id = new_view.get_id()
old_view = coll_view(old_coll, view_id)
if old_view:
compare_field_list(
old_coll, new_coll,
old_view[ANNAL.CURIE.view_fields],
new_view[ANNAL.CURIE.view_fields],
"View %s"%view_id)
# Scan and report on property URI changes in list definitions
for new_list in coll_lists(new_coll):
list_id = new_list.get_id()
old_list = coll_list(old_coll, list_id)
if old_list:
compare_field_list(
old_coll, new_coll,
old_list[ANNAL.CURIE.list_fields],
new_list[ANNAL.CURIE.list_fields],
"List %s"%list_id)
print("")
return status
def post(self, request):
"""
Process options to add or remove a collection in an Annalist site
"""
log.debug("site.post: %r"%(request.POST.lists()))
collections = request.POST.getlist("select", [])
coll_id = collections[0] if collections else "_"
coll_ids = {'ids': ", ".join(collections)}
perm_req = None
perm_scope = None
none_msg = None
many_msg = None
redirect_uri = None
http_response = None
# Process POST option
if "view" in request.POST:
# Collection data is considered part of configuration, hence CONFIG_PERMISSIONS:
perm_req = CONFIG_PERMISSIONS["view"]
# Use Collection or Site permissions:
perm_scope = "all"
none_msg = message.NO_COLLECTION_VIEW
many_msg = message.MANY_COLLECTIONS_VIEW
target_uri = self.view_uri("AnnalistEntityEditView",
coll_id=layout.SITEDATA_ID,
view_id="Collection_view",
type_id="_coll",
entity_id=coll_id,
action="view"
)
redirect_uri = uri_with_params(
target_uri,
{'continuation_url': self.continuation_here()}
)
elif "edit" in request.POST:
perm_req = CONFIG_PERMISSIONS["edit"]
perm_scope = "all"
none_msg = message.NO_COLLECTION_EDIT
many_msg = message.MANY_COLLECTIONS_EDIT
target_uri = self.view_uri("AnnalistEntityEditView",
coll_id=layout.SITEDATA_ID,
view_id="Collection_view",
type_id="_coll",
entity_id=coll_id,
action="edit"
)
redirect_uri = uri_with_params(
target_uri,
{'continuation_url': self.continuation_here()}
)
elif "remove" in request.POST:
perm_req = "DELETE_COLLECTION"
perm_scope = "all" # Collection or site permissions
none_msg = message.NO_COLLECTIONS_REMOVE
elif "new" in request.POST:
perm_req = "CREATE_COLLECTION"
perm_scope = "site" # Site permission required
new_id = request.POST["new_id"]
new_label = request.POST["new_label"]
# Common checks
if none_msg and not collections:
http_response = self.redirect_info(
self.view_uri("AnnalistSiteView"),
info_message=none_msg, info_head=message.NO_ACTION_PERFORMED
)
elif many_msg and len(collections) > 1:
http_response = self.redirect_info(
self.view_uri("AnnalistSiteView"),
info_message=many_msg%coll_ids,
info_head=message.NO_ACTION_PERFORMED
)
elif perm_req:
if perm_scope == "all":
# Check collections for permissions
for cid in collections:
if http_response is None:
site = self.site(host=self.get_request_host())
sitedata = self.site_data()
coll = Collection.load(site, cid, altscope="site")
http_response = (
self.authorize("ADMIN", coll) and # Either of these...
self.authorize(perm_req, coll)
)
coll = None
else:
# Check site only for permissions
http_response = (
self.authorize("ADMIN", None) and
self.authorize(perm_req, None)
)
if http_response is not None:
return http_response
# Perform selected option
if redirect_uri:
log.info("Redirect to %s"%redirect_uri)
return HttpResponseRedirect(redirect_uri)
if "remove" in request.POST:
if layout.SITEDATA_ID in collections:
log.warning("Attempt to delete site data collection %(ids)s"%(coll_ids))
http_response = self.error(self.error403values(scope="DELETE_SITE"))
else:
http_response = ConfirmView.render_form(request,
action_description= message.REMOVE_COLLECTIONS%coll_ids,
action_params= request.POST,
confirmed_action_uri= self.view_uri('AnnalistSiteActionView'),
cancel_action_uri= self.view_uri('AnnalistSiteView'),
title= self.site_data()["title"]
)
return http_response
if "new" in request.POST:
log.info("New collection %s: %s"%(new_id, new_label))
error_message = None
if not new_id:
error_message = message.MISSING_COLLECTION_ID
elif not util.valid_id(new_id):
error_message = message.INVALID_COLLECTION_ID%{'coll_id': new_id}
if error_message:
return self.redirect_error(
self.view_uri("AnnalistSiteView"),
error_message=error_message
)
coll_meta = (
{ RDFS.CURIE.label: new_label
, RDFS.CURIE.comment: ""
})
# Add collection
coll = self.site().add_collection(new_id, coll_meta)
coll.generate_coll_jsonld_context()
user = self.request.user
user_id = user.username
user_uri = "mailto:"+user.email
user_name = "%s %s"%(user.first_name, user.last_name)
user_description = "User %s: permissions for %s in collection %s"%(user_id, user_name, new_id)
coll.create_user_permissions(
user_id, user_uri,
user_name, user_description,
user_permissions=["VIEW", "CREATE", "UPDATE", "DELETE", "CONFIG", "ADMIN"]
)
return self.redirect_info(
self.view_uri("AnnalistSiteView"),
info_message=message.CREATED_COLLECTION_ID%{'coll_id': new_id}
)
# elif "remove" in request.POST:
# collections = request.POST.getlist("select", [])
# if collections:
# # Check authorization
# if layout.SITEDATA_ID in collections:
# log.warning("Attempt to delete site data collection %r"%(collections))
# auth_required = self.error(self.error403values(scope="DELETE_SITE"))
# else:
# auth_required = (
# self.authorize("ADMIN", None) and # either of these..
# self.authorize("DELETE_COLLECTION", None)
# )
# return (
# # Get user to confirm action before actually doing it
# auth_required or
# ConfirmView.render_form(request,
# action_description= message.REMOVE_COLLECTIONS%{'ids': ", ".join(collections)},
# action_params= request.POST,
# confirmed_action_uri= self.view_uri('AnnalistSiteActionView'),
# cancel_action_uri= self.view_uri('AnnalistSiteView'),
# title= self.site_data()["title"]
# )
# )
# else:
# return self.redirect_info(
# self.view_uri("AnnalistSiteView"),
# info_message=message.NO_COLLECTIONS_REMOVE, info_head=message.NO_ACTION_PERFORMED
# )
# if "new" in request.POST:
# # Create new collection with name and label supplied
# new_id = request.POST["new_id"]
# new_label = request.POST["new_label"]
# log.debug("New collection %s: %s"%(new_id, new_label))
# if not new_id:
# return self.redirect_error(
# self.view_uri("AnnalistSiteView"),
# error_message=message.MISSING_COLLECTION_ID
# )
# if not util.valid_id(new_id):
# return self.redirect_error(
# self.view_uri("AnnalistSiteView"),
# error_message=message.INVALID_COLLECTION_ID%{'coll_id': new_id}
# )
# # Create new collection with name and label supplied
# auth_required = (
# self.authorize("ADMIN", None) and # either of these..
# self.authorize("CREATE_COLLECTION", None)
# )
# if auth_required:
# return auth_required
# coll_meta = (
# { RDFS.CURIE.label: new_label
# , RDFS.CURIE.comment: ""
# })
# coll = self.site().add_collection(new_id, coll_meta)
# # Generate initial context
# coll.generate_coll_jsonld_context()
# # Create full permissions in new collection for creating user
# user = self.request.user
# user_id = user.username
# user_uri = "mailto:"+user.email
# user_name = "%s %s"%(user.first_name, user.last_name)
# user_description = "User %s: permissions for %s in collection %s"%(user_id, user_name, new_id)
# coll.create_user_permissions(
# user_id, user_uri,
# user_name, user_description,
# user_permissions=["VIEW", "CREATE", "UPDATE", "DELETE", "CONFIG", "ADMIN"]
# )
# return self.redirect_info(
# self.view_uri("AnnalistSiteView"),
# info_message=message.CREATED_COLLECTION_ID%{'coll_id': new_id}
# )
log.warning("Invalid POST request: %r"%(request.POST.lists()))
return self.error(self.error400values())
def post(self, request):
"""
Process options to add or remove a collection in an Annalist site
"""
log.debug("site.post: %r" % (request.POST.lists()))
collections = request.POST.getlist("select", [])
coll_id = collections[0] if collections else "_"
coll_ids = {'ids': ", ".join(collections)}
perm_req = None
perm_scope = None
none_msg = None
many_msg = None
redirect_uri = None
http_response = None
# Process POST option
if "view" in request.POST:
# Collection data is considered part of configuration, hence CONFIG_PERMISSIONS:
perm_req = CONFIG_PERMISSIONS["view"]
# Use Collection or Site permissions:
perm_scope = "all"
none_msg = message.NO_COLLECTION_VIEW
many_msg = message.MANY_COLLECTIONS_VIEW
target_uri = self.view_uri("AnnalistEntityEditView",
coll_id=layout.SITEDATA_ID,
view_id="Collection_view",
type_id="_coll",
entity_id=coll_id,
action="view")
redirect_uri = uri_with_params(
target_uri, {'continuation_url': self.continuation_here()})
elif "edit" in request.POST:
perm_req = CONFIG_PERMISSIONS["edit"]
perm_scope = "all"
none_msg = message.NO_COLLECTION_EDIT
many_msg = message.MANY_COLLECTIONS_EDIT
target_uri = self.view_uri("AnnalistEntityEditView",
coll_id=layout.SITEDATA_ID,
view_id="Collection_view",
type_id="_coll",
entity_id=coll_id,
action="edit")
redirect_uri = uri_with_params(
target_uri, {'continuation_url': self.continuation_here()})
elif "remove" in request.POST:
perm_req = "DELETE_COLLECTION"
perm_scope = "all" # Collection or site permissions
none_msg = message.NO_COLLECTIONS_REMOVE
elif "new" in request.POST:
perm_req = "CREATE_COLLECTION"
perm_scope = "site" # Site permission required
new_id = request.POST["new_id"]
new_label = request.POST["new_label"]
# Common checks
if none_msg and not collections:
http_response = self.redirect_info(
self.view_uri("AnnalistSiteView"),
info_message=none_msg,
info_head=message.NO_ACTION_PERFORMED)
elif many_msg and len(collections) > 1:
http_response = self.redirect_info(
self.view_uri("AnnalistSiteView"),
info_message=many_msg % coll_ids,
info_head=message.NO_ACTION_PERFORMED)
elif perm_req:
if perm_scope == "all":
# Check collections for permissions
for cid in collections:
if http_response is None:
site = self.site(host=self.get_request_host())
sitedata = self.site_data()
coll = Collection.load(site, cid, altscope="site")
http_response = (
self.authorize("ADMIN", coll)
and # Either of these...
self.authorize(perm_req, coll))
coll = None
else:
# Check site only for permissions
http_response = (self.authorize("ADMIN", None)
and self.authorize(perm_req, None))
if http_response is not None:
return http_response
# Perform selected option
if redirect_uri:
log.info("Redirect to %s" % redirect_uri)
return HttpResponseRedirect(redirect_uri)
if "remove" in request.POST:
if layout.SITEDATA_ID in collections:
log.warning("Attempt to delete site data collection %(ids)s" %
(coll_ids))
http_response = self.error(
self.error403values(scope="DELETE_SITE"))
else:
http_response = ConfirmView.render_form(
request,
action_description=message.REMOVE_COLLECTIONS % coll_ids,
action_params=request.POST,
confirmed_action_uri=self.view_uri(
'AnnalistSiteActionView'),
cancel_action_uri=self.view_uri('AnnalistSiteView'),
title=self.site_data()["title"])
return http_response
if "new" in request.POST:
log.info("New collection %s: %s" % (new_id, new_label))
error_message = None
if not new_id:
error_message = message.MISSING_COLLECTION_ID
elif not util.valid_id(new_id):
error_message = message.INVALID_COLLECTION_ID % {
'coll_id': new_id
}
if error_message:
return self.redirect_error(self.view_uri("AnnalistSiteView"),
error_message=error_message)
coll_meta = ({RDFS.CURIE.label: new_label, RDFS.CURIE.comment: ""})
# Add collection
coll = self.site().add_collection(new_id, coll_meta)
coll.generate_coll_jsonld_context()
user = self.request.user
user_id = user.username
user_uri = "mailto:" + user.email
user_name = "%s %s" % (user.first_name, user.last_name)
user_description = "User %s: permissions for %s in collection %s" % (
user_id, user_name, new_id)
coll.create_user_permissions(user_id,
user_uri,
user_name,
user_description,
user_permissions=[
"VIEW", "CREATE", "UPDATE",
"DELETE", "CONFIG", "ADMIN"
])
return self.redirect_info(
self.view_uri("AnnalistSiteView"),
info_message=message.CREATED_COLLECTION_ID %
{'coll_id': new_id})
# elif "remove" in request.POST:
# collections = request.POST.getlist("select", [])
# if collections:
# # Check authorization
# if layout.SITEDATA_ID in collections:
# log.warning("Attempt to delete site data collection %r"%(collections))
# auth_required = self.error(self.error403values(scope="DELETE_SITE"))
# else:
# auth_required = (
# self.authorize("ADMIN", None) and # either of these..
# self.authorize("DELETE_COLLECTION", None)
# )
# return (
# # Get user to confirm action before actually doing it
# auth_required or
# ConfirmView.render_form(request,
# action_description= message.REMOVE_COLLECTIONS%{'ids': ", ".join(collections)},
# action_params= request.POST,
# confirmed_action_uri= self.view_uri('AnnalistSiteActionView'),
# cancel_action_uri= self.view_uri('AnnalistSiteView'),
# title= self.site_data()["title"]
# )
# )
# else:
# return self.redirect_info(
# self.view_uri("AnnalistSiteView"),
# info_message=message.NO_COLLECTIONS_REMOVE, info_head=message.NO_ACTION_PERFORMED
# )
# if "new" in request.POST:
# # Create new collection with name and label supplied
# new_id = request.POST["new_id"]
# new_label = request.POST["new_label"]
# log.debug("New collection %s: %s"%(new_id, new_label))
# if not new_id:
# return self.redirect_error(
# self.view_uri("AnnalistSiteView"),
# error_message=message.MISSING_COLLECTION_ID
# )
# if not util.valid_id(new_id):
# return self.redirect_error(
# self.view_uri("AnnalistSiteView"),
# error_message=message.INVALID_COLLECTION_ID%{'coll_id': new_id}
# )
# # Create new collection with name and label supplied
# auth_required = (
# self.authorize("ADMIN", None) and # either of these..
# self.authorize("CREATE_COLLECTION", None)
# )
# if auth_required:
# return auth_required
# coll_meta = (
# { RDFS.CURIE.label: new_label
# , RDFS.CURIE.comment: ""
# })
# coll = self.site().add_collection(new_id, coll_meta)
# # Generate initial context
# coll.generate_coll_jsonld_context()
# # Create full permissions in new collection for creating user
# user = self.request.user
# user_id = user.username
# user_uri = "mailto:"+user.email
# user_name = "%s %s"%(user.first_name, user.last_name)
# user_description = "User %s: permissions for %s in collection %s"%(user_id, user_name, new_id)
# coll.create_user_permissions(
# user_id, user_uri,
# user_name, user_description,
# user_permissions=["VIEW", "CREATE", "UPDATE", "DELETE", "CONFIG", "ADMIN"]
# )
# return self.redirect_info(
# self.view_uri("AnnalistSiteView"),
# info_message=message.CREATED_COLLECTION_ID%{'coll_id': new_id}
# )
log.warning("Invalid POST request: %r" % (request.POST.lists()))
return self.error(self.error400values())
