def execute_upgrade():
"""Execute the upgrade from InvenioRDM 2.0 to 3.0.
Please read the disclaimer on this module before thinking about executing
this function!
"""
for record_metadata in RDMRecord.model_cls.query.all():
record = RDMRecord(record_metadata.data, model=record_metadata)
# Updating to new $schema when eventually saved
record.pop("$schema", None)
# Adding empty pids
if record.pids is None:
record.pids = {}
record.commit()
for draft_metadata in RDMDraft.model_cls.query.all():
# Skipping deleted drafts because can't be committed
if draft_metadata.is_deleted:
continue
draft = RDMDraft(draft_metadata.data, model=draft_metadata)
# Updating to new $schema when eventually saved
draft.pop("$schema", None)
# Adding empty pids
if draft.pids is None:
draft.pids = {}
draft.commit()
for parent_metadata in RDMParent.model_cls.query.all():
parent = RDMParent(parent_metadata.data, model=parent_metadata)
# Updating to new $schema when eventually saved
parent.pop("$schema", None)
parent.commit()
# Cleanup associated deleted drafts.
drafts = RDMDraftMetadata.query.filter(
RDMDraftMetadata.is_deleted == True).all() # noqa
for d in drafts:
# Delete all file draft records
RDMFileDraftMetadata.query.filter_by(record_id=d.id).delete()
# Bucket deletion
bucket = d.bucket
d.bucket = None
d.bucket_id = None
# Object and bucket not be removed if it's also associated with the
# record.
r = RDMRecordMetadata.query.filter_by(id=d.id).one_or_none()
if r is None or r.bucket_id != bucket.id:
bucket.remove()
db.session.commit()
def execute_upgrade():
"""Execute the upgrade from InvenioRDM 3.0 to 4.0.
Please read the disclaimer on this module before thinking about executing
this function!
"""
def remove_duplicate_languages(record):
"""Remove duplicate languages."""
if "languages" in record["metadata"]:
serialized_languages = map(
tuple,
map(sorted, map(dict.items, record["metadata"]["languages"])),
)
unique_languages = set(serialized_languages)
languages_list = list(map(dict, unique_languages))
record["metadata"]["languages"] = languages_list
def update_vocabularies(record):
"""Updates languages and resource_type to become vocabularies."""
def get_res_type_vocabulary(data):
"""Returns the id value of the resource type vocabulary."""
if "subtype" in data["resource_type"]:
return data["resource_type"]["subtype"]
elif "type" in data["resource_type"]:
return data["resource_type"]["type"]
def get_language_vocabulary(data):
"""Returns the language as vocabulary."""
return dict(id=data)
def migrate_language(field):
"""Migrates language field."""
for idx, val in enumerate(record["metadata"].get(field, [])):
if "lang" in val:
language_vocab = get_language_vocabulary(val["lang"])
record["metadata"][field][idx]["lang"] = language_vocab
# Migrate resource_type
if "resource_type" in record["metadata"]:
res_type_vocab = get_res_type_vocabulary(record["metadata"])
record["metadata"]["resource_type"] = dict(id=res_type_vocab)
# Migrate resource_type of related_identifiers
for idx, val in enumerate(record["metadata"].get(
"related_identifiers", [])):
if "resource_type" in val:
res_type_vocab = get_res_type_vocabulary(val)
record["metadata"]["related_identifiers"][idx][
"resource_type"] = dict(id=res_type_vocab)
# Migrate languages from additional_descriptions
migrate_language("additional_descriptions")
# Migrate languages from additional_titles
migrate_language("additional_titles")
record.commit()
for record_metadata in RDMRecord.model_cls.query.all():
record = RDMRecord(record_metadata.data, model=record_metadata)
remove_duplicate_languages(record)
# Updating to new $schema when eventually saved
record.pop("$schema", None)
# Adding empty pids
if record.pids is None:
record.pids = {}
record.commit()
for draft_metadata in RDMDraft.model_cls.query.all():
# Skipping deleted drafts because can't be committed
if draft_metadata.is_deleted:
continue
draft = RDMDraft(draft_metadata.data, model=draft_metadata)
remove_duplicate_languages(draft)
# Updating to new $schema when eventually saved
draft.pop("$schema", None)
# Adding empty pids
if draft.pids is None:
draft.pids = {}
draft.commit()
db.session.commit()
# Need to loop again to update the resource type once the scheme is updated
for record_metadata in RDMRecord.model_cls.query.all():
record = RDMRecord(record_metadata.data, model=record_metadata)
update_vocabularies(record)
for draft_metadata in RDMDraft.model_cls.query.all():
# Skipping deleted drafts because can't be committed
if draft_metadata.is_deleted:
continue
draft = RDMDraft(draft_metadata.data, model=draft_metadata)
update_vocabularies(draft)
for parent_metadata in RDMParent.model_cls.query.all():
parent = RDMParent(parent_metadata.data, model=parent_metadata)
# Updating to new $schema when eventually saved
parent.pop("$schema", None)
parent.commit()
db.session.commit()
def execute_upgrade():
"""Execute the upgrade from InvenioRDM 4.0 to 5.0.
Please read the disclaimer on this module before thinking about executing
this function!
"""
def update_roles(creatibutors):
"""Update roles."""
for creatibutor in creatibutors:
role = creatibutor.get("role")
if role:
creatibutor["role"] = {"id": role}
return creatibutors
def update_creators_roles(record):
"""Update creator roles."""
creators = record.get("metadata").get("creators", [])
if creators:
record["metadata"]["creators"] = update_roles(creators)
def update_contributors_roles(record):
"""Update contributors roles."""
contributors = record.get("metadata").get("contributors", [])
if contributors:
record["metadata"]["contributors"] = update_roles(contributors)
def update_additional_titles(record):
"""Update additional titles type."""
add_titles = record.get("metadata").get("additional_titles", [])
for add_title in add_titles:
type_ = add_title.get("type")
# any other type either stays with the previous value or is not
# supported and should fail
if type_ == "alternativetitle":
add_title["type"] = {"id": "alternative-title"}
elif type_ == "translatedtitle":
add_title["type"] = {"id": "translated-title"}
else:
add_title["type"] = {"id": type_}
lang = add_title.get("lang")
if lang:
add_title["lang"] = {"id": lang}
def update_additional_descriptions(record):
"""Update additional descriptions type."""
metadata = record.get("metadata")
add_descriptions = metadata.get("additional_descriptions", [])
for add_desc in add_descriptions:
type_ = add_desc.get("type")
# any other type either stays with the previous value or is not
# supported and should fail
if type_ == "seriesinformation":
add_desc["type"] = {"id": "series-information"}
elif type_ == "tableofcontents":
add_desc["type"] = {"id": "table-of-contentse"}
elif type_ == "technicalinfo":
add_desc["type"] = {"id": "technical-info"}
else:
add_desc["type"] = {"id": type_}
lang = add_desc.get("lang")
if lang:
add_desc["lang"] = {"id": lang}
def update_list_field_vocabulary(record, parent, field):
"""Update related identifiers relation type."""
obj_list = record.get("metadata").get(parent, [])
for obj in obj_list:
obj[field] = {"id": obj[field]}
def update_subjects(record):
"""Updates subjects and keywords."""
subjects = record.get("metadata").get("subjects", [])
vocab_subjects = []
vocab_subjects_ids = []
for subject in subjects:
id_ = subject.get("identifier")
if id_:
if id_ not in vocab_subjects_ids:
vocab_subjects.append({"id": id_})
vocab_subjects_ids.append(id_)
else:
vocab_subjects.append({"subject": subject["subject"]})
if vocab_subjects:
record["metadata"]["subjects"] = vocab_subjects
def update_affiliations(creatibutors):
"""Updates subjects and keywords."""
for idx, creatibutor in enumerate(creatibutors):
affiliations = creatibutor.get("affiliations", [])
vocab_affs = []
for aff in affiliations:
ids = aff.get("identifiers", [])
ror = None
for id_ in ids:
if id_.get("scheme") == "ror":
ror = id_["identifier"]
break
if ror:
vocab_affs.append({"id": ror, "name": aff["name"]})
else:
vocab_affs.append({"name": aff["name"]})
if vocab_affs:
creatibutors[idx]["affiliations"] = vocab_affs
return creatibutors
def update_creators_affiliations(record):
"""Update creator roles."""
creators = record.get("metadata").get("creators", [])
if creators:
record["metadata"]["creators"] = update_affiliations(creators)
def update_contributors_affiliations(record):
"""Update contributors roles."""
contributors = record.get("metadata").get("contributors", [])
if contributors:
record["metadata"]["contributors"] = \
update_affiliations(contributors)
def update_rights(record):
"""Update record rights."""
rights = record.get("metadata").get("rights", [])
for right in rights:
locale = current_app.config.get('BABEL_DEFAULT_LOCALE', 'en')
right["title"] = {locale: right["title"]}
right["description"] = {locale: right["description"]}
def migrate_record(record):
"""Migrates a record/draft to the new schema's values."""
# Force the new jsonschema
record["$schema"] = "local://records/record-v4.0.0.json"
update_creators_roles(record)
update_contributors_roles(record)
update_additional_titles(record)
update_additional_descriptions(record)
update_list_field_vocabulary(record, "related_identifiers",
"relation_type")
update_list_field_vocabulary(record, "dates", "type")
update_subjects(record)
update_creators_affiliations(record)
update_contributors_affiliations(record)
update_rights(record)
return record
print("Migrating records...")
for record_metadata in RDMRecord.model_cls.query.all():
record = RDMRecord(record_metadata.data, model=record_metadata)
record = migrate_record(record)
record.commit()
print("Migrating drafts...")
for draft_metadata in RDMDraft.model_cls.query.all():
# Skipping deleted drafts because can't be committed
if draft_metadata.is_deleted:
continue
draft = RDMDraft(draft_metadata.data, model=draft_metadata)
migrate_record(draft)
draft.commit()
db.session.commit()
print("Records and drafts migrated.")