def execute_upgrade(): """Execute the upgrade from InvenioRDM 2.0 to 3.0. Please read the disclaimer on this module before thinking about executing this function! """ for record_metadata in RDMRecord.model_cls.query.all(): record = RDMRecord(record_metadata.data, model=record_metadata) # Updating to new $schema when eventually saved record.pop("$schema", None) # Adding empty pids if record.pids is None: record.pids = {} record.commit() for draft_metadata in RDMDraft.model_cls.query.all(): # Skipping deleted drafts because can't be committed if draft_metadata.is_deleted: continue draft = RDMDraft(draft_metadata.data, model=draft_metadata) # Updating to new $schema when eventually saved draft.pop("$schema", None) # Adding empty pids if draft.pids is None: draft.pids = {} draft.commit() for parent_metadata in RDMParent.model_cls.query.all(): parent = RDMParent(parent_metadata.data, model=parent_metadata) # Updating to new $schema when eventually saved parent.pop("$schema", None) parent.commit() # Cleanup associated deleted drafts. drafts = RDMDraftMetadata.query.filter( RDMDraftMetadata.is_deleted == True).all() # noqa for d in drafts: # Delete all file draft records RDMFileDraftMetadata.query.filter_by(record_id=d.id).delete() # Bucket deletion bucket = d.bucket d.bucket = None d.bucket_id = None # Object and bucket not be removed if it's also associated with the # record. r = RDMRecordMetadata.query.filter_by(id=d.id).one_or_none() if r is None or r.bucket_id != bucket.id: bucket.remove() db.session.commit()
def execute_upgrade(): """Execute the upgrade from InvenioRDM 3.0 to 4.0. Please read the disclaimer on this module before thinking about executing this function! """ def remove_duplicate_languages(record): """Remove duplicate languages.""" if "languages" in record["metadata"]: serialized_languages = map( tuple, map(sorted, map(dict.items, record["metadata"]["languages"])), ) unique_languages = set(serialized_languages) languages_list = list(map(dict, unique_languages)) record["metadata"]["languages"] = languages_list def update_vocabularies(record): """Updates languages and resource_type to become vocabularies.""" def get_res_type_vocabulary(data): """Returns the id value of the resource type vocabulary.""" if "subtype" in data["resource_type"]: return data["resource_type"]["subtype"] elif "type" in data["resource_type"]: return data["resource_type"]["type"] def get_language_vocabulary(data): """Returns the language as vocabulary.""" return dict(id=data) def migrate_language(field): """Migrates language field.""" for idx, val in enumerate(record["metadata"].get(field, [])): if "lang" in val: language_vocab = get_language_vocabulary(val["lang"]) record["metadata"][field][idx]["lang"] = language_vocab # Migrate resource_type if "resource_type" in record["metadata"]: res_type_vocab = get_res_type_vocabulary(record["metadata"]) record["metadata"]["resource_type"] = dict(id=res_type_vocab) # Migrate resource_type of related_identifiers for idx, val in enumerate(record["metadata"].get( "related_identifiers", [])): if "resource_type" in val: res_type_vocab = get_res_type_vocabulary(val) record["metadata"]["related_identifiers"][idx][ "resource_type"] = dict(id=res_type_vocab) # Migrate languages from additional_descriptions migrate_language("additional_descriptions") # Migrate languages from additional_titles migrate_language("additional_titles") record.commit() for record_metadata in RDMRecord.model_cls.query.all(): record = RDMRecord(record_metadata.data, model=record_metadata) remove_duplicate_languages(record) # Updating to new $schema when eventually saved record.pop("$schema", None) # Adding empty pids if record.pids is None: record.pids = {} record.commit() for draft_metadata in RDMDraft.model_cls.query.all(): # Skipping deleted drafts because can't be committed if draft_metadata.is_deleted: continue draft = RDMDraft(draft_metadata.data, model=draft_metadata) remove_duplicate_languages(draft) # Updating to new $schema when eventually saved draft.pop("$schema", None) # Adding empty pids if draft.pids is None: draft.pids = {} draft.commit() db.session.commit() # Need to loop again to update the resource type once the scheme is updated for record_metadata in RDMRecord.model_cls.query.all(): record = RDMRecord(record_metadata.data, model=record_metadata) update_vocabularies(record) for draft_metadata in RDMDraft.model_cls.query.all(): # Skipping deleted drafts because can't be committed if draft_metadata.is_deleted: continue draft = RDMDraft(draft_metadata.data, model=draft_metadata) update_vocabularies(draft) for parent_metadata in RDMParent.model_cls.query.all(): parent = RDMParent(parent_metadata.data, model=parent_metadata) # Updating to new $schema when eventually saved parent.pop("$schema", None) parent.commit() db.session.commit()
def execute_upgrade(): """Execute the upgrade from InvenioRDM 4.0 to 5.0. Please read the disclaimer on this module before thinking about executing this function! """ def update_roles(creatibutors): """Update roles.""" for creatibutor in creatibutors: role = creatibutor.get("role") if role: creatibutor["role"] = {"id": role} return creatibutors def update_creators_roles(record): """Update creator roles.""" creators = record.get("metadata").get("creators", []) if creators: record["metadata"]["creators"] = update_roles(creators) def update_contributors_roles(record): """Update contributors roles.""" contributors = record.get("metadata").get("contributors", []) if contributors: record["metadata"]["contributors"] = update_roles(contributors) def update_additional_titles(record): """Update additional titles type.""" add_titles = record.get("metadata").get("additional_titles", []) for add_title in add_titles: type_ = add_title.get("type") # any other type either stays with the previous value or is not # supported and should fail if type_ == "alternativetitle": add_title["type"] = {"id": "alternative-title"} elif type_ == "translatedtitle": add_title["type"] = {"id": "translated-title"} else: add_title["type"] = {"id": type_} lang = add_title.get("lang") if lang: add_title["lang"] = {"id": lang} def update_additional_descriptions(record): """Update additional descriptions type.""" metadata = record.get("metadata") add_descriptions = metadata.get("additional_descriptions", []) for add_desc in add_descriptions: type_ = add_desc.get("type") # any other type either stays with the previous value or is not # supported and should fail if type_ == "seriesinformation": add_desc["type"] = {"id": "series-information"} elif type_ == "tableofcontents": add_desc["type"] = {"id": "table-of-contentse"} elif type_ == "technicalinfo": add_desc["type"] = {"id": "technical-info"} else: add_desc["type"] = {"id": type_} lang = add_desc.get("lang") if lang: add_desc["lang"] = {"id": lang} def update_list_field_vocabulary(record, parent, field): """Update related identifiers relation type.""" obj_list = record.get("metadata").get(parent, []) for obj in obj_list: obj[field] = {"id": obj[field]} def update_subjects(record): """Updates subjects and keywords.""" subjects = record.get("metadata").get("subjects", []) vocab_subjects = [] vocab_subjects_ids = [] for subject in subjects: id_ = subject.get("identifier") if id_: if id_ not in vocab_subjects_ids: vocab_subjects.append({"id": id_}) vocab_subjects_ids.append(id_) else: vocab_subjects.append({"subject": subject["subject"]}) if vocab_subjects: record["metadata"]["subjects"] = vocab_subjects def update_affiliations(creatibutors): """Updates subjects and keywords.""" for idx, creatibutor in enumerate(creatibutors): affiliations = creatibutor.get("affiliations", []) vocab_affs = [] for aff in affiliations: ids = aff.get("identifiers", []) ror = None for id_ in ids: if id_.get("scheme") == "ror": ror = id_["identifier"] break if ror: vocab_affs.append({"id": ror, "name": aff["name"]}) else: vocab_affs.append({"name": aff["name"]}) if vocab_affs: creatibutors[idx]["affiliations"] = vocab_affs return creatibutors def update_creators_affiliations(record): """Update creator roles.""" creators = record.get("metadata").get("creators", []) if creators: record["metadata"]["creators"] = update_affiliations(creators) def update_contributors_affiliations(record): """Update contributors roles.""" contributors = record.get("metadata").get("contributors", []) if contributors: record["metadata"]["contributors"] = \ update_affiliations(contributors) def update_rights(record): """Update record rights.""" rights = record.get("metadata").get("rights", []) for right in rights: locale = current_app.config.get('BABEL_DEFAULT_LOCALE', 'en') right["title"] = {locale: right["title"]} right["description"] = {locale: right["description"]} def migrate_record(record): """Migrates a record/draft to the new schema's values.""" # Force the new jsonschema record["$schema"] = "local://records/record-v4.0.0.json" update_creators_roles(record) update_contributors_roles(record) update_additional_titles(record) update_additional_descriptions(record) update_list_field_vocabulary(record, "related_identifiers", "relation_type") update_list_field_vocabulary(record, "dates", "type") update_subjects(record) update_creators_affiliations(record) update_contributors_affiliations(record) update_rights(record) return record print("Migrating records...") for record_metadata in RDMRecord.model_cls.query.all(): record = RDMRecord(record_metadata.data, model=record_metadata) record = migrate_record(record) record.commit() print("Migrating drafts...") for draft_metadata in RDMDraft.model_cls.query.all(): # Skipping deleted drafts because can't be committed if draft_metadata.is_deleted: continue draft = RDMDraft(draft_metadata.data, model=draft_metadata) migrate_record(draft) draft.commit() db.session.commit() print("Records and drafts migrated.")