def import_serial_from_file(sources, rectype):
"""Load serial records from file."""
for idx, source in enumerate(sources, 1):
click.echo("({}/{}) Migrating serial in {}...".format(
idx, len(sources), source.name))
with click.progressbar(json.load(source).items()) as bar:
for key, json_record in bar:
field = json_record.get("legacy_recid", json_record["title"])
click.echo('Importing serial "{0}({1})"...'.format(
field, rectype))
has_children = json_record.get("_migration",
{}).get("children", [])
if has_children:
try:
import_record(json_record,
legacy_id=json_record["title"],
rectype=rectype,
mint_legacy_pid=False)
except Exception as exc:
handler = json_records_exception_handlers.get(
exc.__class__)
if handler:
handler(
exc,
legacy_id=json_record.get("title"),
rectype=rectype,
)
else:
raise exc
def import_loans_from_json(dump_file,
raise_exceptions=False,
rectype="loan",
mint_legacy_pid=True):
"""Imports loan objects from JSON."""
default_location_pid_value, _ = current_app_ils.get_default_location_pid
with click.progressbar(json.load(dump_file)) as bar:
for record in bar:
click.echo('Importing loan "{0}"...'.format(record["legacy_id"]))
try:
patron_pid = validate_user(record)
item = validate_item(record, raise_exception=False)
if not item:
item = current_app_ils.item_record_cls.get_record_by_pid(
MIGRATION_ITEM_PID)
item_pid = {
"value": item.pid.pid_value,
"type": item.pid.pid_type
}
document_pid = validate_document_pid(record, item)
# create a loan
loan_dict = dict(
patron_pid=str(patron_pid),
transaction_location_pid=default_location_pid_value,
transaction_user_pid=str(SystemAgent.id),
document_pid=document_pid,
item_pid=item_pid,
)
loan_dict = provide_valid_loan_state_metadata(
record, loan_dict)
if not loan_dict:
continue
validate_loan(loan_dict, record["item_barcode"],
record["id_crcBORROWER"])
import_record(loan_dict,
rectype=rectype,
legacy_id=record["legacy_id"],
mint_legacy_pid=mint_legacy_pid)
db.session.commit()
except Exception as exc:
handler = json_records_exception_handlers.get(exc.__class__)
if handler:
handler(
exc,
legacy_id=record["legacy_id"],
barcode=record["item_barcode"],
rectype=rectype,
)
if raise_exceptions:
raise exc
else:
db.session.rollback()
raise exc
def import_items_from_json(dump_file, include, rectype="item"):
"""Load items from json file."""
dump_file = dump_file[0]
model, provider = model_provider_by_rectype(rectype)
include_ids = None if include is None else include.split(",")
with click.progressbar(json.load(dump_file)) as bar:
for record in bar:
click.echo('Importing item "{0}({1})"...'.format(
record["barcode"], rectype))
if include_ids is None or record["barcode"] in include_ids:
int_loc_pid_value = get_internal_location_by_legacy_recid(
record["id_crcLIBRARY"]).pid.pid_value
record["internal_location_pid"] = int_loc_pid_value
# find document
try:
record["document_pid"] = get_document_by_legacy_recid(
record["id_bibrec"]).pid.pid_value
except DocumentMigrationError:
error_logger.error(
"ITEM: {0} ERROR: Document {1} not found".format(
record["barcode"], record["id_bibrec"]))
continue
# clean the item JSON
try:
clean_item_record(record)
except ItemMigrationError as e:
click.secho(str(e), fg="red")
error_logger.error("ITEM: {0} ERROR: {1}".format(
record["barcode"], str(e)))
continue
try:
# check if the item already there
item = get_item_by_barcode(record["barcode"])
if item:
click.secho(
"Item {0}) already exists with pid: {1}".format(
record["barcode"], item.pid),
fg="blue",
)
continue
except ItemMigrationError:
try:
import_record(record,
model,
provider,
legacy_id_key="barcode")
db.session.commit()
migrated_logger.warning("ITEM: {0} OK".format(
record["barcode"]))
except Exception as e:
error_logger.error("ITEM: {0} ERROR: {1}".format(
record["barcode"], str(e)))
db.session.rollback()
def import_items_from_json(dump_file, rectype="item"):
"""Load items from json file."""
model, provider = model_provider_by_rectype(rectype)
with click.progressbar(json.load(dump_file)) as bar:
error_logger.error("ITEMS: PROCESSING {0}".format(dump_file))
migrated_logger.warning("ITEMS: PROCESSING {0}".format(dump_file))
for record in bar:
click.echo('Importing item "{0}({1})"...'.format(
record["barcode"], rectype))
set_internal_location_pid(record)
try:
set_document_pid(record)
except DocumentMigrationError:
# there are items on the list which are not to be migrated
# if no document found
continue
# clean the item JSON
try:
clean_item_record(record)
except ItemMigrationError as e:
click.secho(str(e), fg="red")
error_logger.error("ITEM: {0} ERROR: {1}".format(
record["barcode"], str(e)))
continue
# check if the item already there
item = get_item_by_barcode(record["barcode"],
raise_exception=False)
if item:
click.secho(
"Item {0}) already exists with pid: {1}".format(
record["barcode"], item.pid),
fg="blue",
)
continue
else:
try:
import_record(record,
model,
provider,
legacy_id_key="barcode")
db.session.commit()
migrated_logger.warning("ITEM: {0} OK".format(
record["barcode"]))
except Exception as e:
error_logger.error("ITEM: {0} ERROR: {1}".format(
record["barcode"], str(e)))
db.session.rollback()
def create_loan_when_ongoing(ill, legacy_record):
"""Create a loan for the ongoing borrowing request."""
default_location_pid_value, _ = current_app_ils.get_default_location_pid
if ill["status"] != "ON_LOAN":
return
patron_pid = ill["patron_pid"]
item_pid = {"value": ill.pid.pid_value, "type": ill.pid.pid_type}
document_pid = ill["document_pid"]
loan_dict = dict(
patron_pid=str(patron_pid),
transaction_location_pid=default_location_pid_value,
transaction_user_pid=str(SystemAgent.id),
document_pid=document_pid,
item_pid=item_pid,
state="ITEM_ON_LOAN",
start_date=ill["received_date"],
# former due date of borrowing request becomes due date of the loan
end_date=legacy_record.get("due_date", CDS_ILS_FALLBACK_CREATION_DATE))
validate_loan(loan_dict, item_pid["value"], borrower_id=patron_pid)
loan = import_record(loan_dict,
rectype="loan",
legacy_id="ILL loan",
mint_legacy_pid=False)
current_circulation.loan_indexer().index(loan)
db.session.commit()
def import_vendors_from_json(dump_file, rectype="provider"):
"""Imports vendors from JSON data files."""
dump_file = dump_file[0]
click.echo("Importing vendors ..")
with click.progressbar(json.load(dump_file)) as input_data:
ils_records = []
for record in input_data:
record["type"] = "VENDOR"
# Legacy_ids in the .json file can be an array of strings or just
# an integer, but we only accept an array of strings in the schema
if not isinstance(record["legacy_ids"], list):
record["legacy_ids"] = [str(record["legacy_ids"])]
vocabulary_validator.validate(VOCABULARIES_FIELDS, record)
ils_record = import_record(
record,
rectype=rectype,
legacy_id=record["legacy_ids"],
mint_legacy_pid=False,
)
ils_records.append(ils_record)
db.session.commit()
bulk_index_records(ils_records)
def import_ill_borrowing_requests_from_json(dump_file,
raise_exceptions=False,
rectype="borrowing-request"):
"""Imports borrowing requests from JSON data files."""
click.echo("Importing borrowing requests ..")
with click.progressbar(json.load(dump_file)) as input_data:
for record in input_data:
try:
ill = import_record(
clean_record_json(record),
rectype=rectype,
legacy_id=record["legacy_id"],
)
# create a loan for ongoing ILL
create_loan_when_ongoing(ill, record)
except Exception as exc:
handler = json_records_exception_handlers.get(exc.__class__)
if handler:
handler(
exc,
legacy_id=record["legacy_id"],
barcode=record["barcode"],
rectype=rectype,
)
if raise_exceptions:
raise exc
else:
db.session.rollback()
raise exc
db.session.commit()
def import_internal_locations_from_json(
dump_file, include, rectype="internal_location"
):
"""Load parent records from file."""
dump_file = dump_file[0]
model, provider = model_provider_by_rectype(rectype)
library_model, library_provider = model_provider_by_rectype("library")
include_ids = None if include is None else include.split(",")
(
location_pid_value,
_,
) = current_app_ils.get_default_location_pid
with click.progressbar(json.load(dump_file)) as bar:
records = []
for record in bar:
click.echo(
'Importing internal location "{0}({1})"...'.format(
record["legacy_ids"], rectype
)
)
if include_ids is None or record["legacy_id"] in include_ids:
# remove the library type as it is not a part of the data model
library_type = record.pop("type", None)
if not isinstance(record["legacy_ids"], list):
record["legacy_ids"] = [str(record["legacy_ids"])]
if library_type == "external":
# if the type is external => ILL Library
record = import_record(
record,
library_model,
library_provider,
legacy_id_key="legacy_ids",
)
records.append(record)
else:
record["location_pid"] = location_pid_value
record = import_record(
record, model, provider, legacy_id_key="legacy_ids"
)
records.append(record)
# Index all new internal location and libraries records
bulk_index_records(records)
def import_internal_locations_from_json(dump_file,
include,
rectype="internal_location"):
"""Load parent records from file."""
dump_file = dump_file[0]
include_ids = None if include is None else include.split(",")
location_pid_value, _ = current_app_ils.get_default_location_pid
with click.progressbar(json.load(dump_file)) as bar:
records = []
for record in bar:
click.echo('Importing internal location "{0}({1})"...'.format(
record["legacy_ids"], rectype))
if include_ids is None or record["legacy_ids"] in include_ids:
# remove the library type as it is not a part of the data model
library_type = record.pop("type", None)
if not isinstance(record["legacy_ids"], list):
record["legacy_ids"] = [str(record["legacy_ids"])]
if library_type == "external":
# if the type is external => ILL Library
record["type"] = "LIBRARY"
vocabulary_validator.validate(VOCABULARIES_FIELDS, record)
record = import_record(
record,
rectype="provider",
legacy_id=record["legacy_ids"],
mint_legacy_pid=False,
)
records.append(record)
else:
record["location_pid"] = location_pid_value
record = import_record(
record,
rectype="internal_location",
legacy_id=record["legacy_ids"],
mint_legacy_pid=False,
)
records.append(record)
# Index all new internal location and libraries records
bulk_index_records(records)
def import_items_from_json(dump_file, rectype="item"):
"""Load items from json file."""
with click.progressbar(json.load(dump_file)) as bar:
for record in bar:
log_extra = dict(
document_legacy_recid=record["id_bibrec"],
)
click.echo(
'Importing item "{0}({1})"...'.format(
record["barcode"], rectype
)
)
try:
set_internal_location_pid(record)
set_document_pid(record)
# clean the item JSON
clean_item_record(record)
vocabulary_validator.validate(VOCABULARIES_FIELDS, record)
import_record(
record,
rectype=rectype,
legacy_id=record["barcode"],
log_extra=log_extra,
)
except Exception as exc:
handler = json_records_exception_handlers.get(exc.__class__)
if handler:
handler(
exc,
document_legacy_recid=record.get("id_bibrec")
or record.get("document_pid"),
legacy_id=record["barcode"],
rectype=rectype,
),
else:
db.session.rollback()
raise exc
def import_orders_from_json(dump_file,
rectype="acq-order",
raise_exceptions=False):
"""Imports orders from JSON data files."""
click.echo("Importing acquisition orders ..")
with click.progressbar(json.load(dump_file)) as input_data:
for record in input_data:
click.echo('Processing order "{}"...'.format(record["legacy_id"]))
try:
import_record(
migrate_order(record),
rectype=rectype,
legacy_id=record["legacy_id"],
mint_legacy_pid=True,
)
except Exception as exc:
handler = acquisition_order_exception_handler.get(
exc.__class__)
if handler:
handler(
exc,
legacy_id=record["legacy_id"],
barcode=record["barcode"],
rectype=rectype,
)
if raise_exceptions:
raise exc
else:
db.session.rollback()
logger = logging.getLogger(f"{rectype}s_logger")
logger.error(
str(exc),
extra=dict(
legacy_id=record["legacy_id"],
new_pid=None,
status="ERROR",
),
)
raise exc
def import_document_requests_from_json(dump_file, rectype="document-request"):
"""Imports document requests from JSON data files."""
click.echo("Importing document requests ..")
with click.progressbar(json.load(dump_file)) as input_data:
ils_records = []
for record in input_data:
ils_record = import_record(migrate_document_request(record),
rectype=rectype,
legacy_id=record["legacy_id"],
mint_legacy_pid=False)
ils_records.append(ils_record)
db.session.commit()
bulk_index_records(ils_records)
def import_ill_borrowing_requests_from_json(dump_file):
"""Imports borrowing requests from JSON data files."""
dump_file = dump_file[0]
model, provider = model_provider_by_rectype("borrowing-request")
click.echo("Importing borrowing requests ..")
with click.progressbar(json.load(dump_file)) as input_data:
ils_records = []
for record in input_data:
ils_record = import_record(
clean_record_json(record),
model,
provider,
legacy_id_key="legacy_id",
)
ils_records.append(ils_record)
db.session.commit()
bulk_index_records(ils_records)
def import_document_requests_from_json(dump_file):
"""Imports document requests from JSON data files."""
dump_file = dump_file[0]
click.echo("Importing document requests ..")
with click.progressbar(json.load(dump_file)) as input_data:
ils_records = []
for record in input_data:
model, provider = model_provider_by_rectype("document-request")
ils_record = import_record(
migrate_document_request(record),
model,
provider,
legacy_id_key="legacy_id",
)
ils_records.append(ils_record)
db.session.commit()
bulk_index_records(ils_records)
def import_documents_from_record_file(sources, include):
"""Import documents from records file generated by CDS-Migrator-Kit."""
include = include if include is None else include.split(",")
records = []
for idx, source in enumerate(sources, 1):
click.echo("({}/{}) Migrating documents in {}...".format(
idx, len(sources), source.name))
model, provider = model_provider_by_rectype("document")
include_keys = None if include is None else include.split(",")
with click.progressbar(json.load(source).items()) as bar:
records = []
for key, parent in bar:
click.echo('Importing document "{}"...'.format(
parent["legacy_recid"]))
if include_keys is None or key in include_keys:
record = import_record(parent, model, provider)
records.append(record)
# Index all new parent records
bulk_index_records(records)
def import_serial_from_file(sources, rectype):
"""Load serial records from file."""
model, provider = model_provider_by_rectype(rectype)
for idx, source in enumerate(sources, 1):
click.echo("({}/{}) Migrating documents in {}...".format(
idx, len(sources), source.name))
with click.progressbar(json.load(source).items()) as bar:
records = []
for key, json_record in bar:
if "legacy_recid" in json_record:
click.echo('Importing serial "{0}({1})"...'.format(
json_record["legacy_recid"], rectype))
else:
click.echo('Importing parent "{0}({1})"...'.format(
json_record["title"], rectype))
has_children = json_record.get("_migration",
{}).get("children", [])
if has_children:
record = import_record(json_record, model, provider)
records.append(record)
def import_vendors_from_json(dump_file):
"""Imports vendors from JSON data files."""
dump_file = dump_file[0]
model, provider = model_provider_by_rectype("vendor")
click.echo("Importing vendors ..")
with click.progressbar(json.load(dump_file)) as input_data:
ils_records = []
for record in input_data:
# Legacy_ids in the .json file can be an array of strings or just
# an integer, but we only accept an array of strings in the schema
if not isinstance(record["legacy_ids"], list):
record["legacy_ids"] = [str(record["legacy_ids"])]
ils_record = import_record(
record,
model,
provider,
legacy_id_key="legacy_id",
)
ils_records.append(ils_record)
db.session.commit()
bulk_index_records(ils_records)
def import_orders_from_json(dump_file, include=None):
"""Imports orders from JSON data files."""
dump_file = dump_file[0]
click.echo("Importing acquisition orders ..")
with click.progressbar(json.load(dump_file)) as input_data:
ils_records = []
for record in input_data:
model, provider = model_provider_by_rectype("acq-order")
try:
ils_record = import_record(
migrate_order(record),
model,
provider,
legacy_id_key="legacy_id",
)
ils_records.append(ils_record)
except Exception as e:
error_logger.error("ORDER: {0} ERROR: {1}".format(
record["legacy_id"], str(e)))
db.session.rollback()
db.session.commit()
bulk_index_records(ils_records)
def import_loans_from_json(dump_file):
"""Imports loan objects from JSON."""
dump_file = dump_file[0]
document_class = current_app_ils.document_record_cls
loans = []
(
default_location_pid_value,
_,
) = current_app_ils.get_default_location_pid
with click.progressbar(json.load(dump_file)) as bar:
for record in bar:
click.echo('Importing loan "{0}"...'.format(record["legacy_id"]))
user = get_user_by_legacy_id(record["id_crcBORROWER"])
if not user:
# user was deleted, fallback to the AnonymousUser
anonym = current_app.config["ILS_PATRON_ANONYMOUS_CLASS"]
patron_pid = anonym.id
else:
patron_pid = user.pid
try:
item = get_item_by_barcode(record["item_barcode"])
except ItemMigrationError:
records_logger.error(
"LOAN: {0}, ERROR: barcode {1} not found.".format(
record["legacy_id"], record["item_barcode"]
)
)
continue
# Todo uncomment when more data
# raise LoanMigrationError(
# 'no item found with the barcode {} for loan {}'.format(
# record['item_barcode'], record['legacy_id']))
# additional check if the loan refers to the same document
# as it is already attached to the item
document_pid = item.get("document_pid")
document = document_class.get_record_by_pid(document_pid)
if record["legacy_document_id"] is None:
records_logger.error(
"LOAN: {0}, ERROR: document_legacy_recid {1} not found."
.format(
record["legacy_id"], record["legacy_document_id"]
)
)
raise LoanMigrationError(
"no document id for loan {}".format(record["legacy_id"])
)
if (
document.get("legacy_recid", None)
!= record["legacy_document_id"]
):
# this might happen when record merged or migrated,
# the already migrated document should take precedence
click.secho(
"inconsistent document dependencies for loan {}".format(
record["legacy_id"]
),
fg="blue",
)
# create a loan
loan_dict = dict(
patron_pid=str(patron_pid),
transaction_location_pid=default_location_pid_value,
transaction_user_pid=str(SystemAgent.id),
document_pid=document_pid,
item_pid={
"value": item.pid.pid_value,
"type": item.pid.pid_type,
},
)
if record["status"] == "on loan":
loan_dict.update(
dict(
start_date=record["start_date"],
end_date=record["end_date"],
state="ITEM_ON_LOAN",
transaction_date=record["start_date"],
)
)
elif record["status"] == "returned":
loan_dict.update(
dict(
transaction_date=record["returned_on"],
start_date=record["start_date"],
end_date=record["returned_on"],
state="ITEM_RETURNED",
)
)
# loan request
elif (
record["status"] == "waiting" or record["status"] == "pending"
):
loan_dict.update(
dict(
transaction_date=record["request_date"],
request_start_date=record["period_of_interest_from"],
request_expire_date=record["period_of_interest_to"],
state="PENDING",
)
)
# done loan requests became loans, and the rest we can ignore
elif record["status"] in ["proposed", "cancelled", "done"]:
continue
else:
raise LoanMigrationError(
"Unkown loan state for record {0}: {1}".format(
record["legacy_id"], record["state"]
)
)
model, provider = model_provider_by_rectype("loan")
try:
loan = import_record(
loan_dict, model, provider, legacy_id_key=None
)
db.session.commit()
migrated_logger.warning(
"LOAN: {0} OK, new pid: {1}".format(
record["legacy_id"], loan["pid"]
)
)
except Exception as e:
db.session.rollback()
records_logger.error(
"LOAN: {0} ERROR: {1}".format(record["legacy_id"], str(e))
)
return loans
def import_multipart(json_record):
"""Import multipart record."""
document_indexer = current_app_ils.document_indexer
series_indexer = current_app_ils.series_indexer
document_cls = current_app_ils.document_record_cls
multipart_record = None
multipart_id = json_record["_migration"].get("multipart_id")
# volume specific information
volumes = json_record["_migration"]["volumes"]
if multipart_id:
# try to check if the multipart already exists
# (from previous dump file)
multipart_record = get_multipart_by_multipart_id(multipart_id.upper())
# series with record per volume shouldn't have more than one volume
# in the list
if len(volumes) != 1:
raise ManualImportRequired("Matched volumes number incorrect.")
# split json for multipart (series rectype) and
# document (common data for all volumes, to be stored on document rectype)
multipart_json = clean_document_json_for_multipart(json_record,
include_keys=[
"publication_year",
])
publisher = json_record.get("imprint", {}).get("publisher")
if publisher:
multipart_json["publisher"] = publisher
document_json = exclude_multipart_fields(json_record)
document_json["title"] = volumes[0]["title"]
add_cds_url(document_json)
# series with separate record per volume
# (identified together with multipart id)
if not multipart_record:
multipart_record = import_record(
multipart_json,
rectype="multipart",
legacy_id=json_record["legacy_recid"],
# we don't mint the legacy pid for these series, since they
# never were records on legacy, only it's volumes were
mint_legacy_pid=False,
)
try:
# check if the document already exists
legacy_pid_type = current_app.config["CDS_ILS_RECORD_LEGACY_PID_TYPE"]
document_record = get_record_by_legacy_recid(
document_cls, legacy_pid_type, document_json["legacy_recid"])
# try to create relation (should fail if already exists)
create_parent_child_relation(
multipart_record,
document_record,
MULTIPART_MONOGRAPH_RELATION,
volumes[0]["volume"],
)
db.session.commit()
return multipart_record
except PIDDoesNotExistError as e:
document_record = import_record(
document_json,
rectype="document",
legacy_id=document_json["legacy_recid"])
document_indexer.index(document_record)
create_parent_child_relation(
multipart_record,
document_record,
MULTIPART_MONOGRAPH_RELATION,
volumes[0]["volume"],
)
db.session.commit()
# the multipart needs to be indexed immediately,
# because we search multipart_id to match next volumes
series_indexer.index(multipart_record)
RecordRelationIndexer().index(document_record, multipart_record)
return multipart_record
def import_multivolume(json_record):
"""Import multivolume type of multipart."""
document_indexer = current_app_ils.document_indexer
series_indexer = current_app_ils.series_indexer
series_cls, series_pid_provider = model_provider_by_rectype("multipart")
document_cls, document_pid_provider = model_provider_by_rectype("document")
legacy_recid = json_record["legacy_recid"]
# build multipart dict - leave the legacy_recid attached
multipart_json = clean_document_json_for_multipart(
json_record, include_keys=["legacy_recid"])
# prepare json for each volume
document_json_template = exclude_multipart_fields(
json_record, exclude_keys=["legacy_recid"])
volume_list = json_record["_migration"]["volumes"]
try:
get_record_by_legacy_recid(series_cls, legacy_recid)
raise MultipartMigrationError(f"Multipart {legacy_recid} was already "
f"processed. Aborting.")
except PIDDoesNotExistError as e:
multipart_record = import_record(
multipart_json,
series_cls,
series_pid_provider,
legacy_id_key="title",
)
volumes_items_list = json_record["_migration"]["items"]
volumes_identifiers_list = json_record["_migration"]["volumes_identifiers"]
volumes_urls_list = json_record["_migration"]["volumes_urls"]
lists_lenghts = [
len(entry) for entry in [
volumes_urls_list,
volumes_items_list,
volumes_identifiers_list,
]
]
too_many_volumes = any(lists_lenghts) > len(volume_list)
if too_many_volumes:
raise ManualImportRequired(
"Record has more additional volume information "
"entries than the number of indicated volumes")
for volume in volume_list:
replace_fields_in_volume(document_json_template, volume, json_record)
document_record = import_record(
document_json_template,
document_cls,
document_pid_provider,
legacy_id_key="title",
)
document_indexer.index(document_record)
series_indexer.index(multipart_record)
create_parent_child_relation(
multipart_record,
document_record,
MULTIPART_MONOGRAPH_RELATION,
volume.get("volume"),
)
RecordRelationIndexer().index(document_record, multipart_record)
return multipart_record
def import_multivolume(json_record):
"""Import multivolume type of multipart."""
document_indexer = current_app_ils.document_indexer
series_indexer = current_app_ils.series_indexer
series_cls = current_app_ils.series_record_cls
legacy_recid = json_record["legacy_recid"]
# build multipart dict - leave the legacy_recid attached
multipart_json = clean_document_json_for_multipart(
json_record,
include_keys=[
"legacy_recid", "alternative_titles", "publication_year",
"identifiers"
])
# prepare json for each volume
document_json_template = exclude_multipart_fields(
json_record, exclude_keys=["legacy_recid", "alternative_titles"])
volume_list = json_record["_migration"]["volumes"]
try:
legacy_pid_type = current_app.config["CDS_ILS_SERIES_LEGACY_PID_TYPE"]
get_record_by_legacy_recid(series_cls, legacy_pid_type, legacy_recid)
raise MultipartMigrationError(
f"Multipart {legacy_recid} was already processed. Aborting.")
except PIDDoesNotExistError as e:
add_cds_url(multipart_json)
multipart_record = import_record(
multipart_json,
rectype="multipart",
legacy_id=multipart_json["legacy_recid"],
)
series_indexer.index(multipart_record)
volumes_items_list = json_record["_migration"]["items"]
volumes_identifiers_list = json_record["_migration"]["volumes_identifiers"]
volumes_urls_list = json_record["_migration"]["volumes_urls"]
lists_lengths = [
len(entry) for entry in [
volumes_urls_list,
volumes_items_list,
volumes_identifiers_list,
]
]
too_many_volumes = any(lists_lengths) > len(volume_list)
if too_many_volumes:
raise ManualImportRequired(
"Record has more additional volume information "
"entries than the number of indicated volumes")
for volume in volume_list:
replace_fields_in_volume(document_json_template, volume, json_record)
document_record = import_record(
document_json_template,
rectype="document",
legacy_id=json_record["legacy_recid"],
# we don't mint the legacy pid for these documents, since they
# never were records on legacy, only it's parent multipart was
mint_legacy_pid=False,
)
document_indexer.index(document_record)
create_parent_child_relation(
multipart_record,
document_record,
MULTIPART_MONOGRAPH_RELATION,
volume.get("volume"),
)
db.session.commit()
RecordRelationIndexer().index(document_record, multipart_record)
return multipart_record
def import_multipart(json_record):
"""Import multipart record."""
document_indexer = current_app_ils.document_indexer
series_indexer = current_app_ils.series_indexer
series_cls, series_pid_provider = model_provider_by_rectype("multipart")
document_cls, document_pid_provider = model_provider_by_rectype("document")
multipart_record = None
multipart_id = json_record["_migration"].get("multipart_id")
# split json for multipart (series rectype) and
# document (common data for all volumes, to be stored on document rectype)
multipart_json = clean_document_json_for_multipart(json_record)
document_json = exclude_multipart_fields(json_record)
# volume specific information
volumes = json_record["_migration"]["volumes"]
if multipart_id:
# try to check if the multipart already exists
# (from previous dump file)
multipart_record = get_multipart_by_multipart_id(multipart_id)
# series with record per volume shouldn't have more than one volume
# in the list
if len(volumes) != 1:
raise ManualImportRequired("Matched volumes number incorrect.")
# series with separate record per volume
# (identified together with multipart id)
if not multipart_record:
multipart_record = import_record(
multipart_json,
series_cls,
series_pid_provider,
legacy_id_key="title",
)
try:
# check if the document already exists
document_record = get_record_by_legacy_recid(
document_cls, document_json["legacy_recid"])
# try to create relation (should fail if already exists)
create_parent_child_relation(
multipart_record,
document_record,
MULTIPART_MONOGRAPH_RELATION,
volumes[0]["volume"],
)
return multipart_record
except PIDDoesNotExistError as e:
document_record = import_record(document_json, document_cls,
document_pid_provider)
document_indexer.index(document_record)
create_parent_child_relation(
multipart_record,
document_record,
MULTIPART_MONOGRAPH_RELATION,
volumes[0]["volume"],
)
# the multipart needs to be indexed immediately,
# because we search multipart_id to match next volumes
series_indexer.index(multipart_record)
RecordRelationIndexer().index(document_record, multipart_record)
return multipart_record