def import_vendors_from_json(dump_file, rectype="provider"):
"""Imports vendors from JSON data files."""
dump_file = dump_file[0]
click.echo("Importing vendors ..")
with click.progressbar(json.load(dump_file)) as input_data:
ils_records = []
for record in input_data:
record["type"] = "VENDOR"
# Legacy_ids in the .json file can be an array of strings or just
# an integer, but we only accept an array of strings in the schema
if not isinstance(record["legacy_ids"], list):
record["legacy_ids"] = [str(record["legacy_ids"])]
vocabulary_validator.validate(VOCABULARIES_FIELDS, record)
ils_record = import_record(
record,
rectype=rectype,
legacy_id=record["legacy_ids"],
mint_legacy_pid=False,
)
ils_records.append(ils_record)
db.session.commit()
bulk_index_records(ils_records)
def _before_create(self):
"""Perform before create metadata modification."""
document = deepcopy(self.json_data)
for field in self.helper_metadata_fields:
if field in document:
del document[field]
# save the import source
self._set_record_import_source(document)
vocabulary_validator.validate(VOCABULARIES_FIELDS, document)
return document
def _before_create(self, json_series):
"""Perform before create metadata modification."""
series = deepcopy(json_series)
if "volume" in series:
del series["volume"]
# save the import source
self._set_record_import_source(series)
series["mode_of_issuance"] = "SERIAL"
series["series_type"] = "SERIAL"
vocabulary_validator.validate(VOCABULARIES_FIELDS, series)
return series
def import_internal_locations_from_json(dump_file,
include,
rectype="internal_location"):
"""Load parent records from file."""
dump_file = dump_file[0]
include_ids = None if include is None else include.split(",")
location_pid_value, _ = current_app_ils.get_default_location_pid
with click.progressbar(json.load(dump_file)) as bar:
records = []
for record in bar:
click.echo('Importing internal location "{0}({1})"...'.format(
record["legacy_ids"], rectype))
if include_ids is None or record["legacy_ids"] in include_ids:
# remove the library type as it is not a part of the data model
library_type = record.pop("type", None)
if not isinstance(record["legacy_ids"], list):
record["legacy_ids"] = [str(record["legacy_ids"])]
if library_type == "external":
# if the type is external => ILL Library
record["type"] = "LIBRARY"
vocabulary_validator.validate(VOCABULARIES_FIELDS, record)
record = import_record(
record,
rectype="provider",
legacy_id=record["legacy_ids"],
mint_legacy_pid=False,
)
records.append(record)
else:
record["location_pid"] = location_pid_value
record = import_record(
record,
rectype="internal_location",
legacy_id=record["legacy_ids"],
mint_legacy_pid=False,
)
records.append(record)
# Index all new internal location and libraries records
bulk_index_records(records)
def create_record(cls, dump, rectype):
"""Create a new record from dump."""
timestamp, json_data = dump.revisions[-1]
json_data = clean_created_by_field(json_data)
vocabulary_validator.validate(VOCABULARIES_FIELDS, json_data)
add_cover_metadata(json_data)
is_multivolume_record = json_data["_migration"].get(
"multivolume_record", False)
if is_multivolume_record:
click.echo("Multivolume record.")
record = import_multivolume(json_data)
else:
click.echo("Multipart record.")
record = import_multipart(json_data)
# wait for the previous multipart to be indexed
click.echo("Indexing.")
time.sleep(2)
return record
def import_items_from_json(dump_file, rectype="item"):
"""Load items from json file."""
with click.progressbar(json.load(dump_file)) as bar:
for record in bar:
log_extra = dict(
document_legacy_recid=record["id_bibrec"],
)
click.echo(
'Importing item "{0}({1})"...'.format(
record["barcode"], rectype
)
)
try:
set_internal_location_pid(record)
set_document_pid(record)
# clean the item JSON
clean_item_record(record)
vocabulary_validator.validate(VOCABULARIES_FIELDS, record)
import_record(
record,
rectype=rectype,
legacy_id=record["barcode"],
log_extra=log_extra,
)
except Exception as exc:
handler = json_records_exception_handlers.get(exc.__class__)
if handler:
handler(
exc,
document_legacy_recid=record.get("id_bibrec")
or record.get("document_pid"),
legacy_id=record["barcode"],
rectype=rectype,
),
else:
db.session.rollback()
raise exc
def migrate_document_request(record):
"""Create a document request record for ILS."""
state = "PENDING"
new_docreq = dict(
legacy_id=record["legacy_id"],
patron_pid=get_patron_pid(record),
title="Migrated record, no title provided",
state=state,
request_type="LOAN",
medium="PAPER",
)
if record["status"] == "proposal-put aside":
state = "DECLINED"
new_docreq.update(state=state, decline_reason="OTHER")
note = get_acq_ill_notes(record)
if note:
new_docreq.update(note=note)
vocabulary_validator.validate(VOCABULARIES_FIELDS, new_docreq)
return new_docreq
def migrate_order(record):
"""Create a order record for ILS."""
status = get_status(record)
new_order = dict(
legacy_id=record["legacy_id"],
order_lines=[create_order_line(record, status)],
status=status,
)
order_date = record["request_date"]
if order_date:
new_order.update(order_date=get_date(order_date))
else:
new_order.update(order_date="1970-01-01")
# Optional fields
if status == "CANCELLED":
new_order.update(cancel_reason="MIGRATED/UNKNOWN")
grand_total = get_cost(record)
if grand_total:
new_order.update(grand_total=grand_total)
try:
provider_type = 'VENDOR'
# due to migrating the article requests as orders
# (formerly treated as ILLs)
if record['request_type'] == 'article':
provider_type = 'LIBRARY'
provider_legacy_id = record["id_crcLIBRARY"]
# perform merge of libraries and vendors by legacy id
vendor_merge_mapper = {"63": 19, "47": 20, "48": 37}
library_merge_mapper = {"72": 33, "74": 42}
if provider_type == 'VENDOR' and provider_legacy_id in [63, 47, 48]:
provider_legacy_id = vendor_merge_mapper[str(provider_legacy_id)]
provider_type = 'LIBRARY'
elif provider_type == 'LIBRARY' and provider_legacy_id in [72, 74]:
provider_type = 'VENDOR'
provider_legacy_id = library_merge_mapper[str(provider_legacy_id)]
provider = get_provider_by_legacy_id(provider_legacy_id,
provider_type=provider_type,
grand_total=grand_total)
provider_pid = provider.pid.pid_value
except ProviderError as e:
provider_pid = MIGRATION_PROVIDER_PID
new_order.update(provider_pid=provider_pid)
expected_delivery_date = record.get("expected_date")
if expected_delivery_date:
new_order.update(
expected_delivery_date=get_date(expected_delivery_date))
received_date = record.get("arrival_date")
if received_date:
new_order.update(received_date=get_date(received_date))
notes = get_acq_ill_notes(record)
if notes:
new_order.update(notes=notes)
validate_order(new_order)
vocabulary_validator.validate(VOCABULARIES_FIELDS, new_order)
return new_order
def create_record(cls, dump):
"""Create a new record from dump."""
document_cls = current_app_ils.document_record_cls
record_uuid = uuid.uuid4()
timestamp, json_data = dump.revisions[-1]
json_data = clean_created_by_field(json_data)
vocabulary_validator.validate(VOCABULARIES_FIELDS, json_data)
add_cover_metadata(json_data)
add_title_from_conference_info(json_data)
add_cds_url(json_data)
try:
with db.session.begin_nested():
# checks if the document with this legacy_recid already exists
legacy_pid_type = current_app.config[
"CDS_ILS_RECORD_LEGACY_PID_TYPE"
]
# First mint the legacy_recid before assigning the pid. In case
# it fails while importing an existing record we will update it
# and don't want the new pid, since there is already one there
legacy_recid_minter(
json_data["legacy_recid"], legacy_pid_type, record_uuid
)
provider = DocumentIdProvider.create(
object_type="rec",
object_uuid=record_uuid,
)
# requirement from the library
if (
json_data["_migration"]["has_journal"]
and json_data["document_type"] != "PROCEEDINGS"
):
json_data["document_type"] = "SERIAL_ISSUE"
json_data["pid"] = provider.pid.pid_value
document = document_cls.create(json_data, record_uuid)
created_date = json_data.get(
"_created", CDS_ILS_FALLBACK_CREATION_DATE
)
document.model.created = parser.parse(created_date)
document.model.updated = timestamp.replace(tzinfo=None)
document.commit()
db.session.commit()
documents_logger.info(
"CREATED",
extra=dict(
legacy_id=json_data["legacy_recid"],
new_pid=document["pid"],
status="SUCCESS",
),
)
return document
except IlsValidationError as e:
click.secho("Field: {}".format(e.errors[0].res["field"]), fg="red")
click.secho(e.original_exception.message, fg="red")
raise e
except PIDAlreadyExists as e:
allow_updates = current_app.config.get(
"CDS_ILS_MIGRATION_ALLOW_UPDATES"
)
if not allow_updates:
raise e
# update document if already exists with legacy_recid
legacy_pid_type = current_app.config[
"CDS_ILS_RECORD_LEGACY_PID_TYPE"
]
# When updating we don't want to change the pid
if "pid" in json_data:
del json_data["pid"]
document = get_record_by_legacy_recid(
document_cls, legacy_pid_type, json_data["legacy_recid"]
)
document.update(json_data)
document.model.updated = timestamp.replace(tzinfo=None)
document.commit()
db.session.commit()
documents_logger.info(
"UPDATED",
extra=dict(
legacy_id=json_data["legacy_recid"],
new_pid=document["pid"],
status="SUCCESS",
),
)
return document
def create_record(cls, dump, rectype):
"""Create a new record from dump."""
records_logger = logging.getLogger(f"{rectype}s_logger")
series_cls = current_app_ils.series_record_cls
record_uuid = uuid.uuid4()
try:
with db.session.begin_nested():
timestamp, json_data = dump.revisions[-1]
if rectype == 'serial'\
and serial_already_exists(json_data["title"]):
return
json_data = clean_created_by_field(json_data)
vocabulary_validator.validate(VOCABULARIES_FIELDS, json_data)
provider = SeriesIdProvider.create(
object_type="rec",
object_uuid=record_uuid,
)
add_cds_url(json_data)
json_data["pid"] = provider.pid.pid_value
if rectype == "journal":
legacy_pid_type = current_app.config[
"CDS_ILS_SERIES_LEGACY_PID_TYPE"]
legacy_recid_minter(json_data["legacy_recid"],
legacy_pid_type, record_uuid)
add_cover_metadata(json_data)
series = series_cls.create(json_data, record_uuid)
created_date = json_data.get("_created",
CDS_ILS_FALLBACK_CREATION_DATE)
series.model.created = parser.parse(created_date)
series.model.updated = timestamp.replace(tzinfo=None)
series.commit()
db.session.commit()
records_logger.info(
"CREATED",
extra=dict(
new_pid=series["pid"],
status="SUCCESS",
legacy_id=json_data["legacy_recid"],
),
)
return series
except PIDAlreadyExists as e:
allow_updates = current_app.config.get(
"CDS_ILS_MIGRATION_ALLOW_UPDATES")
if not allow_updates:
raise e
# update document if already exists with legacy_recid
legacy_pid_type = current_app.config[
"CDS_ILS_SERIES_LEGACY_PID_TYPE"]
# When updating we don't want to change the pid
if "pid" in json_data:
del json_data["pid"]
series = get_record_by_legacy_recid(series_cls, legacy_pid_type,
json_data["legacy_recid"])
series.update(json_data)
series.model.updated = timestamp.replace(tzinfo=None)
series.commit()
db.session.commit()
records_logger.info(
"UPDATED",
extra=dict(
legacy_id=json_data["legacy_recid"],
new_pid=series["pid"],
status="SUCCESS",
),
)
return series
def clean_record_json(record):
"""Create a record for ILS."""
barcode = (record.get("barcode").replace("No barcode associated",
"").replace(
"No barcode asociated", ""))
status = get_status(record)
try:
if barcode:
item = get_item_by_barcode(barcode)
document_pid = item.get("document_pid")
else:
document_pid = find_correct_document_pid(record)
except ItemMigrationError:
document_pid = find_correct_document_pid(record)
try:
# library_pid
provider = get_provider_by_legacy_id(record["id_crcLIBRARY"],
provider_type="LIBRARY")
provider_pid = provider.pid.pid_value
except ProviderError as e:
provider_pid = MIGRATION_PROVIDER_PID
new_record = dict(
document_pid=document_pid,
legacy_id=record.get("legacy_id"),
provider_pid=provider_pid,
patron_pid=get_patron_pid(record),
status=status,
type=get_type(record),
)
# Optional fields
if status == "CANCELLED":
new_record.update(cancel_reason="MIGRATED/UNKNOWN")
expected_delivery_date = record.get("expected_date")
if expected_delivery_date:
new_record.update(
expected_delivery_date=get_date(expected_delivery_date))
received_date = record.get("arrival_date")
if received_date:
new_record.update(received_date=get_date(received_date))
request_date = record.get("request_date")
if request_date:
new_record.update(request_date=get_date(request_date))
# former return date is the new due date of the borrowing request
due_date = record.get("return_date")
if due_date:
new_record.update(due_date=get_date(due_date))
total = get_cost(record)
if total:
new_record.update(total=total)
notes = get_acq_ill_notes(record)
if notes:
new_record.update(notes=notes)
budget_code = record.get("budget_code")
if budget_code:
new_record.update(budget_code=budget_code)
validate_ill(new_record)
vocabulary_validator.validate(VOCABULARIES_FIELDS, new_record)
return new_record