def create_document(self):
"""Create a new record from dump."""
cleaned_json = self._before_create()
document_class = current_app_ils.document_record_cls
# Reserve record identifier, create record and recid pid in one
# operation.
record_uuid = uuid.uuid4()
try:
with db.session.begin_nested():
provider = DocumentIdProvider.create(
object_type="rec",
object_uuid=record_uuid,
)
cleaned_json["pid"] = provider.pid.pid_value
document = document_class.create(cleaned_json, record_uuid)
created_date = cleaned_json.get("_created")
if created_date:
document.model.created = parser.parse(created_date)
db.session.commit()
return document
except IlsValidationError as e:
click.secho("Field: {}".format(e.errors[0].res["field"]), fg="red")
click.secho(e.original_exception.message, fg="red")
db.session.rollback()
raise e
def create_record(cls, dump):
"""Create a new record from dump."""
record_uuid = uuid.uuid4()
try:
# with db.session.begin_nested():
provider = DocumentIdProvider.create(
object_type="rec",
object_uuid=record_uuid,
)
timestamp, json_data = dump.revisions[-1]
json_data["pid"] = provider.pid.pid_value
json_data = clean_created_by_field(json_data)
document = Document.create(json_data, record_uuid)
document.model.created = dump.created.replace(tzinfo=None)
document.model.updated = timestamp.replace(tzinfo=None)
document.commit()
db.session.commit()
return document
except IlsValidationError as e:
click.secho("Field: {}".format(e.errors[0].res["field"]), fg="red")
click.secho(e.original_exception.message, fg="red")
db.session.rollback()
raise e
def create_multipart_volumes(pid, multipart_legacy_recid, migration_volumes):
"""Create multipart volume documents."""
volumes = {}
# Combine all volume data by volume number
click.echo('Creating volume for {}...'.format(multipart_legacy_recid))
for obj in migration_volumes:
volume_number = obj['volume']
if volume_number not in volumes:
volumes[volume_number] = {}
volume = volumes[volume_number]
for key in obj:
if key != 'volume':
if key in volume:
raise KeyError(
'Duplicate key "{}" for multipart {}'.format(
key, multipart_legacy_recid))
volume[key] = obj[key]
volume_numbers = iter(sorted(volumes.keys()))
# Re-use the current record for the first volume
# TODO review this - there are more cases of multiparts
first_volume = next(volume_numbers)
first = Document.get_record_by_pid(pid)
if 'title' in volumes[first_volume]:
first['title'] = volumes[first_volume]['title']
first['volume'] = first_volume
first['_migration']['multipart_legacy_recid'] = multipart_legacy_recid
# to be tested
if 'legacy_recid' in first:
del first['legacy_recid']
first.commit()
yield first
# Create new records for the rest
for number in volume_numbers:
temp = first.copy()
temp['title'] = volumes[number]['title']
temp['volume'] = number
record_uuid = uuid.uuid4()
provider = DocumentIdProvider.create(
object_type='rec',
object_uuid=record_uuid,
)
temp['pid'] = provider.pid.pid_value
record = Document.create(temp, record_uuid)
record.commit()
yield record
def create_record(cls, dump):
"""Create a new record from dump."""
document_cls = current_app_ils.document_record_cls
record_uuid = uuid.uuid4()
timestamp, json_data = dump.revisions[-1]
json_data = clean_created_by_field(json_data)
add_cover_metadata(json_data)
try:
with db.session.begin_nested():
# checks if the document with this legacy_recid already exists
legacy_recid_minter(json_data["legacy_recid"], record_uuid)
provider = DocumentIdProvider.create(
object_type="rec",
object_uuid=record_uuid,
)
json_data["pid"] = provider.pid.pid_value
document = document_cls.create(json_data, record_uuid)
document.model.created = dump.created.replace(tzinfo=None)
document.model.updated = timestamp.replace(tzinfo=None)
document.commit()
db.session.commit()
return document
except IlsValidationError as e:
click.secho("Field: {}".format(e.errors[0].res["field"]), fg="red")
click.secho(e.original_exception.message, fg="red")
raise e
except PIDAlreadyExists as e:
allow_updates = \
current_app.config.get("CDS_ILS_MIGRATION_ALLOW_UPDATES")
if not allow_updates:
raise e
# update document if already exists with legacy_recid
document = get_record_by_legacy_recid(document_cls,
json_data["legacy_recid"])
document.update(json_data)
document.model.updated = timestamp.replace(tzinfo=None)
document.commit()
db.session.commit()
return document
def create_record(cls, dump):
"""Create a new record from dump."""
# Reserve record identifier, create record and recid pid in one
# operation.
record_uuid = uuid.uuid4()
provider = DocumentIdProvider.create(
object_type='rec',
object_uuid=record_uuid,
)
timestamp, json_data = dump.revisions[-1]
json_data['pid'] = provider.pid.pid_value
try:
document = Document.create(json_data, record_uuid)
document.model.created = dump.created.replace(tzinfo=None)
document.model.updated = timestamp.replace(tzinfo=None)
document.commit()
db.session.commit()
return document
except IlsValidationError as e:
click.secho(e.original_exception.message, fg='red')
def create_record(cls, dump):
"""Create a new record from dump."""
# Reserve record identifier, create record and recid pid in one
# operation.
timestamp, data = dump.latest
record = Record.create(data)
record_uuid = uuid.uuid4()
provider = DocumentIdProvider.create(
object_type='rec',
object_uuid=record_uuid,
)
timestamp, json_data = dump.rest[-1]
json_data['pid'] = provider.pid.pid_value
record.model.json = json_data
record.model.created = dump.created.replace(tzinfo=None)
record.model.updated = timestamp.replace(tzinfo=None)
document = Document.create(record.model.json, record_uuid)
document.commit()
db.session.commit()
return document
def create_record(cls, dump):
"""Create a new record from dump."""
document_cls = current_app_ils.document_record_cls
record_uuid = uuid.uuid4()
timestamp, json_data = dump.revisions[-1]
json_data = clean_created_by_field(json_data)
vocabulary_validator.validate(VOCABULARIES_FIELDS, json_data)
add_cover_metadata(json_data)
add_title_from_conference_info(json_data)
add_cds_url(json_data)
try:
with db.session.begin_nested():
# checks if the document with this legacy_recid already exists
legacy_pid_type = current_app.config[
"CDS_ILS_RECORD_LEGACY_PID_TYPE"
]
# First mint the legacy_recid before assigning the pid. In case
# it fails while importing an existing record we will update it
# and don't want the new pid, since there is already one there
legacy_recid_minter(
json_data["legacy_recid"], legacy_pid_type, record_uuid
)
provider = DocumentIdProvider.create(
object_type="rec",
object_uuid=record_uuid,
)
# requirement from the library
if (
json_data["_migration"]["has_journal"]
and json_data["document_type"] != "PROCEEDINGS"
):
json_data["document_type"] = "SERIAL_ISSUE"
json_data["pid"] = provider.pid.pid_value
document = document_cls.create(json_data, record_uuid)
created_date = json_data.get(
"_created", CDS_ILS_FALLBACK_CREATION_DATE
)
document.model.created = parser.parse(created_date)
document.model.updated = timestamp.replace(tzinfo=None)
document.commit()
db.session.commit()
documents_logger.info(
"CREATED",
extra=dict(
legacy_id=json_data["legacy_recid"],
new_pid=document["pid"],
status="SUCCESS",
),
)
return document
except IlsValidationError as e:
click.secho("Field: {}".format(e.errors[0].res["field"]), fg="red")
click.secho(e.original_exception.message, fg="red")
raise e
except PIDAlreadyExists as e:
allow_updates = current_app.config.get(
"CDS_ILS_MIGRATION_ALLOW_UPDATES"
)
if not allow_updates:
raise e
# update document if already exists with legacy_recid
legacy_pid_type = current_app.config[
"CDS_ILS_RECORD_LEGACY_PID_TYPE"
]
# When updating we don't want to change the pid
if "pid" in json_data:
del json_data["pid"]
document = get_record_by_legacy_recid(
document_cls, legacy_pid_type, json_data["legacy_recid"]
)
document.update(json_data)
document.model.updated = timestamp.replace(tzinfo=None)
document.commit()
db.session.commit()
documents_logger.info(
"UPDATED",
extra=dict(
legacy_id=json_data["legacy_recid"],
new_pid=document["pid"],
status="SUCCESS",
),
)
return document
def create_multipart_volumes(
pid, multipart_legacy_recid, migration_volumes, document_base_metadata
):
"""Create multipart volume documents."""
volumes = {}
# Combine all volume data by volume number
click.echo("Creating volume for {}...".format(multipart_legacy_recid))
for obj in migration_volumes:
volume_number = obj["volume"]
if volume_number not in volumes:
volumes[volume_number] = {}
volume = volumes[volume_number]
if "isbn" in obj:
# the isbn can represent both a document and an eitem
if "isbns" not in volume:
volume["isbns"] = []
volume["isbns"].append({
"value": obj["isbn"],
"is_electronic": bool(obj["is_electronic"])
})
# TODO physical description
elif "barcode" in obj:
# the barcode represents an item
if "items" not in volume:
volume["items"] = []
volume["items"].append({
"barcode": obj["barcode"]
})
else:
# all other fields should be treated as
# additional metadata for the document
for key in obj:
if key != "volume":
if key in volume:
# abort in case of conflict
raise KeyError(
'Duplicate key "{}" for multipart {}'.format(
key, multipart_legacy_recid
)
)
volume[key] = obj[key]
volume_numbers = iter(sorted(volumes.keys()))
inherited_metadata = deepcopy(document_base_metadata)
inherited_metadata["_migration"]["multipart_legacy_recid"] = \
multipart_legacy_recid
inherited_metadata["authors"] = \
inherited_metadata["_migration"]["authors"] \
if "authors" in inherited_metadata["_migration"] else []
inherited_metadata["serial_title"] = inherited_metadata.get("title")
# to be tested
if "legacy_recid" in inherited_metadata:
del inherited_metadata["legacy_recid"]
# Create new records for the rest
for number in volume_numbers:
volume = volumes[number]
temp = inherited_metadata.copy()
if "title" in volume and volume["title"]:
temp["title"] = volume["title"]
temp["volume"] = number
# TODO possibly more fields to merge
record_uuid = uuid.uuid4()
try:
with db.session.begin_nested():
provider = DocumentIdProvider.create(
object_type="rec", object_uuid=record_uuid
)
temp["pid"] = provider.pid.pid_value
record = Document.create(temp, record_uuid)
record.commit()
db.session.commit()
yield record
except IlsValidationError as e:
print("Validation error: {}"
.format(str(e.original_exception.message)))