def create_record(cls, dump): """Create a new record from dump.""" record_uuid = uuid.uuid4() try: # with db.session.begin_nested(): provider = DocumentIdProvider.create( object_type="rec", object_uuid=record_uuid, ) timestamp, json_data = dump.revisions[-1] json_data["pid"] = provider.pid.pid_value json_data = clean_created_by_field(json_data) document = Document.create(json_data, record_uuid) document.model.created = dump.created.replace(tzinfo=None) document.model.updated = timestamp.replace(tzinfo=None) document.commit() db.session.commit() return document except IlsValidationError as e: click.secho("Field: {}".format(e.errors[0].res["field"]), fg="red") click.secho(e.original_exception.message, fg="red") db.session.rollback() raise e
def test_document_creation_refs(app): """Test creation of a document.""" d = dict(pid="a1bc", title="Test title", authors=[dict(full_name="John Doe")], publication_year="2010") doc = Document.create(d) _assert_extra_fields(doc)
def test_journal_relation_from_publication_info(app): """Test journal-document relation from publication info field.""" document_data = { "$schema": "https://127.0.0.1:5000/schemas/documents/document-v1.0.0.json", "created_by": {"type": "script", "value": "test"}, "pid": "4321", "legacy_recid": "1111", "title": "Book: A Book", "document_type": "BOOK", "authors": [{"full_name": "Author Author"}], "abstracts": [{"value": "This is an abstract"}], "language": ["it"], "publication_year": "2020", "identifiers": [{"scheme": "ISBN", "value": "0123456789"}], "cover_metadata": {"ISBN": "0123456789"}, "publication_info": [{"journal_issue": "issue"}], "_migration": { "has_journal": True, "journal_record_legacy_recids": [ { "recid": "1234", "volume": None, } ], }, } journal_data = { "$schema": "https://127.0.0.1:5000/schemas/series/series-v1.0.0.json", "pid": "serid-4", "title": "Dispersion Forces", "authors": ["Buhmann, Stefan Yoshi"], "abstract": "This is a multipart monograph", "mode_of_issuance": "SERIAL", "legacy_recid": "1234", } record_uuid = mint_record_pid( DOCUMENT_PID_TYPE, "pid", {"pid": document_data["pid"]} ) document = Document.create(document_data, record_uuid) record_uuid = mint_record_pid( SERIES_PID_TYPE, "pid", {"pid": journal_data["pid"]} ) journal = Series.create(journal_data, record_uuid) legacy_recid_minter(journal["legacy_recid"], record_uuid) db.session.commit() ri = RecordIndexer() ri.index(document) ri.index(journal) current_search.flush_and_refresh(index="*") link_documents_and_serials() document_rec = Document.get_record_by_pid(document["pid"]) assert "serial" in document_rec.relations
def create_multipart_volumes(pid, multipart_legacy_recid, migration_volumes): """Create multipart volume documents.""" volumes = {} # Combine all volume data by volume number click.echo('Creating volume for {}...'.format(multipart_legacy_recid)) for obj in migration_volumes: volume_number = obj['volume'] if volume_number not in volumes: volumes[volume_number] = {} volume = volumes[volume_number] for key in obj: if key != 'volume': if key in volume: raise KeyError( 'Duplicate key "{}" for multipart {}'.format( key, multipart_legacy_recid)) volume[key] = obj[key] volume_numbers = iter(sorted(volumes.keys())) # Re-use the current record for the first volume # TODO review this - there are more cases of multiparts first_volume = next(volume_numbers) first = Document.get_record_by_pid(pid) if 'title' in volumes[first_volume]: first['title'] = volumes[first_volume]['title'] first['volume'] = first_volume first['_migration']['multipart_legacy_recid'] = multipart_legacy_recid # to be tested if 'legacy_recid' in first: del first['legacy_recid'] first.commit() yield first # Create new records for the rest for number in volume_numbers: temp = first.copy() temp['title'] = volumes[number]['title'] temp['volume'] = number record_uuid = uuid.uuid4() provider = DocumentIdProvider.create( object_type='rec', object_uuid=record_uuid, ) temp['pid'] = provider.pid.pid_value record = Document.create(temp, record_uuid) record.commit() yield record
def test_document_creation_refs(app): """Test creation of a document.""" d = dict( pid="a1bc", created_by={ "type": "script", "value": "demo" }, title="Test title", authors=[dict(full_name="John Doe")], publication_year="2010", document_type="BOOK", ) doc = Document.create(d) _assert_extra_fields(doc)
def test_document_update_refs(app): """Test update of a document.""" d = dict(pid="a1bc", title="Test title", authors=[dict(full_name="John Doe")], publication_year="2010") doc = Document.create(d) del doc["circulation"] del doc["relations"] del doc["eitems"] del doc["items"] del doc["stock"] doc.update(dict(title="Test title 2")) _assert_extra_fields(doc) assert doc["title"] == "Test title 2"
def create_record(cls, dump): """Create a new record from dump.""" # Reserve record identifier, create record and recid pid in one # operation. record_uuid = uuid.uuid4() provider = DocumentIdProvider.create( object_type='rec', object_uuid=record_uuid, ) timestamp, json_data = dump.revisions[-1] json_data['pid'] = provider.pid.pid_value try: document = Document.create(json_data, record_uuid) document.model.created = dump.created.replace(tzinfo=None) document.model.updated = timestamp.replace(tzinfo=None) document.commit() db.session.commit() return document except IlsValidationError as e: click.secho(e.original_exception.message, fg='red')
def create_record(cls, dump): """Create a new record from dump.""" # Reserve record identifier, create record and recid pid in one # operation. timestamp, data = dump.latest record = Record.create(data) record_uuid = uuid.uuid4() provider = DocumentIdProvider.create( object_type='rec', object_uuid=record_uuid, ) timestamp, json_data = dump.rest[-1] json_data['pid'] = provider.pid.pid_value record.model.json = json_data record.model.created = dump.created.replace(tzinfo=None) record.model.updated = timestamp.replace(tzinfo=None) document = Document.create(record.model.json, record_uuid) document.commit() db.session.commit() return document
def testdata(app, db, es_clear, system_user): """Create, index and return test data.""" indexer = RecordIndexer() locations = load_json_from_datadir("locations.json") for location in locations: record = Location.create(location) mint_record_pid(LOCATION_PID_TYPE, "pid", record) record.commit() db.session.commit() indexer.index(record) internal_locations = load_json_from_datadir("internal_locations.json") for internal_location in internal_locations: record = InternalLocation.create(internal_location) mint_record_pid( INTERNAL_LOCATION_PID_TYPE, "pid", record ) record.commit() db.session.commit() indexer.index(record) documents = load_json_from_datadir("documents.json") for doc in documents: record = Document.create(doc) mint_record_pid(DOCUMENT_PID_TYPE, "pid", record) record.commit() db.session.commit() indexer.index(record) items = load_json_from_datadir("items.json") for item in items: record = Item.create(item) mint_record_pid(ITEM_PID_TYPE, "pid", record) record.commit() db.session.commit() indexer.index(record) loans = load_json_from_datadir("loans.json") for loan in loans: record = Loan.create(loan) mint_record_pid(CIRCULATION_LOAN_PID_TYPE, "pid", record) record.commit() db.session.commit() indexer.index(record) series = load_json_from_datadir("series.json") for serie in series: record = Series.create(serie) mint_record_pid(SERIES_PID_TYPE, "pid", record) record.commit() db.session.commit() indexer.index(record) # flush all indices after indexing, otherwise ES won't be ready for tests current_search.flush_and_refresh(index='*') return { "documents": documents, "items": items, "loans": loans, "locations": locations, "series": series, }
def create_multipart_volumes( pid, multipart_legacy_recid, migration_volumes, document_base_metadata ): """Create multipart volume documents.""" volumes = {} # Combine all volume data by volume number click.echo("Creating volume for {}...".format(multipart_legacy_recid)) for obj in migration_volumes: volume_number = obj["volume"] if volume_number not in volumes: volumes[volume_number] = {} volume = volumes[volume_number] if "isbn" in obj: # the isbn can represent both a document and an eitem if "isbns" not in volume: volume["isbns"] = [] volume["isbns"].append({ "value": obj["isbn"], "is_electronic": bool(obj["is_electronic"]) }) # TODO physical description elif "barcode" in obj: # the barcode represents an item if "items" not in volume: volume["items"] = [] volume["items"].append({ "barcode": obj["barcode"] }) else: # all other fields should be treated as # additional metadata for the document for key in obj: if key != "volume": if key in volume: # abort in case of conflict raise KeyError( 'Duplicate key "{}" for multipart {}'.format( key, multipart_legacy_recid ) ) volume[key] = obj[key] volume_numbers = iter(sorted(volumes.keys())) inherited_metadata = deepcopy(document_base_metadata) inherited_metadata["_migration"]["multipart_legacy_recid"] = \ multipart_legacy_recid inherited_metadata["authors"] = \ inherited_metadata["_migration"]["authors"] \ if "authors" in inherited_metadata["_migration"] else [] inherited_metadata["serial_title"] = inherited_metadata.get("title") # to be tested if "legacy_recid" in inherited_metadata: del inherited_metadata["legacy_recid"] # Create new records for the rest for number in volume_numbers: volume = volumes[number] temp = inherited_metadata.copy() if "title" in volume and volume["title"]: temp["title"] = volume["title"] temp["volume"] = number # TODO possibly more fields to merge record_uuid = uuid.uuid4() try: with db.session.begin_nested(): provider = DocumentIdProvider.create( object_type="rec", object_uuid=record_uuid ) temp["pid"] = provider.pid.pid_value record = Document.create(temp, record_uuid) record.commit() db.session.commit() yield record except IlsValidationError as e: print("Validation error: {}" .format(str(e.original_exception.message)))