def _create_record_from_filepath(path, rec_uuid, indexer, versions, verbose): with open(path) as record_file: record_str = record_file.read() record_str = resolve_community_id(record_str) record_str = resolve_block_schema_id(record_str) json_data = json.loads(record_str) b2share_deposit_uuid_minter(rec_uuid, data=json_data) deposit = Deposit.create(json_data, id_=rec_uuid) ObjectVersion.create(deposit.files.bucket, 'myfile', stream=BytesIO(b'mycontent')) deposit.publish() pid, record = deposit.fetch_published() indexer.index(record) if verbose > 0: click.secho('created new record: {}'.format(str(rec_uuid))) last_id = pid.pid_value for i in range(2 * versions): rec_uuid = uuid4() json_data = json.loads(record_str) b2share_deposit_uuid_minter(rec_uuid, data=json_data) deposit2 = Deposit.create(json_data, id_=rec_uuid, version_of=last_id) ObjectVersion.create(deposit2.files.bucket, 'myfile-ver{}'.format(i), stream=BytesIO(b'mycontent')) deposit2.publish() pid, record2 = deposit2.fetch_published() indexer.index(record2) last_id = pid.pid_value if verbose > 0: click.secho('created new version: {}'.format(str(rec_uuid))) return record, deposit
def _create_record_from_filepath(path, rec_uuid, indexer, versions, verbose): with open(path) as record_file: record_str = record_file.read() record_str = resolve_community_id(record_str) record_str = resolve_block_schema_id(record_str) json_data = json.loads(record_str) b2share_deposit_uuid_minter(rec_uuid, data=json_data) deposit = Deposit.create(json_data, id_=rec_uuid) ObjectVersion.create(deposit.files.bucket, 'myfile', stream=BytesIO(b'mycontent')) deposit.publish() pid, record = deposit.fetch_published() indexer.index(record) if verbose > 0: click.secho('created new record: {}'.format(str(rec_uuid))) last_id = pid.pid_value for i in range(2*versions): rec_uuid = uuid4() json_data = json.loads(record_str) b2share_deposit_uuid_minter(rec_uuid, data=json_data) deposit2 = Deposit.create(json_data, id_=rec_uuid, version_of=last_id) ObjectVersion.create(deposit2.files.bucket, 'myfile-ver{}'.format(i), stream=BytesIO(b'mycontent')) deposit2.publish() pid, record2 = deposit2.fetch_published() indexer.index(record2) last_id = pid.pid_value if verbose > 0: click.secho('created new version: {}'.format(str(rec_uuid))) return record, deposit
def test_deposit_create_with_invalid_fields_fails(app, test_records_data): """Test deposit creation without or with an invalid field fails.""" data = test_records_data[0] with app.app_context(): data['publication_state'] = 'published' with pytest.raises(InvalidDepositError): deposit = Deposit.create(deepcopy(data)) with app.app_context(): data['$schema'] = '__garbage__' with pytest.raises(InvalidDepositError): deposit = Deposit.create(deepcopy(data))
def test_deposit_create_with_invalid_community_fails(app, test_records_data): """Test deposit creation without or with an invalid community fails.""" data = test_records_data[0] with app.app_context(): # test with no community del data['community'] with pytest.raises(ValidationError): deposit = Deposit.create(deepcopy(data)) with app.app_context(): # test with an invalid community data['community'] = str(uuid.uuid4()) with pytest.raises(InvalidDepositError): deposit = Deposit.create(deepcopy(data))
def process_v1_record(directory, indexer, base_url, logfile): """ Parse a downloaded file containing records """ with open(os.path.join(directory, '___record___.json'), 'r') as f: file_content = f.read() record_json = json.loads(file_content) recid = str(record_json.get('record_id')) if not record_json.get('domain'): click.secho('Record {} "{}" has no domain, '.format(recid, record_json.get('title')), fg='red') logfile.write("\n********************\n") logfile.write("\nERROR: record {} has no domain, is in limbo\n".format(recid)) logfile.write("\n********************\n") click.secho('Processing record {} "{}"'.format(recid, record_json.get('title'))) record = _process_record(record_json) if record is not None: user = get_or_create_user(record_json['uploaded_by']) with current_app.test_request_context('/', base_url=base_url): current_app.login_manager.reload_user(user) try: deposit = Deposit.create(record) _create_bucket(deposit, record_json, directory, logfile) deposit.publish() _, record = deposit.fetch_published() # index the record indexer.index(record) db.session.commit() except: logfile.write("\n********************") logfile.write("\nERROR while creating record {}\n".format(recid)) logfile.write(traceback.format_exc()) logfile.write("\n********************") click.secho("Finished processing {}".format(record['titles'][0]['title']))
def test_record_publish_adds_no_handles_for_external_files(app, records_data_with_external_pids, test_records_data): """Test that no handle PIDs are created for external files.""" for metadata in test_records_data: with app.app_context(): app.config.update({'FAKE_EPIC_PID': True}) external_pids = records_data_with_external_pids['external_pids'] external_dict = {x['key']: x['ePIC_PID'] for x in external_pids} data = deepcopy(metadata) data['external_pids'] = deepcopy(external_pids) record_uuid = uuid.uuid4() b2share_deposit_uuid_minter(record_uuid, data=data) deposit = Deposit.create(data, id_=record_uuid) ObjectVersion.create(deposit.files.bucket, 'real_file_1.txt', stream=BytesIO(b'mycontent')) ObjectVersion.create(deposit.files.bucket, 'real_file_2.txt', stream=BytesIO(b'mycontent')) deposit.submit() deposit.publish() deposit.commit() _, record = deposit.fetch_published() # external files don't get a handle PID, they already have one # which is stored in record['_deposit']['external_pids'] for f in record.files: if f['key'] in external_dict: assert f.get('ePIC_PID') is None else: assert '0000' in f['ePIC_PID'] # is a new fake PID
def create_deposit(data, creator, files=None): """Create a deposit with the given user as creator.""" with authenticated_user(creator): deposit = Deposit.create(data=deepcopy(data)) if files is not None: for key, value in files.items(): deposit.files[key] = BytesIO(value) return deposit
def test_deposit_create_with_incomplete_metadata(app, test_incomplete_records_data): """Test deposit creation with incomplete metadata succeeds.""" with app.app_context(): for data in test_incomplete_records_data: deposit = Deposit.create(data.incomplete_data) assert ( deposit['publication_state'] == PublicationStates.draft.name) assert (deposit['_deposit']['status'] == 'draft')
def create(data): data = deepcopy(data) record_uuid = uuid.uuid4() # Create persistent identifier b2share_deposit_uuid_minter(record_uuid, data=data) deposit = Deposit.create(data=data, id_=record_uuid, version_of=version_of) if files is not None: for key, value in files.items(): deposit.files[key] = BytesIO(value) return deposit
def test_deposit_submit_with_incomplete_metadata(app, test_incomplete_records_data): """Test deposit submission with incomplete metadata fails.""" for data in test_incomplete_records_data: with app.app_context(): deposit = Deposit.create(data.complete_data) deposit.commit() # make the data incomplete deposit = deposit.patch(data.patch) with pytest.raises(ValidationError): deposit.submit()
def create_deposits(app, test_records_data, creator): """Create test deposits.""" DepositInfo = namedtuple('DepositInfo', ['id', 'data', 'deposit']) with authenticated_user(creator): deposits = [Deposit.create(data=data) for data in deepcopy(test_records_data)] for deposit in deposits: deposit.commit() deposit.commit() return [DepositInfo(dep.id, dep.dumps(), dep) for dep in deposits]
def test_deposit_publish_with_incomplete_metadata( app, test_incomplete_records_data): """Test publication of an incomplete deposit fails.""" for data in test_incomplete_records_data: with app.app_context(): deposit = Deposit.create(data.complete_data) deposit.submit() deposit.commit() # make the data incomplete deposit = deposit.patch(data.patch) with pytest.raises(ValidationError): deposit.publish()
def create_deposits(app, test_records_data, creator): """Create test deposits.""" DepositInfo = namedtuple('DepositInfo', ['deposit_id', 'data', 'deposit']) deposits = [] with authenticated_user(creator): for data in deepcopy(test_records_data): record_uuid = uuid.uuid4() # Create persistent identifier b2share_deposit_uuid_minter(record_uuid, data=data) deposits.append(Deposit.create(data=data, id_=record_uuid)) return [DepositInfo(dep.id, dep.dumps(), dep) for dep in deposits]
def test_record_commit_with_incomplete_metadata(app, test_incomplete_records_data): """Test commit of an incomplete record fails.""" for data in test_incomplete_records_data: with app.app_context(): deposit = Deposit.create(data.complete_data) deposit.submit() deposit.publish() deposit.commit() pid, record = deposit.fetch_published() # make the data incomplete record = record.patch(data.patch) with pytest.raises(ValidationError): record.commit()
def test_record_commit_with_incomplete_metadata(app, test_incomplete_records_data): """Test commit of an incomplete record fails.""" for metadata in test_incomplete_records_data: with app.app_context(): data = deepcopy(metadata.complete_data) record_uuid = uuid.uuid4() b2share_deposit_uuid_minter(record_uuid, data=data) deposit = Deposit.create(data, id_=record_uuid) deposit.submit() deposit.publish() deposit.commit() pid, record = deposit.fetch_published() # make the data incomplete record = record.patch(metadata.patch) with pytest.raises(ValidationError): record.commit()
def create_deposits(app, test_records_data, creator): """Create test deposits.""" DepositInfo = namedtuple( 'DepositInfo', [ 'deposit_id', 'data', 'deposit', # FIXME: replaced by get_deposit, remove it later 'get_deposit' ]) deposits = [] with authenticated_user(creator): for data in deepcopy(test_records_data): record_uuid = uuid.uuid4() # Create persistent identifier b2share_deposit_uuid_minter(record_uuid, data=data) deposits.append(B2ShareDeposit.create(data=data, id_=record_uuid)) return [DepositInfo( dep.id, dep.dumps(), dep, (lambda id: lambda: B2ShareDeposit.get_record(id))(dep.id) ) for dep in deposits]
def _create_records(path, verbose): """Create demo records.""" indexer = RecordIndexer( record_to_index=lambda record: ('records', 'record') ) if verbose > 0: click.secho('Creating records', fg='yellow', bold=True) with db.session.begin_nested(): records_dir = os.path.join(path, 'records') nb_records = 0 for root, dirs, files in os.walk(records_dir): for filename in files: split_filename = os.path.splitext(filename) if split_filename[1] == '.json': rec_uuid = UUID(split_filename[0]) with open(os.path.join(records_dir, root, filename)) as record_file: record_str = record_file.read() record_str = resolve_community_id(record_str) record_str = resolve_block_schema_id(record_str) deposit = Deposit.create(json.loads(record_str), id_=rec_uuid) ObjectVersion.create(deposit.files.bucket, 'myfile', stream=BytesIO(b'mycontent')) deposit.publish() pid, record = deposit.fetch_published() # index the record indexer.index(record) if verbose > 1: click.secho('CREATED RECORD {0}:\n {1}'.format( str(rec_uuid), json.dumps(record, indent=4) )) click.secho('CREATED DEPOSIT {0}:\n {1}'.format( str(rec_uuid), json.dumps(deposit, indent=4) )) nb_records += 1 if verbose > 0: click.secho('Created {} records!'.format(nb_records), fg='green')
def create_deposits(app, test_records_data, creator): """Create test deposits.""" DepositInfo = namedtuple( 'DepositInfo', [ 'deposit_id', 'data', 'deposit', # FIXME: replaced by get_deposit, remove it later 'get_deposit' ]) deposits = [] with authenticated_user(creator): for data in deepcopy(test_records_data): record_uuid = uuid.uuid4() # Create persistent identifier b2share_deposit_uuid_minter(record_uuid, data=data) deposits.append(B2ShareDeposit.create(data=data, id_=record_uuid)) return [ DepositInfo(dep.id, dep.dumps(), dep, (lambda id: lambda: B2ShareDeposit.get_record(id))(dep.id)) for dep in deposits ]