def _create_record_from_filepath(path, rec_uuid, indexer, versions, verbose):
with open(path) as record_file:
record_str = record_file.read()
record_str = resolve_community_id(record_str)
record_str = resolve_block_schema_id(record_str)
json_data = json.loads(record_str)
b2share_deposit_uuid_minter(rec_uuid, data=json_data)
deposit = Deposit.create(json_data, id_=rec_uuid)
ObjectVersion.create(deposit.files.bucket,
'myfile',
stream=BytesIO(b'mycontent'))
deposit.publish()
pid, record = deposit.fetch_published()
indexer.index(record)
if verbose > 0:
click.secho('created new record: {}'.format(str(rec_uuid)))
last_id = pid.pid_value
for i in range(2 * versions):
rec_uuid = uuid4()
json_data = json.loads(record_str)
b2share_deposit_uuid_minter(rec_uuid, data=json_data)
deposit2 = Deposit.create(json_data, id_=rec_uuid, version_of=last_id)
ObjectVersion.create(deposit2.files.bucket,
'myfile-ver{}'.format(i),
stream=BytesIO(b'mycontent'))
deposit2.publish()
pid, record2 = deposit2.fetch_published()
indexer.index(record2)
last_id = pid.pid_value
if verbose > 0:
click.secho('created new version: {}'.format(str(rec_uuid)))
return record, deposit
def _create_record_from_filepath(path, rec_uuid, indexer, versions, verbose):
with open(path) as record_file:
record_str = record_file.read()
record_str = resolve_community_id(record_str)
record_str = resolve_block_schema_id(record_str)
json_data = json.loads(record_str)
b2share_deposit_uuid_minter(rec_uuid, data=json_data)
deposit = Deposit.create(json_data, id_=rec_uuid)
ObjectVersion.create(deposit.files.bucket, 'myfile',
stream=BytesIO(b'mycontent'))
deposit.publish()
pid, record = deposit.fetch_published()
indexer.index(record)
if verbose > 0:
click.secho('created new record: {}'.format(str(rec_uuid)))
last_id = pid.pid_value
for i in range(2*versions):
rec_uuid = uuid4()
json_data = json.loads(record_str)
b2share_deposit_uuid_minter(rec_uuid, data=json_data)
deposit2 = Deposit.create(json_data, id_=rec_uuid,
version_of=last_id)
ObjectVersion.create(deposit2.files.bucket, 'myfile-ver{}'.format(i),
stream=BytesIO(b'mycontent'))
deposit2.publish()
pid, record2 = deposit2.fetch_published()
indexer.index(record2)
last_id = pid.pid_value
if verbose > 0:
click.secho('created new version: {}'.format(str(rec_uuid)))
return record, deposit
def test_deposit_create_with_invalid_fields_fails(app, test_records_data):
"""Test deposit creation without or with an invalid field fails."""
data = test_records_data[0]
with app.app_context():
data['publication_state'] = 'published'
with pytest.raises(InvalidDepositError):
deposit = Deposit.create(deepcopy(data))
with app.app_context():
data['$schema'] = '__garbage__'
with pytest.raises(InvalidDepositError):
deposit = Deposit.create(deepcopy(data))
def test_deposit_create_with_invalid_community_fails(app, test_records_data):
"""Test deposit creation without or with an invalid community fails."""
data = test_records_data[0]
with app.app_context():
# test with no community
del data['community']
with pytest.raises(ValidationError):
deposit = Deposit.create(deepcopy(data))
with app.app_context():
# test with an invalid community
data['community'] = str(uuid.uuid4())
with pytest.raises(InvalidDepositError):
deposit = Deposit.create(deepcopy(data))
def process_v1_record(directory, indexer, base_url, logfile):
"""
Parse a downloaded file containing records
"""
with open(os.path.join(directory, '___record___.json'), 'r') as f:
file_content = f.read()
record_json = json.loads(file_content)
recid = str(record_json.get('record_id'))
if not record_json.get('domain'):
click.secho('Record {} "{}" has no domain, '.format(recid, record_json.get('title')),
fg='red')
logfile.write("\n********************\n")
logfile.write("\nERROR: record {} has no domain, is in limbo\n".format(recid))
logfile.write("\n********************\n")
click.secho('Processing record {} "{}"'.format(recid, record_json.get('title')))
record = _process_record(record_json)
if record is not None:
user = get_or_create_user(record_json['uploaded_by'])
with current_app.test_request_context('/', base_url=base_url):
current_app.login_manager.reload_user(user)
try:
deposit = Deposit.create(record)
_create_bucket(deposit, record_json, directory, logfile)
deposit.publish()
_, record = deposit.fetch_published()
# index the record
indexer.index(record)
db.session.commit()
except:
logfile.write("\n********************")
logfile.write("\nERROR while creating record {}\n".format(recid))
logfile.write(traceback.format_exc())
logfile.write("\n********************")
click.secho("Finished processing {}".format(record['titles'][0]['title']))
def test_record_publish_adds_no_handles_for_external_files(app,
records_data_with_external_pids,
test_records_data):
"""Test that no handle PIDs are created for external files."""
for metadata in test_records_data:
with app.app_context():
app.config.update({'FAKE_EPIC_PID': True})
external_pids = records_data_with_external_pids['external_pids']
external_dict = {x['key']: x['ePIC_PID'] for x in external_pids}
data = deepcopy(metadata)
data['external_pids'] = deepcopy(external_pids)
record_uuid = uuid.uuid4()
b2share_deposit_uuid_minter(record_uuid, data=data)
deposit = Deposit.create(data, id_=record_uuid)
ObjectVersion.create(deposit.files.bucket, 'real_file_1.txt',
stream=BytesIO(b'mycontent'))
ObjectVersion.create(deposit.files.bucket, 'real_file_2.txt',
stream=BytesIO(b'mycontent'))
deposit.submit()
deposit.publish()
deposit.commit()
_, record = deposit.fetch_published()
# external files don't get a handle PID, they already have one
# which is stored in record['_deposit']['external_pids']
for f in record.files:
if f['key'] in external_dict:
assert f.get('ePIC_PID') is None
else:
assert '0000' in f['ePIC_PID'] # is a new fake PID
def create_deposit(data, creator, files=None):
"""Create a deposit with the given user as creator."""
with authenticated_user(creator):
deposit = Deposit.create(data=deepcopy(data))
if files is not None:
for key, value in files.items():
deposit.files[key] = BytesIO(value)
return deposit
def test_deposit_create_with_incomplete_metadata(app,
test_incomplete_records_data):
"""Test deposit creation with incomplete metadata succeeds."""
with app.app_context():
for data in test_incomplete_records_data:
deposit = Deposit.create(data.incomplete_data)
assert (
deposit['publication_state'] == PublicationStates.draft.name)
assert (deposit['_deposit']['status'] == 'draft')
def create(data):
data = deepcopy(data)
record_uuid = uuid.uuid4()
# Create persistent identifier
b2share_deposit_uuid_minter(record_uuid, data=data)
deposit = Deposit.create(data=data, id_=record_uuid,
version_of=version_of)
if files is not None:
for key, value in files.items():
deposit.files[key] = BytesIO(value)
return deposit
def test_deposit_submit_with_incomplete_metadata(app,
test_incomplete_records_data):
"""Test deposit submission with incomplete metadata fails."""
for data in test_incomplete_records_data:
with app.app_context():
deposit = Deposit.create(data.complete_data)
deposit.commit()
# make the data incomplete
deposit = deposit.patch(data.patch)
with pytest.raises(ValidationError):
deposit.submit()
def create_deposits(app, test_records_data, creator):
"""Create test deposits."""
DepositInfo = namedtuple('DepositInfo', ['id', 'data', 'deposit'])
with authenticated_user(creator):
deposits = [Deposit.create(data=data)
for data in deepcopy(test_records_data)]
for deposit in deposits:
deposit.commit()
deposit.commit()
return [DepositInfo(dep.id, dep.dumps(), dep) for dep in deposits]
def test_deposit_publish_with_incomplete_metadata(
app, test_incomplete_records_data):
"""Test publication of an incomplete deposit fails."""
for data in test_incomplete_records_data:
with app.app_context():
deposit = Deposit.create(data.complete_data)
deposit.submit()
deposit.commit()
# make the data incomplete
deposit = deposit.patch(data.patch)
with pytest.raises(ValidationError):
deposit.publish()
def create_deposits(app, test_records_data, creator):
"""Create test deposits."""
DepositInfo = namedtuple('DepositInfo', ['deposit_id', 'data', 'deposit'])
deposits = []
with authenticated_user(creator):
for data in deepcopy(test_records_data):
record_uuid = uuid.uuid4()
# Create persistent identifier
b2share_deposit_uuid_minter(record_uuid, data=data)
deposits.append(Deposit.create(data=data, id_=record_uuid))
return [DepositInfo(dep.id, dep.dumps(), dep) for dep in deposits]
def test_record_commit_with_incomplete_metadata(app,
test_incomplete_records_data):
"""Test commit of an incomplete record fails."""
for data in test_incomplete_records_data:
with app.app_context():
deposit = Deposit.create(data.complete_data)
deposit.submit()
deposit.publish()
deposit.commit()
pid, record = deposit.fetch_published()
# make the data incomplete
record = record.patch(data.patch)
with pytest.raises(ValidationError):
record.commit()
def test_record_commit_with_incomplete_metadata(app,
test_incomplete_records_data):
"""Test commit of an incomplete record fails."""
for metadata in test_incomplete_records_data:
with app.app_context():
data = deepcopy(metadata.complete_data)
record_uuid = uuid.uuid4()
b2share_deposit_uuid_minter(record_uuid, data=data)
deposit = Deposit.create(data, id_=record_uuid)
deposit.submit()
deposit.publish()
deposit.commit()
pid, record = deposit.fetch_published()
# make the data incomplete
record = record.patch(metadata.patch)
with pytest.raises(ValidationError):
record.commit()
def create_deposits(app, test_records_data, creator):
"""Create test deposits."""
DepositInfo = namedtuple(
'DepositInfo', [
'deposit_id',
'data',
'deposit', # FIXME: replaced by get_deposit, remove it later
'get_deposit'
])
deposits = []
with authenticated_user(creator):
for data in deepcopy(test_records_data):
record_uuid = uuid.uuid4()
# Create persistent identifier
b2share_deposit_uuid_minter(record_uuid, data=data)
deposits.append(B2ShareDeposit.create(data=data, id_=record_uuid))
return [DepositInfo(
dep.id, dep.dumps(), dep,
(lambda id: lambda: B2ShareDeposit.get_record(id))(dep.id)
) for dep in deposits]
def _create_records(path, verbose):
"""Create demo records."""
indexer = RecordIndexer(
record_to_index=lambda record: ('records', 'record')
)
if verbose > 0:
click.secho('Creating records', fg='yellow', bold=True)
with db.session.begin_nested():
records_dir = os.path.join(path, 'records')
nb_records = 0
for root, dirs, files in os.walk(records_dir):
for filename in files:
split_filename = os.path.splitext(filename)
if split_filename[1] == '.json':
rec_uuid = UUID(split_filename[0])
with open(os.path.join(records_dir, root,
filename)) as record_file:
record_str = record_file.read()
record_str = resolve_community_id(record_str)
record_str = resolve_block_schema_id(record_str)
deposit = Deposit.create(json.loads(record_str),
id_=rec_uuid)
ObjectVersion.create(deposit.files.bucket, 'myfile',
stream=BytesIO(b'mycontent'))
deposit.publish()
pid, record = deposit.fetch_published()
# index the record
indexer.index(record)
if verbose > 1:
click.secho('CREATED RECORD {0}:\n {1}'.format(
str(rec_uuid), json.dumps(record,
indent=4)
))
click.secho('CREATED DEPOSIT {0}:\n {1}'.format(
str(rec_uuid), json.dumps(deposit,
indent=4)
))
nb_records += 1
if verbose > 0:
click.secho('Created {} records!'.format(nb_records), fg='green')
def create_deposits(app, test_records_data, creator):
"""Create test deposits."""
DepositInfo = namedtuple(
'DepositInfo',
[
'deposit_id',
'data',
'deposit', # FIXME: replaced by get_deposit, remove it later
'get_deposit'
])
deposits = []
with authenticated_user(creator):
for data in deepcopy(test_records_data):
record_uuid = uuid.uuid4()
# Create persistent identifier
b2share_deposit_uuid_minter(record_uuid, data=data)
deposits.append(B2ShareDeposit.create(data=data, id_=record_uuid))
return [
DepositInfo(dep.id, dep.dumps(), dep,
(lambda id: lambda: B2ShareDeposit.get_record(id))(dep.id))
for dep in deposits
]