def locations(db):
"""File system locations."""
tmppath = tempfile.mkdtemp()
arch_tmppath = tempfile.mkdtemp()
loc = Location(
name='testloc',
uri=tmppath,
default=True
)
arch_loc = Location(
name='archive',
uri=arch_tmppath,
default=False
)
db.session.add(loc)
db.session.add(arch_loc)
db.session.commit()
yield {'testloc': tmppath, 'archive': arch_tmppath}
shutil.rmtree(tmppath)
shutil.rmtree(arch_tmppath)
# Delete the cached property value since the tmp archive changes
current_sipstore.__dict__.pop('archive_location', None)
def locations(db, instance_path):
"""File system location."""
default = Location(name='default', uri=instance_path, default=True)
archive = Location(name='archive',
uri=os.path.join(instance_path, 'archive'),
default=False)
db.session.add(default)
db.session.add(archive)
db.session.commit()
return dict((loc.name, loc) for loc in [default, archive])
def test_location_default(app, db):
"""Test location model."""
with db.session.begin_nested():
l1 = Location(name='test1', uri='file:///tmp', default=False)
db.session.add(l1)
assert Location.get_default() is None
with db.session.begin_nested():
l2 = Location(name='test2', uri='file:///tmp', default=True)
l3 = Location(name='test3', uri='file:///tmp', default=True)
db.session.add(l2)
db.session.add(l3)
assert Location.get_default() is None
def load(source, verbose, cache, files, skip, max=None):
"""Load records attach files and index them."""
data = json.load(source)
if isinstance(data, dict):
data = [data]
# to upload remote fulltext files
upload_dir = os.path.join(current_app.instance_path, 'uploads')
try:
os.makedirs(upload_dir)
except FileExistsError:
pass
# initialize file location if needed
if not Location.get_default():
data_dir = os.path.join(current_app.instance_path, 'files')
db.session.add(
Location(name='default', uri='file://' + data_dir, default=True))
db.session.commit()
# create records and index them
click.secho('Creating records...', fg='green')
rec_uuids = load_records_with_files(data, upload_dir, max, verbose, files,
cache, skip)
click.secho('Put %d records for indexing...' % len(rec_uuids), fg='green')
RecordIndexer().bulk_index(rec_uuids)
click.secho('Execute "run" command to process the queue!', fg='yellow')
def loadlocations(force=False):
"""Load default file store and archive location."""
try:
locs = []
uris = [(
'default',
True,
current_app.config['FIXTURES_FILES_LOCATION'],
), (
'archive',
False,
current_app.config['FIXTURES_ARCHIVE_LOCATION'],
)]
for name, default, uri in uris:
if uri.startswith('/') and not exists(uri):
makedirs(uri)
if not Location.query.filter_by(name=name).count():
loc = Location(name=name, uri=uri, default=default)
db.session.add(loc)
locs.append(loc)
db.session.commit()
return locs
except Exception:
db.session.rollback()
raise
def test_SIP_files(db):
"""Test the files methods of API SIP."""
# we create a SIP model
sip = SIP_.create()
db.session.commit()
# We create an API SIP on top of it
api_sip = SIP(sip)
assert len(api_sip.files) == 0
# we setup a file storage
tmppath = tempfile.mkdtemp()
db.session.add(Location(name='default', uri=tmppath, default=True))
db.session.commit()
# we create a file
content = b'test lol\n'
bucket = Bucket.create()
obj = ObjectVersion.create(bucket, 'test.txt', stream=BytesIO(content))
db.session.commit()
# we attach it to the SIP
sf = api_sip.attach_file(obj)
db.session.commit()
assert len(api_sip.files) == 1
assert api_sip.files[0].filepath == 'test.txt'
assert sip.sip_files[0].filepath == 'test.txt'
# finalization
rmtree(tmppath)
def load_locations(force=False):
"""
Load default file store and archive location.
Lifted from https://github.com/zenodo/zenodo
"""
# NOTE: os.path.join returns its 2nd argument if that argument is an
# absolute path which is what we want
files_location = os.path.join(
current_app.instance_path,
current_app.config['FIXTURES_FILES_LOCATION'])
archive_location = os.path.join(
current_app.instance_path,
current_app.config['FIXTURES_ARCHIVE_LOCATION'])
try:
locations = []
uris = [('default', True, files_location),
('archive', False, archive_location)]
for name, default, uri in uris:
if uri.startswith('/') and not os.path.exists(uri):
os.makedirs(uri)
if not Location.query.filter_by(name=name).first():
loc = Location(name=name, uri=uri, default=default)
db.session.add(loc)
locations.append(loc)
db.session.commit()
return locations
except Exception:
db.session.rollback()
raise
def bucket_location(app, db):
"""Create a default location for managing files."""
tmppath = tempfile.mkdtemp()
location = Location(name='default', uri=tmppath, default=True)
db.session.add(location)
db.session.commit()
return location
def test_transfer_cp(db):
"""Test factories.transfer_cp function."""
# first we create a record
recid = uuid.uuid4()
PersistentIdentifier.create(
'recid',
'1337',
object_type='rec',
object_uuid=recid,
status=PIDStatus.REGISTERED)
record = Record.create({'title': 'record test'}, recid)
# we setup a file storage
tmppath = tempfile.mkdtemp()
db.session.add(Location(name='default', uri=tmppath, default=True))
db.session.commit()
# we add a file to the record
bucket = Bucket.create()
content = b'Aaah! A headcrab!!!\n'
record_buckets = RecordsBuckets.create(record=record.model, bucket=bucket)
record.files['crab.txt'] = BytesIO(content)
# test!
rec_dir = join(tmppath, create_accessioned_id('1337', 'recid'))
factories.transfer_cp(record.id, tmppath)
assert isdir(rec_dir)
assert isfile(join(rec_dir, 'crab.txt'))
with open(join(rec_dir, 'crab.txt'), "r") as f:
assert f.read() == content
# finalization
rmtree(tmppath)
def create():
"""Create a location and a bucket for uploading files."""
click.secho('Creating default location for importing files', bold=True)
# Directory where files are stored
directory = current_app.config.get('SONAR_APP_STORAGE_PATH')
if not directory:
directory = current_app.instance_path
directory = join(directory, 'files')
if exists(directory):
shutil.rmtree(directory)
makedirs(directory)
# Remove stored data
ObjectVersion.query.delete()
Bucket.query.delete()
FileInstance.query.delete()
Location.query.delete()
db.session.commit()
# Create location
loc = Location(name='local', uri=directory, default=True)
db.session.add(loc)
db.session.commit()
click.secho('Location #{id} created successfully'.format(id=loc.id),
fg='green')
def app_initialized(app):
"""Flask application with data added."""
d = app.config["DATADIR"] # folder `data`
if os.path.exists(d):
shutil.rmtree(d)
os.makedirs(d)
loc = Location(name="local", uri=d, default=True)
db.session.add(loc)
db.session.commit()
runner = app.test_cli_runner()
runner.invoke(roles_create, ["admin"])
runner.invoke(allow_action, ["superuser-access", "role", "admin"])
record_service = current_rdm_records.records_service
identity = Identity(1)
identity.provides.add(system_process)
rdmrecords = []
for i in range(5):
record_json = fake_record()
rec = record_service.create(identity, record_json)
record_service.publish(rec.id, identity)
rdmrecords.append(rec)
data = {
"rdmrecords": rdmrecords,
}
return {
"app": app,
"data": data,
}
def location():
"""Load default location."""
d = current_app.config['DATADIR']
with db.session.begin_nested():
Location.query.delete()
loc = Location(name='local', uri=d, default=True)
db.session.add(loc)
db.session.commit()
def locations(db):
"""File system locations."""
data_path = tempfile.mkdtemp()
archive_path = tempfile.mkdtemp()
location = Location(name='data', uri=data_path, default=True)
archive_location = Location(name='archive',
uri=archive_path,
default=False)
db.session.add(location)
db.session.add(archive_location)
db.session.commit()
yield {'data': data_path, 'archive': archive_path}
shutil.rmtree(data_path)
shutil.rmtree(archive_path)
def location(app):
"""Create default location."""
tmppath = tempfile.mkdtemp()
with db_.session.begin_nested():
Location.query.delete()
loc = Location(name='local', uri=tmppath, default=True)
db_.session.add(loc)
db_.session.commit()
def test_SIP_create(app, db, mocker):
"""Test the create method from SIP API."""
# we setup a file storage
tmppath = tempfile.mkdtemp()
db.session.add(Location(name='default', uri=tmppath, default=True))
db.session.commit()
# we create a file
content = b'test lol\n'
bucket = Bucket.create()
obj = ObjectVersion.create(bucket, 'test.txt', stream=BytesIO(content))
db.session.commit()
files = [obj]
# setup metadata
mjson = SIPMetadataType(title='JSON Test',
name='json-test',
format='json',
schema='url')
marcxml = SIPMetadataType(title='MARC XML Test',
name='marcxml-test',
format='xml',
schema='uri')
db.session.add(mjson)
db.session.add(marcxml)
metadata = {
'json-test': json.dumps({
'this': 'is',
'not': 'sparta'
}),
'marcxml-test': '<record></record>'
}
# Let's create a SIP
user = create_test_user('*****@*****.**')
agent = {'email': '*****@*****.**', 'ip_address': '1.1.1.1'}
sip = SIP.create(True,
files=files,
metadata=metadata,
user_id=user.id,
agent=agent)
db.session.commit()
assert SIP_.query.count() == 1
assert len(sip.files) == 1
assert len(sip.metadata) == 2
assert SIPFile.query.count() == 1
assert SIPMetadata.query.count() == 2
assert sip.user.id == user.id
assert sip.agent == agent
# we mock the user and the agent to test if the creation works
app.config['SIPSTORE_AGENT_JSONSCHEMA_ENABLED'] = False
mock_current_user = mocker.patch('invenio_sipstore.api.current_user')
type(mock_current_user).is_anonymous = mocker.PropertyMock(
return_value=True)
sip = SIP.create(True, files=files, metadata=metadata)
assert sip.model.user_id is None
assert sip.user is None
assert sip.agent == {}
# finalization
rmtree(tmppath)
def test_location(app, db):
"""Test location model."""
with db.session.begin_nested():
l1 = Location(name='test1', uri='file:///tmp', default=False)
l2 = Location(name='test2', uri='file:///tmp', default=True)
l3 = Location(name='test3', uri='file:///tmp', default=False)
db.session.add(l1)
db.session.add(l2)
db.session.add(l3)
assert Location.get_by_name('test1').name == 'test1'
assert Location.get_by_name('test2').name == 'test2'
assert Location.get_by_name('test3').name == 'test3'
assert Location.get_default().name == 'test2'
assert len(Location.all()) == 3
assert str(Location.get_by_name('test1')) == 'test1'
def test_bucket_create_object(app, db):
"""Test bucket creation."""
with db.session.begin_nested():
l1 = Location(name='test1', uri='file:///tmp/1', default=False)
l2 = Location(name='test2', uri='file:///tmp/2', default=True)
db.session.add(l1)
db.session.add(l2)
assert Location.query.count() == 2
# Simple create
with db.session.begin_nested():
b = Bucket.create()
assert b.id
assert b.default_location == Location.get_default().id
assert b.location == Location.get_default()
assert b.default_storage_class == \
app.config['FILES_REST_DEFAULT_STORAGE_CLASS']
assert b.size == 0
assert b.quota_size is None
assert b.max_file_size is None
assert b.deleted is False
# __repr__ test
assert str(b) == str(b.id)
# Retrieve one
assert Bucket.get(b.id).id == b.id
# Create with location_name and storage class
with db.session.begin_nested():
b = Bucket.create(location=l1, storage_class='A')
assert b.default_location == Location.get_by_name('test1').id
assert b.default_storage_class == 'A'
# Create using location name instead
b = Bucket.create(location=l2.name, storage_class='A')
assert b.default_location == Location.get_by_name('test2').id
# Retrieve one
assert Bucket.all().count() == 3
# Invalid storage class.
pytest.raises(ValueError, Bucket.create, storage_class='X')
def location(db, tmp_path):
"""File system location."""
loc = Location(
name='testloc',
uri=str(tmp_path),
default=True
)
db.session.add(loc)
db.session.commit()
return loc
def init_storage_path(name, uri, default=False):
try:
if uri.startswith("/") and not os.path.exists(uri):
os.makedirs(uri)
loc = Location(name=name, uri=uri, default=default)
db.session.add(loc)
db.session.commit()
return loc
except Exception:
db.session.rollback()
raise
def s3_location(db, s3_testpath):
"""File system location."""
loc = Location(
name='testloc',
uri=s3_testpath,
default=True
)
db.session.add(loc)
db.session.commit()
yield loc
def extra_location(db):
"""File system location."""
tmppath = tempfile.mkdtemp()
loc = Location(name='extra', uri=tmppath, default=False)
db.session.add(loc)
db.session.commit()
yield loc
shutil.rmtree(tmppath)
def dummy_location(db):
"""File system location."""
tmppath = tempfile.mkdtemp()
loc = Location(name='testloc', uri=tmppath, default=True)
db.session.add(loc)
db.session.commit()
yield loc
shutil.rmtree(tmppath)
def files_location(app, db):
try:
from invenio_files_rest.models import Location
loc = Location()
loc.name = 'test'
loc.uri = Path(tempfile.gettempdir()).as_uri()
loc.default = True
db.session.add(loc)
db.session.commit()
except ImportError:
pass
def testapp(base_app, database):
"""Application with just a database.
Pytest-Invenio also initialises ES with the app fixture.
"""
location_obj = Location(name="marctest-location",
uri=tempfile.mkdtemp(),
default=True)
database.session.add(location_obj)
database.session.commit()
yield base_app
def tmp_location(app):
"""File system location."""
with app.app_context():
tmppath = tempfile.mkdtemp()
loc = Location(name='extra', uri=tmppath, default=False)
db.session.add(loc)
db.session.commit()
yield loc
shutil.rmtree(tmppath)
def _create_files_location():
try:
uri = '/tmp/test-workflows'
if uri.startswith('/') and not os.path.exists(uri):
os.makedirs(uri)
loc = Location(name="test-workflows", uri=uri, default=True)
db.session.add(loc)
db.session.commit()
return loc
except Exception:
db.session.rollback()
raise
def location(app, db):
"""Define a location to write SIPs with factories."""
path = os.path.abspath('./tmp/')
try:
os.mkdir(path)
except:
pass
loc = Location(name='archive', uri=path, default=True)
db.session.add(loc)
db.session.commit()
app.config['SIPSTORE_ARCHIVER_LOCATION_NAME'] = 'archive'
yield loc
shutil.rmtree(path)
def init_default_storage_path():
"""Init default file store location."""
try:
uri = current_app.config['BASE_FILES_LOCATION']
if uri.startswith('/') and not os.path.exists(uri):
os.makedirs(uri)
loc = Location(name="default", uri=uri, default=True)
db.session.add(loc)
db.session.commit()
return loc
except Exception:
db.session.rollback()
raise
def loadlocation(force=False):
"""Load default file store location."""
try:
uri = current_app.config['FIXTURES_FILES_LOCATION']
if uri.startswith('/') and not exists(uri):
makedirs(uri)
loc = Location(name='default', uri=uri, default=True, )
db.session.add(loc)
db.session.commit()
return loc
except Exception:
db.session.rollback()
raise
def files():
"""Load files."""
srcroot = dirname(dirname(__file__))
d = current_app.config['DATADIR']
if exists(d):
shutil.rmtree(d)
makedirs(d)
# Clear data
Part.query.delete()
MultipartObject.query.delete()
ObjectVersion.query.delete()
Bucket.query.delete()
FileInstance.query.delete()
Location.query.delete()
db.session.commit()
# Create location
loc = Location(name='local', uri=d, default=True)
db.session.add(loc)
db.session.commit()
# Bucket 0
b1 = Bucket.create(loc)
b1.id = '00000000-0000-0000-0000-000000000000'
for f in ['README.rst', 'LICENSE']:
with open(join(srcroot, f), 'rb') as fp:
ObjectVersion.create(b1, f, stream=fp)
# Bucket 1
b2 = Bucket.create(loc)
b2.id = '11111111-1111-1111-1111-111111111111'
k = 'AUTHORS.rst'
with open(join(srcroot, 'CHANGES.rst'), 'rb') as fp:
ObjectVersion.create(b2, k, stream=fp)
with open(join(srcroot, 'AUTHORS.rst'), 'rb') as fp:
ObjectVersion.create(b2, k, stream=fp)
k = 'RELEASE-NOTES.rst'
with open(join(srcroot, 'RELEASE-NOTES.rst'), 'rb') as fp:
ObjectVersion.create(b2, k, stream=fp)
with open(join(srcroot, 'CHANGES.rst'), 'rb') as fp:
ObjectVersion.create(b2, k, stream=fp)
ObjectVersion.delete(b2.id, k)
# Bucket 2
b2 = Bucket.create(loc)
b2.id = '22222222-2222-2222-2222-222222222222'
db.session.commit()
