def test_location_default(app, db):
"""Test location model."""
with db.session.begin_nested():
l1 = Location(name='test1', uri='file:///tmp', default=False)
db.session.add(l1)
assert Location.get_default() is None
with db.session.begin_nested():
l2 = Location(name='test2', uri='file:///tmp', default=True)
l3 = Location(name='test3', uri='file:///tmp', default=True)
db.session.add(l2)
db.session.add(l3)
assert Location.get_default() is None
def init_default_location():
"""
Add default Location, if not already present.
Used by Travis as well.
"""
if not Location.query.filter(Location.name == 'default').count():
loc = Location()
loc.name = 'default'
loc.default = True
loc.uri = '/virtualenv/files/'
db.session.add(loc)
db.session.commit()
else:
error("Default location already exists.")
def create(cls, data, id_=None):
"""Create a deposit.
Adds bucket creation immediately on deposit creation.
"""
bucket = Bucket.create(
default_location=Location.get_default()
)
try:
schema = data.get("$schema", None) \
.split('/schemas/', 1)[1]
except (IndexError, AttributeError):
return None
if schema:
_deposit_group = \
next(
(depgroup
for dg, depgroup
in current_app.config.get('DEPOSIT_GROUPS').iteritems()
if schema in depgroup['schema']
),
None
)
data["_experiment"] = _deposit_group.get("experiment", "Unknown")
deposit = super(CAPDeposit, cls).create(data, id_=id_)
add_owner_permissions(deposit.id)
RecordsBuckets.create(record=deposit.model, bucket=bucket)
return deposit
def ping_db():
"""Load balancer ping view."""
try:
default_location = Location.get_default()
return 'OK'
except OperationalError:
return 'ERROR'
def ping_files():
"""Load balancer ping view."""
try:
default_location = Location.get_default().uri
test_file_path = join(default_location, 'test.txt')
f = open(test_file_path, 'r')
return "OK"
except (OperationalError, IOError) as e:
return "ERROR"
def test_bucket_create_object(app, db):
"""Test bucket creation."""
with db.session.begin_nested():
l1 = Location(name='test1', uri='file:///tmp/1', default=False)
l2 = Location(name='test2', uri='file:///tmp/2', default=True)
db.session.add(l1)
db.session.add(l2)
assert Location.query.count() == 2
# Simple create
with db.session.begin_nested():
b = Bucket.create()
assert b.id
assert b.default_location == Location.get_default().id
assert b.location == Location.get_default()
assert b.default_storage_class == \
app.config['FILES_REST_DEFAULT_STORAGE_CLASS']
assert b.size == 0
assert b.quota_size is None
assert b.max_file_size is None
assert b.deleted is False
# __repr__ test
assert str(b) == str(b.id)
# Retrieve one
assert Bucket.get(b.id).id == b.id
# Create with location_name and storage class
with db.session.begin_nested():
b = Bucket.create(location=l1, storage_class='A')
assert b.default_location == Location.get_by_name('test1').id
assert b.default_storage_class == 'A'
# Create using location name instead
b = Bucket.create(location=l2.name, storage_class='A')
assert b.default_location == Location.get_by_name('test2').id
# Retrieve one
assert Bucket.all().count() == 3
# Invalid storage class.
pytest.raises(ValueError, Bucket.create, storage_class='X')
def files():
"""Load files."""
srcroot = dirname(dirname(__file__))
d = current_app.config['DATADIR']
if exists(d):
shutil.rmtree(d)
makedirs(d)
# Clear data
Part.query.delete()
MultipartObject.query.delete()
ObjectVersion.query.delete()
Bucket.query.delete()
FileInstance.query.delete()
Location.query.delete()
db.session.commit()
# Create location
loc = Location(name='local', uri=d, default=True)
db.session.add(loc)
db.session.commit()
# Bucket 0
b1 = Bucket.create(loc)
b1.id = '00000000-0000-0000-0000-000000000000'
for f in ['README.rst', 'LICENSE']:
with open(join(srcroot, f), 'rb') as fp:
ObjectVersion.create(b1, f, stream=fp)
# Bucket 1
b2 = Bucket.create(loc)
b2.id = '11111111-1111-1111-1111-111111111111'
k = 'AUTHORS.rst'
with open(join(srcroot, 'CHANGES.rst'), 'rb') as fp:
ObjectVersion.create(b2, k, stream=fp)
with open(join(srcroot, 'AUTHORS.rst'), 'rb') as fp:
ObjectVersion.create(b2, k, stream=fp)
k = 'RELEASE-NOTES.rst'
with open(join(srcroot, 'RELEASE-NOTES.rst'), 'rb') as fp:
ObjectVersion.create(b2, k, stream=fp)
with open(join(srcroot, 'CHANGES.rst'), 'rb') as fp:
ObjectVersion.create(b2, k, stream=fp)
ObjectVersion.delete(b2.id, k)
# Bucket 2
b2 = Bucket.create(loc)
b2.id = '22222222-2222-2222-2222-222222222222'
db.session.commit()
def testapp(base_app, database):
"""Application with just a database.
Pytest-Invenio also initialises ES with the app fixture.
"""
location_obj = Location(
name="marctest-location", uri=tempfile.mkdtemp(), default=True
)
database.session.add(location_obj)
database.session.commit()
InvenioRecords(base_app)
InvenioJSONSchemas(base_app)
yield base_app
def extra_location(db):
"""File system location."""
tmppath = tempfile.mkdtemp()
loc = Location(
name='extra',
uri=tmppath,
default=False
)
db.session.add(loc)
db.session.commit()
yield loc
shutil.rmtree(tmppath)
def create_bucket(bucket_id=None):
with db.session.begin_nested():
loc = Location.get_default()
if not loc:
loc = Location(name='local',
uri=GEO_KNOWLEDGE_HUB_DEFAULT_BUCKET_URL,
default=True)
db.session.add(loc)
bucket = None
if bucket_id:
bucket = db.session.query(Bucket).filter(
Bucket.id == bucket_id).first()
if not bucket:
bucket = Bucket.create(quota_size=100 * 1000 * 1000,
max_file_size=100 * 1000 * 1000,
locked=False)
db.session.commit()
return jsonify({"bucket_id": bucket.id})
def location(db):
"""File system location."""
tmppath = tempfile.mkdtemp()
loc = Location(
name='testloc',
uri=tmppath,
default=True
)
db.session.add(loc)
db.session.commit()
yield loc
shutil.rmtree(tmppath)
def tmp_location(app):
"""File system location."""
with app.app_context():
tmppath = tempfile.mkdtemp()
loc = Location(
name='extra',
uri=tmppath,
default=False
)
db.session.add(loc)
db.session.commit()
yield loc
shutil.rmtree(tmppath)
def init_workflows_storage_path(default=False):
"""Init workflows file store location."""
try:
uri = current_app.config['WORKFLOWS_FILE_LOCATION']
if uri.startswith('/') and not os.path.exists(uri):
os.makedirs(uri)
loc = Location(
name=current_app.config["WORKFLOWS_DEFAULT_FILE_LOCATION_NAME"],
uri=uri,
default=False)
db.session.add(loc)
db.session.commit()
return loc
except Exception:
db.session.rollback()
raise
def load_data(verbose):
"""Load demonstration data."""
# add files location
files_path = os.path.join(current_app.instance_path, 'files')
if os.path.exists(files_path):
rmtree(files_path)
os.mkdir(files_path)
with db.session.begin_nested():
db.session.add(
Location(name='local',
uri=pathlib.Path(files_path).as_uri(),
default=True))
# load the demo
load_demo_data(os.path.join(
os.path.dirname(os.path.realpath(__file__)), 'data'),
verbose=verbose)
db.session.commit()
def init_records_files_storage_path(default=False):
"""Init records file store location."""
try:
uri = os.path.join(current_app.config['BASE_FILES_LOCATION'],
"records", "files")
if uri.startswith('/') and not os.path.exists(uri):
os.makedirs(uri)
loc = Location(
name=current_app.config["RECORDS_DEFAULT_FILE_LOCATION_NAME"],
uri=uri,
default=False)
db.session.add(loc)
db.session.commit()
return loc
except Exception:
db.session.rollback()
raise
def initialize_communities_bucket():
"""Initialize the communities file bucket.
:raises: `invenio_files_rest.errors.FilesException`
"""
bucket_id = UUID(current_app.config['COMMUNITIES_BUCKET_UUID'])
if Bucket.query.get(bucket_id):
raise FilesException("Bucket with UUID {} already exists.".format(
bucket_id))
else:
storage_class = current_app.config['FILES_REST_DEFAULT_STORAGE_CLASS']
location = Location.get_default()
bucket = Bucket(id=bucket_id,
location=location,
default_storage_class=storage_class)
db.session.add(bucket)
db.session.commit()
def cli_location(db):
"""Fixture for invenio file-location.
Adapted to work with `<Flask-obj>.test_cli_runner`.
"""
from invenio_files_rest.models import Location
uri = tempfile.mkdtemp()
location_obj = Location(name="pytest-location", uri=uri, default=True)
db.session.add(location_obj)
db.session.commit()
yield location_obj
# can't use location_obj.uri here, as test_cli_runner expunges
# database-commits on teardown
shutil.rmtree(uri)
def initialize_communities_bucket():
"""Initialize the communities file bucket.
:raises: `invenio_files_rest.errors.FilesException`
"""
bucket_id = UUID(current_app.config['COMMUNITIES_BUCKET_UUID'])
if Bucket.query.get(bucket_id):
raise FilesException("Bucket with UUID {} already exists.".format(
bucket_id))
else:
storage_class = current_app.config['FILES_REST_DEFAULT_STORAGE_CLASS']
location = Location.get_default()
bucket = Bucket(id=bucket_id,
location=location,
default_storage_class=storage_class)
db.session.add(bucket)
db.session.commit()
def files():
"""Load files."""
srcroot = dirname(dirname(__file__))
d = current_app.config['DATADIR']
if exists(d):
shutil.rmtree(d)
makedirs(d)
# Clear data
ObjectVersion.query.delete()
Bucket.query.delete()
FileInstance.query.delete()
Location.query.delete()
db.session.commit()
# Create location
loc = Location(name='local', uri=d, default=True)
db.session.commit()
def location(db):
"""Creates a simple default location for a test.
Scope: function
Use this fixture if your test requires a `files location <https://invenio-
files-rest.readthedocs.io/en/latest/api.html#invenio_files_rest.models.
Location>`_. The location will be a default location with the name
``pytest-location``.
"""
from invenio_files_rest.models import Location
uri = tempfile.mkdtemp()
location_obj = Location(name="pytest-location", uri=uri, default=True)
db.session.add(location_obj)
db.session.commit()
yield location_obj
shutil.rmtree(location_obj.uri)
def create_bucket_from_dir(source_dir, location_obj=None):
"""Create bucket from the specified source directory.
:param source_dir: The directory to create the bucket from.
:param location_obj: Optional location object to use. If None
is specified, get the current default location.
:returns: The new bucket object.
"""
if not location_obj:
from invenio_files_rest.models import Bucket, Location, ObjectVersion
location_obj = Location.get_default() or location
bucket_obj = Bucket.create(location_obj)
for file_name in os.listdir(source_dir):
full_file_path = os.path.join(source_dir, file_name)
if os.path.isdir(full_file_path):
continue
file_obj = open(full_file_path, 'rb')
ObjectVersion.create(bucket_obj, key=file_name, stream=file_obj)
db.session.commit()
return bucket_obj
def get_bucket(self, location=None, storage_class=None, record_id=None):
"""Allows to retrieve bucket for any record(default: self)
Args:
location (str): Bucket location
(default: 'RECORDS_DEFAULT_FILE_LOCATION_NAME') from config
storage_class (str): Bucket storage class
(default: 'RECORDS_DEFAULT_STORAGE_CLASS') from config
record_id (int): record to which bucket is asigned
(default: self)
Returns:
Bucket: if found in db for selected location, storage_class and record_id or
None if there were no bucket.
Raises:
NoResultFound: When provided location was not found.
"""
if not storage_class:
storage_class = current_app.config["RECORDS_DEFAULT_STORAGE_CLASS"]
if not location:
location = current_app.config["RECORDS_DEFAULT_FILE_LOCATION_NAME"]
if not record_id:
record_id = self.id
try:
location_obj = Location.get_by_name(location)
except NoResultFound:
raise NoResultFound(
"Cannot find location %s. Please check if system is configured properly!",
location,
)
bucket = (RecordsBuckets.query.join(Bucket).filter(
RecordsBuckets.record_id == record_id,
Bucket.default_storage_class == storage_class,
Bucket.default_location == location_obj.id,
).one_or_none())
if bucket:
LOGGER.debug("Bucket found", key=bucket.bucket.id, uuid=self.id)
return bucket.bucket
LOGGER.info("Bucket not found", uuid=self.id)
return self._create_bucket(location, storage_class)
def fixtures():
"""Command for working with test data."""
temp_path = os.path.join(os.path.dirname(__file__), 'instance/temp')
demo_files_path = os.path.join(os.path.dirname(__file__), 'demo_files')
# Create location
loc = Location(name='local', uri=temp_path, default=True)
db.session.add(loc)
db.session.commit()
# Example files from the data folder
demo_files = (
'markdown.md',
'csvfile.csv',
'zipfile.zip',
'jsonfile.json',
'xmlfile.xml',
'notebook.ipynb',
'pdffile.pdf',
'jpgfile.jpg',
'pngfile.png',
'pdffile.pdf',
)
rec_uuid = uuid4()
provider = RecordIdProvider.create(object_type='rec', object_uuid=rec_uuid)
data = {
'pid_value': provider.pid.pid_value,
}
record = Record.create(data, id_=rec_uuid)
bucket = Bucket.create()
RecordsBuckets.create(record=record.model, bucket=bucket)
# Add files to the record
for f in demo_files:
with open(os.path.join(demo_files_path, f), 'rb') as fp:
record.files[f] = fp
record.files.flush()
record.commit()
db.session.commit()
def test_communities_load(db, communities_dump, featured_dump, logos_dir):
"""Load the communities JSON dump."""
# Create a single user (with id=1) to meet the community's FK constraint.
create_test_user('*****@*****.**', password='******')
# Initialize communities bucket (requires a default Location)
loc = Location(name='local', uri='file:///tmp', default=True)
db.session.add(loc)
db.session.commit()
initialize_communities_bucket()
# Load the communities
for data in communities_dump:
load_community.delay(data, logos_dir)
# Check if community was loaded correctly
assert Community.query.count() == 1
c = Community.query.first()
assert c.title == 'Invenio Community'
assert c.id == 'invenio'
# Check object metadata
assert ObjectVersion.query.count() == 1 # Check if the logo was created
assert ObjectVersion.query.first().key == 'invenio/logo.jpg'
# Open the original logo from test data and check if checksums match
with open(join(logos_dir, 'invenio.jpg'), 'rb') as fp:
logo_checksum = hashlib.md5(fp.read()).hexdigest()
assert ObjectVersion.query.first().file.checksum == \
'md5:{0}'.format(logo_checksum)
# Load the community featurings
for data in featured_dump:
load_featured.delay(data)
# Make sure they are loaded correctly
assert FeaturedCommunity.query.count() == 1
fc = FeaturedCommunity.query.first()
assert fc.community.id == c.id
def files():
"""Load files."""
bucket_path = os.path.join(os.path.dirname(__file__), 'bucket')
if not os.path.isdir(bucket_path):
os.makedirs(bucket_path)
# Create location
loc = Location(name='local', uri=bucket_path, default=True)
db.session.add(loc)
db.session.commit()
# Bucket
bucket = Bucket.create(location=loc)
# Example files from the data folder
with open('data/metadata.json') as file:
example_records = json.load(file)['metadata']
# Create records
for record in example_records:
create_object(bucket, record)
def _process_files(record, files_metadata):
"""Attach files to a record with a given metadata.
Assumptions:
- The source must be a URL pointing to a tar file.
- All files listed in the metadata are inside the source tar.
- Master files are listed before slaves.
- The reference from the slave to master is done via key.
"""
if not files_metadata:
return
bucket = Bucket.create(location=Location.get_by_name('videos'))
RecordsBuckets.create(record=record.model, bucket=bucket)
response = requests.get(
files_metadata['source'], stream=True, verify=False)
# Throw an error for bad status codes
response.raise_for_status()
with tempfile.NamedTemporaryFile(suffix='.tar', delete=False) as f:
for chunk in response:
f.write(chunk)
tar = tarfile.open(name=f.name)
tar.extractall(path=tempfile.gettempdir())
files_base_dir = os.path.join(tempfile.gettempdir(), tar.getnames()[0])
tar.close()
os.remove(f.name)
for f in files_metadata['metadata']:
obj = ObjectVersion.create(bucket, f['key'])
with open(os.path.join(files_base_dir, f['key']), 'rb') as fp:
obj.set_contents(fp)
for k, v in f['tags'].items():
if k == 'master':
v = ObjectVersion.get(bucket, v).version_id
ObjectVersionTag.create(obj, k, v)
shutil.rmtree(files_base_dir)
record['_files'] = record.files.dumps()
def add_location(name, uri, default):
"""Add a file storage location.
The URI should point to the location where the files will be stored. The
NAME will be used to reference this location.
"""
matching_locations = Location.query.filter(
or_(Location.uri == uri, Location.name == name)).all()
if len(matching_locations) > 0:
if matching_locations[0].name == name:
raise click.BadParameter(
'Another location with the same name already exists.')
else:
raise click.BadParameter(
'Existing location "{}" has the same uri.'.format(
matching_locations[0].name))
if default:
db.session.query(Location).filter(Location.default == True).update(
{Location.default: False})
location = Location(name=name, uri=uri, default=default)
db.session.add(location)
db.session.commit()
def test_location(app, db):
"""Test location model."""
with db.session.begin_nested():
l1 = Location(name='test1', uri='file:///tmp', default=False)
l2 = Location(name='test2', uri='file:///tmp', default=True)
l3 = Location(name='test3', uri='file:///tmp', default=False)
db.session.add(l1)
db.session.add(l2)
db.session.add(l3)
assert Location.get_by_name('test1').name == 'test1'
assert Location.get_by_name('test2').name == 'test2'
assert Location.get_by_name('test3').name == 'test3'
assert Location.get_default().name == 'test2'
assert len(Location.all()) == 3
assert str(Location.get_by_name('test1')) == 'test1'
def test_bucket_create_object(app, db):
"""Test bucket creation."""
with db.session.begin_nested():
l1 = Location(name='test1', uri='file:///tmp/1', default=False)
l2 = Location(name='test2', uri='file:///tmp/2', default=True)
db.session.add(l1)
db.session.add(l2)
assert Location.query.count() == 2
# Simple create
with db.session.begin_nested():
b = Bucket.create()
assert b.id
assert b.default_location == Location.get_default().id
assert b.location == Location.get_default()
assert b.default_storage_class == \
app.config['FILES_REST_DEFAULT_STORAGE_CLASS']
assert b.size == 0
assert b.quota_size is None
assert b.max_file_size is None
assert b.deleted is False
# __repr__ test
assert str(b) == str(b.id)
# Retrieve one
assert Bucket.get(b.id).id == b.id
# Create with location_name and storage class
with db.session.begin_nested():
b = Bucket.create(location=l1, storage_class='A')
assert b.default_location == Location.get_by_name('test1').id
assert b.default_storage_class == 'A'
# Create using location name instead
b = Bucket.create(location=l2.name, storage_class='A')
assert b.default_location == Location.get_by_name('test2').id
# Retrieve one
assert Bucket.all().count() == 3
# Invalid storage class.
pytest.raises(ValueError, Bucket.create, storage_class='X')
def create(cls, data, id_=None, **kwargs):
"""Create a CDS deposit.
Adds bucket creation immediately on deposit creation.
"""
if '_deposit' not in data:
id_ = id_ or uuid.uuid4()
cls.deposit_minter(id_, data)
bucket = Bucket.create(location=Location.get_by_name(
kwargs.get('bucket_location', 'default')))
data['_buckets'] = {'deposit': str(bucket.id)}
data.setdefault('_cds', {})
data['_cds'].setdefault('state', {})
data.setdefault('keywords', [])
data.setdefault('license', [{
'license': 'CERN',
'material': '',
'url': 'http://copyright.web.cern.ch',
}])
data.setdefault('_access', {})
deposit = super(CDSDeposit, cls).create(
data, id_=id_, validator=PartialDraft4Validator)
RecordsBuckets.create(record=deposit.model, bucket=bucket)
return deposit
def _resolve_bucket(cls, deposit, record):
"""Build bucket."""
bucket = Bucket.create(location=Location.get_by_name('videos'))
deposit['_buckets'] = {'deposit': str(bucket.id)}
RecordsBuckets.create(record=deposit.model, bucket=bucket)
record['_buckets'] = deepcopy(deposit['_buckets'])
def add_record(metadata, collection, schema, force, files=[]):
"""Add record."""
collection = Collection.query.filter(Collection.name == collection).first()
if collection is None:
return
data, pid, recid = construct_record(collection, metadata, 1,
{} if force else schema)
d = current_app.config['DATADIR']
buckets = []
data['_files'] = []
for file in files:
bucket = Bucket.create(default_location=Location.get_default())
buckets.append(bucket)
with open(
pkg_resources.resource_filename(
'cap.modules.fixtures',
os.path.join('data', 'files', file)), 'rb') as fp:
obj = ObjectVersion.create(bucket, file, stream=fp)
data['_files'].append({
'bucket': str(obj.bucket_id),
'key': obj.key,
'size': obj.file.size,
'checksum': str(obj.file.checksum),
'version_id': str(obj.version_id),
})
try:
record = Record.create(data, id_=recid)
for bucket in buckets:
rb = RecordsBuckets(record_id=record.id, bucket_id=bucket.id)
db.session.add(rb)
# Invenio-Indexer is delegating the document inferring to
# Invenio-Search which is analysing the string splitting by `/` and
# using `.json` to be sure that it cans understand the mapping.
record['$schema'] = 'mappings/{0}.json'.format(collection.name.lower())
indexer = RecordIndexer()
indexer.index(record)
# Creating permission needs for the record
action_edit_record = RecordUpdateActionNeed(str(recid))
action_read_record = RecordReadActionNeed(str(recid))
action_index_record = RecordIndexActionNeed(str(recid))
# Giving index, read, write permissions to user/creator
db.session.add(ActionUsers.allow(action_edit_record))
db.session.add(ActionUsers.allow(action_read_record))
db.session.add(ActionUsers.allow(action_index_record))
db.session.commit()
print("DONE!!!")
except ValidationError as error:
print("============================")
pprint(error.message)
pprint(error.path)
print("============================")
db.session.rollback()
def files(temp, source):
"""Demo files for testing.
.. note::
This files are *only* for testing.
"""
click.echo('Loading files it may take several minutes.')
if not source:
source = pkg_resources.resource_filename('cds.modules.fixtures',
'data/files.tar.gz')
files = _handle_source(source, temp)
d = current_app.config['DATADIR']
if not exists(d):
makedirs(d)
# Clear data
ObjectVersion.query.delete()
Bucket.query.delete()
FileInstance.query.delete()
Location.query.delete()
db.session.commit()
# Create location
loc = Location(name='local', uri=d, default=True)
db.session.commit()
# Record indexer
indexer = RecordIndexer()
for f in files:
with open(join(source, f), 'rb') as fp:
# Create bucket
bucket = Bucket.create(loc)
# The filename
file_name = basename(f)
# Create object version
ObjectVersion.create(bucket, file_name, stream=fp)
# Attach to dummy records
rec_uuid = uuid.uuid4()
record = {
'_access': {
'read': ['*****@*****.**', 'it-dep']
},
'dummy':
True,
'files': [{
'uri':
'/api/files/{0}/{1}'.format(str(bucket.id), file_name),
'filename':
file_name,
'bucket':
str(bucket.id),
'local':
True
}]
}
# Create PID
current_pidstore.minters['recid'](rec_uuid, record)
# Create record
record = FileRecord.create(record, id_=rec_uuid)
# Index record
indexer.index(record)
# Create records' bucket
RecordsBuckets.create(record=record.model, bucket=bucket)
db.session.commit()
click.echo('DONE :)')
from invenio_files_rest.models import Bucket, Location
from invenio_pidstore.models import PersistentIdentifier, PIDStatus
from invenio_records_files.api import Record, RecordsBuckets
from invenio_sipstore.api import RecordSIP
from invenio_sipstore.models import SIPMetadataType
from six import BytesIO
from invenio_archivematica.tasks import oais_start_transfer
# init
try:
makedirs('/archive/')
except:
pass
locrecords = Location(name='records',
uri='/eos/workspace/o/oais/archivematica-test/records/',
default=True)
locarchive = Location(
name='archive', # this should go in SIPSTORE_ARCHIVER_LOCATION_NAME
uri='/eos/workspace/o/oais/archivematica-test/transfer/')
db.session.add(locrecords)
db.session.add(locarchive)
db.session.commit()
# first we create a metadata type with a schema used by the following record
mtype = SIPMetadataType(
title='Invenio JSON test',
name='invenio-json-test',
format='json',
schema='https://zenodo.org/schemas/deposits/records/record-v1.0.0.json')
db.session.add(mtype)
def bucket_location(app, db):
"""Create a default location for managing files."""
tmppath = tempfile.mkdtemp()
db.session.add(Location(name='default', uri=tmppath, default=True))
db.session.commit()
def location(db, tmp_path):
"""File system location."""
loc = Location(name='testloc', uri=str(tmp_path), default=True)
db.session.add(loc)
db.session.commit()
return loc
def location(location_path, database):
"""File system locations."""
loc = Location(name='testloc', uri=location_path, default=True)
database.session.add(loc)
database.session.commit()
return loc
def add_record(metadata, collection, schema, force, files=[]):
"""Add record."""
collection = Collection.query.filter(
Collection.name == collection).first()
if collection is None:
return
data, pid, recid = construct_record(
collection, metadata, 1, {} if force else schema)
d = current_app.config['DATADIR']
buckets = []
data['_files'] = []
for file in files:
bucket = Bucket.create(default_location=Location.get_default())
buckets.append(bucket)
with open(pkg_resources.resource_filename(
'cap.modules.fixtures', os.path.join('data', 'files', file)
), 'rb') as fp:
obj = ObjectVersion.create(bucket, file, stream=fp)
data['_files'].append({
'bucket': str(obj.bucket_id),
'key': obj.key,
'size': obj.file.size,
'checksum': str(obj.file.checksum),
'version_id': str(obj.version_id),
})
try:
record = Record.create(data, id_=recid)
for bucket in buckets:
rb = RecordsBuckets(record_id=record.id, bucket_id=bucket.id)
db.session.add(rb)
# Invenio-Indexer is delegating the document inferring to
# Invenio-Search which is analysing the string splitting by `/` and
# using `.json` to be sure that it cans understand the mapping.
record['$schema'] = 'mappings/{0}.json'.format(collection.name.lower())
indexer = RecordIndexer()
indexer.index(record)
# Creating permission needs for the record
action_edit_record = RecordUpdateActionNeed(str(recid))
action_read_record = RecordReadActionNeed(str(recid))
action_index_record = RecordIndexActionNeed(str(recid))
# Giving index, read, write permissions to user/creator
db.session.add(ActionUsers.allow(action_edit_record))
db.session.add(ActionUsers.allow(action_read_record))
db.session.add(ActionUsers.allow(action_index_record))
db.session.commit()
print("DONE!!!")
except ValidationError as error:
print("============================")
pprint(error.message)
pprint(error.path)
print("============================")
db.session.rollback()
def test_RecordSIP_create(db, mocker):
"""Test create method from the API class RecordSIP."""
# we setup a file storage
tmppath = tempfile.mkdtemp()
db.session.add(Location(name='default', uri=tmppath, default=True))
# setup metadata
mtype = SIPMetadataType(title='JSON Test',
name='json-test',
format='json',
schema='url://to/schema')
db.session.add(mtype)
db.session.commit()
# first we create a record
recid = uuid.uuid4()
pid = PersistentIdentifier.create('recid',
'1337',
object_type='rec',
object_uuid=recid,
status=PIDStatus.REGISTERED)
mocker.patch('invenio_records.api.RecordBase.validate',
return_value=True,
autospec=True)
record = Record.create(
{
'title': 'record test',
'$schema': 'url://to/schema'
}, recid)
# we add a file to the record
bucket = Bucket.create()
content = b'Test file\n'
RecordsBuckets.create(record=record.model, bucket=bucket)
record.files['test.txt'] = BytesIO(content)
db.session.commit()
# Let's create a SIP
user = create_test_user('*****@*****.**')
agent = {'email': '*****@*****.**', 'ip_address': '1.1.1.1'}
rsip = RecordSIP.create(pid, record, True, user_id=user.id, agent=agent)
db.session.commit()
# test!
assert RecordSIP_.query.count() == 1
assert SIP_.query.count() == 1
assert SIPFile.query.count() == 1
assert SIPMetadata.query.count() == 1
assert len(rsip.sip.files) == 1
assert len(rsip.sip.metadata) == 1
metadata = rsip.sip.metadata[0]
assert metadata.type.format == 'json'
assert '"title": "record test"' in metadata.content
assert rsip.sip.archivable is True
# we try with no files
rsip = RecordSIP.create(pid,
record,
True,
create_sip_files=False,
user_id=user.id,
agent=agent)
assert SIPFile.query.count() == 1
assert SIPMetadata.query.count() == 2
assert len(rsip.sip.files) == 0
assert len(rsip.sip.metadata) == 1
# try with specific SIP metadata type
mtype = SIPMetadataType(title='JSON Test 2',
name='json-test-2',
format='json',
schema=None) # no schema
db.session.add(mtype)
db.session.commit()
rsip = RecordSIP.create(pid,
record,
True,
create_sip_files=False,
user_id=user.id,
agent=agent,
sip_metadata_type='json-test-2')
assert SIPMetadata.query.count() == 3
assert len(rsip.sip.metadata) == 1
assert rsip.sip.metadata[0].type.id == mtype.id
# finalization
rmtree(tmppath)
