def create(cls, data, id_=None, **kwargs):
"""Create a CDS deposit.
Adds bucket creation immediately on deposit creation.
"""
if '_deposit' not in data:
id_ = id_ or uuid.uuid4()
cls.deposit_minter(id_, data)
bucket = Bucket.create(location=Location.get_by_name(
kwargs.get('bucket_location', 'default')))
data['_buckets'] = {'deposit': str(bucket.id)}
data.setdefault('_cds', {})
data['_cds'].setdefault('state', {})
data.setdefault('keywords', [])
data.setdefault('license', [{
'license': 'CERN',
'material': '',
'url': 'http://copyright.web.cern.ch',
}])
if '_access' not in data:
data.setdefault('_access', {})
deposit = super(CDSDeposit,
cls).create(data,
id_=id_,
validator=PartialDraft4Validator)
RecordsBuckets.create(record=deposit.model, bucket=bucket)
return deposit
def _resolve_bucket(cls, deposit, record):
"""Build bucket."""
logging.debug('Creating new buckets, record and deposit.')
bucket = Bucket.create(location=Location.get_by_name('videos'))
deposit['_buckets'] = {'deposit': str(bucket.id)}
RecordsBuckets.create(record=deposit.model, bucket=bucket)
record['_buckets'] = deepcopy(deposit['_buckets'])
db.session.commit()
def test_location(app, db):
"""Test location model."""
with db.session.begin_nested():
l1 = Location(name='test1', uri='file:///tmp', default=False)
l2 = Location(name='test2', uri='file:///tmp', default=True)
l3 = Location(name='test3', uri='file:///tmp', default=False)
db.session.add(l1)
db.session.add(l2)
db.session.add(l3)
assert Location.get_by_name('test1').name == 'test1'
assert Location.get_by_name('test2').name == 'test2'
assert Location.get_by_name('test3').name == 'test3'
assert Location.get_default().name == 'test2'
assert len(Location.all()) == 3
assert str(Location.get_by_name('test1')) == 'test1'
def test_location(app, db):
"""Test location model."""
with db.session.begin_nested():
l1 = Location(name='test1', uri='file:///tmp', default=False)
l2 = Location(name='test2', uri='file:///tmp', default=True)
l3 = Location(name='test3', uri='file:///tmp', default=False)
db.session.add(l1)
db.session.add(l2)
db.session.add(l3)
assert Location.get_by_name('test1').name == 'test1'
assert Location.get_by_name('test2').name == 'test2'
assert Location.get_by_name('test3').name == 'test3'
assert Location.get_default().name == 'test2'
assert len(Location.all()) == 3
assert str(Location.get_by_name('test1')) == 'test1'
def test_bucket_create_object(app, db):
"""Test bucket creation."""
with db.session.begin_nested():
l1 = Location(name='test1', uri='file:///tmp/1', default=False)
l2 = Location(name='test2', uri='file:///tmp/2', default=True)
db.session.add(l1)
db.session.add(l2)
assert Location.query.count() == 2
# Simple create
with db.session.begin_nested():
b = Bucket.create()
assert b.id
assert b.default_location == Location.get_default().id
assert b.location == Location.get_default()
assert b.default_storage_class == \
app.config['FILES_REST_DEFAULT_STORAGE_CLASS']
assert b.size == 0
assert b.quota_size is None
assert b.max_file_size is None
assert b.deleted is False
# __repr__ test
assert str(b) == str(b.id)
# Retrieve one
assert Bucket.get(b.id).id == b.id
# Create with location_name and storage class
with db.session.begin_nested():
b = Bucket.create(location=l1, storage_class='A')
assert b.default_location == Location.get_by_name('test1').id
assert b.default_storage_class == 'A'
# Create using location name instead
b = Bucket.create(location=l2.name, storage_class='A')
assert b.default_location == Location.get_by_name('test2').id
# Retrieve one
assert Bucket.all().count() == 3
# Invalid storage class.
pytest.raises(ValueError, Bucket.create, storage_class='X')
def test_bucket_create_object(app, db):
"""Test bucket creation."""
with db.session.begin_nested():
l1 = Location(name='test1', uri='file:///tmp/1', default=False)
l2 = Location(name='test2', uri='file:///tmp/2', default=True)
db.session.add(l1)
db.session.add(l2)
assert Location.query.count() == 2
# Simple create
with db.session.begin_nested():
b = Bucket.create()
assert b.id
assert b.default_location == Location.get_default().id
assert b.location == Location.get_default()
assert b.default_storage_class == \
app.config['FILES_REST_DEFAULT_STORAGE_CLASS']
assert b.size == 0
assert b.quota_size is None
assert b.max_file_size is None
assert b.deleted is False
# __repr__ test
assert str(b) == str(b.id)
# Retrieve one
assert Bucket.get(b.id).id == b.id
# Create with location_name and storage class
with db.session.begin_nested():
b = Bucket.create(location=l1, storage_class='A')
assert b.default_location == Location.get_by_name('test1').id
assert b.default_storage_class == 'A'
# Create using location name instead
b = Bucket.create(location=l2.name, storage_class='A')
assert b.default_location == Location.get_by_name('test2').id
# Retrieve one
assert Bucket.all().count() == 3
# Invalid storage class.
pytest.raises(ValueError, Bucket.create, storage_class='X')
def get_bucket(self, location=None, storage_class=None, record_id=None):
"""Allows to retrieve bucket for any record(default: self)
Args:
location (str): Bucket location
(default: 'RECORDS_DEFAULT_FILE_LOCATION_NAME') from config
storage_class (str): Bucket storage class
(default: 'RECORDS_DEFAULT_STORAGE_CLASS') from config
record_id (int): record to which bucket is asigned
(default: self)
Returns:
Bucket: if found in db for selected location, storage_class and record_id or
None if there were no bucket.
Raises:
NoResultFound: When provided location was not found.
"""
if not storage_class:
storage_class = current_app.config["RECORDS_DEFAULT_STORAGE_CLASS"]
if not location:
location = current_app.config["RECORDS_DEFAULT_FILE_LOCATION_NAME"]
if not record_id:
record_id = self.id
try:
location_obj = Location.get_by_name(location)
except NoResultFound:
raise NoResultFound(
"Cannot find location %s. Please check if system is configured properly!",
location,
)
bucket = (RecordsBuckets.query.join(Bucket).filter(
RecordsBuckets.record_id == record_id,
Bucket.default_storage_class == storage_class,
Bucket.default_location == location_obj.id,
).one_or_none())
if bucket:
LOGGER.debug("Bucket found", key=bucket.bucket.id, uuid=self.id)
return bucket.bucket
LOGGER.info("Bucket not found", uuid=self.id)
return self._create_bucket(location, storage_class)
def _process_files(record, files_metadata):
"""Attach files to a record with a given metadata.
Assumptions:
- The source must be a URL pointing to a tar file.
- All files listed in the metadata are inside the source tar.
- Master files are listed before slaves.
- The reference from the slave to master is done via key.
"""
if not files_metadata:
return
bucket = Bucket.create(location=Location.get_by_name('videos'))
RecordsBuckets.create(record=record.model, bucket=bucket)
response = requests.get(
files_metadata['source'], stream=True, verify=False)
# Throw an error for bad status codes
response.raise_for_status()
with tempfile.NamedTemporaryFile(suffix='.tar', delete=False) as f:
for chunk in response:
f.write(chunk)
tar = tarfile.open(name=f.name)
tar.extractall(path=tempfile.gettempdir())
files_base_dir = os.path.join(tempfile.gettempdir(), tar.getnames()[0])
tar.close()
os.remove(f.name)
for f in files_metadata['metadata']:
obj = ObjectVersion.create(bucket, f['key'])
with open(os.path.join(files_base_dir, f['key']), 'rb') as fp:
obj.set_contents(fp)
for k, v in f['tags'].items():
if k == 'master':
v = ObjectVersion.get(bucket, v).version_id
ObjectVersionTag.create(obj, k, v)
shutil.rmtree(files_base_dir)
record['_files'] = record.files.dumps()
def _resolve_bucket(cls, deposit, record):
"""Build bucket."""
bucket = Bucket.create(location=Location.get_by_name('videos'))
deposit['_buckets'] = {'deposit': str(bucket.id)}
RecordsBuckets.create(record=deposit.model, bucket=bucket)
record['_buckets'] = deepcopy(deposit['_buckets'])