def test_object_version_tags(app, db, dummy_location):
"""Test object version tags."""
f = FileInstance(uri="f1", size=1, checksum="mychecksum")
db.session.add(f)
db.session.commit()
b = Bucket.create()
obj1 = ObjectVersion.create(b, "test").set_file(f)
ObjectVersionTag.create(obj1, "mykey", "testvalue")
ObjectVersionTag.create(obj1, "another_key", "another value")
db.session.commit()
# Duplicate key
pytest.raises(
IntegrityError, ObjectVersionTag.create, obj1, "mykey", "newvalue")
# Test get
assert ObjectVersionTag.query.count() == 2
assert ObjectVersionTag.get(obj1, "mykey").value == "testvalue"
assert ObjectVersionTag.get_value(obj1.version_id, "another_key") \
== "another value"
assert ObjectVersionTag.get_value(obj1, "invalid") is None
# Test delete
ObjectVersionTag.delete(obj1, "mykey")
assert ObjectVersionTag.query.count() == 1
ObjectVersionTag.delete(obj1, "invalid")
assert ObjectVersionTag.query.count() == 1
# Create or update
ObjectVersionTag.create_or_update(obj1, "another_key", "newval")
ObjectVersionTag.create_or_update(obj1.version_id, "newkey", "testval")
db.session.commit()
assert ObjectVersionTag.get_value(obj1, "another_key") == "newval"
assert ObjectVersionTag.get_value(obj1, "newkey") == "testval"
# Get tags as dictionary
assert obj1.get_tags() == dict(another_key="newval", newkey="testval")
obj2 = ObjectVersion.create(b, 'test2')
assert obj2.get_tags() == dict()
# Copy object version
obj_copy = obj1.copy()
db.session.commit()
assert obj_copy.get_tags() == dict(another_key="newval", newkey="testval")
assert ObjectVersionTag.query.count() == 4
# Cascade delete
ObjectVersion.query.delete()
db.session.commit()
assert ObjectVersionTag.query.count() == 0
def _create_tags(self):
"""Create additional tags."""
# Subtitle file
pattern = re.compile(".*_([a-zA-Z]{2})\.vtt$")
objs = [
o for o in sorted_files_from_bucket(self._bucket)
if pattern.match(o.key)
]
with db.session.begin_nested():
for obj in objs:
# language tag
found = pattern.findall(obj.key)
if len(found) == 1:
lang = found[0]
ObjectVersionTag.create_or_update(obj, 'language', lang)
else:
# clean to be sure there is no some previous value
ObjectVersionTag.delete(obj, 'language')
# other tags
ObjectVersionTag.create_or_update(obj, 'content_type', 'vtt')
ObjectVersionTag.create_or_update(obj, 'context_type',
'subtitle')
ObjectVersionTag.create_or_update(obj, 'media_type',
'subtitle')
# refresh object
db.session.refresh(obj)
# Poster frame
pattern = re.compile('^poster\.(jpg|png)$')
try:
poster = [
o for o in sorted_files_from_bucket(self._bucket)
if pattern.match(o.key)
][0]
except IndexError:
return
ext = pattern.findall(poster.key)[0]
# frame tags
ObjectVersionTag.create_or_update(poster, 'content_type', ext)
ObjectVersionTag.create_or_update(poster, 'context_type', 'poster')
ObjectVersionTag.create_or_update(poster, 'media_type', 'image')
# refresh object
db.session.refresh(poster)
def move_file_into_local(obj, delete=True):
"""Move file from XRootD accessed file system into a local path
:param obj: Object version to make locally available.
:param delete: Whether or not the tmp file should be deleted on exit.
"""
if os.path.exists(obj.file.uri):
yield obj.file.uri
# TODO: remove migration hack
# Check if we are migrating
elif obj.get_tags().get('dfs_path', None):
# This is a special situation!
yield obj.get_tags().get('dfs_path', None)
else:
temp_location = obj.get_tags().get('temp_location', None)
if not temp_location:
temp_folder = tempfile.mkdtemp()
temp_location = os.path.join(temp_folder, 'data')
with open(temp_location, 'wb') as dst:
shutil.copyfileobj(file_opener_xrootd(obj.file.uri, 'rb'), dst)
ObjectVersionTag.create(obj, 'temp_location', temp_location)
db.session.commit()
else:
temp_folder = os.path.dirname(temp_location)
try:
yield temp_location
except:
shutil.rmtree(temp_folder)
ObjectVersionTag.delete(obj, 'temp_location')
db.session.commit()
raise
else:
if delete:
shutil.rmtree(temp_folder)
ObjectVersionTag.delete(obj, 'temp_location')
db.session.commit()
def clean(self, deposit_id, version_id, *args, **kwargs):
"""Undo metadata extraction."""
# 1. Revert patch on record
recid = str(PersistentIdentifier.get('depid', deposit_id).object_uuid)
patch = [{
'op': 'remove',
'path': '/_cds/extracted_metadata',
}]
validator = 'cds.modules.records.validators.PartialDraft4Validator'
try:
patch_record(recid=recid, patch=patch, validator=validator)
except jsonpatch.JsonPatchConflict as c:
logger.warning(
'Failed to apply JSON Patch to deposit {0}: {1}'.format(
recid, c))
# Delete tmp file if any
obj = as_object_version(version_id)
temp_location = obj.get_tags().get('temp_location', None)
if temp_location:
shutil.rmtree(temp_location)
ObjectVersionTag.delete(obj, 'temp_location')
db.session.commit()
def post(self, pid, record, key, files, file_rec, multipart_config,
upload_id, parts):
if multipart_config['upload_id'] != upload_id:
abort(404)
before_upload_complete.send(file_rec,
record=record,
file=file_rec,
multipart_config=multipart_config)
res = current_s3.client.complete_multipart_upload(
bucket=multipart_config['bucket'],
key=multipart_config['key'],
upload_id=upload_id,
parts=parts)
with db.session.begin_nested():
ObjectVersionTag.delete(file_rec.obj, MULTIPART_CONFIG_TAG)
ObjectVersionTag.delete(file_rec.obj, MULTIPART_EXPIRATION_TAG)
etag = 'etag:{}'.format(res['ETag'])
file_rec.obj.file.checksum = etag
file_rec['checksum'] = etag
after_upload_complete.send(file_rec,
record=record,
file=file_rec,
files=files)
files.flush()
record.commit()
db.session.commit()
return jsonify({
'location': file_rec.data['url'],
'checksum': file_rec['checksum']
})
def __delitem__(self, key: ObjectTagKey): # type: ignore
ObjectVersionTag.delete(self._object_version, key.value)
super().pop(key, None)
