def test_object_version_tags(app, db, dummy_location): """Test object version tags.""" f = FileInstance(uri="f1", size=1, checksum="mychecksum") db.session.add(f) db.session.commit() b = Bucket.create() obj1 = ObjectVersion.create(b, "test").set_file(f) ObjectVersionTag.create(obj1, "mykey", "testvalue") ObjectVersionTag.create(obj1, "another_key", "another value") db.session.commit() # Duplicate key pytest.raises( IntegrityError, ObjectVersionTag.create, obj1, "mykey", "newvalue") # Test get assert ObjectVersionTag.query.count() == 2 assert ObjectVersionTag.get(obj1, "mykey").value == "testvalue" assert ObjectVersionTag.get_value(obj1.version_id, "another_key") \ == "another value" assert ObjectVersionTag.get_value(obj1, "invalid") is None # Test delete ObjectVersionTag.delete(obj1, "mykey") assert ObjectVersionTag.query.count() == 1 ObjectVersionTag.delete(obj1, "invalid") assert ObjectVersionTag.query.count() == 1 # Create or update ObjectVersionTag.create_or_update(obj1, "another_key", "newval") ObjectVersionTag.create_or_update(obj1.version_id, "newkey", "testval") db.session.commit() assert ObjectVersionTag.get_value(obj1, "another_key") == "newval" assert ObjectVersionTag.get_value(obj1, "newkey") == "testval" # Get tags as dictionary assert obj1.get_tags() == dict(another_key="newval", newkey="testval") obj2 = ObjectVersion.create(b, 'test2') assert obj2.get_tags() == dict() # Copy object version obj_copy = obj1.copy() db.session.commit() assert obj_copy.get_tags() == dict(another_key="newval", newkey="testval") assert ObjectVersionTag.query.count() == 4 # Cascade delete ObjectVersion.query.delete() db.session.commit() assert ObjectVersionTag.query.count() == 0
def _create_tags(self): """Create additional tags.""" # Subtitle file pattern = re.compile(".*_([a-zA-Z]{2})\.vtt$") objs = [ o for o in sorted_files_from_bucket(self._bucket) if pattern.match(o.key) ] with db.session.begin_nested(): for obj in objs: # language tag found = pattern.findall(obj.key) if len(found) == 1: lang = found[0] ObjectVersionTag.create_or_update(obj, 'language', lang) else: # clean to be sure there is no some previous value ObjectVersionTag.delete(obj, 'language') # other tags ObjectVersionTag.create_or_update(obj, 'content_type', 'vtt') ObjectVersionTag.create_or_update(obj, 'context_type', 'subtitle') ObjectVersionTag.create_or_update(obj, 'media_type', 'subtitle') # refresh object db.session.refresh(obj) # Poster frame pattern = re.compile('^poster\.(jpg|png)$') try: poster = [ o for o in sorted_files_from_bucket(self._bucket) if pattern.match(o.key) ][0] except IndexError: return ext = pattern.findall(poster.key)[0] # frame tags ObjectVersionTag.create_or_update(poster, 'content_type', ext) ObjectVersionTag.create_or_update(poster, 'context_type', 'poster') ObjectVersionTag.create_or_update(poster, 'media_type', 'image') # refresh object db.session.refresh(poster)
def move_file_into_local(obj, delete=True): """Move file from XRootD accessed file system into a local path :param obj: Object version to make locally available. :param delete: Whether or not the tmp file should be deleted on exit. """ if os.path.exists(obj.file.uri): yield obj.file.uri # TODO: remove migration hack # Check if we are migrating elif obj.get_tags().get('dfs_path', None): # This is a special situation! yield obj.get_tags().get('dfs_path', None) else: temp_location = obj.get_tags().get('temp_location', None) if not temp_location: temp_folder = tempfile.mkdtemp() temp_location = os.path.join(temp_folder, 'data') with open(temp_location, 'wb') as dst: shutil.copyfileobj(file_opener_xrootd(obj.file.uri, 'rb'), dst) ObjectVersionTag.create(obj, 'temp_location', temp_location) db.session.commit() else: temp_folder = os.path.dirname(temp_location) try: yield temp_location except: shutil.rmtree(temp_folder) ObjectVersionTag.delete(obj, 'temp_location') db.session.commit() raise else: if delete: shutil.rmtree(temp_folder) ObjectVersionTag.delete(obj, 'temp_location') db.session.commit()
def clean(self, deposit_id, version_id, *args, **kwargs): """Undo metadata extraction.""" # 1. Revert patch on record recid = str(PersistentIdentifier.get('depid', deposit_id).object_uuid) patch = [{ 'op': 'remove', 'path': '/_cds/extracted_metadata', }] validator = 'cds.modules.records.validators.PartialDraft4Validator' try: patch_record(recid=recid, patch=patch, validator=validator) except jsonpatch.JsonPatchConflict as c: logger.warning( 'Failed to apply JSON Patch to deposit {0}: {1}'.format( recid, c)) # Delete tmp file if any obj = as_object_version(version_id) temp_location = obj.get_tags().get('temp_location', None) if temp_location: shutil.rmtree(temp_location) ObjectVersionTag.delete(obj, 'temp_location') db.session.commit()
def post(self, pid, record, key, files, file_rec, multipart_config, upload_id, parts): if multipart_config['upload_id'] != upload_id: abort(404) before_upload_complete.send(file_rec, record=record, file=file_rec, multipart_config=multipart_config) res = current_s3.client.complete_multipart_upload( bucket=multipart_config['bucket'], key=multipart_config['key'], upload_id=upload_id, parts=parts) with db.session.begin_nested(): ObjectVersionTag.delete(file_rec.obj, MULTIPART_CONFIG_TAG) ObjectVersionTag.delete(file_rec.obj, MULTIPART_EXPIRATION_TAG) etag = 'etag:{}'.format(res['ETag']) file_rec.obj.file.checksum = etag file_rec['checksum'] = etag after_upload_complete.send(file_rec, record=record, file=file_rec, files=files) files.flush() record.commit() db.session.commit() return jsonify({ 'location': file_rec.data['url'], 'checksum': file_rec['checksum'] })
def __delitem__(self, key: ObjectTagKey): # type: ignore ObjectVersionTag.delete(self._object_version, key.value) super().pop(key, None)