def create(cls, data, id_=None, **kwargs): """Create a CDS deposit. Adds bucket creation immediately on deposit creation. """ if '_deposit' not in data: id_ = id_ or uuid.uuid4() cls.deposit_minter(id_, data) bucket = Bucket.create(location=Location.get_by_name( kwargs.get('bucket_location', 'default'))) data['_buckets'] = {'deposit': str(bucket.id)} data.setdefault('_cds', {}) data['_cds'].setdefault('state', {}) data.setdefault('keywords', []) data.setdefault('license', [{ 'license': 'CERN', 'material': '', 'url': 'http://copyright.web.cern.ch', }]) if '_access' not in data: data.setdefault('_access', {}) deposit = super(CDSDeposit, cls).create(data, id_=id_, validator=PartialDraft4Validator) RecordsBuckets.create(record=deposit.model, bucket=bucket) return deposit
def _resolve_bucket(cls, deposit, record): """Build bucket.""" logging.debug('Creating new buckets, record and deposit.') bucket = Bucket.create(location=Location.get_by_name('videos')) deposit['_buckets'] = {'deposit': str(bucket.id)} RecordsBuckets.create(record=deposit.model, bucket=bucket) record['_buckets'] = deepcopy(deposit['_buckets']) db.session.commit()
def test_location(app, db): """Test location model.""" with db.session.begin_nested(): l1 = Location(name='test1', uri='file:///tmp', default=False) l2 = Location(name='test2', uri='file:///tmp', default=True) l3 = Location(name='test3', uri='file:///tmp', default=False) db.session.add(l1) db.session.add(l2) db.session.add(l3) assert Location.get_by_name('test1').name == 'test1' assert Location.get_by_name('test2').name == 'test2' assert Location.get_by_name('test3').name == 'test3' assert Location.get_default().name == 'test2' assert len(Location.all()) == 3 assert str(Location.get_by_name('test1')) == 'test1'
def test_location(app, db): """Test location model.""" with db.session.begin_nested(): l1 = Location(name='test1', uri='file:///tmp', default=False) l2 = Location(name='test2', uri='file:///tmp', default=True) l3 = Location(name='test3', uri='file:///tmp', default=False) db.session.add(l1) db.session.add(l2) db.session.add(l3) assert Location.get_by_name('test1').name == 'test1' assert Location.get_by_name('test2').name == 'test2' assert Location.get_by_name('test3').name == 'test3' assert Location.get_default().name == 'test2' assert len(Location.all()) == 3 assert str(Location.get_by_name('test1')) == 'test1'
def test_bucket_create_object(app, db): """Test bucket creation.""" with db.session.begin_nested(): l1 = Location(name='test1', uri='file:///tmp/1', default=False) l2 = Location(name='test2', uri='file:///tmp/2', default=True) db.session.add(l1) db.session.add(l2) assert Location.query.count() == 2 # Simple create with db.session.begin_nested(): b = Bucket.create() assert b.id assert b.default_location == Location.get_default().id assert b.location == Location.get_default() assert b.default_storage_class == \ app.config['FILES_REST_DEFAULT_STORAGE_CLASS'] assert b.size == 0 assert b.quota_size is None assert b.max_file_size is None assert b.deleted is False # __repr__ test assert str(b) == str(b.id) # Retrieve one assert Bucket.get(b.id).id == b.id # Create with location_name and storage class with db.session.begin_nested(): b = Bucket.create(location=l1, storage_class='A') assert b.default_location == Location.get_by_name('test1').id assert b.default_storage_class == 'A' # Create using location name instead b = Bucket.create(location=l2.name, storage_class='A') assert b.default_location == Location.get_by_name('test2').id # Retrieve one assert Bucket.all().count() == 3 # Invalid storage class. pytest.raises(ValueError, Bucket.create, storage_class='X')
def test_bucket_create_object(app, db): """Test bucket creation.""" with db.session.begin_nested(): l1 = Location(name='test1', uri='file:///tmp/1', default=False) l2 = Location(name='test2', uri='file:///tmp/2', default=True) db.session.add(l1) db.session.add(l2) assert Location.query.count() == 2 # Simple create with db.session.begin_nested(): b = Bucket.create() assert b.id assert b.default_location == Location.get_default().id assert b.location == Location.get_default() assert b.default_storage_class == \ app.config['FILES_REST_DEFAULT_STORAGE_CLASS'] assert b.size == 0 assert b.quota_size is None assert b.max_file_size is None assert b.deleted is False # __repr__ test assert str(b) == str(b.id) # Retrieve one assert Bucket.get(b.id).id == b.id # Create with location_name and storage class with db.session.begin_nested(): b = Bucket.create(location=l1, storage_class='A') assert b.default_location == Location.get_by_name('test1').id assert b.default_storage_class == 'A' # Create using location name instead b = Bucket.create(location=l2.name, storage_class='A') assert b.default_location == Location.get_by_name('test2').id # Retrieve one assert Bucket.all().count() == 3 # Invalid storage class. pytest.raises(ValueError, Bucket.create, storage_class='X')
def get_bucket(self, location=None, storage_class=None, record_id=None): """Allows to retrieve bucket for any record(default: self) Args: location (str): Bucket location (default: 'RECORDS_DEFAULT_FILE_LOCATION_NAME') from config storage_class (str): Bucket storage class (default: 'RECORDS_DEFAULT_STORAGE_CLASS') from config record_id (int): record to which bucket is asigned (default: self) Returns: Bucket: if found in db for selected location, storage_class and record_id or None if there were no bucket. Raises: NoResultFound: When provided location was not found. """ if not storage_class: storage_class = current_app.config["RECORDS_DEFAULT_STORAGE_CLASS"] if not location: location = current_app.config["RECORDS_DEFAULT_FILE_LOCATION_NAME"] if not record_id: record_id = self.id try: location_obj = Location.get_by_name(location) except NoResultFound: raise NoResultFound( "Cannot find location %s. Please check if system is configured properly!", location, ) bucket = (RecordsBuckets.query.join(Bucket).filter( RecordsBuckets.record_id == record_id, Bucket.default_storage_class == storage_class, Bucket.default_location == location_obj.id, ).one_or_none()) if bucket: LOGGER.debug("Bucket found", key=bucket.bucket.id, uuid=self.id) return bucket.bucket LOGGER.info("Bucket not found", uuid=self.id) return self._create_bucket(location, storage_class)
def _process_files(record, files_metadata): """Attach files to a record with a given metadata. Assumptions: - The source must be a URL pointing to a tar file. - All files listed in the metadata are inside the source tar. - Master files are listed before slaves. - The reference from the slave to master is done via key. """ if not files_metadata: return bucket = Bucket.create(location=Location.get_by_name('videos')) RecordsBuckets.create(record=record.model, bucket=bucket) response = requests.get( files_metadata['source'], stream=True, verify=False) # Throw an error for bad status codes response.raise_for_status() with tempfile.NamedTemporaryFile(suffix='.tar', delete=False) as f: for chunk in response: f.write(chunk) tar = tarfile.open(name=f.name) tar.extractall(path=tempfile.gettempdir()) files_base_dir = os.path.join(tempfile.gettempdir(), tar.getnames()[0]) tar.close() os.remove(f.name) for f in files_metadata['metadata']: obj = ObjectVersion.create(bucket, f['key']) with open(os.path.join(files_base_dir, f['key']), 'rb') as fp: obj.set_contents(fp) for k, v in f['tags'].items(): if k == 'master': v = ObjectVersion.get(bucket, v).version_id ObjectVersionTag.create(obj, k, v) shutil.rmtree(files_base_dir) record['_files'] = record.files.dumps()
def _resolve_bucket(cls, deposit, record): """Build bucket.""" bucket = Bucket.create(location=Location.get_by_name('videos')) deposit['_buckets'] = {'deposit': str(bucket.id)} RecordsBuckets.create(record=deposit.model, bucket=bucket) record['_buckets'] = deepcopy(deposit['_buckets'])