def test_save(self): with self.app.app_context(): s3 = boto3.client('s3') bucket_name = self.app.config['S3_BUCKET_NAME'] s3.create_bucket(Bucket=bucket_name) metadata = BitStore(publisher="pub_test", package="test_package", body='hi') key = metadata.build_s3_key('datapackage.json') metadata.save_metadata() obs_list = list(s3.list_objects(Bucket=bucket_name, Prefix=key). \ get('Contents')) self.assertEqual(1, len(obs_list)) self.assertEqual(key, obs_list[0]['Key']) res = s3.get_object_acl(Bucket=bucket_name, Key=key) owner_id = res['Owner']['ID'] aws_all_user_group_url = 'http://acs.amazonaws.com/groups/global/AllUsers' full_control = filter(lambda grant: grant['Permission'] == 'FULL_CONTROL', res['Grants']) self.assertEqual(len(full_control), 1) self.assertEqual(full_control[0].get('Grantee')['ID'], owner_id) read_control = filter(lambda grant: grant['Permission'] == 'READ', res['Grants']) self.assertEqual(len(read_control), 1) self.assertEqual(read_control[0].get('Grantee')['URI'], aws_all_user_group_url)
def test_extract_information_from_s3_url(self): metadata = BitStore(publisher="pub_test", package="test_package") s3_key = metadata.build_s3_key("datapackage.json") pub, package, version = BitStore.extract_information_from_s3_url(s3_key) self.assertEqual(pub, 'pub_test') self.assertEqual(package, 'test_package') self.assertEqual(version, 'latest')
def populate_data(publisher_name): data = json.loads(open('fixtures/datapackage.json').read()) data_csv = open('fixtures/data/demo-resource.csv').read() readme = open('fixtures/README.md').read() package = models.Package.query.join(models.Publisher)\ .filter(models.Package.name == "demo-package", models.Publisher.name == publisher_name).first() if package: db.session.delete(models.Package.query.get(package.id)) db.session.commit() publisher = models.Publisher.query.filter_by(name=publisher_name).first() metadata = models.Package(name="demo-package", descriptor=data, readme=readme) metadata.status, metadata.private \ = 'active', False publisher.packages.append(metadata) db.session.add(publisher) db.session.commit() bitstore = BitStore(publisher_name, package='demo-package', body=json.dumps(data)) bitstore.save_metadata() key = bitstore.build_s3_key('demo-resource.csv') bucket_name = app.config['S3_BUCKET_NAME'] s3_client = app.config['S3'] s3_client.put_object(Bucket=bucket_name, Key=key, Body=data_csv, ACL='public-read')
def test_return_none_if_no_readme_found(self): with self.app.app_context(): bit_store = BitStore('test_pub', 'test_package') s3 = boto3.client('s3') bucket_name = self.app.config['S3_BUCKET_NAME'] s3.create_bucket(Bucket=bucket_name) read_me_key = bit_store.build_s3_key('test.md') s3.put_object(Bucket=bucket_name, Key=read_me_key, Body='') self.assertEqual(bit_store.get_readme_object_key(), 'None')
def test_get_empty_metadata_name_for_publisher(self): with self.app.app_context(): s3 = boto3.client('s3') bucket_name = self.app.config['S3_BUCKET_NAME'] s3.create_bucket(Bucket=bucket_name) metadata = BitStore(publisher="pub_test", package="test_package", body='hi') s3.put_object(Bucket=bucket_name, Key='test/key.json', Body=metadata.body) self.assertEqual(0, len(metadata.get_all_metadata_name_for_publisher()))
def test_get_metadata_body(self): with self.app.app_context(): s3 = boto3.client('s3') bucket_name = self.app.config['S3_BUCKET_NAME'] s3.create_bucket(Bucket=bucket_name) metadata = BitStore(publisher="pub_test", package="test_package", body='hi') s3.put_object(Bucket=bucket_name, Key=metadata.build_s3_key('datapackage.json'), Body=metadata.body) self.assertEqual(metadata.body, metadata.get_metadata_body())
def test_should_copy_all_object_from_latest_to_tag(self): numeric_version = 0.8 with self.app.app_context(): bit_store = BitStore('test_pub', 'test_package') s3 = boto3.client('s3') bucket_name = self.app.config['S3_BUCKET_NAME'] s3.create_bucket(Bucket=bucket_name) read_me_key = bit_store.build_s3_key('test.md') data_key = bit_store.build_s3_key('data.csv') metadata_key = bit_store.build_s3_key('datapackage.json') s3.put_object(Bucket=bucket_name, Key=read_me_key, Body='readme') s3.put_object(Bucket=bucket_name, Key=data_key, Body='data') s3.put_object(Bucket=bucket_name, Key=metadata_key, Body='metedata') bit_store.copy_to_new_version(numeric_version) bit_store_numeric = BitStore('test_pub', 'test_package', numeric_version) objects_nu = s3.list_objects(Bucket=bucket_name, Prefix=bit_store_numeric .build_s3_versioned_prefix()) objects_old = s3.list_objects(Bucket=bucket_name, Prefix=bit_store .build_s3_versioned_prefix()) self.assertEqual(len(objects_nu['Contents']), len(objects_old['Contents']))
def finalize_publish(cls, user_id, datapackage_url): ''' Gets the datapackage.json and README from S3 and imports into database. Returns status "queued" if ok, else - None ''' publisher, package, version = BitStore.extract_information_from_s3_url( datapackage_url) if Package.exists(publisher, package): status = check_is_authorized('Package::Update', publisher, package, user_id) else: status = check_is_authorized('Package::Create', publisher, package, user_id) if not status: raise InvalidUsage('Not authorized to upload data', 400) bit_store = BitStore(publisher, package) b = bit_store.get_metadata_body() body = json.loads(b) bit_store.change_acl('public-read') readme = bit_store.get_s3_object(bit_store.get_readme_object_key()) Package.create_or_update(name=package, publisher_name=publisher, descriptor=body, readme=readme) return "queued"
def test_generate_pre_signed_put_obj_url(self): with self.app.app_context(): s3 = boto3.client('s3') bucket_name = self.app.config['S3_BUCKET_NAME'] s3.create_bucket(Bucket=bucket_name) metadata = BitStore(publisher="pub_test", package="test_package", body='hi') post = metadata.generate_pre_signed_post_object('datapackage.json', 123) parsed = urlparse(post['url']) self.assertEqual(parsed.netloc, 's3-{region}.amazonaws.com'. format(region=self.app.config['AWS_REGION'])) self.assertEqual('public-read', post['fields']['acl']) self.assertEqual('text/plain', post['fields']['Content-Type'])
def undelete_data_package(publisher, package): """ DPR data package un-delete operation. This API is responsible for un-mark the mark for delete of data package --- tags: - package parameters: - in: path name: publisher type: string required: true description: publisher name - in: path name: package type: string required: true description: package name - in: header name: Authorization type: string required: true description: JWT Token responses: 500: description: Internal Server Error 200: description: Success Message schema: id: put_package_success properties: status: type: string default: OK """ bitstore = BitStore(publisher=publisher, package=package) status_db = logic.Package.change_status(publisher, package, models.PackageStateEnum.active) try: status_acl = bitstore.change_acl('public-read') except Exception as e: ## TODO roll back changes in db raise InvalidUsage(e.message, 500) if status_acl and status_db: return jsonify({"status": "OK"}), 200
def test_change_acl(self): with self.app.app_context(): public_grants = { 'CanonicalUser': '******', 'Group': 'READ' } private_grants = {'CanonicalUser': '******'} bit_store = BitStore('test_pub', 'test_package', body='test') s3 = boto3.client('s3') bucket_name = self.app.config['S3_BUCKET_NAME'] s3.create_bucket(Bucket=bucket_name) metadata_key = bit_store.build_s3_key('datapackage.json') bit_store.save_metadata() res = s3.get_object_acl(Bucket=bucket_name, Key=metadata_key) owner_id = res['Owner']['ID'] aws_all_user_group_url = 'http://acs.amazonaws.com/groups/global/AllUsers' full_control = filter(lambda grant: grant['Permission'] == 'FULL_CONTROL', res['Grants']) self.assertEqual(len(full_control), 1) self.assertEqual(full_control[0].get('Grantee')['ID'], owner_id) read_control = filter(lambda grant: grant['Permission'] == 'READ', res['Grants']) self.assertEqual(len(read_control), 1) self.assertEqual(read_control[0].get('Grantee')['URI'], aws_all_user_group_url) bit_store.change_acl("private") res = s3.get_object_acl(Bucket=bucket_name, Key=metadata_key) full_control = filter(lambda grant: grant['Permission'] == 'FULL_CONTROL', res['Grants']) self.assertEqual(len(full_control), 1) self.assertEqual(full_control[0].get('Grantee')['ID'], owner_id) read_control = filter(lambda grant: grant['Permission'] == 'READ', res['Grants']) self.assertEqual(len(read_control), 0)
def delete_data_package(publisher, package): """ DPR Data Package Soft Delete Marks Data Package as private --- tags: - package parameters: - in: path name: publisher type: string required: true description: publisher name - in: path name: package type: string required: true description: package name - in: header name: Authorization type: string required: true description: JWT Token responses: 500: description: Internal Server Error 200: description: Success Message schema: id: put_package_success properties: status: type: string default: OK """ bitstore = BitStore(publisher=publisher, package=package) status_db = logic.Package.change_status(publisher, package, models.PackageStateEnum.deleted) try: status_acl = bitstore.change_acl('private') except Exception as e: ## TODO roll back changes in db raise InvalidUsage(e.message, 500) if status_acl and status_db: return jsonify({"status": "OK"}), 200
def test_throw_403_if_not_owner_or_member_of_publisher(self): s3 = boto3.client('s3') s3.create_bucket(Bucket=self.bucket_name) bit_store = BitStore('test_pub', 'test_package') read_me_key = bit_store.build_s3_key('test.md') data_key = bit_store.build_s3_key('data.csv') metadata_key = bit_store.build_s3_key('datapackage.json') s3.put_object(Bucket=self.bucket_name, Key=read_me_key, Body='readme') s3.put_object(Bucket=self.bucket_name, Key=data_key, Body='data') s3.put_object(Bucket=self.bucket_name, Key=metadata_key, Body='metedata') response = self.client.post(self.jwt_url, data=json.dumps({ 'username': self.user_not_allowed_name, 'secret': 'super_secret' }), content_type='application/json') data = json.loads(response.data) jwt_not_allowed = data['token'] auth_not_allowed = "%s" % jwt_not_allowed response = self.client.post( self.url, data=json.dumps({'version': 'tag_one'}), content_type='application/json', headers=dict(Authorization=auth_not_allowed)) self.assertEqual(response.status_code, 403) with self.app.app_context(): data_latest = Package.query.join(Publisher). \ filter(Publisher.name == self.publisher_name, Package.name == self.package).all() self.assertEqual(1, len(data_latest)) bit_store_tagged = BitStore('test_pub', 'test_package', 'tag_one') objects_nu = s3.list_objects( Bucket=self.bucket_name, Prefix=bit_store_tagged.build_s3_versioned_prefix()) self.assertTrue('Contents' not in objects_nu)
def test_delete_data_package(self): with self.app.app_context(): bit_store = BitStore('test_pub', 'test_package') s3 = boto3.client('s3') bucket_name = self.app.config['S3_BUCKET_NAME'] s3.create_bucket(Bucket=bucket_name) read_me_key = bit_store.build_s3_key('test.md') data_key = bit_store.build_s3_key('data.csv') metadata_key = bit_store.build_s3_key('datapackage.json') s3.put_object(Bucket=bucket_name, Key=read_me_key, Body='readme') s3.put_object(Bucket=bucket_name, Key=data_key, Body='data') s3.put_object(Bucket=bucket_name, Key=metadata_key, Body='metedata') status = bit_store.delete_data_package() read_me_res = s3.list_objects(Bucket=bucket_name, Prefix=read_me_key) self.assertTrue('Contents' not in read_me_res) data_res = s3.list_objects(Bucket=bucket_name, Prefix=data_key) self.assertTrue('Contents' not in data_res) self.assertTrue(status)
def test_metadata_s3_key(self): metadata = BitStore(publisher="pub_test", package="test_package") expected = "{t}/pub_test/test_package/_v/latest/datapackage.json". \ format(t=metadata.prefix) self.assertEqual(expected, metadata.build_s3_key('datapackage.json'))
def tag_data_package(publisher, package): """ DPR metadata put operation. This API is responsible for tagging data package --- tags: - package parameters: - in: path name: publisher type: string required: true description: publisher name - in: path name: package type: string required: true description: package name - in: body name: version type: string required: true description: version value - in: header name: Authorization type: string required: true description: JWT Token responses: 400: description: JWT is invalid or req body is not valid 401: description: Invalid Header for JWT 403: description: User not allowed for operation 404: description: User not found 500: description: Internal Server Error 200: description: Success Message schema: id: put_package_success properties: status: type: string description: Status of the operation default: OK """ data = request.get_json() if 'version' not in data: raise InvalidUsage('version not found', 400) bitstore = BitStore(publisher, package) status_db = logic.Package.create_or_update_tag(publisher, package, data['version']) try: status_bitstore = bitstore.copy_to_new_version(data['version']) except Exception as e: ## TODO roll back changes in db raise InvalidUsage(e.message, 500) return jsonify({"status": "OK"}), 200
def get_url(self, data): bitstore = BitStore(data.publisher.name, data.name) datapackage_json_url_in_s3 = bitstore.build_s3_object_url() return datapackage_json_url_in_s3
def __init__(self, package_name, publisher, relative_path, props): self.package_name = package_name self.publisher = publisher self.relative_path = relative_path self.props = props self.bitstore = BitStore(publisher=publisher, package=package_name)
def test_metadata_s3_prefix(self): metadata = BitStore(publisher="pub_test", package="test_package") expected = "{t}/pub_test/test_package".format(t=metadata.prefix) self.assertEqual(expected, metadata.build_s3_base_prefix())