def test_should_copy_all_object_from_latest_to_tag(self): numeric_version = 0.8 with self.app.app_context(): bit_store = BitStore('test_pub', 'test_package') s3 = boto3.client('s3') bucket_name = self.app.config['S3_BUCKET_NAME'] s3.create_bucket(Bucket=bucket_name) read_me_key = bit_store.build_s3_key('test.md') data_key = bit_store.build_s3_key('data.csv') metadata_key = bit_store.build_s3_key('datapackage.json') s3.put_object(Bucket=bucket_name, Key=read_me_key, Body='readme') s3.put_object(Bucket=bucket_name, Key=data_key, Body='data') s3.put_object(Bucket=bucket_name, Key=metadata_key, Body='metedata') bit_store.copy_to_new_version(numeric_version) bit_store_numeric = BitStore('test_pub', 'test_package', numeric_version) objects_nu = s3.list_objects( Bucket=bucket_name, Prefix=bit_store_numeric.build_s3_versioned_prefix()) objects_old = s3.list_objects( Bucket=bucket_name, Prefix=bit_store.build_s3_versioned_prefix()) self.assertEqual(len(objects_nu['Contents']), len(objects_old['Contents']))
def test_save(self): with self.app.app_context(): s3 = boto3.client('s3') bucket_name = self.app.config['S3_BUCKET_NAME'] s3.create_bucket(Bucket=bucket_name) metadata = BitStore(publisher="pub_test", package="test_package", body='hi') key = metadata.build_s3_key('datapackage.json') metadata.save_metadata() obs_list = list(s3.list_objects(Bucket=bucket_name, Prefix=key).\ get('Contents')) self.assertEqual(1, len(obs_list)) self.assertEqual(key, obs_list[0]['Key']) res = s3.get_object_acl(Bucket=bucket_name, Key=key) owner_id = res['Owner']['ID'] aws_all_user_group_url = 'http://acs.amazonaws.com/groups/global/AllUsers' full_control = filter( lambda grant: grant['Permission'] == 'FULL_CONTROL', res['Grants']) self.assertEqual(len(full_control), 1) self.assertEqual(full_control[0].get('Grantee')['ID'], owner_id) read_control = filter(lambda grant: grant['Permission'] == 'READ', res['Grants']) self.assertEqual(len(read_control), 1) self.assertEqual(read_control[0].get('Grantee')['URI'], aws_all_user_group_url)
def populate_data(publisher_name): data = json.loads(open('fixtures/datapackage.json').read()) data_csv = open('fixtures/data/demo-resource.csv').read() readme = open('fixtures/README.md').read() package = Package.query.join(Publisher)\ .filter(Package.name == "demo-package", Publisher.name == publisher_name).first() if package: db.session.delete(Package.query.get(package.id)) db.session.commit() publisher = Publisher.query.filter_by(name=publisher_name).first() metadata = Package(name="demo-package") metadata.descriptor, metadata.status, metadata.private, metadata.readme \ = json.dumps(data), 'active', False, readme publisher.packages.append(metadata) db.session.add(publisher) db.session.commit() bitstore = BitStore(publisher_name, package='demo-package', body=json.dumps(data)) bitstore.save_metadata() key = bitstore.build_s3_key('demo-resource.csv') bucket_name = app.config['S3_BUCKET_NAME'] s3_client = app.config['S3'] s3_client.put_object(Bucket=bucket_name, Key=key, Body=data_csv, ACL='public-read')
def test_return_none_if_no_readme_found(self): with self.app.app_context(): bit_store = BitStore('test_pub', 'test_package') s3 = boto3.client('s3') bucket_name = self.app.config['S3_BUCKET_NAME'] s3.create_bucket(Bucket=bucket_name) read_me_key = bit_store.build_s3_key('test.md') s3.put_object(Bucket=bucket_name, Key=read_me_key, Body='') self.assertEqual(bit_store.get_readme_object_key(), None)
def test_get_metadata_body(self): with self.app.app_context(): s3 = boto3.client('s3') bucket_name = self.app.config['S3_BUCKET_NAME'] s3.create_bucket(Bucket=bucket_name) metadata = BitStore(publisher="pub_test", package="test_package", body='hi') s3.put_object(Bucket=bucket_name, Key=metadata.build_s3_key('datapackage.json'), Body=metadata.body) self.assertEqual(metadata.body, metadata.get_metadata_body())
def test_throw_403_if_not_owner_or_member_of_publisher(self): s3 = boto3.client('s3') s3.create_bucket(Bucket=self.bucket_name) bit_store = BitStore('test_pub', 'test_package') read_me_key = bit_store.build_s3_key('test.md') data_key = bit_store.build_s3_key('data.csv') metadata_key = bit_store.build_s3_key('datapackage.json') s3.put_object(Bucket=self.bucket_name, Key=read_me_key, Body='readme') s3.put_object(Bucket=self.bucket_name, Key=data_key, Body='data') s3.put_object(Bucket=self.bucket_name, Key=metadata_key, Body='metedata') response = self.client.post(self.jwt_url, data=json.dumps({ 'username': self.user_not_allowed_name, 'secret': 'super_secret' }), content_type='application/json') data = json.loads(response.data) jwt_not_allowed = data['token'] auth_not_allowed = "bearer %s" % jwt_not_allowed response = self.client.post( self.url, data=json.dumps({'version': 'tag_one'}), content_type='application/json', headers=dict(Authorization=auth_not_allowed)) self.assertEqual(response.status_code, 403) with self.app.app_context(): data_latest = Package.query.join(Publisher). \ filter(Publisher.name == self.publisher_name, Package.name == self.package).all() self.assertEqual(1, len(data_latest)) bit_store_tagged = BitStore('test_pub', 'test_package', 'tag_one') objects_nu = s3.list_objects( Bucket=self.bucket_name, Prefix=bit_store_tagged.build_s3_versioned_prefix()) self.assertTrue('Contents' not in objects_nu)
def test_delete_data_package(self): with self.app.app_context(): bit_store = BitStore('test_pub', 'test_package') s3 = boto3.client('s3') bucket_name = self.app.config['S3_BUCKET_NAME'] s3.create_bucket(Bucket=bucket_name) read_me_key = bit_store.build_s3_key('test.md') data_key = bit_store.build_s3_key('data.csv') metadata_key = bit_store.build_s3_key('datapackage.json') s3.put_object(Bucket=bucket_name, Key=read_me_key, Body='readme') s3.put_object(Bucket=bucket_name, Key=data_key, Body='data') s3.put_object(Bucket=bucket_name, Key=metadata_key, Body='metedata') status = bit_store.delete_data_package() read_me_res = s3.list_objects(Bucket=bucket_name, Prefix=read_me_key) self.assertTrue('Contents' not in read_me_res) data_res = s3.list_objects(Bucket=bucket_name, Prefix=data_key) self.assertTrue('Contents' not in data_res) self.assertTrue(status)
def test_change_acl(self): with self.app.app_context(): public_grants = {'CanonicalUser': '******', 'Group': 'READ'} private_grants = {'CanonicalUser': '******'} bit_store = BitStore('test_pub', 'test_package', body='test') s3 = boto3.client('s3') bucket_name = self.app.config['S3_BUCKET_NAME'] s3.create_bucket(Bucket=bucket_name) metadata_key = bit_store.build_s3_key('datapackage.json') bit_store.save_metadata() res = s3.get_object_acl(Bucket=bucket_name, Key=metadata_key) owner_id = res['Owner']['ID'] aws_all_user_group_url = 'http://acs.amazonaws.com/groups/global/AllUsers' full_control = filter( lambda grant: grant['Permission'] == 'FULL_CONTROL', res['Grants']) self.assertEqual(len(full_control), 1) self.assertEqual(full_control[0].get('Grantee')['ID'], owner_id) read_control = filter(lambda grant: grant['Permission'] == 'READ', res['Grants']) self.assertEqual(len(read_control), 1) self.assertEqual(read_control[0].get('Grantee')['URI'], aws_all_user_group_url) # for grant in res['Grants']: # self.assertTrue(grant['Permission'] == # public_grants[grant['Grantee']['Type']]) # bit_store.change_acl("private") res = s3.get_object_acl(Bucket=bucket_name, Key=metadata_key) full_control = filter( lambda grant: grant['Permission'] == 'FULL_CONTROL', res['Grants']) self.assertEqual(len(full_control), 1) self.assertEqual(full_control[0].get('Grantee')['ID'], owner_id) read_control = filter(lambda grant: grant['Permission'] == 'READ', res['Grants']) self.assertEqual(len(read_control), 0)
def get_resource(publisher, package, resource): """ DPR resource get operation. This API is responsible for getting resource from S3. --- tags: - package parameters: - in: path name: publisher type: string required: true description: publisher name - in: path name: package type: string required: true description: package name - to retrieve the data package metadata - in: path name: resource type: string required: true description: resource index or name responses: 200: description: Get Data package for one key schema: id: get_data_package properties: data: type: string description: The resource 500: description: Internal Server Error """ try: path = request.path metadata = BitStore(publisher, package) if path.endswith('csv'): resource_key = metadata.build_s3_key(resource + '.csv') data = metadata.get_s3_object(resource_key) def generate(): for row in data.splitlines(): yield row + '\n' return Response(generate()), 200 else: resource_key = metadata.build_s3_key(resource + '.csv') data = metadata.get_s3_object(resource_key) data = csv.DictReader(data.splitlines()) # taking first and adding at the end to avoid last comma first_row = next(data) def generate(): yield '[' for row in data: yield json.dumps(row) + ',' yield json.dumps(first_row) + ']' return Response(generate(), content_type='application/json'), 200 except Exception as e: return handle_error('GENERIC_ERROR', e.message, 500)
def test_metadata_s3_key(self): metadata = BitStore(publisher="pub_test", package="test_package") expected = "{t}/pub_test/test_package/_v/latest/datapackage.json".\ format(t=metadata.prefix) self.assertEqual(expected, metadata.build_s3_key('datapackage.json'))