def test_should_copy_all_object_from_latest_to_tag(self):
numeric_version = 0.8
with self.app.app_context():
bit_store = BitStore('test_pub', 'test_package')
s3 = boto3.client('s3')
bucket_name = self.app.config['S3_BUCKET_NAME']
s3.create_bucket(Bucket=bucket_name)
read_me_key = bit_store.build_s3_key('test.md')
data_key = bit_store.build_s3_key('data.csv')
metadata_key = bit_store.build_s3_key('datapackage.json')
s3.put_object(Bucket=bucket_name, Key=read_me_key, Body='readme')
s3.put_object(Bucket=bucket_name, Key=data_key, Body='data')
s3.put_object(Bucket=bucket_name,
Key=metadata_key,
Body='metedata')
bit_store.copy_to_new_version(numeric_version)
bit_store_numeric = BitStore('test_pub', 'test_package',
numeric_version)
objects_nu = s3.list_objects(
Bucket=bucket_name,
Prefix=bit_store_numeric.build_s3_versioned_prefix())
objects_old = s3.list_objects(
Bucket=bucket_name,
Prefix=bit_store.build_s3_versioned_prefix())
self.assertEqual(len(objects_nu['Contents']),
len(objects_old['Contents']))
def test_save(self):
with self.app.app_context():
s3 = boto3.client('s3')
bucket_name = self.app.config['S3_BUCKET_NAME']
s3.create_bucket(Bucket=bucket_name)
metadata = BitStore(publisher="pub_test",
package="test_package",
body='hi')
key = metadata.build_s3_key('datapackage.json')
metadata.save_metadata()
obs_list = list(s3.list_objects(Bucket=bucket_name, Prefix=key).\
get('Contents'))
self.assertEqual(1, len(obs_list))
self.assertEqual(key, obs_list[0]['Key'])
res = s3.get_object_acl(Bucket=bucket_name, Key=key)
owner_id = res['Owner']['ID']
aws_all_user_group_url = 'http://acs.amazonaws.com/groups/global/AllUsers'
full_control = filter(
lambda grant: grant['Permission'] == 'FULL_CONTROL',
res['Grants'])
self.assertEqual(len(full_control), 1)
self.assertEqual(full_control[0].get('Grantee')['ID'], owner_id)
read_control = filter(lambda grant: grant['Permission'] == 'READ',
res['Grants'])
self.assertEqual(len(read_control), 1)
self.assertEqual(read_control[0].get('Grantee')['URI'],
aws_all_user_group_url)
def populate_data(publisher_name):
data = json.loads(open('fixtures/datapackage.json').read())
data_csv = open('fixtures/data/demo-resource.csv').read()
readme = open('fixtures/README.md').read()
package = Package.query.join(Publisher)\
.filter(Package.name == "demo-package",
Publisher.name == publisher_name).first()
if package:
db.session.delete(Package.query.get(package.id))
db.session.commit()
publisher = Publisher.query.filter_by(name=publisher_name).first()
metadata = Package(name="demo-package")
metadata.descriptor, metadata.status, metadata.private, metadata.readme \
= json.dumps(data), 'active', False, readme
publisher.packages.append(metadata)
db.session.add(publisher)
db.session.commit()
bitstore = BitStore(publisher_name,
package='demo-package',
body=json.dumps(data))
bitstore.save_metadata()
key = bitstore.build_s3_key('demo-resource.csv')
bucket_name = app.config['S3_BUCKET_NAME']
s3_client = app.config['S3']
s3_client.put_object(Bucket=bucket_name,
Key=key,
Body=data_csv,
ACL='public-read')
def test_return_none_if_no_readme_found(self):
with self.app.app_context():
bit_store = BitStore('test_pub', 'test_package')
s3 = boto3.client('s3')
bucket_name = self.app.config['S3_BUCKET_NAME']
s3.create_bucket(Bucket=bucket_name)
read_me_key = bit_store.build_s3_key('test.md')
s3.put_object(Bucket=bucket_name, Key=read_me_key, Body='')
self.assertEqual(bit_store.get_readme_object_key(), None)
def test_get_metadata_body(self):
with self.app.app_context():
s3 = boto3.client('s3')
bucket_name = self.app.config['S3_BUCKET_NAME']
s3.create_bucket(Bucket=bucket_name)
metadata = BitStore(publisher="pub_test",
package="test_package",
body='hi')
s3.put_object(Bucket=bucket_name,
Key=metadata.build_s3_key('datapackage.json'),
Body=metadata.body)
self.assertEqual(metadata.body, metadata.get_metadata_body())
def test_throw_403_if_not_owner_or_member_of_publisher(self):
s3 = boto3.client('s3')
s3.create_bucket(Bucket=self.bucket_name)
bit_store = BitStore('test_pub', 'test_package')
read_me_key = bit_store.build_s3_key('test.md')
data_key = bit_store.build_s3_key('data.csv')
metadata_key = bit_store.build_s3_key('datapackage.json')
s3.put_object(Bucket=self.bucket_name, Key=read_me_key, Body='readme')
s3.put_object(Bucket=self.bucket_name, Key=data_key, Body='data')
s3.put_object(Bucket=self.bucket_name,
Key=metadata_key,
Body='metedata')
response = self.client.post(self.jwt_url,
data=json.dumps({
'username': self.user_not_allowed_name,
'secret': 'super_secret'
}),
content_type='application/json')
data = json.loads(response.data)
jwt_not_allowed = data['token']
auth_not_allowed = "bearer %s" % jwt_not_allowed
response = self.client.post(
self.url,
data=json.dumps({'version': 'tag_one'}),
content_type='application/json',
headers=dict(Authorization=auth_not_allowed))
self.assertEqual(response.status_code, 403)
with self.app.app_context():
data_latest = Package.query.join(Publisher). \
filter(Publisher.name == self.publisher_name,
Package.name == self.package).all()
self.assertEqual(1, len(data_latest))
bit_store_tagged = BitStore('test_pub', 'test_package', 'tag_one')
objects_nu = s3.list_objects(
Bucket=self.bucket_name,
Prefix=bit_store_tagged.build_s3_versioned_prefix())
self.assertTrue('Contents' not in objects_nu)
def test_delete_data_package(self):
with self.app.app_context():
bit_store = BitStore('test_pub', 'test_package')
s3 = boto3.client('s3')
bucket_name = self.app.config['S3_BUCKET_NAME']
s3.create_bucket(Bucket=bucket_name)
read_me_key = bit_store.build_s3_key('test.md')
data_key = bit_store.build_s3_key('data.csv')
metadata_key = bit_store.build_s3_key('datapackage.json')
s3.put_object(Bucket=bucket_name, Key=read_me_key, Body='readme')
s3.put_object(Bucket=bucket_name, Key=data_key, Body='data')
s3.put_object(Bucket=bucket_name,
Key=metadata_key,
Body='metedata')
status = bit_store.delete_data_package()
read_me_res = s3.list_objects(Bucket=bucket_name,
Prefix=read_me_key)
self.assertTrue('Contents' not in read_me_res)
data_res = s3.list_objects(Bucket=bucket_name, Prefix=data_key)
self.assertTrue('Contents' not in data_res)
self.assertTrue(status)
def test_change_acl(self):
with self.app.app_context():
public_grants = {'CanonicalUser': '******', 'Group': 'READ'}
private_grants = {'CanonicalUser': '******'}
bit_store = BitStore('test_pub', 'test_package', body='test')
s3 = boto3.client('s3')
bucket_name = self.app.config['S3_BUCKET_NAME']
s3.create_bucket(Bucket=bucket_name)
metadata_key = bit_store.build_s3_key('datapackage.json')
bit_store.save_metadata()
res = s3.get_object_acl(Bucket=bucket_name, Key=metadata_key)
owner_id = res['Owner']['ID']
aws_all_user_group_url = 'http://acs.amazonaws.com/groups/global/AllUsers'
full_control = filter(
lambda grant: grant['Permission'] == 'FULL_CONTROL',
res['Grants'])
self.assertEqual(len(full_control), 1)
self.assertEqual(full_control[0].get('Grantee')['ID'], owner_id)
read_control = filter(lambda grant: grant['Permission'] == 'READ',
res['Grants'])
self.assertEqual(len(read_control), 1)
self.assertEqual(read_control[0].get('Grantee')['URI'],
aws_all_user_group_url)
# for grant in res['Grants']:
# self.assertTrue(grant['Permission'] ==
# public_grants[grant['Grantee']['Type']])
#
bit_store.change_acl("private")
res = s3.get_object_acl(Bucket=bucket_name, Key=metadata_key)
full_control = filter(
lambda grant: grant['Permission'] == 'FULL_CONTROL',
res['Grants'])
self.assertEqual(len(full_control), 1)
self.assertEqual(full_control[0].get('Grantee')['ID'], owner_id)
read_control = filter(lambda grant: grant['Permission'] == 'READ',
res['Grants'])
self.assertEqual(len(read_control), 0)
def get_resource(publisher, package, resource):
"""
DPR resource get operation.
This API is responsible for getting resource from S3.
---
tags:
- package
parameters:
- in: path
name: publisher
type: string
required: true
description: publisher name
- in: path
name: package
type: string
required: true
description: package name - to retrieve the data package metadata
- in: path
name: resource
type: string
required: true
description: resource index or name
responses:
200:
description: Get Data package for one key
schema:
id: get_data_package
properties:
data:
type: string
description: The resource
500:
description: Internal Server Error
"""
try:
path = request.path
metadata = BitStore(publisher, package)
if path.endswith('csv'):
resource_key = metadata.build_s3_key(resource + '.csv')
data = metadata.get_s3_object(resource_key)
def generate():
for row in data.splitlines():
yield row + '\n'
return Response(generate()), 200
else:
resource_key = metadata.build_s3_key(resource + '.csv')
data = metadata.get_s3_object(resource_key)
data = csv.DictReader(data.splitlines())
# taking first and adding at the end to avoid last comma
first_row = next(data)
def generate():
yield '['
for row in data:
yield json.dumps(row) + ','
yield json.dumps(first_row) + ']'
return Response(generate(), content_type='application/json'), 200
except Exception as e:
return handle_error('GENERIC_ERROR', e.message, 500)
def test_metadata_s3_key(self):
metadata = BitStore(publisher="pub_test", package="test_package")
expected = "{t}/pub_test/test_package/_v/latest/datapackage.json".\
format(t=metadata.prefix)
self.assertEqual(expected, metadata.build_s3_key('datapackage.json'))