def test_should_copy_all_object_from_latest_to_tag(self):
numeric_version = 0.8
with self.app.app_context():
bit_store = BitStore('test_pub', 'test_package')
s3 = boto3.client('s3')
bucket_name = self.app.config['S3_BUCKET_NAME']
s3.create_bucket(Bucket=bucket_name)
read_me_key = bit_store.build_s3_key('test.md')
data_key = bit_store.build_s3_key('data.csv')
metadata_key = bit_store.build_s3_key('datapackage.json')
s3.put_object(Bucket=bucket_name, Key=read_me_key, Body='readme')
s3.put_object(Bucket=bucket_name, Key=data_key, Body='data')
s3.put_object(Bucket=bucket_name, Key=metadata_key, Body='metedata')
bit_store.copy_to_new_version(numeric_version)
bit_store_numeric = BitStore('test_pub', 'test_package',
numeric_version)
objects_nu = s3.list_objects(Bucket=bucket_name,
Prefix=bit_store_numeric
.build_s3_versioned_prefix())
objects_old = s3.list_objects(Bucket=bucket_name,
Prefix=bit_store
.build_s3_versioned_prefix())
self.assertEqual(len(objects_nu['Contents']),
len(objects_old['Contents']))
def test_save(self):
with self.app.app_context():
s3 = boto3.client('s3')
bucket_name = self.app.config['S3_BUCKET_NAME']
s3.create_bucket(Bucket=bucket_name)
metadata = BitStore(publisher="pub_test",
package="test_package",
body='hi')
key = metadata.build_s3_key('datapackage.json')
metadata.save_metadata()
obs_list = list(s3.list_objects(Bucket=bucket_name, Prefix=key). \
get('Contents'))
self.assertEqual(1, len(obs_list))
self.assertEqual(key, obs_list[0]['Key'])
res = s3.get_object_acl(Bucket=bucket_name, Key=key)
owner_id = res['Owner']['ID']
aws_all_user_group_url = 'http://acs.amazonaws.com/groups/global/AllUsers'
full_control = filter(lambda grant: grant['Permission'] == 'FULL_CONTROL', res['Grants'])
self.assertEqual(len(full_control), 1)
self.assertEqual(full_control[0].get('Grantee')['ID'], owner_id)
read_control = filter(lambda grant: grant['Permission'] == 'READ', res['Grants'])
self.assertEqual(len(read_control), 1)
self.assertEqual(read_control[0].get('Grantee')['URI'], aws_all_user_group_url)
def test_extract_information_from_s3_url(self):
metadata = BitStore(publisher="pub_test", package="test_package")
s3_key = metadata.build_s3_key("datapackage.json")
pub, package, version = BitStore.extract_information_from_s3_url(s3_key)
self.assertEqual(pub, 'pub_test')
self.assertEqual(package, 'test_package')
self.assertEqual(version, 'latest')
def test_change_acl(self):
with self.app.app_context():
public_grants = {
'CanonicalUser': '******',
'Group': 'READ'
}
private_grants = {'CanonicalUser': '******'}
bit_store = BitStore('test_pub', 'test_package', body='test')
s3 = boto3.client('s3')
bucket_name = self.app.config['S3_BUCKET_NAME']
s3.create_bucket(Bucket=bucket_name)
metadata_key = bit_store.build_s3_key('datapackage.json')
bit_store.save_metadata()
res = s3.get_object_acl(Bucket=bucket_name, Key=metadata_key)
owner_id = res['Owner']['ID']
aws_all_user_group_url = 'http://acs.amazonaws.com/groups/global/AllUsers'
full_control = filter(lambda grant: grant['Permission'] == 'FULL_CONTROL', res['Grants'])
self.assertEqual(len(full_control), 1)
self.assertEqual(full_control[0].get('Grantee')['ID'], owner_id)
read_control = filter(lambda grant: grant['Permission'] == 'READ', res['Grants'])
self.assertEqual(len(read_control), 1)
self.assertEqual(read_control[0].get('Grantee')['URI'], aws_all_user_group_url)
bit_store.change_acl("private")
res = s3.get_object_acl(Bucket=bucket_name, Key=metadata_key)
full_control = filter(lambda grant: grant['Permission'] == 'FULL_CONTROL', res['Grants'])
self.assertEqual(len(full_control), 1)
self.assertEqual(full_control[0].get('Grantee')['ID'], owner_id)
read_control = filter(lambda grant: grant['Permission'] == 'READ', res['Grants'])
self.assertEqual(len(read_control), 0)
def populate_data(publisher_name):
data = json.loads(open('fixtures/datapackage.json').read())
data_csv = open('fixtures/data/demo-resource.csv').read()
readme = open('fixtures/README.md').read()
package = models.Package.query.join(models.Publisher)\
.filter(models.Package.name == "demo-package",
models.Publisher.name == publisher_name).first()
if package:
db.session.delete(models.Package.query.get(package.id))
db.session.commit()
publisher = models.Publisher.query.filter_by(name=publisher_name).first()
metadata = models.Package(name="demo-package",
descriptor=data,
readme=readme)
metadata.status, metadata.private \
= 'active', False
publisher.packages.append(metadata)
db.session.add(publisher)
db.session.commit()
bitstore = BitStore(publisher_name,
package='demo-package',
body=json.dumps(data))
bitstore.save_metadata()
key = bitstore.build_s3_key('demo-resource.csv')
bucket_name = app.config['S3_BUCKET_NAME']
s3_client = app.config['S3']
s3_client.put_object(Bucket=bucket_name,
Key=key,
Body=data_csv,
ACL='public-read')
def test_return_none_if_no_readme_found(self):
with self.app.app_context():
bit_store = BitStore('test_pub', 'test_package')
s3 = boto3.client('s3')
bucket_name = self.app.config['S3_BUCKET_NAME']
s3.create_bucket(Bucket=bucket_name)
read_me_key = bit_store.build_s3_key('test.md')
s3.put_object(Bucket=bucket_name, Key=read_me_key, Body='')
self.assertEqual(bit_store.get_readme_object_key(), 'None')
def test_delete_data_package(self):
with self.app.app_context():
bit_store = BitStore('test_pub', 'test_package')
s3 = boto3.client('s3')
bucket_name = self.app.config['S3_BUCKET_NAME']
s3.create_bucket(Bucket=bucket_name)
read_me_key = bit_store.build_s3_key('test.md')
data_key = bit_store.build_s3_key('data.csv')
metadata_key = bit_store.build_s3_key('datapackage.json')
s3.put_object(Bucket=bucket_name, Key=read_me_key, Body='readme')
s3.put_object(Bucket=bucket_name, Key=data_key, Body='data')
s3.put_object(Bucket=bucket_name, Key=metadata_key, Body='metedata')
status = bit_store.delete_data_package()
read_me_res = s3.list_objects(Bucket=bucket_name, Prefix=read_me_key)
self.assertTrue('Contents' not in read_me_res)
data_res = s3.list_objects(Bucket=bucket_name, Prefix=data_key)
self.assertTrue('Contents' not in data_res)
self.assertTrue(status)
def test_get_metadata_body(self):
with self.app.app_context():
s3 = boto3.client('s3')
bucket_name = self.app.config['S3_BUCKET_NAME']
s3.create_bucket(Bucket=bucket_name)
metadata = BitStore(publisher="pub_test",
package="test_package",
body='hi')
s3.put_object(Bucket=bucket_name,
Key=metadata.build_s3_key('datapackage.json'),
Body=metadata.body)
self.assertEqual(metadata.body, metadata.get_metadata_body())
def test_throw_403_if_not_owner_or_member_of_publisher(self):
s3 = boto3.client('s3')
s3.create_bucket(Bucket=self.bucket_name)
bit_store = BitStore('test_pub', 'test_package')
read_me_key = bit_store.build_s3_key('test.md')
data_key = bit_store.build_s3_key('data.csv')
metadata_key = bit_store.build_s3_key('datapackage.json')
s3.put_object(Bucket=self.bucket_name, Key=read_me_key, Body='readme')
s3.put_object(Bucket=self.bucket_name, Key=data_key, Body='data')
s3.put_object(Bucket=self.bucket_name, Key=metadata_key, Body='metedata')
response = self.client.post(self.jwt_url,
data=json.dumps({
'username': self.user_not_allowed_name,
'secret': 'super_secret'
}),
content_type='application/json')
data = json.loads(response.data)
jwt_not_allowed = data['token']
auth_not_allowed = "%s" % jwt_not_allowed
response = self.client.post(self.url,
data=json.dumps({
'version': 'tag_one'
}),
content_type='application/json',
headers=dict(Authorization=auth_not_allowed))
self.assertEqual(response.status_code, 403)
with self.app.app_context():
data_latest = Package.query.join(Publisher). \
filter(Publisher.name == self.publisher_name,
Package.name == self.package).all()
self.assertEqual(1, len(data_latest))
bit_store_tagged = BitStore('test_pub', 'test_package',
'tag_one')
objects_nu = s3.list_objects(Bucket=self.bucket_name,
Prefix=bit_store_tagged
.build_s3_versioned_prefix())
self.assertTrue('Contents' not in objects_nu)
def test_metadata_s3_key(self):
metadata = BitStore(publisher="pub_test", package="test_package")
expected = "{t}/pub_test/test_package/_v/latest/datapackage.json". \
format(t=metadata.prefix)
self.assertEqual(expected, metadata.build_s3_key('datapackage.json'))
