def test_save(self):
with self.app.app_context():
s3 = boto3.client('s3')
bucket_name = self.app.config['S3_BUCKET_NAME']
s3.create_bucket(Bucket=bucket_name)
metadata = BitStore(publisher="pub_test",
package="test_package",
body='hi')
key = metadata.build_s3_key('datapackage.json')
metadata.save_metadata()
obs_list = list(s3.list_objects(Bucket=bucket_name, Prefix=key). \
get('Contents'))
self.assertEqual(1, len(obs_list))
self.assertEqual(key, obs_list[0]['Key'])
res = s3.get_object_acl(Bucket=bucket_name, Key=key)
owner_id = res['Owner']['ID']
aws_all_user_group_url = 'http://acs.amazonaws.com/groups/global/AllUsers'
full_control = filter(lambda grant: grant['Permission'] == 'FULL_CONTROL', res['Grants'])
self.assertEqual(len(full_control), 1)
self.assertEqual(full_control[0].get('Grantee')['ID'], owner_id)
read_control = filter(lambda grant: grant['Permission'] == 'READ', res['Grants'])
self.assertEqual(len(read_control), 1)
self.assertEqual(read_control[0].get('Grantee')['URI'], aws_all_user_group_url)
def test_extract_information_from_s3_url(self):
metadata = BitStore(publisher="pub_test", package="test_package")
s3_key = metadata.build_s3_key("datapackage.json")
pub, package, version = BitStore.extract_information_from_s3_url(s3_key)
self.assertEqual(pub, 'pub_test')
self.assertEqual(package, 'test_package')
self.assertEqual(version, 'latest')
def populate_data(publisher_name):
data = json.loads(open('fixtures/datapackage.json').read())
data_csv = open('fixtures/data/demo-resource.csv').read()
readme = open('fixtures/README.md').read()
package = models.Package.query.join(models.Publisher)\
.filter(models.Package.name == "demo-package",
models.Publisher.name == publisher_name).first()
if package:
db.session.delete(models.Package.query.get(package.id))
db.session.commit()
publisher = models.Publisher.query.filter_by(name=publisher_name).first()
metadata = models.Package(name="demo-package",
descriptor=data,
readme=readme)
metadata.status, metadata.private \
= 'active', False
publisher.packages.append(metadata)
db.session.add(publisher)
db.session.commit()
bitstore = BitStore(publisher_name,
package='demo-package',
body=json.dumps(data))
bitstore.save_metadata()
key = bitstore.build_s3_key('demo-resource.csv')
bucket_name = app.config['S3_BUCKET_NAME']
s3_client = app.config['S3']
s3_client.put_object(Bucket=bucket_name,
Key=key,
Body=data_csv,
ACL='public-read')
def test_return_none_if_no_readme_found(self):
with self.app.app_context():
bit_store = BitStore('test_pub', 'test_package')
s3 = boto3.client('s3')
bucket_name = self.app.config['S3_BUCKET_NAME']
s3.create_bucket(Bucket=bucket_name)
read_me_key = bit_store.build_s3_key('test.md')
s3.put_object(Bucket=bucket_name, Key=read_me_key, Body='')
self.assertEqual(bit_store.get_readme_object_key(), 'None')
def test_get_empty_metadata_name_for_publisher(self):
with self.app.app_context():
s3 = boto3.client('s3')
bucket_name = self.app.config['S3_BUCKET_NAME']
s3.create_bucket(Bucket=bucket_name)
metadata = BitStore(publisher="pub_test",
package="test_package",
body='hi')
s3.put_object(Bucket=bucket_name,
Key='test/key.json',
Body=metadata.body)
self.assertEqual(0, len(metadata.get_all_metadata_name_for_publisher()))
def test_get_metadata_body(self):
with self.app.app_context():
s3 = boto3.client('s3')
bucket_name = self.app.config['S3_BUCKET_NAME']
s3.create_bucket(Bucket=bucket_name)
metadata = BitStore(publisher="pub_test",
package="test_package",
body='hi')
s3.put_object(Bucket=bucket_name,
Key=metadata.build_s3_key('datapackage.json'),
Body=metadata.body)
self.assertEqual(metadata.body, metadata.get_metadata_body())
def test_should_copy_all_object_from_latest_to_tag(self):
numeric_version = 0.8
with self.app.app_context():
bit_store = BitStore('test_pub', 'test_package')
s3 = boto3.client('s3')
bucket_name = self.app.config['S3_BUCKET_NAME']
s3.create_bucket(Bucket=bucket_name)
read_me_key = bit_store.build_s3_key('test.md')
data_key = bit_store.build_s3_key('data.csv')
metadata_key = bit_store.build_s3_key('datapackage.json')
s3.put_object(Bucket=bucket_name, Key=read_me_key, Body='readme')
s3.put_object(Bucket=bucket_name, Key=data_key, Body='data')
s3.put_object(Bucket=bucket_name, Key=metadata_key, Body='metedata')
bit_store.copy_to_new_version(numeric_version)
bit_store_numeric = BitStore('test_pub', 'test_package',
numeric_version)
objects_nu = s3.list_objects(Bucket=bucket_name,
Prefix=bit_store_numeric
.build_s3_versioned_prefix())
objects_old = s3.list_objects(Bucket=bucket_name,
Prefix=bit_store
.build_s3_versioned_prefix())
self.assertEqual(len(objects_nu['Contents']),
len(objects_old['Contents']))
def finalize_publish(cls, user_id, datapackage_url):
'''
Gets the datapackage.json and README from S3 and imports into database.
Returns status "queued" if ok, else - None
'''
publisher, package, version = BitStore.extract_information_from_s3_url(
datapackage_url)
if Package.exists(publisher, package):
status = check_is_authorized('Package::Update', publisher, package,
user_id)
else:
status = check_is_authorized('Package::Create', publisher, package,
user_id)
if not status:
raise InvalidUsage('Not authorized to upload data', 400)
bit_store = BitStore(publisher, package)
b = bit_store.get_metadata_body()
body = json.loads(b)
bit_store.change_acl('public-read')
readme = bit_store.get_s3_object(bit_store.get_readme_object_key())
Package.create_or_update(name=package,
publisher_name=publisher,
descriptor=body,
readme=readme)
return "queued"
def test_generate_pre_signed_put_obj_url(self):
with self.app.app_context():
s3 = boto3.client('s3')
bucket_name = self.app.config['S3_BUCKET_NAME']
s3.create_bucket(Bucket=bucket_name)
metadata = BitStore(publisher="pub_test",
package="test_package",
body='hi')
post = metadata.generate_pre_signed_post_object('datapackage.json',
123)
parsed = urlparse(post['url'])
self.assertEqual(parsed.netloc,
's3-{region}.amazonaws.com'.
format(region=self.app.config['AWS_REGION']))
self.assertEqual('public-read', post['fields']['acl'])
self.assertEqual('text/plain', post['fields']['Content-Type'])
def undelete_data_package(publisher, package):
"""
DPR data package un-delete operation.
This API is responsible for un-mark the mark for delete of data package
---
tags:
- package
parameters:
- in: path
name: publisher
type: string
required: true
description: publisher name
- in: path
name: package
type: string
required: true
description: package name
- in: header
name: Authorization
type: string
required: true
description: JWT Token
responses:
500:
description: Internal Server Error
200:
description: Success Message
schema:
id: put_package_success
properties:
status:
type: string
default: OK
"""
bitstore = BitStore(publisher=publisher, package=package)
status_db = logic.Package.change_status(publisher, package,
models.PackageStateEnum.active)
try:
status_acl = bitstore.change_acl('public-read')
except Exception as e:
## TODO roll back changes in db
raise InvalidUsage(e.message, 500)
if status_acl and status_db:
return jsonify({"status": "OK"}), 200
def test_change_acl(self):
with self.app.app_context():
public_grants = {
'CanonicalUser': '******',
'Group': 'READ'
}
private_grants = {'CanonicalUser': '******'}
bit_store = BitStore('test_pub', 'test_package', body='test')
s3 = boto3.client('s3')
bucket_name = self.app.config['S3_BUCKET_NAME']
s3.create_bucket(Bucket=bucket_name)
metadata_key = bit_store.build_s3_key('datapackage.json')
bit_store.save_metadata()
res = s3.get_object_acl(Bucket=bucket_name, Key=metadata_key)
owner_id = res['Owner']['ID']
aws_all_user_group_url = 'http://acs.amazonaws.com/groups/global/AllUsers'
full_control = filter(lambda grant: grant['Permission'] == 'FULL_CONTROL', res['Grants'])
self.assertEqual(len(full_control), 1)
self.assertEqual(full_control[0].get('Grantee')['ID'], owner_id)
read_control = filter(lambda grant: grant['Permission'] == 'READ', res['Grants'])
self.assertEqual(len(read_control), 1)
self.assertEqual(read_control[0].get('Grantee')['URI'], aws_all_user_group_url)
bit_store.change_acl("private")
res = s3.get_object_acl(Bucket=bucket_name, Key=metadata_key)
full_control = filter(lambda grant: grant['Permission'] == 'FULL_CONTROL', res['Grants'])
self.assertEqual(len(full_control), 1)
self.assertEqual(full_control[0].get('Grantee')['ID'], owner_id)
read_control = filter(lambda grant: grant['Permission'] == 'READ', res['Grants'])
self.assertEqual(len(read_control), 0)
def delete_data_package(publisher, package):
"""
DPR Data Package Soft Delete
Marks Data Package as private
---
tags:
- package
parameters:
- in: path
name: publisher
type: string
required: true
description: publisher name
- in: path
name: package
type: string
required: true
description: package name
- in: header
name: Authorization
type: string
required: true
description: JWT Token
responses:
500:
description: Internal Server Error
200:
description: Success Message
schema:
id: put_package_success
properties:
status:
type: string
default: OK
"""
bitstore = BitStore(publisher=publisher, package=package)
status_db = logic.Package.change_status(publisher, package,
models.PackageStateEnum.deleted)
try:
status_acl = bitstore.change_acl('private')
except Exception as e:
## TODO roll back changes in db
raise InvalidUsage(e.message, 500)
if status_acl and status_db:
return jsonify({"status": "OK"}), 200
def test_throw_403_if_not_owner_or_member_of_publisher(self):
s3 = boto3.client('s3')
s3.create_bucket(Bucket=self.bucket_name)
bit_store = BitStore('test_pub', 'test_package')
read_me_key = bit_store.build_s3_key('test.md')
data_key = bit_store.build_s3_key('data.csv')
metadata_key = bit_store.build_s3_key('datapackage.json')
s3.put_object(Bucket=self.bucket_name, Key=read_me_key, Body='readme')
s3.put_object(Bucket=self.bucket_name, Key=data_key, Body='data')
s3.put_object(Bucket=self.bucket_name,
Key=metadata_key,
Body='metedata')
response = self.client.post(self.jwt_url,
data=json.dumps({
'username': self.user_not_allowed_name,
'secret': 'super_secret'
}),
content_type='application/json')
data = json.loads(response.data)
jwt_not_allowed = data['token']
auth_not_allowed = "%s" % jwt_not_allowed
response = self.client.post(
self.url,
data=json.dumps({'version': 'tag_one'}),
content_type='application/json',
headers=dict(Authorization=auth_not_allowed))
self.assertEqual(response.status_code, 403)
with self.app.app_context():
data_latest = Package.query.join(Publisher). \
filter(Publisher.name == self.publisher_name,
Package.name == self.package).all()
self.assertEqual(1, len(data_latest))
bit_store_tagged = BitStore('test_pub', 'test_package', 'tag_one')
objects_nu = s3.list_objects(
Bucket=self.bucket_name,
Prefix=bit_store_tagged.build_s3_versioned_prefix())
self.assertTrue('Contents' not in objects_nu)
def test_delete_data_package(self):
with self.app.app_context():
bit_store = BitStore('test_pub', 'test_package')
s3 = boto3.client('s3')
bucket_name = self.app.config['S3_BUCKET_NAME']
s3.create_bucket(Bucket=bucket_name)
read_me_key = bit_store.build_s3_key('test.md')
data_key = bit_store.build_s3_key('data.csv')
metadata_key = bit_store.build_s3_key('datapackage.json')
s3.put_object(Bucket=bucket_name, Key=read_me_key, Body='readme')
s3.put_object(Bucket=bucket_name, Key=data_key, Body='data')
s3.put_object(Bucket=bucket_name, Key=metadata_key, Body='metedata')
status = bit_store.delete_data_package()
read_me_res = s3.list_objects(Bucket=bucket_name, Prefix=read_me_key)
self.assertTrue('Contents' not in read_me_res)
data_res = s3.list_objects(Bucket=bucket_name, Prefix=data_key)
self.assertTrue('Contents' not in data_res)
self.assertTrue(status)
def test_metadata_s3_key(self):
metadata = BitStore(publisher="pub_test", package="test_package")
expected = "{t}/pub_test/test_package/_v/latest/datapackage.json". \
format(t=metadata.prefix)
self.assertEqual(expected, metadata.build_s3_key('datapackage.json'))
def tag_data_package(publisher, package):
"""
DPR metadata put operation.
This API is responsible for tagging data package
---
tags:
- package
parameters:
- in: path
name: publisher
type: string
required: true
description: publisher name
- in: path
name: package
type: string
required: true
description: package name
- in: body
name: version
type: string
required: true
description: version value
- in: header
name: Authorization
type: string
required: true
description: JWT Token
responses:
400:
description: JWT is invalid or req body is not valid
401:
description: Invalid Header for JWT
403:
description: User not allowed for operation
404:
description: User not found
500:
description: Internal Server Error
200:
description: Success Message
schema:
id: put_package_success
properties:
status:
type: string
description: Status of the operation
default: OK
"""
data = request.get_json()
if 'version' not in data:
raise InvalidUsage('version not found', 400)
bitstore = BitStore(publisher, package)
status_db = logic.Package.create_or_update_tag(publisher, package,
data['version'])
try:
status_bitstore = bitstore.copy_to_new_version(data['version'])
except Exception as e:
## TODO roll back changes in db
raise InvalidUsage(e.message, 500)
return jsonify({"status": "OK"}), 200
def get_url(self, data):
bitstore = BitStore(data.publisher.name, data.name)
datapackage_json_url_in_s3 = bitstore.build_s3_object_url()
return datapackage_json_url_in_s3
def __init__(self, package_name, publisher, relative_path, props):
self.package_name = package_name
self.publisher = publisher
self.relative_path = relative_path
self.props = props
self.bitstore = BitStore(publisher=publisher, package=package_name)
def test_metadata_s3_prefix(self):
metadata = BitStore(publisher="pub_test", package="test_package")
expected = "{t}/pub_test/test_package".format(t=metadata.prefix)
self.assertEqual(expected, metadata.build_s3_base_prefix())
