def test_should_update_data_package_if_preexists(self):
with self.app.test_request_context():
pub = Publisher.query.filter_by(name=self.publisher_one).one()
pub.packages.append(
Package(name=self.package_two,
version='tag_one',
readme='old_readme'))
db.session.add(pub)
db.session.commit()
latest_data = Package.query.join(Publisher). \
filter(Publisher.name == self.publisher_one,
Package.name == self.package_two,
Package.version == 'latest').one()
tagged_data = Package.query.join(Publisher). \
filter(Publisher.name == self.publisher_one,
Package.name == self.package_two,
Package.version == 'tag_one').one()
self.assertNotEqual(latest_data.readme, tagged_data.readme)
Package.create_or_update_version(self.publisher_one, self.package_two,
'tag_one')
tagged_data = Package.query.join(Publisher). \
filter(Publisher.name == self.publisher_one,
Package.name == self.package_two,
Package.version == 'tag_one').one()
self.assertEqual(latest_data.readme, tagged_data.readme)
def test_data_package_page_load_without_views(self):
descriptor = {"data": [], "resources": []}
with self.app.app_context():
publisher = Publisher(name=self.publisher)
metadata = Package(name=self.package)
metadata.descriptor = json.dumps(descriptor)
publisher.packages.append(metadata)
db.session.add(publisher)
db.session.commit()
rv = self.client.get('/{publisher}/{package}'.\
format(publisher=self.publisher,
package=self.package))
self.assertNotEqual(404, rv.status_code)
self.assertIn('Data Files', rv.data.decode("utf8"))
# cheks handsontable load
self.assertIn('handsontable', rv.data.decode("utf8"))
# cheks Vega graph load
self.assertIn('vega', rv.data.decode("utf8"))
rv = self.client.get('/non-existing/demo-package')
self.assertEqual(404, rv.status_code)
# cheks handsontable not loaded
self.assertNotIn('handsontable', rv.data)
# cheks graph not loaded
self.assertNotIn('vega', rv.data)
def test_adds_local_urls(self):
descriptor = {
'name': 'test',
'resources': [{
'path': 'first.csv'
}, {
'path': 'second.csv'
}]
}
with self.app.app_context():
db.drop_all()
db.create_all()
publisher = Publisher(name=self.publisher)
metadata = Package(name=self.package)
metadata.descriptor = json.dumps(descriptor)
publisher.packages.append(metadata)
db.session.add(publisher)
db.session.commit()
response = self.client.get('/api/package/%s/%s'%\
(self.publisher, self.package))
catalog = Catalog(json.loads(response.data))
dataset = catalog.construct_dataset('http://example.com/')
self.assertEqual(dataset.\
get('resources')[0].get('localurl'),
'http://example.com/api/dataproxy/demo/demo-package/r/first.csv')
self.assertEqual(dataset.\
get('resources')[1].get('localurl'),
'http://example.com/api/dataproxy/demo/demo-package/r/second.csv')
def populate_data(publisher_name):
data = json.loads(open('fixtures/datapackage.json').read())
data_csv = open('fixtures/data/demo-resource.csv').read()
readme = open('fixtures/README.md').read()
package = Package.query.join(Publisher)\
.filter(Package.name == "demo-package",
Publisher.name == publisher_name).first()
if package:
db.session.delete(Package.query.get(package.id))
db.session.commit()
publisher = Publisher.query.filter_by(name=publisher_name).first()
metadata = Package(name="demo-package")
metadata.descriptor, metadata.status, metadata.private, metadata.readme \
= json.dumps(data), 'active', False, readme
publisher.packages.append(metadata)
db.session.add(publisher)
db.session.commit()
bitstore = BitStore(publisher_name,
package='demo-package',
body=json.dumps(data))
bitstore.save_metadata()
key = bitstore.build_s3_key('demo-resource.csv')
bucket_name = app.config['S3_BUCKET_NAME']
s3_client = app.config['S3']
s3_client.put_object(Bucket=bucket_name,
Key=key,
Body=data_csv,
ACL='public-read')
def setUp(self):
self.publisher = 'demo'
self.package = 'demo-package'
self.publisher_one = 'test_publisher1'
self.publisher_two = 'test_publisher2'
self.package_one = 'test_package1'
self.package_two = 'test_package2'
self.package_three = 'test_package3'
self.descriptor = json.loads(open('fixtures/datapackage.json').read())
self.app = create_app()
self.app.app_context().push()
with self.app.test_request_context():
db.drop_all()
db.create_all()
create_test_package(self.publisher, self.package, self.descriptor)
user1 = User(name=self.publisher_one)
publisher1 = Publisher(name=self.publisher_one)
association1 = PublisherUser(role=UserRoleEnum.owner)
association1.publisher = publisher1
user1.publishers.append(association1)
user2 = User(name=self.publisher_two)
publisher2 = Publisher(name=self.publisher_two)
association2 = PublisherUser(role=UserRoleEnum.owner)
association2.publisher = publisher2
user2.publishers.append(association2)
metadata1 = Package(name=self.package_one)
tag1 = PackageTag(descriptor=dict(name='test_one'))
metadata1.tags.append(tag1)
publisher1.packages.append(metadata1)
metadata2 = Package(name=self.package_two)
tag2 = PackageTag(descriptor=dict(name='test_two'))
metadata2.tags.append(tag2)
publisher1.packages.append(metadata2)
metadata3 = Package(name=self.package_one)
tag3 = PackageTag(descriptor=dict(name='test_three'))
metadata3.tags.append(tag3)
publisher2.packages.append(metadata3)
metadata4 = Package(name=self.package_two)
tag4 = PackageTag(descriptor=dict(name='test_four'))
metadata4.tags.append(tag4)
publisher2.packages.append(metadata4)
metadata5 = Package(name=self.package_three)
tag5 = PackageTag(descriptor=dict(name='test_four'))
metadata5.tags.append(tag5)
publisher2.packages.append(metadata5)
db.session.add(user1)
db.session.add(user2)
db.session.commit()
def setUp(self):
self.app = create_app()
self.client = self.app.test_client()
with self.app.app_context():
self.bucket_name = self.app.config['S3_BUCKET_NAME']
db.drop_all()
db.create_all()
self.user = User()
self.user.id = self.user_id
self.user.email, self.user.name, self.user.secret = \
'*****@*****.**', self.publisher_name, 'super_secret'
self.publisher = Publisher(name=self.publisher_name)
association = PublisherUser(role=UserRoleEnum.owner)
association.publisher = self.publisher
metadata = Package(name=self.package)
self.publisher.packages.append(metadata)
self.user.publishers.append(association)
self.user_not_allowed = User()
self.user_not_allowed.id = self.user_not_allowed_id
self.user_not_allowed.email, self.user_not_allowed.name, \
self.user_not_allowed.secret = \
'*****@*****.**', self.user_not_allowed_name, 'super_secret'
self.publisher_not_allowed = Publisher(
name=self.user_not_allowed_name)
association_not_allowed = PublisherUser(role=UserRoleEnum.owner)
association_not_allowed.publisher = self.publisher_not_allowed
metadata = Package(name=self.package)
self.publisher_not_allowed.packages.append(metadata)
self.user_not_allowed.publishers.append(association_not_allowed)
self.user_member = User()
self.user_member.id = self.user_member_id
self.user_member.email, self.user_member.name, self.user_member.secret = \
'*****@*****.**', self.user_member_name, 'super_secret'
association_member = PublisherUser(role=UserRoleEnum.member)
association_member.publisher = self.publisher
self.user_member.publishers.append(association_member)
db.session.add(self.user)
db.session.add(self.user_not_allowed)
db.session.commit()
response = self.client.post(self.jwt_url,
data=json.dumps({
'username': self.publisher_name,
'secret': 'super_secret'
}),
content_type='application/json')
data = json.loads(response.data)
self.jwt = data['token']
self.auth = "bearer %s" % self.jwt
def test_return_200_if_meta_data_found(self):
descriptor = {'name': 'test description'}
with self.app.app_context():
publisher = Publisher(name=self.publisher)
metadata = Package(name=self.package)
metadata.descriptor = json.dumps(descriptor)
publisher.packages.append(metadata)
db.session.add(publisher)
db.session.commit()
response = self.client.\
get('/api/package/%s/%s' % (self.publisher, self.package))
data = json.loads(response.data)
self.assertEqual(response.status_code, 200)
def test_should_populate_new_versioned_data_package(self):
Package.create_or_update_version(self.publisher_one, self.package_two,
'tag_one')
latest_data = Package.query.join(Publisher). \
filter(Publisher.name == self.publisher_one,
Package.name == self.package_two,
Package.version == 'latest').one()
tagged_data = Package.query.join(Publisher). \
filter(Publisher.name == self.publisher_one,
Package.name == self.package_two,
Package.version == 'tag_one').one()
self.assertEqual(latest_data.name, tagged_data.name)
self.assertEqual('tag_one', tagged_data.version)
def test_update_fields_if_instance_present(self):
metadata = Package.query.join(Publisher)\
.filter(Publisher.name == self.publisher_one,
Package.name == self.package_one).one()
self.assertEqual(json.loads(metadata.descriptor)['name'], "test_one")
Package.create_or_update(self.package_one,
self.publisher_one,
descriptor=json.dumps(dict(name='sub')),
private=True)
metadata = Package.query.join(Publisher) \
.filter(Publisher.name == self.publisher_one,
Package.name == self.package_one).one()
self.assertEqual(json.loads(metadata.descriptor)['name'], "sub")
self.assertEqual(metadata.private, True)
def test_return_generic_error_if_descriptor_is_not_json(self):
descriptor = 'test description'
with self.app.app_context():
publisher = Publisher(name='pub')
metadata = Package(name=self.package)
metadata.descriptor = descriptor
publisher.packages.append(metadata)
db.session.add(publisher)
db.session.commit()
response = self.client\
.get('/api/package/%s/%s' % ('pub', self.package))
data = json.loads(response.data)
self.assertEqual(response.status_code, 500)
self.assertEqual(data['error_code'], 'GENERIC_ERROR')
def test_return_empty_string_if_readme_not_there(self):
descriptor = {'name': 'test description'}
with self.app.app_context():
publisher = Publisher(name=self.publisher)
metadata = Package(name=self.package)
metadata.descriptor = json.dumps(descriptor)
publisher.packages.append(metadata)
db.session.add(publisher)
db.session.commit()
response = self.client.get('/api/package/%s/%s'%\
(self.publisher, self.package))
data = json.loads(response.data)
self.assertEqual(response.status_code, 200)
self.assertEqual(data['readme'], '')
def setUp(self):
self.publisher = 'test_publisher'
self.package1 = 'test_package1'
self.package2 = 'test_package2'
self.app = create_app()
self.client = self.app.test_client()
with self.app.app_context():
db.drop_all()
db.create_all()
publisher = Publisher(name=self.publisher)
metadata1 = Package(name=self.package1)
metadata2 = Package(name=self.package2)
publisher.packages.append(metadata1)
publisher.packages.append(metadata2)
db.session.add(publisher)
db.session.commit()
def test_get_views_if_views_dont_exist(self):
descriptor = {'name': 'test', 'resources': []}
with self.app.app_context():
db.drop_all()
db.create_all()
publisher = Publisher(name=self.publisher)
metadata = Package(name=self.package)
metadata.descriptor = json.dumps(descriptor)
publisher.packages.append(metadata)
db.session.add(publisher)
db.session.commit()
response = self.client.get('/api/package/%s/%s'%\
(self.publisher, self.package))
catalog = Catalog(json.loads(response.data))
views = catalog.get_views()
self.assertEqual(views, [])
def setUp(self):
self.app = create_app()
# self.app.app_context().push()
self.client = self.app.test_client()
with self.app.app_context():
db.drop_all()
db.create_all()
self.user = User()
self.user.id = self.user_id
self.user.email, self.user.name, self.user.secret = \
'*****@*****.**', self.publisher, 'super_secret'
publisher = Publisher(name=self.publisher)
association = PublisherUser(role=UserRoleEnum.owner)
publisher.packages.append(Package(name=self.package))
association.publisher = publisher
self.user.publishers.append(association)
db.session.add(self.user)
db.session.commit()
response = self.client.post(self.jwt_url,
data=json.dumps({
'username': self.publisher,
'secret': 'super_secret'
}),
content_type='application/json')
data = json.loads(response.data)
self.jwt = data['token']
def test_adds_empty_readme_if_there_is_not(self):
descriptor = {'name': 'test', 'resources': []}
with self.app.app_context():
db.drop_all()
db.create_all()
publisher = Publisher(name=self.publisher)
metadata = Package(name=self.package)
metadata.descriptor = json.dumps(descriptor)
publisher.packages.append(metadata)
db.session.add(publisher)
db.session.commit()
response = self.client.get('/api/package/%s/%s'%\
(self.publisher, self.package))
catalog = Catalog(json.loads(response.data))
dataset = catalog.construct_dataset()
self.assertEqual(dataset.get('readme'), '')
def setup_class(self):
self.publisher_one = 'test_publisher1'
self.publisher_two = 'test_publisher2'
self.package_one = 'test_package1'
self.package_two = 'test_package2'
self.package_three = 'test_package3'
self.app = create_app()
self.app.app_context().push()
with self.app.test_request_context():
db.drop_all()
db.create_all()
user1 = User(name=self.publisher_one)
publisher1 = Publisher(name=self.publisher_one)
association1 = PublisherUser(role=UserRoleEnum.owner)
association1.publisher = publisher1
user1.publishers.append(association1)
user2 = User(name=self.publisher_two)
publisher2 = Publisher(name=self.publisher_two)
association2 = PublisherUser(role=UserRoleEnum.owner)
association2.publisher = publisher2
user2.publishers.append(association2)
metadata1 = Package(name=self.package_one)
tag1 = PackageTag(descriptor=dict(name='test_one'))
metadata1.tags.append(tag1)
publisher1.packages.append(metadata1)
db.session.add(user1)
db.session.add(user2)
db.session.commit()
def test_insert_if_not_present(self):
pub = self.publisher_two
name = "custom_name"
metadata = Package.query.join(Publisher) \
.filter(Publisher.name == pub,
Package.name == name).all()
self.assertEqual(len(metadata), 0)
Package.create_or_update(name,
pub,
descriptor=json.dumps(dict(name='sub')),
private=True)
metadata = Package.query.join(Publisher) \
.filter(Publisher.name == pub,
Package.name == name).all()
self.assertEqual(len(metadata), 1)
def wrapped(*args, **kwargs):
entity_str, action_str = action.split("::")
user_id, instance = None, None
jwt_status, user_info = get_user_from_jwt(request,
app.config['API_KEY'])
if jwt_status:
user_id = user_info['user']
if entity_str == 'Package':
publisher_name, package_name = kwargs['publisher'], kwargs[
'package']
instance = Package.get_package(publisher_name, package_name)
elif entity_str == 'Publisher':
publisher_name = kwargs['publisher']
instance = Publisher.query.filter_by(name=publisher_name).one()
else:
return handle_error(
"INVALID_ENTITY",
"{e} is not a valid one".format(e=entity_str), 401)
status = ia(user_id, instance, action)
if not status:
return handle_error("NOT_ALLOWED",
"The operation is not allowed", 403)
return f(*args, **kwargs)
def setUp(self):
self.app = create_app()
self.app.app_context().push()
with self.app.test_request_context():
db.drop_all()
db.create_all()
self.user = User(id=11,
name=self.user_name,
secret='supersecret',
auth0_id="123|auth0")
self.publisher = Publisher(name=self.user_name)
self.publisher.packages.append(Package(name='test_package'))
association = PublisherUser(role=UserRoleEnum.owner)
association.publisher = self.publisher
self.user.publishers.append(association)
self.publisher1 = Publisher(name="test_publisher")
self.publisher1.packages.append(Package(name='test_package'))
association1 = PublisherUser(role=UserRoleEnum.member)
association1.publisher = self.publisher1
self.user.publishers.append(association1)
db.session.add(self.user)
self.sysadmin = User(id=12, name='admin', sysadmin=True)
db.session.add(self.sysadmin)
self.random_user = User(id=13, name='random')
db.session.add(self.random_user)
self.publisher2 = Publisher(name="test_publisher1", private=True)
self.publisher2.packages.append(
Package(name='test_package', private=True))
db.session.add(self.publisher2)
self.publisher3 = Publisher(name="test_publisher2", private=False)
self.publisher3.packages.append(Package(name='test_package'))
db.session.add(self.publisher3)
db.session.commit()
def test_construct_dataset(self):
descriptor = json.loads(open('fixtures/datapackage.json').read())
with self.app.app_context():
db.drop_all()
db.create_all()
publisher = Publisher(name=self.publisher)
metadata = Package(name=self.package)
metadata.descriptor = json.dumps(descriptor)
publisher.packages.append(metadata)
db.session.add(publisher)
db.session.commit()
response = self.client.get('/api/package/%s/%s'%\
(self.publisher, self.package))
catalog = Catalog(json.loads(response.data))
dataset = catalog.construct_dataset()
self.assertEqual(dataset.get('name'), descriptor.get('name'))
self.assertEqual(dataset.get('owner'), self.publisher)
self.assertIn('localurl', dataset.get('resources')[0])
self.assertNotEqual(len(dataset.get('views')), 0)
def test_return_true_if_delete_data_package_success(self):
status = Package.delete_data_package(self.publisher_one,
self.package_one)
self.assertTrue(status)
data = Package.query.join(Publisher). \
filter(Publisher.name == self.publisher_one,
Package.name == self.package_one).all()
self.assertEqual(0, len(data))
data = Publisher.query.all()
self.assertEqual(1, len(data))
def test_home_shows_packages(self):
descriptor = json.loads(open('fixtures/datapackage.json').read())
with self.app.app_context():
publisher = Publisher(name='core')
package = Package(name='gold-prices', descriptor=descriptor)
publisher.packages.append(package)
db.session.add(publisher)
db.session.commit()
rv = self.client.get('/')
self.assertEqual(rv.status_code, 200)
self.assertTrue('DEMO - CBOE Volatility Index' in rv.data)
def test_data_package_page_loads_if_descriptor_has_bad_licenses(self):
descriptor = json.loads(open('fixtures/datapackage.json').read())
descriptor['licenses'] = {'url': 'test/url', 'type': 'Test'}
with self.app.app_context():
publisher = Publisher(name=self.publisher)
package = Package(name=self.package, descriptor=descriptor)
publisher.packages.append(package)
db.session.add(publisher)
db.session.commit()
rv = self.client.get('/%s/%s' % (self.publisher, self.package))
self.assertEqual(200, rv.status_code)
def test_return_all_metadata_is_there(self):
descriptor = {'name': 'test description'}
readme = 'README'
with self.app.app_context():
publisher = Publisher(name=self.publisher)
metadata = Package(name=self.package)
metadata.descriptor = json.dumps(descriptor)
metadata.readme = readme
publisher.packages.append(metadata)
db.session.add(publisher)
db.session.commit()
response = self.client.get('/api/package/%s/%s'%\
(self.publisher, self.package))
data = json.loads(response.data)
self.assertEqual(response.status_code, 200)
self.assertEqual(len(data), 5)
self.assertEqual(data['descriptor']['name'], 'test description')
self.assertEqual(data['readme'], 'README')
self.assertEqual(data['id'], 1)
self.assertEqual(data['name'], self.package)
self.assertEqual(data['publisher'], self.publisher)
def create_test_package(publisher='demo', package='demo-package', descriptor={}, readme=''):
user = User(name=publisher, secret='supersecret', email='*****@*****.**')
publisher = Publisher(name=publisher)
association = PublisherUser(role=UserRoleEnum.owner)
association.publisher = publisher
user.publishers.append(association)
package = Package(name=package, descriptor=descriptor, readme=readme)
publisher.packages.append(package)
db.session.add(user)
db.session.commit()
def setUp(self):
self.app = create_app()
self.app.app_context().push()
with self.app.test_request_context():
db.drop_all()
db.create_all()
self.pub1_name = 'pub1'
self.pub2_name = 'pub2'
self.pub1 = Publisher(name=self.pub1_name)
self.pub2 = Publisher(name=self.pub2_name)
self.pub1.packages.append(
Package(name='pack1',
descriptor='{"title": "pack1 details one"}',
readme="Big readme one"))
self.pub1.packages.append(
Package(name='pack2',
descriptor='{"title": "pack2 details two"}',
readme="Big readme two"))
self.pub1.packages.append(
Package(name='pack3',
descriptor='{"title": "pack3 details three"}'))
self.pub2.packages.append(
Package(name='pack4',
descriptor='{"title": "pack4 details four"}'))
self.pub2.packages.append(
Package(name='pack5',
descriptor='{"title": "pack5 details five"}'))
self.pub2.packages.append(
Package(name='pack6',
descriptor='{"title": "pack6 details six"}'))
db.session.add(self.pub1)
db.session.add(self.pub2)
db.session.commit()
def undelete_data_package(publisher, package):
"""
DPR data package un-delete operation.
This API is responsible for un-mark the mark for delete of data package
---
tags:
- package
parameters:
- in: path
name: publisher
type: string
required: true
description: publisher name
- in: path
name: package
type: string
required: true
description: package name
- in: header
name: Authorization
type: string
required: true
description: >
Jwt token in format of "bearer {token}.
The token can be generated from /api/auth/token"
responses:
500:
description: Internal Server Error
200:
description: Success Message
schema:
id: put_package_success
properties:
status:
type: string
default: OK
"""
try:
bitstore = BitStore(publisher=publisher, package=package)
status_acl = bitstore.change_acl('public-read')
status_db = Package.change_status(publisher, package,
PackageStateEnum.active)
if status_acl and status_db:
return jsonify({"status": "OK"}), 200
if not status_acl:
raise Exception('Failed to change acl')
if not status_db:
raise Exception('Failed to change status')
except Exception as e:
app.logger.error(e)
return handle_error('GENERIC_ERROR', e.message, 500)
def test_change_status(self):
data = Package.query.join(Publisher). \
filter(Publisher.name == self.publisher_one,
Package.name == self.package_one).one()
self.assertEqual(PackageStateEnum.active, data.status)
Package.change_status(self.publisher_one, self.package_one,
PackageStateEnum.deleted)
data = Package.query.join(Publisher). \
filter(Publisher.name == self.publisher_one,
Package.name == self.package_one).one()
self.assertEqual(PackageStateEnum.deleted, data.status)
Package.change_status(self.publisher_one,
self.package_one,
status=PackageStateEnum.active)
data = Package.query.join(Publisher). \
filter(Publisher.name == self.publisher_one,
Package.name == self.package_one).one()
self.assertEqual(PackageStateEnum.active, data.status)
def test_should_not_visible_after_soft_delete(self):
descriptor = {'name': 'test description'}
with self.app.app_context():
publisher = Publisher(name=self.publisher)
metadata = Package(name=self.package)
metadata.tags.append(PackageTag(descriptor=descriptor))
publisher.packages.append(metadata)
db.session.add(publisher)
db.session.commit()
logic.Package.delete(self.publisher, self.package)
response = self.client. \
get('/api/package/%s/%s' % (self.publisher, self.package))
self.assertEqual(response.status_code, 404)
def test_get_views(self):
descriptor = {
'name': 'test',
'resources': [],
'views': [{
"type": "graph"
}]
}
with self.app.app_context():
db.drop_all()
db.create_all()
publisher = Publisher(name=self.publisher)
metadata = Package(name=self.package)
metadata.descriptor = json.dumps(descriptor)
publisher.packages.append(metadata)
db.session.add(publisher)
db.session.commit()
response = self.client.get('/api/package/%s/%s'%\
(self.publisher, self.package))
catalog = Catalog(json.loads(response.data))
views = catalog.get_views()
self.assertNotEqual(len(views), 0)
self.assertEqual(views[0].get('type'), 'graph')
