def test_should_update_data_package_if_preexists(self): with self.app.test_request_context(): pub = Publisher.query.filter_by(name=self.publisher_one).one() pub.packages.append( Package(name=self.package_two, version='tag_one', readme='old_readme')) db.session.add(pub) db.session.commit() latest_data = Package.query.join(Publisher). \ filter(Publisher.name == self.publisher_one, Package.name == self.package_two, Package.version == 'latest').one() tagged_data = Package.query.join(Publisher). \ filter(Publisher.name == self.publisher_one, Package.name == self.package_two, Package.version == 'tag_one').one() self.assertNotEqual(latest_data.readme, tagged_data.readme) Package.create_or_update_version(self.publisher_one, self.package_two, 'tag_one') tagged_data = Package.query.join(Publisher). \ filter(Publisher.name == self.publisher_one, Package.name == self.package_two, Package.version == 'tag_one').one() self.assertEqual(latest_data.readme, tagged_data.readme)
def test_data_package_page_load_without_views(self): descriptor = {"data": [], "resources": []} with self.app.app_context(): publisher = Publisher(name=self.publisher) metadata = Package(name=self.package) metadata.descriptor = json.dumps(descriptor) publisher.packages.append(metadata) db.session.add(publisher) db.session.commit() rv = self.client.get('/{publisher}/{package}'.\ format(publisher=self.publisher, package=self.package)) self.assertNotEqual(404, rv.status_code) self.assertIn('Data Files', rv.data.decode("utf8")) # cheks handsontable load self.assertIn('handsontable', rv.data.decode("utf8")) # cheks Vega graph load self.assertIn('vega', rv.data.decode("utf8")) rv = self.client.get('/non-existing/demo-package') self.assertEqual(404, rv.status_code) # cheks handsontable not loaded self.assertNotIn('handsontable', rv.data) # cheks graph not loaded self.assertNotIn('vega', rv.data)
def test_adds_local_urls(self): descriptor = { 'name': 'test', 'resources': [{ 'path': 'first.csv' }, { 'path': 'second.csv' }] } with self.app.app_context(): db.drop_all() db.create_all() publisher = Publisher(name=self.publisher) metadata = Package(name=self.package) metadata.descriptor = json.dumps(descriptor) publisher.packages.append(metadata) db.session.add(publisher) db.session.commit() response = self.client.get('/api/package/%s/%s'%\ (self.publisher, self.package)) catalog = Catalog(json.loads(response.data)) dataset = catalog.construct_dataset('http://example.com/') self.assertEqual(dataset.\ get('resources')[0].get('localurl'), 'http://example.com/api/dataproxy/demo/demo-package/r/first.csv') self.assertEqual(dataset.\ get('resources')[1].get('localurl'), 'http://example.com/api/dataproxy/demo/demo-package/r/second.csv')
def populate_data(publisher_name): data = json.loads(open('fixtures/datapackage.json').read()) data_csv = open('fixtures/data/demo-resource.csv').read() readme = open('fixtures/README.md').read() package = Package.query.join(Publisher)\ .filter(Package.name == "demo-package", Publisher.name == publisher_name).first() if package: db.session.delete(Package.query.get(package.id)) db.session.commit() publisher = Publisher.query.filter_by(name=publisher_name).first() metadata = Package(name="demo-package") metadata.descriptor, metadata.status, metadata.private, metadata.readme \ = json.dumps(data), 'active', False, readme publisher.packages.append(metadata) db.session.add(publisher) db.session.commit() bitstore = BitStore(publisher_name, package='demo-package', body=json.dumps(data)) bitstore.save_metadata() key = bitstore.build_s3_key('demo-resource.csv') bucket_name = app.config['S3_BUCKET_NAME'] s3_client = app.config['S3'] s3_client.put_object(Bucket=bucket_name, Key=key, Body=data_csv, ACL='public-read')
def setUp(self): self.publisher = 'demo' self.package = 'demo-package' self.publisher_one = 'test_publisher1' self.publisher_two = 'test_publisher2' self.package_one = 'test_package1' self.package_two = 'test_package2' self.package_three = 'test_package3' self.descriptor = json.loads(open('fixtures/datapackage.json').read()) self.app = create_app() self.app.app_context().push() with self.app.test_request_context(): db.drop_all() db.create_all() create_test_package(self.publisher, self.package, self.descriptor) user1 = User(name=self.publisher_one) publisher1 = Publisher(name=self.publisher_one) association1 = PublisherUser(role=UserRoleEnum.owner) association1.publisher = publisher1 user1.publishers.append(association1) user2 = User(name=self.publisher_two) publisher2 = Publisher(name=self.publisher_two) association2 = PublisherUser(role=UserRoleEnum.owner) association2.publisher = publisher2 user2.publishers.append(association2) metadata1 = Package(name=self.package_one) tag1 = PackageTag(descriptor=dict(name='test_one')) metadata1.tags.append(tag1) publisher1.packages.append(metadata1) metadata2 = Package(name=self.package_two) tag2 = PackageTag(descriptor=dict(name='test_two')) metadata2.tags.append(tag2) publisher1.packages.append(metadata2) metadata3 = Package(name=self.package_one) tag3 = PackageTag(descriptor=dict(name='test_three')) metadata3.tags.append(tag3) publisher2.packages.append(metadata3) metadata4 = Package(name=self.package_two) tag4 = PackageTag(descriptor=dict(name='test_four')) metadata4.tags.append(tag4) publisher2.packages.append(metadata4) metadata5 = Package(name=self.package_three) tag5 = PackageTag(descriptor=dict(name='test_four')) metadata5.tags.append(tag5) publisher2.packages.append(metadata5) db.session.add(user1) db.session.add(user2) db.session.commit()
def setUp(self): self.app = create_app() self.client = self.app.test_client() with self.app.app_context(): self.bucket_name = self.app.config['S3_BUCKET_NAME'] db.drop_all() db.create_all() self.user = User() self.user.id = self.user_id self.user.email, self.user.name, self.user.secret = \ '*****@*****.**', self.publisher_name, 'super_secret' self.publisher = Publisher(name=self.publisher_name) association = PublisherUser(role=UserRoleEnum.owner) association.publisher = self.publisher metadata = Package(name=self.package) self.publisher.packages.append(metadata) self.user.publishers.append(association) self.user_not_allowed = User() self.user_not_allowed.id = self.user_not_allowed_id self.user_not_allowed.email, self.user_not_allowed.name, \ self.user_not_allowed.secret = \ '*****@*****.**', self.user_not_allowed_name, 'super_secret' self.publisher_not_allowed = Publisher( name=self.user_not_allowed_name) association_not_allowed = PublisherUser(role=UserRoleEnum.owner) association_not_allowed.publisher = self.publisher_not_allowed metadata = Package(name=self.package) self.publisher_not_allowed.packages.append(metadata) self.user_not_allowed.publishers.append(association_not_allowed) self.user_member = User() self.user_member.id = self.user_member_id self.user_member.email, self.user_member.name, self.user_member.secret = \ '*****@*****.**', self.user_member_name, 'super_secret' association_member = PublisherUser(role=UserRoleEnum.member) association_member.publisher = self.publisher self.user_member.publishers.append(association_member) db.session.add(self.user) db.session.add(self.user_not_allowed) db.session.commit() response = self.client.post(self.jwt_url, data=json.dumps({ 'username': self.publisher_name, 'secret': 'super_secret' }), content_type='application/json') data = json.loads(response.data) self.jwt = data['token'] self.auth = "bearer %s" % self.jwt
def test_return_200_if_meta_data_found(self): descriptor = {'name': 'test description'} with self.app.app_context(): publisher = Publisher(name=self.publisher) metadata = Package(name=self.package) metadata.descriptor = json.dumps(descriptor) publisher.packages.append(metadata) db.session.add(publisher) db.session.commit() response = self.client.\ get('/api/package/%s/%s' % (self.publisher, self.package)) data = json.loads(response.data) self.assertEqual(response.status_code, 200)
def test_should_populate_new_versioned_data_package(self): Package.create_or_update_version(self.publisher_one, self.package_two, 'tag_one') latest_data = Package.query.join(Publisher). \ filter(Publisher.name == self.publisher_one, Package.name == self.package_two, Package.version == 'latest').one() tagged_data = Package.query.join(Publisher). \ filter(Publisher.name == self.publisher_one, Package.name == self.package_two, Package.version == 'tag_one').one() self.assertEqual(latest_data.name, tagged_data.name) self.assertEqual('tag_one', tagged_data.version)
def test_update_fields_if_instance_present(self): metadata = Package.query.join(Publisher)\ .filter(Publisher.name == self.publisher_one, Package.name == self.package_one).one() self.assertEqual(json.loads(metadata.descriptor)['name'], "test_one") Package.create_or_update(self.package_one, self.publisher_one, descriptor=json.dumps(dict(name='sub')), private=True) metadata = Package.query.join(Publisher) \ .filter(Publisher.name == self.publisher_one, Package.name == self.package_one).one() self.assertEqual(json.loads(metadata.descriptor)['name'], "sub") self.assertEqual(metadata.private, True)
def test_return_generic_error_if_descriptor_is_not_json(self): descriptor = 'test description' with self.app.app_context(): publisher = Publisher(name='pub') metadata = Package(name=self.package) metadata.descriptor = descriptor publisher.packages.append(metadata) db.session.add(publisher) db.session.commit() response = self.client\ .get('/api/package/%s/%s' % ('pub', self.package)) data = json.loads(response.data) self.assertEqual(response.status_code, 500) self.assertEqual(data['error_code'], 'GENERIC_ERROR')
def test_return_empty_string_if_readme_not_there(self): descriptor = {'name': 'test description'} with self.app.app_context(): publisher = Publisher(name=self.publisher) metadata = Package(name=self.package) metadata.descriptor = json.dumps(descriptor) publisher.packages.append(metadata) db.session.add(publisher) db.session.commit() response = self.client.get('/api/package/%s/%s'%\ (self.publisher, self.package)) data = json.loads(response.data) self.assertEqual(response.status_code, 200) self.assertEqual(data['readme'], '')
def setUp(self): self.publisher = 'test_publisher' self.package1 = 'test_package1' self.package2 = 'test_package2' self.app = create_app() self.client = self.app.test_client() with self.app.app_context(): db.drop_all() db.create_all() publisher = Publisher(name=self.publisher) metadata1 = Package(name=self.package1) metadata2 = Package(name=self.package2) publisher.packages.append(metadata1) publisher.packages.append(metadata2) db.session.add(publisher) db.session.commit()
def test_get_views_if_views_dont_exist(self): descriptor = {'name': 'test', 'resources': []} with self.app.app_context(): db.drop_all() db.create_all() publisher = Publisher(name=self.publisher) metadata = Package(name=self.package) metadata.descriptor = json.dumps(descriptor) publisher.packages.append(metadata) db.session.add(publisher) db.session.commit() response = self.client.get('/api/package/%s/%s'%\ (self.publisher, self.package)) catalog = Catalog(json.loads(response.data)) views = catalog.get_views() self.assertEqual(views, [])
def setUp(self): self.app = create_app() # self.app.app_context().push() self.client = self.app.test_client() with self.app.app_context(): db.drop_all() db.create_all() self.user = User() self.user.id = self.user_id self.user.email, self.user.name, self.user.secret = \ '*****@*****.**', self.publisher, 'super_secret' publisher = Publisher(name=self.publisher) association = PublisherUser(role=UserRoleEnum.owner) publisher.packages.append(Package(name=self.package)) association.publisher = publisher self.user.publishers.append(association) db.session.add(self.user) db.session.commit() response = self.client.post(self.jwt_url, data=json.dumps({ 'username': self.publisher, 'secret': 'super_secret' }), content_type='application/json') data = json.loads(response.data) self.jwt = data['token']
def test_adds_empty_readme_if_there_is_not(self): descriptor = {'name': 'test', 'resources': []} with self.app.app_context(): db.drop_all() db.create_all() publisher = Publisher(name=self.publisher) metadata = Package(name=self.package) metadata.descriptor = json.dumps(descriptor) publisher.packages.append(metadata) db.session.add(publisher) db.session.commit() response = self.client.get('/api/package/%s/%s'%\ (self.publisher, self.package)) catalog = Catalog(json.loads(response.data)) dataset = catalog.construct_dataset() self.assertEqual(dataset.get('readme'), '')
def setup_class(self): self.publisher_one = 'test_publisher1' self.publisher_two = 'test_publisher2' self.package_one = 'test_package1' self.package_two = 'test_package2' self.package_three = 'test_package3' self.app = create_app() self.app.app_context().push() with self.app.test_request_context(): db.drop_all() db.create_all() user1 = User(name=self.publisher_one) publisher1 = Publisher(name=self.publisher_one) association1 = PublisherUser(role=UserRoleEnum.owner) association1.publisher = publisher1 user1.publishers.append(association1) user2 = User(name=self.publisher_two) publisher2 = Publisher(name=self.publisher_two) association2 = PublisherUser(role=UserRoleEnum.owner) association2.publisher = publisher2 user2.publishers.append(association2) metadata1 = Package(name=self.package_one) tag1 = PackageTag(descriptor=dict(name='test_one')) metadata1.tags.append(tag1) publisher1.packages.append(metadata1) db.session.add(user1) db.session.add(user2) db.session.commit()
def test_insert_if_not_present(self): pub = self.publisher_two name = "custom_name" metadata = Package.query.join(Publisher) \ .filter(Publisher.name == pub, Package.name == name).all() self.assertEqual(len(metadata), 0) Package.create_or_update(name, pub, descriptor=json.dumps(dict(name='sub')), private=True) metadata = Package.query.join(Publisher) \ .filter(Publisher.name == pub, Package.name == name).all() self.assertEqual(len(metadata), 1)
def wrapped(*args, **kwargs): entity_str, action_str = action.split("::") user_id, instance = None, None jwt_status, user_info = get_user_from_jwt(request, app.config['API_KEY']) if jwt_status: user_id = user_info['user'] if entity_str == 'Package': publisher_name, package_name = kwargs['publisher'], kwargs[ 'package'] instance = Package.get_package(publisher_name, package_name) elif entity_str == 'Publisher': publisher_name = kwargs['publisher'] instance = Publisher.query.filter_by(name=publisher_name).one() else: return handle_error( "INVALID_ENTITY", "{e} is not a valid one".format(e=entity_str), 401) status = ia(user_id, instance, action) if not status: return handle_error("NOT_ALLOWED", "The operation is not allowed", 403) return f(*args, **kwargs)
def setUp(self): self.app = create_app() self.app.app_context().push() with self.app.test_request_context(): db.drop_all() db.create_all() self.user = User(id=11, name=self.user_name, secret='supersecret', auth0_id="123|auth0") self.publisher = Publisher(name=self.user_name) self.publisher.packages.append(Package(name='test_package')) association = PublisherUser(role=UserRoleEnum.owner) association.publisher = self.publisher self.user.publishers.append(association) self.publisher1 = Publisher(name="test_publisher") self.publisher1.packages.append(Package(name='test_package')) association1 = PublisherUser(role=UserRoleEnum.member) association1.publisher = self.publisher1 self.user.publishers.append(association1) db.session.add(self.user) self.sysadmin = User(id=12, name='admin', sysadmin=True) db.session.add(self.sysadmin) self.random_user = User(id=13, name='random') db.session.add(self.random_user) self.publisher2 = Publisher(name="test_publisher1", private=True) self.publisher2.packages.append( Package(name='test_package', private=True)) db.session.add(self.publisher2) self.publisher3 = Publisher(name="test_publisher2", private=False) self.publisher3.packages.append(Package(name='test_package')) db.session.add(self.publisher3) db.session.commit()
def test_construct_dataset(self): descriptor = json.loads(open('fixtures/datapackage.json').read()) with self.app.app_context(): db.drop_all() db.create_all() publisher = Publisher(name=self.publisher) metadata = Package(name=self.package) metadata.descriptor = json.dumps(descriptor) publisher.packages.append(metadata) db.session.add(publisher) db.session.commit() response = self.client.get('/api/package/%s/%s'%\ (self.publisher, self.package)) catalog = Catalog(json.loads(response.data)) dataset = catalog.construct_dataset() self.assertEqual(dataset.get('name'), descriptor.get('name')) self.assertEqual(dataset.get('owner'), self.publisher) self.assertIn('localurl', dataset.get('resources')[0]) self.assertNotEqual(len(dataset.get('views')), 0)
def test_return_true_if_delete_data_package_success(self): status = Package.delete_data_package(self.publisher_one, self.package_one) self.assertTrue(status) data = Package.query.join(Publisher). \ filter(Publisher.name == self.publisher_one, Package.name == self.package_one).all() self.assertEqual(0, len(data)) data = Publisher.query.all() self.assertEqual(1, len(data))
def test_home_shows_packages(self): descriptor = json.loads(open('fixtures/datapackage.json').read()) with self.app.app_context(): publisher = Publisher(name='core') package = Package(name='gold-prices', descriptor=descriptor) publisher.packages.append(package) db.session.add(publisher) db.session.commit() rv = self.client.get('/') self.assertEqual(rv.status_code, 200) self.assertTrue('DEMO - CBOE Volatility Index' in rv.data)
def test_data_package_page_loads_if_descriptor_has_bad_licenses(self): descriptor = json.loads(open('fixtures/datapackage.json').read()) descriptor['licenses'] = {'url': 'test/url', 'type': 'Test'} with self.app.app_context(): publisher = Publisher(name=self.publisher) package = Package(name=self.package, descriptor=descriptor) publisher.packages.append(package) db.session.add(publisher) db.session.commit() rv = self.client.get('/%s/%s' % (self.publisher, self.package)) self.assertEqual(200, rv.status_code)
def test_return_all_metadata_is_there(self): descriptor = {'name': 'test description'} readme = 'README' with self.app.app_context(): publisher = Publisher(name=self.publisher) metadata = Package(name=self.package) metadata.descriptor = json.dumps(descriptor) metadata.readme = readme publisher.packages.append(metadata) db.session.add(publisher) db.session.commit() response = self.client.get('/api/package/%s/%s'%\ (self.publisher, self.package)) data = json.loads(response.data) self.assertEqual(response.status_code, 200) self.assertEqual(len(data), 5) self.assertEqual(data['descriptor']['name'], 'test description') self.assertEqual(data['readme'], 'README') self.assertEqual(data['id'], 1) self.assertEqual(data['name'], self.package) self.assertEqual(data['publisher'], self.publisher)
def create_test_package(publisher='demo', package='demo-package', descriptor={}, readme=''): user = User(name=publisher, secret='supersecret', email='*****@*****.**') publisher = Publisher(name=publisher) association = PublisherUser(role=UserRoleEnum.owner) association.publisher = publisher user.publishers.append(association) package = Package(name=package, descriptor=descriptor, readme=readme) publisher.packages.append(package) db.session.add(user) db.session.commit()
def setUp(self): self.app = create_app() self.app.app_context().push() with self.app.test_request_context(): db.drop_all() db.create_all() self.pub1_name = 'pub1' self.pub2_name = 'pub2' self.pub1 = Publisher(name=self.pub1_name) self.pub2 = Publisher(name=self.pub2_name) self.pub1.packages.append( Package(name='pack1', descriptor='{"title": "pack1 details one"}', readme="Big readme one")) self.pub1.packages.append( Package(name='pack2', descriptor='{"title": "pack2 details two"}', readme="Big readme two")) self.pub1.packages.append( Package(name='pack3', descriptor='{"title": "pack3 details three"}')) self.pub2.packages.append( Package(name='pack4', descriptor='{"title": "pack4 details four"}')) self.pub2.packages.append( Package(name='pack5', descriptor='{"title": "pack5 details five"}')) self.pub2.packages.append( Package(name='pack6', descriptor='{"title": "pack6 details six"}')) db.session.add(self.pub1) db.session.add(self.pub2) db.session.commit()
def undelete_data_package(publisher, package): """ DPR data package un-delete operation. This API is responsible for un-mark the mark for delete of data package --- tags: - package parameters: - in: path name: publisher type: string required: true description: publisher name - in: path name: package type: string required: true description: package name - in: header name: Authorization type: string required: true description: > Jwt token in format of "bearer {token}. The token can be generated from /api/auth/token" responses: 500: description: Internal Server Error 200: description: Success Message schema: id: put_package_success properties: status: type: string default: OK """ try: bitstore = BitStore(publisher=publisher, package=package) status_acl = bitstore.change_acl('public-read') status_db = Package.change_status(publisher, package, PackageStateEnum.active) if status_acl and status_db: return jsonify({"status": "OK"}), 200 if not status_acl: raise Exception('Failed to change acl') if not status_db: raise Exception('Failed to change status') except Exception as e: app.logger.error(e) return handle_error('GENERIC_ERROR', e.message, 500)
def test_change_status(self): data = Package.query.join(Publisher). \ filter(Publisher.name == self.publisher_one, Package.name == self.package_one).one() self.assertEqual(PackageStateEnum.active, data.status) Package.change_status(self.publisher_one, self.package_one, PackageStateEnum.deleted) data = Package.query.join(Publisher). \ filter(Publisher.name == self.publisher_one, Package.name == self.package_one).one() self.assertEqual(PackageStateEnum.deleted, data.status) Package.change_status(self.publisher_one, self.package_one, status=PackageStateEnum.active) data = Package.query.join(Publisher). \ filter(Publisher.name == self.publisher_one, Package.name == self.package_one).one() self.assertEqual(PackageStateEnum.active, data.status)
def test_should_not_visible_after_soft_delete(self): descriptor = {'name': 'test description'} with self.app.app_context(): publisher = Publisher(name=self.publisher) metadata = Package(name=self.package) metadata.tags.append(PackageTag(descriptor=descriptor)) publisher.packages.append(metadata) db.session.add(publisher) db.session.commit() logic.Package.delete(self.publisher, self.package) response = self.client. \ get('/api/package/%s/%s' % (self.publisher, self.package)) self.assertEqual(response.status_code, 404)
def test_get_views(self): descriptor = { 'name': 'test', 'resources': [], 'views': [{ "type": "graph" }] } with self.app.app_context(): db.drop_all() db.create_all() publisher = Publisher(name=self.publisher) metadata = Package(name=self.package) metadata.descriptor = json.dumps(descriptor) publisher.packages.append(metadata) db.session.add(publisher) db.session.commit() response = self.client.get('/api/package/%s/%s'%\ (self.publisher, self.package)) catalog = Catalog(json.loads(response.data)) views = catalog.get_views() self.assertNotEqual(len(views), 0) self.assertEqual(views[0].get('type'), 'graph')