class DatasetTestCase(unittest.TestCase): def setUp(self): make_test_app() self.ds = Dataset(SIMPLE_MODEL) def tearDown(self): tear_down_test_app() def test_load_model_properties(self): assert self.ds.name == SIMPLE_MODEL["dataset"]["name"], self.ds.name assert self.ds.label == SIMPLE_MODEL["dataset"]["label"], self.ds.label def test_load_model_dimensions(self): assert len(self.ds.dimensions) == 4, self.ds.dimensions assert isinstance(self.ds["time"], ValueDimension), self.ds["time"] assert isinstance(self.ds["field"], ValueDimension), self.ds["field"] assert isinstance(self.ds["to"], ComplexDimension), self.ds["to"] assert isinstance(self.ds["function"], ComplexDimension), self.ds["function"] assert len(self.ds.metrics) == 1, self.ds.metrics assert isinstance(self.ds["amount"], Metric), self.ds["amount"] def test_value_dimensions_as_attributes(self): self.ds.generate() dim = self.ds["field"] assert isinstance(dim.column.type, UnicodeText), dim.column assert "field" == dim.column.name, dim.column assert dim.name == "field", dim.name assert dim.source_column == SIMPLE_MODEL["mapping"]["field"]["column"], dim.source_column assert dim.label == SIMPLE_MODEL["mapping"]["field"]["label"], dim.label assert dim.default == None, dim.default assert dim.dataset == self.ds, dim.dataset assert dim.datatype == "string", dim.datatype assert not hasattr(dim, "table") assert not hasattr(dim, "alias") def test_generate_db_entry_table(self): self.ds.generate() assert self.ds.table.name == "test_entry", self.ds.table.name cols = self.ds.table.c assert "id" in cols assert isinstance(cols["id"].type, Integer) # TODO: assert "time" in cols assert isinstance(cols["time"].type, UnicodeText) assert "amount" in cols assert isinstance(cols["amount"].type, Float) assert "field" in cols assert isinstance(cols["field"].type, UnicodeText) assert "to_id" in cols assert isinstance(cols["to_id"].type, Integer) assert "function_id" in cols assert isinstance(cols["function_id"].type, Integer) self.assertRaises(KeyError, cols.__getitem__, "foo")
def create(): require.dataset.create() dataset = request_data() data = validate_dataset(dataset) if Dataset.by_name(data['name']) is not None: raise Invalid(SchemaNode(String(), name='name'), _("A dataset with this identifer already exists!")) dataset = Dataset({'dataset': data, 'model': {}}) dataset.managers.append(current_user) db.session.add(dataset) db.session.commit() return view(dataset.name)
def get_or_create_dataset(model): """ Based on a provided model we get the model (if it doesn't exist we create it). """ dataset = Dataset.by_name(model['dataset']['name']) # If the dataset wasn't found we create it if dataset is None: dataset = Dataset(model) db.session.add(dataset) db.session.commit() log.info("Dataset: %s", dataset.name) return dataset
def test_delete_dataset_requires_auth(self): name = self.cra.name url = url_for('datasets_api.delete', name=name) res = self.client.delete(url, query_string={}) assert '403' in res.status, res.status ds = Dataset.by_name(name) assert ds is not None, ds
def test_create_dataset(self): url = url_for('datasets_api.create') res = self.client.post(url, data=json.dumps({}), query_string=self.auth_qs, headers={'content-type': 'application/json'}) assert '400' in res.status, res.status assert 'errors' in res.json, res.json params = { 'name': 'testds', 'label': 'Test Dataset', 'category': 'budget', 'description': 'I\'m a banana!', 'currency': 'EUR' } data = json.dumps(params) res = self.client.post(url, data=data, query_string=self.auth_qs, headers={'content-type': 'application/json'}) assert "200" in res.status, res.status assert res.json['name'] == 'testds', res.json ds = Dataset.by_name('testds') assert ds.label == params['label'], ds
def list_cubes(self): cubes = [] for dataset in Dataset.all_by_account(None): if not dataset.has_model: continue cubes.append({'name': dataset.name, 'label': dataset.label}) return cubes
def query_index(): q = Dataset.all_by_account(current_user, order=False) q = q.order_by(Dataset.updated_at.desc()) # Filter by languages if they have been provided for language in request.args.getlist('languages'): l = aliased(DatasetLanguage) q = q.join(l, Dataset._languages) q = q.filter(l.code == language) # Filter by territories if they have been provided for territory in request.args.getlist('territories'): t = aliased(DatasetTerritory) q = q.join(t, Dataset._territories) q = q.filter(t.code == territory) # Return a list of languages as dicts with code, count, url and label languages = [{'code': code, 'count': count, 'label': LANGUAGES.get(code)} for (code, count) in DatasetLanguage.dataset_counts(q)] territories = [{'code': code, 'count': count, 'label': COUNTRIES.get(code)} for (code, count) in DatasetTerritory.dataset_counts(q)] pager = Pager(q, limit=15) return pager, languages, territories
def list_cubes(self): cubes = [] for dataset in Dataset.all_by_account(None): if not dataset.has_model: continue cubes.append({"name": dataset.name, "label": dataset.label}) return cubes
def test_delete_dataset(self): name = self.cra.name url = url_for('datasets_api.delete', name=name) res = self.client.delete(url, query_string=self.auth_qs) assert '410' in res.status, res.status ds = Dataset.by_name(name) assert ds is None, ds
def query_index(): q = Dataset.all_by_account(current_user, order=False) q = q.order_by(Dataset.updated_at.desc()) # Filter by languages if they have been provided for language in request.args.getlist('languages'): l = aliased(DatasetLanguage) q = q.join(l, Dataset._languages) q = q.filter(l.code == language) # Filter by territories if they have been provided for territory in request.args.getlist('territories'): t = aliased(DatasetTerritory) q = q.join(t, Dataset._territories) q = q.filter(t.code == territory) # Filter by account if one has been provided for account in request.args.getlist('account'): a = aliased(Account) q = q.join(a, Dataset.managers) q = q.filter(a.name == account) # Return a list of languages as dicts with code, count, url and label languages = [{'code': code, 'count': count, 'label': LANGUAGES.get(code)} for (code, count) in DatasetLanguage.dataset_counts(q)] territories = [{'code': code, 'count': count, 'label': COUNTRIES.get(code)} for (code, count) in DatasetTerritory.dataset_counts(q)] pager = Pager(q, limit=15) return pager, languages, territories
def test_view_fields_empty(self): cra = Dataset.by_name('cra') cra.fields = {} db.session.commit() url = url_for('datasets_api.structure', name='cra') res = self.client.get(url) fields = res.json.get('fields') assert 'cap_or_cur' not in fields, res.json
def setUp(self): super(TestLoad, self).setUp() data_manager._index = None model = meta_fixture('cra') self.ds = Dataset(model) db.session.add(self.ds) db.session.commit() self.cra_url = csvimport_fixture_path('../data', 'cra.csv')
def load_from_url(dataset_name, url): with flask_app.app_context(): dataset = Dataset.by_name(dataset_name) if dataset is None: log.error("Dataset not found: %s", dataset_name) return source = tasks.extract_url(dataset, url) if source is not None: load_from_source.delay(dataset_name, source.name)
def test_publish(self): cra = Dataset.by_name('cra') cra.private = True db.session.commit() url = url_for('datasets_api.view', name='cra') res = self.client.get(url) assert '403' in res.status, res.status res = self.client.get(url, query_string={'api_key': self.user.api_key}) assert '200' in res.status, res.status data = res.json.copy() data['category'] = 'budget' data['private'] = False response = self.client.post(url, data=json.dumps(data), headers={'content-type': 'application/json'}, query_string={'api_key': self.user.api_key}) assert '200' in response.status, response.json cra = Dataset.by_name('cra') assert cra.private is False, cra.private
def load_from_source(dataset_name, source_name): with flask_app.app_context(): dataset = Dataset.by_name(dataset_name) if source_name is None: return artifact = tasks.transform_source(dataset, source_name) if artifact is None: return tasks.load(dataset, source_name=source_name)
def setUp(self): make_test_app() self.engine = core.db.engine self.meta = core.db.metadata #MetaData() self.meta.bind = self.engine self.ds = Dataset(SIMPLE_MODEL) self.reader = csv.DictReader(StringIO(TEST_DATA)) self.entity = self.ds['to'] self.classifier = self.ds['function']
def list_cubes(self): cubes = [] for dataset in Dataset.all_by_account(None): if not len(dataset.model.axes): continue cubes.append({ 'name': dataset.name, 'label': dataset.label }) return cubes
def import_fixture(name): meta_fp = csvimport_fixture_file(name, 'meta.json') model_fp = csvimport_fixture_file(name, 'model.json') meta = json.load(meta_fp) if model_fp: meta['model'] = json.load(model_fp) dataset = Dataset(meta) db.session.add(dataset) data_path = csvimport_fixture_path(name, 'data.csv') db.session.commit() return dataset, data_path
def test_publish(self): cra = Dataset.by_name('cra') cra.private = True db.session.commit() url = url_for('datasets_api.view', name='cra') res = self.client.get(url) assert '403' in res.status, res.status res = self.client.get(url, query_string={'api_key': self.user.api_key}) assert '200' in res.status, res.status data = res.json.copy() data['category'] = 'budget' data['private'] = False response = self.client.post( url, data=json.dumps(data), headers={'content-type': 'application/json'}, query_string={'api_key': self.user.api_key}) assert '200' in response.status, response.json cra = Dataset.by_name('cra') assert cra.private is False, cra.private
def authz(): obj = Dataset.by_name(request.args.get('dataset')) etag_cache_keygen(obj, private=True) if obj is None: return jsonify({ 'read': False, 'update': False }) return jsonify({ 'read': dataset.read(obj), 'update': dataset.update(obj) })
def test_update_invalid_label(self): data = {'name': 'cra', 'label': '', 'description': 'I\'m a banana', 'currency': 'GBP'} res = self.client.post(url_for('datasets_api.update', name='cra'), data=json.dumps(data), headers={'content-type': 'application/json'}, query_string={'api_key': self.user.api_key}) assert '400' in res.status, res.status assert 'Shorter than' in res.data, res.json cra = Dataset.by_name('cra') assert cra.label != '', cra.label
def load_from_source(dataset_name, source_name): with flask_app.app_context(): dataset = Dataset.by_name(dataset_name) if dataset is None: log.error("Dataset not found: %s", dataset_name) return if source_name is None: log.error("No source specified: %s", dataset_name) return source = tasks.transform_source(dataset, source_name) if source is None: return tasks.load(dataset, source_name=source_name)
def test_update_invalid_currency(self): data = {'name': 'cra', 'label': 'Common Rough Act', 'description': 'I\'m a banana', 'category': 'budget', 'currency': 'glass pearls'} res = self.client.post(url_for('datasets_api.update', name='cra'), data=json.dumps(data), headers={'content-type': 'application/json'}, query_string={'api_key': self.user.api_key}) assert 'not one of' in res.data, res.json cra = Dataset.by_name('cra') assert cra.currency == 'GBP', cra.label
def test_update(self): data = {'name': 'cra', 'label': 'Common Rough Act', 'description': 'I\'m a banana', 'currency': 'EUR', 'languages': ['en'], 'territories': ['GB'], 'category': 'budget'} res = self.client.post(url_for('datasets_api.update', name='cra'), data=json.dumps(data), headers={'content-type': 'application/json'}, query_string={'api_key': self.user.api_key}) cra = Dataset.by_name('cra') assert cra.label == 'Common Rough Act', (cra.label, res.json) assert cra.currency == 'EUR', (cra.currency, res.json)
def load_from_source(dataset_name, source_name): with flask_app.app_context(): dataset = Dataset.by_name(dataset_name) if dataset is None: log.error("Dataset not found: %s", dataset_name) return if source_name is None: log.error("No source specified: %s", dataset_name) return artifact = tasks.transform_source(dataset, source_name) if artifact is None: return tasks.load(dataset, source_name=source_name)
def test_update_invalid_territory(self): data = {'name': 'cra', 'label': 'CRA', 'territories': ['su'], 'description': 'I\'m a banana', 'currency': 'GBP'} response = self.client.post(url_for('datasets_api.update', name='cra'), data=json.dumps(data), headers={'content-type': 'application/json'}, query_string={'api_key': self.user.api_key}) assert '400' in response.status, response.status assert 'updated' not in response.data cra = Dataset.by_name('cra') assert 'su' not in cra.territories
def test_update_invalid_category(self): data = {'name': 'cra', 'label': 'Common Rough Act', 'description': 'I\'m a banana', 'currency': 'EUR', 'languages': ['en'], 'territories': ['GB'], 'category': 'foo'} res = self.client.post(url_for('datasets_api.update', name='cra'), data=json.dumps(data), headers={'content-type': 'application/json'}, query_string={'api_key': self.user.api_key}) assert '400' in res.status, res.status assert 'not one of' in res.data, res.json cra = Dataset.by_name('cra') assert cra.label != 'Common Rough Act', cra.label
def test_update_invalid_currency(self): data = { 'name': 'cra', 'label': 'Common Rough Act', 'description': 'I\'m a banana', 'category': 'budget', 'currency': 'glass pearls' } res = self.client.post(url_for('datasets_api.update', name='cra'), data=json.dumps(data), headers={'content-type': 'application/json'}, query_string={'api_key': self.user.api_key}) assert 'not one of' in res.data, res.json cra = Dataset.by_name('cra') assert cra.currency == 'GBP', cra.label
def test_update_invalid_label(self): data = { 'name': 'cra', 'label': '', 'description': 'I\'m a banana', 'currency': 'GBP' } res = self.client.post(url_for('datasets_api.update', name='cra'), data=json.dumps(data), headers={'content-type': 'application/json'}, query_string={'api_key': self.user.api_key}) assert '400' in res.status, res.status assert 'Shorter than' in res.data, res.json cra = Dataset.by_name('cra') assert cra.label != '', cra.label
def cube(self, name, locale=None): dataset = Dataset.by_name(name) if name is None: raise NoSuchCubeError("Unknown dataset %s" % name, name) measures, dimensions, mappings = [], [], {} aggregates = [MeasureAggregate('num_entries', label='Numer of entries', function='count')] for measure in dataset.model.measures: cubes_measure = Measure(measure.name, label=measure.label) measures.append(cubes_measure) aggregate = MeasureAggregate(measure.name, label=measure.label, measure=measure.name, function='sum') aggregates.append(aggregate) mappings[measure.name] = measure.column for dimension in dataset.model.dimensions: attributes = [] for attr in dimension.attributes: attributes.append(attr.name) mappings[attr.path] = attr.column meta = { 'label': dimension.label, 'name': dimension.name, 'levels': [{ 'name': dimension.name, 'label': dimension.label, # 'key': 'name', 'attributes': attributes }] } dimensions.append(create_dimension(meta)) return Cube(name=dataset.name, fact=dataset.fact_table.table.name, aggregates=aggregates, measures=measures, label=dataset.label, description=dataset.description, dimensions=dimensions, store=self.store, mappings=mappings)
def test_update(self): data = { 'name': 'cra', 'label': 'Common Rough Act', 'description': 'I\'m a banana', 'currency': 'EUR', 'languages': ['en'], 'territories': ['GB'], 'category': 'budget' } res = self.client.post(url_for('datasets_api.update', name='cra'), data=json.dumps(data), headers={'content-type': 'application/json'}, query_string={'api_key': self.user.api_key}) cra = Dataset.by_name('cra') assert cra.label == 'Common Rough Act', (cra.label, res.json) assert cra.currency == 'EUR', (cra.currency, res.json)
def test_update_invalid_territory(self): data = { 'name': 'cra', 'label': 'CRA', 'territories': ['su'], 'description': 'I\'m a banana', 'currency': 'GBP' } response = self.client.post( url_for('datasets_api.update', name='cra'), data=json.dumps(data), headers={'content-type': 'application/json'}, query_string={'api_key': self.user.api_key}) assert '400' in response.status, response.status assert 'updated' not in response.data cra = Dataset.by_name('cra') assert 'su' not in cra.territories
def test_update_invalid_category(self): data = { 'name': 'cra', 'label': 'Common Rough Act', 'description': 'I\'m a banana', 'currency': 'EUR', 'languages': ['en'], 'territories': ['GB'], 'category': 'foo' } res = self.client.post(url_for('datasets_api.update', name='cra'), data=json.dumps(data), headers={'content-type': 'application/json'}, query_string={'api_key': self.user.api_key}) assert '400' in res.status, res.status assert 'not one of' in res.data, res.json cra = Dataset.by_name('cra') assert cra.label != 'Common Rough Act', cra.label
def test_create_dataset(self): url = url_for('datasets_api.create') res = self.client.post(url, data=json.dumps({}), query_string=self.auth_qs, headers={'content-type': 'application/json'}) assert '400' in res.status, res.status assert 'errors' in res.json, res.json params = {'name': 'testds', 'label': 'Test Dataset', 'category': 'budget', 'description': 'I\'m a banana!', 'currency': 'EUR'} data = json.dumps(params) res = self.client.post(url, data=data, query_string=self.auth_qs, headers={'content-type': 'application/json'}) assert "200" in res.status, res.status assert res.json['name'] == 'testds', res.json ds = Dataset.by_name('testds') assert ds.label == params['label'], ds
class ComplexDimensionTestCase(unittest.TestCase): def setUp(self): make_test_app() self.engine = core.db.engine self.meta = core.db.metadata #MetaData() self.meta.bind = self.engine self.ds = Dataset(SIMPLE_MODEL) self.reader = csv.DictReader(StringIO(TEST_DATA)) self.entity = self.ds['to'] self.classifier = self.ds['function'] def tearDown(self): tear_down_test_app() def test_basic_properties(self): self.ds.generate() assert self.entity.name=='to', self.entity.name assert self.classifier.name=='function', self.classifier.name assert self.entity.scheme=='entity', self.entity.scheme assert self.classifier.scheme=='funny', self.classifier.scheme def test_generated_tables(self): assert not hasattr(self.entity, 'table'), self.entity self.ds.generate() assert hasattr(self.entity, 'table'), self.entity assert self.entity.table.name=='test_' + self.entity.scheme, self.entity.table.name assert hasattr(self.entity, 'alias') assert self.entity.alias.name==self.entity.name, self.entity.alias.name cols = self.entity.table.c assert 'id' in cols self.assertRaises(KeyError, cols.__getitem__, 'field') def test_attributes_exist_on_object(self): assert len(self.entity.attributes)==3, self.entity.attributes self.assertRaises(KeyError, self.entity.__getitem__, 'field') assert self.entity['name'].name=='name' assert self.entity['name'].datatype=='string' assert self.entity['const'].default=='true' def test_attributes_exist_on_table(self): self.ds.generate() assert hasattr(self.entity, 'table'), self.entity assert 'name' in self.entity.table.c, self.entity.table.c assert 'label' in self.entity.table.c, self.entity.table.c
def has_cube(self, name): dataset = Dataset.by_name(name) if dataset is None: return False return dataset.has_model
def authz(): obj = Dataset.by_name(request.args.get("dataset")) if obj is None: return jsonify({"read": False, "update": False}) return jsonify({"read": dataset.read(obj), "update": dataset.update(obj)})
def authz(): obj = Dataset.by_name(request.args.get('dataset')) if obj is None: return jsonify({'read': False, 'update': False}) return jsonify({'read': dataset.read(obj), 'update': dataset.update(obj)})
def setUp(self): make_test_app() self.ds = Dataset(SIMPLE_MODEL) self.engine = core.db.engine self.ds.generate() self.reader = csv.DictReader(StringIO(TEST_DATA))
def create_budget_data_package(url, user, private): try: bdpkg = BudgetDataPackage(url) except Exception as problem: # Lots of different types of problems can arise with a # BudgetDataPackage, but their message should be understandable # so we catch just any Exception and email it's message to the user log.error("Failed to parse budget data package: {0}".format( problem.message)) return [] sources = [] for (idx, resource) in enumerate(bdpkg.resources): dataset = Dataset.by_name(bdpkg.name) if dataset is None: # Get information from the descriptior file for the given # resource (at index idx) info = get_dataset_info_from_descriptor(bdpkg, idx) # Set the dataset name based on the previously computed one info['dataset']['name'] = bdpkg.name # Create the model from the resource schema model = create_model_from_schema(resource.schema) # Set the default value for the time to the fiscal year of the # resource, because it isn't included in the budget CSV so we # won't be able to load it along with the data. model['time']['default_value'] = resource.fiscalYear # Add the model as the mapping info['mapping'] = model # Create the dataset dataset = Dataset(info) dataset.managers.append(user) dataset.private = private db.session.add(dataset) db.session.commit() else: if not dataset.can_update(user): log.error( "User {0} not permitted to update dataset {1}".format( user.name, bdpkg.name)) return [] if 'url' in resource: resource_url = resource.url elif 'path' in resource: if 'base' in bdpkg: resource_url = urlparse.urljoin(bdpkg.base, resource.path) else: resource_url = urlparse.urljoin(url, resource.path) else: log.error('Url not found') return [] # We do not re-add old sources so if we find the same source # we don't do anything, else we create the source and append it # to the source list for dataset_source in dataset.sources: if dataset_source.url == resource_url: break else: source = Source(dataset=dataset, creator=user, url=resource_url) db.session.add(source) db.session.commit() sources.append(source) return sources
def cube(self, name, locale=None, namespace=None): dataset = Dataset.by_name(name) if name is None: raise NoSuchCubeError("Unknown dataset %s" % name, name) measures, dimensions, mappings = [], [], {} aggregates = [MeasureAggregate("fact_count", label="Number of entries", function="count")] for measure in dataset.model.measures: cubes_measure = Measure(measure.name, label=measure.label) measures.append(cubes_measure) aggregate = MeasureAggregate( measure.name + "_sum", label=measure.label, measure=measure.name, function="sum" ) aggregates.append(aggregate) mappings[measure.name] = measure.column for dimension in dataset.model.dimensions: attributes, last_col = [], None for attr in dimension.attributes: attributes.append({"name": attr.name, "label": attr.label}) mappings[attr.path] = last_col = attr.column # Workaround because the cubes mapper shortens references # for single-attribute dimensions to just the dimension name. if len(attributes) == 1: mappings[dimension.name] = last_col # Translate into cubes' categories cardinality = "high" if dimension.cardinality: if dimension.cardinality < 6: cardinality = "tiny" elif dimension.cardinality < 51: cardinality = "low" elif dimension.cardinality < 1001: cardinality = "medium" meta = { "label": dimension.label, "name": dimension.name, "cardinality": cardinality, "levels": [ { "name": dimension.name, "label": dimension.label, "cardinality": cardinality, "attributes": attributes, } ], } if dimension.key_attribute: meta["levels"][0]["key"] = dimension.key_attribute.name if dimension.label_attribute: meta["levels"][0]["label_attribute"] = dimension.label_attribute.name meta["levels"][0]["order_attribute"] = dimension.label_attribute.name dimensions.append(Dimension.from_metadata(meta)) cube = Cube( name=dataset.name, fact=dataset.fact_table.table.name, aggregates=aggregates, measures=measures, label=dataset.label, description=dataset.description, dimensions=dimensions, store=self.store, mappings=mappings, ) link_cube(cube, locale, provider=self, namespace=namespace) return cube
def get_dataset(name): dataset = obj_or_404(Dataset.by_name(name)) require.dataset.read(dataset) return dataset
def setUp(self): make_test_app() self.ds = Dataset(SIMPLE_MODEL)
def cube(self, name, locale=None, namespace=None): dataset = Dataset.by_name(name) if name is None: raise NoSuchCubeError("Unknown dataset %s" % name, name) measures, dimensions, mappings = [], [], {} aggregates = [ MeasureAggregate('fact_count', label='Number of entries', function='count') ] for measure in dataset.model.measures: cubes_measure = Measure(measure.name, label=measure.label) measures.append(cubes_measure) aggregate = MeasureAggregate(measure.name + '_sum', label=measure.label, measure=measure.name, function='sum') aggregates.append(aggregate) mappings[measure.name] = measure.column for dimension in dataset.model.dimensions: attributes, last_col = [], None for attr in dimension.attributes: attributes.append({'name': attr.name, 'label': attr.label}) mappings[attr.path] = last_col = attr.column # Workaround because the cubes mapper shortens references # for single-attribute dimensions to just the dimension name. if len(attributes) == 1: mappings[dimension.name] = last_col # Translate into cubes' categories cardinality = 'high' if dimension.cardinality: if dimension.cardinality < 6: cardinality = 'tiny' elif dimension.cardinality < 51: cardinality = 'low' elif dimension.cardinality < 1001: cardinality = 'medium' meta = { 'label': dimension.label, 'name': dimension.name, 'cardinality': cardinality, 'levels': [{ 'name': dimension.name, 'label': dimension.label, 'cardinality': cardinality, 'attributes': attributes }] } if dimension.key_attribute: meta['levels'][0]['key'] = dimension.key_attribute.name if dimension.label_attribute: meta['levels'][0]['label_attribute'] = \ dimension.label_attribute.name meta['levels'][0]['order_attribute'] = \ dimension.label_attribute.name dimensions.append(Dimension.from_metadata(meta)) cube = Cube(name=dataset.name, fact=dataset.fact_table.table.name, aggregates=aggregates, measures=measures, label=dataset.label, description=dataset.description, dimensions=dimensions, store=self.store, mappings=mappings) link_cube(cube, locale, provider=self, namespace=namespace) return cube
def authz(): obj = Dataset.by_name(request.args.get('dataset')) etag_cache_keygen(obj, private=True) if obj is None: return jsonify({'read': False, 'update': False}) return jsonify({'read': dataset.read(obj), 'update': dataset.update(obj)})
class DatasetLoadTestCase(unittest.TestCase): def setUp(self): make_test_app() self.ds = Dataset(SIMPLE_MODEL) self.engine = core.db.engine self.ds.generate() self.reader = csv.DictReader(StringIO(TEST_DATA)) def tearDown(self): tear_down_test_app() def test_load_all(self): self.ds.load_all(self.reader) resn = self.engine.execute(self.ds.table.select()).fetchall() assert len(resn) == 6, resn row0 = resn[0] assert row0["time"] == "2010", row0.items() assert row0["amount"] == 200, row0.items() assert row0["field"] == "foo", row0.items() def test_flush(self): self.ds.load_all(self.reader) resn = self.engine.execute(self.ds.table.select()).fetchall() assert len(resn) == 6, resn self.ds.flush() resn = self.engine.execute(self.ds.table.select()).fetchall() assert len(resn) == 0, resn def test_drop(self): tn = self.engine.table_names() assert "test_entry" in tn, tn assert "test_entity" in tn, tn assert "test_funny" in tn, tn self.ds.drop() tn = self.engine.table_names() assert "test_entry" not in tn, tn assert "test_entity" not in tn, tn assert "test_funny" not in tn, tn def test_aggregate_simple(self): self.ds.load_all(self.reader) res = self.ds.aggregate() assert res["summary"]["num_entries"] == 6, res assert res["summary"]["amount"] == 2690.0, res def test_aggregate_basic_cut(self): self.ds.load_all(self.reader) res = self.ds.aggregate(cuts=[("field", u"foo")]) assert res["summary"]["num_entries"] == 3, res assert res["summary"]["amount"] == 1000, res def test_aggregate_or_cut(self): self.ds.load_all(self.reader) res = self.ds.aggregate(cuts=[("field", u"foo"), ("field", u"bar")]) assert res["summary"]["num_entries"] == 4, res assert res["summary"]["amount"] == 1190, res def test_aggregate_dimensions_drilldown(self): self.ds.load_all(self.reader) res = self.ds.aggregate(drilldowns=["function"]) assert res["summary"]["num_entries"] == 6, res assert res["summary"]["amount"] == 2690, res assert len(res["drilldown"]) == 2, res["drilldown"] def test_aggregate_two_dimensions_drilldown(self): self.ds.load_all(self.reader) res = self.ds.aggregate(drilldowns=["function", "field"]) # pprint(res) assert res["summary"]["num_entries"] == 6, res assert res["summary"]["amount"] == 2690, res assert len(res["drilldown"]) == 5, res["drilldown"] def test_materialize_table(self): self.ds.load_all(self.reader) itr = self.ds.materialize() tbl = list(itr) assert len(tbl) == 6, tbl row = tbl[0] assert isinstance(row["field"], unicode), row assert isinstance(row["function"], dict), row assert isinstance(row["to"], dict), row