def testEntityTableGeneration(self):
"""Test that single-concept tables are generated correctly."""
table_data = self.data_source_obj.GetTableData(
data_source.QueryParameters(
data_source.QueryParameters.CONCEPT_QUERY, ['category2']))
# Make sure quotes are properly escaped
self.assertEqual(table_data.rows,
[['california'], ['maine\'s'], ['oregon']])
def testTotalsSliceTableGeneration(self):
"""Test that slice tables are generated correctly with total values."""
table_data = self.data_source_obj.GetTableData(
data_source.QueryParameters(
data_source.QueryParameters.SLICE_QUERY,
['category1', 'metric1', 'metric2', 'metric3']))
self.assertEqual(table_data.rows,
[['red', 21 + 33, (98.0 + 90.0) / 2.0, 2]])
def testSliceTableGeneration(self):
"""Test that slice tables are generated correctly."""
table_data = self.data_source_obj.GetTableData(
data_source.QueryParameters(
data_source.QueryParameters.SLICE_QUERY,
['metric3', 'category2', 'metric1', 'metric2']))
self.assertEqual(
table_data.rows,
[[3, 'california', 89 + 99 + 293, (321.0 + 231.0 + 12.0) / 3.0],
[2, 'maine\'s', 293 + 932,
(32.0 + 48.0) / 2.0], [1, 'oregon', 32, 33]])
def PopulateDataset(data_source_obj, verbose):
"""Create a DSPL dataset from a data source.
Loops through the set of possible slices (provided by the _CalculateSlices
function), creating the necessary DSPL concept, slice, and table objects as
needed.
The following naming convention is used:
DSPL concept ID := DataSource column ID
DSPL table ID for concept tables := DSPL concept ID + "_table"
DSPL slice ID := "slice_" + n, where n=0,1,2,...
DSPL table ID for slice tables := DSPL slice ID + "_table"
DSPL table file name := DSPL table ID + ".csv"
Args:
data_source_obj: An object that implements the DataSource interface
verbose: Print out status messages to stdout
Returns:
A DSPL DataSet object
"""
column_bundle = data_source_obj.GetColumnBundle()
dataset = dspl_model.DataSet(verbose=verbose)
# Add standard imports
dataset.AddImport(
dspl_model.Import(
namespace_id='entity',
namespace_url=(
'http://www.google.com/publicdata/dataset/google/entity')))
dataset.AddImport(
dspl_model.Import(
namespace_id='geo',
namespace_url=(
'http://www.google.com/publicdata/dataset/google/geo')))
dataset.AddImport(
dspl_model.Import(
namespace_id='geo_us',
namespace_url=(
'http://www.google.com/publicdata/dataset/google/geo/us')))
dataset.AddImport(
dspl_model.Import(
namespace_id='quantity',
namespace_url=(
'http://www.google.com/publicdata/dataset/google/quantity')))
dataset.AddImport(
dspl_model.Import(
namespace_id='time',
namespace_url=(
'http://www.google.com/publicdata/dataset/google/time')))
dataset.AddImport(
dspl_model.Import(
namespace_id='unit',
namespace_url=(
'http://www.google.com/publicdata/dataset/google/unit')))
# Store concept ID to column ID mappings for imported dimension concepts
dimension_map = {}
# Generate concept metadata
for column in column_bundle.GetColumnIterator():
if column.slice_role == 'metric':
metric_concept = dspl_model.Concept(
concept_id=column.column_id,
concept_extension_reference=column.concept_extension,
data_type=column.data_type)
dataset.AddConcept(metric_concept)
else:
# Column corresponds to a dimension concept
if column.concept_ref:
# Dimension concept is imported; no need to enumerate instances
dimension_concept = dspl_model.Concept(
concept_id=column.concept_ref,
concept_reference=column.concept_ref,
data_type=column.data_type)
dimension_map[column.concept_ref] = column.column_id
else:
# Dimension defined inside the dataset; need to enumerate instances
if verbose:
print('Enumerating instances of \'%s\' concept' %
(column.column_id))
if column.parent_ref:
parent_column = column_bundle.GetColumnByID(
column.parent_ref)
query_column_ids = [column.column_id, column.parent_ref]
else:
parent_column = None
query_column_ids = [column.column_id]
concept_table_rows = data_source_obj.GetTableData(
data_source.QueryParameters(
query_type=data_source.QueryParameters.CONCEPT_QUERY,
column_ids=query_column_ids))
dataset.AddTable(
_CreateConceptTable(column, concept_table_rows,
parent_column, verbose))
dimension_concept = dspl_model.Concept(
concept_id=column.column_id,
concept_extension_reference=column.concept_extension,
data_type=column.data_type,
table_ref='%s_table' % (column.column_id))
if column.parent_ref:
# Add in parent reference property
dimension_concept.properties.append(
dspl_model.Property(column.parent_ref, True))
dataset.AddConcept(dimension_concept)
# Generate slice metadata
for i, slice_column_set in enumerate(_CalculateSlices(column_bundle)):
if verbose:
print('Evaluating slice: %s' %
([c.column_id for c in slice_column_set]))
dimension_ids = []
metric_ids = []
for column in slice_column_set:
if column.slice_role == 'dimension':
if column.concept_ref:
dimension_ids.append(column.concept_ref)
else:
dimension_ids.append(column.column_id)
else:
if column.concept_ref:
metric_ids.append(column.concept_ref)
else:
metric_ids.append(column.column_id)
# Execute slice query
if verbose:
print('Getting slice values')
slice_table_rows = data_source_obj.GetTableData(
data_source.QueryParameters(
query_type=data_source.QueryParameters.SLICE_QUERY,
column_ids=[c.column_id for c in slice_column_set]))
# Add slice and table metadata to dataset model
slice_table = _CreateSliceTable(slice_column_set, 'slice_%d_table' % i,
'slice_%d_table.csv' % i,
slice_table_rows, verbose)
dataset.AddTable(slice_table)
new_slice = dspl_model.Slice(slice_id='slice_%d' % (i),
dimension_refs=dimension_ids,
metric_refs=metric_ids,
dimension_map=dimension_map,
table_ref='slice_%d_table' % i)
dataset.AddSlice(new_slice)
return dataset