def _get_dataset_for_year_with_computed_attributes(self, year):
dataset = self._get_dataset(dataset_name='test',
cache_directory=self.temp_cache_path,
year=year)
dataset_junior = DatasetJunior(dataset=dataset, name='test')
indicator = Indicator(dataset_name='test',
attribute='opus_core.test.attribute')
dataset_junior.compute(indicator=indicator, year=year)
indicator = Indicator(dataset_name='test',
attribute='opus_core.test.attribute2')
dataset_junior.compute(indicator=indicator, year=year)
return dataset, dataset_junior
def test_create_indicator(self):
indicator_path = self.source_data.get_indicator_directory()
self.assert_(not os.path.exists(indicator_path))
self.source_data.years = range(1980, 1984)
indicator = Indicator(dataset_name='test',
attribute='opus_core.test.attribute')
indicator2 = Indicator(dataset_name='test',
attribute='opus_core.test.attribute2')
maker = Maker(project_name='test', test=True)
computed_indicators = maker.create_batch(indicators={
'attr1': indicator,
'attr2': indicator2
},
source_data=self.source_data)
for style in [Table.ALL, Table.PER_YEAR, Table.PER_ATTRIBUTE]:
table = Table(
indicator_directory=self.source_data.get_indicator_directory(),
output_type='csv',
output_style=style)
table._create_input_stores(range(1980, 1984))
viz_results = table.visualize(
indicators_to_visualize=['attr1', 'attr2'],
computed_indicators=computed_indicators)
for viz_result in viz_results:
if style == Table.ALL:
file_name = 'test_table-%i_1980-1983_attr1-attr2.csv' % style
elif style == Table.PER_YEAR:
file_name = 'test_table-%i_%i_attr1-attr2.csv' % (
style, viz_result.years[0])
elif style == Table.PER_ATTRIBUTE:
if viz_result.indicators[0].indicator.name == 'attribute':
name = 'attr1'
else:
name = 'attr2'
file_name = 'test_table-%i_1980-1983_%s.csv' % (style,
name)
self.assertEqual(
os.path.join(
viz_result.storage_location, viz_result.table_name +
'.' + viz_result.file_extension),
os.path.join(indicator_path, file_name))
def get_indicator(self,
indicator_name,
dataset_name,
indicator_definition=None):
if indicator_definition is not None:
attribute, source = indicator_definition
else:
indicator_nodes = get_available_indicator_nodes(self.project)
for indicator_node in indicator_nodes:
dataset, name = get_variable_dataset_and_name(indicator_node)
if name == indicator_name and dataset == dataset_name:
attribute = (indicator_node.text or '').strip()
source = indicator_node.get('source')
break
else:
raise Exception('Could not find an indicator %s for dataset %s'\
%(indicator_name, dataset_name))
# Make sure that expressions are prepended by their names
# WAS the line below, but it fails if the expression includes an argument like 'function=mean'
#if attribute.find('=') == -1 and source == 'expression':
if is_anonymous_autogen_name(VariableName(attribute).get_short_name()):
attribute = str(indicator_name) + '=' + attribute
new_indicator = Indicator(name=indicator_name,
dataset_name=dataset_name,
attribute=attribute)
return new_indicator
def go():
from opus_gui.results_manager.run.indicator_framework.representations.indicator import Indicator
import os
indicators = {
'zone_jobs':
Indicator(dataset_name='zone',
attribute='urbansim.zone.number_of_jobs')
}
from opus_core.configurations.dataset_pool_configuration import DatasetPoolConfiguration
from opus_gui.results_manager.run.indicator_framework.maker.source_data import SourceData
project_name = 'eugene_gridcell'
run_name1 = 'base_year_data'
years = [1980, 1980]
source_data = SourceData(
cache_directory=paths.get_opus_data_path_path(project_name, run_name1),
#comparison_cache_directory = '',
years=years,
dataset_pool_configuration=DatasetPoolConfiguration(
package_order=['urbansim', 'opus_core'], ),
project_name=project_name)
################################################################
#COMPUTE indicators
################################################################
from opus_gui.results_manager.run.indicator_framework.maker.maker import Maker
maker = Maker(project_name)
computed_indicators = maker.create_batch(indicators=indicators,
source_data=source_data)
############################################
#VISUALIZE the resulting computed indicators
############################################
from opus_gui.results_manager.run.indicator_framework.visualizer.visualization_factory import VisualizationFactory
visualizer = VisualizationFactory()
visualizations = []
maps = [('zone_jobs', 'matplotlib_chart')]
for item in maps:
vistype = 'mapnik_map' # default to map
if type(item) == type(()): item, vistype = item
print "Generating indicator %s" % item
visualizations += visualizer.visualize(
indicators_to_visualize=[
item
], # override default indicators to visualize (all)
computed_indicators=computed_indicators,
visualization_type=vistype,
name=item)
print "Done generating indicators."
return dict((v.name, v.get_file_path()) for v in visualizations)
def test__indicator_expressions(self):
maker = Maker(project_name='test', test=True)
indicator = Indicator(attribute='2 * opus_core.test.attribute',
dataset_name='test')
computed_indicator = maker.create(indicator=indicator,
source_data=self.source_data)
storage_location = os.path.join(
self.source_data.get_indicator_directory(), '_stored_data', '1980')
self.assert_(os.path.exists(os.path.join(storage_location, 'test')))
store = StorageFactory().get_storage(type='flt_storage',
storage_location=storage_location)
cols = sorted(store.get_column_names(table_name='test'))
self.assertTrue(len(cols[0]) > 10)
truncated_cols = copy(cols)
truncated_cols[0] = cols[0][:10]
self.assertEqual(truncated_cols, ['autogenvar', 'id'])
expected_attribute_vals = [10, 12, 14, 16]
data = store.load_table(table_name='test', column_names=[cols[0]])
self.assertEqual(expected_attribute_vals, list(data[cols[0]]))
def test_create_indicator_multiple_years(self):
indicator_path = os.path.join(self.temp_cache_path, 'indicators')
self.assert_(not os.path.exists(indicator_path))
self.source_data.years = range(1980, 1984)
indicator = Indicator(dataset_name='test',
attribute='opus_core.test.attribute')
maker = Maker(project_name='test', test=True)
maker.create(indicator=indicator, source_data=self.source_data)
for year in range(1980, 1984):
storage_location = os.path.join(
self.source_data.get_indicator_directory(), '_stored_data',
repr(year))
self.assert_(os.path.exists(os.path.join(storage_location,
'test')))
store = StorageFactory().get_storage(
type='flt_storage', storage_location=storage_location)
cols = store.get_column_names(table_name='test')
self.assertEqual(sorted(cols), sorted(['attribute', 'id']))
id_vals = [1, 2, 3, 4]
attribute_vals = [5, 6, 7, 8]
attribute_vals_1983 = [10, 12, 14, 16]
data = store.load_table(table_name='test', column_names=cols)
self.assertEqual(id_vals, list(data['id']))
if year == 1983:
self.assertEqual(attribute_vals_1983, list(data['attribute']))
else:
self.assertEqual(attribute_vals, list(data['attribute']))
def skip_test_create_indicator(self):
indicator = Indicator(
dataset_name = 'opus_core',
attribute = 'urbansim.gridcell.population'
)
maker = Maker(project_name = 'test', test = True)
computed_indicators = maker.create_batch(
indicators = {'population':indicator},
source_data = self.source_data)
indicator_path = os.path.join(self.temp_cache_path, 'indicators')
self.assert_(not os.path.exists(indicator_path))
map = MapnikMap(
indicator_directory = self.source_data.get_indicator_directory(),
name = 'map_of_opus_core.population(gridcell)')
map.create(False)
viz_result = map.visualize(
indicators_to_visualize = ['population'],
computed_indicators = computed_indicators)[0]
self.assertTrue(os.path.exists(
os.path.join(viz_result.storage_location,
viz_result.table_name + '.' + viz_result.file_extension)))
def test__integrity_checker(self):
maker = Maker(project_name='test', test=True)
'''package does not exist'''
indicator = Indicator(attribute='package.test.attribute',
dataset_name='test')
self.assertRaises(IntegrityError, maker.create, indicator,
self.source_data)
def skip_test_create_indicator2(self):
# if the environment variable DISPLAY isn't defined, exit this test
if 'DISPLAY' not in os.environ:
return
indicator_path = self.source_data.get_indicator_directory()
self.assert_(not os.path.exists(indicator_path))
self.source_data.years = range(1980, 1984)
indicator = Indicator(dataset_name='test',
attribute='opus_core.test.attribute')
indicator2 = Indicator(dataset_name='test',
attribute='opus_core.test.attribute2')
maker = Maker(project_name='test', test=True)
computed_indicators = maker.create_batch(
indicators={
'attr1': indicator,
'attr2': indicator2
},
source_data=self.source_data)
chart = MatplotlibChart(
name='test_chart',
indicator_directory=self.source_data.get_indicator_directory())
chart._create_input_stores(range(1980, 1984))
viz_results = chart.visualize(
indicators_to_visualize=['attr1', 'attr2'],
computed_indicators=computed_indicators)
for viz_result in viz_results:
if viz_result.indicators[0].indicator.name == 'attribute':
name = 'attr1'
else:
name = 'attr2'
file_name = 'test_chart_1980-1983_%s.png' % name
self.assertEqual(
os.path.join(
viz_result.storage_location, viz_result.table_name +
'.' + viz_result.file_extension),
os.path.join(indicator_path, file_name))
def test__get_indicator_path(self):
indicator = Indicator(attribute='opus_core.test.population',
dataset_name='test')
computed_indicator = ComputedIndicator(source_data=self.source_data,
indicator=indicator,
dataset_name='test',
primary_keys=['id'])
returned_path = computed_indicator.get_file_name()
expected_path = 'test__population.csv'
self.assertEqual(returned_path, expected_path)
def test__indicator_expressions_with_two_variables(self):
maker = Maker(project_name='test', test=True)
indicator = Indicator(
attribute=
'2 * opus_core.test.attribute - opus_core.test.attribute2',
dataset_name='test')
computed_indicator = maker.create(indicator=indicator,
source_data=self.source_data)
storage_location = os.path.join(
self.source_data.get_indicator_directory(), '_stored_data', '1980')
self.assert_(os.path.exists(os.path.join(storage_location, 'test')))
store = StorageFactory().get_storage(type='flt_storage',
storage_location=storage_location)
cols = store.get_column_names(table_name='test')
attr_col = [col for col in cols if col != 'id'][0]
expected_attribute_vals = [-40, -48, -56, -64]
data = store.load_table(table_name='test', column_names=[attr_col])
self.assertEqual(expected_attribute_vals, list(data[attr_col]))
def test__output_types(self):
output_types = ['csv', 'tab', 'fixed_field']
try:
import dbfpy
except ImportError:
pass
else:
output_types.append('dbf')
try:
test_db_name = 'test_db_for_indicator_framework'
database_config = DatabaseConfiguration(
database_name=test_db_name,
test=True,
)
server = DatabaseServer(database_config)
server.drop_database(database_name=test_db_name)
server.create_database(database_name=test_db_name)
except:
has_sql = False
else:
has_sql = True
output_types.append('sql')
indicator = Indicator(dataset_name='test',
attribute='opus_core.test.attribute')
maker = Maker(project_name='test', test=True)
computed_indicators = maker.create_batch(
indicators={'attr1': indicator}, source_data=self.source_data)
for output_type in output_types:
kwargs = {}
if output_type == 'sql':
kwargs['storage_location'] = database_config
elif output_type == 'fixed_field':
kwargs[
'fixed_field_format'] = '<fixed_field><field name="attribute_1980" format="10f" /></fixed_field>'
table = Table(
indicator_directory=self.source_data.get_indicator_directory(),
output_type=output_type,
**kwargs)
table._create_input_stores(self.source_data.years)
viz_result = table.visualize(
indicators_to_visualize=['attr1'],
computed_indicators=computed_indicators)[0]
if output_type in ['csv', 'dbf', 'tab', 'fixed_field']:
self.assertTrue(
os.path.exists(
os.path.join(
viz_result.storage_location,
viz_result.table_name + '.' +
viz_result.file_extension)))
elif output_type == 'sql':
self.assertTrue(server.has_database(test_db_name))
db = server.get_database(test_db_name)
self.assertTrue(
db.table_exists(table_name=viz_result.table_name))
if has_sql:
server.drop_database(database_name=test_db_name)
def main():
from opus_gui.results_manager.run.indicator_framework.representations.indicator import Indicator
indicators = {
'zone_population':
Indicator(dataset_name='zone', attribute='urbansim.zone.population'),
'gridcell_population':
Indicator(dataset_name='gridcell',
attribute='urbansim.gridcell.population'),
'zone_industrial_sqft':
Indicator(dataset_name='zone',
attribute='urbansim.zone.industrial_sqft'),
'gridcell_number_of_jobs':
Indicator(dataset_name='gridcell',
attribute='urbansim.gridcell.number_of_jobs',
name='jobs'),
'zone_number_of_jobs':
Indicator(dataset_name='zone',
attribute='urbansim.gridcell.number_of_jobs',
name='zone_jobs'),
#Expression example (comparison to baseyear)
'large_area_population_change':
Indicator(
dataset_name='large_area',
name='de_population_change',
attribute=
'psrc.large_area.de_population_DDDD - psrc.large_area.de_population_2000',
),
#example using regional-level aggregators
'alldata_home_based_jobs':
Indicator(
attribute=
'alldata.aggregate_all(urbansim.zone.number_of_home_based_jobs)',
dataset_name='alldata',
name='number_of_home_based_jobs'),
}
#################################################################
#DEFINE data source
#################################################################
# define any number of cache directories and/or years
# over which the indicators are computed
from opus_core.configurations.dataset_pool_configuration import DatasetPoolConfiguration
from opus_gui.results_manager.run.indicator_framework.maker.source_data import SourceData
result_template = SourceData(
cache_directory=r'D:\urbansim_cache\run_1090.2006_11_14_12_12',
comparison_cache_directory=
r'D:\urbansim_cache\run_1091.2006_11_14_12_12',
years=[2000, 2010],
dataset_pool_configuration=DatasetPoolConfiguration(
package_order=['urbansim', 'opus_core'], ),
name='run_1090')
################################################################
#COMPUTE indicators
################################################################
# setup an indicator Maker that will compute a set of indicators
# for a given result template
from opus_gui.results_manager.run.indicator_framework.maker.maker import Maker
maker = Maker()
computed_indicators = maker.create_batch(indicators=indicators,
result_template=result_template)
############################################
#VISUALIZE the resulting computed indicators
############################################
from opus_gui.results_manager.run.indicator_framework.visualizer.visualization_factory import VisualizationFactory
from opus_core.database_management.configurations.database_configuration import DatabaseConfiguration
visualizer = VisualizationFactory()
visualizations = []
# View an indicator as a Map
maps = ['zone_population', 'gridcell_population']
visualizations += visualizer.visualize(
indicators_to_visualize=
maps, #override default indicators to visualize (all)
computed_indicators=computed_indicators,
visualization_type='mapnik_map',
name='my_maps')
# View an indicator as a matplotlib Chart
charts = ['gridcell_population']
visualizations += visualizer.visualize(
indicators_to_visualize=charts,
computed_indicators=computed_indicators,
visualization_type='matplotlib_chart',
years=[2010], #override default years to visualize (all)
name='charts')
# Write an indicator as a Table
tables = [
'zone_industrial_sqft', 'large_area_population_change',
'alldata_home_based_jobs'
]
for output_type in ['tab', 'cvs', 'dbf']:
visualizations += visualizer.visualize(
indicators_to_visualize=tables,
computed_indicators=computed_indicators,
visualization_type='table',
output_type=output_type,
name='tables',
)
# Write a set of indicators sharing a dataset as a Dataset Table
indicators_in_dataset_table = ['zone_population', 'zone_industrial_sqft']
visualizations += visualizer.visualize(
indicators_to_visualize=indicators_in_dataset_table,
computed_indicators=computed_indicators,
visualization_type='dataset_table',
output_type='csv',
exclude_condition=
'urbansim.zone.population<100', #this accepts any opus expression
name='dataset_table',
)
################################################################
#Generate a REPORT with the visualizations
################################################################
from opus_gui.results_manager.run.indicator_framework.reporter.report_factory import ReportFactory
reporter = ReportFactory()
reporter.generate_report(visualized_indicators=visualized_indicators,
report_type='basic',
open_immediately=True,
storage_location='c:/my_reports')
# Opus/UrbanSim urban simulation software.
# Copyright (C) 2005-2009 University of Washington
# See opus_core/LICENSE
#############################################################
#DEFINE indicators
#############################################################
from opus_gui.results_manager.run.indicator_framework.representations.indicator import Indicator
indicators = {
'zone_population':Indicator(
dataset_name = 'zone',
attribute = 'urbansim.zone.population'),
'gridcell_population':Indicator(
dataset_name = 'gridcell',
attribute = 'urbansim.gridcell.population'),
'zone_industrial_sqft':Indicator(
dataset_name = 'zone',
attribute = 'urbansim.zone.industrial_sqft'),
'gridcell_number_of_jobs':Indicator(
dataset_name = 'gridcell',
attribute = 'urbansim.gridcell.number_of_jobs',
name = 'jobs'),
'zone_number_of_jobs':Indicator(
dataset_name = 'zone',
attribute = 'urbansim.gridcell.number_of_jobs',