def _get_dataset_for_year_with_computed_attributes(self, year): dataset = self._get_dataset(dataset_name='test', cache_directory=self.temp_cache_path, year=year) dataset_junior = DatasetJunior(dataset=dataset, name='test') indicator = Indicator(dataset_name='test', attribute='opus_core.test.attribute') dataset_junior.compute(indicator=indicator, year=year) indicator = Indicator(dataset_name='test', attribute='opus_core.test.attribute2') dataset_junior.compute(indicator=indicator, year=year) return dataset, dataset_junior
def test_create_indicator(self): indicator_path = self.source_data.get_indicator_directory() self.assert_(not os.path.exists(indicator_path)) self.source_data.years = range(1980, 1984) indicator = Indicator(dataset_name='test', attribute='opus_core.test.attribute') indicator2 = Indicator(dataset_name='test', attribute='opus_core.test.attribute2') maker = Maker(project_name='test', test=True) computed_indicators = maker.create_batch(indicators={ 'attr1': indicator, 'attr2': indicator2 }, source_data=self.source_data) for style in [Table.ALL, Table.PER_YEAR, Table.PER_ATTRIBUTE]: table = Table( indicator_directory=self.source_data.get_indicator_directory(), output_type='csv', output_style=style) table._create_input_stores(range(1980, 1984)) viz_results = table.visualize( indicators_to_visualize=['attr1', 'attr2'], computed_indicators=computed_indicators) for viz_result in viz_results: if style == Table.ALL: file_name = 'test_table-%i_1980-1983_attr1-attr2.csv' % style elif style == Table.PER_YEAR: file_name = 'test_table-%i_%i_attr1-attr2.csv' % ( style, viz_result.years[0]) elif style == Table.PER_ATTRIBUTE: if viz_result.indicators[0].indicator.name == 'attribute': name = 'attr1' else: name = 'attr2' file_name = 'test_table-%i_1980-1983_%s.csv' % (style, name) self.assertEqual( os.path.join( viz_result.storage_location, viz_result.table_name + '.' + viz_result.file_extension), os.path.join(indicator_path, file_name))
def get_indicator(self, indicator_name, dataset_name, indicator_definition=None): if indicator_definition is not None: attribute, source = indicator_definition else: indicator_nodes = get_available_indicator_nodes(self.project) for indicator_node in indicator_nodes: dataset, name = get_variable_dataset_and_name(indicator_node) if name == indicator_name and dataset == dataset_name: attribute = (indicator_node.text or '').strip() source = indicator_node.get('source') break else: raise Exception('Could not find an indicator %s for dataset %s'\ %(indicator_name, dataset_name)) # Make sure that expressions are prepended by their names # WAS the line below, but it fails if the expression includes an argument like 'function=mean' #if attribute.find('=') == -1 and source == 'expression': if is_anonymous_autogen_name(VariableName(attribute).get_short_name()): attribute = str(indicator_name) + '=' + attribute new_indicator = Indicator(name=indicator_name, dataset_name=dataset_name, attribute=attribute) return new_indicator
def go(): from opus_gui.results_manager.run.indicator_framework.representations.indicator import Indicator import os indicators = { 'zone_jobs': Indicator(dataset_name='zone', attribute='urbansim.zone.number_of_jobs') } from opus_core.configurations.dataset_pool_configuration import DatasetPoolConfiguration from opus_gui.results_manager.run.indicator_framework.maker.source_data import SourceData project_name = 'eugene_gridcell' run_name1 = 'base_year_data' years = [1980, 1980] source_data = SourceData( cache_directory=paths.get_opus_data_path_path(project_name, run_name1), #comparison_cache_directory = '', years=years, dataset_pool_configuration=DatasetPoolConfiguration( package_order=['urbansim', 'opus_core'], ), project_name=project_name) ################################################################ #COMPUTE indicators ################################################################ from opus_gui.results_manager.run.indicator_framework.maker.maker import Maker maker = Maker(project_name) computed_indicators = maker.create_batch(indicators=indicators, source_data=source_data) ############################################ #VISUALIZE the resulting computed indicators ############################################ from opus_gui.results_manager.run.indicator_framework.visualizer.visualization_factory import VisualizationFactory visualizer = VisualizationFactory() visualizations = [] maps = [('zone_jobs', 'matplotlib_chart')] for item in maps: vistype = 'mapnik_map' # default to map if type(item) == type(()): item, vistype = item print "Generating indicator %s" % item visualizations += visualizer.visualize( indicators_to_visualize=[ item ], # override default indicators to visualize (all) computed_indicators=computed_indicators, visualization_type=vistype, name=item) print "Done generating indicators." return dict((v.name, v.get_file_path()) for v in visualizations)
def test__indicator_expressions(self): maker = Maker(project_name='test', test=True) indicator = Indicator(attribute='2 * opus_core.test.attribute', dataset_name='test') computed_indicator = maker.create(indicator=indicator, source_data=self.source_data) storage_location = os.path.join( self.source_data.get_indicator_directory(), '_stored_data', '1980') self.assert_(os.path.exists(os.path.join(storage_location, 'test'))) store = StorageFactory().get_storage(type='flt_storage', storage_location=storage_location) cols = sorted(store.get_column_names(table_name='test')) self.assertTrue(len(cols[0]) > 10) truncated_cols = copy(cols) truncated_cols[0] = cols[0][:10] self.assertEqual(truncated_cols, ['autogenvar', 'id']) expected_attribute_vals = [10, 12, 14, 16] data = store.load_table(table_name='test', column_names=[cols[0]]) self.assertEqual(expected_attribute_vals, list(data[cols[0]]))
def test_create_indicator_multiple_years(self): indicator_path = os.path.join(self.temp_cache_path, 'indicators') self.assert_(not os.path.exists(indicator_path)) self.source_data.years = range(1980, 1984) indicator = Indicator(dataset_name='test', attribute='opus_core.test.attribute') maker = Maker(project_name='test', test=True) maker.create(indicator=indicator, source_data=self.source_data) for year in range(1980, 1984): storage_location = os.path.join( self.source_data.get_indicator_directory(), '_stored_data', repr(year)) self.assert_(os.path.exists(os.path.join(storage_location, 'test'))) store = StorageFactory().get_storage( type='flt_storage', storage_location=storage_location) cols = store.get_column_names(table_name='test') self.assertEqual(sorted(cols), sorted(['attribute', 'id'])) id_vals = [1, 2, 3, 4] attribute_vals = [5, 6, 7, 8] attribute_vals_1983 = [10, 12, 14, 16] data = store.load_table(table_name='test', column_names=cols) self.assertEqual(id_vals, list(data['id'])) if year == 1983: self.assertEqual(attribute_vals_1983, list(data['attribute'])) else: self.assertEqual(attribute_vals, list(data['attribute']))
def skip_test_create_indicator(self): indicator = Indicator( dataset_name = 'opus_core', attribute = 'urbansim.gridcell.population' ) maker = Maker(project_name = 'test', test = True) computed_indicators = maker.create_batch( indicators = {'population':indicator}, source_data = self.source_data) indicator_path = os.path.join(self.temp_cache_path, 'indicators') self.assert_(not os.path.exists(indicator_path)) map = MapnikMap( indicator_directory = self.source_data.get_indicator_directory(), name = 'map_of_opus_core.population(gridcell)') map.create(False) viz_result = map.visualize( indicators_to_visualize = ['population'], computed_indicators = computed_indicators)[0] self.assertTrue(os.path.exists( os.path.join(viz_result.storage_location, viz_result.table_name + '.' + viz_result.file_extension)))
def test__integrity_checker(self): maker = Maker(project_name='test', test=True) '''package does not exist''' indicator = Indicator(attribute='package.test.attribute', dataset_name='test') self.assertRaises(IntegrityError, maker.create, indicator, self.source_data)
def skip_test_create_indicator2(self): # if the environment variable DISPLAY isn't defined, exit this test if 'DISPLAY' not in os.environ: return indicator_path = self.source_data.get_indicator_directory() self.assert_(not os.path.exists(indicator_path)) self.source_data.years = range(1980, 1984) indicator = Indicator(dataset_name='test', attribute='opus_core.test.attribute') indicator2 = Indicator(dataset_name='test', attribute='opus_core.test.attribute2') maker = Maker(project_name='test', test=True) computed_indicators = maker.create_batch( indicators={ 'attr1': indicator, 'attr2': indicator2 }, source_data=self.source_data) chart = MatplotlibChart( name='test_chart', indicator_directory=self.source_data.get_indicator_directory()) chart._create_input_stores(range(1980, 1984)) viz_results = chart.visualize( indicators_to_visualize=['attr1', 'attr2'], computed_indicators=computed_indicators) for viz_result in viz_results: if viz_result.indicators[0].indicator.name == 'attribute': name = 'attr1' else: name = 'attr2' file_name = 'test_chart_1980-1983_%s.png' % name self.assertEqual( os.path.join( viz_result.storage_location, viz_result.table_name + '.' + viz_result.file_extension), os.path.join(indicator_path, file_name))
def test__get_indicator_path(self): indicator = Indicator(attribute='opus_core.test.population', dataset_name='test') computed_indicator = ComputedIndicator(source_data=self.source_data, indicator=indicator, dataset_name='test', primary_keys=['id']) returned_path = computed_indicator.get_file_name() expected_path = 'test__population.csv' self.assertEqual(returned_path, expected_path)
def test__indicator_expressions_with_two_variables(self): maker = Maker(project_name='test', test=True) indicator = Indicator( attribute= '2 * opus_core.test.attribute - opus_core.test.attribute2', dataset_name='test') computed_indicator = maker.create(indicator=indicator, source_data=self.source_data) storage_location = os.path.join( self.source_data.get_indicator_directory(), '_stored_data', '1980') self.assert_(os.path.exists(os.path.join(storage_location, 'test'))) store = StorageFactory().get_storage(type='flt_storage', storage_location=storage_location) cols = store.get_column_names(table_name='test') attr_col = [col for col in cols if col != 'id'][0] expected_attribute_vals = [-40, -48, -56, -64] data = store.load_table(table_name='test', column_names=[attr_col]) self.assertEqual(expected_attribute_vals, list(data[attr_col]))
def test__output_types(self): output_types = ['csv', 'tab', 'fixed_field'] try: import dbfpy except ImportError: pass else: output_types.append('dbf') try: test_db_name = 'test_db_for_indicator_framework' database_config = DatabaseConfiguration( database_name=test_db_name, test=True, ) server = DatabaseServer(database_config) server.drop_database(database_name=test_db_name) server.create_database(database_name=test_db_name) except: has_sql = False else: has_sql = True output_types.append('sql') indicator = Indicator(dataset_name='test', attribute='opus_core.test.attribute') maker = Maker(project_name='test', test=True) computed_indicators = maker.create_batch( indicators={'attr1': indicator}, source_data=self.source_data) for output_type in output_types: kwargs = {} if output_type == 'sql': kwargs['storage_location'] = database_config elif output_type == 'fixed_field': kwargs[ 'fixed_field_format'] = '<fixed_field><field name="attribute_1980" format="10f" /></fixed_field>' table = Table( indicator_directory=self.source_data.get_indicator_directory(), output_type=output_type, **kwargs) table._create_input_stores(self.source_data.years) viz_result = table.visualize( indicators_to_visualize=['attr1'], computed_indicators=computed_indicators)[0] if output_type in ['csv', 'dbf', 'tab', 'fixed_field']: self.assertTrue( os.path.exists( os.path.join( viz_result.storage_location, viz_result.table_name + '.' + viz_result.file_extension))) elif output_type == 'sql': self.assertTrue(server.has_database(test_db_name)) db = server.get_database(test_db_name) self.assertTrue( db.table_exists(table_name=viz_result.table_name)) if has_sql: server.drop_database(database_name=test_db_name)
def main(): from opus_gui.results_manager.run.indicator_framework.representations.indicator import Indicator indicators = { 'zone_population': Indicator(dataset_name='zone', attribute='urbansim.zone.population'), 'gridcell_population': Indicator(dataset_name='gridcell', attribute='urbansim.gridcell.population'), 'zone_industrial_sqft': Indicator(dataset_name='zone', attribute='urbansim.zone.industrial_sqft'), 'gridcell_number_of_jobs': Indicator(dataset_name='gridcell', attribute='urbansim.gridcell.number_of_jobs', name='jobs'), 'zone_number_of_jobs': Indicator(dataset_name='zone', attribute='urbansim.gridcell.number_of_jobs', name='zone_jobs'), #Expression example (comparison to baseyear) 'large_area_population_change': Indicator( dataset_name='large_area', name='de_population_change', attribute= 'psrc.large_area.de_population_DDDD - psrc.large_area.de_population_2000', ), #example using regional-level aggregators 'alldata_home_based_jobs': Indicator( attribute= 'alldata.aggregate_all(urbansim.zone.number_of_home_based_jobs)', dataset_name='alldata', name='number_of_home_based_jobs'), } ################################################################# #DEFINE data source ################################################################# # define any number of cache directories and/or years # over which the indicators are computed from opus_core.configurations.dataset_pool_configuration import DatasetPoolConfiguration from opus_gui.results_manager.run.indicator_framework.maker.source_data import SourceData result_template = SourceData( cache_directory=r'D:\urbansim_cache\run_1090.2006_11_14_12_12', comparison_cache_directory= r'D:\urbansim_cache\run_1091.2006_11_14_12_12', years=[2000, 2010], dataset_pool_configuration=DatasetPoolConfiguration( package_order=['urbansim', 'opus_core'], ), name='run_1090') ################################################################ #COMPUTE indicators ################################################################ # setup an indicator Maker that will compute a set of indicators # for a given result template from opus_gui.results_manager.run.indicator_framework.maker.maker import Maker maker = Maker() computed_indicators = maker.create_batch(indicators=indicators, result_template=result_template) ############################################ #VISUALIZE the resulting computed indicators ############################################ from opus_gui.results_manager.run.indicator_framework.visualizer.visualization_factory import VisualizationFactory from opus_core.database_management.configurations.database_configuration import DatabaseConfiguration visualizer = VisualizationFactory() visualizations = [] # View an indicator as a Map maps = ['zone_population', 'gridcell_population'] visualizations += visualizer.visualize( indicators_to_visualize= maps, #override default indicators to visualize (all) computed_indicators=computed_indicators, visualization_type='mapnik_map', name='my_maps') # View an indicator as a matplotlib Chart charts = ['gridcell_population'] visualizations += visualizer.visualize( indicators_to_visualize=charts, computed_indicators=computed_indicators, visualization_type='matplotlib_chart', years=[2010], #override default years to visualize (all) name='charts') # Write an indicator as a Table tables = [ 'zone_industrial_sqft', 'large_area_population_change', 'alldata_home_based_jobs' ] for output_type in ['tab', 'cvs', 'dbf']: visualizations += visualizer.visualize( indicators_to_visualize=tables, computed_indicators=computed_indicators, visualization_type='table', output_type=output_type, name='tables', ) # Write a set of indicators sharing a dataset as a Dataset Table indicators_in_dataset_table = ['zone_population', 'zone_industrial_sqft'] visualizations += visualizer.visualize( indicators_to_visualize=indicators_in_dataset_table, computed_indicators=computed_indicators, visualization_type='dataset_table', output_type='csv', exclude_condition= 'urbansim.zone.population<100', #this accepts any opus expression name='dataset_table', ) ################################################################ #Generate a REPORT with the visualizations ################################################################ from opus_gui.results_manager.run.indicator_framework.reporter.report_factory import ReportFactory reporter = ReportFactory() reporter.generate_report(visualized_indicators=visualized_indicators, report_type='basic', open_immediately=True, storage_location='c:/my_reports')
# Opus/UrbanSim urban simulation software. # Copyright (C) 2005-2009 University of Washington # See opus_core/LICENSE ############################################################# #DEFINE indicators ############################################################# from opus_gui.results_manager.run.indicator_framework.representations.indicator import Indicator indicators = { 'zone_population':Indicator( dataset_name = 'zone', attribute = 'urbansim.zone.population'), 'gridcell_population':Indicator( dataset_name = 'gridcell', attribute = 'urbansim.gridcell.population'), 'zone_industrial_sqft':Indicator( dataset_name = 'zone', attribute = 'urbansim.zone.industrial_sqft'), 'gridcell_number_of_jobs':Indicator( dataset_name = 'gridcell', attribute = 'urbansim.gridcell.number_of_jobs', name = 'jobs'), 'zone_number_of_jobs':Indicator( dataset_name = 'zone', attribute = 'urbansim.gridcell.number_of_jobs',