def get_indicator(self,
indicator_name,
dataset_name,
indicator_definition=None):
if indicator_definition is not None:
attribute, source = indicator_definition
else:
indicator_nodes = get_available_indicator_nodes(self.project)
for indicator_node in indicator_nodes:
dataset, name = get_variable_dataset_and_name(indicator_node)
if name == indicator_name and dataset == dataset_name:
attribute = (indicator_node.text or '').strip()
source = indicator_node.get('source')
break
else:
raise Exception('Could not find an indicator %s for dataset %s'\
%(indicator_name, dataset_name))
# Make sure that expressions are prepended by their names
# WAS the line below, but it fails if the expression includes an argument like 'function=mean'
#if attribute.find('=') == -1 and source == 'expression':
if is_anonymous_autogen_name(VariableName(attribute).get_short_name()):
attribute = str(indicator_name) + '=' + attribute
new_indicator = Indicator(name=indicator_name,
dataset_name=dataset_name,
attribute=attribute)
return new_indicator
def get_indicator(self, indicator_name, dataset_name, indicator_definition = None):
terse_name = None
if indicator_definition is not None:
attribute, source = indicator_definition
else:
indicator_nodes = get_available_indicator_nodes(self.project)
for indicator_node in indicator_nodes:
dataset, name, tn = get_variable_parts(indicator_node)
if name == indicator_name and dataset == dataset_name:
attribute = (indicator_node.text or '').strip()
source = indicator_node.get('source')
terse_name = tn
break
else:
raise Exception('Could not find an indicator %s for dataset %s'\
%(indicator_name, dataset_name))
# Make sure that expressions are prepended by their names
# WAS the line below, but it fails if the expression includes an argument like 'function=mean'
#if attribute.find('=') == -1 and source == 'expression':
if is_anonymous_autogen_name(VariableName(attribute).get_short_name()):
attribute = str(indicator_name) + '='+ attribute
new_indicator = Indicator(name = indicator_name,
dataset_name = dataset_name,
attribute = attribute,
terse_name = terse_name)
return new_indicator
def _setup_available_indicators(self):
indicator_nodes = get_available_indicator_nodes(self.project)
current_row = self.indicator_table.currentRow()
self.indicator_table.clear()
self.indicator_table.setColumnCount(3)
self.indicator_table.setRowCount(len(indicator_nodes))
col = QTableWidgetItem()
col.setText('Name')
self.indicator_table.setHorizontalHeaderItem(0,col)
col = QTableWidgetItem()
col.setText('Dataset')
self.indicator_table.setHorizontalHeaderItem(1,col)
col = QTableWidgetItem()
col.setText('Definition')
self.indicator_table.setHorizontalHeaderItem(2,col)
for i, indicator in enumerate(indicator_nodes):
row = QTableWidgetItem()
self.indicator_table.setVerticalHeaderItem(i, row)
dataset, name = get_variable_dataset_and_name(indicator)
item = QTableWidgetItem()
item.setText(name)
self.indicator_table.setItem(i,0,item)
item = QTableWidgetItem()
item.setText(dataset)
self.indicator_table.setItem(i,1,item)
item = QTableWidgetItem()
item.setText(indicator.text or '')
self.indicator_table.setItem(i,2,item)
if current_row is None or current_row == -1:
current_row = 0
self.indicator_table.resizeRowsToContents()
self.indicator_table.resizeColumnsToContents()
self.indicator_table.setCurrentCell(current_row,0)
self.on_indicator_table_itemSelectionChanged()
def _setup_available_indicators(self):
indicator_nodes = get_available_indicator_nodes(self.project)
current_row = self.indicator_table.currentRow()
self.indicator_table.clear()
self.indicator_table.setColumnCount(3)
self.indicator_table.setRowCount(len(indicator_nodes))
col = QTableWidgetItem()
col.setText('Name')
self.indicator_table.setHorizontalHeaderItem(0, col)
col = QTableWidgetItem()
col.setText('Dataset')
self.indicator_table.setHorizontalHeaderItem(1, col)
col = QTableWidgetItem()
col.setText('Definition')
self.indicator_table.setHorizontalHeaderItem(2, col)
for i, indicator in enumerate(indicator_nodes):
row = QTableWidgetItem()
self.indicator_table.setVerticalHeaderItem(i, row)
dataset, name = get_variable_dataset_and_name(indicator)
item = QTableWidgetItem()
item.setText(name)
self.indicator_table.setItem(i, 0, item)
item = QTableWidgetItem()
item.setText(dataset)
self.indicator_table.setItem(i, 1, item)
item = QTableWidgetItem()
item.setText(indicator.text or '')
self.indicator_table.setItem(i, 2, item)
if current_row is None or current_row == -1:
current_row = 0
self.indicator_table.resizeRowsToContents()
self.indicator_table.resizeColumnsToContents()
self.indicator_table.setCurrentCell(current_row, 0)
self.on_indicator_table_itemSelectionChanged()
def on_pb_generate_results_released(self):
run_name = self.current_run
indicator_name = self.current_indicator
indicator_dataset = self.current_indicator_dataset
start_year = int(self.current_year)
end_year = start_year
if run_name is None or indicator_name is None or start_year is None:
return
key = (run_name, indicator_name, start_year)
if key in self.already_browsed:
if not self.generating_results:
(tab_widget,map_widget) = self.already_browsed[key]
# self.swap_visualizations(map_widget, tab_widget)
self.pb_generate_results.setText('Results Generated')
else:
self.queued_results = ('swap', (map_widget, tab_widget))
return
self.pb_generate_results.setEnabled(False)
self.pb_generate_results.setText('Generating Results...')
indicator_nodes = get_available_indicator_nodes(self.project)
dataset = None
for indicator_node in indicator_nodes:
ind_dataset, name = get_variable_dataset_and_name(indicator_node)
if name == indicator_name and ind_dataset == indicator_dataset:
dataset = ind_dataset
break
if dataset is None:
raise Exception('Could not find dataset for indicator %s' % indicator_name)
table_params = {
'name': None,
'output_type' : 'tab',
'indicators' : [indicator_name],
}
map_params = {'name':None,
'indicators':[indicator_name]}
visualizations = [
('table_per_year', dataset, table_params),
('mapnik_map', dataset, map_params)
]
batch_processor = BatchProcessor(self.project)
batch_processor.guiElement = self
batch_processor.set_data(
visualizations = visualizations,
source_data_name = run_name,
years = range(start_year, end_year + 1))
if not self.generating_results:
self.generating_results = True
logger.log_note('Generating results for %s on run %s for year %indicator_node'%(run_name, indicator_name, start_year))
self.running_key = key
self.batch_processor = batch_processor
batch_processor_thread = OpusGuiThread(
parentThread = get_mainwindow_instance(),
parentGuiElement = self,
thread_object = self.batch_processor)
# Use this signal from the thread if it is capable of producing its own status signal
self.connect(batch_processor_thread, SIGNAL("runFinished(PyQt_PyObject)"), self._run_finished)
self.connect(batch_processor_thread, SIGNAL("runError(PyQt_PyObject)"), self._run_error)
batch_processor_thread.start()
else:
self.queued_results = (key, batch_processor)
def on_pb_urbancanvas_clicked(self):
run_name = self.current_run
indicator_name = self.current_indicator
indicator_dataset = self.current_indicator_dataset
if indicator_dataset != 'parcel':
MessageBox.information(mainwindow = self, text = 'Not a parcel variable. Only parcel variables can be sent to UrbanCanvas')
else:
start_year = int(self.current_year)
end_year = start_year
if run_name is None or indicator_name is None or start_year is None:
return
key = (run_name, indicator_name, start_year)
self.pb_urbancanvas.setText('Sending to UrbanCanvas...')
indicator_nodes = get_available_indicator_nodes(self.project)
dataset = None
for indicator_node in indicator_nodes:
ind_dataset, name = get_variable_dataset_and_name(indicator_node)
if name == indicator_name and ind_dataset == indicator_dataset:
dataset = ind_dataset
break
if dataset is None:
raise Exception('Could not find dataset for indicator %s' % indicator_name)
table_params = {
'name': None,
'output_type' : 'tab',
'indicators' : [indicator_name],
}
expression_library = self.project.xml_config.get_expression_library()
expression = expression_library[(dataset,name)]
logger.log_note(expression)
base_year = end_year
project_name = self.project.name
opus_data_path = self.project.xml_config.get_opus_data_path()
logger.log_note(base_year)
logger.log_note(project_name)
logger.log_note(opus_data_path)
interface = IndicatorFrameworkInterface(self.project)
source_data = interface.get_source_data(
source_data_name = run_name,
years = [end_year,]
)
cache = os.path.join(source_data.cache_directory,str(end_year))
logger.log_note(cache)
storage = StorageFactory().get_storage('flt_storage',storage_location=cache)
dataset_pool = DatasetPool(storage=storage, package_order=[project_name,'urbansim_parcel','urbansim','opus_core'])
parcels = dataset_pool.get_dataset('parcel')
parcel_ids = pd.Series(parcels.get_attribute('parcel_id'))
values = pd.Series(parcels.compute_variables([expression],dataset_pool=dataset_pool).astype('float'))
parcels = pd.DataFrame({"parcel_id":parcel_ids,"vl_values":values})
#parcels.set_index(keys='parcel_id',inplace=True)
#parcels["vl_values"][parcels["vl_values"]==0] = np.nan
parcels = parcels[parcels["vl_values"]>0]
os.chdir(os.path.join(opus_data_path,project_name))
parcels.to_csv('results_browser_indicator.csv',index=False)
#np.savez('results_browser_indicator',parcel_id=parcels.vl_values.index.values.astype('int32'),values=parcels.vl_values.values.astype('int32'))
##############UNCOMMENT IF WEBSERVICE IS DESIRED
# parcels.save('variable_library.pkl') ##I believe 'save' was just deprectated in pandas- its now to_pickle or some such thing... change this later
# web_service_path = os.path.join(os.getenv("OPUS_HOME"),'src',project_name,'scripts','web_service.py')
# logger.log_note(web_service_path)
# p = subprocess.Popen([sys.executable,web_service_path])
# MessageBox.information(mainwindow = self, text = 'Click OK when done viewing in UrbanCanvas')
# p.kill()
# self.pb_urbancanvas.setText('View in UrbanCanvas')
MessageBox.information(mainwindow = self, text = 'Variable exported to the project data directory for viewing in UrbanCanvas')
self.pb_urbancanvas.setText('View in UrbanCanvas')
def on_pb_urbancanvas_clicked(self):
run_name = self.current_run
indicator_name = self.current_indicator
indicator_dataset = self.current_indicator_dataset
if indicator_dataset != 'parcel':
MessageBox.information(
mainwindow=self,
text=
'Not a parcel variable. Only parcel variables can be sent to UrbanCanvas'
)
else:
start_year = int(self.current_year)
end_year = start_year
if run_name is None or indicator_name is None or start_year is None:
return
key = (run_name, indicator_name, start_year)
self.pb_urbancanvas.setText('Sending to UrbanCanvas...')
indicator_nodes = get_available_indicator_nodes(self.project)
dataset = None
for indicator_node in indicator_nodes:
ind_dataset, name = get_variable_dataset_and_name(
indicator_node)
if name == indicator_name and ind_dataset == indicator_dataset:
dataset = ind_dataset
break
if dataset is None:
raise Exception('Could not find dataset for indicator %s' %
indicator_name)
table_params = {
'name': None,
'output_type': 'tab',
'indicators': [indicator_name],
}
expression_library = self.project.xml_config.get_expression_library(
)
expression = expression_library[(dataset, name)]
logger.log_note(expression)
base_year = end_year
project_name = self.project.name
opus_data_path = self.project.xml_config.get_opus_data_path()
logger.log_note(base_year)
logger.log_note(project_name)
logger.log_note(opus_data_path)
interface = IndicatorFrameworkInterface(self.project)
source_data = interface.get_source_data(source_data_name=run_name,
years=[
end_year,
])
cache = os.path.join(source_data.cache_directory, str(end_year))
logger.log_note(cache)
storage = StorageFactory().get_storage('flt_storage',
storage_location=cache)
dataset_pool = DatasetPool(storage=storage,
package_order=[
project_name, 'urbansim_parcel',
'urbansim', 'opus_core'
])
parcels = dataset_pool.get_dataset('parcel')
parcel_ids = pd.Series(parcels.get_attribute('parcel_id'))
values = pd.Series(
parcels.compute_variables(
[expression], dataset_pool=dataset_pool).astype('float'))
parcels = pd.DataFrame({
"parcel_id": parcel_ids,
"vl_values": values
})
#parcels.set_index(keys='parcel_id',inplace=True)
#parcels["vl_values"][parcels["vl_values"]==0] = np.nan
parcels = parcels[parcels["vl_values"] > 0]
os.chdir(os.path.join(opus_data_path, project_name))
parcels.to_csv('results_browser_indicator.csv', index=False)
#np.savez('results_browser_indicator',parcel_id=parcels.vl_values.index.values.astype('int32'),values=parcels.vl_values.values.astype('int32'))
##############UNCOMMENT IF WEBSERVICE IS DESIRED
# parcels.save('variable_library.pkl') ##I believe 'save' was just deprectated in pandas- its now to_pickle or some such thing... change this later
# web_service_path = os.path.join(os.getenv("OPUS_HOME"),'src',project_name,'scripts','web_service.py')
# logger.log_note(web_service_path)
# p = subprocess.Popen([sys.executable,web_service_path])
# MessageBox.information(mainwindow = self, text = 'Click OK when done viewing in UrbanCanvas')
# p.kill()
# self.pb_urbancanvas.setText('View in UrbanCanvas')
MessageBox.information(
mainwindow=self,
text=
'Variable exported to the project data directory for viewing in UrbanCanvas'
)
self.pb_urbancanvas.setText('View in UrbanCanvas')
def on_pb_generate_results_clicked(self):
run_name = self.current_run
indicator_name = self.current_indicator
indicator_dataset = self.current_indicator_dataset
start_year = int(self.current_year)
end_year = start_year
if run_name is None or indicator_name is None or start_year is None:
return
key = (run_name, indicator_name, start_year)
if key in self.already_browsed:
if not self.generating_results:
(tab_widget, map_widget) = self.already_browsed[key]
# self.swap_visualizations(map_widget, tab_widget)
self.pb_generate_results.setText('Results Generated')
else:
self.queued_results = ('swap', (map_widget, tab_widget))
return
self.pb_generate_results.setEnabled(False)
self.pb_generate_results.setText('Generating Results...')
indicator_nodes = get_available_indicator_nodes(self.project)
dataset = None
for indicator_node in indicator_nodes:
ind_dataset, name = get_variable_dataset_and_name(indicator_node)
if name == indicator_name and ind_dataset == indicator_dataset:
dataset = ind_dataset
break
if dataset is None:
raise Exception('Could not find dataset for indicator %s' %
indicator_name)
table_params = {
'name': None,
'output_type': 'tab',
'indicators': [indicator_name],
}
map_params = {'name': None, 'indicators': [indicator_name]}
visualizations = [('table_per_year', dataset, table_params),
('mapnik_map', dataset, map_params)]
batch_processor = BatchProcessor(self.project)
batch_processor.guiElement = self
batch_processor.set_data(visualizations=visualizations,
source_data_name=run_name,
years=range(start_year, end_year + 1))
if not self.generating_results:
self.generating_results = True
logger.log_note(
'Generating results for %s on run %s for year %indicator_node'
% (run_name, indicator_name, start_year))
self.running_key = key
self.batch_processor = batch_processor
batch_processor_thread = OpusGuiThread(
parentThread=get_mainwindow_instance(),
parentGuiElement=self,
thread_object=self.batch_processor)
# Use this signal from the thread if it is capable of producing its own status signal
self.connect(batch_processor_thread,
SIGNAL("runFinished(PyQt_PyObject)"),
self._run_finished)
self.connect(batch_processor_thread,
SIGNAL("runError(PyQt_PyObject)"), self._run_error)
batch_processor_thread.start()
else:
self.queued_results = (key, batch_processor)
def _setup_indicators(self, existing_indicators = []):
if self.dataset_name is None: return
indicator_nodes = get_available_indicator_nodes(self.project)
# self.xml_helper.get_available_indicator_names(attributes = ['dataset'])
current_row = self.twAvailableIndicators.currentRow()
self.twAvailableIndicators.clear()
self.twAvailableIndicators.setColumnCount(2)
self.twAvailableIndicators.horizontalHeader().setStretchLastSection(True)
self.twIndicatorsToVisualize.horizontalHeader().setStretchLastSection(True)
while self.twAvailableIndicators.rowCount() > 0:
self.twAvailableIndicators.removeRow(0)
while self.twIndicatorsToVisualize.rowCount() > 0:
self.twIndicatorsToVisualize.removeRow(0)
# self.twAvailableIndicators.setRowCount(len(indicators) - len(existing_indicators))
col = QTableWidgetItem()
col.setText(QString('Name'))
self.twAvailableIndicators.setHorizontalHeaderItem(0,col)
#col = QTableWidgetItem()
#col.setText(QString('Dataset'))
#self.twAvailableIndicators.setHorizontalHeaderItem(1,col)
col = QTableWidgetItem()
col.setText(QString('Definition'))
self.twAvailableIndicators.setHorizontalHeaderItem(1,col)
self.indicator_nodes = {}
for indicator in indicator_nodes:
dataset, name = get_variable_dataset_and_name(indicator)
self.indicator_nodes[name] = indicator
if name not in existing_indicators:
if self.dataset_name == dataset:
item = QTableWidgetItem()
item.setText(name)
row = self.twAvailableIndicators.rowCount()
self.twAvailableIndicators.insertRow(row)
#self.twAvailableIndicators.setVerticalHeaderItem(row,QTableWidgetItem())
self.twAvailableIndicators.setItem(row,0,item)
#item = QTableWidgetItem()
#item.setText(dataset)
#self.twAvailableIndicators.setItem(i,1,item)
item = QTableWidgetItem()
item.setText(indicator.text or '')
self.twAvailableIndicators.setItem(row,1,item)
else:
if self.dataset_name != dataset:
logger.log_warning('Visualization configured incorrectly. Cannot have indicators for different datasets. Skipping indicator %s'%str(name))
continue
item = QTableWidgetItem()
item.setText(name)
row = self.twIndicatorsToVisualize.rowCount()
self.twIndicatorsToVisualize.insertRow(row)
self.twIndicatorsToVisualize.setItem(row, 0, item)
if current_row is None or current_row == -1:
current_row = 0
self.twAvailableIndicators.setCurrentCell(current_row,0)
def _setup_indicators(self, existing_indicators=[]):
if self.dataset_name is None: return
indicator_nodes = get_available_indicator_nodes(self.project)
# self.xml_helper.get_available_indicator_names(attributes = ['dataset'])
current_row = self.twAvailableIndicators.currentRow()
self.twAvailableIndicators.clear()
self.twAvailableIndicators.setColumnCount(2)
self.twAvailableIndicators.horizontalHeader().setStretchLastSection(
True)
self.twIndicatorsToVisualize.horizontalHeader().setStretchLastSection(
True)
while self.twAvailableIndicators.rowCount() > 0:
self.twAvailableIndicators.removeRow(0)
while self.twIndicatorsToVisualize.rowCount() > 0:
self.twIndicatorsToVisualize.removeRow(0)
# self.twAvailableIndicators.setRowCount(len(indicators) - len(existing_indicators))
col = QTableWidgetItem()
col.setText(QString('Name'))
self.twAvailableIndicators.setHorizontalHeaderItem(0, col)
#col = QTableWidgetItem()
#col.setText(QString('Dataset'))
#self.twAvailableIndicators.setHorizontalHeaderItem(1,col)
col = QTableWidgetItem()
col.setText(QString('Definition'))
self.twAvailableIndicators.setHorizontalHeaderItem(1, col)
self.indicator_nodes = {}
for indicator in indicator_nodes:
dataset, name = get_variable_dataset_and_name(indicator)
self.indicator_nodes[name] = indicator
if name not in existing_indicators:
if self.dataset_name == dataset:
item = QTableWidgetItem()
item.setText(name)
row = self.twAvailableIndicators.rowCount()
self.twAvailableIndicators.insertRow(row)
#self.twAvailableIndicators.setVerticalHeaderItem(row,QTableWidgetItem())
self.twAvailableIndicators.setItem(row, 0, item)
#item = QTableWidgetItem()
#item.setText(dataset)
#self.twAvailableIndicators.setItem(i,1,item)
item = QTableWidgetItem()
item.setText(indicator.text or '')
self.twAvailableIndicators.setItem(row, 1, item)
else:
if self.dataset_name != dataset:
logger.log_warning(
'Visualization configured incorrectly. Cannot have indicators for different datasets. Skipping indicator %s'
% str(name))
continue
item = QTableWidgetItem()
item.setText(name)
row = self.twIndicatorsToVisualize.rowCount()
self.twIndicatorsToVisualize.insertRow(row)
self.twIndicatorsToVisualize.setItem(row, 0, item)
if current_row is None or current_row == -1:
current_row = 0
self.twAvailableIndicators.setCurrentCell(current_row, 0)
