def _scanForRuns(self):
data_path = self.project.data_path()
if not os.path.exists(data_path):
MessageBox.warning(mainwindow = self.base_widget,
text="Project data path %s doesn't exist. " % data_path + \
"Simulation runs in the Results tab cannot be updated." )
return
run_manager = get_run_manager()
run_manager.clean_runs()
self._sync_base_year_data(run_manager)
run_manager.close()
added_runs, removed_runs = sync_available_runs(self.project)
added_msg = removed_msg = None
if len(added_runs) > 0:
added_msg = ('The following simulation runs have been '
'automatically added to the results manager:\n\n%s'
% '\n'.join(added_runs))
if len(removed_runs) > 0:
removed_msg = ('The following simulation runs have been '
'automatically removed from the results manager:\n\n%s'
% '\n'.join(removed_runs))
if added_msg or removed_msg:
## The idea is to leave the run information to services db & cache, and
## we don't need to save the newly added runs, once we set the temporary
# self.project.dirty = True
text = 'The list of simulation runs has been automatically updated.'
detailed_text = '%s\n%s' % (added_msg or '', removed_msg or '')
logger.log_status(text+'\n'+detailed_text)
def delete_run(self, run_node, force=False):
'''
Remove a run both from the services database and from the model.
@param run_node (Element): the node to remove.
'''
# Prevent the user from removing base years
cache_directory = run_node.find('cache_directory').text
if cache_directory.endswith('base_year_data') and not force:
msg = ('Removing the base year data directory is restricted from '
'within OpusGUI since doing so will make it impossible to '
'run any simulations or estimations.')
MessageBox.warning(mainwindow = self.view,
text = 'Cannot remove base year data',
detailed_text = msg)
return
try:
run_manager = get_run_manager()
run_id = run_node.get('run_id')
try:
run_id = int(run_id)
except:
run_id = -1
run_manager.delete_everything_for_this_run(run_id, cache_directory)
run_manager.close()
self.project.delete_node(run_node)
# self.model.remove_node(run_node)
except Exception, ex: # TODO catch more specific error?
MessageBox.warning(self.view, 'Could not remove run', str(ex))
def on_buttonBox_accepted(self):
viz_params = self._get_viz_spec()
if viz_params is None:
self.close()
return
dataset_name = viz_params['dataset_name']
viz_type = viz_params['visualization_type']
close = True
if (viz_type == 'mapnik_map' or viz_type == 'mapnik_animated_map') and dataset_name not in self.spatial_datasets:
MessageBox.warning(mainwindow = self.mainwindow,
text = "That indicator cannot be visualized as a map.",
detailed_text = ('The dataset %s is either not spatial or cannot be '
'rendered as a grid. If the latter, please try '
'exporting to an external GIS tool.'%dataset_name))
close = False
else:
self._update_xml_from_dict(self.base_node, viz_params)
update_batch_indicator_visualization(self.base_node)
if close:
self.close()
def check_variable(self, check='syntax'):
'''
validate the variable and display the result in i dialog box
if check_syntax is True a syntax check is performed, otherwise a data
check is performed.
'''
if check == 'syntax':
func = batch_check_syntax
elif check == 'data':
func = batch_check_data
else:
raise ValueError(
'check_variable() got an unknown value for argument "check"; "%s"'
% check)
self._update_variable_from_fields()
dummy, result, msgs = func([
self.variable,
], self.validator)[0]
if result is True:
text = '%s check OK' % check
MessageBox.information(mainwindow=self, text=text)
else:
text = 'Encountered a %s error' % check
MessageBox.warning(mainwindow=self,
text=text,
detailed_text='\n '.join(msgs))
def on_syntaxButton_clicked(self):
node = self._checkImportNodeMsg()
if node is None:
return
MessageBox.information(self, 'Check passed!',
self.ATTENTION)
def addParam(self):
assert self.has_selected_item()
item = self.selected_item()
node = item.node
window = addParamGui(self.manager.base_widget, None)
window.setModal(True)
window.show()
if window.exec_() == window.Accepted:
name = str(window.nameEdit.text())
typ = str(window.typeComboBox.currentText())
default = str(window.defaultEdit.text())
required = str(window.requiredComboBox.currentText())
attributes = {'name': name, 'param_type': typ, 'required': required}
node = Element('param', attributes)
node.text = default
if self.model.insertRow(0, self.selected_index(), node) == False:
MessageBox.error(mainwindow = self.view,
text = 'Parameter Name Exists',
detailed_text = ('The parameter name to add is "%s" '
'but there is already a parameter with that name.' %
name))
return
self.view.expand(self.selected_index())
def runErrorFromThread(self,errorMessage):
self.running = False
self.paused = False
self.pbnStartModel.setText(QString("Start Simulation Run..."))
MessageBox.warning(mainwindow = self.mainwindow,
text = "There was a problem running the simulation.",
detailed_text = errorMessage)
def execToolConfig(self):
'''
Show the dialog box for executing a "tool config"
A tool config is has a pointer to an existing tool (a "tool hook") but
can provide an alternative configuration.
'''
assert self.has_selected_item()
# First we need to get the hooked node that we want to run
node = self.selected_item().node
hooked_tool_name = node.find('tool_hook').text
hooked_tool_node = get_tool_node_by_name(self.project, hooked_tool_name)
if hooked_tool_node is None:
MessageBox.error(mainwindow = self.view,
text = 'Invalid tool hook',
detailed_text = ('This tool config points to a tool named "%s" '
'but there is no tool with that name in this project.' %
hooked_tool_name))
return
# Open up a GUI element and populate with variable's
tool_lib_node = get_tool_library_node(self.project)
params = {'tool_path': get_path_to_tool_modules(self.project)}
window = ExecuteToolGui(parent_widget = self.manager.base_widget,
tool_node = hooked_tool_node,
tool_config = node,
tool_library_node = tool_lib_node,
params=params,
model=self.model)
window.setModal(True)
window.show()
def execBatch(self):
''' Present a dialog to execute a set of tool configurations '''
assert self.has_selected_item()
# Node representing the set to execute
tool_set_node = self.selected_item().node
# map tool config nodes in set -> name of the hooked node
tool_config_to_tool_name = {}
tool_config_nodes = tool_set_node[:]
for tool_config_node in tool_config_nodes:
hook_node = tool_config_node.find('tool_hook')
hooked_tool_name = str(hook_node.text or '').strip()
hooked_tool_node = get_tool_node_by_name(self.project, hooked_tool_name)
module_name = hooked_tool_node.find('class_module').text
try:
module_name = hooked_tool_node.find('class_module').text
tool_path = get_path_to_tool_modules(self.project)
import_path = tool_path + '.' + module_name
importString = "from %s import opusRun" % (import_path)
exec(importString)
tool_config_to_tool_name[tool_config_node] = import_path
except Exception, e:
MessageBox.error(mainwindow = self.view,
text = 'Invalid module name',
detailed_text = ('This tool points to a module named "%s", ' % import_path + \
'but there is no module with that name, or module returned import error: %s. ' \
% formatExceptionInfo()
))
return
def execToolFile(self):
"""
Show a dialog box that lets the user configure and execute a
tool.
"""
assert self.has_selected_item()
tool_lib_node = get_tool_library_node(self.project)
params = {"tool_path": get_path_to_tool_modules(self.project)}
tool_node = self.selected_item().node
try:
module_name = tool_node.find("class_module").text
import_path = params["tool_path"] + "." + module_name
importString = "from %s import opusRun" % (import_path)
exec (importString)
except Exception, e:
print e
MessageBox.error(
mainwindow=self.view,
text="Invalid module name",
detailed_text=(
'This tool points to a module named "%s" '
"but there is no module with that name, or module returned import error." % import_path
),
)
return
def execToolFile(self):
'''
Show a dialog box that lets the user configure and execute a
tool.
'''
assert self.has_selected_item()
tool_lib_node = get_tool_library_node(self.project)
params = {'tool_path': get_path_to_tool_modules(self.project)}
tool_node = self.selected_item().node
try:
module_name = tool_node.find('class_module').text
import_path = params['tool_path'] + '.' + module_name
importString = "from %s import opusRun" % (import_path)
exec(importString)
except Exception, e:
print e
MessageBox.error(mainwindow = self.view,
text = 'Invalid module name',
detailed_text = ('This tool points to a module named "%s", ' % import_path + \
'but there is no module with that name, or module returned import error: %s. ' \
% formatExceptionInfo()
))
return
def on_buttonBox_accepted(self):
viz_params = self._get_viz_spec()
if viz_params is None:
self.close()
return
dataset_name = viz_params['dataset_name']
viz_type = viz_params['visualization_type']
close = True
if (viz_type == 'mapnik_map' or viz_type == 'mapnik_animated_map'
) and dataset_name not in self.spatial_datasets:
MessageBox.warning(
mainwindow=self.mainwindow,
text="That indicator cannot be visualized as a map.",
detailed_text=(
'The dataset %s is either not spatial or cannot be '
'rendered as a grid. If the latter, please try '
'exporting to an external GIS tool.' % dataset_name))
close = False
else:
self._update_xml_from_dict(self.base_node, viz_params)
update_batch_indicator_visualization(self.base_node)
if close:
self.close()
def runErrorFromThread(self, errorMessage):
self.running = False
self.paused = False
self.pbnStartModel.setText(QString("Start Estimation..."))
MessageBox.warning(mainwindow=self,
text="There was a problem with the estimation.",
detailed_text=errorMessage)
def importXMLFromFile(self):
''' NO DOCUMENTATION '''
# print "importXMLFromFile"
# First, prompt the user for the filename to read in
start_dir = paths.get_project_configs_path()
configDialog = QFileDialog()
filter_str = QString("*.xml")
fd = configDialog.getOpenFileName(
self.manager.base_widget, "Please select an XML file to import...",
start_dir, filter_str)
# Check for cancel
if len(fd) == 0:
return
fileName = QString(fd)
# Pass that in to create a new XMLConfiguration
try:
options = ''
xml_tree = load_xml_file(str(fileName))
xml_node = xml_tree.getroot()
options = ' or another node in the tree view'
self.import_from_node(xml_node)
except Exception, e:
MessageBox.error(mainwindow=self.view,
text='Cannot insert XML file.',
detailed_text='XML insert failed. '
'%s. '
'Please select another XML file%s.' %
(e, options))
return
def __init__(self, parent_widget):
QDialog.__init__(self, parent_widget)
self.setupUi(self)
settings_directory = os.path.join(os.environ['OPUS_HOME'], 'settings')
self._config_filename = os.path.join(settings_directory, 'database_server_configurations.xml')
try:
root = ElementTree(file=self._config_filename).getroot()
view = XmlView(self)
model = XmlModel(root)
delegate = XmlItemDelegate(view)
view.setModel(model)
# Turns out that Qt Garbage collects the model (and delegate) if we don't explicitly
# bind it to a Python object in addition to using the PyQt .setModel() method.
view._model = model
view._delegate = delegate
view.setItemDelegate(delegate)
view.openDefaultItems()
self.gridlayout.addWidget(view)
self.tree_view = view
self.xml_root = root
return
except IOError, ex:
MessageBox.error(mainwindow = self,
text = 'Could not initialize Database Settings',
detailed_text = str(ex))
self.xml_root = None
self._config_filename = ''
self.configFile = None
def _checkImportNodeMsg(self):
try:
return self._checkImportNode()
except Exception, e:
MessageBox.error(self, 'This is not a valid XML structure.', \
'''%s.\n\n%s''' % (e, self.ATTENTION))
return None
def execToolConfig(self):
'''
Show the dialog box for executing a "tool config"
A tool config is has a pointer to an existing tool (a "tool hook") but
can provide an alternative configuration.
'''
assert self.has_selected_item()
# First we need to get the hooked node that we want to run
node = self.selected_item().node
hooked_tool_name = node.find('tool_hook').text
hooked_tool_node = get_tool_node_by_name(self.project,
hooked_tool_name)
if hooked_tool_node is None:
MessageBox.error(
mainwindow=self.view,
text='Invalid tool hook',
detailed_text=(
'This tool config points to a tool named "%s" '
'but there is no tool with that name in this project.' %
hooked_tool_name))
return
# Open up a GUI element and populate with variable's
tool_lib_node = get_tool_library_node(self.project)
params = {'tool_path': get_path_to_tool_modules(self.project)}
window = ExecuteToolGui(parent_widget=self.manager.base_widget,
tool_node=hooked_tool_node,
tool_config=node,
tool_library_node=tool_lib_node,
params=params)
window.setModal(True)
window.show()
def _checkImportNodeMsg(self):
try:
return self._checkImportNode()
except Exception, e:
MessageBox.error(self, 'This is not a valid XML structure.', \
'''%s.\n\n%s''' % (e, self.ATTENTION))
return None
def init_for_variable(self, variable, validator, existing_variables):
''' Prepare the editor to edit a variable.
@param variable The variable to edit (dict)
@param validator A VariableValidator object used for data/syntax checking
@param existing_variables a list of tuples with (dataset, variable name) for all existing
variables. Used for checking for unique names.
'''
self.original_variable = variable # keep a reference to the original node...
self.variable = variable.copy(
) # ...but manipulate a temporary variable
self.validator = validator
self.existing_variables = existing_variables or []
self.leVarName.setText(variable['name'])
index = self.cboVarType.findText(variable['source'])
if index < 0: # variable source was not in the combo box
text = 'Failed to edit variable %s' % variable['name']
details = (
'Variable "%s" (dataset: %s) has an unknown type value (%s).\n'
'Please select a new type and update/save the variable.' %
(variable['name'], variable['dataset'], variable['source']))
MessageBox.warning(self, text, details)
self.cboVarType.setCurrentIndex(index)
if variable['use'] == 'model variable':
self.rbUseModel.setChecked(True)
elif variable['use'] == 'indicator':
self.rbUseIndicator.setChecked(True)
else:
self.rbUseBoth.setChecked(True)
self.le_var_def.document().setPlainText(variable['definition'])
self.leVarName.selectAll()
self.leVarName.setFocus()
def addParam(self):
assert self.has_selected_item()
item = self.selected_item()
node = item.node
window = addParamGui(self.manager.base_widget, None)
window.setModal(True)
window.show()
if window.exec_() == window.Accepted:
name = str(window.nameEdit.text())
typ = str(window.typeComboBox.currentText())
default = str(window.defaultEdit.text())
required = str(window.requiredComboBox.currentText())
attributes = {
'name': name,
'param_type': typ,
'required': required
}
node = Element('param', attributes)
node.text = default
if self.model.insertRow(0, self.selected_index(), node) == False:
MessageBox.error(
mainwindow=self.view,
text='Parameter Name Exists',
detailed_text=(
'The parameter name to add is "%s" '
'but there is already a parameter with that name.' %
name))
return
self.view.expand(self.selected_index())
def execBatch(self):
''' Present a dialog to execute a set of tool configurations '''
assert self.has_selected_item()
# Node representing the set to execute
tool_set_node = self.selected_item().node
# map tool config nodes in set -> name of the hooked node
tool_config_to_tool_name = {}
tool_config_nodes = tool_set_node[:]
for tool_config_node in tool_config_nodes:
hook_node = tool_config_node.find('tool_hook')
hooked_tool_name = str(hook_node.text or '').strip()
hooked_tool_node = get_tool_node_by_name(self.project,
hooked_tool_name)
module_name = hooked_tool_node.find('class_module').text
try:
module_name = hooked_tool_node.find('class_module').text
tool_path = get_path_to_tool_modules(self.project)
import_path = tool_path + '.' + module_name
importString = "from %s import opusRun" % (import_path)
exec(importString)
tool_config_to_tool_name[tool_config_node] = import_path
except Exception, e:
MessageBox.error(
mainwindow=self.view,
text='Invalid module name',
detailed_text=
('This tool points to a module named "%s" '
'but there is no module with that name, or module returned import error.'
% import_path))
return
def init_for_variable(self, variable, validator, existing_variables):
''' Prepare the editor to edit a variable.
@param variable The variable to edit (dict)
@param validator A VariableValidator object used for data/syntax checking
@param existing_variables a list of tuples with (dataset, variable name) for all existing
variables. Used for checking for unique names.
'''
self.original_variable = variable # keep a reference to the original node...
self.variable = variable.copy() # ...but manipulate a temporary variable
self.validator = validator
self.existing_variables = existing_variables or []
self.leVarName.setText(variable['name'])
self.le_var_def.document().setPlainText(variable['definition'])
index = self.cboVarType.findText(variable['source'])
if index < 0: # variable source was not in the combo box
text = 'Failed to edit variable %s' % variable['name']
details = ('Variable "%s" (dataset: %s) has an unknown type value (%s).\n'
'Please select a new type and update/save the variable.' %
(variable['name'], variable['dataset'], variable['source']))
MessageBox.warning(self, text, details)
# This triggers _update_variable_from_fields and hence requires
# name and definition to be set already
self.cboVarType.setCurrentIndex(index)
if variable['use'] == 'model variable':
self.rbUseModel.setChecked(True)
elif variable['use'] == 'indicator':
self.rbUseIndicator.setChecked(True)
else:
self.rbUseBoth.setChecked(True)
self.leVarName.selectAll()
self.leVarName.setFocus()
def execToolFile(self):
'''
Show a dialog box that lets the user configure and execute a
tool.
'''
assert self.has_selected_item()
tool_lib_node = get_tool_library_node(self.project)
params = {'tool_path': get_path_to_tool_modules(self.project)}
tool_node = self.selected_item().node
try:
module_name = tool_node.find('class_module').text
import_path = params['tool_path'] + '.' + module_name
importString = "from %s import opusRun" % (import_path)
exec(importString)
except Exception, e:
print e
MessageBox.error(
mainwindow=self.view,
text='Invalid module name',
detailed_text=
('This tool points to a module named "%s" '
'but there is no module with that name, or module returned import error.'
% import_path))
return
def _get_viz_spec(self):
viz_type = self.cboVizType.currentText()
translation = self._get_output_types(viz_type)
dataset_name = self.dataset_name
output_type = QString(translation[str(self.cboOutputType.currentText())])
indicators = QString(str(self._get_column_values(column = 0)))
vals = {
'indicators': indicators,
'output_type': output_type,
'dataset_name': dataset_name,
'visualization_type': QString(self._get_type_mapper()[str(viz_type)])
}
if output_type == 'mapnik_map' or output_type == 'mapnik_animated_map':
vals['mapnik_bucket_labels'] = self.mapnik_options['mapnik_bucket_labels']
vals['mapnik_bucket_colors'] = self.mapnik_options['mapnik_bucket_colors']
vals['mapnik_bucket_ranges'] = self.mapnik_options['mapnik_bucket_ranges']
vals['mapnik_resolution'] = self.mapnik_options['mapnik_resolution']
vals['mapnik_page_dims'] = self.mapnik_options['mapnik_page_dims']
vals['mapnik_map_lower_left'] = self.mapnik_options['mapnik_map_lower_left']
vals['mapnik_map_upper_right'] = self.mapnik_options['mapnik_map_upper_right']
vals['mapnik_legend_lower_left'] = self.mapnik_options['mapnik_legend_lower_left']
vals['mapnik_legend_upper_right'] = self.mapnik_options['mapnik_legend_upper_right']
elif output_type == 'fixed_field':
try:
fixed_field_params = QString(str(self._get_column_values(column = 1)))
except:
errorInfo = formatExceptionInfo()
logger.log_error(errorInfo)
MessageBox.error(mainwindow = self,
text = 'Could not get fixed field parameters for all columns',
detailed_text = '')
return None
vals['fixed_field_specification'] = fixed_field_params
vals['id_format'] = self.leOption1.text()
elif output_type == 'sql':
vals['database_name'] = self.leOption1.text()
elif output_type == 'esri':
vals['storage_location'] = self.leOption1.text()
elif output_type in ('tab', 'xls'):
if self.rbSingleTable.isChecked():
output_style = Table.ALL
elif self.rbTablePerIndicator.isChecked():
output_style = Table.PER_ATTRIBUTE
elif self.rbTablePerYear.isChecked():
output_style = Table.PER_YEAR
vals['output_style'] = QString(str(output_style))
if self.appendTypeCheckBox.isChecked():
vals['append_col_type'] = True
else:
vals['append_col_type'] = False
if output_type == 'xls':
vals['storage_location'] = self.leOption1.text()
return vals
def on_buttonBox_accepted(self):
try:
ElementTree(self.xml_root).write(self._config_filename)
except IOError, ex:
MessageBox.error(self,
text = 'A disk error occured when saving the ' +
'settings.',
detailed_text = str(ex))
def on_buttonBox_accepted(self):
try:
ElementTree(self.xml_root).write(self._config_filename, pretty_print=True, with_tail=False)
except IOError, ex:
MessageBox.error(self,
text = 'A disk error occured when saving the ' +
'settings.',
detailed_text = str(ex))
def import_run(self, cache_directory, run_info={}):
run_manager = get_run_manager()
retval, retmsg = run_manager.import_run_from_cache(cache_directory)
if not retval:
MessageBox.warning(mainwindow = self.base_widget,
text = retmsg,
detailed_text = '')
else:
sync_available_runs(self.project)
def on_buttonBox_accepted(self):
path = str(self.lePath.text())
if not os.path.exists(path):
msg = 'Cannot import run from %s: path does not exist' % path
logger.log_warning(msg)
MessageBox.warning(mainwindow=self, text=msg, detailed_text='')
else:
self.resultManagerBase.import_run(path)
self.close()
def on_buttonBox_accepted(self):
path = str(self.lePath.text())
if not os.path.exists(path):
msg = 'Cannot import run from %s: path does not exist' % path
logger.log_warning(msg)
MessageBox.warning(mainwindow = self,
text = msg,
detailed_text = '')
else:
self.resultManagerBase.import_run(path)
self.close()
def _validate_names(self, show_error_message=False):
''' go through all nest and equation names and ensure that there are no collisions.
Returns True if all the names are valid and False otherwise.
If @param show_error_message is True an error message of the name errors is displayed.'''
# Check for colliding names among the nest and equations
colliding_names = set()
nodes_to_inspect = self.submodel_node.findall('.//nest')
nodes_to_inspect.extend(self.submodel_node.findall('.//equation'))
for inspected_node in nodes_to_inspect:
# get all sibling names with the same tag
sibling_names = [
node.get('name') for node in inspected_node.getparent() if
node is not inspected_node and node.tag == inspected_node.tag
]
# if there is a name collision, add the name to the set of found colliding names
if inspected_node.get('name') in sibling_names:
parent_node = inspected_node.getparent()
if parent_node.tag == 'nest':
desc = '<%s>/%s' % (parent_node.get('name'),
inspected_node.get('name'))
else:
desc = '%s' % inspected_node.get('name')
colliding_names.add(desc)
# the concept of colliding names might be confusing so we want to be clear on what is
# happening and (more importantly) how to solve it
if colliding_names:
if not show_error_message:
return False
str_collide_list = ''.join(
['<li>%s</li>\n' % name for name in colliding_names])
short_msg = 'Name collisions found.'
longer_msg = ''' <qt> Colliding names:
<b> <ul> %s </ul> </b>
<p>A name collision is when there are two items with the same name, the same type and the same level.</p>
For example:
<ul>
<li>MY_NEST</li>
<li><ul><li>MY_EQUATION</li><li>MY_EQUATION</li></ul></li>
</ul>
<p>will cause a name collision, while this example;</p>
<ul>
<li>MY_NEST</li>
<li><ul><li>MY_EQUATION</li></ul></li>
<li>MY_OTHER_NEST</li> <li>
<ul><li>MY_EQUATION</li></ul></li>
</ul>
<p>is fine since the two equations with the same name are on different levels.</p>
<p>To correct this error please give unique names for the above mentioned equations
and/or nests.</p></qt>''' % str_collide_list
MessageBox.warning(self, short_msg, longer_msg)
return False
return True
def on_indicatorBox(self):
indicator_name = str(self.diagnostic_indicator_name.currentText())
dataset_name = self.diagnostic_dataset_name.currentText()
indicator_type = str(self.diagnostic_indicator_type.currentText())
year = str(self.diagnostic_year.currentText())
if year == "":
return
year = int(year)
if dataset_name not in self.spatial_datasets and indicator_type == "map":
MessageBox.warning(
mainwindow=self.mainwindow,
text="That indicator cannot be visualized as a map.",
detailed_text=(
"The dataset %s is either not spatial or cannot be "
"rendered as a grid. If the latter, please try "
"exporting to an external GIS tool." % dataset_name
),
)
return
cache_directory = self.model.config["cache_directory"]
params = {"indicators": [indicator_name]}
if indicator_type == "table":
indicator_type = "tab"
params["output_type"] = "tab"
# JLM > commented this out for when implemented animations
# else:
# indicator_type = 'mapnik_map'
visualizations = [(indicator_type, str(dataset_name), params)]
self.batch_processor = BatchProcessor(self.project)
self.batch_processor.guiElement = self
self.batch_processor.set_data(
visualizations=visualizations,
source_data_name=self.model.run_name,
years=[year, year],
cache_directory=cache_directory,
)
self.diagnosticThread = OpusGuiThread(
parentThread=self.mainwindow, parentGuiElement=self, thread_object=self.batch_processor
)
# Use this signal from the thread if it is capable of producing its own status signal
QObject.connect(self.diagnosticThread, SIGNAL("runFinished(PyQt_PyObject)"), self.visualizationsCreated)
QObject.connect(self.diagnosticThread, SIGNAL("runError(PyQt_PyObject)"), self.runErrorFromThread)
self.diagnosticThread.start()
def on_indicatorBox(self):
indicator_name = str(self.diagnostic_indicator_name.currentText())
dataset_name = self.diagnostic_dataset_name.currentText()
indicator_type = str(self.diagnostic_indicator_type.currentText())
year = str(self.diagnostic_year.currentText())
if year == '': return
year = int(year)
if dataset_name not in self.spatial_datasets and indicator_type == 'map':
MessageBox.warning(
mainwindow=self.mainwindow,
text="That indicator cannot be visualized as a map.",
detailed_text=(
'The dataset %s is either not spatial or cannot be '
'rendered as a grid. If the latter, please try '
'exporting to an external GIS tool.' % dataset_name))
return
cache_directory = self.model.config['cache_directory']
params = {'indicators': [indicator_name]}
if indicator_type == 'table':
indicator_type = 'tab'
params['output_type'] = 'tab'
# JLM > commented this out for when implemented animations
# else:
# indicator_type = 'mapnik_map'
visualizations = [(indicator_type, str(dataset_name), params)]
self.batch_processor = BatchProcessor(self.project)
self.batch_processor.guiElement = self
self.batch_processor.set_data(visualizations=visualizations,
source_data_name=self.model.run_name,
years=[year, year],
cache_directory=cache_directory)
self.diagnosticThread = OpusGuiThread(
parentThread=self.mainwindow,
parentGuiElement=self,
thread_object=self.batch_processor)
# Use this signal from the thread if it is capable of producing its own status signal
QObject.connect(self.diagnosticThread,
SIGNAL("runFinished(PyQt_PyObject)"),
self.visualizationsCreated)
QObject.connect(self.diagnosticThread,
SIGNAL("runError(PyQt_PyObject)"),
self.runErrorFromThread)
self.diagnosticThread.start()
def load_table(self, visualization, limit=10000):
storage = StorageFactory().get_storage(
type='%s_storage' % visualization.output_type,
storage_location=visualization.storage_location)
table_data = storage.load_table(table_name=visualization.table_name)
try:
primary_keys = visualization.indicators[0].primary_keys
except:
primary_keys = []
keys = primary_keys + [
key for key in table_data.keys() if key not in primary_keys
]
num_rows = min(len(table_data[keys[0]]), limit)
num_cols = len(keys)
self.tableWidget.clear()
self.tableWidget.setColumnCount(num_cols)
self.tableWidget.setRowCount(num_rows)
j = 0
for key in keys:
col = QTableWidgetItem()
col.setText(QString(key))
self.tableWidget.setHorizontalHeaderItem(j, col)
j += 1
self.tableWidget.resizeColumnsToContents()
order = sorted(enumerate(table_data[keys[0]]), lambda (i, v),
(j, v2): int(v * 100) - int(v2 * 100))
for i, (idx, v) in enumerate(order):
row = QTableWidgetItem()
self.tableWidget.setVerticalHeaderItem(i, row)
j = 0
for key in keys:
item = QTableWidgetItem()
item.setText(QString(str(table_data[key][idx])))
self.tableWidget.setItem(i, j, item)
j += 1
if i > limit:
msg = 'The table %s has been truncated to %i rows because of memory limitations.' % (
visualization.table_name, limit)
detailed_msg = '<qt>To view the full results, open the following file:<br><br><small>%s</small></qt>' % visualization.get_file_path(
)
MessageBox.warning(mainwindow=self,
text=msg,
detailed_text=detailed_msg)
break
def _show_problem_variables(self):
''' shows the current list of problem variables '''
if not self.problem_variables:
MessageBox.information(self, 'All tested variables passed.')
return
msg = '<qt>'
for dummy, err_msg in self.problem_variables:
msg += '<br/>'.join(err_msg)
msg += '<br/><br/>'
msg += '</qt>'
txt = 'Found problems with %d of the variables' % len(self.problem_variables)
MessageBox.warning(self, txt, msg)
def __init__(self, parent_widget):
QDialog.__init__(self, parent_widget)
self.setupUi(self)
desc = '\n'.join((
'%(PROTOCOL_TAG)s: Database protocol. Double-click and hold to see the available options.',
'%(HOST_NAME_TAG)s: Host name (and database name for Postgres).',
' * Postgres: Use the format "host_name/database_name" (without quotes).',
' - Leave host_name empty to connect to localhost.',
' - Leave database_name empty (but retain the slash) to connect',
' to the default database for the database user.',
' * Other protocols: Enter the host name only.',
'%(USER_NAME_TAG)s: Database user name',
'%(PASSWORD_TAG)s: Database password', '',
'To add a new database configuration, please edit',
'%(conf_file_path)s'))
desc = QtGui.QApplication.translate("DatabaseSettingsEditGui", desc,
None,
QtGui.QApplication.UnicodeUTF8)
desc = unicode(desc) % dict(
[(n, eval('DatabaseServerConfiguration.%s' % n))
for n in dir(DatabaseServerConfiguration)
if re.match('.*_TAG$', n)] +
[('conf_file_path',
DatabaseServerConfiguration.get_default_configuration_file_path(
))])
desc = QtCore.QString(desc)
self.description.setText(desc)
self._config_filename = DatabaseServerConfiguration.get_default_configuration_file_path(
)
try:
self.xml_root = ElementTree(file=self._config_filename).getroot()
self.base_widget = self.variableBox
self.xml_controller = XmlController_DatabaseConfig(self)
# Turns out that Qt Garbage collects the model (and delegate) if we don't explicitly
# bind it to a Python object in addition to using the PyQt .setModel() method.
self.tree_view = self.xml_controller.view
return
except IOError, ex:
MessageBox.error(mainwindow=self,
text='Could not initialize Database Settings',
detailed_text=str(ex))
self.xml_root = None
self._config_filename = ''
self.configFile = None
def __init__(self, parent_widget):
QDialog.__init__(self, parent_widget)
self.setupUi(self)
desc = "\n".join(
(
"%(PROTOCOL_TAG)s: Database protocol. Double-click and hold to see the available options.",
"%(HOST_NAME_TAG)s: Host name (and database name for Postgres).",
' * Postgres: Use the format "host_name/database_name" (without quotes).',
" - Leave host_name empty to connect to localhost.",
" - Leave database_name empty (but retain the slash) to connect",
" to the default database for the database user.",
" * Other protocols: Enter the host name only.",
"%(USER_NAME_TAG)s: Database user name",
"%(PASSWORD_TAG)s: Database password",
"",
"To add a new database configuration, please edit",
"%(conf_file_path)s",
)
)
desc = QtGui.QApplication.translate("DatabaseSettingsEditGui", desc, None, QtGui.QApplication.UnicodeUTF8)
desc = unicode(desc) % dict(
[
(n, eval("DatabaseServerConfiguration.%s" % n))
for n in dir(DatabaseServerConfiguration)
if re.match(".*_TAG$", n)
]
+ [("conf_file_path", DatabaseServerConfiguration.get_default_configuration_file_path())]
)
desc = QtCore.QString(desc)
self.description.setText(desc)
self._config_filename = DatabaseServerConfiguration.get_default_configuration_file_path()
try:
self.xml_root = ElementTree(file=self._config_filename).getroot()
self.base_widget = self.variableBox
self.xml_controller = XmlController_DatabaseConfig(self)
# Turns out that Qt Garbage collects the model (and delegate) if we don't explicitly
# bind it to a Python object in addition to using the PyQt .setModel() method.
self.tree_view = self.xml_controller.view
return
except IOError, ex:
MessageBox.error(mainwindow=self, text="Could not initialize Database Settings", detailed_text=str(ex))
self.xml_root = None
self._config_filename = ""
self.configFile = None
def _show_problem_variables(self):
''' shows the current list of problem variables '''
if not self.problem_variables:
MessageBox.information(self, 'All tested variables passed.')
return
msg = '<qt>'
for dummy, err_msg in self.problem_variables:
msg += '<br/>'.join(err_msg)
msg += '<br/><br/>'
msg += '</qt>'
txt = 'Found problems with %d of the variables' % len(
self.problem_variables)
MessageBox.warning(self, txt, msg)
def _validate_names(self, show_error_message = False):
''' go through all nest and equation names and ensure that there are no collisions.
Returns True if all the names are valid and False otherwise.
If @param show_error_message is True an error message of the name errors is displayed.'''
# Check for colliding names among the nest and equations
colliding_names = set()
nodes_to_inspect = self.submodel_node.findall('.//nest')
nodes_to_inspect.extend(self.submodel_node.findall('.//equation'))
for inspected_node in nodes_to_inspect:
# get all sibling names with the same tag
sibling_names = [node.get('name') for node in inspected_node.getparent() if
node is not inspected_node and node.tag == inspected_node.tag]
# if there is a name collision, add the name to the set of found colliding names
if inspected_node.get('name') in sibling_names:
parent_node = inspected_node.getparent()
if parent_node.tag == 'nest':
desc = '<%s>/%s' % (parent_node.get('name'), inspected_node.get('name'))
else:
desc = '%s' % inspected_node.get('name')
colliding_names.add(desc)
# the concept of colliding names might be confusing so we want to be clear on what is
# happening and (more importantly) how to solve it
if colliding_names:
if not show_error_message:
return False
str_collide_list = ''.join(['<li>%s</li>\n' % name for name in colliding_names])
short_msg = 'Name collisions found.'
longer_msg = ''' <qt> Colliding names:
<b> <ul> %s </ul> </b>
<p>A name collision is when there are two items with the same name, the same type and the same level.</p>
For example:
<ul>
<li>MY_NEST</li>
<li><ul><li>MY_EQUATION</li><li>MY_EQUATION</li></ul></li>
</ul>
<p>will cause a name collision, while this example;</p>
<ul>
<li>MY_NEST</li>
<li><ul><li>MY_EQUATION</li></ul></li>
<li>MY_OTHER_NEST</li> <li>
<ul><li>MY_EQUATION</li></ul></li>
</ul>
<p>is fine since the two equations with the same name are on different levels.</p>
<p>To correct this error please give unique names for the above mentioned equations
and/or nests.</p></qt>'''% str_collide_list
MessageBox.warning(self, short_msg, longer_msg)
return False
return True
def load_table(self, visualization, limit = 10000):
storage = StorageFactory().get_storage(
type = '%s_storage'%visualization.output_type,
storage_location = visualization.storage_location)
table_data = storage.load_table(
table_name = visualization.table_name)
try:
primary_keys = visualization.indicators[0].primary_keys
except:
primary_keys = []
keys = primary_keys + [key for key in table_data.keys()
if key not in primary_keys]
num_rows = min(len(table_data[keys[0]]), limit)
num_cols = len(keys)
self.tableWidget.clear()
self.tableWidget.setColumnCount(num_cols)
self.tableWidget.setRowCount(num_rows)
j = 0
for key in keys:
col = QTableWidgetItem()
col.setText(QString(key))
self.tableWidget.setHorizontalHeaderItem(j,col)
j += 1
self.tableWidget.resizeColumnsToContents()
order = sorted(enumerate(table_data[keys[0]]), lambda (i,v),(j,v2): int(v*100)-int(v2*100))
for i, (idx,v) in enumerate(order):
row = QTableWidgetItem()
self.tableWidget.setVerticalHeaderItem(i,row)
j = 0
for key in keys:
item = QTableWidgetItem()
item.setText(QString(str(table_data[key][idx])))
self.tableWidget.setItem(i,j,item)
j += 1
if i > limit:
msg = 'The table %s has been truncated to %i rows because of memory limitations.'%(visualization.table_name,limit)
detailed_msg = '<qt>To view the full results, open the following file:<br><br><small>%s</small></qt>'%visualization.get_file_path()
MessageBox.warning(mainwindow = self,
text = msg,
detailed_text = detailed_msg)
break
def update_model_nested_structure(self, node = None):
'''
Create an XML representation to use for the argument "nested_structure" used by Nested Logit
Models (NLM). The argument is used in the NLMs init() method.
@param node submodel node to construct a nested structure from (default self.submodel_node)
@raise RuntimeError: If the model node could not be updated
@return the created nested_structure node (useful for tests)
'''
node = self.submodel_node if node is None else node
if node.find('nest') is None: # can't create a nested structure if there are no nests
return None
counter = {'current count': 1} # pass object ref to keep increments down the recursive chain
try:
new_nested_structure_node = self._create_nested_structure_xml(node, counter)
except ValueError, ex:
MessageBox.error(self, 'Not all nests have equations assigned to them.', str(ex))
return None
def send_to_urbancanvas(self):
'''
Sends to UrbanCanvas for visualization.
'''
self._update_variable_from_fields()
func = batch_check_data
dummy, result, msgs = func([self.variable,], self.validator)[0]
expression = dummy['definition']
if dummy['dataset'] == 'parcel':
from opus_core.storage_factory import StorageFactory
from opus_core.datasets.dataset_pool import DatasetPool
import os, sys
base_year = self.validator.project.xml_config.get_estimation_configuration()['base_year']
project_name = self.validator.project.name
opus_data_path = self.validator.project.xml_config.get_opus_data_path()
logger.log_note(base_year)
logger.log_note(project_name)
logger.log_note(opus_data_path)
cache = os.path.join(opus_data_path,project_name,'base_year_data',str(base_year))
logger.log_note(cache)
storage = StorageFactory().get_storage('flt_storage',storage_location=cache)
dataset_pool = DatasetPool(storage=storage, package_order=[project_name,'urbansim_parcel','urbansim','opus_core'])
parcels = dataset_pool.get_dataset('parcel')
parcel_ids = pd.Series(parcels.get_attribute('parcel_id'))
values = pd.Series(parcels.compute_variables([expression],dataset_pool=dataset_pool).astype('float'))
parcels = pd.DataFrame({"parcel_id":parcel_ids,"vl_values":values})
#parcels.set_index(keys='parcel_id',inplace=True)
#parcels["vl_values"][parcels["vl_values"]==0] = np.nan
parcels = parcels[parcels["vl_values"]>0]
os.chdir(os.path.join(opus_data_path,project_name))
parcels.to_csv('variable_library_indicator.csv',index=False)
#np.savez('variable_library_indicator',parcel_id=parcels.vl_values.index.values.astype('int32'),values=parcels.vl_values.values.astype('int32'))
##############UNCOMMENT IF WEBSERVICE IS DESIRED
# parcels.save('variable_library.pkl') ##I believe 'save' was just deprectated in pandas- its now to_pickle or some such thing... change this later
# web_service_path = os.path.join(os.getenv("OPUS_HOME"),'src',project_name,'scripts','web_service.py')
# logger.log_note(web_service_path)
# p = subprocess.Popen([sys.executable,web_service_path])
# MessageBox.information(mainwindow = self, text = 'Click OK when done viewing in UrbanCanvas')
# p.kill()
MessageBox.information(mainwindow = self, text = 'Variable exported to the project data directory for viewing in UrbanCanvas')
else:
MessageBox.information(mainwindow = self, text = 'Not a parcel variable. Only parcel variables can be sent to UrbanCanvas')
def send_to_urbancanvas(self):
'''
Sends to UrbanCanvas for visualization.
'''
self._update_variable_from_fields()
func = batch_check_data
dummy, result, msgs = func([self.variable,], self.validator)[0]
expression = dummy['definition']
if dummy['dataset'] == 'parcel':
from opus_core.storage_factory import StorageFactory
from opus_core.datasets.dataset_pool import DatasetPool
import os, sys
base_year = self.validator.project.xml_config.get_estimation_configuration()['base_year']
project_name = self.validator.project.name
opus_data_path = self.validator.project.xml_config.get_opus_data_path()
logger.log_note(base_year)
logger.log_note(project_name)
logger.log_note(opus_data_path)
cache = os.path.join(opus_data_path,project_name,'base_year_data',str(base_year))
logger.log_note(cache)
storage = StorageFactory().get_storage('flt_storage',storage_location=cache)
dataset_pool = DatasetPool(storage=storage, package_order=[project_name,'urbansim_parcel','urbansim','opus_core'])
parcels = dataset_pool.get_dataset('parcel')
parcel_ids = pd.Series(parcels.get_attribute('parcel_id'))
values = pd.Series(parcels.compute_variables([expression],dataset_pool=dataset_pool).astype('float'))
parcels = pd.DataFrame({"parcel_id":parcel_ids,"vl_values":values})
parcels.set_index(keys='parcel_id',inplace=True)
#parcels["vl_values"][parcels["vl_values"]==0] = np.nan
parcels = parcels[parcels["vl_values"]>0]
os.chdir(os.path.join(opus_data_path,project_name))
np.savez('variable_library_indicator',parcel_id=parcels.vl_values.index.values.astype('int32'),values=parcels.vl_values.values.astype('int32'))
##############UNCOMMENT IF WEBSERVICE IS DESIRED
# parcels.save('variable_library.pkl') ##I believe 'save' was just deprectated in pandas- its now to_pickle or some such thing... change this later
# web_service_path = os.path.join(os.getenv("OPUS_HOME"),'src',project_name,'scripts','web_service.py')
# logger.log_note(web_service_path)
# p = subprocess.Popen([sys.executable,web_service_path])
# MessageBox.information(mainwindow = self, text = 'Click OK when done viewing in UrbanCanvas')
# p.kill()
MessageBox.information(mainwindow = self, text = 'Variable exported to the project data directory for viewing in UrbanCanvas')
else:
MessageBox.information(mainwindow = self, text = 'Not a parcel variable. Only parcel variables can be sent to UrbanCanvas')
def _update_variable_from_fields(self):
''' update the variable with values from the gui widgets '''
self.variable['name'] = str(self.leVarName.text())
self.variable['source'] = str(self.cboVarType.currentText())
self.variable['definition'] = str(
self.le_var_def.document().toPlainText())
try:
v = VariableName(self.variable['definition'])
dataset_name = v.get_dataset_name()
interaction_set_names = v.get_interaction_set_names()
except (SyntaxError, ValueError):
MessageBox.error(
mainwindow=self,
text='parse error for variable',
detailed_text=
'setting dataset name for this variable to <unknown>')
dataset_name = '<unknown>'
interaction_set_names = None
if dataset_name is None and interaction_set_names is not None:
# It's an interaction set. Look up possible names in available_datasets
names = get_available_dataset_names(self.validator.project)
n1 = interaction_set_names[0] + '_x_' + interaction_set_names[1]
if n1 in names:
dataset_name = n1
else:
n2 = interaction_set_names[1] + '_x_' + interaction_set_names[0]
if n2 in names:
dataset_name = n2
else:
MessageBox.error(
mainwindow=self,
text=
'unable to find an interaction set in available_datasets for this variable',
detailed_text=
"tried %s and %s \nbut couldn't find either name in available_datasets \nsetting dataset_name to <unknown>"
% (n1, n2))
dataset_name = '<unknown>'
self.variable['dataset'] = dataset_name
if self.rbUseModel.isChecked():
self.variable['use'] = 'model variable'
elif self.rbUseIndicator.isChecked():
self.variable['use'] = 'indicator'
else:
self.variable['use'] = 'both'
def okToCloseProject(self):
'''
Called before an operation that causes this project to close.
If the project contains changes; ask if the user wants to save them,
discard them or cancel the operation.
@return: True if the user wishes to proceed with the closing operation.
'''
if not self.project.dirty: return True
question = 'Do you want to save your changes before closing the project?'
user_answer = common_dialogs.save_before_close(question)
if user_answer == common_dialogs.YES:
ok_flag, msg = self.project.save() # cancels on failed save
if not ok_flag:
MessageBox.error(self, "Could not save project", str(msg))
return False
return True
elif user_answer == common_dialogs.NO:
self.project.dirty = False
return True
else:
return False
def check_variable(self, check = 'syntax'):
'''
validate the variable and display the result in i dialog box
if check_syntax is True a syntax check is performed, otherwise a data
check is performed.
'''
if check == 'syntax':
func = batch_check_syntax
elif check == 'data':
func = batch_check_data
else:
raise ValueError('check_variable() got an unknown value for argument "check"; "%s"' % check)
self._update_variable_from_fields()
dummy, result, msgs = func([self.variable,], self.validator)[0]
if result is True:
text = '%s check OK' % check
MessageBox.information(mainwindow = self, text = text)
else:
text = 'Encountered a %s error' % check
MessageBox.warning(mainwindow = self, text = text, detailed_text = '\n '.join(msgs))
def _scanForRuns(self):
run_manager = get_run_manager()
run_manager.clean_runs()
run_manager.close()
added_runs, removed_runs = update_available_runs(self.project)
added_msg = removed_msg = None
if len(added_runs) > 0:
added_msg = ('The following simulation runs have been '
'automatically added to the results manager:\n\n%s'
% '\n'.join(added_runs))
if len(removed_runs) > 0:
removed_msg = ('The following simulation runs have been '
'automatically removed from the results manager:\n\n%s'
% '\n'.join(removed_runs))
if added_msg or removed_msg:
self.project.dirty = True
text = 'The list of simulation runs has been automatically updated.'
detailed_text = '%s\n\n%s' % (added_msg or '', removed_msg or '')
MessageBox.information(mainwindow = self.base_widget,
text = text,
detailed_text = detailed_text)
def _scanForRuns(self):
run_manager = get_run_manager()
run_manager.clean_runs()
run_manager.close()
added_runs, removed_runs = update_available_runs(self.project)
added_msg = removed_msg = None
if len(added_runs) > 0:
added_msg = ('The following simulation runs have been '
'automatically added to the results manager:\n\n%s' %
'\n'.join(added_runs))
if len(removed_runs) > 0:
removed_msg = (
'The following simulation runs have been '
'automatically removed from the results manager:\n\n%s' %
'\n'.join(removed_runs))
if added_msg or removed_msg:
self.project.dirty = True
text = 'The list of simulation runs has been automatically updated.'
detailed_text = '%s\n\n%s' % (added_msg or '', removed_msg or '')
MessageBox.information(mainwindow=self.base_widget,
text=text,
detailed_text=detailed_text)
def _update_variable_from_fields(self):
''' update the variable with values from the gui widgets '''
self.variable['name'] = str(self.leVarName.text())
self.variable['source'] = str(self.cboVarType.currentText())
self.variable['definition'] = str(self.le_var_def.document().toPlainText())
try:
v = VariableName(self.variable['definition'])
dataset_name = v.get_dataset_name()
interaction_set_names = v.get_interaction_set_names()
except (SyntaxError, ValueError):
MessageBox.error(mainwindow = self,
text = 'parse error for variable',
detailed_text = 'setting dataset name for this variable to <unknown>')
dataset_name = '<unknown>'
interaction_set_names = None
if dataset_name is None and interaction_set_names is not None:
# It's an interaction set. Look up possible names in available_datasets
names = get_available_dataset_names(self.validator.project)
n1 = interaction_set_names[0] + '_x_' + interaction_set_names[1]
if n1 in names:
dataset_name = n1
else:
n2 = interaction_set_names[1] + '_x_' + interaction_set_names[0]
if n2 in names:
dataset_name = n2
else:
MessageBox.error(mainwindow = self,
text = 'unable to find an interaction set in available_datasets for this variable',
detailed_text = "tried %s and %s \nbut couldn't find either name in available_datasets \nsetting dataset_name to <unknown>" % (n1,n2) )
dataset_name = '<unknown>'
self.variable['dataset'] = dataset_name
if self.rbUseModel.isChecked():
self.variable['use'] = 'model variable'
elif self.rbUseIndicator.isChecked():
self.variable['use'] = 'indicator'
else:
self.variable['use'] = 'both'
def update_model_nested_structure(self, node=None):
'''
Create an XML representation to use for the argument "nested_structure" used by Nested Logit
Models (NLM). The argument is used in the NLMs init() method.
@param node submodel node to construct a nested structure from (default self.submodel_node)
@raise RuntimeError: If the model node could not be updated
@return the created nested_structure node (useful for tests)
'''
node = self.submodel_node if node is None else node
if node.find(
'nest'
) is None: # can't create a nested structure if there are no nests
return None
counter = {
'current count': 1
} # pass object ref to keep increments down the recursive chain
try:
new_nested_structure_node = self._create_nested_structure_xml(
node, counter)
except ValueError, ex:
MessageBox.error(self,
'Not all nests have equations assigned to them.',
str(ex))
return None
def on_buttonBox_accepted(self):
path = str(self.lePath.text())
if not os.path.exists(path):
msg = 'Cannot import, %s does not exist' % path
logger.log_warning(msg)
MessageBox.warning(mainwindow=self, text=msg, detailed_text='')
else:
cache_directory = path
years = []
for dir in os.listdir(cache_directory):
if len(dir) == 4 and dir.isdigit():
years.append(int(dir))
if years == []:
msg = 'Cannot import, %s has no run data' % path
logger.log_warning(msg)
MessageBox.warning(mainwindow=self, text=msg, detailed_text='')
else:
start_year = min(years)
end_year = max(years)
project_name = os.environ['OPUSPROJECTNAME']
run_name = os.path.basename(path)
server_config = ServicesDatabaseConfiguration()
run_manager = RunManager(server_config)
run_id = run_manager._get_new_run_id()
resources = {
'cache_directory': cache_directory,
'description': '',
'years': (start_year, end_year),
'project_name': project_name
}
try:
run_manager.add_row_to_history(run_id=run_id,
resources=resources,
status='done',
run_name=run_name)
update_available_runs(self.project)
logger.log_status(
'Added run %s of project %s to run_activity table' %
(run_name, project_name))
except:
errorInfo = formatExceptionInfo()
logger.log_error(errorInfo)
MessageBox.error(
mainwindow=self,
text=
'Could not add run %s of project %s to run_activity table'
% (run_name, project_name),
detailed_text=errorInfo)
self.close()
def on_buttonBox_accepted(self):
path = str(self.lePath.text())
if not os.path.exists(path):
msg = 'Cannot import, %s does not exist' % path
logger.log_warning(msg)
MessageBox.warning(mainwindow = self,
text = msg,
detailed_text = '')
else:
cache_directory = path
years = []
for dir in os.listdir(cache_directory):
if len(dir) == 4 and dir.isdigit():
years.append(int(dir))
if years == []:
msg = 'Cannot import, %s has no run data'%path
logger.log_warning(msg)
MessageBox.warning(mainwindow = self,
text = msg,
detailed_text = '')
else:
start_year = min(years)
end_year = max(years)
project_name = os.environ['OPUSPROJECTNAME']
run_name = os.path.basename(path)
server_config = ServicesDatabaseConfiguration()
run_manager = RunManager(server_config)
run_id = run_manager._get_new_run_id()
resources = {
'cache_directory': cache_directory,
'description': '',
'years': (start_year, end_year),
'project_name': project_name
}
try:
run_manager.add_row_to_history(run_id = run_id,
resources = resources,
status = 'done',
run_name = run_name)
update_available_runs(self.project)
logger.log_status('Added run %s of project %s to run_activity table'%(run_name, project_name))
except:
errorInfo = formatExceptionInfo()
logger.log_error(errorInfo)
MessageBox.error(mainwindow = self,
text = 'Could not add run %s of project %s to run_activity table'%(run_name, project_name),
detailed_text = errorInfo)
self.close()
def _run_error(self,errorMessage):
text = 'Error in computing or displaying indicator'
MessageBox.error(mainwindow = self, text = text, detailed_text = errorMessage)
def _sync_base_year_data(self, run_manager=None):
"""
synchronize base_year_data information in xml_configuration with
run_activity table. Information in xml_configuration takes
precedent, because we assume users don't directly modify data in
serivecs.run_activity table
"""
# TODO baseyear_dir is somewhat hard-coded; it would be better to read
# from xml_configuration instead, but there is no such node currently
run_name = 'base_year_data'
baseyear_dir = os.path.join(self.project.data_path(), run_name)
baseyear_dir = os.path.normpath(baseyear_dir)
if not os.path.exists(baseyear_dir):
MessageBox.warning(mainwindow = self.base_widget,
text="base_year_data directory %s doesn't exist. " % baseyear_dir
)
return
import glob
years = [int(os.path.basename(year_dir)) for year_dir in
glob.glob(os.path.join(baseyear_dir, '[0-9][0-9][0-9][0-9]'))]
if not years:
MessageBox.warning(mainwindow = self.base_widget,
text="base_year_data directory %s doesn't contain any year sub-directory. " % baseyear_dir
)
return
start_year = min(years)
end_year = max(years)
base_year = end_year # default to the last year in baseyear_dir
# and update it with information found in scenario_manager
scenario_manager_node = self.project.find('scenario_manager')
for base_year_node in scenario_manager_node.findall('.//base_year'):
try:
base_year = int(base_year_node.text.strip())
break
except (TypeError, ValueError):
continue
resources = {
'cache_directory': baseyear_dir,
'description': 'base year data',
'base_year': base_year,
'years': (start_year, end_year)
}
if run_manager is None: run_manager = get_run_manager()
base_year_data_db = run_manager.get_runs(run_name=run_name,
process_name=get_host_name())
if len(base_year_data_db) == 0:
run_id = run_manager._get_new_run_id()
elif len(base_year_data_db) >= 1:
for idx, row in enumerate(base_year_data_db):
if idx==0:
run_id = row[0]
else:
run_manager.delete_everything_for_this_run(row[0])
resources_db = run_manager.get_resources_for_run_id_from_history(run_id)
if resources_db.get('cache_directory', '') == baseyear_dir and \
resources_db.get('base_year', -1) == base_year:
#all good, we don't need to do anything
return
else:
resources_db.merge(resources)
resources = resources_db
run_manager.add_row_to_history(run_id = run_id,
resources = resources,
status = 'done',
run_name = run_name)
def on_pb_urbancanvas_clicked(self):
run_name = self.current_run
indicator_name = self.current_indicator
indicator_dataset = self.current_indicator_dataset
if indicator_dataset != 'parcel':
MessageBox.information(
mainwindow=self,
text=
'Not a parcel variable. Only parcel variables can be sent to UrbanCanvas'
)
else:
start_year = int(self.current_year)
end_year = start_year
if run_name is None or indicator_name is None or start_year is None:
return
key = (run_name, indicator_name, start_year)
self.pb_urbancanvas.setText('Sending to UrbanCanvas...')
indicator_nodes = get_available_indicator_nodes(self.project)
dataset = None
for indicator_node in indicator_nodes:
ind_dataset, name = get_variable_dataset_and_name(
indicator_node)
if name == indicator_name and ind_dataset == indicator_dataset:
dataset = ind_dataset
break
if dataset is None:
raise Exception('Could not find dataset for indicator %s' %
indicator_name)
table_params = {
'name': None,
'output_type': 'tab',
'indicators': [indicator_name],
}
expression_library = self.project.xml_config.get_expression_library(
)
expression = expression_library[(dataset, name)]
logger.log_note(expression)
base_year = end_year
project_name = self.project.name
opus_data_path = self.project.xml_config.get_opus_data_path()
logger.log_note(base_year)
logger.log_note(project_name)
logger.log_note(opus_data_path)
interface = IndicatorFrameworkInterface(self.project)
source_data = interface.get_source_data(source_data_name=run_name,
years=[
end_year,
])
cache = os.path.join(source_data.cache_directory, str(end_year))
logger.log_note(cache)
storage = StorageFactory().get_storage('flt_storage',
storage_location=cache)
dataset_pool = DatasetPool(storage=storage,
package_order=[
project_name, 'urbansim_parcel',
'urbansim', 'opus_core'
])
parcels = dataset_pool.get_dataset('parcel')
parcel_ids = pd.Series(parcels.get_attribute('parcel_id'))
values = pd.Series(
parcels.compute_variables(
[expression], dataset_pool=dataset_pool).astype('float'))
parcels = pd.DataFrame({
"parcel_id": parcel_ids,
"vl_values": values
})
#parcels.set_index(keys='parcel_id',inplace=True)
#parcels["vl_values"][parcels["vl_values"]==0] = np.nan
parcels = parcels[parcels["vl_values"] > 0]
os.chdir(os.path.join(opus_data_path, project_name))
parcels.to_csv('results_browser_indicator.csv', index=False)
#np.savez('results_browser_indicator',parcel_id=parcels.vl_values.index.values.astype('int32'),values=parcels.vl_values.values.astype('int32'))
##############UNCOMMENT IF WEBSERVICE IS DESIRED
# parcels.save('variable_library.pkl') ##I believe 'save' was just deprectated in pandas- its now to_pickle or some such thing... change this later
# web_service_path = os.path.join(os.getenv("OPUS_HOME"),'src',project_name,'scripts','web_service.py')
# logger.log_note(web_service_path)
# p = subprocess.Popen([sys.executable,web_service_path])
# MessageBox.information(mainwindow = self, text = 'Click OK when done viewing in UrbanCanvas')
# p.kill()
# self.pb_urbancanvas.setText('View in UrbanCanvas')
MessageBox.information(
mainwindow=self,
text=
'Variable exported to the project data directory for viewing in UrbanCanvas'
)
self.pb_urbancanvas.setText('View in UrbanCanvas')
def _run_error(self, errorMessage):
text = 'Error in computing or displaying indicator'
MessageBox.error(mainwindow=self,
text=text,
detailed_text=errorMessage)
