def test_DataMenu_import_data(mocker, tmpdir, core, project, module_menu):
# Make a source directory with some files
config_tmpdir = tmpdir.mkdir("config")
mock_dir = Directory(str(config_tmpdir))
mocker.patch("dtocean_core.utils.database.UserDataDirectory",
return_value=mock_dir)
data_menu = DataMenu()
var_tree = Tree()
project = deepcopy(project)
mod_name = "Mock Module"
core.register_level(project, core._markers["register"], "Mock Module")
module_menu.activate(core, project, mod_name)
mod_branch = var_tree.get_branch(core, project, mod_name)
mod_branch.read_test_data(core, project,
os.path.join(dir_path, "inputs_wp2_tidal.pkl"))
pre_length = len(project._pool)
dts_path = os.path.join(str(tmpdir), "test.dts")
data_menu.export_data(core, project, dts_path)
assert os.path.isfile(dts_path)
data_menu.import_data(core, project, dts_path)
assert len(project._pool) > pre_length
def test_filter_interface(core, project, tree):
project = deepcopy(project)
data_menu = DataMenu()
module_menu = ModuleMenu()
project_menu = ProjectMenu()
tree = Tree()
data_menu.select_database(project, "local")
project_menu.initiate_pipeline(core, project)
project_menu.initiate_options(core, project)
filter_branch = tree.get_branch(core, project,
'Database Filtering Interface')
new_var = filter_branch.get_input_variable(core, project,
"device.selected_name")
new_var.set_raw_interface(core, "ExampleWaveDevice")
new_var.read(core, project)
module_menu.activate(core, project, "Hydrodynamics")
project_menu.initiate_filter(core, project)
assert True
def _set_variables(self, box_number):
self.varBox.clear()
if box_number < 0: return
interface_name = self._mod_names[box_number]
tree = Tree()
if self._var_ids is None:
var_id_dict = {}
else:
var_id_dict = self._var_ids
branch = tree.get_branch(self._shell.core, self._shell.project,
interface_name)
var_inputs = branch.get_inputs(self._shell.core, self._shell.project)
unique_vars = OrderedSet(var_inputs)
var_names = []
for var_id in unique_vars:
var_meta = self._shell.core.get_metadata(var_id)
if "SimpleData" in var_meta.structure:
if var_meta.types is None:
errStr = ("Variable {} with SimpleData structure "
"requires types meta data to be "
"set").format(var_id)
raise ValueError(errStr)
if ("int" in var_meta.types or "float" in var_meta.types
or "str" in var_meta.types):
title = var_meta.title
if var_meta.units is not None:
title = "{} ({})".format(title, var_meta.units[0])
var_names.append(title)
if title not in var_id_dict:
var_id_dict[title] = var_id
self._var_ids = var_id_dict
self.varBox.addItems(var_names)
self.varBox.setCurrentIndex(-1)
return
def _set_variables(self, shell, active_interfaces, include_str=False):
self.varBox.clear()
tree = Tree()
all_var_names = []
var_id_dict = {}
for interface_name in active_interfaces:
branch = tree.get_branch(shell.core, shell.project, interface_name)
var_inputs = branch.get_inputs(shell.core, shell.project)
var_outputs = branch.get_outputs(shell.core, shell.project)
unique_vars = OrderedSet(var_inputs + var_outputs)
var_names = []
var_names = []
for var_id in unique_vars:
var_meta = shell.core.get_metadata(var_id)
if "SimpleData" in var_meta.structure:
if var_meta.types is None:
errStr = ("Variable {} with SimpleData structure "
"requires types meta data to be "
"set").format(var_id)
raise ValueError(errStr)
if "int" in var_meta.types or "float" in var_meta.types:
var_names.append(var_meta.title)
if include_str and "str" in var_meta.types:
var_names.append(var_meta.title)
if var_meta.title not in var_id_dict:
var_id_dict[var_meta.title] = var_id
all_var_names.extend(var_names)
self._var_ids = var_id_dict
self.varBox.addItems(all_var_names)
self.varBox.setCurrentIndex(-1)
return
def test_ModuleMenu_execute_current_nothemes(core, project, module_menu):
var_tree = Tree()
project = deepcopy(project)
mod_name = "Mock Module"
module_menu.activate(core, project, mod_name)
mod_branch = var_tree.get_branch(core, project, mod_name)
mod_branch.read_test_data(core, project,
os.path.join(dir_path, "inputs_wp2_tidal.pkl"))
module_menu.execute_current(core, project, execute_themes=False)
assert True
def __init__(self, title,
hub_name,
branch_cls,
active=True,
branch_order=None):
self._title = title
self._tree = Tree()
self._hub_name = hub_name
self._BranchCls = branch_cls
self._active = active
self._branch_order = branch_order
self._items = []
return
def wave_project(core):
'''Share a Project object'''
project_menu = ProjectMenu()
var_tree = Tree()
new_project = project_menu.new_project(core, "test wave")
options_branch = var_tree.get_branch(core, new_project,
"System Type Selection")
device_type = options_branch.get_input_variable(core, new_project,
"device.system_type")
device_type.set_raw_interface(core, "Wave Floating")
device_type.read(core, new_project)
return new_project
def __init__(self, parent):
super(PipeLine, self).__init__(parent)
self._tree = Tree()
# Root branches
self._branch_map = None
self._items = []
# Test data picker
self._test_data_picker = TestDataPicker(self)
self._test_data_picker.setModal(True)
self._active_thread = None
self._init_title()
return
def __init__(self, parent):
super(PipeLine, self).__init__(parent)
self._tree = Tree()
self._model = None
self._proxy = None
# Root branches
self._branch_map = None
self._controls = []
# Test data picker
self._test_data_picker = TestDataPicker(self)
self._test_data_picker.setModal(True)
self._init_model()
self._init_title()
return
def __init__(self,
address,
title,
view,
model,
proxy,
hub_name,
branch_cls,
active=True,
branch_order=None):
super(HubControl, self).__init__(address, title, view, model, proxy)
self._tree = Tree()
self._hub_name = hub_name
self._BranchCls = branch_cls
self._active = active
self._branch_order = branch_order
return
def project(core):
'''Share a Project object'''
project_title = "Test"
project_menu = ProjectMenu()
var_tree = Tree()
new_project = project_menu.new_project(core, project_title)
options_branch = var_tree.get_branch(core, new_project,
"System Type Selection")
device_type = options_branch.get_input_variable(core, new_project,
"device.system_type")
device_type.set_raw_interface(core, "Tidal Fixed")
device_type.read(core, new_project)
project_menu.initiate_pipeline(core, new_project)
return new_project
def test_options_interface_sites(core, wave_project):
project_menu = ProjectMenu()
var_tree = Tree()
interface_name = "Site and System Options"
project = deepcopy(wave_project)
project_menu.activate(core, project, interface_name)
site_system_branch = var_tree.get_branch(core, project, interface_name)
available_sites = site_system_branch.get_input_variable(
core, project, "hidden.available_sites")
available_sites_dict = {
"id": [1, 2],
"site_name": ["a", "b"],
"lease_area_proj4_string": ["a", "b"]
}
available_sites_df = pd.DataFrame(available_sites_dict)
available_sites.set_raw_interface(core, available_sites_df)
available_sites.read(core, project)
can_execute = project_menu.is_executable(core, project, interface_name)
if not can_execute:
inputs = site_system_branch.get_input_status(core, project)
pprint(inputs)
assert can_execute
connector = _get_connector(project, "project")
interface = connector.get_interface(core, project, interface_name)
interface.connect()
assert interface.data.site_names.equals(available_sites_df["site_name"])
assert interface.data.corridor_selected == False
assert interface.data.lease_selected == False
def test_ThemeMenu_execute_all(core, project, theme_menu):
var_tree = Tree()
project = deepcopy(project)
theme_menu.activate(core, project, "Mock Theme")
theme_menu.activate(core, project, "Mock Theme 2")
theme_menu.execute_all(core, project)
theme_branch = var_tree.get_branch(core, project, "Mock Theme")
var = theme_branch.get_output_variable(core, project,
'project.capex_total')
result = var.get_value(core, project)
assert result == 1
theme_branch = var_tree.get_branch(core, project, "Mock Theme 2")
var = theme_branch.get_output_variable(core, project,
'project.discounted_capex')
result = var.get_value(core, project)
assert result == 2
def test_system_type_interface(core, wave_project):
project_menu = ProjectMenu()
var_tree = Tree()
interface_name = "System Type Selection"
project = deepcopy(wave_project)
system_type_branch = var_tree.get_branch(core, project, interface_name)
can_execute = project_menu.is_executable(core, project, interface_name)
if not can_execute:
inputs = system_type_branch.get_input_status(core, project)
pprint(inputs)
assert can_execute
connector = _get_connector(project, "project")
interface = connector.get_interface(core, project, interface_name)
interface.connect()
assert interface.data.output == True
def test_options_interface_systems(core, wave_project):
project_menu = ProjectMenu()
var_tree = Tree()
interface_name = "Site and System Options"
project = deepcopy(wave_project)
project_menu.activate(core, project, interface_name)
site_system_branch = var_tree.get_branch(core, project, interface_name)
available_systems = site_system_branch.get_input_variable(
core, project, "hidden.available_systems")
available_systems_dict = {
"id": [1, 2],
"description": ["a", "b"],
"device_type": ["a", "Wave Floating"]
}
available_systems_df = pd.DataFrame(available_systems_dict)
available_systems.set_raw_interface(core, available_systems_df)
available_systems.read(core, project)
can_execute = project_menu.is_executable(core, project, interface_name)
if not can_execute:
inputs = site_system_branch.get_input_status(core, project)
pprint(inputs)
assert can_execute
connector = _get_connector(project, "project")
interface = connector.get_interface(core, project, interface_name)
interface.connect()
assert interface.data.system_names.equals(pd.Series(["b"]))
class HubRoot(object):
def __init__(self, title,
hub_name,
branch_cls,
active=True,
branch_order=None):
self._title = title
self._tree = Tree()
self._hub_name = hub_name
self._BranchCls = branch_cls
self._active = active
self._branch_order = branch_order
self._items = []
return
def _activate(self, parent, shell, hub_branches):
if self._branch_order is None:
branch_order = hub_branches
else:
branch_order = [branch for branch in self._branch_order
if branch in hub_branches]
for branch_name in branch_order:
branch = self._tree.get_branch(shell.core,
shell.project,
branch_name)
new_item = self._BranchCls(parent,
branch,
self._title,
branch_name)
if self._active:
new_item._activate(shell)
else:
new_item.setDisabled(True)
self._items.append(new_item)
return
def tree():
'''Share a Tree object'''
new_tree = Tree()
return new_tree
class HubControl(BaseControl):
def __init__(self,
address,
title,
view,
model,
proxy,
hub_name,
branch_cls,
active=True,
branch_order=None):
super(HubControl, self).__init__(address, title, view, model, proxy)
self._tree = Tree()
self._hub_name = hub_name
self._BranchCls = branch_cls
self._active = active
self._branch_order = branch_order
return
def _activate(self, shell, parent):
retrieve_branch = True
if (self._hub_name in shell.queued_interfaces
and shell.queued_interfaces[self._hub_name] is not None):
hub_branches = shell.queued_interfaces[self._hub_name]
retrieve_branch = False
else:
hub_branches = self._tree.get_available_branches(
shell.core, shell.project, [self._hub_name])
if self._branch_order is None:
branch_order = hub_branches
else:
branch_order = [
branch for branch in self._branch_order
if branch in hub_branches
]
for branch_name in branch_order:
if retrieve_branch:
branch = self._tree.get_branch(shell.core, shell.project,
branch_name)
else:
branch = None
# Model item
address = self._address + "." + branch_name
user_dict = {
"address": address,
"visible": True,
"section": "branch",
"status": None
}
name_item = QtGui.QStandardItem(branch_name)
name_item.setData(address, QtCore.Qt.UserRole)
name_item.setData((user_dict, ), 33)
parent.appendRow(name_item)
# Controller
new_control = self._BranchCls(address, branch_name, self._view,
self._model, self._proxy,
self._title, branch)
new_control._init_ui(name_item)
if self._active:
new_control._activate(shell, name_item)
else:
name_item.setFlags(QtCore.Qt.NoItemFlags)
self._controls.append(new_control)
return
class PipeLine(PipeLineDock):
error_detected = QtCore.pyqtSignal(object, object, object)
def __init__(self, parent):
super(PipeLine, self).__init__(parent)
self._tree = Tree()
self._model = None
self._proxy = None
# Root branches
self._branch_map = None
self._controls = []
# Test data picker
self._test_data_picker = TestDataPicker(self)
self._test_data_picker.setModal(True)
self._init_model()
self._init_title()
return
def _init_model(self):
self._model = QtGui.QStandardItemModel()
self._model.setColumnCount(1)
self._proxy = PipelineFilterProxyModel()
self._proxy.setSourceModel(self._model)
self._proxy.setDynamicSortFilter(True)
self.treeView.setModel(self._proxy)
self.filterLineEdit.textChanged.connect(self._update_filter)
self.clearButton.clicked.connect(self._clear_filter)
return
def _init_title(self):
self.setWindowTitle("Pipeline")
self.treeView.setSortingEnabled(False)
self._model.setHeaderData(0, QtCore.Qt.Horizontal, "Waiting...")
return
def _set_branch_map(self, branch_map):
self._branch_map = branch_map
return
def _set_title(self, title):
self._model.setHeaderData(0, QtCore.Qt.Horizontal, title)
return
def _draw(self, shell):
self._clear()
section_address = ""
for branch_dict in self._branch_map:
HubCls = branch_dict["hub"]
args = []
if "args" in branch_dict: args.extend(branch_dict["args"])
# Model item
if HubCls == SectionControl:
section_address = branch_dict["name"]
address = section_address
section = "section"
else:
address = section_address + "." + branch_dict["name"]
section = "hub"
user_dict = {
"address": address,
"visible": True,
"section": section,
"status": None
}
name_item = QtGui.QStandardItem(branch_dict["name"])
name_item.setData(address, QtCore.Qt.UserRole)
name_item.setData((user_dict, ), 33)
self._model.appendRow(name_item)
# Controller
new_control = HubCls(address, branch_dict["name"], self.treeView,
self._model, self._proxy, *args)
new_control._init_ui(name_item)
new_control._activate(shell, name_item)
self._controls.append(new_control)
return
def _expand(self, shell):
for controller in self._controls:
controller._expand(shell)
return
def _refresh(self, shell, expand=True):
self._draw(shell)
if expand: self._expand(shell)
return
def _clear(self):
root = self._model.invisibleRootItem()
root.removeRows(0, root.rowCount())
self._controls = []
return
def _find_controller(self,
proxy_index=None,
controller_title=None,
controller_class=None,
root=None):
if proxy_index is None and controller_title is None:
err_str = ("Either argument 'proxy_index' or 'controller_title' "
"must be provided and 'proxy_index' will take "
"precedence")
raise ValueError(err_str)
if root is None: root = self
if proxy_index is not None:
match = self._proxy.data(proxy_index, QtCore.Qt.UserRole)
search_attr = "_address"
else:
match = controller_title
search_attr = "_title"
for controller in root._controls:
if getattr(controller, search_attr) == match:
if (controller_class is None
or isinstance(controller, controller_class)):
return controller
result = self._find_controller(proxy_index, controller_title,
controller_class, controller)
if result is not None: return result
return None
def _make_menus(self, shell, position):
proxy_indexes = self.treeView.selectedIndexes()
proxy_index = proxy_indexes[0]
controller = self._find_controller(proxy_index)
if isinstance(controller, InputBranchControl):
# Set the widget action
self._test_data_picker.buttonBox.button(
QtGui.QDialogButtonBox.Ok).clicked.disconnect()
self._test_data_picker.buttonBox.button(
QtGui.QDialogButtonBox.Ok).clicked.connect(
lambda: self._read_test_data(shell, controller))
self._test_data_picker.buttonBox.button(
QtGui.QDialogButtonBox.Ok).clicked.connect(
self._test_data_picker.close)
# Build module and theme context menu
if controller._hub_title in ["Modules", "Assessment"]:
menu = QtGui.QMenu()
# Enable tooltips
tips = lambda action: QtGui.QToolTip.showText(
QtGui.QCursor.pos(), action.toolTip(), menu,
menu.actionGeometry(action))
menu.hovered.connect(tips)
if controller._hub_title == "Modules":
# Can't reset or insepct unless the interface has executed
connector = _get_connector(shell.project, "modules")
active = connector.is_interface_completed(
shell.core, shell.project, controller._title)
action = menu.addAction('Inspect',
lambda: controller._inspect(shell))
action.setToolTip('Inspect results following '
'execution of this module')
action.setEnabled(active)
action = menu.addAction('Reset',
lambda: controller._reset(shell))
action.setToolTip('Reset simulation prior to '
'execution of this module')
action.setEnabled(active)
menu.addAction('Load test data...',
self._test_data_picker.show)
menu.exec_(self.treeView.mapToGlobal(position))
elif isinstance(controller, OutputBranchControl):
# Build module context menu
if controller._hub_title == "Modules":
menu = QtGui.QMenu()
menu.addAction('Inspect', lambda: controller._inspect(shell))
menu.exec_(self.treeView.mapToGlobal(position))
return
def _set_top_item(self):
index = self._controls[0]._get_index_from_address()
proxy_index = self._proxy.mapFromSource(index)
self.treeView.clicked.emit(proxy_index)
return
def _read_auto(self, shell):
self._tree.read_auto(shell.core, shell.project)
return
@QtCore.pyqtSlot(str)
def _update_filter(self, text):
search = QtCore.QRegExp(text, QtCore.Qt.CaseInsensitive,
QtCore.QRegExp.RegExp)
self._proxy.setFilterRegExp(search)
return
@QtCore.pyqtSlot(str)
def _repeat_filter(self):
text = self.filterLineEdit.text()
search = QtCore.QRegExp(text, QtCore.Qt.CaseInsensitive,
QtCore.QRegExp.RegExp)
self._proxy.setFilterRegExp(search)
return
@QtCore.pyqtSlot()
def _clear_filter(self):
self.filterLineEdit.clear()
self._update_filter("")
return
@QtCore.pyqtSlot(object, object)
def _read_test_data(self, shell, controller):
if shell._active_thread is not None: shell._active_thread.wait()
shell.read_test_data(controller,
str(self._test_data_picker.pathLineEdit.text()),
self._test_data_picker.overwriteBox.isChecked())
shell._active_thread.error_detected.connect(self._emit_error)
return
@QtCore.pyqtSlot(object, object, object)
def _emit_error(self, etype, evalue, etraceback):
self.error_detected.emit(etype, evalue, etraceback)
return
class PipeLine(PipeLineDock):
error_detected = QtCore.pyqtSignal(object, object, object)
def __init__(self, parent):
super(PipeLine, self).__init__(parent)
self._tree = Tree()
# Root branches
self._branch_map = None
self._items = []
# Test data picker
self._test_data_picker = TestDataPicker(self)
self._test_data_picker.setModal(True)
self._active_thread = None
self._init_title()
return
def _init_title(self):
self.setWindowTitle("Pipeline")
self.treeWidget.setSortingEnabled(False)
self.treeWidget.headerItem().setText(0, "Waiting...")
return
def _set_branch_map(self, branch_map):
self._branch_map = branch_map
return
def _set_title(self, title):
self.treeWidget.headerItem().setText(0, title)
return
def _draw(self, shell):
self._clear()
for branch_dict in self._branch_map:
HubCls = branch_dict["hub"]
args = []
if "args" in branch_dict: args.extend(branch_dict["args"])
new_item = HubCls(self.treeWidget,
branch_dict["name"],
*args)
new_item._activate(shell)
self._items.append(new_item)
return
def _expand(self, shell):
for item in self._items:
item._expand(shell)
return
def _refresh(self, shell, expand=True):
self._draw(shell)
if expand: self._expand(shell)
return
def _clear(self):
root = self.treeWidget.invisibleRootItem()
for widget in self._items:
widget._clear()
if not isinstance(widget, HiddenHub):
root.removeChild(widget)
self._items = []
return
def _find_item(self, item_title, item_class=None, root_item=None):
if root_item is None: root_item = self
for item in root_item._items:
if item._title == item_title:
if item_class is None or isinstance(item, item_class):
return item
result = self._find_item(item_title, item_class, item)
if result is not None: return result
return None
def _make_menus(self, shell, position):
item = self.treeWidget.currentItem()
if isinstance(item, InputBranchItem):
# Set the widget action
self._test_data_picker.buttonBox.button(
QtGui.QDialogButtonBox.Ok).clicked.disconnect()
self._test_data_picker.buttonBox.button(
QtGui.QDialogButtonBox.Ok).clicked.connect(
lambda: self._read_test_data(shell, item))
self._test_data_picker.buttonBox.button(
QtGui.QDialogButtonBox.Ok).clicked.connect(
self._test_data_picker.close)
# Build module and theme context menu
if item._hub_title in ["Modules", "Assessment"]:
menu = QtGui.QMenu()
if item._hub_title == "Modules":
menu.addAction('Inspect', lambda: item._inspect(shell))
menu.addAction('Reset', lambda: item._reset(shell))
menu.addAction('Load test data...',
self._test_data_picker.show)
menu.exec_(self.treeWidget.mapToGlobal(position))
elif isinstance(item, OutputBranchItem):
# Build module context menu
if item._hub_title == "Modules":
menu = QtGui.QMenu()
menu.addAction('Inspect', lambda: item._inspect(shell))
menu.exec_(self.treeWidget.mapToGlobal(position))
return
def _set_top_item(self):
self.treeWidget.itemClicked.emit(self.treeWidget.topLevelItem(0), -1)
return
def _read_auto(self, shell):
self._tree.read_auto(shell.core,
shell.project)
return
@QtCore.pyqtSlot(object, object)
def _read_test_data(self, shell, item):
self._active_thread = ThreadReadTest(
item,
shell,
str(self._test_data_picker.pathLineEdit.text()),
self._test_data_picker.overwriteBox.isChecked())
self._active_thread.start()
self._active_thread.error_detected.connect(self._emit_error)
self._active_thread.finished.connect(self._clear_active_thread)
return
@QtCore.pyqtSlot()
def _clear_active_thread(self):
self._active_thread = None
return
@QtCore.pyqtSlot(object, object, object)
def _emit_error(self, etype, evalue, etraceback):
self.error_detected.emit(etype, evalue, etraceback)
return
def var_tree():
return Tree()
