def _open_and_execute_project(self): """Opens a project and executes all DAGs in that project.""" logger = HeadlessLogger() project_dir = self._args.project project_file_path = pathlib.Path(project_dir, ".spinetoolbox", "project.json").resolve() try: with project_file_path.open() as project_file: try: project_dict = json.load(project_file) except json.decoder.JSONDecodeError: logger.msg_error.emit(f"Error in project file {project_file_path}. Invalid JSON.") return _Status.ERROR except OSError: logger.msg_error.emit(f"Project file {project_file_path} missing") return _Status.ERROR executable_items, dag_handler = open_project(project_dict, project_dir, logger) if executable_items is None: return _Status.ERROR dags = dag_handler.dags() for dag in dags: node_successors = dag_handler.node_successors(dag) if not node_successors: logger.msg_error.emit("The project contains a graph that is not a Directed Acyclic Graph.") return _Status.ERROR items_in_dag = tuple(item for item in executable_items if item.name in dag.nodes) execution_permits = {item_name: True for item_name in dag.nodes} engine = SpineEngine(items_in_dag, node_successors, execution_permits) engine.run() if engine.state() == SpineEngineState.FAILED: return _Status.ERROR return _Status.OK
def execute_dag(self, dag, execution_permits, dag_identifier): """Executes given dag. Args: dag (DiGraph): Executed DAG execution_permits (dict): Dictionary, where keys are node names in dag and value is a boolean dag_identifier (str): Identifier number for printing purposes """ node_successors = self.dag_handler.node_successors(dag) if not node_successors: self._logger.msg_warning.emit( "<b>Graph {0} is not a Directed Acyclic Graph</b>".format( dag_identifier)) self._logger.msg.emit("Items in graph: {0}".format(", ".join( dag.nodes()))) edges = [ "{0} -> {1}".format(*edge) for edge in self.dag_handler.edges_causing_loops(dag) ] self._logger.msg.emit( "Please edit connections in Design View to execute it. " "Possible fix: remove connection(s) {0}.".format( ", ".join(edges))) return items = [ self._project_item_model.get_item( name).project_item.execution_item() for name in node_successors ] self.engine = SpineEngine(items, node_successors, execution_permits) self.engine.dag_node_execution_finished.connect( self._notify_item_for_finished_execution) self.dag_execution_about_to_start.emit(self.engine) self._logger.msg.emit("<b>Starting DAG {0}</b>".format(dag_identifier)) self._logger.msg.emit("Order: {0}".format(" -> ".join( list(node_successors)))) self.engine.run() outcome = { SpineEngineState.USER_STOPPED: "stopped by the user", SpineEngineState.FAILED: "failed", SpineEngineState.COMPLETED: "completed successfully", }[self.engine.state()] self._logger.msg.emit("<b>DAG {0} {1}</b>".format( dag_identifier, outcome)) self.engine.dag_node_execution_finished.disconnect( self._notify_item_for_finished_execution)
def test_execution_permits(self): """Tests that the middle item of an item triplet is not executed when its execution permit is False.""" mock_item_a = self._mock_item("item_a", resources_forward=[self.url_a_fw], resources_backward=[self.url_a_bw]) mock_item_b = self._mock_item("item_b", resources_forward=[self.url_b_fw], resources_backward=[self.url_b_bw]) mock_item_c = self._mock_item("item_c", resources_forward=[self.url_c_fw], resources_backward=[self.url_c_bw]) items = { "item_a": mock_item_a, "item_b": mock_item_b, "item_c": mock_item_c } connections = [ { "from": ("item_a", "right"), "to": ("item_b", "left") }, { "from": ("item_b", "bottom"), "to": ("item_c", "left") }, ] successors = {"item_a": ["item_b"], "item_b": ["item_c"]} execution_permits = {"item_a": True, "item_b": False, "item_c": True} engine = SpineEngine(items=items, connections=connections, node_successors=successors, execution_permits=execution_permits) engine._make_item = lambda name, direction: engine._items[name] engine.run() item_a_execute_args = [[], [self.url_b_bw]] item_b_skip_execution_args = [[self.url_a_fw], [self.url_c_bw]] item_c_execute_calls = [call([self.url_b_fw], [])] self.assert_execute_call(mock_item_a.execute, item_a_execute_args) mock_item_a.exclude_execution.assert_not_called() mock_item_b.execute.assert_not_called() self.assert_execute_call(mock_item_b.exclude_execution, item_b_skip_execution_args) mock_item_b.output_resources.assert_called() mock_item_c.execute.assert_has_calls(item_c_execute_calls) mock_item_c.exclude_execution.assert_not_called() self.assertEqual(engine.state(), SpineEngineState.COMPLETED)
def test_fork_execution_succeeds(self): """Tests execution with three items in a fork.""" mock_item_a = self._mock_item("item_a", resources_forward=[self.url_a_fw], resources_backward=[self.url_a_bw]) mock_item_b = self._mock_item("item_b", resources_forward=[self.url_b_fw], resources_backward=[self.url_b_bw]) mock_item_c = self._mock_item("item_c", resources_forward=[self.url_c_fw], resources_backward=[self.url_c_bw]) items = { "item_a": mock_item_a, "item_b": mock_item_b, "item_c": mock_item_c } connections = [ { "from": ("item_a", "right"), "to": ("item_b", "left") }, { "from": ("item_b", "bottom"), "to": ("item_c", "left") }, ] successors = {"item_a": ["item_b", "item_c"]} execution_permits = {"item_a": True, "item_b": True, "item_c": True} engine = SpineEngine(items=items, connections=connections, node_successors=successors, execution_permits=execution_permits) engine._make_item = lambda name, direction: engine._items[name] engine.run() item_a_execute_args = [[], [self.url_b_bw, self.url_c_bw]] item_b_execute_calls = [call([self.url_a_fw], [])] item_c_execute_calls = [call([self.url_a_fw], [])] self.assert_execute_call(mock_item_a.execute, item_a_execute_args) mock_item_a.exclude_execution.assert_not_called() mock_item_b.execute.assert_has_calls(item_b_execute_calls) mock_item_b.exclude_execution.assert_not_called() mock_item_c.execute.assert_has_calls(item_c_execute_calls) mock_item_c.exclude_execution.assert_not_called() self.assertEqual(engine.state(), SpineEngineState.COMPLETED)
class SpineToolboxProject(MetaObject): """Class for Spine Toolbox projects.""" dag_execution_finished = Signal() dag_execution_about_to_start = Signal("QVariant") """Emitted just before an engine runs. Provides a reference to the engine.""" project_execution_about_to_start = Signal() """Emitted just before the entire project is executed.""" def __init__(self, toolbox, name, description, p_dir, project_item_model, settings, logger): """ Args: toolbox (ToolboxUI): toolbox of this project name (str): Project name description (str): Project description p_dir (str): Project directory project_item_model (ProjectItemModel): project item tree model settings (QSettings): Toolbox settings logger (LoggerInterface): a logger instance """ super().__init__(name, description) self._toolbox = toolbox self._project_item_model = project_item_model self._logger = logger self._settings = settings self.dag_handler = DirectedGraphHandler() self.db_mngr = SpineDBManager(logger, self) self.engine = None self._execution_stopped = True self._dag_execution_list = None self._dag_execution_permits_list = None self._dag_execution_index = None self.project_dir = None # Full path to project directory self.config_dir = None # Full path to .spinetoolbox directory self.items_dir = None # Full path to items directory self.config_file = None # Full path to .spinetoolbox/project.json file if not self._create_project_structure(p_dir): self._logger.msg_error.emit( "Creating project directory " "structure to <b>{0}</b> failed".format(p_dir)) def connect_signals(self): """Connect signals to slots.""" self.dag_handler.dag_simulation_requested.connect( self.notify_changes_in_dag) self.dag_execution_finished.connect(self.execute_next_dag) def _create_project_structure(self, directory): """Makes the given directory a Spine Toolbox project directory. Creates directories and files that are common to all projects. Args: directory (str): Abs. path to a directory that should be made into a project directory Returns: bool: True if project structure was created successfully, False otherwise """ self.project_dir = directory self.config_dir = os.path.abspath( os.path.join(self.project_dir, ".spinetoolbox")) self.items_dir = os.path.abspath(os.path.join(self.config_dir, "items")) self.config_file = os.path.abspath( os.path.join(self.config_dir, PROJECT_FILENAME)) try: create_dir(self.project_dir) # Make project directory except OSError: self._logger.msg_error.emit("Creating directory {0} failed".format( self.project_dir)) return False try: create_dir(self.config_dir) # Make project config directory except OSError: self._logger.msg_error.emit("Creating directory {0} failed".format( self.config_dir)) return False try: create_dir(self.items_dir) # Make project items directory except OSError: self._logger.msg_error.emit("Creating directory {0} failed".format( self.items_dir)) return False return True def change_name(self, name): """Changes project name. Args: name (str): New project name """ super().set_name(name) # Update Window Title self._toolbox.setWindowTitle("Spine Toolbox -- {} --".format( self.name)) self._logger.msg.emit("Project name changed to <b>{0}</b>".format( self.name)) def save(self, tool_def_paths): """Collects project information and objects into a dictionary and writes it to a JSON file. Args: tool_def_paths (list): List of absolute paths to tool specification files Returns: bool: True or False depending on success """ project_dict = dict() # Dictionary for storing project info project_dict["version"] = LATEST_PROJECT_VERSION project_dict["name"] = self.name project_dict["description"] = self.description project_dict["tool_specifications"] = tool_def_paths # Compute connections directly from Links on scene connections = list() for link in self._toolbox.ui.graphicsView.links(): src_connector = link.src_connector src_anchor = src_connector.position src_name = src_connector.parent_name() dst_connector = link.dst_connector dst_anchor = dst_connector.position dst_name = dst_connector.parent_name() conn = { "from": [src_name, src_anchor], "to": [dst_name, dst_anchor] } connections.append(conn) project_dict["connections"] = connections items_dict = dict() # Dictionary for storing project items # Traverse all items in project model by category for category_item in self._project_item_model.root().children(): category = category_item.name category_dict = items_dict[category] = dict() for item in self._project_item_model.items(category): category_dict[item.name] = item.project_item.item_dict() # Save project to file saved_dict = dict(project=project_dict, objects=items_dict) # Write into JSON file with open(self.config_file, "w") as fp: json.dump(saved_dict, fp, indent=4) return True def load(self, objects_dict): """Populates project item model with items loaded from project file. Args: objects_dict (dict): Dictionary containing all project items in JSON format Returns: bool: True if successful, False otherwise """ self._logger.msg.emit("Loading project items...") empty = True for category_name, category_dict in objects_dict.items(): items = [] for name, item_dict in category_dict.items(): item_dict.pop("short name", None) item_dict["name"] = name items.append(item_dict) empty = False self.add_project_items(category_name, *items, verbosity=False) if empty: self._logger.msg_warning.emit("Project has no items") return True def load_tool_specification_from_file(self, jsonfile): """Returns a Tool specification from a definition file. Args: jsonfile (str): Path of the tool specification definition file Returns: ToolSpecification or None if reading the file failed """ try: with open(jsonfile, "r") as fp: try: definition = json.load(fp) except ValueError: self._logger.msg_error.emit( "Tool specification file not valid") logging.exception("Loading JSON data failed") return None except FileNotFoundError: self._logger.msg_error.emit( "Tool specification file <b>{0}</b> does not exist".format( jsonfile)) return None # Path to main program relative to definition file includes_main_path = definition.get("includes_main_path", ".") path = os.path.normpath( os.path.join(os.path.dirname(jsonfile), includes_main_path)) return self.load_tool_specification_from_dict(definition, path) def load_tool_specification_from_dict(self, definition, path): """Returns a Tool specification from a definition dictionary. Args: definition (dict): Dictionary with the tool definition path (str): Directory where main program file is located Returns: ToolSpecification, NoneType """ try: _tooltype = definition["tooltype"].lower() except KeyError: self._logger.msg_error.emit( "No tool type defined in tool definition file. Supported types " "are 'python', 'gams', 'julia' and 'executable'") return None if _tooltype == "julia": return JuliaTool.load(self._toolbox, path, definition, self._settings, self._logger) if _tooltype == "python": return PythonTool.load(self._toolbox, path, definition, self._settings, self._logger) if _tooltype == "gams": return GAMSTool.load(path, definition, self._settings, self._logger) if _tooltype == "executable": return ExecutableTool.load(path, definition, self._settings, self._logger) self._logger.msg_warning.emit( "Tool type <b>{}</b> not available".format(_tooltype)) return None def add_project_items(self, category_name, *items, set_selected=False, verbosity=True): """Adds item to project. Args: category_name (str): The items' category items (dict): one or more dict of items to add set_selected (bool): Whether to set item selected after the item has been added to project verbosity (bool): If True, prints message """ category_ind = self._project_item_model.find_category(category_name) if not category_ind: self._logger.msg_error.emit( "Category {0} not found".format(category_name)) return category_item = self._project_item_model.item(category_ind) item_maker = category_item.item_maker() for item_dict in items: try: item = item_maker(**item_dict, toolbox=self._toolbox, project=self, logger=self._logger) tree_item = LeafProjectTreeItem(item, self._toolbox) except TypeError: self._logger.msg_error.emit( "Loading project item <b>{0}</b> into category <b>{1}</b> failed. " "This is most likely caused by an outdated project file.". format(item_dict["name"], category_name)) continue self._project_item_model.insert_item(tree_item, category_ind) # Append new node to networkx graph self.add_to_dag(item.name) if verbosity: self._logger.msg.emit( "{0} <b>{1}</b> added to project.".format( item.item_type(), item.name)) if set_selected: self.set_item_selected(tree_item) def add_to_dag(self, item_name): """Add new node (project item) to the directed graph.""" self.dag_handler.add_dag_node(item_name) def set_item_selected(self, item): """Sets item selected and shows its info screen. Args: item (BaseProjectTreeItem): Project item to select """ ind = self._project_item_model.find_item(item.name) self._toolbox.ui.treeView_project.setCurrentIndex(ind) def execute_dags(self, dags, execution_permits): """Executes given dags. Args: dags (Sequence(DiGraph)) execution_permits (Sequence(dict)) """ self._execution_stopped = False self._dag_execution_list = list(dags) self._dag_execution_permits_list = list(execution_permits) self._dag_execution_index = 0 self.execute_next_dag() @Slot() def execute_next_dag(self): """Executes next dag in the execution list.""" if self._execution_stopped: return try: dag = self._dag_execution_list[self._dag_execution_index] execution_permits = self._dag_execution_permits_list[ self._dag_execution_index] except IndexError: return dag_identifier = f"{self._dag_execution_index + 1}/{len(self._dag_execution_list)}" self.execute_dag(dag, execution_permits, dag_identifier) self._dag_execution_index += 1 self.dag_execution_finished.emit() def execute_dag(self, dag, execution_permits, dag_identifier): """Executes given dag. Args: dag (DiGraph): Executed DAG execution_permits (dict): Dictionary, where keys are node names in dag and value is a boolean dag_identifier (str): Identifier number for printing purposes """ node_successors = self.dag_handler.node_successors(dag) if not node_successors: self._logger.msg_warning.emit( "<b>Graph {0} is not a Directed Acyclic Graph</b>".format( dag_identifier)) self._logger.msg.emit("Items in graph: {0}".format(", ".join( dag.nodes()))) edges = [ "{0} -> {1}".format(*edge) for edge in self.dag_handler.edges_causing_loops(dag) ] self._logger.msg.emit( "Please edit connections in Design View to execute it. " "Possible fix: remove connection(s) {0}.".format( ", ".join(edges))) return items = [ self._project_item_model.get_item(name).project_item for name in node_successors ] self.engine = SpineEngine(items, node_successors, execution_permits) self.dag_execution_about_to_start.emit(self.engine) self._logger.msg.emit("<b>Starting DAG {0}</b>".format(dag_identifier)) self._logger.msg.emit("Order: {0}".format(" -> ".join( list(node_successors)))) self.engine.run() outcome = { SpineEngineState.USER_STOPPED: "stopped by the user", SpineEngineState.FAILED: "failed", SpineEngineState.COMPLETED: "completed successfully", }[self.engine.state()] self._logger.msg.emit("<b>DAG {0} {1}</b>".format( dag_identifier, outcome)) def execute_selected(self): """Executes DAGs corresponding to all selected project items.""" self._toolbox.ui.textBrowser_eventlog.verticalScrollBar().setValue( self._toolbox.ui.textBrowser_eventlog.verticalScrollBar().maximum( )) if not self.dag_handler.dags(): self._logger.msg_warning.emit("Project has no items to execute") return # Get selected item selected_indexes = self._toolbox.ui.treeView_project.selectedIndexes() if not selected_indexes: self._logger.msg_warning.emit( "Please select a project item and try again.") return dags = set() executable_item_names = list() for ind in selected_indexes: item = self._project_item_model.item(ind) executable_item_names.append(item.name) dag = self.dag_handler.dag_with_node(item.name) if not dag: self._logger.msg_error.emit( "[BUG] Could not find a graph containing {0}. " "<b>Please reopen the project.</b>".format(item.name)) continue dags.add(dag) execution_permit_list = list() for dag in dags: execution_permits = dict() for item_name in dag.nodes: execution_permits[ item_name] = item_name in executable_item_names execution_permit_list.append(execution_permits) self._logger.msg.emit("") self._logger.msg.emit( "-------------------------------------------------") self._logger.msg.emit( "<b>Executing Selected Directed Acyclic Graphs</b>") self._logger.msg.emit( "-------------------------------------------------") self.execute_dags(dags, execution_permit_list) def execute_project(self): """Executes all dags in the project.""" self.project_execution_about_to_start.emit() dags = self.dag_handler.dags() if not dags: self._logger.msg_warning.emit("Project has no items to execute") return execution_permit_list = list() for dag in dags: execution_permit_list.append( {item_name: True for item_name in dag.nodes}) self._logger.msg.emit("") self._logger.msg.emit("--------------------------------------------") self._logger.msg.emit("<b>Executing All Directed Acyclic Graphs</b>") self._logger.msg.emit("--------------------------------------------") self.execute_dags(dags, execution_permit_list) def stop(self): """Stops execution. Slot for the main window Stop tool button in the toolbar.""" if self._execution_stopped: self._logger.msg.emit("No execution in progress") return self._logger.msg.emit("Stopping...") self._execution_stopped = True if self.engine: self.engine.stop() def export_graphs(self): """Exports all valid directed acyclic graphs in project to GraphML files.""" if not self.dag_handler.dags(): self._logger.msg_warning.emit("Project has no graphs to export") return i = 0 for g in self.dag_handler.dags(): fn = str(i) + ".graphml" path = os.path.join(self.project_dir, fn) if not self.dag_handler.export_to_graphml(g, path): self._logger.msg_warning.emit( "Exporting graph nr. {0} failed. Not a directed acyclic graph" .format(i)) else: self._logger.msg.emit("Graph nr. {0} exported to {1}".format( i, path)) i += 1 @Slot("QVariant") def notify_changes_in_dag(self, dag): """Notifies the items in given dag that the dag has changed.""" node_successors = self.dag_handler.node_successors(dag) if not node_successors: # Not a dag, invalidate workflow edges = self.dag_handler.edges_causing_loops(dag) for node in dag.nodes(): ind = self._project_item_model.find_item(node) project_item = self._project_item_model.item(ind).project_item project_item.invalidate_workflow(edges) return # Make resource map and run simulation node_predecessors = inverted(node_successors) for rank, item_name in enumerate(node_successors): item = self._project_item_model.get_item(item_name).project_item resources = [] for parent_name in node_predecessors.get(item_name, set()): parent_item = self._project_item_model.get_item( parent_name).project_item resources += parent_item.output_resources_forward() item.handle_dag_changed(rank, resources) def notify_changes_in_all_dags(self): """Notifies all items of changes in all dags in the project.""" for g in self.dag_handler.dags(): self.notify_changes_in_dag(g) def notify_changes_in_containing_dag(self, item): """Notifies items in dag containing the given item that the dag has changed.""" dag = self.dag_handler.dag_with_node(item) # Some items trigger this method while they are being initialized # but before they have been added to any DAG. # In those cases we don't need to notify other items. if dag: self.notify_changes_in_dag(dag) elif self._project_item_model.find_item(item) is not None: self._logger.msg_error.emit( f"[BUG] Could not find a graph containing {item}. <b>Please reopen the project.</b>" ) @property def settings(self): return self._settings
def test_filter_stacks(self): """Tests filter stacks are properly applied.""" with TemporaryDirectory() as temp_dir: url = "sqlite:///" + os.path.join(temp_dir, "db.sqlite") db_map = DiffDatabaseMapping(url, create=True) import_scenarios(db_map, (("scen1", True), ("scen2", True))) import_tools(db_map, ("toolA", )) db_map.commit_session("Add test data.") db_map.connection.close() url_a_fw = _make_resource(url) url_b_fw = _make_resource("db:///url_b_fw") url_c_bw = _make_resource("db:///url_c_bw") mock_item_a = self._mock_item("item_a", resources_forward=[url_a_fw], resources_backward=[]) mock_item_b = self._mock_item("item_b", resources_forward=[url_b_fw], resources_backward=[]) mock_item_c = self._mock_item("item_c", resources_forward=[], resources_backward=[url_c_bw]) items = { "item_a": mock_item_a, "item_b": mock_item_b, "item_c": mock_item_c } connections = [ { "from": ("item_a", "right"), "to": ("item_b", "left"), "resource_filters": { url_a_fw.label: { "scenario_filter": [1, 2], "tool_filter": [1] } }, }, { "from": ("item_b", "bottom"), "to": ("item_c", "left") }, ] successors = {"item_a": ["item_b"], "item_b": ["item_c"]} execution_permits = { "item_a": True, "item_b": True, "item_c": True } engine = SpineEngine(items=items, connections=connections, node_successors=successors, execution_permits=execution_permits) engine._make_item = lambda name, direction: engine._items[name] with patch("spine_engine.spine_engine.create_timestamp" ) as mock_create_timestamp: mock_create_timestamp.return_value = "timestamp" engine.run() # Check that item_b has been executed two times, with the right filters self.assertEqual(len(mock_item_b.fwd_flt_stacks), 2) self.assertEqual(len(mock_item_b.bwd_flt_stacks), 2) self.assertIn( (scenario_filter_config("scen1"), tool_filter_config("toolA")), mock_item_b.fwd_flt_stacks) self.assertIn( (scenario_filter_config("scen2"), tool_filter_config("toolA")), mock_item_b.fwd_flt_stacks) self.assertIn( (execution_filter_config({ "execution_item": "item_b", "scenarios": ["scen1"], "timestamp": "timestamp" }), ), mock_item_b.bwd_flt_stacks, ) self.assertIn( (execution_filter_config({ "execution_item": "item_b", "scenarios": ["scen2"], "timestamp": "timestamp" }), ), mock_item_b.bwd_flt_stacks, ) # Check that item_c has also been executed two times self.assertEqual(len(mock_item_c.fwd_flt_stacks), 2)
class SpineToolboxProject(MetaObject): """Class for Spine Toolbox projects.""" dag_execution_finished = Signal() dag_execution_about_to_start = Signal("QVariant") """Emitted just before an engine runs. Provides a reference to the engine.""" project_execution_about_to_start = Signal() """Emitted just before the entire project is executed.""" def __init__( self, toolbox, name, description, p_dir, project_item_model, settings, embedded_julia_console, embedded_python_console, logger, ): """ Args: toolbox (ToolboxUI): toolbox of this project name (str): Project name description (str): Project description p_dir (str): Project directory project_item_model (ProjectItemModel): project item tree model settings (QSettings): Toolbox settings embedded_julia_console (JuliaREPLWidget): a Julia console widget for execution in the embedded console embedded_python_console (PythonReplWidget): a Python console widget for execution in the embedded console logger (LoggerInterface): a logger instance """ super().__init__(name, description) self._toolbox = toolbox self._project_item_model = project_item_model self._logger = logger self._settings = settings self._embedded_julia_console = embedded_julia_console self._embedded_python_console = embedded_python_console self.dag_handler = DirectedGraphHandler() self.db_mngr = SpineDBManager(settings, logger, self) self.engine = None self._execution_stopped = True self._dag_execution_list = None self._dag_execution_permits_list = None self._dag_execution_index = None self.project_dir = None # Full path to project directory self.config_dir = None # Full path to .spinetoolbox directory self.items_dir = None # Full path to items directory self.config_file = None # Full path to .spinetoolbox/project.json file self._toolbox.undo_stack.clear() if not self._create_project_structure(p_dir): self._logger.msg_error.emit( "Creating project directory " "structure to <b>{0}</b> failed".format(p_dir)) def connect_signals(self): """Connect signals to slots.""" self.dag_handler.dag_simulation_requested.connect( self.notify_changes_in_dag) self.dag_execution_finished.connect(self.execute_next_dag) def _create_project_structure(self, directory): """Makes the given directory a Spine Toolbox project directory. Creates directories and files that are common to all projects. Args: directory (str): Abs. path to a directory that should be made into a project directory Returns: bool: True if project structure was created successfully, False otherwise """ self.project_dir = directory self.config_dir = os.path.abspath( os.path.join(self.project_dir, ".spinetoolbox")) self.items_dir = os.path.abspath(os.path.join(self.config_dir, "items")) self.config_file = os.path.abspath( os.path.join(self.config_dir, PROJECT_FILENAME)) try: create_dir(self.project_dir) # Make project directory except OSError: self._logger.msg_error.emit("Creating directory {0} failed".format( self.project_dir)) return False try: create_dir(self.config_dir) # Make project config directory except OSError: self._logger.msg_error.emit("Creating directory {0} failed".format( self.config_dir)) return False try: create_dir(self.items_dir) # Make project items directory except OSError: self._logger.msg_error.emit("Creating directory {0} failed".format( self.items_dir)) return False return True def call_set_name(self, name): self._toolbox.undo_stack.push(SetProjectNameCommand(self, name)) def call_set_description(self, description): self._toolbox.undo_stack.push( SetProjectDescriptionCommand(self, description)) def set_name(self, name): """Changes project name. Args: name (str): New project name """ super().set_name(name) self._toolbox.update_window_title() # Remove entry with the old name from File->Open recent menu self._toolbox.remove_path_from_recent_projects(self.project_dir) # Add entry with the new name back to File->Open recent menu self._toolbox.update_recent_projects() self._logger.msg.emit("Project name changed to <b>{0}</b>".format( self.name)) def set_description(self, description): super().set_description(description) msg = "Project description " if description: msg += f"changed to <b>{description}</b>" else: msg += "cleared" self._logger.msg.emit(msg) @staticmethod def get_connections(links): connections = list() for link in links: src_connector = link.src_connector src_anchor = src_connector.position src_name = src_connector.parent_name() dst_connector = link.dst_connector dst_anchor = dst_connector.position dst_name = dst_connector.parent_name() conn = { "from": [src_name, src_anchor], "to": [dst_name, dst_anchor] } connections.append(conn) return connections def save(self, spec_paths): """Collects project information and objects into a dictionary and writes it to a JSON file. Args: spec_paths (list): List of absolute paths to specification files Returns: bool: True or False depending on success """ project_dict = dict() # Dictionary for storing project info project_dict["version"] = LATEST_PROJECT_VERSION project_dict["name"] = self.name project_dict["description"] = self.description project_dict["tool_specifications"] = spec_paths # Compute connections directly from Links on scene project_dict["connections"] = self.get_connections( self._toolbox.ui.graphicsView.links()) items_dict = dict() # Dictionary for storing project items # Traverse all items in project model by category for category_item in self._project_item_model.root().children(): category = category_item.name category_dict = items_dict[category] = dict() for item in self._project_item_model.items(category): category_dict[item.name] = item.project_item.item_dict() # Save project to file saved_dict = dict(project=project_dict, objects=items_dict) # Write into JSON file with open(self.config_file, "w") as fp: json.dump(saved_dict, fp, indent=4) return True def load(self, objects_dict): """Populates project item model with items loaded from project file. Args: objects_dict (dict): Dictionary containing all project items in JSON format Returns: bool: True if successful, False otherwise """ self._logger.msg.emit("Loading project items...") empty = True for category_name, category_dict in objects_dict.items(): if category_name not in CATEGORIES: self._logger.msg_warning.emit( f"The project contains an unknown project item category: '{category_name}'. " "Moving on...") continue items_in_category = dict() for name, item_dict in category_dict.items(): item_dict.pop("short name", None) item_dict["name"] = name try: item_type = item_dict.pop("type") except KeyError: item_type = _legacy_type_from_category(category_name) items_in_category.setdefault(item_type, list()).append(item_dict) empty = False for item_type, items in items_in_category.items(): if not self.make_and_add_project_items( item_type, items, verbosity=False): return False if empty: self._logger.msg_warning.emit("Project has no items") return True def add_project_items(self, item_type, *items, set_selected=False, verbosity=True): """Pushes an AddProjectItemsCommand to the toolbox undo stack. """ if not items: return self._toolbox.undo_stack.push( AddProjectItemsCommand(self, item_type, items, set_selected=set_selected, verbosity=verbosity)) def make_project_tree_items(self, item_type, items): """Creates and returns a dictionary mapping category indexes to a list of corresponding LeafProjectTreeItem instances. Args: item_type (str): item type items (list): one or more dicts of items to add Returns: dict(QModelIndex, list(LeafProjectTreeItem)) """ factory = self._toolbox.item_factories.get(item_type) if factory is None: self._logger.msg_error.emit( f"Unknown item type <b>{item_type}</b>") for item in items: self._logger.msg_error.emit( f"Loading project item <b>{item['name']}</b> failed") return {None: None} project_items_by_category = {} for item_dict in items: try: project_item = factory.make_item(self._toolbox, self, self._logger, **item_dict) except TypeError: self._logger.msg_error.emit( f"Creating <b>{item_type}</b> project item <b>{item_dict['name']}</b> failed. " "This is most likely caused by an outdated project file.") continue except KeyError as error: self._logger.msg_error.emit( f"Creating <b>{item_type}</b> project item <b>{item_dict['name']}</b> failed. " f"This is most likely caused by an outdated or corrupted project file " f"(missing JSON key: {str(error)}).") continue project_items_by_category.setdefault(project_item.item_category(), list()).append(project_item) project_tree_items = {} for category, project_items in project_items_by_category.items(): category_ind = self._project_item_model.find_category(category) # NOTE: category_ind might be None, and needs to be handled caller side project_tree_items[category_ind] = [ LeafProjectTreeItem(project_item, self._toolbox) for project_item in project_items ] return project_tree_items def do_add_project_tree_items(self, category_ind, *project_tree_items, set_selected=False, verbosity=True): """Adds LeafProjectTreeItem instances to project. Args: category_ind (QModelIndex): The category index project_tree_items (LeafProjectTreeItem): one or more LeafProjectTreeItem instances to add set_selected (bool): Whether to set item selected after the item has been added to project verbosity (bool): If True, prints message """ for project_tree_item in project_tree_items: project_item = project_tree_item.project_item self._project_item_model.insert_item(project_tree_item, category_ind) factory = self._toolbox.item_factories[project_item.item_type()] factory.activate_project_item(self._toolbox, project_item) # Append new node to networkx graph self.add_to_dag(project_item.name) if verbosity: self._logger.msg.emit("{0} <b>{1}</b> added to project".format( project_item.item_type(), project_item.name)) if set_selected: item = list(project_tree_items)[-1] self.set_item_selected(item) def set_item_selected(self, item): """ Selects the given item. Args: item (LeafProjectTreeItem) """ ind = self._project_item_model.find_item(item.name) self._toolbox.ui.treeView_project.setCurrentIndex(ind) def make_and_add_project_items(self, item_type, items, set_selected=False, verbosity=True): """Adds items to project at loading. Args: item_type (str): Item type e.g. "Tool" items (list): one or more dict of items to add set_selected (bool): Whether to set item selected after the item has been added to project verbosity (bool): If True, prints message """ for category_ind, project_tree_items in self.make_project_tree_items( item_type, items).items(): if not category_ind: continue self.do_add_project_tree_items(category_ind, *project_tree_items, set_selected=set_selected, verbosity=verbosity) return True def add_to_dag(self, item_name): """Add new node (project item) to the directed graph.""" self.dag_handler.add_dag_node(item_name) def remove_all_items(self): """Pushes a RemoveAllProjectItemsCommand to the toolbox undo stack. """ items_per_category = self._project_item_model.items_per_category() if not any(v for v in items_per_category.values()): self._logger.msg.emit("No project items to remove") return delete_data = int( self._settings.value("appSettings/deleteData", defaultValue="0")) != 0 msg = "Remove all items from project?" if not delete_data: msg += "Item data directory will still be available in the project directory after this operation." else: msg += "<br><br><b>Warning: Item data will be permanently lost after this operation.</b>" message_box = QMessageBox( QMessageBox.Question, "Remove All Items", msg, buttons=QMessageBox.Ok | QMessageBox.Cancel, parent=self._toolbox, ) message_box.button(QMessageBox.Ok).setText("Remove Items") answer = message_box.exec_() if answer != QMessageBox.Ok: return links = self._toolbox.ui.graphicsView.links() self._toolbox.undo_stack.push( RemoveAllProjectItemsCommand(self, items_per_category, links, delete_data=delete_data)) def remove_item(self, name, check_dialog=False): """Pushes a RemoveProjectItemCommand to the toolbox undo stack. Args: name (str): Item's name check_dialog (bool): If True, shows 'Are you sure?' message box """ delete_data = int( self._settings.value("appSettings/deleteData", defaultValue="0")) != 0 if check_dialog: msg = "Remove item <b>{}</b> from project? ".format(name) if not delete_data: msg += "Item data directory will still be available in the project directory after this operation." else: msg += "<br><br><b>Warning: Item data will be permanently lost after this operation.</b>" msg += "<br><br>Tip: Remove items by pressing 'Delete' key to bypass this dialog." # noinspection PyCallByClass, PyTypeChecker message_box = QMessageBox( QMessageBox.Question, "Remove Item", msg, buttons=QMessageBox.Ok | QMessageBox.Cancel, parent=self._toolbox, ) message_box.button(QMessageBox.Ok).setText("Remove Item") answer = message_box.exec_() if answer != QMessageBox.Ok: return self._toolbox.undo_stack.push( RemoveProjectItemCommand(self, name, delete_data=delete_data)) def do_remove_item(self, name): """Removes item from project given its name. This method is used when closing the existing project for opening a new one. Args: name (str): Item's name """ ind = self._project_item_model.find_item(name) category_ind = ind.parent() item = self._project_item_model.item(ind) self._remove_item(category_ind, item) def _remove_item(self, category_ind, item, delete_data=False): """ Removes LeafProjectTreeItem from project. Args: category_ind (QModelIndex): The category index item (LeafProjectTreeItem): the item to remove delete_data (bool): If set to True, deletes the directories and data associated with the item """ try: data_dir = item.project_item.data_dir except AttributeError: data_dir = None # Remove item from project model if not self._project_item_model.remove_item(item, parent=category_ind): self._logger.msg_error.emit( "Removing item <b>{0}</b> from project failed".format( item.name)) # Remove item icon and connected links (QGraphicsItems) from scene icon = item.project_item.get_icon() self._toolbox.ui.graphicsView.remove_icon(icon) self.dag_handler.remove_node_from_graph(item.name) item.project_item.tear_down() if delete_data: if data_dir: # Remove data directory and all its contents self._logger.msg.emit( "Removing directory <b>{0}</b>".format(data_dir)) try: if not erase_dir(data_dir): self._logger.msg_error.emit("Directory does not exist") except OSError: self._logger.msg_error.emit( "[OSError] Removing directory failed. Check directory permissions." ) self._logger.msg.emit( "Item <b>{0}</b> removed from project".format(item.name)) def execute_dags(self, dags, execution_permits): """Executes given dags. Args: dags (Sequence(DiGraph)) execution_permits (Sequence(dict)) """ self._execution_stopped = False self._dag_execution_list = list(dags) self._dag_execution_permits_list = list(execution_permits) self._dag_execution_index = 0 self.execute_next_dag() @Slot() def execute_next_dag(self): """Executes next dag in the execution list.""" if self._execution_stopped: return try: dag = self._dag_execution_list[self._dag_execution_index] execution_permits = self._dag_execution_permits_list[ self._dag_execution_index] except IndexError: return dag_identifier = f"{self._dag_execution_index + 1}/{len(self._dag_execution_list)}" self.execute_dag(dag, execution_permits, dag_identifier) self._dag_execution_index += 1 self.dag_execution_finished.emit() def execute_dag(self, dag, execution_permits, dag_identifier): """Executes given dag. Args: dag (DiGraph): Executed DAG execution_permits (dict): Dictionary, where keys are node names in dag and value is a boolean dag_identifier (str): Identifier number for printing purposes """ node_successors = self.dag_handler.node_successors(dag) if not node_successors: self._logger.msg_warning.emit( "<b>Graph {0} is not a Directed Acyclic Graph</b>".format( dag_identifier)) self._logger.msg.emit("Items in graph: {0}".format(", ".join( dag.nodes()))) edges = [ "{0} -> {1}".format(*edge) for edge in self.dag_handler.edges_causing_loops(dag) ] self._logger.msg.emit( "Please edit connections in Design View to execute it. " "Possible fix: remove connection(s) {0}.".format( ", ".join(edges))) return items = [ self._project_item_model.get_item( name).project_item.execution_item() for name in node_successors ] self.engine = SpineEngine(items, node_successors, execution_permits) self.engine.dag_node_execution_finished.connect( self._notify_item_for_finished_execution) self.dag_execution_about_to_start.emit(self.engine) self._logger.msg.emit("<b>Starting DAG {0}</b>".format(dag_identifier)) self._logger.msg.emit("Order: {0}".format(" -> ".join( list(node_successors)))) self.engine.run() outcome = { SpineEngineState.USER_STOPPED: "stopped by the user", SpineEngineState.FAILED: "failed", SpineEngineState.COMPLETED: "completed successfully", }[self.engine.state()] self._logger.msg.emit("<b>DAG {0} {1}</b>".format( dag_identifier, outcome)) self.engine.dag_node_execution_finished.disconnect( self._notify_item_for_finished_execution) def execute_selected(self): """Executes DAGs corresponding to all selected project items.""" if not self.dag_handler.dags(): self._logger.msg_warning.emit("Project has no items to execute") return # Get selected item selected_indexes = self._toolbox.ui.treeView_project.selectedIndexes() if not selected_indexes: self._logger.msg_warning.emit( "Please select a project item and try again.") return dags = set() executable_item_names = list() for ind in selected_indexes: item = self._project_item_model.item(ind) executable_item_names.append(item.name) dag = self.dag_handler.dag_with_node(item.name) if not dag: self._logger.msg_error.emit( "[BUG] Could not find a graph containing {0}. " "<b>Please reopen the project.</b>".format(item.name)) continue dags.add(dag) execution_permit_list = list() for dag in dags: execution_permits = dict() for item_name in dag.nodes: execution_permits[ item_name] = item_name in executable_item_names execution_permit_list.append(execution_permits) self._logger.msg.emit("") self._logger.msg.emit( "-------------------------------------------------") self._logger.msg.emit( "<b>Executing Selected Directed Acyclic Graphs</b>") self._logger.msg.emit( "-------------------------------------------------") self.execute_dags(dags, execution_permit_list) for name in executable_item_names: # Make sure transient files and file pattern resources get updated throughout the DAG self.notify_changes_in_containing_dag(name) def execute_project(self): """Executes all dags in the project.""" self.project_execution_about_to_start.emit() dags = self.dag_handler.dags() if not dags: self._logger.msg_warning.emit("Project has no items to execute") return execution_permit_list = list() for dag in dags: execution_permit_list.append( {item_name: True for item_name in dag.nodes}) self._logger.msg.emit("") self._logger.msg.emit("--------------------------------------------") self._logger.msg.emit("<b>Executing All Directed Acyclic Graphs</b>") self._logger.msg.emit("--------------------------------------------") self.execute_dags(dags, execution_permit_list) def stop(self): """Stops execution. Slot for the main window Stop tool button in the toolbar.""" if self._execution_stopped: self._logger.msg.emit("No execution in progress") return self._logger.msg.emit("Stopping...") self._execution_stopped = True if self.engine: self.engine.stop() def export_graphs(self): """Exports all valid directed acyclic graphs in project to GraphML files.""" if not self.dag_handler.dags(): self._logger.msg_warning.emit("Project has no graphs to export") return i = 0 for g in self.dag_handler.dags(): fn = str(i) + ".graphml" path = os.path.join(self.project_dir, fn) if not self.dag_handler.export_to_graphml(g, path): self._logger.msg_warning.emit( "Exporting graph nr. {0} failed. Not a directed acyclic graph" .format(i)) else: self._logger.msg.emit("Graph nr. {0} exported to {1}".format( i, path)) i += 1 @Slot("QVariant") def notify_changes_in_dag(self, dag): """Notifies the items in given dag that the dag has changed.""" node_successors = self.dag_handler.node_successors(dag) if not node_successors: # Not a dag, invalidate workflow edges = self.dag_handler.edges_causing_loops(dag) for node in dag.nodes(): ind = self._project_item_model.find_item(node) project_item = self._project_item_model.item(ind).project_item project_item.invalidate_workflow(edges) return # Make resource map and run simulation node_predecessors = inverted(node_successors) for rank, item_name in enumerate(node_successors): item = self._project_item_model.get_item(item_name).project_item resources = [] for parent_name in node_predecessors.get(item_name, set()): parent_item = self._project_item_model.get_item( parent_name).project_item resources += parent_item.resources_for_direct_successors() item.handle_dag_changed(rank, resources) def notify_changes_in_all_dags(self): """Notifies all items of changes in all dags in the project.""" for g in self.dag_handler.dags(): self.notify_changes_in_dag(g) def notify_changes_in_containing_dag(self, item): """Notifies items in dag containing the given item that the dag has changed.""" dag = self.dag_handler.dag_with_node(item) # Some items trigger this method while they are being initialized # but before they have been added to any DAG. # In those cases we don't need to notify other items. if dag: self.notify_changes_in_dag(dag) @property def settings(self): return self._settings @Slot(str, "QVariant", "QVariant") def _notify_item_for_finished_execution(self, item_name, execution_direction, engine_state): """Notifies a project item that its execution counterpart has been executed successfully.""" item = self._project_item_model.get_item(item_name) if item is None: return item.project_item.executed_successfully(execution_direction, engine_state) def direct_successors(self, item): """Returns a list of direct successor nodes for given project item.""" item_name = item.name dags = self.dag_handler.dags() for dag in dags: successors = self.dag_handler.node_successors(dag) items_successors = successors.get(item_name) if items_successors is not None: return [ self._project_item_model.get_item(successor).project_item for successor in items_successors ] return []