def test_pipeline_execution_bad_notebook(pipeline_dir): # Construct 4-node pipeline where node 3 (nodebook) produces a failure node1 = NotebookNode("node1", num_outputs=2) node2 = PythonNode("node2", num_outputs=2, input_nodes=[node1]) node3 = NotebookNode("node3", num_outputs=2, input_nodes=[node1], fail=True) node4 = NotebookNode("node4", num_outputs=2, input_nodes=[node2, node3]) nodes = [node1, node2, node3, node4] processed_nodes = [node1, node2] unprocessed_nodes = [node3, node4] pipeline = construct_pipeline("p1", nodes=nodes, location=pipeline_dir) with pytest.raises(RuntimeError) as e: LocalPipelineProcessor(pipeline_dir).process(pipeline) assert "Error processing operation node3" in str(e.value) # Confirm outputs (and non-outputs) for node in processed_nodes: for output in node.outputs: assert os.path.exists(os.path.join(pipeline_dir, output)) for node in unprocessed_nodes: for output in node.outputs: assert not os.path.exists(os.path.join(pipeline_dir, output))
def test_pipeline_execution_order_in_simple_pipeline(): expected_operation_names = ["f", "a", "c", "g"] pipeline_json = _read_pipeline_resource( "resources/sample_pipelines/pipeline_dependency_simple.json") pipeline = PipelineParser().parse(pipeline_json) current_ordered_operation_names = _get_operation_names( pipeline.operations.values()) assert current_ordered_operation_names != expected_operation_names operations = LocalPipelineProcessor._sort_operations( operations_by_id=pipeline.operations) ordered_operation_names = _get_operation_names(operations) assert ordered_operation_names == expected_operation_names
def test_pipeline_execution(pipeline_dir): # Construct 4-node pipeline consisting of 3 notebooks and 1 python script. # This pipeline is "diamond shaped" with node1 feeding nodes 2 and 3, each then # feeding node4. node1 = NotebookNode("node1", num_outputs=2) node2 = PythonNode("node2", num_outputs=2, input_nodes=[node1]) node3 = NotebookNode("node3", num_outputs=2, input_nodes=[node1]) node4 = NotebookNode("node4", num_outputs=2, input_nodes=[node2, node3]) nodes = [node1, node2, node3, node4] pipeline = construct_pipeline("p1", nodes=nodes, location=pipeline_dir) LocalPipelineProcessor(pipeline_dir).process(pipeline) # Confirm outputs for node in nodes: for output in node.outputs: assert os.path.exists(os.path.join(pipeline_dir, output))
def test_pipeline_execution_missing_kernelspec(pipeline_dir): # Construct 4-node pipeline consisting of 3 notebooks and 1 python script. # This pipeline is "diamond shaped" with node1 feeding nodes 2 and 3, each then # feeding node4. node1 = NotebookNode("node1", num_outputs=2) node2 = PythonNode("node2", num_outputs=2, input_nodes=[node1]) node3 = NotebookNode("node3", num_outputs=2, input_nodes=[node1]) node4 = NotebookNode("node4", num_outputs=2, input_nodes=[node2, node3]) nodes = [node1, node2, node3, node4] pipeline = construct_pipeline("p1", nodes=nodes, location=pipeline_dir) node1nb_file = os.path.join(pipeline_dir, pipeline.operations[node1.id].filename) nb = nbformat.read(node1nb_file, 4) nb["metadata"].pop("kernelspec") nbformat.write(nb, node1nb_file) with pytest.raises(RuntimeError) as e: LocalPipelineProcessor(pipeline_dir).process(pipeline) assert ("Error processing operation node1 (node1.ipynb): No kernel " "name found in notebook and no override provided." in str(e.value))