def test_pipeline_get_envs(): # Ensure pipeline operation env lists are properly converted to dictionaries. pipeline_definitions = _read_pipeline_resource( 'pipeline_dependency_complex.json') pipeline = PipelineParser().parse(pipeline_definitions) local_processor = LocalPipelineProcessor() for op in pipeline.operations.values(): op_envs = local_processor._get_envs(op) assert op_envs['OP_NAME'] == op.name
def test_pipeline_execution_bad_notebook(pipeline_dir): # Construct 4-node pipeline where node 3 (nodebook) produces a failure node1 = NotebookNode("node1", num_outputs=2) node2 = PythonNode("node2", num_outputs=2, input_nodes=[node1]) node3 = NotebookNode("node3", num_outputs=2, input_nodes=[node1], fail=True) node4 = NotebookNode("node4", num_outputs=2, input_nodes=[node2, node3]) nodes = [node1, node2, node3, node4] processed_nodes = [node1, node2] unprocessed_nodes = [node3, node4] pipeline = construct_pipeline("p1", nodes=nodes, location=pipeline_dir) with pytest.raises(RuntimeError) as e: LocalPipelineProcessor(pipeline_dir).process(pipeline) assert 'Error processing operation node3' in str(e.value) # Confirm outputs (and non-outputs) for node in processed_nodes: for output in node.outputs: assert os.path.exists(os.path.join(pipeline_dir, output)) for node in unprocessed_nodes: for output in node.outputs: assert not os.path.exists(os.path.join(pipeline_dir, output))
def test_pipeline_execution(pipeline_dir): # Construct 4-node pipeline consisting of 3 notebooks and 1 python script. # This pipeline is "diamond shaped" with node1 feeding nodes 2 and 3, each then # feeding node4. node1 = NotebookNode("node1", num_outputs=2) node2 = PythonNode("node2", num_outputs=2, input_nodes=[node1]) node3 = NotebookNode("node3", num_outputs=2, input_nodes=[node1]) node4 = NotebookNode("node4", num_outputs=2, input_nodes=[node2, node3]) nodes = [node1, node2, node3, node4] pipeline = construct_pipeline("p1", nodes=nodes, location=pipeline_dir) LocalPipelineProcessor(pipeline_dir).process(pipeline) # Confirm outputs for node in nodes: for output in node.outputs: assert os.path.exists(os.path.join(pipeline_dir, output))
def test_pipeline_execution_missing_kernelspec(pipeline_dir): # Construct 4-node pipeline consisting of 3 notebooks and 1 python script. # This pipeline is "diamond shaped" with node1 feeding nodes 2 and 3, each then # feeding node4. node1 = NotebookNode("node1", num_outputs=2) node2 = PythonNode("node2", num_outputs=2, input_nodes=[node1]) node3 = NotebookNode("node3", num_outputs=2, input_nodes=[node1]) node4 = NotebookNode("node4", num_outputs=2, input_nodes=[node2, node3]) nodes = [node1, node2, node3, node4] pipeline = construct_pipeline("p1", nodes=nodes, location=pipeline_dir) node1nb_file = os.path.join(pipeline_dir, pipeline.operations[node1.id].filename) nb = nbformat.read(node1nb_file, 4) nb['metadata'].pop('kernelspec') nbformat.write(nb, node1nb_file) with pytest.raises(RuntimeError) as e: LocalPipelineProcessor(pipeline_dir).process(pipeline) assert 'Error processing operation node1 (node1.ipynb): No kernel ' \ 'name found in notebook and no override provided.' in str(e.value)