def __check(expected_dot_path, oozie_xml_path, detail_level, show_input_ports=False, show_output_ports=False): actual_oozie_xml = read_as_string(__name__, oozie_xml_path) actual_dot = convert_oozie_to_dot(actual_oozie_xml, detail_level, show_input_ports, show_output_ports, True) actual = SimplifiedDotGraph.from_dot(actual_dot) expected_dot = read_as_string(__name__, expected_dot_path) expected = SimplifiedDotGraph.from_dot(expected_dot) assert expected == actual, 'Problem when analyzing file "{}", '\ 'namely: {} != {}'.format(expected_dot_path, expected, actual)
def check_changed(dir_name): path = ['data', 'low_score_nodes_remover', dir_name] src_dir = os.path.join(*(path + ['pipeline.yaml'])) pipeline_yaml = read_as_string(__name__, src_dir) pipeline = Pipeline.from_yaml_dump(pipeline_yaml) remover = LowScoreNodesRemover(ImportanceScoreMap(DetailLevel.medium)) actual = remover.run(pipeline) expected_yaml = read_as_string(__name__, os.path.join(*(path + ['expected.yaml']))) expected = Pipeline.from_yaml_dump(expected_yaml) assert expected == actual
def check(oozie_workflow_file_path, expected_pipeline_file_path): actual_pipeline = convert_oozie_yaml_to_pipeline(oozie_workflow_file_path) expected_pipeline_yaml = read_as_string( __name__, expected_pipeline_file_path) expected = Pipeline.from_yaml_dump(expected_pipeline_yaml) assert expected == actual_pipeline, 'expected={},\nactual={}'\ .format(expected, actual_pipeline)
def check(oozie_workflow_file_path, expected_pipeline_file_path): actual_pipeline = convert_oozie_yaml_to_pipeline(oozie_workflow_file_path) expected_pipeline_yaml = read_as_string(__name__, expected_pipeline_file_path) expected = Pipeline.from_yaml_dump(expected_pipeline_yaml) assert expected == actual_pipeline, 'expected={},\nactual={}'\ .format(expected, actual_pipeline)
def check_no_changes(dir_name): src_dir = os.path.join('data', 'low_score_nodes_remover', dir_name, 'pipeline.yaml') pipeline_yaml = read_as_string(__name__, src_dir) pipeline = Pipeline.from_yaml_dump(pipeline_yaml) remover = LowScoreNodesRemover(ImportanceScoreMap(DetailLevel.medium)) actual = remover.run(pipeline) assert pipeline == actual
def convert_to_dot(pipeline_file_relative_path, detail_level=DetailLevel.medium, show_input_ports=False, show_output_ports=False): pipeline_yaml = read_as_string(__name__, pipeline_file_relative_path) pipeline = Pipeline.from_yaml_dump(pipeline_yaml) dot_converter = Converter( detail_level, show_input_ports, show_output_ports) return dot_converter.run(pipeline)
def check(yaml_file_path): graph1 = OozieGraph.from_yaml_dump( read_as_string(__name__, yaml_file_path)) yaml_str1 = graph1.to_yaml_dump() graph2 = OozieGraph.from_yaml_dump(yaml_str1) yaml_str2 = graph2.to_yaml_dump() assert yaml_str1 == yaml_str2 assert graph1 == graph2
def check(expected_dot_path, pipeline_path, detail_level, show_input_ports=False, show_output_ports=False): actual_dot = convert_to_dot(pipeline_path, detail_level, show_input_ports, show_output_ports) actual = SimplifiedDotGraph.from_dot(actual_dot) expected_dot = read_as_string(__name__, expected_dot_path) expected = SimplifiedDotGraph.from_dot(expected_dot) assert expected == actual, '{} != {}'.format(expected, actual)
def convert_to_dot(pipeline_file_relative_path, detail_level=DetailLevel.medium, show_input_ports=False, show_output_ports=False): pipeline_yaml = read_as_string(__name__, pipeline_file_relative_path) pipeline = Pipeline.from_yaml_dump(pipeline_yaml) dot_converter = Converter(detail_level, show_input_ports, show_output_ports) return dot_converter.run(pipeline)
def __convert_to_images(oozie_file_path): oozie_xml = read_as_string(__name__, oozie_file_path) vertical_orientation = False show_input_ports = True show_output_ports = True for detail_level in DetailLevel: TestEndToEndGenerateImages.__convert_to_image(oozie_file_path, oozie_xml, detail_level, show_input_ports, show_output_ports, vertical_orientation) detail_level = DetailLevel.highest show_input_ports = False show_output_ports = False TestEndToEndGenerateImages.__convert_to_image(oozie_file_path, oozie_xml, detail_level, show_input_ports, show_output_ports, vertical_orientation) vertical_orientation = True TestEndToEndGenerateImages.__convert_to_image(oozie_file_path, oozie_xml, detail_level, show_input_ports, show_output_ports, vertical_orientation)
def __convert_to_images(oozie_file_path): oozie_xml = read_as_string(__name__, oozie_file_path) vertical_orientation = False show_input_ports = True show_output_ports = True for detail_level in DetailLevel: TestEndToEndGenerateImages.__convert_to_image( oozie_file_path, oozie_xml, detail_level, show_input_ports, show_output_ports, vertical_orientation) detail_level = DetailLevel.highest show_input_ports = False show_output_ports = False TestEndToEndGenerateImages.__convert_to_image(oozie_file_path, oozie_xml, detail_level, show_input_ports, show_output_ports, vertical_orientation) vertical_orientation = True TestEndToEndGenerateImages.__convert_to_image(oozie_file_path, oozie_xml, detail_level, show_input_ports, show_output_ports, vertical_orientation)
def convert_oozie_yaml_to_pipeline(oozie_workflow_file_path): oozie_yaml = read_as_string(__name__, oozie_workflow_file_path) oozie_graph = OozieGraph.from_yaml_dump(oozie_yaml) pipeline = convert(oozie_graph, IISPipelineConverter()) return pipeline
def check(oozie_workflow_file_path, expected_pipeline_file_path): oozie_workflow = read_as_string(__name__, oozie_workflow_file_path) actual = vipe.oozie.reader.reader.read(oozie_workflow) expected_yaml = read_as_string(__name__, expected_pipeline_file_path) expected = OozieGraph.from_yaml_dump(expected_yaml) assert expected == actual
def check(relative_input_path, expected_graph): input_dot = read_as_string(__name__, relative_input_path) actual_graph = SimplifiedDotGraph.from_dot(input_dot) assert expected_graph == actual_graph, \ '{} != {}'.format(expected_graph, actual_graph)
def check(pipeline_file_path, data_dict): pipeline_yaml = read_as_string(__name__, pipeline_file_path) pipeline = Pipeline.from_yaml_dump(pipeline_yaml) actual_data = PipelineData.from_pipeline(pipeline) expected_data = PipelineData.from_basic_data_types(data_dict) assert expected_data == actual_data