def __check(expected_dot_path, oozie_xml_path, detail_level,
show_input_ports=False, show_output_ports=False):
actual_oozie_xml = read_as_string(__name__, oozie_xml_path)
actual_dot = convert_oozie_to_dot(actual_oozie_xml, detail_level,
show_input_ports, show_output_ports,
True)
actual = SimplifiedDotGraph.from_dot(actual_dot)
expected_dot = read_as_string(__name__, expected_dot_path)
expected = SimplifiedDotGraph.from_dot(expected_dot)
assert expected == actual, 'Problem when analyzing file "{}", '\
'namely: {} != {}'.format(expected_dot_path, expected, actual)
def check_changed(dir_name):
path = ['data', 'low_score_nodes_remover', dir_name]
src_dir = os.path.join(*(path + ['pipeline.yaml']))
pipeline_yaml = read_as_string(__name__, src_dir)
pipeline = Pipeline.from_yaml_dump(pipeline_yaml)
remover = LowScoreNodesRemover(ImportanceScoreMap(DetailLevel.medium))
actual = remover.run(pipeline)
expected_yaml = read_as_string(__name__,
os.path.join(*(path + ['expected.yaml'])))
expected = Pipeline.from_yaml_dump(expected_yaml)
assert expected == actual
def check_changed(dir_name):
path = ['data', 'low_score_nodes_remover', dir_name]
src_dir = os.path.join(*(path + ['pipeline.yaml']))
pipeline_yaml = read_as_string(__name__, src_dir)
pipeline = Pipeline.from_yaml_dump(pipeline_yaml)
remover = LowScoreNodesRemover(ImportanceScoreMap(DetailLevel.medium))
actual = remover.run(pipeline)
expected_yaml = read_as_string(__name__,
os.path.join(*(path + ['expected.yaml'])))
expected = Pipeline.from_yaml_dump(expected_yaml)
assert expected == actual
def __check(expected_dot_path,
oozie_xml_path,
detail_level,
show_input_ports=False,
show_output_ports=False):
actual_oozie_xml = read_as_string(__name__, oozie_xml_path)
actual_dot = convert_oozie_to_dot(actual_oozie_xml, detail_level,
show_input_ports, show_output_ports,
True)
actual = SimplifiedDotGraph.from_dot(actual_dot)
expected_dot = read_as_string(__name__, expected_dot_path)
expected = SimplifiedDotGraph.from_dot(expected_dot)
assert expected == actual, 'Problem when analyzing file "{}", '\
'namely: {} != {}'.format(expected_dot_path, expected, actual)
def check(oozie_workflow_file_path, expected_pipeline_file_path):
actual_pipeline = convert_oozie_yaml_to_pipeline(oozie_workflow_file_path)
expected_pipeline_yaml = read_as_string(
__name__, expected_pipeline_file_path)
expected = Pipeline.from_yaml_dump(expected_pipeline_yaml)
assert expected == actual_pipeline, 'expected={},\nactual={}'\
.format(expected, actual_pipeline)
def check(oozie_workflow_file_path, expected_pipeline_file_path):
actual_pipeline = convert_oozie_yaml_to_pipeline(oozie_workflow_file_path)
expected_pipeline_yaml = read_as_string(__name__,
expected_pipeline_file_path)
expected = Pipeline.from_yaml_dump(expected_pipeline_yaml)
assert expected == actual_pipeline, 'expected={},\nactual={}'\
.format(expected, actual_pipeline)
def check_no_changes(dir_name):
src_dir = os.path.join('data', 'low_score_nodes_remover', dir_name,
'pipeline.yaml')
pipeline_yaml = read_as_string(__name__, src_dir)
pipeline = Pipeline.from_yaml_dump(pipeline_yaml)
remover = LowScoreNodesRemover(ImportanceScoreMap(DetailLevel.medium))
actual = remover.run(pipeline)
assert pipeline == actual
def convert_to_dot(pipeline_file_relative_path,
detail_level=DetailLevel.medium,
show_input_ports=False, show_output_ports=False):
pipeline_yaml = read_as_string(__name__, pipeline_file_relative_path)
pipeline = Pipeline.from_yaml_dump(pipeline_yaml)
dot_converter = Converter(
detail_level, show_input_ports, show_output_ports)
return dot_converter.run(pipeline)
def check(yaml_file_path):
graph1 = OozieGraph.from_yaml_dump(
read_as_string(__name__, yaml_file_path))
yaml_str1 = graph1.to_yaml_dump()
graph2 = OozieGraph.from_yaml_dump(yaml_str1)
yaml_str2 = graph2.to_yaml_dump()
assert yaml_str1 == yaml_str2
assert graph1 == graph2
def check_no_changes(dir_name):
src_dir = os.path.join('data', 'low_score_nodes_remover',
dir_name, 'pipeline.yaml')
pipeline_yaml = read_as_string(__name__, src_dir)
pipeline = Pipeline.from_yaml_dump(pipeline_yaml)
remover = LowScoreNodesRemover(ImportanceScoreMap(DetailLevel.medium))
actual = remover.run(pipeline)
assert pipeline == actual
def check(expected_dot_path, pipeline_path, detail_level,
show_input_ports=False, show_output_ports=False):
actual_dot = convert_to_dot(pipeline_path, detail_level,
show_input_ports, show_output_ports)
actual = SimplifiedDotGraph.from_dot(actual_dot)
expected_dot = read_as_string(__name__, expected_dot_path)
expected = SimplifiedDotGraph.from_dot(expected_dot)
assert expected == actual, '{} != {}'.format(expected, actual)
def convert_to_dot(pipeline_file_relative_path,
detail_level=DetailLevel.medium,
show_input_ports=False,
show_output_ports=False):
pipeline_yaml = read_as_string(__name__, pipeline_file_relative_path)
pipeline = Pipeline.from_yaml_dump(pipeline_yaml)
dot_converter = Converter(detail_level, show_input_ports,
show_output_ports)
return dot_converter.run(pipeline)
def check(expected_dot_path,
pipeline_path,
detail_level,
show_input_ports=False,
show_output_ports=False):
actual_dot = convert_to_dot(pipeline_path, detail_level, show_input_ports,
show_output_ports)
actual = SimplifiedDotGraph.from_dot(actual_dot)
expected_dot = read_as_string(__name__, expected_dot_path)
expected = SimplifiedDotGraph.from_dot(expected_dot)
assert expected == actual, '{} != {}'.format(expected, actual)
def __convert_to_images(oozie_file_path):
oozie_xml = read_as_string(__name__, oozie_file_path)
vertical_orientation = False
show_input_ports = True
show_output_ports = True
for detail_level in DetailLevel:
TestEndToEndGenerateImages.__convert_to_image(oozie_file_path,
oozie_xml, detail_level,
show_input_ports, show_output_ports,
vertical_orientation)
detail_level = DetailLevel.highest
show_input_ports = False
show_output_ports = False
TestEndToEndGenerateImages.__convert_to_image(oozie_file_path,
oozie_xml, detail_level,
show_input_ports, show_output_ports,
vertical_orientation)
vertical_orientation = True
TestEndToEndGenerateImages.__convert_to_image(oozie_file_path,
oozie_xml, detail_level,
show_input_ports, show_output_ports,
vertical_orientation)
def __convert_to_images(oozie_file_path):
oozie_xml = read_as_string(__name__, oozie_file_path)
vertical_orientation = False
show_input_ports = True
show_output_ports = True
for detail_level in DetailLevel:
TestEndToEndGenerateImages.__convert_to_image(
oozie_file_path, oozie_xml, detail_level, show_input_ports,
show_output_ports, vertical_orientation)
detail_level = DetailLevel.highest
show_input_ports = False
show_output_ports = False
TestEndToEndGenerateImages.__convert_to_image(oozie_file_path,
oozie_xml, detail_level,
show_input_ports,
show_output_ports,
vertical_orientation)
vertical_orientation = True
TestEndToEndGenerateImages.__convert_to_image(oozie_file_path,
oozie_xml, detail_level,
show_input_ports,
show_output_ports,
vertical_orientation)
def convert_oozie_yaml_to_pipeline(oozie_workflow_file_path):
oozie_yaml = read_as_string(__name__, oozie_workflow_file_path)
oozie_graph = OozieGraph.from_yaml_dump(oozie_yaml)
pipeline = convert(oozie_graph, IISPipelineConverter())
return pipeline
def check(oozie_workflow_file_path, expected_pipeline_file_path):
oozie_workflow = read_as_string(__name__, oozie_workflow_file_path)
actual = vipe.oozie.reader.reader.read(oozie_workflow)
expected_yaml = read_as_string(__name__, expected_pipeline_file_path)
expected = OozieGraph.from_yaml_dump(expected_yaml)
assert expected == actual
def check(relative_input_path, expected_graph):
input_dot = read_as_string(__name__, relative_input_path)
actual_graph = SimplifiedDotGraph.from_dot(input_dot)
assert expected_graph == actual_graph, \
'{} != {}'.format(expected_graph, actual_graph)
def check(pipeline_file_path, data_dict):
pipeline_yaml = read_as_string(__name__, pipeline_file_path)
pipeline = Pipeline.from_yaml_dump(pipeline_yaml)
actual_data = PipelineData.from_pipeline(pipeline)
expected_data = PipelineData.from_basic_data_types(data_dict)
assert expected_data == actual_data
def convert_oozie_yaml_to_pipeline(oozie_workflow_file_path):
oozie_yaml = read_as_string(__name__, oozie_workflow_file_path)
oozie_graph = OozieGraph.from_yaml_dump(oozie_yaml)
pipeline = convert(oozie_graph, IISPipelineConverter())
return pipeline
def check(pipeline_file_path, data_dict):
pipeline_yaml = read_as_string(__name__, pipeline_file_path)
pipeline = Pipeline.from_yaml_dump(pipeline_yaml)
actual_data = PipelineData.from_pipeline(pipeline)
expected_data = PipelineData.from_basic_data_types(data_dict)
assert expected_data == actual_data
def check(relative_input_path, expected_graph):
input_dot = read_as_string(__name__, relative_input_path)
actual_graph = SimplifiedDotGraph.from_dot(input_dot)
assert expected_graph == actual_graph, \
'{} != {}'.format(expected_graph, actual_graph)