def test_parse_els_multiple_line_with_back_references(self):
# Should remain unchanged, as the conversion from a comma-separated string to a List will
# occur before writing to file.
prop_file = tempfile.NamedTemporaryFile("w", delete=False)
prop_file.write("""
#comment
key=value,value2,${test}
key2=value
key3=refer${key2}
key4=refer${key5}
key5=test
""")
prop_file.close()
job_properties = {"test": "answer"}
props = PropertySet(config={},
job_properties=job_properties,
action_node_properties={})
expected = {
"key": "value,value2,answer",
"key2": "value",
"key3": "refervalue",
"key4": "refer${key5}", # no forward-references
"key5": "test",
}
self.assertEqual(expected,
el_utils.parse_els(prop_file.name, props=props))
def test_parse_els_no_file(self):
expected_properties = {}
props = PropertySet(job_properties={"key": "value"},
config={},
action_node_properties={})
self.assertEqual(expected_properties,
el_utils.parse_els(None, props=props))
def read_config_replace_el(self):
"""
Reads configuration properties to config dictionary.
Replaces EL properties within.
:return: None
"""
self.config = el_utils.parse_els(properties_file=self.config_file,
props=self.props)
def read_and_update_job_properties_replace_el(self):
"""
Reads job properties and updates job_properties dictionary with the read values
Replaces EL job_properties within.
:return: None
"""
self.job_properties.update(
el_utils.parse_els(properties_file=self.job_properties_file,
props=self.props))
def test_parse_els_file(self):
import tempfile
prop_file = tempfile.NamedTemporaryFile("w", delete=False)
prop_file.write("#comment\n" "key=value")
prop_file.close()
params = {"test": "answer"}
expected = {"test": "answer", "key": "value"}
self.assertEqual(expected, el_utils.parse_els(prop_file.name, params))
def test_parse_els_file_list(self):
# Should remain unchanged, as the conversion from a comma-separated string to a List will
# occur before writing to file.
import tempfile
prop_file = tempfile.NamedTemporaryFile("w", delete=False)
prop_file.write("#comment\n" "key=value,value2")
prop_file.close()
params = {"test": "answer"}
expected = {"test": "answer", "key": "value,value2"}
self.assertEqual(expected, el_utils.parse_els(prop_file.name, params))
def test_parse_els_file(self):
prop_file = tempfile.NamedTemporaryFile("w", delete=False)
prop_file.write("#comment\n" "key=value")
prop_file.close()
job_properties = {"test": "answer"}
props = PropertySet(job_properties=job_properties,
config={},
action_node_properties={})
expected = {"key": "value"}
self.assertEqual(expected,
el_utils.parse_els(prop_file.name, props=props))
def __init__(
self,
dag_name: str,
input_directory_path: str,
output_directory_path: str,
action_mapper: Dict[str, Type[ActionMapper]],
control_mapper: Dict[str, Type[BaseMapper]],
template_name: str = "workflow.tpl",
user: str = None,
start_days_ago: int = None,
schedule_interval: str = None,
output_dag_name: str = None,
):
"""
:param input_directory_path: Oozie workflow directory.
:param output_directory_path: Desired output directory.
:param user: Username. # TODO remove me and use real ${user} EL
:param start_days_ago: Desired DAG start date, expressed as number of days ago from the present day
:param schedule_interval: Desired DAG schedule interval, expressed as number of days
:param dag_name: Desired output DAG name.
"""
# Each OozieParser class corresponds to one workflow, where one can get
# the workflow's required dependencies (imports), operator relations,
# and operator execution sequence.
self.input_directory_path = input_directory_path
self.output_directory_path = output_directory_path
self.start_days_ago = start_days_ago
self.schedule_interval = schedule_interval
self.dag_name = dag_name
self.template_name = template_name
self.configuration_properties_file = os.path.join(
input_directory_path, CONFIGURATION_PROPERTIES)
self.job_properties_file = os.path.join(input_directory_path,
JOB_PROPERTIES)
self.output_dag_name = (
os.path.join(output_directory_path, output_dag_name)
if output_dag_name else
os.path.join(output_directory_path, self.dag_name) + ".py")
params = {"user.name": user or os.environ["USER"]}
params = self.add_properties_to_params(params)
params = el_utils.parse_els(self.configuration_properties_file, params)
self.params = params
self.parser = parser.OozieParser(
input_directory_path=input_directory_path,
output_directory_path=output_directory_path,
params=params,
dag_name=dag_name,
action_mapper=action_mapper,
control_mapper=control_mapper,
)
def test_parse_els_file_list(self):
# Should remain unchanged, as the conversion from a comma-separated string to a List will
# occur before writing to file.
prop_file = tempfile.NamedTemporaryFile("w", delete=False)
prop_file.write("#comment\n" "key=value,value2,${test}")
prop_file.close()
job_properties = {"test": "answer"}
props = PropertySet(config={},
job_properties=job_properties,
action_node_properties={})
expected = {"key": "value,value2,answer"}
self.assertEqual(expected,
el_utils.parse_els(prop_file.name, props=props))
def add_properties_to_params(self, params: Dict[str, str]):
"""
Template method, can be overridden.
"""
return el_utils.parse_els(self.job_properties_file, params)
def test_parse_els_no_file(self):
params = {"test": "answer"}
self.assertEqual(params, el_utils.parse_els(None, params))