def test_parse_els_multiple_line_with_back_references(self): # Should remain unchanged, as the conversion from a comma-separated string to a List will # occur before writing to file. prop_file = tempfile.NamedTemporaryFile("w", delete=False) prop_file.write(""" #comment key=value,value2,${test} key2=value key3=refer${key2} key4=refer${key5} key5=test """) prop_file.close() job_properties = {"test": "answer"} props = PropertySet(config={}, job_properties=job_properties, action_node_properties={}) expected = { "key": "value,value2,answer", "key2": "value", "key3": "refervalue", "key4": "refer${key5}", # no forward-references "key5": "test", } self.assertEqual(expected, el_utils.parse_els(prop_file.name, props=props))
def test_parse_els_no_file(self): expected_properties = {} props = PropertySet(job_properties={"key": "value"}, config={}, action_node_properties={}) self.assertEqual(expected_properties, el_utils.parse_els(None, props=props))
def read_config_replace_el(self): """ Reads configuration properties to config dictionary. Replaces EL properties within. :return: None """ self.config = el_utils.parse_els(properties_file=self.config_file, props=self.props)
def read_and_update_job_properties_replace_el(self): """ Reads job properties and updates job_properties dictionary with the read values Replaces EL job_properties within. :return: None """ self.job_properties.update( el_utils.parse_els(properties_file=self.job_properties_file, props=self.props))
def test_parse_els_file(self): import tempfile prop_file = tempfile.NamedTemporaryFile("w", delete=False) prop_file.write("#comment\n" "key=value") prop_file.close() params = {"test": "answer"} expected = {"test": "answer", "key": "value"} self.assertEqual(expected, el_utils.parse_els(prop_file.name, params))
def test_parse_els_file_list(self): # Should remain unchanged, as the conversion from a comma-separated string to a List will # occur before writing to file. import tempfile prop_file = tempfile.NamedTemporaryFile("w", delete=False) prop_file.write("#comment\n" "key=value,value2") prop_file.close() params = {"test": "answer"} expected = {"test": "answer", "key": "value,value2"} self.assertEqual(expected, el_utils.parse_els(prop_file.name, params))
def test_parse_els_file(self): prop_file = tempfile.NamedTemporaryFile("w", delete=False) prop_file.write("#comment\n" "key=value") prop_file.close() job_properties = {"test": "answer"} props = PropertySet(job_properties=job_properties, config={}, action_node_properties={}) expected = {"key": "value"} self.assertEqual(expected, el_utils.parse_els(prop_file.name, props=props))
def __init__( self, dag_name: str, input_directory_path: str, output_directory_path: str, action_mapper: Dict[str, Type[ActionMapper]], control_mapper: Dict[str, Type[BaseMapper]], template_name: str = "workflow.tpl", user: str = None, start_days_ago: int = None, schedule_interval: str = None, output_dag_name: str = None, ): """ :param input_directory_path: Oozie workflow directory. :param output_directory_path: Desired output directory. :param user: Username. # TODO remove me and use real ${user} EL :param start_days_ago: Desired DAG start date, expressed as number of days ago from the present day :param schedule_interval: Desired DAG schedule interval, expressed as number of days :param dag_name: Desired output DAG name. """ # Each OozieParser class corresponds to one workflow, where one can get # the workflow's required dependencies (imports), operator relations, # and operator execution sequence. self.input_directory_path = input_directory_path self.output_directory_path = output_directory_path self.start_days_ago = start_days_ago self.schedule_interval = schedule_interval self.dag_name = dag_name self.template_name = template_name self.configuration_properties_file = os.path.join( input_directory_path, CONFIGURATION_PROPERTIES) self.job_properties_file = os.path.join(input_directory_path, JOB_PROPERTIES) self.output_dag_name = ( os.path.join(output_directory_path, output_dag_name) if output_dag_name else os.path.join(output_directory_path, self.dag_name) + ".py") params = {"user.name": user or os.environ["USER"]} params = self.add_properties_to_params(params) params = el_utils.parse_els(self.configuration_properties_file, params) self.params = params self.parser = parser.OozieParser( input_directory_path=input_directory_path, output_directory_path=output_directory_path, params=params, dag_name=dag_name, action_mapper=action_mapper, control_mapper=control_mapper, )
def test_parse_els_file_list(self): # Should remain unchanged, as the conversion from a comma-separated string to a List will # occur before writing to file. prop_file = tempfile.NamedTemporaryFile("w", delete=False) prop_file.write("#comment\n" "key=value,value2,${test}") prop_file.close() job_properties = {"test": "answer"} props = PropertySet(config={}, job_properties=job_properties, action_node_properties={}) expected = {"key": "value,value2,answer"} self.assertEqual(expected, el_utils.parse_els(prop_file.name, props=props))
def add_properties_to_params(self, params: Dict[str, str]): """ Template method, can be overridden. """ return el_utils.parse_els(self.job_properties_file, params)
def test_parse_els_no_file(self): params = {"test": "answer"} self.assertEqual(params, el_utils.parse_els(None, params))