def test_parse_els_file(self):
import tempfile
prop_file = tempfile.NamedTemporaryFile('w', delete=False)
prop_file.write('#comment\n' 'key=value')
prop_file.close()
params = {'test': 'answer'}
expected = {'test': 'answer', 'key': 'value'}
self.assertEqual(expected, el_utils.parse_els(prop_file.name, params))
def test_parse_els_file(self):
import tempfile
prop_file = tempfile.NamedTemporaryFile("w", delete=False)
prop_file.write("#comment\n" "key=value")
prop_file.close()
params = {"test": "answer"}
expected = {"test": "answer", "key": "value"}
self.assertEqual(expected, el_utils.parse_els(prop_file.name, params))
def __init__(
self,
dag_name: str,
input_directory_path: str,
output_directory_path: str,
action_mapper: Dict[str, Type[ActionMapper]],
control_mapper: Dict[str, Type[BaseMapper]],
user: str = None,
start_days_ago: int = None,
schedule_interval: str = None,
):
"""
:param input_directory_path: Oozie workflow directory.
:param output_directory_path: Desired output directory.
:param user: Username. # TODO remove me and use real ${user} EL
:param start_days_ago: Desired DAG start date, expressed as number of days ago from the present day
:param schedule_interval: Desired DAG schedule interval, expressed as number of days
:param dag_name: Desired output DAG name.
"""
# Each OozieParser class corresponds to one workflow, where one can get
# the workflow's required dependencies (imports), operator relations,
# and operator execution sequence.
self.input_directory_path = input_directory_path
self.output_directory_path = output_directory_path
self.start_days_ago = start_days_ago
self.schedule_interval = schedule_interval
self.dag_name = dag_name
self.configuration_properties_file = os.path.join(
input_directory_path, "configuration.properties")
self.job_properties_file = os.path.join(input_directory_path,
"job.properties")
self.output_dag_name = os.path.join(output_directory_path,
self.dag_name) + ".py"
params = {"user.name": user or os.environ["USER"]}
params = self.add_properties_to_params(params)
params = el_utils.parse_els(self.configuration_properties_file, params)
self.params = params
self.parser = parser.OozieParser(
input_directory_path=input_directory_path,
output_directory_path=output_directory_path,
params=params,
dag_name=dag_name,
action_mapper=action_mapper,
control_mapper=control_mapper,
)
def main():
args = parse_args(sys.argv[1:])
in_file_name = args.input
out_file_name = args.output
params = {'user.name': args.user or os.environ['USER']}
params = el_utils.parse_els(args.properties, params)
# Each OozieParser class corresponds to one workflow, where one can get
# the workflow's required dependencies (imports), operator relations,
# and operator execution sequence.
parser = oozie_parser.OozieParser(oozie_wflow=in_file_name, params=params)
parser.parse_workflow()
relations = parser.get_relations()
depens = parser.get_dependencies()
ops = parser.get_operators()
parser.update_trigger_rules()
create_dag_file(ops, depens, relations, params, out_file_name)
def test_parse_els_no_file(self):
params = {"test": "answer"}
self.assertEqual(params, el_utils.parse_els(None, params))
def add_properties_to_params(self, params: Dict[str, str]):
"""
Template method, can be overridden.
"""
return el_utils.parse_els(self.job_properties_file, params)
def test_parse_els_no_file(self):
params = {'test': 'answer'}
self.assertEqual(params, el_utils.parse_els(None, params))