def __init__(
self,
oozie_node: ET.Element,
name: str,
trigger_rule: str = TriggerRule.ALL_SUCCESS,
params: Dict[str, str] = None,
**kwargs,
):
ActionMapper.__init__(self, oozie_node, name, trigger_rule, **kwargs)
self.params = params or {}
self.trigger_rule = trigger_rule
self.java_class = ""
self.java_jar = ""
self.job_name = None
self.jars = []
self.properties = {}
self.application_args = []
self.file_extractor = FileExtractor(oozie_node=oozie_node,
params=self.params)
self.archive_extractor = ArchiveExtractor(oozie_node=oozie_node,
params=self.params)
self.prepare_command = None
self.hdfs_files = []
self.hdfs_archives = []
self.dataproc_jars = []
def test_replace_el(self):
# Given
self.job_properties["var1"] = "value1"
self.job_properties["var2"] = "value2"
# language=XML
node_str = """
<pig>
<file>/path/with/el/${var1}</file>
<file>/path/with/el/${var2}</file>
<file>/path/with/two/els/${var1}/${var2}</file>
</pig>
"""
oozie_node = ET.fromstring(node_str)
file_extractor = FileExtractor(oozie_node=oozie_node, props=self.props)
# When
file_extractor.parse_node()
# Then
self.assertEqual(
[
"hdfs:///path/with/el/{{var1}}",
"hdfs:///path/with/el/{{var2}}",
"hdfs:///path/with/two/els/{{var1}}/{{var2}}",
],
file_extractor.hdfs_files,
)
def __init__(
self,
oozie_node: Element,
name: str,
dag_name: str,
props: PropertySet,
jar_files: List[str],
**kwargs,
):
ActionMapper.__init__(self,
oozie_node=oozie_node,
dag_name=dag_name,
name=name,
props=props,
**kwargs)
self.file_extractor = FileExtractor(oozie_node=oozie_node,
props=self.props)
self.archive_extractor = ArchiveExtractor(oozie_node=oozie_node,
props=self.props)
self.main_class: Optional[str] = None
self.java_opts: List[str] = []
self.args: Optional[List[str]] = None
self.hdfs_files: Optional[List[str]] = None
self.hdfs_archives: Optional[List[str]] = None
self.prepare_extension: PrepareMapperExtension = PrepareMapperExtension(
self)
self.jar_files: List[str] = jar_files if jar_files else []
self.jar_files_in_hdfs: List[str] = []
self._get_jar_files_in_hdfs_full_paths()
def test_add_absolute_file(self):
# Given
file_extractor = FileExtractor(oozie_node=Element("fake"),
params=self.default_params)
# When
file_extractor.add_file("/test_file")
# Then
self.assertEqual(file_extractor.files, ["/test_file"])
self.assertEqual(file_extractor.hdfs_files, ["hdfs:///test_file"])
def test_add_relative_file(self):
# Given
file_extractor = FileExtractor(oozie_node=Element("fake"),
params=self.default_params)
# When
file_extractor.add_file("test_file")
# Then
self.assertEqual(file_extractor.files, ["test_file"])
self.assertEqual(file_extractor.hdfs_files,
["hdfs:///user/pig/examples/pig_test_node/test_file"])
def test_add_file_extra_hash(self):
# Given
file_extractor = FileExtractor(oozie_node=Element("fake"),
params=self.default_params)
# When
with self.assertRaises(Exception) as context:
file_extractor.add_file("/test_file#4rarear#")
# Then
self.assertEqual(
"There should be maximum one '#' in the path /test_file#4rarear#",
str(context.exception))
def __init__(self, oozie_node: Element, name: str, dag_name: str, props: PropertySet, **kwargs):
ActionMapper.__init__(
self, oozie_node=oozie_node, name=name, dag_name=dag_name, props=props, **kwargs
)
self.params_dict: Dict[str, str] = {}
self.file_extractor = FileExtractor(oozie_node=oozie_node, props=self.props)
self.archive_extractor = ArchiveExtractor(oozie_node=oozie_node, props=self.props)
self.name_node: Optional[str] = None
self.hdfs_files: Optional[List[str]] = None
self.hdfs_archives: Optional[List[str]] = None
self.prepare_extension: PrepareMapperExtension = PrepareMapperExtension(self)
def __init__(self, oozie_node: ET.Element, name: str, props: PropertySet, **kwargs):
ActionMapper.__init__(self, oozie_node=oozie_node, name=name, props=props, **kwargs)
self.java_class: Optional[str] = None
self.java_jar: Optional[str] = None
self.job_name: Optional[str] = None
self.jars: List[str] = []
self.application_args: List[str] = []
self.file_extractor = FileExtractor(oozie_node=oozie_node, props=self.props)
self.archive_extractor = ArchiveExtractor(oozie_node=oozie_node, props=self.props)
self.hdfs_files: List[str] = []
self.hdfs_archives: List[str] = []
self.dataproc_jars: List[str] = []
self.spark_opts: Dict[str, str] = {}
self.prepare_extension: PrepareMapperExtension = PrepareMapperExtension(self)
def __init__(self, oozie_node: Element, name: str, props: PropertySet,
**kwargs):
ActionMapper.__init__(self,
oozie_node=oozie_node,
name=name,
props=props,
**kwargs)
self.params_dict: Dict[str, str] = {}
self.file_extractor = FileExtractor(oozie_node=oozie_node,
props=self.props)
self.archive_extractor = ArchiveExtractor(oozie_node=oozie_node,
props=self.props)
self._parse_oozie_node()
self.prepare_extension: PrepareMapperExtension = PrepareMapperExtension(
self)
def __init__(self, oozie_node: Element, name: str, props: PropertySet,
**kwargs):
ActionMapper.__init__(self,
oozie_node=oozie_node,
name=name,
props=props,
**kwargs)
self.variables: Optional[Dict[str, str]] = None
self.query: Optional[str] = None
self.script: Optional[str] = None
self.hdfs_files: Optional[List[str]] = None
self.hdfs_archives: Optional[List[str]] = None
self.file_extractor = FileExtractor(oozie_node=oozie_node,
props=self.props)
self.archive_extractor = ArchiveExtractor(oozie_node=oozie_node,
props=self.props)
self.prepare_extension: PrepareMapperExtension = PrepareMapperExtension(
self)
def __init__(
self,
oozie_node: Element,
name: str,
trigger_rule: str = TriggerRule.ALL_SUCCESS,
params=None,
**kwargs,
):
ActionMapper.__init__(self, oozie_node=oozie_node, name=name, trigger_rule=trigger_rule, **kwargs)
if params is None:
params = dict()
self.params = params
self.trigger_rule = trigger_rule
self.properties = {}
self.params_dict = {}
self.file_extractor = FileExtractor(oozie_node=oozie_node, params=params)
self.archive_extractor = ArchiveExtractor(oozie_node=oozie_node, params=params)
self._parse_oozie_node()
def test_add_multiple_files(self):
# Given
file_extractor = FileExtractor(oozie_node=Element("fake"),
props=self.props)
# When
file_extractor.add_file("/test_file")
file_extractor.add_file("test_file2")
file_extractor.add_file("/test_file3")
# Then
self.assertEqual(file_extractor.files,
["/test_file", "test_file2", "/test_file3"])
self.assertEqual(
file_extractor.hdfs_files,
[
"hdfs:///test_file",
"hdfs:///user/pig/examples/pig_test_node/test_file2",
"hdfs:///test_file3"
],
)
def test_add_hash_files(self):
# Given
file_extractor = FileExtractor(oozie_node=Element("fake"),
params=self.default_params)
# When
file_extractor.add_file("/test_file#test3_link")
file_extractor.add_file("test_file2#test_link")
file_extractor.add_file("/test_file3")
# Then
self.assertEqual(
file_extractor.files,
["/test_file#test3_link", "test_file2#test_link", "/test_file3"])
self.assertEqual(
file_extractor.hdfs_files,
[
"hdfs:///test_file#test3_link",
"hdfs:///user/pig/examples/pig_test_node/test_file2#test_link",
"hdfs:///test_file3",
],
)
def test_replace_el(self):
# Given
params = {"var1": "value1", "var2": "value2", **self.default_params}
# language=XML
node_str = """
<pig>
<file>/path/with/el/${var1}</file>
<file>/path/with/el/${var2}</file>
<file>/path/with/two/els/${var1}/${var2}</file>
</pig>
"""
oozie_node = ET.fromstring(node_str)
file_extractor = FileExtractor(oozie_node=oozie_node, params=params)
# When
file_extractor.parse_node()
# Then
self.assertEqual(
file_extractor.hdfs_files,
[
"hdfs:///path/with/el/value1",
"hdfs:///path/with/el/value2",
"hdfs:///path/with/two/els/value1/value2",
],
)