def test_normalize_path_red_path(self, oozie_path):
cluster = "my-cluster"
region = "europe-west3"
job_properties = {"nameNode": "hdfs://localhost:8020"}
config = {"dataproc_cluster": cluster, "gcp_region": region}
with self.assertRaisesRegex(ParseException, "Unknown path format. "):
normalize_path(oozie_path,
props=PropertySet(config=config,
job_properties=job_properties))
def prepare_move_command(node: Element, params):
source = normalize_path(node.attrib[FS_TAG_SOURCE], params)
target = normalize_path(node.attrib[FS_TAG_TARGET],
params,
allow_no_schema=True)
command = "fs -mv {source} {target}".format(source=shlex.quote(source),
target=shlex.quote(target))
return command
def test_normalize_path_red_path_allowed_no_schema(self, oozie_path):
cluster = "my-cluster"
region = "europe-west3"
params = {
"nameNode": "hdfs://localhost:8020",
"dataproc_cluster": cluster,
"gcp_region": region
}
with self.assertRaisesRegex(ParseException, "Unknown path format. "):
normalize_path(oozie_path, params, allow_no_schema=True)
def on_parse_node(self):
super().on_parse_node()
self.git_uri = get_tag_el_text(self.oozie_node, TAG_GIT_URI)
self.git_branch = get_tag_el_text(self.oozie_node, TAG_BRANCH)
destination_uri = get_tag_el_text(self.oozie_node,
tag=TAG_DESTINATION_URI)
if destination_uri:
self.destination_path = normalize_path(destination_uri,
props=self.props,
translated=True)
key_path_uri = get_tag_el_text(self.oozie_node, tag=TAG_KEY_PATH)
self.key_path = (normalize_path(
key_path_uri, props=self.props, translated=True)
if key_path_uri else None)
def parse_prepare_node(self) -> Tuple[List[str], List[str]]:
"""
<prepare>
<delete path="[PATH]"/>
...
<mkdir path="[PATH]"/>
...
</prepare>
"""
delete_paths = []
mkdir_paths = []
prepare_node = xml_utils.find_node_by_tag(self.mapper.oozie_node,
"prepare")
if prepare_node:
# If there exists a prepare node, there will only be one, according
# to oozie xml schema
for node in prepare_node:
node_path = normalize_path(node.attrib["path"],
props=self.mapper.props)
if node.tag == "delete":
delete_paths.append(node_path)
elif node.tag == "mkdir":
mkdir_paths.append(node_path)
else:
raise Exception(f"Unknown XML node in prepare: {node.tag}")
return delete_paths, mkdir_paths
def test_normalize_path_green_path(self, oozie_path, expected_result):
cluster = "my-cluster"
region = "europe-west3"
job_properties = {"nameNode": "hdfs://localhost:8020"}
config = {"dataproc_cluster": cluster, "gcp_region": region}
result = normalize_path(oozie_path,
props=PropertySet(
config=config,
job_properties=job_properties))
self.assertEqual(expected_result, result)
def prepare_chgrp_command(node: Element, params):
path = normalize_path(node.attrib[FS_TAG_PATH], params)
group = node.attrib[FS_TAG_GROUP]
recursive = node.find(FS_TAG_RECURSIVE) is not None
extra_param = "-R" if recursive else ""
command = "fs -chgrp {extra} {group} {path}".format(
extra=extra_param, group=shlex.quote(group), path=shlex.quote(path))
return command
def test_normalize_path_green_path(self, oozie_path, expected_result):
cluster = "my-cluster"
region = "europe-west3"
params = {
"nameNode": "hdfs://localhost:8020",
"dataproc_cluster": cluster,
"gcp_region": region
}
result = normalize_path(oozie_path, params)
self.assertEqual(expected_result, result)
def prepare_chmod_command(node: Element, params):
path = normalize_path(node.attrib[FS_TAG_PATH], params)
permission = node.attrib[FS_TAG_PERMISSIONS]
# TODO: Add support for dirFiles Reference: GH issues #80
# dirFiles = bool_value(node, FS_TAG _DIRFILES)
recursive = node.find(FS_TAG_RECURSIVE) is not None
extra_param = "-R" if recursive else ""
command = "fs -chmod {extra} {permission} {path}".format(
extra=extra_param,
path=shlex.quote(path),
permission=shlex.quote(permission))
return command
def parse_prepare_node(
oozie_node: ET.Element,
params: Dict[str, str]) -> Tuple[List[str], List[str]]:
"""
<prepare>
<delete path="[PATH]"/>
...
<mkdir path="[PATH]"/>
...
</prepare>
"""
delete_paths = []
mkdir_paths = []
prepare_nodes = xml_utils.find_nodes_by_tag(oozie_node, "prepare")
if prepare_nodes:
# If there exists a prepare node, there will only be one, according
# to oozie xml schema
for node in prepare_nodes[0]:
node_path = normalize_path(node.attrib["path"], params=params)
if node.tag == "delete":
delete_paths.append(node_path)
else:
mkdir_paths.append(node_path)
return delete_paths, mkdir_paths
def prepare_touchz_command(node: Element, params):
path = normalize_path(node.attrib[FS_TAG_PATH], params)
command = "fs -touchz {path}".format(path=shlex.quote(path))
return command
def prepare_delete_command(node: Element, props: PropertySet):
path = normalize_path(node.attrib[FS_TAG_PATH], props=props)
command = "fs -rm -f -r {path}".format(path=shlex.quote(path))
return command
