def test_error_rendering(self, input_sentence, translation, kwargs):
translated = translate(input_sentence, functions_module="functions")
self.assertEqual(translated, translation)
with self.assertRaises(AirflowException):
Template(translated).render(**kwargs)
def replace_url_el(url: str, props: PropertySet, allow_no_schema=False) -> str:
"""
Transforms url by replacing EL-expression with equivalent jinja templates.
If schema validation is required then props should include proper name-node.
For example:
input: '{$nameNode}/users/{$userName}/dir
url_with_var: `{{nameNode}}/users/{{userName}}/dir
In this case to validate url schema props should contain `nameNode` value.
"""
url_with_var = el_parser.translate(url)
name_node, _ = _resolve_name_node(url_with_var, props)
if name_node:
url_parts = urlparse(name_node)
else:
url_parts = urlparse(url_with_var)
allowed_schemas = {"hdfs", ""} if allow_no_schema else {"hdfs"}
if url_parts.scheme not in allowed_schemas:
raise ParseException(
f"Unknown path format. The URL should be provided in the following format: "
f"hdfs://localhost:9200/path. Current value: {url_with_var}"
)
return url_with_var
def test_rendering(self, input_sentence, translation, output_sentence, kwargs):
translated = translate(input_sentence, functions_module="functions")
self.assertEqual(translated, translation)
render = Template(translated).render(**kwargs)
self.assertEqual(render, output_sentence)
def extract_properties_from_configuration_node(
config_node: ET.Element) -> Dict[str, str]:
"""Extracts configuration properties from ``configuration`` node"""
properties_dict: Dict[str, str] = dict()
for property_node in config_node.findall(TAG_PROPERTY):
name_node = property_node.find(TAG_NAME)
value_node = property_node.find(TAG_VALUE)
if name_node is None or value_node is None:
raise ParseException(
'Element "property" should have direct children elements: name, value. One of them does not '
"exist. Make sure the configuration element is valid.")
name = name_node.text
value = value_node.text
if not name:
raise ParseException(
'Element "name" should have content, however its value is empty. Make sure the element has '
"the correct content.")
if not value:
raise ParseException(
'Element "value" should have content, however its value is empty. Make sure the element has '
"the correct content.")
properties_dict[name] = el_parser.translate(value)
return properties_dict
def normalize_path(url: str, props: PropertySet, allow_no_schema=False, translated=False) -> str:
"""
Transforms url by replacing EL-expression with equivalent jinja templates
and returns only the path part of the url. If schema validation is
required then props should include proper name-node. If translated is set to True
then passed url is supposed to be a valid jinja expression.
For example:
input: '{$nameNode}/users/{$userName}/dir
url_with_var: `{{nameNode}}/users/{{userName}}/dir
In this case to validate url schema props should contain `nameNode` value.
"""
url_with_var = url if translated else el_parser.translate(url)
name_node, shift = _resolve_name_node(url_with_var, props)
if name_node:
url_parts = urlparse(name_node)
output = url_with_var[shift:]
else:
url_parts = urlparse(url_with_var)
output = url_parts.path
allowed_schemas = {"hdfs", ""} if allow_no_schema else {"hdfs"}
if url_parts.scheme not in allowed_schemas:
raise ParseException(
f"Unknown path format. The URL should be provided in the following format: "
f"hdfs://localhost:9200/path. Current value: {url_with_var}"
)
return output
def get_command(self) -> str:
cmd_txt = xml_utils.get_tag_el_text(self.oozie_node, TAG_CMD)
args = xml_utils.get_tags_el_array_from_text(self.oozie_node, TAG_ARG)
if not cmd_txt:
raise Exception(f"Missing or empty command node in SSH action {self.oozie_node}")
cmd = " ".join([cmd_txt] + [shlex.quote(x) for x in args])
cmd = el_parser.translate(cmd, quote=True)
return cmd
def _parse_oozie_node(self):
self.resource_manager = get_tag_el_text(self.oozie_node, TAG_RESOURCE)
self.name_node = get_tag_el_text(self.oozie_node, TAG_NAME)
cmd_txt = get_tag_el_text(self.oozie_node, TAG_CMD)
args = get_tags_el_array_from_text(self.oozie_node, TAG_ARG)
cmd = " ".join([cmd_txt] + [x for x in args])
self.bash_command = el_parser.translate(cmd, quote=False)
self.pig_command = f"sh {self.bash_command}"
def extract_param_values_from_action_node(oozie_node: Element):
param_nodes = xml_utils.find_nodes_by_tag(oozie_node, TAG_PARAM)
new_params = {}
for node in param_nodes:
if not node.text:
continue
param = el_parser.translate(node.text)
key, _, value = param.partition("=")
new_params[key] = value
return new_params
def get_tags_el_array_from_text(root: ET.Element, tag: str) -> List[str]:
"""
If nodes exist in the oozie_node with the tag specified in tag, it
will build an array of text values for all matching nodes. While doing it
it will attempt to resolve EL expressions in the text values.
"""
tags_array = []
node_array = find_nodes_by_tag(root=root, tag=tag)
if node_array:
for node in node_array:
if node.text is not None:
tags_array.append(el_parser.translate(node.text))
return tags_array
def get_tag_el_text(root: ET.Element,
tag: str,
default: str = None) -> Optional[str]:
"""
If a node exists in the oozie_node with the tag specified in tag, it
will attempt to replace the EL (if it exists) with the corresponding
variable. If no EL var is found, it just returns text. However, if the
tag is not found under oozie_node, then return default. If there are
more than one with the specified tag, it uses the first one found.
"""
var = find_node_by_tag(root, tag)
if var is not None and var.text is not None:
# Only check the first one
return el_parser.translate(var.text)
return default
def _evaluate_properties_line(line: str, known_values: dict, props: PropertySet) -> Tuple[str, str]:
"""
Evaluates single line from properties file using already known values from the file and
values from passed property set.
"""
key, value = line.split("=", 1)
translation = el_parser.translate(value)
tmp = deepcopy(known_values)
tmp.update(props.merged)
env = Environment(undefined=StrictUndefined)
try:
translation = env.from_string(translation).render(**tmp)
except UndefinedError:
translation = value
return key.strip(), translation.strip()
def test_translations(self, input_sentence, output_sentence):
translation = translate(input_sentence)
output_sentence = output_sentence
self.assertEqual(translation, output_sentence)
def test_translations(self, output_sentence, input_sentence):
translation = translate(input_sentence, functions_module="")
output_sentence = output_sentence
self.assertEqual(translation, output_sentence)
