def test_action_gen_xml_java_opts(self): # Contains java_opts data = {u'name': u'java-fc05', u'properties': {u'files': [], u'job_xml': [], u'jar_path': u'/user/romain/hadoop-mapreduce-examples.jar', u'java_opts': [{u'value': u'-debug -Da -Db=1'}], u'retry_max': [], u'retry_interval': [], u'job_properties': [], u'capture_output': False, u'main_class': u'MyClass', u'arguments': [], u'prepares': [], u'credentials': [], u'sla': [{u'value': False, u'key': u'enabled'}, {u'value': u'${nominal_time}', u'key': u'nominal-time'}, {u'value': u'', u'key': u'should-start'}, {u'value': u'${30 * MINUTES}', u'key': u'should-end'}, {u'value': u'', u'key': u'max-duration'}, {u'value': u'', u'key': u'alert-events'}, {u'value': u'', u'key': u'alert-contact'}, {u'value': u'', u'key': u'notification-msg'}, {u'value': u'', u'key': u'upstream-apps'}], u'archives': []}, u'actionParametersFetched': False, u'id': u'fc05d86f-9f07-7a8d-6256-e6abfa87cf77', u'type': u'java-widget', u'children': [{u'to': u'33430f0f-ebfa-c3ec-f237-3e77efa03d0a'}, {u'error': u'17c9c895-5a16-7443-bb81-f34b30b21548'}], u'actionParameters': []} java_node = Node(data) node_mapping = {"fc05d86f-9f07-7a8d-6256-e6abfa87cf77": java_node, "33430f0f-ebfa-c3ec-f237-3e77efa03d0a": java_node, "17c9c895-5a16-7443-bb81-f34b30b21548": java_node} # Last 2 are actually kill and ok nodes xml = java_node.to_xml(node_mapping=node_mapping) xml = [row.strip() for row in xml.split('\n')] assert_false("<java-opts>[{u'value': u'-debug -Da -Db=1'}]</java-opts>" in xml, xml) assert_true("<java-opts>-debug -Da -Db=1</java-opts>" in xml, xml)
def action_parameters(request): response = {'status': -1} parameters = set() try: node_data = json.loads(request.POST.get('node', '{}')) parameters = parameters.union(set(Node(node_data).find_parameters())) script_path = node_data.get('properties', {}).get('script_path', {}) if script_path: script_path = script_path.replace('hdfs://', '') if request.fs.do_as_user(request.user, request.fs.exists, script_path): data = request.fs.do_as_user(request.user, request.fs.read, script_path, 0, 16 * 1024**2) if node_data['type'] in ('hive', 'hive2'): parameters = parameters.union( set(find_dollar_braced_variables(data))) elif node_data['type'] == 'pig': parameters = parameters.union( set(find_dollar_variables(data))) response['status'] = 0 response['parameters'] = list(parameters) except Exception, e: response['message'] = str(e)
def action_parameters(request): response = {'status': -1} parameters = set() try: node_data = json.loads(request.POST.get('node', '{}')) parameters = parameters.union(set(Node(node_data).find_parameters())) script_path = node_data.get('properties', {}).get('script_path', {}) if script_path: script_path = script_path.replace('hdfs://', '') if request.fs.do_as_user(request.user, request.fs.exists, script_path): data = request.fs.do_as_user(request.user, request.fs.read, script_path, 0, 16 * 1024 ** 2) if node_data['type'] in ('hive', 'hive2'): parameters = parameters.union(set(find_dollar_braced_variables(data))) elif node_data['type'] == 'pig': parameters = parameters.union(set(find_dollar_variables(data))) elif node_data['type'] == 'hive-document': notebook = Notebook(document=Document2.objects.get_by_uuid(user=request.user, uuid=node_data['properties']['uuid'])) parameters = parameters.union(set(find_dollar_braced_variables(notebook.get_str()))) elif node_data['type'] == 'sqoop-document': notebook = Notebook(document=Document2.objects.get_by_uuid(user=request.user, uuid=node_data['properties']['uuid'])) parameters = parameters.union(set(find_dollar_braced_variables(notebook.get_str()))) elif node_data['type'] == 'spark-document': notebook = Notebook(document=Document2.objects.get_by_uuid(user=request.user, uuid=node_data['properties']['uuid'])) for arg in notebook.get_data()['snippets'][0]['properties']['spark_arguments']: parameters = parameters.union(set(find_dollar_braced_variables(arg))) response['status'] = 0 response['parameters'] = list(parameters) except Exception, e: response['message'] = str(e)
def test_ignore_dead_fork_link(self): data = {'id': 1, 'type': 'fork', 'children': [{'to': 1, 'id': 1}, {'to': 2, 'id': 2}], 'properties': {}, 'name': 'my-fork'} # to --> 2 does not exist fork = Node(data) node_mapping = {1: fork} # Point to ourself assert_equal(['<fork', 'name="my-fork">', '<path', 'start="my-fork"', '/>', '</fork>'], fork.to_xml(node_mapping=node_mapping).split())
def __init__(self): self.deployment_dir = '/tmp/test' self.nodes = [ Node({ 'id': '1', 'type': 'hive-document', 'properties': { 'jdbc_url': u"${wf:actionData('shell-31b5')['hiveserver']}", 'password': u'test' } }) ]
def __init__(self): self.deployment_dir = '/tmp/test' self.nodes = [ Node({ 'id': '1', 'type': 'hive-document', 'properties': { 'jdbc_url': u'jdbc:hive2://test-replace-url:12345/default', 'password': u'test' } }) ]
def test_ignore_dead_fork_link(self): data = { 'id': 1, 'type': 'fork', 'children': [{ 'to': 1, 'id': 1 }, { 'to': 2, 'id': 2 }], 'properties': {}, 'name': 'my-fork' } # to --> 2 does not exist fork = Node(data) node_mapping = {1: fork} # Point to ourself assert_equal([ '<fork', 'name="my-fork">', '<path', 'start="my-fork"', '/>', '</fork>' ], fork.to_xml(node_mapping=node_mapping).split())
def __init__(self): self.deployment_dir = deployment_dir self.nodes = [ Node({'id': '1', 'type': 'mapreduce', 'properties': {'jar_path': jar_1}}), Node({'id': '2', 'type': 'mapreduce', 'properties': {'jar_path': jar_2}}), Node({'id': '3', 'type': 'java', 'properties': {'jar_path': jar_3}}), Node({'id': '4', 'type': 'java', 'properties': {'jar_path': jar_4}}), # Workspace relative paths Node({'id': '5', 'type': 'java', 'properties': {'jar_path': jar_5}}), Node({'id': '6', 'type': 'java', 'properties': {'jar_path': jar_6}}) ]
def test_action_gen_xml_prepare(self): # Prepare has a value data = { u'properties': { u'files': [], u'job_xml': [], u'parameters': [], u'retry_interval': [], u'retry_max': [], u'job_properties': [], u'arguments': [], u'prepares': [{u'type': u'mkdir', u'value': u'/my_dir'}], u'credentials': [], u'script_path': u'my_pig.pig', u'sla': [{u'key': u'enabled', u'value': False}, {u'key': u'nominal-time', u'value': u'${nominal_time}'}, {u'key': u'should-start', u'value': u''}, {u'key': u'should-end', u'value': u'${30 * MINUTES}'}, {u'key': u'max-duration', u'value': u''}, {u'key': u'alert-events', u'value': u''}, {u'key': u'alert-contact', u'value': u''}, {u'key': u'notification-msg', u'value': u''}, {u'key': u'upstream-apps', u'value': u''}], u'archives': [] }, u'type': u'pig-widget', u'id': u'c59d1947-7ce0-ef34-22b2-d64b9fc5bf9a', u'name': u'pig-c59d', "children":[{"to": "c59d1947-7ce0-ef34-22b2-d64b9fc5bf9a"}, {"error": "c59d1947-7ce0-ef34-22b2-d64b9fc5bf9a"}] } pig_node = Node(data) node_mapping = {"c59d1947-7ce0-ef34-22b2-d64b9fc5bf9a": pig_node} xml = pig_node.to_xml(node_mapping=node_mapping) xml = [row.strip() for row in xml.split()] assert_true(u'<prepare>' in xml, xml) assert_true(u'<mkdir' in xml, xml) assert_true(u'path="${nameNode}/my_dir"/>' in xml, xml) # Prepare has empty value and is skipped pig_node.data['properties']['prepares'] = [{u'type': u'mkdir', u'value': u''}] xml = pig_node.to_xml(node_mapping=node_mapping) xml = [row.strip() for row in xml.split()] assert_false(u'<prepare>' in xml, xml) assert_false(u'<mkdir' in xml, xml) # Prepare has a value and an empty value pig_node.data['properties']['prepares'] = [{u'type': u'mkdir', u'value': u'/my_dir'}, {u'type': u'rm', u'value': u''}] xml = pig_node.to_xml(node_mapping=node_mapping) xml = [row.strip() for row in xml.split()] assert_true(u'<prepare>' in xml, xml) assert_true(u'<mkdir' in xml, xml) assert_true(u'path="${nameNode}/my_dir"/>' in xml, xml) assert_false(u'<rm' in xml, xml)
def test_action_gen_xml_java_opts(self): # Contains java_opts data = { u'name': u'java-fc05', u'properties': { u'files': [], u'job_xml': [], u'jar_path': u'/user/romain/hadoop-mapreduce-examples.jar', u'java_opts': [{ u'value': u'-debug -Da -Db=1' }], u'retry_max': [], u'retry_interval': [], u'job_properties': [], u'capture_output': False, u'main_class': u'MyClass', u'arguments': [], u'prepares': [], u'credentials': [], u'sla': [{ u'value': False, u'key': u'enabled' }, { u'value': u'${nominal_time}', u'key': u'nominal-time' }, { u'value': u'', u'key': u'should-start' }, { u'value': u'${30 * MINUTES}', u'key': u'should-end' }, { u'value': u'', u'key': u'max-duration' }, { u'value': u'', u'key': u'alert-events' }, { u'value': u'', u'key': u'alert-contact' }, { u'value': u'', u'key': u'notification-msg' }, { u'value': u'', u'key': u'upstream-apps' }], u'archives': [] }, u'actionParametersFetched': False, u'id': u'fc05d86f-9f07-7a8d-6256-e6abfa87cf77', u'type': u'java-widget', u'children': [{ u'to': u'33430f0f-ebfa-c3ec-f237-3e77efa03d0a' }, { u'error': u'17c9c895-5a16-7443-bb81-f34b30b21548' }], u'actionParameters': [] } java_node = Node(data) node_mapping = { "fc05d86f-9f07-7a8d-6256-e6abfa87cf77": java_node, "33430f0f-ebfa-c3ec-f237-3e77efa03d0a": java_node, "17c9c895-5a16-7443-bb81-f34b30b21548": java_node } # Last 2 are actually kill and ok nodes xml = java_node.to_xml(node_mapping=node_mapping) xml = [row.strip() for row in xml.split('\n')] assert_false( "<java-opts>[{u'value': u'-debug -Da -Db=1'}]</java-opts>" in xml, xml) assert_true("<java-opts>-debug -Da -Db=1</java-opts>" in xml, xml)
def test_action_gen_xml_prepare(self): # Prepare has a value data = { u'properties': { u'files': [], u'job_xml': [], u'parameters': [], u'retry_interval': [], u'retry_max': [], u'job_properties': [], u'arguments': [], u'prepares': [{ u'type': u'mkdir', u'value': u'/my_dir' }], u'credentials': [], u'script_path': u'my_pig.pig', u'sla': [{ u'key': u'enabled', u'value': False }, { u'key': u'nominal-time', u'value': u'${nominal_time}' }, { u'key': u'should-start', u'value': u'' }, { u'key': u'should-end', u'value': u'${30 * MINUTES}' }, { u'key': u'max-duration', u'value': u'' }, { u'key': u'alert-events', u'value': u'' }, { u'key': u'alert-contact', u'value': u'' }, { u'key': u'notification-msg', u'value': u'' }, { u'key': u'upstream-apps', u'value': u'' }], u'archives': [] }, u'type': u'pig-widget', u'id': u'c59d1947-7ce0-ef34-22b2-d64b9fc5bf9a', u'name': u'pig-c59d', "children": [{ "to": "c59d1947-7ce0-ef34-22b2-d64b9fc5bf9a" }, { "error": "c59d1947-7ce0-ef34-22b2-d64b9fc5bf9a" }] } pig_node = Node(data) node_mapping = {"c59d1947-7ce0-ef34-22b2-d64b9fc5bf9a": pig_node} xml = pig_node.to_xml(node_mapping=node_mapping) xml = [row.strip() for row in xml.split()] assert_true(u'<prepare>' in xml, xml) assert_true(u'<mkdir' in xml, xml) assert_true(u'path="${nameNode}/my_dir"/>' in xml, xml) # Prepare has empty value and is skipped pig_node.data['properties']['prepares'] = [{ u'type': u'mkdir', u'value': u'' }] xml = pig_node.to_xml(node_mapping=node_mapping) xml = [row.strip() for row in xml.split()] assert_false(u'<prepare>' in xml, xml) assert_false(u'<mkdir' in xml, xml) # Prepare has a value and an empty value pig_node.data['properties']['prepares'] = [{ u'type': u'mkdir', u'value': u'/my_dir' }, { u'type': u'rm', u'value': u'' }] xml = pig_node.to_xml(node_mapping=node_mapping) xml = [row.strip() for row in xml.split()] assert_true(u'<prepare>' in xml, xml) assert_true(u'<mkdir' in xml, xml) assert_true(u'path="${nameNode}/my_dir"/>' in xml, xml) assert_false(u'<rm' in xml, xml)