def test_action_gen_xml_java_opts(self):
# Contains java_opts
data = {u'name': u'java-fc05', u'properties': {u'files': [], u'job_xml': [], u'jar_path': u'/user/romain/hadoop-mapreduce-examples.jar', u'java_opts': [{u'value': u'-debug -Da -Db=1'}], u'retry_max': [], u'retry_interval': [], u'job_properties': [], u'capture_output': False, u'main_class': u'MyClass', u'arguments': [], u'prepares': [], u'credentials': [], u'sla': [{u'value': False, u'key': u'enabled'}, {u'value': u'${nominal_time}', u'key': u'nominal-time'}, {u'value': u'', u'key': u'should-start'}, {u'value': u'${30 * MINUTES}', u'key': u'should-end'}, {u'value': u'', u'key': u'max-duration'}, {u'value': u'', u'key': u'alert-events'}, {u'value': u'', u'key': u'alert-contact'}, {u'value': u'', u'key': u'notification-msg'}, {u'value': u'', u'key': u'upstream-apps'}], u'archives': []}, u'actionParametersFetched': False, u'id': u'fc05d86f-9f07-7a8d-6256-e6abfa87cf77', u'type': u'java-widget', u'children': [{u'to': u'33430f0f-ebfa-c3ec-f237-3e77efa03d0a'}, {u'error': u'17c9c895-5a16-7443-bb81-f34b30b21548'}], u'actionParameters': []}
java_node = Node(data)
node_mapping = {"fc05d86f-9f07-7a8d-6256-e6abfa87cf77": java_node, "33430f0f-ebfa-c3ec-f237-3e77efa03d0a": java_node, "17c9c895-5a16-7443-bb81-f34b30b21548": java_node} # Last 2 are actually kill and ok nodes
xml = java_node.to_xml(node_mapping=node_mapping)
xml = [row.strip() for row in xml.split('\n')]
assert_false("<java-opts>[{u'value': u'-debug -Da -Db=1'}]</java-opts>" in xml, xml)
assert_true("<java-opts>-debug -Da -Db=1</java-opts>" in xml, xml)
def action_parameters(request):
response = {'status': -1}
parameters = set()
try:
node_data = json.loads(request.POST.get('node', '{}'))
parameters = parameters.union(set(Node(node_data).find_parameters()))
script_path = node_data.get('properties', {}).get('script_path', {})
if script_path:
script_path = script_path.replace('hdfs://', '')
if request.fs.do_as_user(request.user, request.fs.exists,
script_path):
data = request.fs.do_as_user(request.user, request.fs.read,
script_path, 0, 16 * 1024**2)
if node_data['type'] in ('hive', 'hive2'):
parameters = parameters.union(
set(find_dollar_braced_variables(data)))
elif node_data['type'] == 'pig':
parameters = parameters.union(
set(find_dollar_variables(data)))
response['status'] = 0
response['parameters'] = list(parameters)
except Exception, e:
response['message'] = str(e)
def action_parameters(request):
response = {'status': -1}
parameters = set()
try:
node_data = json.loads(request.POST.get('node', '{}'))
parameters = parameters.union(set(Node(node_data).find_parameters()))
script_path = node_data.get('properties', {}).get('script_path', {})
if script_path:
script_path = script_path.replace('hdfs://', '')
if request.fs.do_as_user(request.user, request.fs.exists, script_path):
data = request.fs.do_as_user(request.user, request.fs.read, script_path, 0, 16 * 1024 ** 2)
if node_data['type'] in ('hive', 'hive2'):
parameters = parameters.union(set(find_dollar_braced_variables(data)))
elif node_data['type'] == 'pig':
parameters = parameters.union(set(find_dollar_variables(data)))
elif node_data['type'] == 'hive-document':
notebook = Notebook(document=Document2.objects.get_by_uuid(user=request.user, uuid=node_data['properties']['uuid']))
parameters = parameters.union(set(find_dollar_braced_variables(notebook.get_str())))
elif node_data['type'] == 'sqoop-document':
notebook = Notebook(document=Document2.objects.get_by_uuid(user=request.user, uuid=node_data['properties']['uuid']))
parameters = parameters.union(set(find_dollar_braced_variables(notebook.get_str())))
elif node_data['type'] == 'spark-document':
notebook = Notebook(document=Document2.objects.get_by_uuid(user=request.user, uuid=node_data['properties']['uuid']))
for arg in notebook.get_data()['snippets'][0]['properties']['spark_arguments']:
parameters = parameters.union(set(find_dollar_braced_variables(arg)))
response['status'] = 0
response['parameters'] = list(parameters)
except Exception, e:
response['message'] = str(e)
def test_ignore_dead_fork_link(self):
data = {'id': 1, 'type': 'fork', 'children': [{'to': 1, 'id': 1}, {'to': 2, 'id': 2}], 'properties': {}, 'name': 'my-fork'} # to --> 2 does not exist
fork = Node(data)
node_mapping = {1: fork} # Point to ourself
assert_equal(['<fork', 'name="my-fork">', '<path', 'start="my-fork"', '/>', '</fork>'], fork.to_xml(node_mapping=node_mapping).split())
def __init__(self):
self.deployment_dir = '/tmp/test'
self.nodes = [
Node({
'id': '1',
'type': 'hive-document',
'properties': {
'jdbc_url':
u"${wf:actionData('shell-31b5')['hiveserver']}",
'password': u'test'
}
})
]
def __init__(self):
self.deployment_dir = '/tmp/test'
self.nodes = [
Node({
'id': '1',
'type': 'hive-document',
'properties': {
'jdbc_url':
u'jdbc:hive2://test-replace-url:12345/default',
'password': u'test'
}
})
]
def test_ignore_dead_fork_link(self):
data = {
'id': 1,
'type': 'fork',
'children': [{
'to': 1,
'id': 1
}, {
'to': 2,
'id': 2
}],
'properties': {},
'name': 'my-fork'
} # to --> 2 does not exist
fork = Node(data)
node_mapping = {1: fork} # Point to ourself
assert_equal([
'<fork', 'name="my-fork">', '<path', 'start="my-fork"', '/>',
'</fork>'
],
fork.to_xml(node_mapping=node_mapping).split())
def __init__(self):
self.deployment_dir = deployment_dir
self.nodes = [
Node({'id': '1', 'type': 'mapreduce', 'properties': {'jar_path': jar_1}}),
Node({'id': '2', 'type': 'mapreduce', 'properties': {'jar_path': jar_2}}),
Node({'id': '3', 'type': 'java', 'properties': {'jar_path': jar_3}}),
Node({'id': '4', 'type': 'java', 'properties': {'jar_path': jar_4}}),
# Workspace relative paths
Node({'id': '5', 'type': 'java', 'properties': {'jar_path': jar_5}}),
Node({'id': '6', 'type': 'java', 'properties': {'jar_path': jar_6}})
]
def test_action_gen_xml_prepare(self):
# Prepare has a value
data = {
u'properties': {
u'files': [], u'job_xml': [], u'parameters': [], u'retry_interval': [], u'retry_max': [], u'job_properties': [], u'arguments': [],
u'prepares': [{u'type': u'mkdir', u'value': u'/my_dir'}],
u'credentials': [], u'script_path': u'my_pig.pig',
u'sla': [{u'key': u'enabled', u'value': False}, {u'key': u'nominal-time', u'value': u'${nominal_time}'}, {u'key': u'should-start', u'value': u''}, {u'key': u'should-end', u'value': u'${30 * MINUTES}'}, {u'key': u'max-duration', u'value': u''}, {u'key': u'alert-events', u'value': u''}, {u'key': u'alert-contact', u'value': u''}, {u'key': u'notification-msg', u'value': u''}, {u'key': u'upstream-apps', u'value': u''}],
u'archives': []
},
u'type': u'pig-widget',
u'id': u'c59d1947-7ce0-ef34-22b2-d64b9fc5bf9a',
u'name': u'pig-c59d',
"children":[{"to": "c59d1947-7ce0-ef34-22b2-d64b9fc5bf9a"}, {"error": "c59d1947-7ce0-ef34-22b2-d64b9fc5bf9a"}]
}
pig_node = Node(data)
node_mapping = {"c59d1947-7ce0-ef34-22b2-d64b9fc5bf9a": pig_node}
xml = pig_node.to_xml(node_mapping=node_mapping)
xml = [row.strip() for row in xml.split()]
assert_true(u'<prepare>' in xml, xml)
assert_true(u'<mkdir' in xml, xml)
assert_true(u'path="${nameNode}/my_dir"/>' in xml, xml)
# Prepare has empty value and is skipped
pig_node.data['properties']['prepares'] = [{u'type': u'mkdir', u'value': u''}]
xml = pig_node.to_xml(node_mapping=node_mapping)
xml = [row.strip() for row in xml.split()]
assert_false(u'<prepare>' in xml, xml)
assert_false(u'<mkdir' in xml, xml)
# Prepare has a value and an empty value
pig_node.data['properties']['prepares'] = [{u'type': u'mkdir', u'value': u'/my_dir'}, {u'type': u'rm', u'value': u''}]
xml = pig_node.to_xml(node_mapping=node_mapping)
xml = [row.strip() for row in xml.split()]
assert_true(u'<prepare>' in xml, xml)
assert_true(u'<mkdir' in xml, xml)
assert_true(u'path="${nameNode}/my_dir"/>' in xml, xml)
assert_false(u'<rm' in xml, xml)
def test_action_gen_xml_java_opts(self):
# Contains java_opts
data = {
u'name':
u'java-fc05',
u'properties': {
u'files': [],
u'job_xml': [],
u'jar_path':
u'/user/romain/hadoop-mapreduce-examples.jar',
u'java_opts': [{
u'value': u'-debug -Da -Db=1'
}],
u'retry_max': [],
u'retry_interval': [],
u'job_properties': [],
u'capture_output':
False,
u'main_class':
u'MyClass',
u'arguments': [],
u'prepares': [],
u'credentials': [],
u'sla': [{
u'value': False,
u'key': u'enabled'
}, {
u'value': u'${nominal_time}',
u'key': u'nominal-time'
}, {
u'value': u'',
u'key': u'should-start'
}, {
u'value': u'${30 * MINUTES}',
u'key': u'should-end'
}, {
u'value': u'',
u'key': u'max-duration'
}, {
u'value': u'',
u'key': u'alert-events'
}, {
u'value': u'',
u'key': u'alert-contact'
}, {
u'value': u'',
u'key': u'notification-msg'
}, {
u'value': u'',
u'key': u'upstream-apps'
}],
u'archives': []
},
u'actionParametersFetched':
False,
u'id':
u'fc05d86f-9f07-7a8d-6256-e6abfa87cf77',
u'type':
u'java-widget',
u'children': [{
u'to': u'33430f0f-ebfa-c3ec-f237-3e77efa03d0a'
}, {
u'error': u'17c9c895-5a16-7443-bb81-f34b30b21548'
}],
u'actionParameters': []
}
java_node = Node(data)
node_mapping = {
"fc05d86f-9f07-7a8d-6256-e6abfa87cf77": java_node,
"33430f0f-ebfa-c3ec-f237-3e77efa03d0a": java_node,
"17c9c895-5a16-7443-bb81-f34b30b21548": java_node
} # Last 2 are actually kill and ok nodes
xml = java_node.to_xml(node_mapping=node_mapping)
xml = [row.strip() for row in xml.split('\n')]
assert_false(
"<java-opts>[{u'value': u'-debug -Da -Db=1'}]</java-opts>"
in xml, xml)
assert_true("<java-opts>-debug -Da -Db=1</java-opts>" in xml, xml)
def test_action_gen_xml_prepare(self):
# Prepare has a value
data = {
u'properties': {
u'files': [],
u'job_xml': [],
u'parameters': [],
u'retry_interval': [],
u'retry_max': [],
u'job_properties': [],
u'arguments': [],
u'prepares': [{
u'type': u'mkdir',
u'value': u'/my_dir'
}],
u'credentials': [],
u'script_path':
u'my_pig.pig',
u'sla': [{
u'key': u'enabled',
u'value': False
}, {
u'key': u'nominal-time',
u'value': u'${nominal_time}'
}, {
u'key': u'should-start',
u'value': u''
}, {
u'key': u'should-end',
u'value': u'${30 * MINUTES}'
}, {
u'key': u'max-duration',
u'value': u''
}, {
u'key': u'alert-events',
u'value': u''
}, {
u'key': u'alert-contact',
u'value': u''
}, {
u'key': u'notification-msg',
u'value': u''
}, {
u'key': u'upstream-apps',
u'value': u''
}],
u'archives': []
},
u'type':
u'pig-widget',
u'id':
u'c59d1947-7ce0-ef34-22b2-d64b9fc5bf9a',
u'name':
u'pig-c59d',
"children": [{
"to": "c59d1947-7ce0-ef34-22b2-d64b9fc5bf9a"
}, {
"error": "c59d1947-7ce0-ef34-22b2-d64b9fc5bf9a"
}]
}
pig_node = Node(data)
node_mapping = {"c59d1947-7ce0-ef34-22b2-d64b9fc5bf9a": pig_node}
xml = pig_node.to_xml(node_mapping=node_mapping)
xml = [row.strip() for row in xml.split()]
assert_true(u'<prepare>' in xml, xml)
assert_true(u'<mkdir' in xml, xml)
assert_true(u'path="${nameNode}/my_dir"/>' in xml, xml)
# Prepare has empty value and is skipped
pig_node.data['properties']['prepares'] = [{
u'type': u'mkdir',
u'value': u''
}]
xml = pig_node.to_xml(node_mapping=node_mapping)
xml = [row.strip() for row in xml.split()]
assert_false(u'<prepare>' in xml, xml)
assert_false(u'<mkdir' in xml, xml)
# Prepare has a value and an empty value
pig_node.data['properties']['prepares'] = [{
u'type': u'mkdir',
u'value': u'/my_dir'
}, {
u'type': u'rm',
u'value': u''
}]
xml = pig_node.to_xml(node_mapping=node_mapping)
xml = [row.strip() for row in xml.split()]
assert_true(u'<prepare>' in xml, xml)
assert_true(u'<mkdir' in xml, xml)
assert_true(u'path="${nameNode}/my_dir"/>' in xml, xml)
assert_false(u'<rm' in xml, xml)
