def test_convert_mapreduce(self):
wf = Workflow.objects.new_workflow(self.user)
wf.save()
Workflow.objects.initialize(wf)
Link.objects.filter(parent__workflow=wf).delete()
action = add_node(wf, 'action-name-1', 'mapreduce', [wf.start], {
'description': 'Test MR job design',
'files': '[]',
'jar_path': '/user/hue/oozie/examples/lib/hadoop-examples.jar',
'job_properties': '[{"name": "sleep.job.map.sleep.time", "value": "5"}, {"name": "sleep.job.reduce.sleep.time", "value": "10"}]',
'prepares': '[{"value":"${output}","type":"delete"},{"value":"/test","type":"mkdir"}]',
'archives': '[]',
})
Link(parent=action, child=wf.end, name="ok").save()
# Setting doc.last_modified to older date
doc = Document.objects.get(id=wf.doc.get().id)
Document.objects.filter(id=doc.id).update(last_modified=datetime.strptime('2000-01-01T00:00:00Z', '%Y-%m-%dT%H:%M:%SZ'))
doc = Document.objects.get(id=doc.id)
try:
# Test that corresponding doc2 is created after convert
assert_false(Document2.objects.filter(owner=self.user, type='query-mapreduce').exists())
converter = DocumentConverter(self.user)
converter.convert()
doc2 = Document2.objects.get(owner=self.user, type='query-mapreduce')
# Verify snippet values
assert_equal('ready', doc2.data_dict['snippets'][0]['status'])
assert_equal('/user/hue/oozie/examples/lib/hadoop-examples.jar', doc2.data_dict['snippets'][0]['properties']['app_jar'])
assert_equal(['sleep.job.map.sleep.time=5', 'sleep.job.reduce.sleep.time=10'], doc2.data_dict['snippets'][0]['properties']['hadoopProperties'])
finally:
wf.delete()
def test_convert_shell(self):
wf = Workflow.objects.new_workflow(self.user)
wf.save()
Workflow.objects.initialize(wf)
Link.objects.filter(parent__workflow=wf).delete()
action = add_node(wf, 'action-name-1', 'shell', [wf.start], {
u'job_xml': 'my-job.xml',
u'files': '["hello.py"]',
u'name': 'Shell',
u'job_properties': '[{"name": "mapred.job.queue.name", "value": "test"}]',
u'capture_output': 'on',
u'command': 'hello.py',
u'archives': '[{"dummy": "", "name": "test.zip"}]',
u'prepares': '[]',
u'params': '[{"type": "argument", "value": "baz"}, {"type": "env-var", "value": "foo=bar"}]',
u'description': 'Execute a Python script printing its arguments'
})
Link(parent=action, child=wf.end, name="ok").save()
# Setting doc.last_modified to older date
doc = Document.objects.get(id=wf.doc.get().id)
Document.objects.filter(id=doc.id).update(last_modified=datetime.strptime('2000-01-01T00:00:00Z', '%Y-%m-%dT%H:%M:%SZ'))
doc = Document.objects.get(id=doc.id)
try:
if IS_HUE_4.get():
# Test that corresponding doc2 is created after convert
assert_false(Document2.objects.filter(owner=self.user, type='query-shell').exists())
converter = DocumentConverter(self.user)
converter.convert()
doc2 = Document2.objects.get(owner=self.user, type='query-shell')
# Verify snippet values
assert_equal('ready', doc2.data_dict['snippets'][0]['status'])
assert_equal('hello.py', doc2.data_dict['snippets'][0]['properties']['command_path'])
assert_equal(['baz'], doc2.data_dict['snippets'][0]['properties']['arguments'])
assert_equal(['foo=bar'], doc2.data_dict['snippets'][0]['properties']['env_var'])
assert_equal(['mapred.job.queue.name=test'], doc2.data_dict['snippets'][0]['properties']['hadoopProperties'])
assert_equal(['test.zip'], doc2.data_dict['snippets'][0]['properties']['archives'])
assert_equal([{'type': 'file', 'path': 'hello.py'}], doc2.data_dict['snippets'][0]['properties']['files'])
assert_equal(True, doc2.data_dict['snippets'][0]['properties']['capture_output'])
else:
# Test that corresponding doc2 is created after convert
assert_false(Document2.objects.filter(owner=self.user, type='link-workflow').exists())
converter = DocumentConverter(self.user)
converter.convert()
doc2 = Document2.objects.get(owner=self.user, type='link-workflow')
# Verify absolute_url
response = self.client.get(doc2.get_absolute_url())
assert_equal(200, response.status_code)
assert_equal(doc.last_modified.strftime('%Y-%m-%dT%H:%M:%S'), doc2.last_modified.strftime('%Y-%m-%dT%H:%M:%S'))
finally:
wf.delete()
def test_convert_mapreduce(self):
wf = Workflow.objects.new_workflow(self.user)
wf.save()
Workflow.objects.initialize(wf)
Link.objects.filter(parent__workflow=wf).delete()
action = add_node(wf, 'action-name-1', 'mapreduce', [wf.start], {
'description': 'Test MR job design',
'files': '[]',
'jar_path': '/user/hue/oozie/examples/lib/hadoop-examples.jar',
'job_properties': '[{"name": "sleep.job.map.sleep.time", "value": "5"}, {"name": "sleep.job.reduce.sleep.time", "value": "10"}]',
'prepares': '[{"value":"${output}","type":"delete"},{"value":"/test","type":"mkdir"}]',
'archives': '[]',
})
Link(parent=action, child=wf.end, name="ok").save()
# Setting doc.last_modified to older date
doc = Document.objects.get(id=wf.doc.get().id)
Document.objects.filter(id=doc.id).update(last_modified=datetime.strptime('2000-01-01T00:00:00Z', '%Y-%m-%dT%H:%M:%SZ'))
doc = Document.objects.get(id=doc.id)
try:
if IS_HUE_4.get():
# Test that corresponding doc2 is created after convert
assert_false(Document2.objects.filter(owner=self.user, type='query-mapreduce').exists())
converter = DocumentConverter(self.user)
converter.convert()
doc2 = Document2.objects.get(owner=self.user, type='query-mapreduce')
# Verify snippet values
assert_equal('ready', doc2.data_dict['snippets'][0]['status'])
assert_equal('/user/hue/oozie/examples/lib/hadoop-examples.jar', doc2.data_dict['snippets'][0]['properties']['app_jar'])
assert_equal(['sleep.job.map.sleep.time=5', 'sleep.job.reduce.sleep.time=10'], doc2.data_dict['snippets'][0]['properties']['hadoopProperties'])
else:
# Test that corresponding doc2 is created after convert
assert_false(Document2.objects.filter(owner=self.user, type='link-workflow').exists())
converter = DocumentConverter(self.user)
converter.convert()
doc2 = Document2.objects.get(owner=self.user, type='link-workflow')
# Verify absolute_url
response = self.client.get(doc2.get_absolute_url())
assert_equal(200, response.status_code)
assert_equal(doc.last_modified.strftime('%Y-%m-%dT%H:%M:%S'), doc2.last_modified.strftime('%Y-%m-%dT%H:%M:%S'))
finally:
wf.delete()
def test_convert_java(self):
wf = Workflow.objects.new_workflow(self.user)
wf.save()
Workflow.objects.initialize(wf)
Link.objects.filter(parent__workflow=wf).delete()
action = add_node(wf, 'action-name-1', 'java', [wf.start], {
'name': 'MyTeragen',
"description": "Generate N number of records",
"main_class": "org.apache.hadoop.examples.terasort.TeraGen",
"args": "1000 ${output_dir}/teragen",
"files": '["my_file","my_file2"]',
"job_xml": "",
"java_opts": "-Dexample-property=natty",
"jar_path": "/user/hue/oozie/workspaces/lib/hadoop-examples.jar",
'job_properties': '[{"name": "mapred.job.queue.name", "value": "test"}]',
"prepares": '[{"value":"/test","type":"mkdir"}]',
"archives": '[{"dummy":"","name":"my_archive"},{"dummy":"","name":"my_archive2"}]',
"capture_output": True,
})
Link(parent=action, child=wf.end, name="ok").save()
# Setting doc.last_modified to older date
doc = Document.objects.get(id=wf.doc.get().id)
Document.objects.filter(id=doc.id).update(
last_modified=datetime.strptime('2000-01-01T00:00:00Z', '%Y-%m-%dT%H:%M:%SZ'))
doc = Document.objects.get(id=doc.id)
try:
# Test that corresponding doc2 is created after convert
assert_false(Document2.objects.filter(owner=self.user, type='query-java').exists())
converter = DocumentConverter(self.user)
converter.convert()
doc2 = Document2.objects.get(owner=self.user, type='query-java')
# Verify snippet values
assert_equal('ready', doc2.data_dict['snippets'][0]['status'])
assert_equal('/user/hue/oozie/workspaces/lib/hadoop-examples.jar', doc2.data_dict['snippets'][0]['properties']['app_jar'])
assert_equal('org.apache.hadoop.examples.terasort.TeraGen', doc2.data_dict['snippets'][0]['properties']['class'])
assert_equal('1000 ${output_dir}/teragen', doc2.data_dict['snippets'][0]['properties']['args'])
assert_equal('-Dexample-property=natty', doc2.data_dict['snippets'][0]['properties']['java_opts'])
assert_equal(['mapred.job.queue.name=test'], doc2.data_dict['snippets'][0]['properties']['hadoopProperties'])
assert_equal(['my_archive', 'my_archive2'], doc2.data_dict['snippets'][0]['properties']['archives'])
assert_equal([{'type': 'file', 'path': 'my_file'}, {'type': 'file', 'path': 'my_file2'}], doc2.data_dict['snippets'][0]['properties']['files'])
assert_equal(True, doc2.data_dict['snippets'][0]['properties']['capture_output'])
finally:
wf.delete()
def test_convert_shell(self):
wf = Workflow.objects.new_workflow(self.user)
wf.save()
Workflow.objects.initialize(wf)
Link.objects.filter(parent__workflow=wf).delete()
action = add_node(wf, 'action-name-1', 'shell', [wf.start], {
u'job_xml': 'my-job.xml',
u'files': '["hello.py"]',
u'name': 'Shell',
u'job_properties': '[{"name": "mapred.job.queue.name", "value": "test"}]',
u'capture_output': True,
u'command': 'hello.py',
u'archives': '[{"dummy": "", "name": "test.zip"}]',
u'prepares': '[]',
u'params': '[{"type": "argument", "value": "baz"}, {"type": "env-var", "value": "foo=bar"}]',
u'description': 'Execute a Python script printing its arguments'
})
Link(parent=action, child=wf.end, name="ok").save()
# Setting doc.last_modified to older date
doc = Document.objects.get(id=wf.doc.get().id)
Document.objects.filter(id=doc.id).update(last_modified=datetime.strptime('2000-01-01T00:00:00Z', '%Y-%m-%dT%H:%M:%SZ'))
doc = Document.objects.get(id=doc.id)
try:
# Test that corresponding doc2 is created after convert
assert_false(Document2.objects.filter(owner=self.user, type='query-shell').exists())
converter = DocumentConverter(self.user)
converter.convert()
doc2 = Document2.objects.get(owner=self.user, type='query-shell')
# Verify snippet values
assert_equal('ready', doc2.data_dict['snippets'][0]['status'])
assert_equal('hello.py', doc2.data_dict['snippets'][0]['properties']['command_path'])
assert_equal(['baz'], doc2.data_dict['snippets'][0]['properties']['arguments'])
assert_equal(['foo=bar'], doc2.data_dict['snippets'][0]['properties']['env_var'])
assert_equal(['mapred.job.queue.name=test'], doc2.data_dict['snippets'][0]['properties']['hadoopProperties'])
assert_equal(['test.zip'], doc2.data_dict['snippets'][0]['properties']['archives'])
assert_equal([{'type': 'file', 'path': 'hello.py'}], doc2.data_dict['snippets'][0]['properties']['files'])
assert_equal(True, doc2.data_dict['snippets'][0]['properties']['capture_output'])
finally:
wf.delete()
def test_convert_java(self):
wf = Workflow.objects.new_workflow(self.user)
wf.save()
Workflow.objects.initialize(wf)
Link.objects.filter(parent__workflow=wf).delete()
action = add_node(
wf, 'action-name-1', 'java', [wf.start], {
'name': 'MyTeragen',
"description": "Generate N number of records",
"main_class": "org.apache.hadoop.examples.terasort.TeraGen",
"args": "1000 ${output_dir}/teragen",
"files": '["my_file","my_file2"]',
"job_xml": "",
"java_opts": "-Dexample-property=natty",
"jar_path":
"/user/hue/oozie/workspaces/lib/hadoop-examples.jar",
'job_properties':
'[{"name": "mapred.job.queue.name", "value": "test"}]',
"prepares": '[{"value":"/test","type":"mkdir"}]',
"archives":
'[{"dummy":"","name":"my_archive"},{"dummy":"","name":"my_archive2"}]',
"capture_output": True,
})
Link(parent=action, child=wf.end, name="ok").save()
# Setting doc.last_modified to older date
doc = Document.objects.get(id=wf.doc.get().id)
Document.objects.filter(
id=doc.id).update(last_modified=datetime.strptime(
'2000-01-01T00:00:00Z', '%Y-%m-%dT%H:%M:%SZ'))
doc = Document.objects.get(id=doc.id)
try:
# Test that corresponding doc2 is created after convert
assert_false(
Document2.objects.filter(owner=self.user,
type='query-java').exists())
converter = DocumentConverter(self.user)
converter.convert()
doc2 = Document2.objects.get(owner=self.user, type='query-java')
# Verify snippet values
assert_equal('ready', doc2.data_dict['snippets'][0]['status'])
assert_equal(
'/user/hue/oozie/workspaces/lib/hadoop-examples.jar',
doc2.data_dict['snippets'][0]['properties']['app_jar'])
assert_equal('org.apache.hadoop.examples.terasort.TeraGen',
doc2.data_dict['snippets'][0]['properties']['class'])
assert_equal('1000 ${output_dir}/teragen',
doc2.data_dict['snippets'][0]['properties']['args'])
assert_equal(
'-Dexample-property=natty',
doc2.data_dict['snippets'][0]['properties']['java_opts'])
assert_equal([
'mapred.job.queue.name=test'
], doc2.data_dict['snippets'][0]['properties']['hadoopProperties'])
assert_equal(
['my_archive', 'my_archive2'],
doc2.data_dict['snippets'][0]['properties']['archives'])
assert_equal([{
'type': 'file',
'path': 'my_file'
}, {
'type': 'file',
'path': 'my_file2'
}], doc2.data_dict['snippets'][0]['properties']['files'])
assert_equal(
True,
doc2.data_dict['snippets'][0]['properties']['capture_output'])
finally:
wf.delete()
