def test_create_mapreduce_streaming(self):
mr_action = """
<streaming>
<mapper>/usr/bin/cat</mapper>
<reducer>/usr/bin/wc</reducer>
</streaming>"""
mr_workflow = mrw.MapReduceWorkFlowCreator()
mr_workflow.build_workflow_xml(self.prepare, self.job_xml,
self.configuration, self.files,
self.archives, self.streaming)
res = mr_workflow.get_built_workflow_xml()
self.assertIn(mr_action, res)
mr_workflow = mrw.MapReduceWorkFlowCreator()
mr_workflow.build_workflow_xml(self.prepare, self.job_xml,
self.configuration, self.files,
self.archives)
res = mr_workflow.get_built_workflow_xml()
self.assertNotIn(mr_action, res)
mr_workflow = mrw.MapReduceWorkFlowCreator()
with testtools.ExpectedException(ex.NotFoundException):
mr_workflow.build_workflow_xml(self.prepare, self.job_xml,
self.configuration, self.files,
self.archives, {'bogus': 'element'})
def test_create_mapreduce_workflow(self):
mr_workflow = mrw.MapReduceWorkFlowCreator()
mr_workflow.build_workflow_xml(self.prepare, self.job_xml,
self.configuration, self.files,
self.archives)
res = mr_workflow.get_built_workflow_xml()
mr_action = """ <map-reduce>
<job-tracker>${jobTracker}</job-tracker>
<name-node>${nameNode}</name-node>
<prepare>
<delete path="delete_dir_1"/>
<delete path="delete_dir_2"/>
<mkdir path="mkdir_1"/>
</prepare>
<job-xml>job_xml.xml</job-xml>
<configuration>
<property>
<name>conf_param_1</name>
<value>conf_value_1</value>
</property>
<property>
<name>conf_param_2</name>
<value>conf_value_3</value>
</property>
</configuration>
<file>file1</file>
<file>file2</file>
<archive>arch1</archive>
</map-reduce>"""
self.assertIn(mr_action, res)
def get_workflow_xml(self, cluster, job_configs, input_data, output_data,
hdfs_user):
proxy_configs = job_configs.get('proxy_configs')
job_dict = {'configs': self.get_configs(input_data, output_data,
proxy_configs)}
self.update_job_dict(job_dict, job_configs)
creator = mapreduce_workflow.MapReduceWorkFlowCreator()
creator.build_workflow_xml(configuration=job_dict['configs'],
streaming=self._get_streaming(job_dict))
return creator.get_built_workflow_xml()
