def _get_mapreduce_mapper(self, job_properties, config): return mapreduce_mapper.MapReduceMapper( oozie_node=self.mapreduce_node, name="test_id", dag_name="DAG_NAME_B", props=PropertySet(job_properties=job_properties, config=config), )
def _get_mapreduce_mapper(self, job_properties, config): return mapreduce_mapper.MapReduceMapper( oozie_node=self.mapreduce_node, name="test_id", dag_name="DAG_NAME_B", props=PropertySet(job_properties=job_properties, config=config), input_directory_path="/tmp/input-directory-path/", )
def test_to_tasks_and_relations(self): mapper = mapreduce_mapper.MapReduceMapper( oozie_node=self.mapreduce_node, name="test_id", dag_name="DAG_NAME_B", props=PropertySet( job_properties={"nameNode": "hdfs://"}, config={ "dataproc_cluster": "my-cluster", "gcp_region": "europe-west3", "hadoop_jars": "hdfs:///user/mapred/examples/mapreduce/lib/wordcount.jar", "hadoop_main_class": "WordCount", }, ), ) mapper.on_parse_node() tasks, relations = mapper.to_tasks_and_relations() self.assertEqual( [ Task( task_id="test_id", template_name="mapreduce.tpl", template_params={ "props": PropertySet( config={ "dataproc_cluster": "my-cluster", "gcp_region": "europe-west3", "hadoop_jars": "hdfs:///user/mapred/examples/mapreduce/lib/wordcount.jar", "hadoop_main_class": "WordCount", }, job_properties={"nameNode": "hdfs://"}, action_node_properties={ "mapred.mapper.new-api": "true", "mapred.reducer.new-api": "true", "mapred.job.queue.name": "${queueName}", "mapreduce.job.map.class": "WordCount$Map", "mapreduce.job.reduce.class": "WordCount$Reduce", "mapreduce.job.output.key.class": "org.apache.hadoop.io.Text", "mapreduce.job.output.value.class": "org.apache.hadoop.io.IntWritable", "mapreduce.input.fileinputformat." "inputdir": "/user/mapred/${examplesRoot}/mapreduce/input", "mapreduce.output.fileoutputformat." "outputdir": "/user/mapred/${examplesRoot}/mapreduce/output", }, ), "params_dict": {}, "hdfs_files": [], "hdfs_archives": [], "action_node_properties": { "mapred.mapper.new-api": "true", "mapred.reducer.new-api": "true", "mapred.job.queue.name": "${queueName}", "mapreduce.job.map.class": "WordCount$Map", "mapreduce.job.reduce.class": "WordCount$Reduce", "mapreduce.job.output.key.class": "org.apache.hadoop.io.Text", "mapreduce.job.output.value.class": "org.apache.hadoop.io.IntWritable", "mapreduce.input.fileinputformat.inputdir": "/user/mapred/${examplesRoot}/" "mapreduce/input", "mapreduce.output.fileoutputformat.outputdir": "/user/mapred/${examplesRoot}/" "mapreduce/output", }, }, ) ], tasks, ) self.assertEqual(relations, [])