コード例 #1
0
ファイル: test_support.py プロジェクト: crs4/pydoop
 def test_record_reader(self):
     job_conf = {'this.is.not.used': '22'}
     hs = HadoopSimulatorLocal(TFactory(record_reader=TrivialRecordReader))
     foname = 'map_reduce.out'
     with self._mkf(foname) as fout:
         hs.run(None, fout, job_conf, 0)
         self.assertTrue(os.stat(fout.name).st_size > 0)
コード例 #2
0
ファイル: test_support.py プロジェクト: wtj/pydoop
 def test_map_only(self):
     job_conf = {'this.is.not.used': '22'}
     hs = HadoopSimulatorLocal(TFactory())
     with open(self.fname, 'r') as fin:
         with self._mkf('map_only.out') as fout:
             hs.run(fin, fout, job_conf, 0)
             self.assertTrue(os.stat(fout.name).st_size > 0)
コード例 #3
0
ファイル: test_support.py プロジェクト: crs4/pydoop
 def test_map_only(self):
     job_conf = {'this.is.not.used': '22'}
     hs = HadoopSimulatorLocal(TFactory(), loglevel=logging.CRITICAL)
     with open(self.fname, 'r') as fin:
         with self._mkf('map_only.out', 'w') as fout:
             hs.run(fin, fout, job_conf, 0)
             self.assertTrue(os.stat(fout.name).st_size > 0)
コード例 #4
0
ファイル: test_support.py プロジェクト: wtj/pydoop
 def test_record_reader(self):
     job_conf = {'this.is.not.used': '22'}
     hs = HadoopSimulatorLocal(TFactory(record_reader=TrivialRecordReader))
     foname = 'map_reduce.out'
     with self._mkf(foname) as fout:
         hs.run(None, fout, job_conf, 0)
         self.assertTrue(os.stat(fout.name).st_size > 0)
コード例 #5
0
ファイル: run.py プロジェクト: crs4/pydoop
def run_local_minimal(logger):
    hs = HadoopSimulatorLocal(factory=factory_minimal, logger=logger,
                              loglevel=logger.level)
    data_in, data_out, conf, input_split, output_dir = create_configuration()
    hs.run(open(data_in), open(data_out, 'wb'), conf)
    dump_counters(hs, logger)
    check_results(data_in, data_out, logger)
    clean_up(data_out, output_dir)
コード例 #6
0
ファイル: run.py プロジェクト: muhammadyaseen/pydoop
def run_local_minimal(logger):
    hs = HadoopSimulatorLocal(factory=factory_minimal,
                              logger=logger,
                              loglevel=logger.level)
    data_in, data_out, conf, input_split, output_dir = create_configuration()
    hs.run(open(data_in), open(data_out, 'wb'), conf)
    dump_counters(hs, logger)
    check_results(data_in, data_out, logger)
    clean_up(data_out, output_dir)
コード例 #7
0
ファイル: run.py プロジェクト: crs4/pydoop
def run_local_full(logger):
    data_in, data_out, conf, input_split, output_dir = create_configuration()
    hsl = HadoopSimulatorLocal(factory=factory_full, logger=logger,
                               loglevel=logger.level)
    hsl.run(None, None, conf, input_split=input_split, num_reducers=1)
    data_out = os.path.join(output_dir,
                            'part-r-%05d' % int(conf["mapred.task.partition"]))
    dump_counters(hsl, logger)
    check_results(data_in, data_out, logger)
    clean_up(data_out, output_dir)
コード例 #8
0
ファイル: run.py プロジェクト: muhammadyaseen/pydoop
def run_local_full(logger):
    data_in, data_out, conf, input_split, output_dir = create_configuration()
    hsl = HadoopSimulatorLocal(factory=factory_full,
                               logger=logger,
                               loglevel=logger.level)
    hsl.run(None, None, conf, input_split=input_split, num_reducers=1)
    data_out = os.path.join(output_dir,
                            'part-r-%05d' % int(conf["mapred.task.partition"]))
    dump_counters(hsl, logger)
    check_results(data_in, data_out, logger)
    clean_up(data_out, output_dir)
コード例 #9
0
 def test_map_combiner_reduce(self):
     job_conf = {'this.is.not.used': '22'}
     hs = HadoopSimulatorLocal(TFactory(combiner=TReducer))
     foname = 'map_combiner_reduce.out'
     with open(self.fname, 'r') as fin:
         with self._mkf(foname) as fout:
             hs.run(fin, fout, job_conf, 1)
     with open(self._mkfn(foname)) as f:
         for l in f:
             k, c = l.strip().split()
             self.assertEqual(COUNTS[k], int(c))
コード例 #10
0
 def test_map_combiner_reduce(self):
     job_conf = {'this.is.not.used': '22'}
     hs = HadoopSimulatorLocal(TFactory(combiner=TReducer))
     foname = 'map_combiner_reduce.out'
     with open(self.fname, 'r') as fin:
         with self._mkf(foname) as fout:
             hs.run(fin, fout, job_conf, 1)
     with open(self._mkfn(foname)) as f:
         for l in f:
             k, c = l.strip().split()
             self.assertEqual(COUNTS[k], int(c))
コード例 #11
0
ファイル: test_support.py プロジェクト: crs4/pydoop
 def test_map_reduce(self):
     job_conf = {'this.is.not.used': '22'}
     hs = HadoopSimulatorLocal(TFactory(), loglevel=logging.CRITICAL)
     foname = 'map_reduce.out'
     with open(self.fname, 'r') as fin:
         with self._mkf(foname) as fout:
             hs.run(fin, fout, job_conf, 1)
             self.assertTrue(os.stat(fout.name).st_size > 0)
     with open(self._mkfn(foname)) as f:
         for l in f:
             k, c = l.strip().split()
             self.assertEqual(COUNTS[k], int(c))
コード例 #12
0
ファイル: test_support.py プロジェクト: muhammadyaseen/pydoop
 def test_map_reduce(self):
     job_conf = {'this.is.not.used': '22'}
     hs = HadoopSimulatorLocal(TFactory(), loglevel=logging.CRITICAL)
     foname = 'map_reduce.out'
     with open(self.fname, 'r') as fin:
         with self._mkf(foname) as fout:
             hs.run(fin, fout, job_conf, 1)
             self.assertTrue(os.stat(fout.name).st_size > 0)
     with open(self._mkfn(foname)) as f:
         for l in f:
             k, c = l.strip().split()
             self.assertEqual(COUNTS[k], int(c))
コード例 #13
0
 def test_map_reduce_with_counters(self):
     job_conf = {'this.is.not.used': '22'}
     hs = HadoopSimulatorLocal(TFactory(reducer_class=TReducerWithCounters))
     foname = 'map_reduce.out'
     with open(self.fname, 'r') as fin:
         with self._mkf(foname) as fout:
             hs.run(fin, fout, job_conf, 1)
             self.assertTrue(os.stat(fout.name).st_size > 0)
     with open(self._mkfn(foname)) as f:
         for l in f:
             k, c = l.strip().split()
             if "COUNTER_" in k:
                 ck = int(k[8:]) - 1
                 key = COUNTS.keys()[ck]
                 self.assertEqual(COUNTS[key], int(c))
             else:
                 self.assertEqual(COUNTS[k], int(c))
コード例 #14
0
ファイル: test_support.py プロジェクト: wtj/pydoop
 def test_map_reduce_with_counters(self):
     job_conf = {'this.is.not.used': '22'}
     hs = HadoopSimulatorLocal(TFactory(reducer_class=TReducerWithCounters))
     foname = 'map_reduce.out'
     with open(self.fname, 'r') as fin:
         with self._mkf(foname) as fout:
             hs.run(fin, fout, job_conf, 1)
             self.assertTrue(os.stat(fout.name).st_size > 0)
     with open(self._mkfn(foname)) as f:
         for l in f:
             k, c = l.strip().split()
             if "COUNTER_" in k:
                 ck = int(k[8:]) - 1
                 key = COUNTS.keys()[ck]
                 self.assertEqual(COUNTS[key], int(c))
             else:
                 self.assertEqual(COUNTS[k], int(c))
コード例 #15
0
ファイル: run.py プロジェクト: muhammadyaseen/pydoop
def run_local_avro(logger, avro_in='v', avro_out=None):
    mapper, reducer = AVRO_MAPPERS[avro_in], AVRO_REDUCERS[avro_out]
    schema_k_out = STATS_SCHEMA_STR if avro_out in {'k', 'kv'} else None
    schema_v_out = STATS_SCHEMA_STR if avro_out in {'v', 'kv'} else None
    file_in = USERS_PETS_FN if avro_in == 'kv' else AVRO_FN
    factory = pp.Factory(mapper_class=mapper, reducer_class=reducer)
    simulator = HadoopSimulatorLocal(factory, logger, logging.INFO,
                                     AvroContext, avro_in, avro_out,
                                     schema_k_out, schema_v_out)
    with open(file_in, 'rb') as fin, open(DATA_OUT, 'wb') as fout:
        simulator.run(fin, fout, {}, num_reducers=1)
    dump_counters(simulator, logger)
    if avro_out:
        data_out_des = DATA_OUT + '-des'
        avro_container_dump_results.main(DATA_OUT, data_out_des, avro_out)
        avro_check_results.main(USERS_CSV_FN, data_out_des)
    else:
        avro_check_results.main(USERS_CSV_FN, DATA_OUT)
コード例 #16
0
ファイル: run.py プロジェクト: crs4/pydoop
def run_local_avro(logger, avro_in='v', avro_out=None):
    mapper, reducer = AVRO_MAPPERS[avro_in], AVRO_REDUCERS[avro_out]
    schema_k_out = STATS_SCHEMA_STR if avro_out in {'k', 'kv'} else None
    schema_v_out = STATS_SCHEMA_STR if avro_out in {'v', 'kv'} else None
    file_in = USERS_PETS_FN if avro_in == 'kv' else AVRO_FN
    factory = pp.Factory(mapper_class=mapper, reducer_class=reducer)
    simulator = HadoopSimulatorLocal(
        factory, logger, logging.INFO, AvroContext,
        avro_in, avro_out, schema_k_out, schema_v_out
    )
    with open(file_in, 'rb') as fin, open(DATA_OUT, 'wb') as fout:
        simulator.run(fin, fout, {}, num_reducers=1)
    dump_counters(simulator, logger)
    if avro_out:
        data_out_des = DATA_OUT + '-des'
        avro_container_dump_results.main(DATA_OUT, data_out_des, avro_out)
        avro_check_results.main(USERS_CSV_FN, data_out_des)
    else:
        avro_check_results.main(USERS_CSV_FN, DATA_OUT)
コード例 #17
0
    def test_map_reduce_with_counters(self):
        job_conf = {'this.is.not.used': '22'}
        hs = HadoopSimulatorLocal(TFactory(reducer_class=TReducerWithCounters))
        foname = 'map_reduce.out'

        with open(self.fname, 'r') as fin:
            with self._mkf(foname) as fout:
                hs.run(fin, fout, job_conf, 1)

        with open(self._mkfn(foname)) as f:
            sum_ = 0
            counter_value = 0
            for l in f:
                k, c = l.strip().split()
                if "COUNTER_" in k:
                    counter_value = int(c)
                    self.assertEqual(sum_, counter_value)
                else:
                    sum_ += int(c)
                    self.assertEqual(COUNTS[k], int(c))
コード例 #18
0
    def test_map_reduce_with_counters(self):
        job_conf = {'this.is.not.used': '22'}
        hs = HadoopSimulatorLocal(TFactory(reducer_class=TReducerWithCounters))
        foname = 'map_reduce.out'

        with open(self.fname, 'r') as fin:
            with self._mkf(foname) as fout:
                hs.run(fin, fout, job_conf, 1)

        with open(self._mkfn(foname)) as f:
            sum_ = 0
            counter_value = 0
            for l in f:
                k, c = l.strip().split()
                if "COUNTER_" in k:
                    counter_value = int(c)
                    self.assertEqual(sum_, counter_value)
                else:
                    sum_ += int(c)
                    self.assertEqual(COUNTS[k], int(c))
コード例 #19
0
 def test_map_only(self):
     job_conf = {'this.is.not.used': '22'}
     hs = HadoopSimulatorLocal(TFactory())
     with open(self.fname, 'r') as fin:
         with self._mkf('map_only.out') as fout:
             hs.run(fin, fout, job_conf, 0)