def check_results(data_in, data_out, logger): local_wc = pts.LocalWordCount(data_in) logger.info("checking results") with open(data_out, 'r') as f: res = local_wc.check(f.read()) if res.startswith("ERROR"): # FIXME: change local_wc to raise an exception logger.error(res) raise RuntimeError(res) logger.info(res)
def main(argv): logger = logging.getLogger("main") logger.setLevel(logging.INFO) input_dir = argv[1] output_dir = argv[2] logger.info("checking results") lwc = pts.LocalWordCount(input_dir) measured_res = get_res(output_dir) expected_res = lwc.expected_output logger.info(check(measured_res, expected_res))
def main(): parser = make_parser() opt, _ = parser.parse_args() logger = logging.getLogger("main") logger.setLevel(logging.INFO) logger.info("running word count") wc_output = run_wc(opt) logger.info("running filter") filter_output = run_filter(opt, wc_output) logger.info("checking results") res = hadut.collect_output(filter_output) local_wc = pts.LocalWordCount(opt.input, min_occurrence=opt.threshold) logger.info(local_wc.check(res))
def main(argv): parser = make_parser() args = parser.parse_args(argv) update_conf(args) logger = logging.getLogger("main") logger.setLevel(logging.INFO) runner = hadut.PipesRunner(prefix=PREFIX, logger=logger) with open(args.pipes_exe) as f: pipes_code = pts.add_sys_path(f.read()) runner.set_input(args.local_input, put=True) runner.set_exe(pipes_code) runner.run(properties=CONF, hadoop_conf_dir=HADOOP_CONF_DIR, logger=logger) res = runner.collect_output() runner.clean() local_wc = pts.LocalWordCount(args.local_input) logging.info(local_wc.check(res))
def check_wordcount(mr_out_dir, stop_words=None): output = hadut.collect_output(mr_out_dir) local_wc = pts.LocalWordCount(DEFAULT_INPUT_DIR, stop_words=stop_words) res = local_wc.check(output) return res.startswith("OK") # FIXME: change local_wc to raise an exception
def main(argv): logger = logging.getLogger("main") logger.setLevel(logging.DEBUG) with Timer() as total_time: parser = make_parser() args = parser.parse_args(argv) if args.dataset: print args.dataset create_dataset(logger, args.dataset) if args.script: piped_code_file = args.script else: piped_code_file = DEFAULT_SCRIPT if not os.path.exists(piped_code_file): raise IOError("script {0} not found !!!".format(piped_code_file)) with open(piped_code_file) as f: pipes_code = pts.add_sys_path(f.read()) dataset = [d for d in os.listdir("dataset") if d.endswith("MB")] dataset.sort(cmp=lambda x, y: cmp(int(x.replace("MB", "")), int(y.replace("MB", "")))) logger.info(" Uploading dataset: { %s }", ', '.join(dataset)) if not hadut.path_exists(os.path.join(DATASET_DIR)): logger.info(" dataset folder created") hdfs.mkdir(DATASET_DIR) for data_filename in dataset: source_path = os.path.join(DATASET_DIR, data_filename) dest_path = os.path.join(DATASET_DIR, data_filename) if not hadut.path_exists(os.path.join(DATASET_DIR, data_filename)): logger.info(" -> uploading %s...", source_path) hdfs.put(source_path, dest_path) update_conf(args) results = dict() for data_input in dataset: with Timer() as t: runner = hadut.PipesRunner(prefix=PREFIX, logger=logger) logger.info("Running the script %s with data input %s..", piped_code_file, data_input) data_input_path = os.path.join(DATASET_DIR, data_input) runner.set_input(data_input_path, put=False) runner.set_exe(pipes_code) runner.run(properties=CONF, hadoop_conf_dir=HADOOP_CONF_DIR, logger=logger) res = runner.collect_output() print data_input_path local_wc = pts.LocalWordCount(data_input_path) logging.info(local_wc.check(res)) # print res # runner.clean() results[data_input] = (t.secs, t.msecs) print "\n\n RESULTs" print "=" * (len(piped_code_file) + 15) print " * script: {0}".format(piped_code_file) print " * mappers: {0}".format(CONF["mapred.map.tasks"]) print " * reducers: {0}".format(CONF["mapred.reduce.tasks"]) print " * dataset: [{0}]".format(",".join(dataset)) print " * times (input -> secs):" for data_input in dataset: print " - {0} -> {1} secs.".format(data_input, results[data_input][0]) print "\n => Total execution time: {0}".format(total_time.secs) print "=" * (len(piped_code_file) + 15) print "\n"