def main(argv):
parser = make_parser()
opt, _ = parser.parse_args(argv)
if opt.output is not sys.stdout:
opt.output = open(opt.output, 'w')
logger = logging.getLogger("main")
logger.setLevel(logging.DEBUG)
runner = hadut.PipesRunner(prefix=PREFIX, logger=logger)
with open(LOCAL_MR_SCRIPT) as f:
pipes_code = pts.add_sys_path(f.read())
runner.set_input(opt.input, put=True)
runner.set_exe(pipes_code)
mr_options = get_mr_options(opt, runner.wd)
runner.run(properties=mr_options,
hadoop_conf_dir=HADOOP_CONF_DIR,
logger=logger)
mr_output = runner.collect_output()
runner.clean()
d = pts.parse_mr_output(mr_output, vtype=int)
ip_list = sorted(d.iteritems(), key=operator.itemgetter(1), reverse=True)
if opt.n_top:
ip_list = ip_list[:opt.n_top]
for ip, count in ip_list:
opt.output.write("%s\t%d\n" % (ip, count))
if opt.output is not sys.stdout:
opt.output.close()
def run_filter(opt, input_):
runner = hadut.PipesRunner(prefix=PREFIX)
options = BASE_MR_OPTIONS.copy()
options.update({
MR_JOB_NAME: "filter",
MR_IN_CLASS: "%s.SequenceFileInputFormat" % MRLIB,
MR_REDUCE_TASKS: "0",
"filter.occurrence.threshold": opt.threshold,
})
with open(LOCAL_FILTER_SCRIPT) as f:
pipes_code = pts.adapt_script(f.read())
runner.set_input(input_)
runner.set_exe(pipes_code)
runner.run(properties=options, hadoop_conf_dir=HADOOP_CONF_DIR)
return runner.output
def run_wc(opt):
runner = hadut.PipesRunner(prefix=PREFIX)
options = BASE_MR_OPTIONS.copy()
options.update({
MR_JOB_NAME: "wordcount",
MR_OUT_CLASS: "%s.SequenceFileOutputFormat" % MRLIB,
MR_OUT_COMPRESS_TYPE: "NONE",
MR_REDUCE_TASKS: "2",
})
with open(LOCAL_WC_SCRIPT) as f:
pipes_code = pts.adapt_script(f.read())
runner.set_input(opt.input, put=True)
runner.set_exe(pipes_code)
runner.run(properties=options, hadoop_conf_dir=HADOOP_CONF_DIR)
return runner.output
def run_filter(opt, input_):
runner = hadut.PipesRunner(prefix=PREFIX)
options = BASE_MR_OPTIONS.copy()
options.update({
# [TODO] replace student_id with your id, e.g. 2011-12345
MR_JOB_NAME: "filter_2018-26190",
MR_IN_CLASS: "%s.SequenceFileInputFormat" % MRLIB,
MR_REDUCE_TASKS: "1",
})
with open(LOCAL_FILTER_SCRIPT) as f:
pipes_code = pts.adapt_script(f.read())
runner.set_input(input_)
runner.set_exe(pipes_code)
runner.run(properties=options, hadoop_conf_dir=HADOOP_CONF_DIR)
return runner.output
def run_dst(opt):
runner = hadut.PipesRunner(prefix=PREFIX)
options = BASE_MR_OPTIONS.copy()
options.update({
# [TODO] replace student_id with your id, e.g. 2011-12345
MR_JOB_NAME: "dst_count_2018-26190",
MR_OUT_CLASS: "%s.SequenceFileOutputFormat" % MRLIB,
MR_OUT_COMPRESS_TYPE: "NONE",
MR_REDUCE_TASKS: "2",
})
with open(LOCAL_DST_SCRIPT) as f:
pipes_code = pts.adapt_script(f.read())
runner.set_input(opt.input, put=True)
runner.set_exe(pipes_code)
runner.run(properties=options, hadoop_conf_dir=HADOOP_CONF_DIR)
return runner.output
def main(argv):
parser = make_parser()
args = parser.parse_args(argv)
update_conf(args)
logger = logging.getLogger("main")
logger.setLevel(logging.INFO)
runner = hadut.PipesRunner(prefix=PREFIX, logger=logger)
with open(args.pipes_exe) as f:
pipes_code = pts.add_sys_path(f.read())
runner.set_input(args.local_input, put=True)
runner.set_exe(pipes_code)
runner.run(properties=CONF, hadoop_conf_dir=HADOOP_CONF_DIR, logger=logger)
res = runner.collect_output()
runner.clean()
local_wc = pts.LocalWordCount(args.local_input)
logging.info(local_wc.check(res))
def main(argv):
logger = logging.getLogger("main")
logger.setLevel(logging.INFO)
local_input = argv[1]
with open(MR_SCRIPT) as f:
pipes_code = pts.add_sys_path(f.read())
runner = hadut.PipesRunner(prefix=PREFIX, logger=logger)
runner.set_input(local_input, put=True)
runner.set_exe(pipes_code)
runner.run()
res = runner.collect_output()
runner.clean()
hdfs.rmr(HDFS_WD)
logger.info("checking results")
expected_res = local_vc(local_input)
logger.info(check(res, expected_res))
def main(argv):
parser = make_parser()
args = parser.parse_args(argv)
update_conf(args)
logger = logging.getLogger("main")
logger.setLevel(logging.INFO)
runner = hadut.PipesRunner(prefix=PREFIX, logger=logger)
with open(args.pipes_exe) as f:
pipes_code = pts.adapt_script(f.read())
runner.set_input(args.local_input, put=True)
runner.set_exe(pipes_code)
runner.run(properties=CONF, hadoop_conf_dir=HADOOP_CONF_DIR, logger=logger)
res = runner.collect_output()
if not os.getenv("DEBUG"):
runner.clean()
with open("results/result_nsf.txt", "w") as f_out:
f_out.write(res)
def main(argv):
logger = logging.getLogger("main")
logger.setLevel(logging.DEBUG)
with Timer() as total_time:
parser = make_parser()
args = parser.parse_args(argv)
if args.dataset:
print args.dataset
create_dataset(logger, args.dataset)
if args.script:
piped_code_file = args.script
else:
piped_code_file = DEFAULT_SCRIPT
if not os.path.exists(piped_code_file):
raise IOError("script {0} not found !!!".format(piped_code_file))
with open(piped_code_file) as f:
pipes_code = pts.add_sys_path(f.read())
dataset = [d for d in os.listdir("dataset") if d.endswith("MB")]
dataset.sort(cmp=lambda x, y: cmp(int(x.replace("MB", "")),
int(y.replace("MB", ""))))
logger.info(" Uploading dataset: { %s }", ', '.join(dataset))
if not hadut.path_exists(os.path.join(DATASET_DIR)):
logger.info(" dataset folder created")
hdfs.mkdir(DATASET_DIR)
for data_filename in dataset:
source_path = os.path.join(DATASET_DIR, data_filename)
dest_path = os.path.join(DATASET_DIR, data_filename)
if not hadut.path_exists(os.path.join(DATASET_DIR, data_filename)):
logger.info(" -> uploading %s...", source_path)
hdfs.put(source_path, dest_path)
update_conf(args)
results = dict()
for data_input in dataset:
with Timer() as t:
runner = hadut.PipesRunner(prefix=PREFIX, logger=logger)
logger.info("Running the script %s with data input %s..",
piped_code_file, data_input)
data_input_path = os.path.join(DATASET_DIR, data_input)
runner.set_input(data_input_path, put=False)
runner.set_exe(pipes_code)
runner.run(properties=CONF,
hadoop_conf_dir=HADOOP_CONF_DIR,
logger=logger)
res = runner.collect_output()
print data_input_path
local_wc = pts.LocalWordCount(data_input_path)
logging.info(local_wc.check(res))
# print res
# runner.clean()
results[data_input] = (t.secs, t.msecs)
print "\n\n RESULTs"
print "=" * (len(piped_code_file) + 15)
print " * script: {0}".format(piped_code_file)
print " * mappers: {0}".format(CONF["mapred.map.tasks"])
print " * reducers: {0}".format(CONF["mapred.reduce.tasks"])
print " * dataset: [{0}]".format(",".join(dataset))
print " * times (input -> secs):"
for data_input in dataset:
print " - {0} -> {1} secs.".format(data_input,
results[data_input][0])
print "\n => Total execution time: {0}".format(total_time.secs)
print "=" * (len(piped_code_file) + 15)
print "\n"
