def pagerank(input_directory, output_directory, iterations, **kwargs):
if output_directory is None:
output_directory = utils.sibling_directory(
input_directory, "%(dirname)s_with_pageranks")
config_list = []
current_input = input_directory
current_output = None
for iteration in xrange(iterations):
if iteration == iterations - 1:
current_output = output_directory
else:
current_output = utils.sibling_directory(
input_directory,
"%(dirname)s_pagerank_iteration_" + str(iteration + 1))
(input_url, output_url) = utils.generate_urls(
current_input, current_output)
config = utils.mapreduce_job(
input_dir = input_url,
output_dir = output_url,
map_function = "PageRankMapFunction",
reduce_function = "PageRankReduceFunction"
)
config_list.append(config)
current_input = current_output
pagerank_config = utils.run_in_sequence(*config_list)
return pagerank_config
def pagerank(input_directory, output_directory, iterations, hdfs, **kwargs):
if output_directory is None:
output_directory = utils.sibling_directory(
input_directory, "%(dirname)s_with_pageranks")
config_list = []
current_input = input_directory
current_output = None
for iteration in xrange(iterations):
if iteration == iterations - 1:
current_output = output_directory
else:
current_output = utils.sibling_directory(
input_directory,
"%(dirname)s_pagerank_iteration_" + str(iteration + 1))
(input_url, output_url) = utils.generate_urls(current_input,
current_output, hdfs)
config = utils.mapreduce_job(input_dir=input_url,
output_dir=output_url,
map_function="PageRankMapFunction",
reduce_function="PageRankReduceFunction")
config_list.append(config)
current_input = current_output
pagerank_config = utils.run_in_sequence(*config_list)
return pagerank_config
def cloudburst(
input_directory,
output_directory,
hdfs,
min_read_len,
max_read_len,
max_align_diff,
redundancy,
allow_differences,
block_size,
**kwargs
):
if output_directory is None:
output_directory = utils.sibling_directory(input_directory, "%(dirname)s_cloudburst_aligned")
intermediate_directory = utils.sibling_directory(input_directory, "%(dirname)s_cloudburst_unmerged")
(input_url, intermediate_url) = utils.generate_urls(input_directory, intermediate_directory, hdfs)
cloudburst_config = utils.mapreduce_job(
input_dir=input_url,
output_dir=intermediate_url,
map_function="CloudBurstMapFunction",
reduce_function="CloudBurstReduceFunction",
partition_function="CloudBurstPartitionFunction",
)
seed_len = min_read_len / (max_align_diff + 1)
flank_len = max_read_len - seed_len + max_align_diff
cloudburst_params = {
"CLOUDBURST_MIN_READ_LEN": min_read_len,
"CLOUDBURST_MAX_READ_LEN": max_read_len,
"CLOUDBURST_MAX_ALIGN_DIFF": max_align_diff,
"CLOUDBURST_SEED_LEN": seed_len,
"CLOUDBURST_FLANK_LEN": flank_len,
"CLOUDBURST_REDUNDANCY": redundancy,
"CLOUDBURST_ALLOW_DIFFERENCES": int(allow_differences),
"CLOUDBURST_BLOCK_SIZE": block_size,
}
if "params" not in cloudburst_config:
cloudburst_config["params"] = {}
for key, value in cloudburst_params.items():
cloudburst_config["params"][key] = value
mergefiles_config = merge_files(intermediate_directory, output_directory, hdfs)
return utils.run_in_sequence(cloudburst_config, mergefiles_config)
def rdrand(username, hdfs, input_directory, output_directory, **kwargs):
if output_directory is None:
output_directory = "%s/outputs" % (username)
rdrand_output_directory = "%s/rdrand" % (output_directory)
merged_output_directory = "%s/final_output" % (output_directory)
(input_url,
rdrand_output_url) = utils.generate_urls(input_directory,
rdrand_output_directory, hdfs)
merged_output_url = utils.generate_url(merged_output_directory, hdfs)
rdrand_config = utils.mapreduce_job(
input_dir=input_url,
output_dir=rdrand_output_url,
map_function="PassThroughMapFunction",
reduce_function="CountDuplicateKeysReduceFunction",
partition_function="UniformPartitionFunction")
rdrand_params = {
"SKIP_PHASE_ZERO": 1, # Don't sample...
"INTERMEDIATE_TO_INPUT_RATIO": 3.0, #... instead assume ratio of 3
"MAP_INPUT_FORMAT_READER": "RdRandFormatReader", # 64-bit fragments
"REDUCE_INPUT_FORMAT_READER":
"FixedSizeKVPairFormatReader", # no header
"REDUCE_INPUT_FIXED_KEY_LENGTH": 16, # 128-bit intermediate keys...
"REDUCE_INPUT_FIXED_VALUE_LENGTH": 0, # ... with empty values
"WRITE_WITHOUT_HEADERS.phase_one": 1 # no headers
}
if "params" not in rdrand_config:
rdrand_config["params"] = {}
for key, value in rdrand_params.items():
rdrand_config["params"][key] = value
# Run a second job to merge all duplicate key information into a single
# output file for better readability.
mergefiles_config = merge_files(rdrand_output_directory,
merged_output_directory, hdfs)
return utils.run_in_sequence(rdrand_config, mergefiles_config)
def rdrand(username, input_directory, output_directory, **kwargs):
if output_directory is None:
output_directory = "%s/outputs" % (username)
rdrand_output_directory = "%s/rdrand" % (output_directory)
merged_output_directory = "%s/final_output" % (output_directory)
(input_url, rdrand_output_url) = utils.generate_urls(
input_directory, rdrand_output_directory)
merged_output_url = utils.generate_url(merged_output_directory)
rdrand_config = utils.mapreduce_job(
input_dir = input_url,
output_dir = rdrand_output_url,
map_function = "PassThroughMapFunction",
reduce_function = "CountDuplicateKeysReduceFunction",
partition_function = "UniformPartitionFunction")
rdrand_params = {
"SKIP_PHASE_ZERO": 1, # Don't sample...
"INTERMEDIATE_TO_INPUT_RATIO": 3.0, #... instead assume ratio of 3
"MAP_INPUT_FORMAT_READER" : "RdRandFormatReader", # 64-bit fragments
"REDUCE_INPUT_FORMAT_READER": "FixedSizeKVPairFormatReader", # no header
"REDUCE_INPUT_FIXED_KEY_LENGTH": 16, # 128-bit intermediate keys...
"REDUCE_INPUT_FIXED_VALUE_LENGTH": 0, # ... with empty values
"WRITE_WITHOUT_HEADERS.phase_one": 1 # no headers
}
if "params" not in rdrand_config:
rdrand_config["params"] = {}
for key, value in rdrand_params.items():
rdrand_config["params"][key] = value
# Run a second job to merge all duplicate key information into a single
# output file for better readability.
mergefiles_config = merge_files(
rdrand_output_directory, merged_output_directory)
return utils.run_in_sequence(rdrand_config, mergefiles_config)
def tuple_length_count(input_directory, output_directory, **kwargs):
if output_directory is None:
output_directory = utils.sibling_directory(
input_directory, "tuple_length_counter_%(dirname)s")
intermediate_directory = utils.sibling_directory(
input_directory, "unmerged_counts_%(dirname)s")
(input_url, output_url) = utils.generate_urls(
input_directory, intermediate_directory)
tuple_length_config = utils.mapreduce_job(
input_dir = input_url,
output_dir = output_url,
map_function = "TupleLengthCounterMapFunction",
reduce_function = "SumValuesReduceFunction")
merge_files_config = merge_files.merge_files(
intermediate_directory, output_directory)
config = utils.run_in_sequence(tuple_length_config, merge_files_config)
return config
