def test_merge_files(self):
file_to_read1 = "FMI.txt"
file_read1 = open(file_to_read1, "r")
content1 = file_read1.read()
file_read1.seek(0)
file_to_write = "merge.txt"
file_write = open(file_to_write, "w")
file_to_read2 = "Hack Bulgaria.txt"
file_read2 = open(file_to_read2, "r")
content2 = file_read2.read()
file_read2.seek(0)
data = content1 + content2
file_write.write(data)
print(merge_files("FMI.txt", "Hack Bulgaria.txt", file_to_write))
self.assertEqual(file_read1.read() + file_read2.read(), merge_files("FMI.txt", "Hack Bulgaria.txt", file_to_write))
file_read1.close()
file_read2.close()
file_write.close()
def cloudburst(
input_directory,
output_directory,
hdfs,
min_read_len,
max_read_len,
max_align_diff,
redundancy,
allow_differences,
block_size,
**kwargs
):
if output_directory is None:
output_directory = utils.sibling_directory(input_directory, "%(dirname)s_cloudburst_aligned")
intermediate_directory = utils.sibling_directory(input_directory, "%(dirname)s_cloudburst_unmerged")
(input_url, intermediate_url) = utils.generate_urls(input_directory, intermediate_directory, hdfs)
cloudburst_config = utils.mapreduce_job(
input_dir=input_url,
output_dir=intermediate_url,
map_function="CloudBurstMapFunction",
reduce_function="CloudBurstReduceFunction",
partition_function="CloudBurstPartitionFunction",
)
seed_len = min_read_len / (max_align_diff + 1)
flank_len = max_read_len - seed_len + max_align_diff
cloudburst_params = {
"CLOUDBURST_MIN_READ_LEN": min_read_len,
"CLOUDBURST_MAX_READ_LEN": max_read_len,
"CLOUDBURST_MAX_ALIGN_DIFF": max_align_diff,
"CLOUDBURST_SEED_LEN": seed_len,
"CLOUDBURST_FLANK_LEN": flank_len,
"CLOUDBURST_REDUNDANCY": redundancy,
"CLOUDBURST_ALLOW_DIFFERENCES": int(allow_differences),
"CLOUDBURST_BLOCK_SIZE": block_size,
}
if "params" not in cloudburst_config:
cloudburst_config["params"] = {}
for key, value in cloudburst_params.items():
cloudburst_config["params"][key] = value
mergefiles_config = merge_files(intermediate_directory, output_directory, hdfs)
return utils.run_in_sequence(cloudburst_config, mergefiles_config)
def rdrand(username, hdfs, input_directory, output_directory, **kwargs):
if output_directory is None:
output_directory = "%s/outputs" % (username)
rdrand_output_directory = "%s/rdrand" % (output_directory)
merged_output_directory = "%s/final_output" % (output_directory)
(input_url,
rdrand_output_url) = utils.generate_urls(input_directory,
rdrand_output_directory, hdfs)
merged_output_url = utils.generate_url(merged_output_directory, hdfs)
rdrand_config = utils.mapreduce_job(
input_dir=input_url,
output_dir=rdrand_output_url,
map_function="PassThroughMapFunction",
reduce_function="CountDuplicateKeysReduceFunction",
partition_function="UniformPartitionFunction")
rdrand_params = {
"SKIP_PHASE_ZERO": 1, # Don't sample...
"INTERMEDIATE_TO_INPUT_RATIO": 3.0, #... instead assume ratio of 3
"MAP_INPUT_FORMAT_READER": "RdRandFormatReader", # 64-bit fragments
"REDUCE_INPUT_FORMAT_READER":
"FixedSizeKVPairFormatReader", # no header
"REDUCE_INPUT_FIXED_KEY_LENGTH": 16, # 128-bit intermediate keys...
"REDUCE_INPUT_FIXED_VALUE_LENGTH": 0, # ... with empty values
"WRITE_WITHOUT_HEADERS.phase_one": 1 # no headers
}
if "params" not in rdrand_config:
rdrand_config["params"] = {}
for key, value in rdrand_params.items():
rdrand_config["params"][key] = value
# Run a second job to merge all duplicate key information into a single
# output file for better readability.
mergefiles_config = merge_files(rdrand_output_directory,
merged_output_directory, hdfs)
return utils.run_in_sequence(rdrand_config, mergefiles_config)
def rdrand(username, input_directory, output_directory, **kwargs):
if output_directory is None:
output_directory = "%s/outputs" % (username)
rdrand_output_directory = "%s/rdrand" % (output_directory)
merged_output_directory = "%s/final_output" % (output_directory)
(input_url, rdrand_output_url) = utils.generate_urls(
input_directory, rdrand_output_directory)
merged_output_url = utils.generate_url(merged_output_directory)
rdrand_config = utils.mapreduce_job(
input_dir = input_url,
output_dir = rdrand_output_url,
map_function = "PassThroughMapFunction",
reduce_function = "CountDuplicateKeysReduceFunction",
partition_function = "UniformPartitionFunction")
rdrand_params = {
"SKIP_PHASE_ZERO": 1, # Don't sample...
"INTERMEDIATE_TO_INPUT_RATIO": 3.0, #... instead assume ratio of 3
"MAP_INPUT_FORMAT_READER" : "RdRandFormatReader", # 64-bit fragments
"REDUCE_INPUT_FORMAT_READER": "FixedSizeKVPairFormatReader", # no header
"REDUCE_INPUT_FIXED_KEY_LENGTH": 16, # 128-bit intermediate keys...
"REDUCE_INPUT_FIXED_VALUE_LENGTH": 0, # ... with empty values
"WRITE_WITHOUT_HEADERS.phase_one": 1 # no headers
}
if "params" not in rdrand_config:
rdrand_config["params"] = {}
for key, value in rdrand_params.items():
rdrand_config["params"][key] = value
# Run a second job to merge all duplicate key information into a single
# output file for better readability.
mergefiles_config = merge_files(
rdrand_output_directory, merged_output_directory)
return utils.run_in_sequence(rdrand_config, mergefiles_config)
def tuple_length_count(input_directory, output_directory, **kwargs):
if output_directory is None:
output_directory = utils.sibling_directory(
input_directory, "tuple_length_counter_%(dirname)s")
intermediate_directory = utils.sibling_directory(
input_directory, "unmerged_counts_%(dirname)s")
(input_url, output_url) = utils.generate_urls(
input_directory, intermediate_directory)
tuple_length_config = utils.mapreduce_job(
input_dir = input_url,
output_dir = output_url,
map_function = "TupleLengthCounterMapFunction",
reduce_function = "SumValuesReduceFunction")
merge_files_config = merge_files.merge_files(
intermediate_directory, output_directory)
config = utils.run_in_sequence(tuple_length_config, merge_files_config)
return config
path_name = args.in_path
loss_weights = [args.weight_class, args.weight_regress]
# if True it retrains, wlse it reads a model
retrain = True # train the NN (~20 min), otherwise it needs a model from disc.
# name of the model for saving or loading
model_files = 'selu_mode'
# make the output directories
os.mkdir(os.getcwd() + out_path)
out_path = os.getcwd() + out_path
###########################################
# get the data !!! ########################
(X_train, is_MC_train, ref_train, bmass_train, Y_train) = merge_files(
path_name, 'train', n_vertex=n_vertex)
(X_test, is_MC, ref_test, bmass_test, Y_test) = merge_files(
path_name, 'test', n_vertex=n_vertex)
# though shall preprocess (var = 1, mean = 0) for gradiend based learning!
scaler = preprocessing.StandardScaler().fit(X_train)
X_scaled_train = scaler.transform(X_train)
X_scaled_test = scaler.transform(X_test)
############################################
# the below weights make sure that MC is not used for the second loss,
# which is the mass regression
background = 1 - is_MC_train
all_events = np.ones(background.shape)
