def do_get_avg_detection_time_job(avg_penalty_list, sub_pipe):
"""
:param avg_penalty_list: [... avg_penalty_id ...],
:param sub_pipe:
:return:
"""
"""
avg_penalty_pattern:
[model_type_str, trunc_num_step_str, penalty_str]
monitor pattern:
************** ****************
[graph_name_str, monitor_info_str, monitor_str]
"""
models = get_instances(avg_penalty_pattern, model_type_str,
avg_penalty_list, sep)
trunc_num_step_list = get_instances(avg_penalty_pattern,
trunc_num_step_str, avg_penalty_list,
sep)
msg_list = [None] * len(avg_penalty_list)
for mdl_idx in range(0, len(models)):
model = models[mdl_idx]
avg_penalty_id = avg_penalty_list[mdl_idx]
"""
Make sure the existence of related graph_title files.
"""
graph_names = [
unit_sep.join([model, str(iid)]) for iid in instance_num_range
]
graph_ids = get_graph_instances(graph_dir, graph_name_str, graph_names,
graph_name_str, sep)
graph_paths = [
path.join(graph_dir, graph_id) for graph_id in graph_ids
]
chk_res = [path.exists(graph_path) for graph_path in graph_paths]
if not all(chk_res):
msg_list[mdl_idx] = 'Missing required graph_title files.' \
'Cannot finish {0:s}'.format(avg_penalty_id)
print msg_list[mdl_idx]
continue
"""
Make sure the existence of related monitor files.
"""
monitor_info_list = [
monitor_info_fmt.format(trunc_num_step_list[mdl_idx], method)
for method in methods
]
monitor_ids = [
sep.join(item)
for item in product(graph_names, monitor_info_list, [monitor_str])
]
monitor_paths = [
path.join(monitor_dir, monitor_id) for monitor_id in monitor_ids
]
chk_res = [path.exists(monitor_path) for monitor_path in monitor_paths]
if not all(chk_res):
msg_list[mdl_idx] = 'Missing required monitor files.' \
'Cannot finish {0:s}'.format(avg_penalty_id)
print msg_list[mdl_idx]
continue
"""
Now, all related files exist.
"""
avg_penalty_path = path.join(penalty_dir, avg_penalty_id)
avg_res = [0] * (len(methods) * trunc_point_num)
for j in range(0, len(graph_paths)):
adj_table = get_adjtable(graph_paths[j])
"""
monitors = [
...
...
[step * 1] ... [step * trunc_point_num] ---- method_k
...
...
]
"""
monitors = []
for monitor_path in monitor_paths[j * len(methods):(j + 1) *
len(methods)]:
monitors.extend(pickle_from_disk(monitor_path))
"""
avg_res = [
...
res_1, ..., res_(trunc point num) ---- method_k
...
]
"""
avg_res = map(add, avg_res, get_penalty(adj_table, monitors))
avg_res = [
round(float(item) / len(graph_paths), 2) for item in avg_res
]
"""
Construct message.
avg_penalty_pattern: [model_type_str, trunc_num_step_str, penalty_str]
"""
msg_parts = avg_penalty_id.split(sep)
msg_content = '{0:s}:{1:s}:{2:s}\n'.format(
msg_parts[0],
msg_parts[1],
msg_parts[2],
)
base_method = methods[0]
base_line = avg_res[trunc_point_num - 1]
msg_content += '{0:s}:{1:.2f}\n'.format(base_method, base_line)
for mtd_idx in range(1, len(methods)):
tmp = (mtd_idx + 1) * trunc_point_num - 1
msg_content += '{0:s}:{1:.2f} ({2:.2f}%)\n'.format(
methods[mtd_idx], avg_res[tmp],
((avg_res[tmp] - base_line) * 100.0) / base_line)
msg_list[mdl_idx] = msg_content
print msg_content
if trunc_point_num > 1:
"""
avg_res = [
...
avg_res_1, ..., avg_res_(trunc point num) ---- method_k
...
]
"""
pickle_to_disk(avg_penalty_path, avg_res)
elif trunc_point_num == 1:
"""
Baseline and stretch ratio.
"""
write_to_disk(avg_penalty_path, msg_content)
if sub_pipe:
sub_pipe.send(msg_list)
def do_get_avg_detection_time_job(avg_detec_time_list, sub_pipe):
"""
:param avg_detec_time_list: [... avg_detec_time_id ...],
:param sub_pipe:
:return:
"""
"""
avg_detection_time_pattern:
[graph_name_str,
trunc_num_step_str,
infection_info_str,
snapshot_num_str,
avg_detection_time_str]
monitor_pattern:
****************
[graph_name_str, monitor_info_str, monitor_str]
"""
graph_id_list = get_graph_instances(graph_dir, graph_name_str,
avg_detec_time_list,
avg_detection_time_pattern, sep)
trunc_num_step_list = get_instances(avg_detection_time_pattern,
trunc_num_step_str,
avg_detec_time_list, sep)
monitor_list = get_instances(avg_detection_time_pattern, monitor_pattern,
avg_detec_time_list, sep)
msg_list = [None] * len(avg_detec_time_list)
for graph_id_idx in range(0, len(graph_id_list)):
graph_id = graph_id_list[graph_id_idx]
trunc_num_step = trunc_num_step_list[graph_id_idx]
monitor_id = monitor_list[graph_id_idx]
avg_detec_time_id = avg_detec_time_list[graph_id_idx]
"""
Make sure the existence of graph_title file.
"""
graph_path = path.join(graph_dir, graph_id)
if not path.exists(graph_path):
msg_list[graph_id_idx] = 'Missing {0:s}. Cannot finish {1:s}'. \
format(graph_path, avg_detec_time_id)
print msg_list[graph_id_idx]
continue
"""
Make sure the existence of related monitor file.
"""
monitor_info_list = [
monitor_info_fmt.format(trunc_num_step, method)
for method in methods
]
monitor_id_list = [
monitor_id.replace(monitor_info_str, monitor_info)
for monitor_info in monitor_info_list
]
monitor_paths = [
path.join(monitor_dir, monitor_id)
for monitor_id in monitor_id_list
]
chk_res = [path.exists(monitor_path) for monitor_path in monitor_paths]
if not all(chk_res):
msg_list[graph_id_idx] = 'Missing monitor. Cannot finish {0:s}'. \
format(avg_detec_time_id)
print msg_list[graph_id_idx]
continue
"""
Now, all related files exist.
"""
avg_detec_time_path = path.join(avg_detection_time_dir,
avg_detec_time_id)
adj_table = get_adjtable(path.join(graph_dir, graph_id))
"""
monitors = [
...
...
[step * 1] ... [step * trunc_point_num] ---- method_k
...
...
]
"""
monitors = []
for monitor_path in monitor_paths:
monitors.extend(pickle_from_disk(monitor_path))
"""
res = [
...
res_1, ..., res_(trunc point num) ---- method_k
...
]
"""
res = [
round(float(item), 2) for item in get_avg_detection_time(
adj_table, monitors, snapshot_num)
]
"""
Construct message.
avg_detection_time_pattern:
[graph_name_str, trunc_num_step_str, infection_info_str, snapshot_num_str, avg_detection_time_str]
"""
msg_parts = avg_detec_time_id.split(sep)
msg_content = '{0:s}:{1:s}:{2:s}:{3:s}\n'.format(
msg_parts[0], msg_parts[1], msg_parts[2], msg_parts[3])
base_method = methods[0]
base_line = res[trunc_point_num - 1]
msg_content += '{0:s}:{1:.2f}\n'.format(base_method, base_line)
for mtd_idx in range(1, len(methods)):
tmp = (mtd_idx + 1) * trunc_point_num - 1
msg_content += '{0:s}:{1:.2f} ({2:.2f}%)\n'.format(
methods[mtd_idx], res[tmp],
((res[tmp] - base_line) * 100.0) / base_line)
msg_list[graph_id_idx] = msg_content
print msg_content
if trunc_point_num > 1:
"""
res = [
...
res_1, ..., res_(trunc point num) ---- method_k
...
]
"""
pickle_to_disk(avg_detec_time_path, res)
elif trunc_point_num == 1:
"""
Baseline and stretch ratio.
"""
write_to_disk(avg_detec_time_path, msg_content)
if sub_pipe:
sub_pipe.send(msg_list)
def do_get_penalty_job(penalty_list, sub_pipe=None):
"""
:param penalty_list: [... penalty_k ...],
:param sub_pipe: Corresponding to a master pipe owned by the master process,
if sub_pipe is not empty, we put some information into it and send to the master pipe.
:return: write [..., penalty val_(i*scale) , ...] to penalty_path
penalty val_(i*scale) : the penalty value of i*scale monitor
"""
"""
penalty pattern:
[graph_name_str, trunc_num_step_str, penalty_str]
monitor pattern:
****************
[graph_name_str, monitor_info_str, monitor_str]
"""
trunc_num_step_list = get_instances(penalty_pattern, trunc_num_step_str,
penalty_list, sep)
monitor_list = get_instances(penalty_pattern, monitor_pattern,
penalty_list, sep)
graph_id_list = get_graph_instances(graph_dir, graph_name_str,
penalty_list, penalty_pattern, sep)
msg_list = [None] * len(penalty_list)
for graph_id_idx in range(0, len(graph_id_list)):
graph_id = graph_id_list[graph_id_idx]
trunc_num_step = trunc_num_step_list[graph_id_idx]
monitor_id = monitor_list[graph_id_idx]
penalty_id = penalty_list[graph_id_idx]
"""
Make sure the existence of graph_title file.
"""
graph_path = path.join(graph_dir, graph_id)
if not path.exists(graph_path):
msg_list[graph_id_idx] = 'Missing {0:s}. Cannot finish {1:s}'.\
format(graph_path, penalty_id)
print msg_list[graph_id_idx]
continue
"""
Make sure the existence of related monitor file.
"""
monitor_info_list = [
monitor_info_fmt.format(trunc_num_step, method)
for method in methods
]
monitor_id_list = [
monitor_id.replace(monitor_info_str, monitor_info)
for monitor_info in monitor_info_list
]
monitor_paths = [
path.join(monitor_dir, monitor_id)
for monitor_id in monitor_id_list
]
chk_res = [path.exists(monitor_path) for monitor_path in monitor_paths]
if not all(chk_res):
msg_list[graph_id_idx] = 'Missing monitor. Cannot finish {0:s}'. \
format(penalty_id)
print msg_list[graph_id_idx]
continue
"""
Now, all related files exist.
"""
penalty_path = path.join(penalty_dir, penalty_id)
adj_table = get_adjtable(graph_path)
"""
monitors = [
...
...
[step * 1] ... [step * trunc_point_num] ---- method_k
...
...
]
"""
monitors = []
for monitor_path in monitor_paths:
monitors.extend(pickle_from_disk(monitor_path))
"""
res = [
...
res_1, ..., res_(trunc point num) ---- method_k
...
]
"""
res = [
round(float(item), 2) for item in get_penalty(adj_table, monitors)
]
"""
Construct message.
penalty pattern:
[graph_name_str, trunc_num_step_str, penalty_str]
"""
msg_parts = penalty_id.split(sep)
msg_content = '{0:s}:{1:s}:{2:s}\n'.format(
msg_parts[0],
msg_parts[1],
msg_parts[2],
)
base_method = methods[0]
base_line = res[trunc_point_num - 1]
msg_content += '{0:s}:{1:.2f}\n'.format(base_method, base_line)
for mtd_idx in range(1, len(methods)):
tmp = (mtd_idx + 1) * trunc_point_num - 1
msg_content += '{0:s}:{1:.2f} ({2:.2f}%)\n'.format(
methods[mtd_idx], res[tmp],
((res[tmp] - base_line) * 100.0) / base_line)
msg_list[graph_id_idx] = msg_content
print msg_content
if trunc_point_num > 1:
"""
res = [
...
res_1, ..., res_(trunc point num) ---- method_k
...
]
"""
pickle_to_disk(penalty_path, res)
elif trunc_point_num == 1:
"""
Baseline and stretch ratio.
"""
write_to_disk(penalty_path, msg_content)
if sub_pipe:
sub_pipe.send(msg_list)