def do_get_avg_detection_time_job(avg_penalty_list, sub_pipe): """ :param avg_penalty_list: [... avg_penalty_id ...], :param sub_pipe: :return: """ """ avg_penalty_pattern: [model_type_str, trunc_num_step_str, penalty_str] monitor pattern: ************** **************** [graph_name_str, monitor_info_str, monitor_str] """ models = get_instances(avg_penalty_pattern, model_type_str, avg_penalty_list, sep) trunc_num_step_list = get_instances(avg_penalty_pattern, trunc_num_step_str, avg_penalty_list, sep) msg_list = [None] * len(avg_penalty_list) for mdl_idx in range(0, len(models)): model = models[mdl_idx] avg_penalty_id = avg_penalty_list[mdl_idx] """ Make sure the existence of related graph_title files. """ graph_names = [ unit_sep.join([model, str(iid)]) for iid in instance_num_range ] graph_ids = get_graph_instances(graph_dir, graph_name_str, graph_names, graph_name_str, sep) graph_paths = [ path.join(graph_dir, graph_id) for graph_id in graph_ids ] chk_res = [path.exists(graph_path) for graph_path in graph_paths] if not all(chk_res): msg_list[mdl_idx] = 'Missing required graph_title files.' \ 'Cannot finish {0:s}'.format(avg_penalty_id) print msg_list[mdl_idx] continue """ Make sure the existence of related monitor files. """ monitor_info_list = [ monitor_info_fmt.format(trunc_num_step_list[mdl_idx], method) for method in methods ] monitor_ids = [ sep.join(item) for item in product(graph_names, monitor_info_list, [monitor_str]) ] monitor_paths = [ path.join(monitor_dir, monitor_id) for monitor_id in monitor_ids ] chk_res = [path.exists(monitor_path) for monitor_path in monitor_paths] if not all(chk_res): msg_list[mdl_idx] = 'Missing required monitor files.' \ 'Cannot finish {0:s}'.format(avg_penalty_id) print msg_list[mdl_idx] continue """ Now, all related files exist. """ avg_penalty_path = path.join(penalty_dir, avg_penalty_id) avg_res = [0] * (len(methods) * trunc_point_num) for j in range(0, len(graph_paths)): adj_table = get_adjtable(graph_paths[j]) """ monitors = [ ... ... [step * 1] ... [step * trunc_point_num] ---- method_k ... ... ] """ monitors = [] for monitor_path in monitor_paths[j * len(methods):(j + 1) * len(methods)]: monitors.extend(pickle_from_disk(monitor_path)) """ avg_res = [ ... res_1, ..., res_(trunc point num) ---- method_k ... ] """ avg_res = map(add, avg_res, get_penalty(adj_table, monitors)) avg_res = [ round(float(item) / len(graph_paths), 2) for item in avg_res ] """ Construct message. avg_penalty_pattern: [model_type_str, trunc_num_step_str, penalty_str] """ msg_parts = avg_penalty_id.split(sep) msg_content = '{0:s}:{1:s}:{2:s}\n'.format( msg_parts[0], msg_parts[1], msg_parts[2], ) base_method = methods[0] base_line = avg_res[trunc_point_num - 1] msg_content += '{0:s}:{1:.2f}\n'.format(base_method, base_line) for mtd_idx in range(1, len(methods)): tmp = (mtd_idx + 1) * trunc_point_num - 1 msg_content += '{0:s}:{1:.2f} ({2:.2f}%)\n'.format( methods[mtd_idx], avg_res[tmp], ((avg_res[tmp] - base_line) * 100.0) / base_line) msg_list[mdl_idx] = msg_content print msg_content if trunc_point_num > 1: """ avg_res = [ ... avg_res_1, ..., avg_res_(trunc point num) ---- method_k ... ] """ pickle_to_disk(avg_penalty_path, avg_res) elif trunc_point_num == 1: """ Baseline and stretch ratio. """ write_to_disk(avg_penalty_path, msg_content) if sub_pipe: sub_pipe.send(msg_list)
def do_get_avg_detection_time_job(avg_detec_time_list, sub_pipe): """ :param avg_detec_time_list: [... avg_detec_time_id ...], :param sub_pipe: :return: """ """ avg_detection_time_pattern: [graph_name_str, trunc_num_step_str, infection_info_str, snapshot_num_str, avg_detection_time_str] monitor_pattern: **************** [graph_name_str, monitor_info_str, monitor_str] """ graph_id_list = get_graph_instances(graph_dir, graph_name_str, avg_detec_time_list, avg_detection_time_pattern, sep) trunc_num_step_list = get_instances(avg_detection_time_pattern, trunc_num_step_str, avg_detec_time_list, sep) monitor_list = get_instances(avg_detection_time_pattern, monitor_pattern, avg_detec_time_list, sep) msg_list = [None] * len(avg_detec_time_list) for graph_id_idx in range(0, len(graph_id_list)): graph_id = graph_id_list[graph_id_idx] trunc_num_step = trunc_num_step_list[graph_id_idx] monitor_id = monitor_list[graph_id_idx] avg_detec_time_id = avg_detec_time_list[graph_id_idx] """ Make sure the existence of graph_title file. """ graph_path = path.join(graph_dir, graph_id) if not path.exists(graph_path): msg_list[graph_id_idx] = 'Missing {0:s}. Cannot finish {1:s}'. \ format(graph_path, avg_detec_time_id) print msg_list[graph_id_idx] continue """ Make sure the existence of related monitor file. """ monitor_info_list = [ monitor_info_fmt.format(trunc_num_step, method) for method in methods ] monitor_id_list = [ monitor_id.replace(monitor_info_str, monitor_info) for monitor_info in monitor_info_list ] monitor_paths = [ path.join(monitor_dir, monitor_id) for monitor_id in monitor_id_list ] chk_res = [path.exists(monitor_path) for monitor_path in monitor_paths] if not all(chk_res): msg_list[graph_id_idx] = 'Missing monitor. Cannot finish {0:s}'. \ format(avg_detec_time_id) print msg_list[graph_id_idx] continue """ Now, all related files exist. """ avg_detec_time_path = path.join(avg_detection_time_dir, avg_detec_time_id) adj_table = get_adjtable(path.join(graph_dir, graph_id)) """ monitors = [ ... ... [step * 1] ... [step * trunc_point_num] ---- method_k ... ... ] """ monitors = [] for monitor_path in monitor_paths: monitors.extend(pickle_from_disk(monitor_path)) """ res = [ ... res_1, ..., res_(trunc point num) ---- method_k ... ] """ res = [ round(float(item), 2) for item in get_avg_detection_time( adj_table, monitors, snapshot_num) ] """ Construct message. avg_detection_time_pattern: [graph_name_str, trunc_num_step_str, infection_info_str, snapshot_num_str, avg_detection_time_str] """ msg_parts = avg_detec_time_id.split(sep) msg_content = '{0:s}:{1:s}:{2:s}:{3:s}\n'.format( msg_parts[0], msg_parts[1], msg_parts[2], msg_parts[3]) base_method = methods[0] base_line = res[trunc_point_num - 1] msg_content += '{0:s}:{1:.2f}\n'.format(base_method, base_line) for mtd_idx in range(1, len(methods)): tmp = (mtd_idx + 1) * trunc_point_num - 1 msg_content += '{0:s}:{1:.2f} ({2:.2f}%)\n'.format( methods[mtd_idx], res[tmp], ((res[tmp] - base_line) * 100.0) / base_line) msg_list[graph_id_idx] = msg_content print msg_content if trunc_point_num > 1: """ res = [ ... res_1, ..., res_(trunc point num) ---- method_k ... ] """ pickle_to_disk(avg_detec_time_path, res) elif trunc_point_num == 1: """ Baseline and stretch ratio. """ write_to_disk(avg_detec_time_path, msg_content) if sub_pipe: sub_pipe.send(msg_list)
def do_get_penalty_job(penalty_list, sub_pipe=None): """ :param penalty_list: [... penalty_k ...], :param sub_pipe: Corresponding to a master pipe owned by the master process, if sub_pipe is not empty, we put some information into it and send to the master pipe. :return: write [..., penalty val_(i*scale) , ...] to penalty_path penalty val_(i*scale) : the penalty value of i*scale monitor """ """ penalty pattern: [graph_name_str, trunc_num_step_str, penalty_str] monitor pattern: **************** [graph_name_str, monitor_info_str, monitor_str] """ trunc_num_step_list = get_instances(penalty_pattern, trunc_num_step_str, penalty_list, sep) monitor_list = get_instances(penalty_pattern, monitor_pattern, penalty_list, sep) graph_id_list = get_graph_instances(graph_dir, graph_name_str, penalty_list, penalty_pattern, sep) msg_list = [None] * len(penalty_list) for graph_id_idx in range(0, len(graph_id_list)): graph_id = graph_id_list[graph_id_idx] trunc_num_step = trunc_num_step_list[graph_id_idx] monitor_id = monitor_list[graph_id_idx] penalty_id = penalty_list[graph_id_idx] """ Make sure the existence of graph_title file. """ graph_path = path.join(graph_dir, graph_id) if not path.exists(graph_path): msg_list[graph_id_idx] = 'Missing {0:s}. Cannot finish {1:s}'.\ format(graph_path, penalty_id) print msg_list[graph_id_idx] continue """ Make sure the existence of related monitor file. """ monitor_info_list = [ monitor_info_fmt.format(trunc_num_step, method) for method in methods ] monitor_id_list = [ monitor_id.replace(monitor_info_str, monitor_info) for monitor_info in monitor_info_list ] monitor_paths = [ path.join(monitor_dir, monitor_id) for monitor_id in monitor_id_list ] chk_res = [path.exists(monitor_path) for monitor_path in monitor_paths] if not all(chk_res): msg_list[graph_id_idx] = 'Missing monitor. Cannot finish {0:s}'. \ format(penalty_id) print msg_list[graph_id_idx] continue """ Now, all related files exist. """ penalty_path = path.join(penalty_dir, penalty_id) adj_table = get_adjtable(graph_path) """ monitors = [ ... ... [step * 1] ... [step * trunc_point_num] ---- method_k ... ... ] """ monitors = [] for monitor_path in monitor_paths: monitors.extend(pickle_from_disk(monitor_path)) """ res = [ ... res_1, ..., res_(trunc point num) ---- method_k ... ] """ res = [ round(float(item), 2) for item in get_penalty(adj_table, monitors) ] """ Construct message. penalty pattern: [graph_name_str, trunc_num_step_str, penalty_str] """ msg_parts = penalty_id.split(sep) msg_content = '{0:s}:{1:s}:{2:s}\n'.format( msg_parts[0], msg_parts[1], msg_parts[2], ) base_method = methods[0] base_line = res[trunc_point_num - 1] msg_content += '{0:s}:{1:.2f}\n'.format(base_method, base_line) for mtd_idx in range(1, len(methods)): tmp = (mtd_idx + 1) * trunc_point_num - 1 msg_content += '{0:s}:{1:.2f} ({2:.2f}%)\n'.format( methods[mtd_idx], res[tmp], ((res[tmp] - base_line) * 100.0) / base_line) msg_list[graph_id_idx] = msg_content print msg_content if trunc_point_num > 1: """ res = [ ... res_1, ..., res_(trunc point num) ---- method_k ... ] """ pickle_to_disk(penalty_path, res) elif trunc_point_num == 1: """ Baseline and stretch ratio. """ write_to_disk(penalty_path, msg_content) if sub_pipe: sub_pipe.send(msg_list)