def main_method(args): sim_output = Path(args.sim_output) if not sim_output.exists(): get_logger().error("%s does not exist", sim_output) return 1 output = Path(args.output) / 'inferred-demand' output.mkdir(parents=True, exist_ok=True) with multiprocessing.Pool(processes=os.cpu_count()) as pool: results = list() for node_dir in sim_output.iterdir(): if not node_dir.is_dir(): continue node_name_dir = map_utils.find_ncp_folder(node_dir) if node_name_dir is None: get_logger().debug("No NCP folder found in %s", node_name_dir) continue results.append( pool.apply_async(func=process_node, args=[output, node_name_dir])) for result in results: result.wait()
def main_method(args): sim_output = Path(args.sim_output) if not sim_output.exists(): get_logger().error("%s does not exist", sim_output) return 1 output = Path(args.output) output.mkdir(parents=True, exist_ok=True) # node_name -> (xs, ys) data = dict() min_time = None for node_dir in sim_output.iterdir(): max_diff = None if not node_dir.is_dir(): continue get_logger().debug("Processing node %s", node_dir) node_name_dir = map_utils.find_ncp_folder(node_dir) if node_name_dir is None: get_logger().debug("No NCP folder found") continue xs = list() ys = list() for time_dir in node_name_dir.iterdir(): if not time_dir.is_dir(): continue directory_time = int(time_dir.stem) get_logger().debug("\t\tProcessing time %s", time_dir) resource_report_file = time_dir / 'resourceReport-SHORT.json' if resource_report_file.exists(): try: with open(resource_report_file, 'r') as f: resource_report = json.load(f) except json.decoder.JSONDecodeError: get_logger().warning("Problem reading %s, skipping", resource_report_file) report_time = int(resource_report['timestamp']) if report_time < 1: # skip reports that we don't have a rational time for continue # directory_time should be greater than or equal to report_time diff = map_utils.timestamp_to_seconds(directory_time - report_time) xs.append(directory_time) ys.append(diff) if min_time is None or directory_time < min_time: min_time = directory_time if max_diff is None or diff > max_diff: max_diff = diff data[node_dir.stem] = (xs, ys) get_logger().info("Maximum difference for %s is %f seconds", node_dir.stem, max_diff) output_graph(output, data, min_time)
def process_node(first_timestamp, all_apps, data_load, data_demand_short, data_demand_long, data_capacity, node_dir, container_node): """ Args: first_timestamp(datetime.datetime): when the simulation started """ node_all_times = set() node_containers = set() node_name_dir = map_utils.find_ncp_folder(node_dir) if node_name_dir is None: get_logger().debug("No NCP folder found in %s", node_name_dir) return ncp = map_utils.node_name_from_dir(node_name_dir) first_timestamp_ms = first_timestamp.timestamp() * 1000 get_logger().debug("Processing ncp folder %s from %s. NCP name: %s", node_name_dir, node_dir, ncp) for time_dir in node_name_dir.iterdir(): if not time_dir.is_dir(): continue time = int(time_dir.stem) if time < first_timestamp_ms: # ignore data before the start of the simulation continue node_all_times.add(time) resource_report_file = time_dir / 'resourceReport-SHORT.json' if resource_report_file.exists(): try: with open(resource_report_file, 'r') as f: resource_report = json.load(f) process_resource_report(ncp, data_load, data_demand_short, data_capacity, node_containers, time, resource_report) except json.decoder.JSONDecodeError: get_logger().warning("Problem reading %s, skipping", resource_report_file) resource_report_file = time_dir / 'resourceReport-LONG.json' if resource_report_file.exists(): try: with open(resource_report_file, 'r') as f: resource_report = json.load(f) process_resource_report(ncp, None, data_demand_long, None, node_containers, time, resource_report) except json.decoder.JSONDecodeError: get_logger().warning("Problem reading %s, skipping", resource_report_file) fill_in_missing_apps(all_apps, data_load, ncp, node_all_times) fill_in_missing_apps(all_apps, data_demand_short, ncp, node_all_times) fill_in_missing_apps(all_apps, data_demand_long, ncp, node_all_times) fill_in_missing_containers(all_apps, data_capacity, node_all_times, node_containers) for container in node_containers: container_node[container] = ncp
def main_method(args): sim_output = Path(args.sim_output) if not sim_output.exists(): get_logger().error("%s does not exist", sim_output) return 1 scenario_dir = Path(args.scenario) if not scenario_dir.exists(): get_logger().error("%s does not exist", scenario_dir) return 1 with open(args.first_timestamp_file) as f: ts_str = f.readline().strip() first_timestamp = map_utils.log_timestamp_to_datetime(ts_str) first_timestamp_ms = first_timestamp.timestamp() * 1000 get_logger().info("Simulation started at %s -> %d", first_timestamp, first_timestamp_ms) output = Path(args.output) output.mkdir(parents=True, exist_ok=True) show_load_demand_attr = args.show_load_demand_attr if args.show_load_demand_attr != '' else None all_services = map_utils.gather_all_services(scenario_dir) with multiprocessing.Pool(processes=os.cpu_count()) as pool: results = list() for node_dir in sim_output.iterdir(): if not node_dir.is_dir(): continue node_name_dir = map_utils.find_ncp_folder(node_dir) if node_name_dir is None: get_logger().debug("No NCP folder found in %s", node_dir) continue get_logger().debug(f"node_dir: {node_dir}") results.append(pool.apply_async(func=process_node, args=[output, first_timestamp_ms, all_services, node_name_dir, show_load_demand_attr])) for result in results: result.wait()
def main_method(args): sim_output = Path(args.sim_output) if not sim_output.exists(): get_logger().error("%s does not exist", sim_output) return 1 with open(args.first_timestamp_file) as f: ts_str = f.readline().strip() first_timestamp = map_utils.log_timestamp_to_datetime(ts_str) first_timestamp_ms = first_timestamp.timestamp() * 1000 get_logger().info("Simulation started at %s -> %d", first_timestamp, first_timestamp_ms) output = Path(args.output) / 'rlg-resource-report-lag' output.mkdir(parents=True, exist_ok=True) for node_dir in sim_output.iterdir(): max_diff = None if not node_dir.is_dir(): continue get_logger().debug("Processing node %s", node_dir) node_name_dir = map_utils.find_ncp_folder(node_dir) if node_name_dir is None: get_logger().debug("No NCP folder found") continue ncp = map_utils.node_name_from_dir(node_name_dir) all_known_nodes = set() xs = list() ys = dict() # report node -> series of diffs for time_dir in sorted(node_name_dir.iterdir()): if not time_dir.is_dir(): continue directory_time = int(time_dir.stem) get_logger().debug("\t\tProcessing time %s", time_dir) resource_report_file = time_dir / 'regionResourceReports-SHORT.json' if resource_report_file.exists(): try: with open(resource_report_file, 'r') as f: resource_reports = json.load(f) except json.decoder.JSONDecodeError: get_logger().warning("Problem reading %s, skipping", resource_report_file) continue xs.append(directory_time) seen_nodes = set() for resource_report in resource_reports: report_time = int(resource_report['timestamp']) if report_time < 1: # skip reports that we don't have a rational time for diff = None else: diff = map_utils.timestamp_to_seconds(directory_time - report_time) node = resource_report['nodeName']['name'] if node in seen_nodes: get_logger().warning( "Saw multiple reports from %s in %s, skipping the second one", node, time_dir) continue seen_nodes.add(node) all_known_nodes.add(node) # default value is setup to ensure that newly discovered # nodes have a list of values the same length as the other lists node_series = ys.get(node, [None] * (len(xs) - 1)) node_series.append(diff) ys[node] = node_series get_logger().debug("Added %s to %s xs: %d node_series: %d", diff, node, len(xs), len(node_series)) if max_diff is None or diff > max_diff: max_diff = diff # make sure we skip values for any nodes that we should have seen missing_nodes = all_known_nodes - seen_nodes if len(missing_nodes) > 0: get_logger().debug("Missing nodes: %s Seen: %s", missing_nodes, seen_nodes) for missing_node in missing_nodes: # the key must exist by now node_series = ys[missing_node] node_series.append(None) ys[missing_node] = node_series get_logger().debug("Added None to %s", missing_node) for node, node_series in ys.items(): if len(xs) != len(node_series): raise RuntimeError( f"List sizes not correct for {node} {len(xs)} != {len(node_series)}" ) if len(xs) > 0: get_logger().info("Maximum diff for %s is %s ms", ncp, max_diff) output_graph(output, ncp, xs, ys, first_timestamp_ms)
def process_node(all_services, node_dir): """ Arguments: all_services(set): names of all services node_dir(Path): directory to the node Returns: str: node name or None if not found NodeNetworkData: network load data from resource summaries or None on an error NodeNetworkData: network demand data from resource summaries or None on an error NodeNetworkData: network load data from resource report or None on an error int: first timestamp """ node_name_dir = map_utils.find_ncp_folder(node_dir) if node_name_dir is None: get_logger().debug("No NCP folder found in %s", node_dir) return None, None, None, None, None node_name = map_utils.node_name_from_dir(node_name_dir) network_summary_load = NodeNetworkData(all_services) network_summary_demand = NodeNetworkData(all_services) network_report_load = NodeNetworkData(all_services) first_timestamp = None get_logger().debug("Processing ncp folder %s from %s. NCP name: %s", node_name_dir, node_dir, node_name) for time_dir in node_name_dir.iterdir(): if not time_dir.is_dir(): continue time = int(time_dir.stem) if first_timestamp is None or time < first_timestamp: first_timestamp = time resource_summary_file = time_dir / 'resourceSummary-LONG.json' if resource_summary_file.exists(): try: with open(resource_summary_file, 'r') as f: resource_summary = json.load(f) if 'networkLoad' in resource_summary: network_load = resource_summary['networkLoad'] network_summary_load.add_data(time, network_load) if 'networkDemand' in resource_summary: network_demand = resource_summary['networkDemand'] network_summary_demand.add_data(time, network_demand) except json.decoder.JSONDecodeError: get_logger().warning("Problem reading %s, skipping", resource_summary_file) resource_report_file = time_dir / 'resourceReport-SHORT.json' if resource_report_file.exists(): try: with open(resource_report_file, 'r') as f: resource_report = json.load(f) if 'networkLoad' in resource_report: network_load = resource_report['networkLoad'] network_report_load.add_data(time, network_load) except json.decoder.JSONDecodeError: get_logger().warning("Problem reading %s, skipping", resource_summary_file) return node_name, network_summary_load, network_summary_demand, network_report_load, first_timestamp