def main_method(args):
sim_output = Path(args.sim_output)
if not sim_output.exists():
get_logger().error("%s does not exist", sim_output)
return 1
output = Path(args.output) / 'inferred-demand'
output.mkdir(parents=True, exist_ok=True)
with multiprocessing.Pool(processes=os.cpu_count()) as pool:
results = list()
for node_dir in sim_output.iterdir():
if not node_dir.is_dir():
continue
node_name_dir = map_utils.find_ncp_folder(node_dir)
if node_name_dir is None:
get_logger().debug("No NCP folder found in %s", node_name_dir)
continue
results.append(
pool.apply_async(func=process_node,
args=[output, node_name_dir]))
for result in results:
result.wait()
def main_method(args):
sim_output = Path(args.sim_output)
if not sim_output.exists():
get_logger().error("%s does not exist", sim_output)
return 1
output = Path(args.output)
output.mkdir(parents=True, exist_ok=True)
# node_name -> (xs, ys)
data = dict()
min_time = None
for node_dir in sim_output.iterdir():
max_diff = None
if not node_dir.is_dir():
continue
get_logger().debug("Processing node %s", node_dir)
node_name_dir = map_utils.find_ncp_folder(node_dir)
if node_name_dir is None:
get_logger().debug("No NCP folder found")
continue
xs = list()
ys = list()
for time_dir in node_name_dir.iterdir():
if not time_dir.is_dir():
continue
directory_time = int(time_dir.stem)
get_logger().debug("\t\tProcessing time %s", time_dir)
resource_report_file = time_dir / 'resourceReport-SHORT.json'
if resource_report_file.exists():
try:
with open(resource_report_file, 'r') as f:
resource_report = json.load(f)
except json.decoder.JSONDecodeError:
get_logger().warning("Problem reading %s, skipping",
resource_report_file)
report_time = int(resource_report['timestamp'])
if report_time < 1:
# skip reports that we don't have a rational time for
continue
# directory_time should be greater than or equal to report_time
diff = map_utils.timestamp_to_seconds(directory_time -
report_time)
xs.append(directory_time)
ys.append(diff)
if min_time is None or directory_time < min_time:
min_time = directory_time
if max_diff is None or diff > max_diff:
max_diff = diff
data[node_dir.stem] = (xs, ys)
get_logger().info("Maximum difference for %s is %f seconds",
node_dir.stem, max_diff)
output_graph(output, data, min_time)
def process_node(first_timestamp, all_apps, data_load, data_demand_short, data_demand_long, data_capacity, node_dir, container_node):
"""
Args:
first_timestamp(datetime.datetime): when the simulation started
"""
node_all_times = set()
node_containers = set()
node_name_dir = map_utils.find_ncp_folder(node_dir)
if node_name_dir is None:
get_logger().debug("No NCP folder found in %s", node_name_dir)
return
ncp = map_utils.node_name_from_dir(node_name_dir)
first_timestamp_ms = first_timestamp.timestamp() * 1000
get_logger().debug("Processing ncp folder %s from %s. NCP name: %s", node_name_dir, node_dir, ncp)
for time_dir in node_name_dir.iterdir():
if not time_dir.is_dir():
continue
time = int(time_dir.stem)
if time < first_timestamp_ms:
# ignore data before the start of the simulation
continue
node_all_times.add(time)
resource_report_file = time_dir / 'resourceReport-SHORT.json'
if resource_report_file.exists():
try:
with open(resource_report_file, 'r') as f:
resource_report = json.load(f)
process_resource_report(ncp, data_load, data_demand_short, data_capacity, node_containers, time, resource_report)
except json.decoder.JSONDecodeError:
get_logger().warning("Problem reading %s, skipping", resource_report_file)
resource_report_file = time_dir / 'resourceReport-LONG.json'
if resource_report_file.exists():
try:
with open(resource_report_file, 'r') as f:
resource_report = json.load(f)
process_resource_report(ncp, None, data_demand_long, None, node_containers, time, resource_report)
except json.decoder.JSONDecodeError:
get_logger().warning("Problem reading %s, skipping", resource_report_file)
fill_in_missing_apps(all_apps, data_load, ncp, node_all_times)
fill_in_missing_apps(all_apps, data_demand_short, ncp, node_all_times)
fill_in_missing_apps(all_apps, data_demand_long, ncp, node_all_times)
fill_in_missing_containers(all_apps, data_capacity, node_all_times, node_containers)
for container in node_containers:
container_node[container] = ncp
def main_method(args):
sim_output = Path(args.sim_output)
if not sim_output.exists():
get_logger().error("%s does not exist", sim_output)
return 1
scenario_dir = Path(args.scenario)
if not scenario_dir.exists():
get_logger().error("%s does not exist", scenario_dir)
return 1
with open(args.first_timestamp_file) as f:
ts_str = f.readline().strip()
first_timestamp = map_utils.log_timestamp_to_datetime(ts_str)
first_timestamp_ms = first_timestamp.timestamp() * 1000
get_logger().info("Simulation started at %s -> %d", first_timestamp, first_timestamp_ms)
output = Path(args.output)
output.mkdir(parents=True, exist_ok=True)
show_load_demand_attr = args.show_load_demand_attr if args.show_load_demand_attr != '' else None
all_services = map_utils.gather_all_services(scenario_dir)
with multiprocessing.Pool(processes=os.cpu_count()) as pool:
results = list()
for node_dir in sim_output.iterdir():
if not node_dir.is_dir():
continue
node_name_dir = map_utils.find_ncp_folder(node_dir)
if node_name_dir is None:
get_logger().debug("No NCP folder found in %s", node_dir)
continue
get_logger().debug(f"node_dir: {node_dir}")
results.append(pool.apply_async(func=process_node, args=[output, first_timestamp_ms, all_services, node_name_dir, show_load_demand_attr]))
for result in results:
result.wait()
def main_method(args):
sim_output = Path(args.sim_output)
if not sim_output.exists():
get_logger().error("%s does not exist", sim_output)
return 1
with open(args.first_timestamp_file) as f:
ts_str = f.readline().strip()
first_timestamp = map_utils.log_timestamp_to_datetime(ts_str)
first_timestamp_ms = first_timestamp.timestamp() * 1000
get_logger().info("Simulation started at %s -> %d", first_timestamp,
first_timestamp_ms)
output = Path(args.output) / 'rlg-resource-report-lag'
output.mkdir(parents=True, exist_ok=True)
for node_dir in sim_output.iterdir():
max_diff = None
if not node_dir.is_dir():
continue
get_logger().debug("Processing node %s", node_dir)
node_name_dir = map_utils.find_ncp_folder(node_dir)
if node_name_dir is None:
get_logger().debug("No NCP folder found")
continue
ncp = map_utils.node_name_from_dir(node_name_dir)
all_known_nodes = set()
xs = list()
ys = dict() # report node -> series of diffs
for time_dir in sorted(node_name_dir.iterdir()):
if not time_dir.is_dir():
continue
directory_time = int(time_dir.stem)
get_logger().debug("\t\tProcessing time %s", time_dir)
resource_report_file = time_dir / 'regionResourceReports-SHORT.json'
if resource_report_file.exists():
try:
with open(resource_report_file, 'r') as f:
resource_reports = json.load(f)
except json.decoder.JSONDecodeError:
get_logger().warning("Problem reading %s, skipping",
resource_report_file)
continue
xs.append(directory_time)
seen_nodes = set()
for resource_report in resource_reports:
report_time = int(resource_report['timestamp'])
if report_time < 1:
# skip reports that we don't have a rational time for
diff = None
else:
diff = map_utils.timestamp_to_seconds(directory_time -
report_time)
node = resource_report['nodeName']['name']
if node in seen_nodes:
get_logger().warning(
"Saw multiple reports from %s in %s, skipping the second one",
node, time_dir)
continue
seen_nodes.add(node)
all_known_nodes.add(node)
# default value is setup to ensure that newly discovered
# nodes have a list of values the same length as the other lists
node_series = ys.get(node, [None] * (len(xs) - 1))
node_series.append(diff)
ys[node] = node_series
get_logger().debug("Added %s to %s xs: %d node_series: %d",
diff, node, len(xs), len(node_series))
if max_diff is None or diff > max_diff:
max_diff = diff
# make sure we skip values for any nodes that we should have seen
missing_nodes = all_known_nodes - seen_nodes
if len(missing_nodes) > 0:
get_logger().debug("Missing nodes: %s Seen: %s",
missing_nodes, seen_nodes)
for missing_node in missing_nodes:
# the key must exist by now
node_series = ys[missing_node]
node_series.append(None)
ys[missing_node] = node_series
get_logger().debug("Added None to %s", missing_node)
for node, node_series in ys.items():
if len(xs) != len(node_series):
raise RuntimeError(
f"List sizes not correct for {node} {len(xs)} != {len(node_series)}"
)
if len(xs) > 0:
get_logger().info("Maximum diff for %s is %s ms", ncp, max_diff)
output_graph(output, ncp, xs, ys, first_timestamp_ms)
def process_node(all_services, node_dir):
"""
Arguments:
all_services(set): names of all services
node_dir(Path): directory to the node
Returns:
str: node name or None if not found
NodeNetworkData: network load data from resource summaries or None on an error
NodeNetworkData: network demand data from resource summaries or None on an error
NodeNetworkData: network load data from resource report or None on an error
int: first timestamp
"""
node_name_dir = map_utils.find_ncp_folder(node_dir)
if node_name_dir is None:
get_logger().debug("No NCP folder found in %s", node_dir)
return None, None, None, None, None
node_name = map_utils.node_name_from_dir(node_name_dir)
network_summary_load = NodeNetworkData(all_services)
network_summary_demand = NodeNetworkData(all_services)
network_report_load = NodeNetworkData(all_services)
first_timestamp = None
get_logger().debug("Processing ncp folder %s from %s. NCP name: %s",
node_name_dir, node_dir, node_name)
for time_dir in node_name_dir.iterdir():
if not time_dir.is_dir():
continue
time = int(time_dir.stem)
if first_timestamp is None or time < first_timestamp:
first_timestamp = time
resource_summary_file = time_dir / 'resourceSummary-LONG.json'
if resource_summary_file.exists():
try:
with open(resource_summary_file, 'r') as f:
resource_summary = json.load(f)
if 'networkLoad' in resource_summary:
network_load = resource_summary['networkLoad']
network_summary_load.add_data(time, network_load)
if 'networkDemand' in resource_summary:
network_demand = resource_summary['networkDemand']
network_summary_demand.add_data(time, network_demand)
except json.decoder.JSONDecodeError:
get_logger().warning("Problem reading %s, skipping",
resource_summary_file)
resource_report_file = time_dir / 'resourceReport-SHORT.json'
if resource_report_file.exists():
try:
with open(resource_report_file, 'r') as f:
resource_report = json.load(f)
if 'networkLoad' in resource_report:
network_load = resource_report['networkLoad']
network_report_load.add_data(time, network_load)
except json.decoder.JSONDecodeError:
get_logger().warning("Problem reading %s, skipping",
resource_summary_file)
return node_name, network_summary_load, network_summary_demand, network_report_load, first_timestamp