def main(argv=None):
if argv is None:
argv = sys.argv[1:]
class ArgumentParserWithDefaults(argparse.ArgumentParser):
'''
From https://stackoverflow.com/questions/12151306/argparse-way-to-include-default-values-in-help
'''
def add_argument(self, *args, help=None, default=None, **kwargs):
if help is not None:
kwargs['help'] = help
if default is not None and args[0] != '-h':
kwargs['default'] = default
if help is not None:
kwargs['help'] += ' (default: {})'.format(default)
super().add_argument(*args, **kwargs)
parser = ArgumentParserWithDefaults(
formatter_class=argparse.RawTextHelpFormatter)
parser.add_argument("-l",
"--logconfig",
dest="logconfig",
help="logging configuration (default: logging.json)",
default='logging.json')
parser.add_argument("--debug",
dest="debug",
help="Enable interactive debugger on error",
action='store_true')
parser.add_argument("--demand",
dest="demand",
help="Directory to read the demand information from",
required=True)
parser.add_argument("--output",
dest="output",
help="Directory to write the demand information to",
required=True)
parser.add_argument("--step-size",
dest="step_size",
help="Number of milliseconds between samples",
type=int,
default=10000)
args = parser.parse_args(argv)
map_utils.setup_logging(default_path=args.logconfig)
if 'multiprocessing' in sys.modules:
import multiprocessing_logging
multiprocessing_logging.install_mp_handler()
if args.debug:
import pdb, traceback
try:
return main_method(args)
except:
extype, value, tb = sys.exc_info()
traceback.print_exc()
pdb.post_mortem(tb)
else:
return main_method(args)
def main(argv=None):
if argv is None:
argv = sys.argv[1:]
class ArgumentParserWithDefaults(argparse.ArgumentParser):
'''
From https://stackoverflow.com/questions/12151306/argparse-way-to-include-default-values-in-help
'''
def add_argument(self, *args, help=None, default=None, **kwargs):
if help is not None:
kwargs['help'] = help
if default is not None and args[0] != '-h':
kwargs['default'] = default
if help is not None:
kwargs['help'] += ' (default: {})'.format(default)
super().add_argument(*args, **kwargs)
parser = ArgumentParserWithDefaults(
formatter_class=argparse.RawTextHelpFormatter)
parser.add_argument("-l",
"--logconfig",
dest="logconfig",
help="logging configuration (default: logging.json)",
default='logging.json')
parser.add_argument("--debug",
dest="debug",
help="Enable interactive debugger on error",
action='store_true')
parser.add_argument("-s",
"--sim-output",
dest="sim_output",
help="Chart output directory (Required)",
required=True)
parser.add_argument("-o",
"--output",
dest="output",
help="Output directory (Required)",
required=True)
parser.add_argument(
"--first-timestamp-file",
dest="first_timestamp_file",
help=
"Path to file containing the log timestamp that the simulation started",
required=True)
args = parser.parse_args(argv)
map_utils.setup_logging(default_path=args.logconfig)
if args.debug:
import pdb, traceback
try:
return main_method(args)
except:
extype, value, tb = sys.exc_info()
traceback.print_exc()
pdb.post_mortem(tb)
else:
return main_method(args)
def main(argv=None):
if argv is None:
argv = sys.argv[1:]
parser = argparse.ArgumentParser()
parser.add_argument("-l",
"--logconfig",
dest="logconfig",
help="logging configuration (default: logging.json)",
default='logging.json')
parser.add_argument("-s",
"--sim-output",
dest="sim_output",
help="Hifi sim output directory (Required)",
required=True)
parser.add_argument("-o",
"--output",
dest="output",
help="Output file (defaults to pull_stats.csv)",
default='pull_stats.csv')
args = parser.parse_args(argv)
map_utils.setup_logging(default_path=args.logconfig)
sim_dir = Path(args.sim_output)
with open(args.output, 'w') as csvfile:
writer = csv.writer(csvfile)
writer.writerow([
'node', 'initial_pull_timestamp', 'finished_timestamp',
'time_to_pull_seconds', 'attempts', 'result', 'image'
])
for server_dir in sim_dir.iterdir():
if server_dir.is_dir():
node = server_dir.name
agent_dir = server_dir / 'agent'
for agent_logfile in agent_dir.glob('map-agent*.log'):
get_logger().debug("Processing %s", agent_logfile)
process_logfile(writer, node, agent_logfile)
def main(argv=None):
if argv is None:
argv = sys.argv[1:]
parser = argparse.ArgumentParser()
parser.add_argument("-l",
"--logconfig",
dest="logconfig",
help="logging configuration (default: logging.json)",
default='logging.json')
parser.add_argument(
"--first-timestamp-file",
dest="first_timestamp_file",
help=
"Path to file containing the log timestamp that the simulation started",
required=True)
parser.add_argument("-s",
"--sim-output",
dest="sim_output",
help="Chart output directory (Required)",
required=True)
parser.add_argument("-o",
"--output",
dest="output",
help="Output directory (Required)",
required=True)
parser.add_argument("-c",
"--container-capacity",
dest="container_queue_length_capacity",
help="QueueLength capacity per container",
default=None)
parser.add_argument(
"--process-latency-data",
dest="process_latency_data",
action='store_true',
help=
"If present, show queue length estimate from processing_latency.csv files."
)
args = parser.parse_args(argv)
map_utils.setup_logging(default_path=args.logconfig)
with open(args.first_timestamp_file) as f:
ts_str = f.readline().strip()
first_timestamp = map_utils.log_timestamp_to_datetime(ts_str)
first_timestamp_ms = first_timestamp.timestamp() * 1000
get_logger().info("Run started at %s", first_timestamp)
sim_output = Path(args.sim_output)
if not sim_output.exists():
get_logger().error("%s does not exist", sim_output)
return 1
output = Path(args.output)
output.mkdir(parents=True, exist_ok=True)
container_queue_length_capacity = int(
args.container_queue_length_capacity
) if args.container_queue_length_capacity else None
process_latency_data = args.process_latency_data
# app -> node_attr -> container_name -> timestamp -> value
data_load = dict()
data_demand = dict()
container_processing_latency = dict()
for node_dir in sim_output.iterdir():
if not node_dir.is_dir():
continue
get_logger().debug("Processing node %s", node_dir)
agent_dir = node_dir / 'agent'
if not agent_dir.is_dir():
continue
get_logger().debug("\tAgent dir %s", agent_dir)
for node_name_dir in agent_dir.iterdir():
if not node_name_dir.is_dir():
continue
for time_dir in node_name_dir.iterdir():
if not time_dir.is_dir():
continue
get_logger().debug("\t\tProcessing time %s", time_dir)
resource_report_file = time_dir / 'resourceReport-SHORT.json'
if resource_report_file.exists():
try:
with open(resource_report_file, 'r') as f:
resource_report = json.load(f)
time = int(time_dir.stem)
if 'containerReports' in resource_report:
for container_name, container_report in resource_report[
'containerReports'].items():
container_name_short = container_name.split(
'.')[0]
process_container_report(
data_load, data_demand,
container_name_short, time,
container_report)
except json.decoder.JSONDecodeError:
get_logger().warning("Problem reading %s, skipping",
resource_report_file)
container_data_dir = agent_dir / 'container_data'
if process_latency_data:
for container_data_service_dir in container_data_dir.iterdir():
match = re.search(r'^(.+)\.([^\._]+)_(.+)$',
container_data_service_dir.name)
if match:
artifact = match.group(2)
for container_data_service_name_dir in container_data_service_dir.iterdir(
):
match = re.search(r'^([^\.]+)\.map\.dcomp$',
container_data_service_name_dir.name)
if match:
container_name = match.group(1)
for container_data_service_name_instance_dir in container_data_service_name_dir.iterdir(
):
processing_latency_file = container_data_service_name_instance_dir / 'app_metrics_data' / 'processing_latency.csv'
container_instance_latency_data = dict()
with open(processing_latency_file, "r") as f:
reader = csv.DictReader(f)
for row in reader:
event = row['event']
time_start = int(row['time_received'])
time_end = int(row['time_ack_sent'])
if event == 'request_success':
container_instance_latency_data[
time_start] = 1
container_instance_latency_data[
time_end] = -1
container_processing_latency.setdefault(
artifact, dict()).setdefault(
container_name, list()).append(
container_instance_latency_data)
container_expected_queue_lengths = accumulate_expected_queue_length(
container_processing_latency)
output_graphs(first_timestamp_ms, output, data_load,
container_expected_queue_lengths, "load",
container_queue_length_capacity)
def main(argv=None):
if argv is None:
argv = sys.argv[1:]
parser = argparse.ArgumentParser()
parser.add_argument("-l",
"--logconfig",
dest="logconfig",
help="logging configuration (default: logging.json)",
default='logging.json')
parser.add_argument("-c",
"--chart-output",
dest="chart_output",
help="Output of MAPChartGeneration(Required)",
required=True)
parser.add_argument("-o",
"--output",
dest="output",
help="Output directory (Required)",
required=True)
parser.add_argument("--interactive",
dest="interactive",
action="store_true",
help="If specified, display the plots")
args = parser.parse_args(argv)
map_utils.setup_logging(default_path=args.logconfig)
chart_output = Path(args.chart_output)
if not chart_output.exists():
get_logger().error("%s does not exist", chart_output)
return 1
output_dir = Path(args.output)
output_dir.mkdir(parents=True, exist_ok=True)
client_demand_dir = chart_output / 'client_demand'
frames = list()
for file in client_demand_dir.glob('num_clients-*.csv'):
match = re.match(r'^num_clients-(.*)\.csv$', file.name)
if not match:
continue
service = match.group(1)
df = pd.read_csv(file)
df['service'] = service
frames.append(df)
data = pd.concat(frames, ignore_index=True)
data['time_minutes'] = (data['time'] - data['time'].min()) / 1000 / 60
all_services = data['service'].unique()
fig, ax = map_utils.subplots()
ax.set_title('number of clients per service over time')
ax.set_ylabel('number of clients')
ax.set_xlabel('time (minutes)')
for service in all_services:
plot_data = data.loc[(data['service'] == service)]
ax.step(plot_data['time_minutes'],
plot_data['num clients'],
where='post',
label=service)
handles, labels = ax.get_legend_handles_labels()
lgd = ax.legend(handles,
labels,
bbox_to_anchor=(1.04, 1),
loc="upper left")
if args.interactive:
plt.show()
else:
output_name = output_dir / 'client_demand.png'
plt.savefig(output_name.as_posix(),
format='png',
bbox_extra_artists=(lgd, ),
bbox_inches='tight')
plt.close(fig)
def main(argv=None):
if argv is None:
argv = sys.argv[1:]
parser = argparse.ArgumentParser()
parser.add_argument("-l",
"--logconfig",
dest="logconfig",
help="logging configuration (default: logging.json)",
default='logging.json')
parser.add_argument("-c",
"--chart_output",
dest="chart_output",
help="Chart output directory (Required)",
required=True)
parser.add_argument("-o",
"--output",
dest="output",
help="Output directory (Required)",
required=True)
parser.add_argument(
"--first-timestamp-file",
dest="first_timestamp_file",
help=
"Path to file containing the log timestamp that the simulation started",
required=True)
parser.add_argument(
"--sample-duration",
dest="sample_duration",
help=
"Number of minutes between samples of plans for expected percentages plots",
default=1,
type=float)
parser.add_argument("--scenario",
dest="scenario",
help="Scenario directory (Required)",
required=True)
args = parser.parse_args(argv)
map_utils.setup_logging(default_path=args.logconfig)
chart_output = Path(args.chart_output)
if not chart_output.exists():
get_logger().error("%s does not exist", chart_output)
return 1
with open(args.first_timestamp_file) as f:
ts_str = f.readline().strip()
first_timestamp = map_utils.log_timestamp_to_datetime(ts_str)
get_logger().info("Simulation started at %s", first_timestamp)
output_dir = Path(args.output)
output_dir.mkdir(parents=True, exist_ok=True)
dcop_file = chart_output / 'dcop_plan_updates/all_dcop_plans.csv'
dcop_plans = pd.read_csv(dcop_file, na_values="?")
dcop_plans.fillna(0, inplace=True)
dcop_plans.sort_values(by=['time'], inplace=True)
rlg_file = chart_output / 'rlg_plan_updates/all_rlg_overflow_plans.csv'
rlg_plans = pd.read_csv(rlg_file, na_values="?")
rlg_plans.fillna(0, inplace=True)
rlg_plans.sort_values(by=['time'], inplace=True)
dns_file = chart_output / 'dns_region_plan_updates/all_dns_region_plans.csv'
dns_plans = pd.read_csv(dns_file, na_values="?")
dns_plans.fillna(0, inplace=True)
dns_plans.sort_values(by=['time'], inplace=True)
first_timestamp_ms = first_timestamp.timestamp() * 1000
dcop_plans['time_minutes'] = (dcop_plans['time'] -
first_timestamp_ms) / 1000 / 60
rlg_plans['time_minutes'] = (rlg_plans['time'] -
first_timestamp_ms) / 1000 / 60
dns_plans['time_minutes'] = (dns_plans['time'] -
first_timestamp_ms) / 1000 / 60
all_regions = np.union1d(dcop_plans['plan_region'].unique(),
rlg_plans['plan_region'].unique())
all_services = np.union1d(dcop_plans['service'].unique(),
rlg_plans['service'].unique())
for region in all_regions:
for service in all_services:
plot_region_service_plan_type(output_dir, "DCOP", dcop_plans,
all_regions, region, service)
plot_region_service_plan_type(output_dir, "RLG", rlg_plans,
all_regions, region, service)
plot_region_service_plan_type(output_dir, "DNS", dns_plans,
all_regions, region, service)
scenario_dir = Path(args.scenario)
if not scenario_dir.exists():
get_logger().error("%s does not exist", scenario_dir)
return 1
client_regions = compute_client_regions(scenario_dir)
# plot expected percentages
sample_duration = args.sample_duration
for source_region in client_regions:
for service in all_services:
times, region_weights = compute_expected(sample_duration,
all_regions, 'DCOP',
dcop_plans, source_region,
service)
plot_expected(output_dir, 'DCOP', source_region, service, times,
region_weights)
times, region_weights = compute_expected(sample_duration,
all_regions, 'RLG',
rlg_plans, source_region,
service)
plot_expected(output_dir, 'RLG', source_region, service, times,
region_weights)
times, region_weights = compute_expected(sample_duration,
all_regions, 'DNS',
dns_plans, source_region,
service)
plot_expected(output_dir, 'DNS', source_region, service, times,
region_weights)
get_logger().info("Finished")
def main(argv=None):
if argv is None:
argv = sys.argv[1:]
parser = argparse.ArgumentParser()
parser.add_argument("-l",
"--logconfig",
dest="logconfig",
help="logging configuration (default: logging.json)",
default='logging.json')
parser.add_argument("-L",
"--log-file",
dest="log_file_path",
help="Log file with timestamps",
required=True)
parser.add_argument(
"-b",
"--testbed",
dest="testbed",
help=
"The testbed used for the run. This is used to obtain time zone information.",
default='')
parser.add_argument("-s",
"--run-seconds-start",
dest="run_seconds_start",
help="Number of seconds into run to start inspection",
default='0')
parser.add_argument(
"-i",
"--interval",
dest="run_seconds_interval",
help="Interval number of seconds within run to inspect",
default='')
parser.add_argument(
"-d",
"--identify-log-delays",
dest="log_warn_delay",
help=
"Print a warning between consecutive log statements separated by a delay larger than the value specified in seconds.",
default='')
parser.add_argument("-f",
"--regex-filter",
dest="regex_filter",
help="Pattern to use for filtering of log statements",
default='')
parser.add_argument(
"-t",
"--show-run-seconds",
dest="show_run_seconds",
action="store_true",
help="Adds the number of seconds into the run to each log statement",
default='')
parser.add_argument("-o",
"--output-file",
dest="log_output_file_path",
help="Log statements output from inspection interval",
default='')
parser.add_argument(
"-p",
"--occurrence-plot",
dest="occurrence_plot_output_path",
help="Create a graph showing occurrences of log messages in time",
default='')
parser.add_argument(
"--first-timestamp-file",
dest="first_timestamp_file",
help=
"Path to file containing the log timestamp that the simulation started",
required=True)
args = parser.parse_args(argv)
map_utils.setup_logging(default_path=args.logconfig)
# parse arguments
log_file_path = args.log_file_path
testbed = (None if args.testbed == '' else args.testbed)
run_seconds_start = float(args.run_seconds_start)
run_seconds_interval = (None if args.run_seconds_interval == '' else float(
args.run_seconds_interval))
log_warn_delay = (None if args.log_warn_delay == '' else timedelta(
seconds=float(args.log_warn_delay)))
regex_filter = (None if args.regex_filter == '' else re.compile(
args.regex_filter))
show_run_seconds = args.show_run_seconds
log_output_file_path = args.log_output_file_path
occurrence_plot_output_path = (None if args.occurrence_plot_output_path
== '' else args.occurrence_plot_output_path)
with open(args.first_timestamp_file) as f:
ts_str = f.readline().strip()
first_timestamp = map_utils.log_timestamp_to_datetime(ts_str)
get_logger().info("Simulation started at %s", first_timestamp)
time_zone = None
#testbed_to_time_zone_map = {'emulab' : tz.tzoffset('MDT', -21600), 'dcomp' : tz.tzoffset('UTC', 0)}
#testbed_to_time_zone_map = {'emulab' : timezone(-timedelta(hours=6), 'mdt'), 'dcomp' : timezone.utc}
testbed_to_time_zone_map = {
'emulab': timezone('US/Mountain'),
'dcomp': timezone('UTC')
}
if (testbed != None):
if (testbed in testbed_to_time_zone_map):
time_zone = testbed_to_time_zone_map.get(testbed.lower())
get_logger().info("Found time zone for testbed '%s': %s.", testbed,
time_zone)
else:
get_logger().fatal("Could not map testbed '%s' to timezone.",
testbed)
exit(1)
get_logger().info("Log File: %s", log_file_path)
with open(log_file_path, "r") as log_file:
log_line_occurrence_times = list()
ref_time = first_timestamp
get_logger().info(" Ref time String: %s",
map_utils.datetime_to_string(ref_time))
get_logger().info(" Ref time epoch ms: %s",
str(datetime_to_epoch_ms(ref_time)))
get_logger().info("")
get_logger().info("")
start_time_within_run = ref_time + timedelta(seconds=run_seconds_start)
get_logger().info("At %s seconds into run: ", str(run_seconds_start))
get_logger().info(" Time within run: %s",
map_utils.datetime_to_string(start_time_within_run))
get_logger().info(" Epoch ms: %s",
str(datetime_to_epoch_ms(start_time_within_run)))
get_logger().info("")
end_time_within_run = (None if run_seconds_interval == None else (
ref_time +
timedelta(seconds=(run_seconds_start + run_seconds_interval))))
if (end_time_within_run != None):
get_logger().info("At " +
str(run_seconds_start + run_seconds_interval) +
" seconds into run: ")
get_logger().info(
" Time within run: %s",
map_utils.datetime_to_string(end_time_within_run))
get_logger().info(" Epoch ms: %s",
str(datetime_to_epoch_ms(end_time_within_run)))
get_logger().info("\n\n\n")
get_logger().info(
"<==================== Log Output ====================>\n")
if (log_output_file_path != ''):
log_output_file = open(log_output_file_path, "w")
log_output_file.write("Log statements for run time range \n")
log_output_file.write(" Start Time (s): " +
str(run_seconds_start) + "\n")
if (run_seconds_interval != None):
log_output_file.write(" End Time (s): " +
str(run_seconds_start +
run_seconds_interval) + "\n")
log_output_file.write("\n")
if (log_warn_delay != None):
log_output_file.write(
" Max time delay without warning (s): " +
str(log_warn_delay) + "\n")
log_output_file.write(
"====================================================================================================\n\n\n\n"
)
else:
log_output_file = None
log_lines_printed = 0
prev_line = ''
prev_line_time = None
last_line_matches = False
for line in log_file:
output_line = line
line_time = map_utils.log_line_to_time(line, time_zone)
if (line_time != None):
if (line_time >= start_time_within_run and
(regex_filter == None or re.search(regex_filter, line))):
line_prefix = ""
if (show_run_seconds):
line_run_seconds = (line_time - ref_time).seconds
output_line = str(
line_run_seconds) + ": " + output_line
if (log_warn_delay != None and prev_line_time != None):
log_delay = line_time - prev_line_time
if (log_output_file != None
and log_delay > log_warn_delay):
log_output_file.write("<--- Time delay " +
str(log_delay) + " > " +
str(log_warn_delay) +
" --->\n")
if (log_lines_printed < 1):
get_logger().info(line)
if (log_output_file != None):
log_output_file.write(output_line)
record_output_line_time(ref_time, line_time,
log_line_occurrence_times)
prev_line = line
prev_line_time = line_time
log_lines_printed += 1
last_line_matches = True
else:
last_line_matches = False
if (end_time_within_run != None
and line_time > end_time_within_run):
break
else:
# write a statements without a valid timestamp if and only if the the most recent line with a valid time was printed
# this allows multiline log statements such as exceptions to be printed entirely
if (last_line_matches):
if (log_output_file != None):
log_output_file.write(output_line)
get_logger().info(" ......")
get_logger().info(prev_line)
# close the output file if it was specified
if (log_output_file != None):
log_output_file.close()
if (occurrence_plot_output_path != None):
if (regex_filter == None):
title = "Times of all log lines"
else:
title = "Times of log lines matching regex '" + regex_filter.pattern + "'"
plot_log_line_occurrence(occurrence_plot_output_path, title,
log_line_occurrence_times)
def main(argv=None):
if argv is None:
argv = sys.argv[1:]
class ArgumentParserWithDefaults(argparse.ArgumentParser):
'''
From https://stackoverflow.com/questions/12151306/argparse-way-to-include-default-values-in-help
'''
def add_argument(self, *args, help=None, default=None, **kwargs):
if help is not None:
kwargs['help'] = help
if default is not None and args[0] != '-h':
kwargs['default'] = default
if help is not None:
kwargs['help'] += ' (default: {})'.format(default)
super().add_argument(*args, **kwargs)
parser = ArgumentParserWithDefaults(
formatter_class=argparse.RawTextHelpFormatter)
parser.add_argument("-l",
"--logconfig",
dest="logconfig",
help="logging configuration (default: logging.json)",
default='logging.json')
parser.add_argument("--debug",
dest="debug",
help="Enable interactive debugger on error",
action='store_true')
parser.add_argument(
"-i",
"--input-file",
dest="load_capacity_csv_file",
help="Path to the input load-[attribute]-[region].csv file",
required=True)
args = parser.parse_args(argv)
map_utils.setup_logging(default_path=args.logconfig)
load_capacity_csv_file = Path(args.load_capacity_csv_file)
with open(load_capacity_csv_file) as f:
reader = csv.DictReader(f)
areas = dict()
start_time = None
end_time = None
prev_row = None
prev_row_values = dict()
for row in reader:
get_logger().debug(f"row: {row}")
current_time = float(row['relative minutes'])
if not start_time:
start_time = current_time
end_time = current_time
if prev_row:
prev_time = float(prev_row['relative minutes'])
delta_time = current_time - prev_time
average_values_for_delta = dict()
for label in row.keys():
if label != 'relative minutes':
if label in prev_row_values:
prev_value = float(prev_row_values[label])
current_value = float(
row[label]) if row[label] != '' else prev_value
average_values_for_delta[label] = (current_value +
prev_value) / 2
for label in average_values_for_delta.keys():
value = average_values_for_delta[label]
if label not in areas:
areas[label] = 0
areas[label] += value * delta_time
for label in row.keys():
if label != 'relative minutes':
if row[label] != '':
prev_row_values[label] = row[label]
get_logger().debug(f"prev_row_values: {prev_row_values}")
prev_row = row
total_delta_time = end_time - start_time
get_logger().info(f"time: {total_delta_time}")
labels = list(areas.keys())
labels.sort()
capacity_sum_label = ""
capacity_sum = 0
load_sum_label = ""
load_sum = 0
for label in labels:
value = areas[label]
value_per_time = value / total_delta_time
get_logger().info(f"{label}")
get_logger().info(f" area: {value}")
get_logger().info(
f" area / time: {value_per_time} = {value} / {total_delta_time}"
)
if "capacity" in label:
if not capacity_sum_label == "":
capacity_sum_label += " + "
capacity_sum_label += label
capacity_sum += value
else:
if not load_sum_label == "":
load_sum_label += " + "
load_sum_label += label
load_sum += value
get_logger().info(f"{capacity_sum_label}")
get_logger().info(f" area: {capacity_sum}")
get_logger().info(
f" area / time: {capacity_sum / total_delta_time} = {capacity_sum} / {total_delta_time}"
)
get_logger().info(f"{load_sum_label}")
get_logger().info(f" area: {load_sum}")
get_logger().info(
f" area / time: {load_sum / total_delta_time} = {load_sum} / {total_delta_time}"
)
def main(argv=None):
if argv is None:
argv = sys.argv[1:]
class ArgumentParserWithDefaults(argparse.ArgumentParser):
'''
From https://stackoverflow.com/questions/12151306/argparse-way-to-include-default-values-in-help
'''
def add_argument(self, *args, help=None, default=None, **kwargs):
if help is not None:
kwargs['help'] = help
if default is not None and args[0] != '-h':
kwargs['default'] = default
if help is not None:
kwargs['help'] += ' (default: {})'.format(default)
super().add_argument(*args, **kwargs)
parser = ArgumentParserWithDefaults(
formatter_class=argparse.RawTextHelpFormatter)
parser.add_argument("-l",
"--logconfig",
dest="logconfig",
help="logging configuration (default: logging.json)",
default='logging.json')
parser.add_argument("--debug",
dest="debug",
help="Enable interactive debugger on error",
action='store_true')
parser.add_argument("--input",
dest="input",
help="Path to all_client_requests.csv",
required=True)
parser.add_argument(
"--first-timestamp-file",
dest="first_timestamp_file",
help=
"Path to file containing the log timestamp that the simulation started",
required=True)
parser.add_argument("-o",
"--output",
dest="output",
help="Output directory",
required=True)
parser.add_argument("-s",
"--sim-output",
dest="sim_output",
help="Sim output directory (Required)",
required=True)
parser.add_argument(
"--window-duration",
dest="window_duration",
help=
"Number of minutes to use for the window to compute percentages over",
default=1,
type=float)
args = parser.parse_args(argv)
map_utils.setup_logging(default_path=args.logconfig)
with open(args.first_timestamp_file) as f:
ts_str = f.readline().strip()
first_timestamp = map_utils.log_timestamp_to_datetime(ts_str)
get_logger().info("Simulation started at %s", first_timestamp)
all_requests_path = Path(args.input)
if not all_requests_path.exists():
get_logger().error("Cannot read all requests file at %s",
all_requests_path)
return -1
output = Path(args.output)
output.mkdir(parents=True, exist_ok=True)
sim_output = Path(args.sim_output)
regions = dict()
window_duration = datetime.timedelta(minutes=args.window_duration)
scenario_dir = sim_output / 'inputs' / 'scenario'
node_to_region_dict = map_utils.gather_region_info(scenario_dir)
_, ip_to_node_dict = summarize_hifi.parse_node_info(sim_output)
get_logger().debug(f"node_to_region_dict = {node_to_region_dict}")
get_logger().debug(f"ip_to_node_dict = {ip_to_node_dict}")
# region --> service --> window --> ClientDnsRequestCounts
requests_region_service_counts = dict()
read_all_requests_file(all_requests_path, first_timestamp, window_duration,
node_to_region_dict, requests_region_service_counts)
read_weighted_dns_files(sim_output, first_timestamp, window_duration,
ip_to_node_dict, requests_region_service_counts)
output_requests_csv_per_region(requests_region_service_counts,
first_timestamp, output)
output_requests_graph_per_region(requests_region_service_counts,
first_timestamp, output)
def main(argv=None):
if argv is None:
argv = sys.argv[1:]
parser = argparse.ArgumentParser()
parser.add_argument("-l",
"--logconfig",
dest="logconfig",
help="logging configuration (default: logging.json)",
default='logging.json')
parser.add_argument("-c",
"--chart_output",
dest="chart_output",
help="Chart output directory (Required)",
required=True)
parser.add_argument("-o",
"--output",
dest="output",
help="Output directory (Required)",
required=True)
args = parser.parse_args(argv)
map_utils.setup_logging(default_path=args.logconfig)
chart_output = Path(args.chart_output)
if not chart_output.exists():
get_logger().error("%s does not exist", chart_output)
return 1
output_dir = Path(args.output)
output_dir.mkdir(parents=True, exist_ok=True)
chart_output_dns = chart_output / 'dns'
get_logger().debug("chart_output_dns: %s", chart_output_dns)
all_regions = set()
all_services = set()
service_region_dns_resolution_counts = dict()
earliest_time = None
for dns_req_count_file in chart_output_dns.glob('dns_req_count_*.csv'):
match = re.match("dns_req_count_(.+)--(.+)\.csv",
os.path.basename(dns_req_count_file))
if match:
region = match.group(1)
service = match.group(2)
get_logger().info(
"Found file '%s' for region '%s' and service '%s'",
dns_req_count_file, region, service)
all_regions.add(region)
all_services.add(service)
dns_resolution_counts = pd.read_csv(dns_req_count_file,
na_values="?")
service_region_dns_resolution_counts.setdefault(
service, dict())[region] = dns_resolution_counts
if earliest_time is None:
earliest_time = dns_resolution_counts['time'].min()
else:
earliest_time = min(earliest_time,
dns_resolution_counts['time'].min())
for service in all_services:
for region in all_regions:
service_region_dns_resolution_counts[service][region][
'time_minutes'] = (service_region_dns_resolution_counts[
service][region]['time'] - earliest_time) / 1000 / 60
get_logger().info("Output plots for service '%s' and region '%s'",
service, region)
plot_dns_region_resolution_counts(
output_dir,
service_region_dns_resolution_counts[service][region],
all_regions, region, service)
plot_dns_container_resolution_counts(
output_dir,
service_region_dns_resolution_counts[service][region], region,
service)
def main(argv=None):
if argv is None:
argv = sys.argv[1:]
parser = argparse.ArgumentParser()
parser.add_argument("-l",
"--logconfig",
dest="logconfig",
help="logging configuration (default: logging.json)",
default='logging.json')
parser.add_argument("-s",
"--sim-output",
dest="sim_output",
help="Chart output directory (Required)",
required=True)
parser.add_argument("-o",
"--output",
dest="output",
help="Output directory (Required)",
required=True)
parser.add_argument(
"--first-timestamp-file",
dest="first_timestamp_file",
help=
"Path to file containing the log timestamp that the simulation started",
required=True)
args = parser.parse_args(argv)
map_utils.setup_logging(default_path=args.logconfig)
sim_output = Path(args.sim_output)
if not sim_output.exists():
get_logger().error("%s does not exist", sim_output)
return 1
with open(args.first_timestamp_file) as f:
ts_str = f.readline().strip()
first_timestamp = map_utils.log_timestamp_to_datetime(ts_str)
get_logger().info("Simulation started at %s", first_timestamp)
first_timestamp_ms = first_timestamp.timestamp() * 1000
output = Path(args.output)
output.mkdir(parents=True, exist_ok=True)
# app -> node_attr -> container_name -> timestamp -> value
data_load = dict()
data_demand = dict()
for node_dir in sim_output.iterdir():
if not node_dir.is_dir():
continue
get_logger().debug("Processing node %s", node_dir)
agent_dir = node_dir / 'agent'
if agent_dir.is_dir():
get_logger().debug("\tAgent dir %s", agent_dir)
for node_name_dir in agent_dir.iterdir():
if not node_name_dir.is_dir():
continue
for time_dir in node_name_dir.iterdir():
if not time_dir.is_dir():
continue
add_time_dir_data(time_dir, data_load, data_demand)
else:
for time_dir in node_dir.iterdir():
if not time_dir.is_dir():
continue
add_time_dir_data(time_dir, data_load, data_demand)
output_graphs(first_timestamp_ms, output, data_load, "load")
output_graphs(first_timestamp_ms, output, data_demand, "demand")
def main(argv=None):
if argv is None:
argv = sys.argv[1:]
parser = argparse.ArgumentParser()
parser.add_argument("-l",
"--logconfig",
dest="logconfig",
help="logging configuration (default: logging.json)",
default='logging.json')
parser.add_argument("-c",
"--chart_output",
dest="chart_output",
help="Chart output directory (Required)",
required=True)
parser.add_argument("-o",
"--output",
dest="output",
help="Output directory (Required)",
required=True)
parser.add_argument("--interactive",
dest="interactive",
action="store_true",
help="If specified, display the plots")
args = parser.parse_args(argv)
map_utils.setup_logging(default_path=args.logconfig)
chart_output = Path(args.chart_output)
if not chart_output.exists():
get_logger().error("%s does not exist", chart_output)
return 1
output_dir = Path(args.output)
load_dir = chart_output / 'load'
apps = set()
attributes = set()
for f in load_dir.glob('ncp_load-*.csv'):
match = re.match(r'ncp_load-(?P<app>.*)-(?P<attr>\S+)\.csv', f.name)
if match:
apps.add(match.group('app'))
attributes.add(match.group('attr'))
else:
get_logger().debug("No match %s", f.name)
get_logger().info("apps %s", apps)
get_logger().info("attributes %s", attributes)
for app in apps:
for attr in attributes:
load_file = chart_output / 'load' / 'ncp_load-{0}-{1}.csv'.format(
app, attr)
load_data = pd.read_csv(load_file, na_values="?")
load_data.set_index('time')
load_data['time_minutes'] = (load_data['time'] -
load_data['time'].min()) / 1000 / 60
fig, ax = map_utils.subplots()
ax.set_title('service: {0} attribute: {1}'.format(app, attr))
ax.set_xlabel('time (minutes)')
ax.set_xlim(left=0, right=load_data['time_minutes'].max())
for col in load_data.columns.values:
if 'time' != col and 'time_minutes' != col:
plt.plot(load_data['time_minutes'], load_data[col])
handles, labels = ax.get_legend_handles_labels()
lgd = ax.legend(handles,
labels,
bbox_to_anchor=(1.04, 1),
loc="upper left")
output_name = output_dir / 'load-per-server-{0}-{1}.png'.format(
app, attr)
if args.interactive:
plt.show()
else:
plt.savefig(output_name.as_posix(),
format='png',
bbox_extra_artists=(lgd, ),
bbox_inches='tight')
plt.close(fig)
def main(argv=None):
if argv is None:
argv = sys.argv[1:]
parser = argparse.ArgumentParser()
parser.add_argument("-l",
"--logconfig",
dest="logconfig",
help="logging configuration (default: logging.json)",
default='logging.json')
parser.add_argument("-L",
"--log-directory",
dest="log_dir_path",
help="Log file directory",
required=True)
parser.add_argument(
"-b",
"--testbed",
dest="testbed",
help=
"The testbed used for the run. This is used to obtain time zone information.",
default='')
global min_discrepancy_interval_duration
parser.add_argument(
"-d",
"--min-interval-duration",
dest="min_duration",
help=
"The minimum missing data interval duration allowed for intervals to output",
default=str(min_discrepancy_interval_duration))
args = parser.parse_args(argv)
map_utils.setup_logging(default_path=args.logconfig)
# parse arguments
log_dir_path = Path(args.log_dir_path)
testbed = (None if args.testbed == '' else args.testbed)
min_discrepancy_interval_duration = float(args.min_duration)
time_zone = None
testbed_to_time_zone_map = {
'emulab': timezone('US/Mountain'),
'dcomp': timezone('UTC')
}
if (testbed != None):
if (testbed in testbed_to_time_zone_map):
time_zone = testbed_to_time_zone_map.get(testbed.lower())
get_logger().info("Found time zone for testbed '%s': %s.", testbed,
time_zone)
else:
get_logger().fatal("Could not map testbed '%s' to timezone.",
testbed)
exit(1)
parse_state = ParseState()
get_logger().info("Log directory: %s", log_dir_path)
logfiles = sorted(log_dir_path.glob("map-agent*.log"),
key=lambda f: f.stat().st_mtime)
with fileinput.input(files=logfiles) as log_file:
log_file_time_reference_line = log_file.readline()
ref_time = map_utils.log_line_to_time(log_file_time_reference_line,
time_zone)
if (ref_time == None):
get_logger().fatal(
"Failed to parse time in log statements. You might need to specify a testbed to determine the timezone or remove the testbed specification if the log statements include the timezone."
)
exit(1)
get_logger().info(" Ref line: %s",
log_file_time_reference_line.strip())
get_logger().info(" Ref time String: %s",
map_utils.datetime_to_string(ref_time))
get_logger().info(" Ref time epoch ms: %s",
str(datetime_to_epoch_ms(ref_time)))
get_logger().info("\n\n")
for line in log_file:
line_time = map_utils.log_line_to_time(line, time_zone)
if line_time:
parse_line(ref_time, line_time, line, parse_state)
clear_finished_flows(ref_time, line_time, parse_state)
check_for_network_discrepancies(ref_time, line_time,
parse_state)
else:
get_logger().debug("Line has no readable timestamp: %s", line)
get_logger().debug("parse_state.active_containers: %s",
str(parse_state.active_containers))
get_logger().info("Outputting Summary")
get_logger().info("Minutes with missing data: %s",
str(parse_state.container_nic_discrepancy_times))
disrepancy_initervals = compute_discrepancy_intervals(
parse_state.container_nic_discrepancy_times,
timedelta(seconds=min_time_between_discrepancy_intervals) /
timedelta(minutes=1))
filter_discrepancy_intervals(
disrepancy_initervals,
timedelta(seconds=min_discrepancy_interval_duration) /
timedelta(minutes=1))
get_logger().info("Intervals with missing data: %s",
str(disrepancy_initervals))
total_intervals = dict()
greater_than_1 = dict()
greater_than_3 = dict()
for node, node_data in disrepancy_initervals.items():
for nic, intervals in node_data.items():
total_intervals[node] = total_intervals.get(node,
0) + len(intervals)
for interval in intervals:
if interval.duration > 1:
greater_than_1[node] = greater_than_1.get(node, 0) + 1
if interval.duration > 3:
greater_than_3[node] = greater_than_3.get(node, 0) + 1
get_logger().info(
"Summary total intervals: %s intervals greater than 1 minute: %s intervals greater than 3 minutes %s",
total_intervals, greater_than_1, greater_than_3)
get_logger().debug("Final parse_state: %s", str(parse_state))
