def main(argv=None): if argv is None: argv = sys.argv[1:] class ArgumentParserWithDefaults(argparse.ArgumentParser): ''' From https://stackoverflow.com/questions/12151306/argparse-way-to-include-default-values-in-help ''' def add_argument(self, *args, help=None, default=None, **kwargs): if help is not None: kwargs['help'] = help if default is not None and args[0] != '-h': kwargs['default'] = default if help is not None: kwargs['help'] += ' (default: {})'.format(default) super().add_argument(*args, **kwargs) parser = ArgumentParserWithDefaults( formatter_class=argparse.RawTextHelpFormatter) parser.add_argument("-l", "--logconfig", dest="logconfig", help="logging configuration (default: logging.json)", default='logging.json') parser.add_argument("--debug", dest="debug", help="Enable interactive debugger on error", action='store_true') parser.add_argument("--demand", dest="demand", help="Directory to read the demand information from", required=True) parser.add_argument("--output", dest="output", help="Directory to write the demand information to", required=True) parser.add_argument("--step-size", dest="step_size", help="Number of milliseconds between samples", type=int, default=10000) args = parser.parse_args(argv) map_utils.setup_logging(default_path=args.logconfig) if 'multiprocessing' in sys.modules: import multiprocessing_logging multiprocessing_logging.install_mp_handler() if args.debug: import pdb, traceback try: return main_method(args) except: extype, value, tb = sys.exc_info() traceback.print_exc() pdb.post_mortem(tb) else: return main_method(args)
def main(argv=None): if argv is None: argv = sys.argv[1:] class ArgumentParserWithDefaults(argparse.ArgumentParser): ''' From https://stackoverflow.com/questions/12151306/argparse-way-to-include-default-values-in-help ''' def add_argument(self, *args, help=None, default=None, **kwargs): if help is not None: kwargs['help'] = help if default is not None and args[0] != '-h': kwargs['default'] = default if help is not None: kwargs['help'] += ' (default: {})'.format(default) super().add_argument(*args, **kwargs) parser = ArgumentParserWithDefaults( formatter_class=argparse.RawTextHelpFormatter) parser.add_argument("-l", "--logconfig", dest="logconfig", help="logging configuration (default: logging.json)", default='logging.json') parser.add_argument("--debug", dest="debug", help="Enable interactive debugger on error", action='store_true') parser.add_argument("-s", "--sim-output", dest="sim_output", help="Chart output directory (Required)", required=True) parser.add_argument("-o", "--output", dest="output", help="Output directory (Required)", required=True) parser.add_argument( "--first-timestamp-file", dest="first_timestamp_file", help= "Path to file containing the log timestamp that the simulation started", required=True) args = parser.parse_args(argv) map_utils.setup_logging(default_path=args.logconfig) if args.debug: import pdb, traceback try: return main_method(args) except: extype, value, tb = sys.exc_info() traceback.print_exc() pdb.post_mortem(tb) else: return main_method(args)
def main(argv=None): if argv is None: argv = sys.argv[1:] parser = argparse.ArgumentParser() parser.add_argument("-l", "--logconfig", dest="logconfig", help="logging configuration (default: logging.json)", default='logging.json') parser.add_argument("-s", "--sim-output", dest="sim_output", help="Hifi sim output directory (Required)", required=True) parser.add_argument("-o", "--output", dest="output", help="Output file (defaults to pull_stats.csv)", default='pull_stats.csv') args = parser.parse_args(argv) map_utils.setup_logging(default_path=args.logconfig) sim_dir = Path(args.sim_output) with open(args.output, 'w') as csvfile: writer = csv.writer(csvfile) writer.writerow([ 'node', 'initial_pull_timestamp', 'finished_timestamp', 'time_to_pull_seconds', 'attempts', 'result', 'image' ]) for server_dir in sim_dir.iterdir(): if server_dir.is_dir(): node = server_dir.name agent_dir = server_dir / 'agent' for agent_logfile in agent_dir.glob('map-agent*.log'): get_logger().debug("Processing %s", agent_logfile) process_logfile(writer, node, agent_logfile)
def main(argv=None): if argv is None: argv = sys.argv[1:] parser = argparse.ArgumentParser() parser.add_argument("-l", "--logconfig", dest="logconfig", help="logging configuration (default: logging.json)", default='logging.json') parser.add_argument( "--first-timestamp-file", dest="first_timestamp_file", help= "Path to file containing the log timestamp that the simulation started", required=True) parser.add_argument("-s", "--sim-output", dest="sim_output", help="Chart output directory (Required)", required=True) parser.add_argument("-o", "--output", dest="output", help="Output directory (Required)", required=True) parser.add_argument("-c", "--container-capacity", dest="container_queue_length_capacity", help="QueueLength capacity per container", default=None) parser.add_argument( "--process-latency-data", dest="process_latency_data", action='store_true', help= "If present, show queue length estimate from processing_latency.csv files." ) args = parser.parse_args(argv) map_utils.setup_logging(default_path=args.logconfig) with open(args.first_timestamp_file) as f: ts_str = f.readline().strip() first_timestamp = map_utils.log_timestamp_to_datetime(ts_str) first_timestamp_ms = first_timestamp.timestamp() * 1000 get_logger().info("Run started at %s", first_timestamp) sim_output = Path(args.sim_output) if not sim_output.exists(): get_logger().error("%s does not exist", sim_output) return 1 output = Path(args.output) output.mkdir(parents=True, exist_ok=True) container_queue_length_capacity = int( args.container_queue_length_capacity ) if args.container_queue_length_capacity else None process_latency_data = args.process_latency_data # app -> node_attr -> container_name -> timestamp -> value data_load = dict() data_demand = dict() container_processing_latency = dict() for node_dir in sim_output.iterdir(): if not node_dir.is_dir(): continue get_logger().debug("Processing node %s", node_dir) agent_dir = node_dir / 'agent' if not agent_dir.is_dir(): continue get_logger().debug("\tAgent dir %s", agent_dir) for node_name_dir in agent_dir.iterdir(): if not node_name_dir.is_dir(): continue for time_dir in node_name_dir.iterdir(): if not time_dir.is_dir(): continue get_logger().debug("\t\tProcessing time %s", time_dir) resource_report_file = time_dir / 'resourceReport-SHORT.json' if resource_report_file.exists(): try: with open(resource_report_file, 'r') as f: resource_report = json.load(f) time = int(time_dir.stem) if 'containerReports' in resource_report: for container_name, container_report in resource_report[ 'containerReports'].items(): container_name_short = container_name.split( '.')[0] process_container_report( data_load, data_demand, container_name_short, time, container_report) except json.decoder.JSONDecodeError: get_logger().warning("Problem reading %s, skipping", resource_report_file) container_data_dir = agent_dir / 'container_data' if process_latency_data: for container_data_service_dir in container_data_dir.iterdir(): match = re.search(r'^(.+)\.([^\._]+)_(.+)$', container_data_service_dir.name) if match: artifact = match.group(2) for container_data_service_name_dir in container_data_service_dir.iterdir( ): match = re.search(r'^([^\.]+)\.map\.dcomp$', container_data_service_name_dir.name) if match: container_name = match.group(1) for container_data_service_name_instance_dir in container_data_service_name_dir.iterdir( ): processing_latency_file = container_data_service_name_instance_dir / 'app_metrics_data' / 'processing_latency.csv' container_instance_latency_data = dict() with open(processing_latency_file, "r") as f: reader = csv.DictReader(f) for row in reader: event = row['event'] time_start = int(row['time_received']) time_end = int(row['time_ack_sent']) if event == 'request_success': container_instance_latency_data[ time_start] = 1 container_instance_latency_data[ time_end] = -1 container_processing_latency.setdefault( artifact, dict()).setdefault( container_name, list()).append( container_instance_latency_data) container_expected_queue_lengths = accumulate_expected_queue_length( container_processing_latency) output_graphs(first_timestamp_ms, output, data_load, container_expected_queue_lengths, "load", container_queue_length_capacity)
def main(argv=None): if argv is None: argv = sys.argv[1:] parser = argparse.ArgumentParser() parser.add_argument("-l", "--logconfig", dest="logconfig", help="logging configuration (default: logging.json)", default='logging.json') parser.add_argument("-c", "--chart-output", dest="chart_output", help="Output of MAPChartGeneration(Required)", required=True) parser.add_argument("-o", "--output", dest="output", help="Output directory (Required)", required=True) parser.add_argument("--interactive", dest="interactive", action="store_true", help="If specified, display the plots") args = parser.parse_args(argv) map_utils.setup_logging(default_path=args.logconfig) chart_output = Path(args.chart_output) if not chart_output.exists(): get_logger().error("%s does not exist", chart_output) return 1 output_dir = Path(args.output) output_dir.mkdir(parents=True, exist_ok=True) client_demand_dir = chart_output / 'client_demand' frames = list() for file in client_demand_dir.glob('num_clients-*.csv'): match = re.match(r'^num_clients-(.*)\.csv$', file.name) if not match: continue service = match.group(1) df = pd.read_csv(file) df['service'] = service frames.append(df) data = pd.concat(frames, ignore_index=True) data['time_minutes'] = (data['time'] - data['time'].min()) / 1000 / 60 all_services = data['service'].unique() fig, ax = map_utils.subplots() ax.set_title('number of clients per service over time') ax.set_ylabel('number of clients') ax.set_xlabel('time (minutes)') for service in all_services: plot_data = data.loc[(data['service'] == service)] ax.step(plot_data['time_minutes'], plot_data['num clients'], where='post', label=service) handles, labels = ax.get_legend_handles_labels() lgd = ax.legend(handles, labels, bbox_to_anchor=(1.04, 1), loc="upper left") if args.interactive: plt.show() else: output_name = output_dir / 'client_demand.png' plt.savefig(output_name.as_posix(), format='png', bbox_extra_artists=(lgd, ), bbox_inches='tight') plt.close(fig)
def main(argv=None): if argv is None: argv = sys.argv[1:] parser = argparse.ArgumentParser() parser.add_argument("-l", "--logconfig", dest="logconfig", help="logging configuration (default: logging.json)", default='logging.json') parser.add_argument("-c", "--chart_output", dest="chart_output", help="Chart output directory (Required)", required=True) parser.add_argument("-o", "--output", dest="output", help="Output directory (Required)", required=True) parser.add_argument( "--first-timestamp-file", dest="first_timestamp_file", help= "Path to file containing the log timestamp that the simulation started", required=True) parser.add_argument( "--sample-duration", dest="sample_duration", help= "Number of minutes between samples of plans for expected percentages plots", default=1, type=float) parser.add_argument("--scenario", dest="scenario", help="Scenario directory (Required)", required=True) args = parser.parse_args(argv) map_utils.setup_logging(default_path=args.logconfig) chart_output = Path(args.chart_output) if not chart_output.exists(): get_logger().error("%s does not exist", chart_output) return 1 with open(args.first_timestamp_file) as f: ts_str = f.readline().strip() first_timestamp = map_utils.log_timestamp_to_datetime(ts_str) get_logger().info("Simulation started at %s", first_timestamp) output_dir = Path(args.output) output_dir.mkdir(parents=True, exist_ok=True) dcop_file = chart_output / 'dcop_plan_updates/all_dcop_plans.csv' dcop_plans = pd.read_csv(dcop_file, na_values="?") dcop_plans.fillna(0, inplace=True) dcop_plans.sort_values(by=['time'], inplace=True) rlg_file = chart_output / 'rlg_plan_updates/all_rlg_overflow_plans.csv' rlg_plans = pd.read_csv(rlg_file, na_values="?") rlg_plans.fillna(0, inplace=True) rlg_plans.sort_values(by=['time'], inplace=True) dns_file = chart_output / 'dns_region_plan_updates/all_dns_region_plans.csv' dns_plans = pd.read_csv(dns_file, na_values="?") dns_plans.fillna(0, inplace=True) dns_plans.sort_values(by=['time'], inplace=True) first_timestamp_ms = first_timestamp.timestamp() * 1000 dcop_plans['time_minutes'] = (dcop_plans['time'] - first_timestamp_ms) / 1000 / 60 rlg_plans['time_minutes'] = (rlg_plans['time'] - first_timestamp_ms) / 1000 / 60 dns_plans['time_minutes'] = (dns_plans['time'] - first_timestamp_ms) / 1000 / 60 all_regions = np.union1d(dcop_plans['plan_region'].unique(), rlg_plans['plan_region'].unique()) all_services = np.union1d(dcop_plans['service'].unique(), rlg_plans['service'].unique()) for region in all_regions: for service in all_services: plot_region_service_plan_type(output_dir, "DCOP", dcop_plans, all_regions, region, service) plot_region_service_plan_type(output_dir, "RLG", rlg_plans, all_regions, region, service) plot_region_service_plan_type(output_dir, "DNS", dns_plans, all_regions, region, service) scenario_dir = Path(args.scenario) if not scenario_dir.exists(): get_logger().error("%s does not exist", scenario_dir) return 1 client_regions = compute_client_regions(scenario_dir) # plot expected percentages sample_duration = args.sample_duration for source_region in client_regions: for service in all_services: times, region_weights = compute_expected(sample_duration, all_regions, 'DCOP', dcop_plans, source_region, service) plot_expected(output_dir, 'DCOP', source_region, service, times, region_weights) times, region_weights = compute_expected(sample_duration, all_regions, 'RLG', rlg_plans, source_region, service) plot_expected(output_dir, 'RLG', source_region, service, times, region_weights) times, region_weights = compute_expected(sample_duration, all_regions, 'DNS', dns_plans, source_region, service) plot_expected(output_dir, 'DNS', source_region, service, times, region_weights) get_logger().info("Finished")
def main(argv=None): if argv is None: argv = sys.argv[1:] parser = argparse.ArgumentParser() parser.add_argument("-l", "--logconfig", dest="logconfig", help="logging configuration (default: logging.json)", default='logging.json') parser.add_argument("-L", "--log-file", dest="log_file_path", help="Log file with timestamps", required=True) parser.add_argument( "-b", "--testbed", dest="testbed", help= "The testbed used for the run. This is used to obtain time zone information.", default='') parser.add_argument("-s", "--run-seconds-start", dest="run_seconds_start", help="Number of seconds into run to start inspection", default='0') parser.add_argument( "-i", "--interval", dest="run_seconds_interval", help="Interval number of seconds within run to inspect", default='') parser.add_argument( "-d", "--identify-log-delays", dest="log_warn_delay", help= "Print a warning between consecutive log statements separated by a delay larger than the value specified in seconds.", default='') parser.add_argument("-f", "--regex-filter", dest="regex_filter", help="Pattern to use for filtering of log statements", default='') parser.add_argument( "-t", "--show-run-seconds", dest="show_run_seconds", action="store_true", help="Adds the number of seconds into the run to each log statement", default='') parser.add_argument("-o", "--output-file", dest="log_output_file_path", help="Log statements output from inspection interval", default='') parser.add_argument( "-p", "--occurrence-plot", dest="occurrence_plot_output_path", help="Create a graph showing occurrences of log messages in time", default='') parser.add_argument( "--first-timestamp-file", dest="first_timestamp_file", help= "Path to file containing the log timestamp that the simulation started", required=True) args = parser.parse_args(argv) map_utils.setup_logging(default_path=args.logconfig) # parse arguments log_file_path = args.log_file_path testbed = (None if args.testbed == '' else args.testbed) run_seconds_start = float(args.run_seconds_start) run_seconds_interval = (None if args.run_seconds_interval == '' else float( args.run_seconds_interval)) log_warn_delay = (None if args.log_warn_delay == '' else timedelta( seconds=float(args.log_warn_delay))) regex_filter = (None if args.regex_filter == '' else re.compile( args.regex_filter)) show_run_seconds = args.show_run_seconds log_output_file_path = args.log_output_file_path occurrence_plot_output_path = (None if args.occurrence_plot_output_path == '' else args.occurrence_plot_output_path) with open(args.first_timestamp_file) as f: ts_str = f.readline().strip() first_timestamp = map_utils.log_timestamp_to_datetime(ts_str) get_logger().info("Simulation started at %s", first_timestamp) time_zone = None #testbed_to_time_zone_map = {'emulab' : tz.tzoffset('MDT', -21600), 'dcomp' : tz.tzoffset('UTC', 0)} #testbed_to_time_zone_map = {'emulab' : timezone(-timedelta(hours=6), 'mdt'), 'dcomp' : timezone.utc} testbed_to_time_zone_map = { 'emulab': timezone('US/Mountain'), 'dcomp': timezone('UTC') } if (testbed != None): if (testbed in testbed_to_time_zone_map): time_zone = testbed_to_time_zone_map.get(testbed.lower()) get_logger().info("Found time zone for testbed '%s': %s.", testbed, time_zone) else: get_logger().fatal("Could not map testbed '%s' to timezone.", testbed) exit(1) get_logger().info("Log File: %s", log_file_path) with open(log_file_path, "r") as log_file: log_line_occurrence_times = list() ref_time = first_timestamp get_logger().info(" Ref time String: %s", map_utils.datetime_to_string(ref_time)) get_logger().info(" Ref time epoch ms: %s", str(datetime_to_epoch_ms(ref_time))) get_logger().info("") get_logger().info("") start_time_within_run = ref_time + timedelta(seconds=run_seconds_start) get_logger().info("At %s seconds into run: ", str(run_seconds_start)) get_logger().info(" Time within run: %s", map_utils.datetime_to_string(start_time_within_run)) get_logger().info(" Epoch ms: %s", str(datetime_to_epoch_ms(start_time_within_run))) get_logger().info("") end_time_within_run = (None if run_seconds_interval == None else ( ref_time + timedelta(seconds=(run_seconds_start + run_seconds_interval)))) if (end_time_within_run != None): get_logger().info("At " + str(run_seconds_start + run_seconds_interval) + " seconds into run: ") get_logger().info( " Time within run: %s", map_utils.datetime_to_string(end_time_within_run)) get_logger().info(" Epoch ms: %s", str(datetime_to_epoch_ms(end_time_within_run))) get_logger().info("\n\n\n") get_logger().info( "<==================== Log Output ====================>\n") if (log_output_file_path != ''): log_output_file = open(log_output_file_path, "w") log_output_file.write("Log statements for run time range \n") log_output_file.write(" Start Time (s): " + str(run_seconds_start) + "\n") if (run_seconds_interval != None): log_output_file.write(" End Time (s): " + str(run_seconds_start + run_seconds_interval) + "\n") log_output_file.write("\n") if (log_warn_delay != None): log_output_file.write( " Max time delay without warning (s): " + str(log_warn_delay) + "\n") log_output_file.write( "====================================================================================================\n\n\n\n" ) else: log_output_file = None log_lines_printed = 0 prev_line = '' prev_line_time = None last_line_matches = False for line in log_file: output_line = line line_time = map_utils.log_line_to_time(line, time_zone) if (line_time != None): if (line_time >= start_time_within_run and (regex_filter == None or re.search(regex_filter, line))): line_prefix = "" if (show_run_seconds): line_run_seconds = (line_time - ref_time).seconds output_line = str( line_run_seconds) + ": " + output_line if (log_warn_delay != None and prev_line_time != None): log_delay = line_time - prev_line_time if (log_output_file != None and log_delay > log_warn_delay): log_output_file.write("<--- Time delay " + str(log_delay) + " > " + str(log_warn_delay) + " --->\n") if (log_lines_printed < 1): get_logger().info(line) if (log_output_file != None): log_output_file.write(output_line) record_output_line_time(ref_time, line_time, log_line_occurrence_times) prev_line = line prev_line_time = line_time log_lines_printed += 1 last_line_matches = True else: last_line_matches = False if (end_time_within_run != None and line_time > end_time_within_run): break else: # write a statements without a valid timestamp if and only if the the most recent line with a valid time was printed # this allows multiline log statements such as exceptions to be printed entirely if (last_line_matches): if (log_output_file != None): log_output_file.write(output_line) get_logger().info(" ......") get_logger().info(prev_line) # close the output file if it was specified if (log_output_file != None): log_output_file.close() if (occurrence_plot_output_path != None): if (regex_filter == None): title = "Times of all log lines" else: title = "Times of log lines matching regex '" + regex_filter.pattern + "'" plot_log_line_occurrence(occurrence_plot_output_path, title, log_line_occurrence_times)
def main(argv=None): if argv is None: argv = sys.argv[1:] class ArgumentParserWithDefaults(argparse.ArgumentParser): ''' From https://stackoverflow.com/questions/12151306/argparse-way-to-include-default-values-in-help ''' def add_argument(self, *args, help=None, default=None, **kwargs): if help is not None: kwargs['help'] = help if default is not None and args[0] != '-h': kwargs['default'] = default if help is not None: kwargs['help'] += ' (default: {})'.format(default) super().add_argument(*args, **kwargs) parser = ArgumentParserWithDefaults( formatter_class=argparse.RawTextHelpFormatter) parser.add_argument("-l", "--logconfig", dest="logconfig", help="logging configuration (default: logging.json)", default='logging.json') parser.add_argument("--debug", dest="debug", help="Enable interactive debugger on error", action='store_true') parser.add_argument( "-i", "--input-file", dest="load_capacity_csv_file", help="Path to the input load-[attribute]-[region].csv file", required=True) args = parser.parse_args(argv) map_utils.setup_logging(default_path=args.logconfig) load_capacity_csv_file = Path(args.load_capacity_csv_file) with open(load_capacity_csv_file) as f: reader = csv.DictReader(f) areas = dict() start_time = None end_time = None prev_row = None prev_row_values = dict() for row in reader: get_logger().debug(f"row: {row}") current_time = float(row['relative minutes']) if not start_time: start_time = current_time end_time = current_time if prev_row: prev_time = float(prev_row['relative minutes']) delta_time = current_time - prev_time average_values_for_delta = dict() for label in row.keys(): if label != 'relative minutes': if label in prev_row_values: prev_value = float(prev_row_values[label]) current_value = float( row[label]) if row[label] != '' else prev_value average_values_for_delta[label] = (current_value + prev_value) / 2 for label in average_values_for_delta.keys(): value = average_values_for_delta[label] if label not in areas: areas[label] = 0 areas[label] += value * delta_time for label in row.keys(): if label != 'relative minutes': if row[label] != '': prev_row_values[label] = row[label] get_logger().debug(f"prev_row_values: {prev_row_values}") prev_row = row total_delta_time = end_time - start_time get_logger().info(f"time: {total_delta_time}") labels = list(areas.keys()) labels.sort() capacity_sum_label = "" capacity_sum = 0 load_sum_label = "" load_sum = 0 for label in labels: value = areas[label] value_per_time = value / total_delta_time get_logger().info(f"{label}") get_logger().info(f" area: {value}") get_logger().info( f" area / time: {value_per_time} = {value} / {total_delta_time}" ) if "capacity" in label: if not capacity_sum_label == "": capacity_sum_label += " + " capacity_sum_label += label capacity_sum += value else: if not load_sum_label == "": load_sum_label += " + " load_sum_label += label load_sum += value get_logger().info(f"{capacity_sum_label}") get_logger().info(f" area: {capacity_sum}") get_logger().info( f" area / time: {capacity_sum / total_delta_time} = {capacity_sum} / {total_delta_time}" ) get_logger().info(f"{load_sum_label}") get_logger().info(f" area: {load_sum}") get_logger().info( f" area / time: {load_sum / total_delta_time} = {load_sum} / {total_delta_time}" )
def main(argv=None): if argv is None: argv = sys.argv[1:] class ArgumentParserWithDefaults(argparse.ArgumentParser): ''' From https://stackoverflow.com/questions/12151306/argparse-way-to-include-default-values-in-help ''' def add_argument(self, *args, help=None, default=None, **kwargs): if help is not None: kwargs['help'] = help if default is not None and args[0] != '-h': kwargs['default'] = default if help is not None: kwargs['help'] += ' (default: {})'.format(default) super().add_argument(*args, **kwargs) parser = ArgumentParserWithDefaults( formatter_class=argparse.RawTextHelpFormatter) parser.add_argument("-l", "--logconfig", dest="logconfig", help="logging configuration (default: logging.json)", default='logging.json') parser.add_argument("--debug", dest="debug", help="Enable interactive debugger on error", action='store_true') parser.add_argument("--input", dest="input", help="Path to all_client_requests.csv", required=True) parser.add_argument( "--first-timestamp-file", dest="first_timestamp_file", help= "Path to file containing the log timestamp that the simulation started", required=True) parser.add_argument("-o", "--output", dest="output", help="Output directory", required=True) parser.add_argument("-s", "--sim-output", dest="sim_output", help="Sim output directory (Required)", required=True) parser.add_argument( "--window-duration", dest="window_duration", help= "Number of minutes to use for the window to compute percentages over", default=1, type=float) args = parser.parse_args(argv) map_utils.setup_logging(default_path=args.logconfig) with open(args.first_timestamp_file) as f: ts_str = f.readline().strip() first_timestamp = map_utils.log_timestamp_to_datetime(ts_str) get_logger().info("Simulation started at %s", first_timestamp) all_requests_path = Path(args.input) if not all_requests_path.exists(): get_logger().error("Cannot read all requests file at %s", all_requests_path) return -1 output = Path(args.output) output.mkdir(parents=True, exist_ok=True) sim_output = Path(args.sim_output) regions = dict() window_duration = datetime.timedelta(minutes=args.window_duration) scenario_dir = sim_output / 'inputs' / 'scenario' node_to_region_dict = map_utils.gather_region_info(scenario_dir) _, ip_to_node_dict = summarize_hifi.parse_node_info(sim_output) get_logger().debug(f"node_to_region_dict = {node_to_region_dict}") get_logger().debug(f"ip_to_node_dict = {ip_to_node_dict}") # region --> service --> window --> ClientDnsRequestCounts requests_region_service_counts = dict() read_all_requests_file(all_requests_path, first_timestamp, window_duration, node_to_region_dict, requests_region_service_counts) read_weighted_dns_files(sim_output, first_timestamp, window_duration, ip_to_node_dict, requests_region_service_counts) output_requests_csv_per_region(requests_region_service_counts, first_timestamp, output) output_requests_graph_per_region(requests_region_service_counts, first_timestamp, output)
def main(argv=None): if argv is None: argv = sys.argv[1:] parser = argparse.ArgumentParser() parser.add_argument("-l", "--logconfig", dest="logconfig", help="logging configuration (default: logging.json)", default='logging.json') parser.add_argument("-c", "--chart_output", dest="chart_output", help="Chart output directory (Required)", required=True) parser.add_argument("-o", "--output", dest="output", help="Output directory (Required)", required=True) args = parser.parse_args(argv) map_utils.setup_logging(default_path=args.logconfig) chart_output = Path(args.chart_output) if not chart_output.exists(): get_logger().error("%s does not exist", chart_output) return 1 output_dir = Path(args.output) output_dir.mkdir(parents=True, exist_ok=True) chart_output_dns = chart_output / 'dns' get_logger().debug("chart_output_dns: %s", chart_output_dns) all_regions = set() all_services = set() service_region_dns_resolution_counts = dict() earliest_time = None for dns_req_count_file in chart_output_dns.glob('dns_req_count_*.csv'): match = re.match("dns_req_count_(.+)--(.+)\.csv", os.path.basename(dns_req_count_file)) if match: region = match.group(1) service = match.group(2) get_logger().info( "Found file '%s' for region '%s' and service '%s'", dns_req_count_file, region, service) all_regions.add(region) all_services.add(service) dns_resolution_counts = pd.read_csv(dns_req_count_file, na_values="?") service_region_dns_resolution_counts.setdefault( service, dict())[region] = dns_resolution_counts if earliest_time is None: earliest_time = dns_resolution_counts['time'].min() else: earliest_time = min(earliest_time, dns_resolution_counts['time'].min()) for service in all_services: for region in all_regions: service_region_dns_resolution_counts[service][region][ 'time_minutes'] = (service_region_dns_resolution_counts[ service][region]['time'] - earliest_time) / 1000 / 60 get_logger().info("Output plots for service '%s' and region '%s'", service, region) plot_dns_region_resolution_counts( output_dir, service_region_dns_resolution_counts[service][region], all_regions, region, service) plot_dns_container_resolution_counts( output_dir, service_region_dns_resolution_counts[service][region], region, service)
def main(argv=None): if argv is None: argv = sys.argv[1:] parser = argparse.ArgumentParser() parser.add_argument("-l", "--logconfig", dest="logconfig", help="logging configuration (default: logging.json)", default='logging.json') parser.add_argument("-s", "--sim-output", dest="sim_output", help="Chart output directory (Required)", required=True) parser.add_argument("-o", "--output", dest="output", help="Output directory (Required)", required=True) parser.add_argument( "--first-timestamp-file", dest="first_timestamp_file", help= "Path to file containing the log timestamp that the simulation started", required=True) args = parser.parse_args(argv) map_utils.setup_logging(default_path=args.logconfig) sim_output = Path(args.sim_output) if not sim_output.exists(): get_logger().error("%s does not exist", sim_output) return 1 with open(args.first_timestamp_file) as f: ts_str = f.readline().strip() first_timestamp = map_utils.log_timestamp_to_datetime(ts_str) get_logger().info("Simulation started at %s", first_timestamp) first_timestamp_ms = first_timestamp.timestamp() * 1000 output = Path(args.output) output.mkdir(parents=True, exist_ok=True) # app -> node_attr -> container_name -> timestamp -> value data_load = dict() data_demand = dict() for node_dir in sim_output.iterdir(): if not node_dir.is_dir(): continue get_logger().debug("Processing node %s", node_dir) agent_dir = node_dir / 'agent' if agent_dir.is_dir(): get_logger().debug("\tAgent dir %s", agent_dir) for node_name_dir in agent_dir.iterdir(): if not node_name_dir.is_dir(): continue for time_dir in node_name_dir.iterdir(): if not time_dir.is_dir(): continue add_time_dir_data(time_dir, data_load, data_demand) else: for time_dir in node_dir.iterdir(): if not time_dir.is_dir(): continue add_time_dir_data(time_dir, data_load, data_demand) output_graphs(first_timestamp_ms, output, data_load, "load") output_graphs(first_timestamp_ms, output, data_demand, "demand")
def main(argv=None): if argv is None: argv = sys.argv[1:] parser = argparse.ArgumentParser() parser.add_argument("-l", "--logconfig", dest="logconfig", help="logging configuration (default: logging.json)", default='logging.json') parser.add_argument("-c", "--chart_output", dest="chart_output", help="Chart output directory (Required)", required=True) parser.add_argument("-o", "--output", dest="output", help="Output directory (Required)", required=True) parser.add_argument("--interactive", dest="interactive", action="store_true", help="If specified, display the plots") args = parser.parse_args(argv) map_utils.setup_logging(default_path=args.logconfig) chart_output = Path(args.chart_output) if not chart_output.exists(): get_logger().error("%s does not exist", chart_output) return 1 output_dir = Path(args.output) load_dir = chart_output / 'load' apps = set() attributes = set() for f in load_dir.glob('ncp_load-*.csv'): match = re.match(r'ncp_load-(?P<app>.*)-(?P<attr>\S+)\.csv', f.name) if match: apps.add(match.group('app')) attributes.add(match.group('attr')) else: get_logger().debug("No match %s", f.name) get_logger().info("apps %s", apps) get_logger().info("attributes %s", attributes) for app in apps: for attr in attributes: load_file = chart_output / 'load' / 'ncp_load-{0}-{1}.csv'.format( app, attr) load_data = pd.read_csv(load_file, na_values="?") load_data.set_index('time') load_data['time_minutes'] = (load_data['time'] - load_data['time'].min()) / 1000 / 60 fig, ax = map_utils.subplots() ax.set_title('service: {0} attribute: {1}'.format(app, attr)) ax.set_xlabel('time (minutes)') ax.set_xlim(left=0, right=load_data['time_minutes'].max()) for col in load_data.columns.values: if 'time' != col and 'time_minutes' != col: plt.plot(load_data['time_minutes'], load_data[col]) handles, labels = ax.get_legend_handles_labels() lgd = ax.legend(handles, labels, bbox_to_anchor=(1.04, 1), loc="upper left") output_name = output_dir / 'load-per-server-{0}-{1}.png'.format( app, attr) if args.interactive: plt.show() else: plt.savefig(output_name.as_posix(), format='png', bbox_extra_artists=(lgd, ), bbox_inches='tight') plt.close(fig)
def main(argv=None): if argv is None: argv = sys.argv[1:] parser = argparse.ArgumentParser() parser.add_argument("-l", "--logconfig", dest="logconfig", help="logging configuration (default: logging.json)", default='logging.json') parser.add_argument("-L", "--log-directory", dest="log_dir_path", help="Log file directory", required=True) parser.add_argument( "-b", "--testbed", dest="testbed", help= "The testbed used for the run. This is used to obtain time zone information.", default='') global min_discrepancy_interval_duration parser.add_argument( "-d", "--min-interval-duration", dest="min_duration", help= "The minimum missing data interval duration allowed for intervals to output", default=str(min_discrepancy_interval_duration)) args = parser.parse_args(argv) map_utils.setup_logging(default_path=args.logconfig) # parse arguments log_dir_path = Path(args.log_dir_path) testbed = (None if args.testbed == '' else args.testbed) min_discrepancy_interval_duration = float(args.min_duration) time_zone = None testbed_to_time_zone_map = { 'emulab': timezone('US/Mountain'), 'dcomp': timezone('UTC') } if (testbed != None): if (testbed in testbed_to_time_zone_map): time_zone = testbed_to_time_zone_map.get(testbed.lower()) get_logger().info("Found time zone for testbed '%s': %s.", testbed, time_zone) else: get_logger().fatal("Could not map testbed '%s' to timezone.", testbed) exit(1) parse_state = ParseState() get_logger().info("Log directory: %s", log_dir_path) logfiles = sorted(log_dir_path.glob("map-agent*.log"), key=lambda f: f.stat().st_mtime) with fileinput.input(files=logfiles) as log_file: log_file_time_reference_line = log_file.readline() ref_time = map_utils.log_line_to_time(log_file_time_reference_line, time_zone) if (ref_time == None): get_logger().fatal( "Failed to parse time in log statements. You might need to specify a testbed to determine the timezone or remove the testbed specification if the log statements include the timezone." ) exit(1) get_logger().info(" Ref line: %s", log_file_time_reference_line.strip()) get_logger().info(" Ref time String: %s", map_utils.datetime_to_string(ref_time)) get_logger().info(" Ref time epoch ms: %s", str(datetime_to_epoch_ms(ref_time))) get_logger().info("\n\n") for line in log_file: line_time = map_utils.log_line_to_time(line, time_zone) if line_time: parse_line(ref_time, line_time, line, parse_state) clear_finished_flows(ref_time, line_time, parse_state) check_for_network_discrepancies(ref_time, line_time, parse_state) else: get_logger().debug("Line has no readable timestamp: %s", line) get_logger().debug("parse_state.active_containers: %s", str(parse_state.active_containers)) get_logger().info("Outputting Summary") get_logger().info("Minutes with missing data: %s", str(parse_state.container_nic_discrepancy_times)) disrepancy_initervals = compute_discrepancy_intervals( parse_state.container_nic_discrepancy_times, timedelta(seconds=min_time_between_discrepancy_intervals) / timedelta(minutes=1)) filter_discrepancy_intervals( disrepancy_initervals, timedelta(seconds=min_discrepancy_interval_duration) / timedelta(minutes=1)) get_logger().info("Intervals with missing data: %s", str(disrepancy_initervals)) total_intervals = dict() greater_than_1 = dict() greater_than_3 = dict() for node, node_data in disrepancy_initervals.items(): for nic, intervals in node_data.items(): total_intervals[node] = total_intervals.get(node, 0) + len(intervals) for interval in intervals: if interval.duration > 1: greater_than_1[node] = greater_than_1.get(node, 0) + 1 if interval.duration > 3: greater_than_3[node] = greater_than_3.get(node, 0) + 1 get_logger().info( "Summary total intervals: %s intervals greater than 1 minute: %s intervals greater than 3 minutes %s", total_intervals, greater_than_1, greater_than_3) get_logger().debug("Final parse_state: %s", str(parse_state))