def assign_loglines_to_experiments(loglines, experiments, battor_power_syncs): logger.info("Assigning loglines to experiments") logcat_power_syncs = get_power_syncs(None, experiments=experiments) for idx, expt in enumerate(experiments): logger.info("\t%d/%d" % (idx, len(experiments))) for measurement in expt.get_measurements(): op = measurement.operation start = measurement.results.get(op + "Begin") end = measurement.results.get(op + "End") start_logline, start_idx = common.nearest_battor_logline_to_logcat_timestamp( loglines, start, battor_power_syncs, logcat_power_syncs ) end_logline, end_idx = common.nearest_battor_logline_to_logcat_timestamp( loglines, end, battor_power_syncs, logcat_power_syncs ) # Filter out all measurements that have current or voltage readings above normal # measurement_loglines = loglines[start_idx:end_idx] # if not all([x.current <= 1650 for x in measurement_loglines]) or \ # not all([x.voltage <= 7200 for x in measurement_loglines]): # iface_m = getattr(expt, measurement.interface) # logger.warn("Removing measurement due to high voltage/current") # setattr(iface_m, op, None) # else: if start_logline and end_logline: logger.info("\t\tAssigned %s" % (op)) setattr(measurement, "loglines", loglines[start_idx:end_idx]) setattr(measurement, "start_idx", start_idx) setattr(measurement, "end_idx", end_idx) else: logger.info("\t\tNo loglines found for measurement. Removing it") expt.remove_measurement(measurement) continue measurement.start_logline_idx = start_idx measurement.end_logline_idx = end_idx measurement.process()
def process(args): # First, we read the config config = json.loads(open(args.config).read()) # Get the list of files of each type battor_files, dev1_files, dev2_files, server_files, handover_files = post_processing.get_files(config) # Build schedule schedule = post_processing.get_schedule(args) user_dict = post_processing.build_user_dict(args) sorted_user_dict_by_value = pycommons.sort_dict(user_dict, 1) user_order = [x[0] for x in sorted_user_dict_by_value] # Create colors per user common.create_color_dict(user_order) overall_experiments = {} for date, b_f, d1_f, d2_f, s_f, h_f in itertools.izip(config['dates'], battor_files, dev1_files, dev2_files, server_files, handover_files): date_str = str(date) assert schedule[date], 'Schedule does not contain entry for date: %s' % (date_str) user = schedule[date] logger.info("Processing: %s (%s)" % (date_str, user)) # Merge all the separate log files together experiments = post_processing.merge_logs(date, d1_f, d2_f, s_f, h_f) # Find all power syncs as reported by logcat logcat_power_syncs = device1.get_power_syncs(None, experiments=experiments) # Logcat power syncs contain all the power syncs sorted by 'powerSyncBegin' key # Once we find the BattOr power syncs, we can line these up # Get all the BattOr loglines loglines = common.parse_battor_loglines(b_f, **config) sample_rate = common.get_sample_rate(b_f) down_sample = config['battor']['down_sample'] # Get the start and end of each BattOr power sync # This is only the start and end of the *edge* corresponding to 'powerSyncEnd' key # in the device1's logcat logs battor_power_syncs = common.find_battor_power_syncs(date, loglines, sample_rate=sample_rate, down_sample=down_sample, logcat_power_syncs=logcat_power_syncs, **config['edge']) # Now we have both logcat power syncs and BattOr power syncs # We now have a way to anchor BattOr loglines with android timestamps # Build a handover object for the day's file handover = Handover(h_f) for data in handover.data: os_timestamp = data.os_timestamp nearest_logline, logline_idx = common.nearest_battor_logline_to_logcat_timestamp(loglines, os_timestamp, battor_power_syncs, logcat_power_syncs)