def _GeneratorFromPath(path): """Create an event generator for file or directory at given path string.""" if not path: raise ValueError('path must be a valid string') if IsTensorFlowEventsFile(path): return event_file_loader.EventFileLoader(path) else: return directory_watcher.DirectoryWatcher( path, event_file_loader.EventFileLoader, IsTensorFlowEventsFile)
def _process_health_pill_event(self, node_name_set, mapping, target_step, file_path): """Creates health pills out of data in an event. Creates health pills out of the event and adds them to the mapping. Args: node_name_set: A set of node names that are relevant. mapping: The mapping from node name to event_accumulator.HealthPillEvents. This object may be destructively modified. target_step: The target step at which to obtain health pills. file_path: The path to the file with health pill events. Returns: Whether we should stop reading events because future events are no longer relevant. """ events_loader = event_file_loader.EventFileLoader(file_path) for event in events_loader.Load(): if not event.HasField('summary'): logging.warning( 'An event in a debugger events file lacks a summary.') continue if event.step < target_step: # This event is not of the relevant step. We perform this check # first because the majority of events will be eliminated from # consideration by this check. continue if event.step > target_step: # We have passed the relevant step. No need to read more events. return True for value in event.summary.value: # Since we seek health pills for a specific step, this function # returns 1 health pill per node per step. The wall time is the # seconds since the epoch. health_pill = self._process_health_pill_value( node_name_set, event.wall_time, event.step, value) if not health_pill: continue mapping[health_pill.node_name].append(health_pill) # Keep reading events. return False
def main(): global MAX_EVENT_NUM, EVENT_SPACING parser = argparse.ArgumentParser() parser.add_argument('-i', '--input', help="Input file name", required=True) parser.add_argument('-o', '--output', help="Output file name", required=True) parser.add_argument("-max", "--max_event_number", help="the limit number of event log to save", action="store", type=int) parser.add_argument("-space", "--max_space", help="the setting of event space", action="store", type=int) args = parser.parse_args() if args.max_event_number: MAX_EVENT_NUM = args.max_event_number if args.max_space: EVENT_SPACING = args.max_space inputfile = args.input outputfile = args.output # inputfile = '/data/dlp_test/mao_tbs/491/train/events.out.tfevents.1501763386.ai-work-2' # outputfile = '/data/dlp_test/mao_tbs/491/test' _generator = event_file_loader.EventFileLoader(inputfile) i = 0 # image_values = [] indexes = [] file_versions = [] graph_defs = [] meta_graph_defs = [] tagged_run_metadatas = [] for event in _generator.Load(): records = defaultdict(int) if event.HasField('summary'): count = 0 for value in event.summary.value: if value.HasField('image'): records['image'] += 1 value.ClearField('image') # print event.ListFields() elif value.HasField('simple_value'): records['simple_value'] += 1 # print value.tag elif value.HasField('histo'): records['histo'] += 1 elif value.HasField('obsolete_old_style_histogram'): records['old_style_histo'] += 1 elif value.HasField('audio'): records['audio'] += 1 elif value.HasField('tensor'): # almost no data records['tensor'] += 1 # print value.tag count += 1 elif event.HasField('file_version'): file_versions.append(i) elif event.HasField('graph_def'): graph_defs.append(i) elif event.HasField('meta_graph_def'): meta_graph_defs.append(i) elif event.HasField('tagged_run_metadata'): tagged_run_metadatas.append(i) indexes.append(len(records.keys())) i += 1 # for k in image_values: # print k, image_values[k]["id"], image_values[k]["count"] # events[image_values[k]["id"]].summary.value[image_values[k]["count"]].image.CopyFrom(image_values[k]["value"]) # print indexes print 'file_versions:', file_versions print 'graph_defs:', graph_defs print 'meta_graph_defs:', meta_graph_defs print 'tagged_run_metadatas:', tagged_run_metadatas meta_events = file_versions + graph_defs + meta_graph_defs + tagged_run_metadatas print "events:", len(indexes) # 结尾索引 max_field_len = max(indexes) max_index = list(reversed(indexes)).index(max_field_len) max_index = len(indexes) - max_index # indexes = [i for i in range(len(indexes)) if indexes[i] == max_field_len] # print "信息量多的event:", len(indexes) # store_list = find_max_equal_diffence_array(indexes) # print "event的最大等差数列:", store_list # compression_ratio = int(len(store_list) / MAX_EVENT_NUM) # 开头索引 index = 0 for i in range(len(indexes)): if indexes[i] >= min_field_num: index = i break # 算法基本假设: 所有信息量多event是以MAX_EVENT_NUM为间隔增长的,默认100 store_list = [i for i in range(index, max_index)] compression_ratio = int(len(store_list) / MAX_EVENT_NUM) compression_ratio = compression_ratio if compression_ratio else 1 print "数列的压缩率:", compression_ratio if compression_ratio > 1: spacing_public_contract = find_largest_public_contract(EVENT_SPACING) rate = int(compression_ratio / EVENT_SPACING) if rate > 1: setup_num = rate * EVENT_SPACING else: setup_num = filter(lambda x: x <= compression_ratio, spacing_public_contract)[-1] store_list = store_list[::setup_num] # for k in last_image: # events[store_list[-1]].summary.value[last_image[k]["count"]].image.CopyFrom(last_image[k]["value"]) # events = [events[i] for i in store_list] ew = event_file_writer.EventFileWriter(outputfile) out_json = defaultdict(list) i = 0 event_count = 1 _generator = event_file_loader.EventFileLoader(inputfile) for event in _generator.Load(): # 清除图片 if event.HasField('summary'): for value in event.summary.value: # 不删除最后一张图片 if value.HasField('image') and (i != store_list[-1]): value.ClearField('image') if i in store_list: event_count += 1 ew.add_event(event) for v in event.summary.value: if v.HasField('simple_value'): out_json[v.tag].append(v.simple_value) elif i in meta_events: event_count += 1 ew.add_event(event) i += 1 ew.flush() ew.close() print event_count
def generate_event_from_file(filepath): return event_file_loader.EventFileLoader(filepath).Load()
def generator_from_event_file(event_file): """Returns a generator that yields events from an event file.""" return event_file_loader.EventFileLoader(event_file).Load()
def _LoaderForTestFile(self, filename): return event_file_loader.EventFileLoader( os.path.join(self.get_temp_dir(), filename))