def write_stats(ranges, results, prefix="stats"): attrs = list(ranges.keys()) df = DataFile( os.path.join(base_path, '%s_%s.dat' % (prefix, '_'.join(attrs))), ['entries', 'lookups', 'hits', 'total time', 'lookup time', 'scans'] + attr_names("iter time") + attr_names("iter thruput") + attr_names("file size") + attr_names("scan thruput") + attrs, overwrite=True) for config, reps in results: cfg = dict(config) # TODO: avg. over the reps iter_times = Entry("iter time") iter_tps = Entry("iter thruput") file_sizes = Entry("file size") scan_tps = Entry("scan thruput") for data in reps: db_path, entries, lookups, hits, total_time, num_scans, scans_time, lookup_time, iter_time, iter_tp = data.split( ", ") iter_times.add(int(iter_time)) iter_tps.add(int(iter_tp)) file_sizes.add(int(os.path.getsize(db_path))) scan_tps.add(float(scans_time) / float(num_scans)) for attr in ranges: df[attr] = cfg[attr] df['entries'] = entries df['lookups'] = lookups df['hits'] = hits df['total time'] = total_time df['lookup time'] = lookup_time df['scans'] = num_scans # df['file size'] = os.path.getsize(db_path) for attr in attr_names("iter time"): df[attr] = getattr(iter_times, attr.split()[-1]) for attr in attr_names("iter thruput"): df[attr] = getattr(iter_tps, attr.split()[-1]) for attr in attr_names("file size"): df[attr] = getattr(file_sizes, attr.split()[-1]) for attr in attr_names("scan thruput"): df[attr] = getattr(scan_tps, attr.split()[-1]) print file_sizes print iter_times print iter_tps print scan_tps df.save() df.close()
def save(self): print "saving summary of '" + self.getName() + "'..." summaryDir = self.getSummaryDir() if os.path.isdir(summaryDir): shutil.rmtree(summaryDir, True, False) mkdirRec(summaryDir) dataFile = DataFile() filename = "" for worldType in AVAILABLE_WORLDS: print "-- saving data of '" + worldType + "'" worldDir = os.path.join(summaryDir,worldType) mkdirRec(worldDir) for robotCount in AVAILABLE_ROBOT_COUNTS: if self.meanCoverageEvents_[worldType][robotCount].hasData(): filename = self.getSaveFilename(worldDir, COVERAGE_EVENTS_FILE, robotCount) dataFile.reset() dataFile.addComment("# [coverage meanTimestamp(usec)]") data = self.meanCoverageEvents_[worldType][robotCount].getMean() dataFile.setDataAs(data[:2], "fl") dataFile.save(filename) if self.meanTimeEvents_[worldType][robotCount].hasData(): filename = self.getSaveFilename(worldDir, TIME_EVENTS_FILE, robotCount) dataFile.reset() dataFile.addComment("# [coverage meanTimestamp(usec)]") data = self.meanTimeEvents_[worldType][robotCount].getMean() dataFile.setDataAs(data[:2], "fl") dataFile.save(filename) if self.meanTileTimeBetweenVisits_[worldType][robotCount].hasData(): filename = self.getSaveFilename(worldDir, MEAN_TILE_TIME_BEWTEEN_VISITS_FILE, robotCount) dataFile.reset() dataFile.addComment("# [x y meanTimeBetweenVisits(usec)]") data = self.meanTileTimeBetweenVisits_[worldType][robotCount].getMeanGrid() dataFile.setDataAs(data[:3], "iil") dataFile.save(filename) if self.meanTileVisits_[worldType][robotCount].hasData(): filename = self.getSaveFilename(worldDir, TILE_VISITS_FILE, robotCount) dataFile.reset() dataFile.addComment("# [x y meanVisitCount]") data = self.meanTileVisits_[worldType][robotCount].getMeanGrid() dataFile.setDataAs(data[:3], "iif") dataFile.save(filename) if self.meanFinalCoverage_[worldType][robotCount].hasData(): filename = self.getSaveFilename(worldDir, FINAL_COVERAGE_FILE, robotCount) dataFile.reset() dataFile.addComment("# [coverage timeStamp(usec)]") data = self.meanFinalCoverage_[worldType][robotCount].getMean() dataFile.setDataAs(data[:2], "fl") dataFile.save(filename)
def write_stats(ranges, results, prefix="stats"): attrs = list(ranges.keys()) df = DataFile(os.path.join(base_path, '%s_%s.dat' % (prefix, '_'.join(attrs))), ['entries', 'lookups', 'hits', 'total time', 'lookup time', 'scans'] + attr_names("iter time") + attr_names("iter thruput") + attr_names("file size") + attr_names("scan thruput") + attrs, overwrite=True) for config, reps in results: cfg = dict(config) # TODO: avg. over the reps iter_times = Entry("iter time") iter_tps = Entry("iter thruput") file_sizes = Entry("file size") scan_tps = Entry("scan thruput") for data in reps: db_path, entries, lookups, hits, total_time, num_scans, scans_time, lookup_time, iter_time, iter_tp = data.split(", ") iter_times.add(int(iter_time)) iter_tps.add(int(iter_tp)) file_sizes.add(int(os.path.getsize(db_path))) scan_tps.add(float(scans_time)/float(num_scans)) for attr in ranges: df[attr] = cfg[attr] df['entries'] = entries df['lookups'] = lookups df['hits'] = hits df['total time'] = total_time df['lookup time'] = lookup_time df['scans'] = num_scans # df['file size'] = os.path.getsize(db_path) for attr in attr_names("iter time"): df[attr] = getattr(iter_times, attr.split()[-1]) for attr in attr_names("iter thruput"): df[attr] = getattr(iter_tps, attr.split()[-1]) for attr in attr_names("file size"): df[attr] = getattr(file_sizes, attr.split()[-1]) for attr in attr_names("scan thruput"): df[attr] = getattr(scan_tps, attr.split()[-1]) print file_sizes print iter_times print iter_tps print scan_tps df.save() df.close()
def exec_ranges(cmd, args, range_attrs, num_reps=10, pbs_queue=None, pbs_name='storagesim', config=None): # range_attrs is in the form: # range_val: [list of possible values] # # when several range_vals and value lists are given, all possible # permutations are executed base_path = args['base_dir'] if config != None: df = DataFile(os.path.join(config, 'config_ranges.dat'), list(range_attrs.keys()), overwrite=True) outputs = [] for attr_p in permutations(range_attrs): # set the attrs accordingly for this exec names = [] for attr, val in attr_p: args[attr] = val names.append('%s_%s' % (str(attr), str(val))) if config != None: df[attr] = val if config != None: df.save() args['base_dir'] = os.path.join(base_path, '%s' % ('_'.join(names))) if not os.path.exists(args['base_dir']): os.mkdir(args['base_dir']) outputs.append((attr_p, exec_repetitions(cmd, args, num_reps=num_reps, pbs_queue=pbs_queue, pbs_name=pbs_name))) if config != None: df.close() return outputs
def read_and_save_flash(self, filename, with_bootloader): _df = DataFile([]) _df.data = self.read_all_flash(with_bootloader) print 'Read', len(_df.data), 'bytes' _df.save(filename)
#write 234000 256.21552423 us/byte # 255.434597666 us/byte #l = Loader('/dev/tty.wchusbserial14230', 57600) #l = Loader('/dev/tty.wchusbserial14230', 230400) l = Loader('/dev/tty.wchusbserial14220', 153600) print l.dev.get_about() l.read_and_save_flash('flash_with_loader.hex', True) l.read_and_save_flash('flash_without_loader.hex', False) l.write_flash(fw.data) l.dev.start_app() 1 / 0 # df = DataFile([]) # # df.load('flash.hex') # df.save('flash2.hex') dev = Bootloader('/dev/tty.wchusbserial14230', 57600) print dev.sync(10) print dev.get_about() eeprom = dev.read_eeprom(0, 1024) # flash = dev.read_flash(0x7000, 0x1000) flash = dev.read_flash(0x0000, 32 * 1024) # print_dump(flash) print_dump(flash) df = DataFile(flash) df.save('flash.hex') dev.start_app()
class ProjectData: def __init__(self, project): self.folder = os.path.dirname(project) self.project_file = project self.current_file = 0 self.current_label = 0 self.modified = False self.bad_files = [] self.datafile = None self.config = None self.read_conf() self.read_file() def read(self): if self.modified: self.read_conf() self.modified = False self.read_file() if self.current_label >= len(self.config["labels"]): self.current_label = 0 def read_conf(self): try: with open(self.project_file) as in_file: self.config = json.load(in_file) except IOError: logger.error("Unable to read {}: permission denied".format( self.project_file)) exit(2) def read_file(self): if set(self.config["files"]) == set(self.bad_files): dialogs.report_no_files() exit(3) current = self.config["files"][self.current_file] file_path = os.path.join(self.folder, current) if current in self.bad_files or not os.path.exists(file_path): if not os.path.exists(file_path): self.datafile = None self.bad_files.append(current) dialogs.notify_read_error(current) self.next_file() self.read_file() return try: self.datafile = DataFile(file_path, self.config["labels"]) self.insert_header() except (UnrecognizedFormatError, BadFileError): self.datafile = None self.bad_files.append(current) dialogs.notify_read_error(current) self.next_file() self.read_file() def insert_header(self): header = self.datafile.get_data_header() if str(header) not in self.config.keys(): self.config[str(header)] = { "plot": [[i] for i in self.datafile.get_data_columns()], "normalize": [], "functions": [] } self.modified = True self.save_config() def save_file(self): self.datafile.save() def save_config(self): if self.modified: try: with open(self.project_file, 'w') as out_file: json.dump(self.config, out_file) except IOError: logger.error("Unable to write {}: permission denied".format( self.project_file)) exit(2) self.modified = False def next_label(self): self.current_label = (self.current_label + 1) % len( self.config["labels"]) def prev_label(self): length = len(self.config["labels"]) self.current_label = (self.current_label - 1 + length) % length def get_current_label(self): label = self.config["labels"][self.current_label] color = self.config["colors"][self.current_label] return label, color def get_label_color(self, label): index = self.config["labels"].index(label) return self.config["colors"][index] def get_labels_info(self): return self.config["labels"], self.config["colors"] def set_labels_info(self, names, colors): self.config["labels"] = names self.config["colors"] = colors self.modified = True if self.current_label >= len(names): self.current_label = 0 def get_plot_info(self): header = self.datafile.get_data_header() conf = self.config[str(header)] return conf["plot"], conf["normalize"] def set_plot_info(self, plot_set, normalize): header = self.datafile.get_data_header() conf = self.config[str(header)] conf["plot"] = plot_set conf["normalize"] = normalize self.modified = True def next_file(self): self.current_file = (self.current_file + 1) % len(self.config["files"]) def prev_file(self): self.current_file = (self.current_file - 1 + len( self.config["files"])) % len(self.config["files"]) def get_functions(self): header = self.datafile.get_data_header() return self.config[str(header)]["functions"] def add_function(self, fs): header = self.datafile.get_data_header() new_header = header + [fs.name] self.config[str(new_header)] = self.config[str(header)] self.config[str(new_header)]["functions"].append(fs.name) self.datafile.add_function(fs) self.modified = True def remove_function(self, index): header = self.datafile.get_data_header() conf = self.config[str(header)] f_name = conf["functions"][index] f_col = self.datafile.get_data_columns()[header.index(f_name)] for plot in conf["plot"]: if f_col in plot: plot.remove(f_col) for i, col in enumerate(plot): if col > f_col: plot[i] = col - 1 self.datafile.remove_function(f_name) del conf["functions"][index] self.modified = True
class FilesData: def __init__(self, files): self.files_list = files self.config_list = [] self.current_file = 0 self.current_label = 0 self.modified = False self.bad_files = [] self.datafile = None self.config = None self.init() self.read() def init(self): for file in self.files_list: conf = file + ".json" file_conf = conf if os.path.exists(conf) else None self.config_list.append(file_conf) # noinspection PyTypeChecker def read(self): conf_path = self.config_list[self.current_file] if conf_path: self.read_conf() self.read_file() else: self.config = { "labels": ["Label"], "colors": ["#1f77b4"], } self.read_file() self.config["plot"] = [[i] for i in self.datafile.get_data_columns()] self.config["normalize"] = [] self.config["functions"] = [] if self.current_label >= len(self.config["labels"]): self.current_label = 0 self.modified = False def read_conf(self): conf_path = self.config_list[self.current_file] try: with open(conf_path) as in_file: self.config = json.load(in_file) except IOError: logger.error( "Unable to read {}: permission denied".format(conf_path)) exit(2) def read_file(self): if set(self.files_list) == set(self.bad_files): dialogs.report_no_files() exit(3) current = self.files_list[self.current_file] if current in self.bad_files: self.next_file() self.read_file() return try: self.datafile = DataFile(current, self.config["labels"]) except (UnrecognizedFormatError, BadFileError): self.datafile = None self.bad_files.append(current) dialogs.notify_read_error(os.path.basename(current)) self.next_file() self.read_file() def save_file(self): self.datafile.save() def save_config(self): if self.modified: conf_path = self.files_list[self.current_file] + ".json" try: with open(conf_path, 'w') as out_file: json.dump(self.config, out_file) except IOError: logger.error( "Unable to write {}: permission denied".format(conf_path)) exit(2) self.config_list[self.current_file] = conf_path self.modified = False def next_label(self): self.current_label = (self.current_label + 1) % len( self.config["labels"]) def prev_label(self): length = len(self.config["labels"]) self.current_label = (self.current_label - 1 + length) % length def get_current_label(self): label = self.config["labels"][self.current_label] color = self.config["colors"][self.current_label] return label, color def get_label_color(self, label): index = self.config["labels"].index(label) return self.config["colors"][index] def get_labels_info(self): return self.config["labels"], self.config["colors"] def set_labels_info(self, names, colors): self.config["labels"] = names self.config["colors"] = colors self.modified = True if self.current_label >= len(names): self.current_label = 0 def get_plot_info(self): return self.config["plot"], self.config["normalize"] def set_plot_info(self, plot_set, normalize): self.config["plot"] = plot_set self.config["normalize"] = normalize self.modified = True def next_file(self): self.current_file = (self.current_file + 1) % len(self.files_list) def prev_file(self): self.current_file = (self.current_file - 1 + len(self.files_list)) % len(self.files_list) def get_functions(self): return self.config["functions"] def add_function(self, fs): self.datafile.add_function(fs) self.config["functions"].append(fs.name) self.modified = True def remove_function(self, index): f_name = self.config["functions"][index] header = self.datafile.get_data_header() f_col = self.datafile.get_data_columns()[header.index(f_name)] for plot in self.config["plot"]: if f_col in plot: plot.remove(f_col) for i, col in enumerate(plot): if col > f_col: plot[i] = col - 1 self.datafile.remove_function(f_name) del self.config["functions"][index] self.modified = True