def write_stats(ranges, results, prefix="stats"):
attrs = list(ranges.keys())
df = DataFile(
os.path.join(base_path, '%s_%s.dat' % (prefix, '_'.join(attrs))),
['entries', 'lookups', 'hits', 'total time', 'lookup time', 'scans'] +
attr_names("iter time") + attr_names("iter thruput") +
attr_names("file size") + attr_names("scan thruput") + attrs,
overwrite=True)
for config, reps in results:
cfg = dict(config)
# TODO: avg. over the reps
iter_times = Entry("iter time")
iter_tps = Entry("iter thruput")
file_sizes = Entry("file size")
scan_tps = Entry("scan thruput")
for data in reps:
db_path, entries, lookups, hits, total_time, num_scans, scans_time, lookup_time, iter_time, iter_tp = data.split(
", ")
iter_times.add(int(iter_time))
iter_tps.add(int(iter_tp))
file_sizes.add(int(os.path.getsize(db_path)))
scan_tps.add(float(scans_time) / float(num_scans))
for attr in ranges:
df[attr] = cfg[attr]
df['entries'] = entries
df['lookups'] = lookups
df['hits'] = hits
df['total time'] = total_time
df['lookup time'] = lookup_time
df['scans'] = num_scans
# df['file size'] = os.path.getsize(db_path)
for attr in attr_names("iter time"):
df[attr] = getattr(iter_times, attr.split()[-1])
for attr in attr_names("iter thruput"):
df[attr] = getattr(iter_tps, attr.split()[-1])
for attr in attr_names("file size"):
df[attr] = getattr(file_sizes, attr.split()[-1])
for attr in attr_names("scan thruput"):
df[attr] = getattr(scan_tps, attr.split()[-1])
print file_sizes
print iter_times
print iter_tps
print scan_tps
df.save()
df.close()
def save(self):
print "saving summary of '" + self.getName() + "'..."
summaryDir = self.getSummaryDir()
if os.path.isdir(summaryDir):
shutil.rmtree(summaryDir, True, False)
mkdirRec(summaryDir)
dataFile = DataFile()
filename = ""
for worldType in AVAILABLE_WORLDS:
print "-- saving data of '" + worldType + "'"
worldDir = os.path.join(summaryDir,worldType)
mkdirRec(worldDir)
for robotCount in AVAILABLE_ROBOT_COUNTS:
if self.meanCoverageEvents_[worldType][robotCount].hasData():
filename = self.getSaveFilename(worldDir, COVERAGE_EVENTS_FILE, robotCount)
dataFile.reset()
dataFile.addComment("# [coverage meanTimestamp(usec)]")
data = self.meanCoverageEvents_[worldType][robotCount].getMean()
dataFile.setDataAs(data[:2], "fl")
dataFile.save(filename)
if self.meanTimeEvents_[worldType][robotCount].hasData():
filename = self.getSaveFilename(worldDir, TIME_EVENTS_FILE, robotCount)
dataFile.reset()
dataFile.addComment("# [coverage meanTimestamp(usec)]")
data = self.meanTimeEvents_[worldType][robotCount].getMean()
dataFile.setDataAs(data[:2], "fl")
dataFile.save(filename)
if self.meanTileTimeBetweenVisits_[worldType][robotCount].hasData():
filename = self.getSaveFilename(worldDir, MEAN_TILE_TIME_BEWTEEN_VISITS_FILE, robotCount)
dataFile.reset()
dataFile.addComment("# [x y meanTimeBetweenVisits(usec)]")
data = self.meanTileTimeBetweenVisits_[worldType][robotCount].getMeanGrid()
dataFile.setDataAs(data[:3], "iil")
dataFile.save(filename)
if self.meanTileVisits_[worldType][robotCount].hasData():
filename = self.getSaveFilename(worldDir, TILE_VISITS_FILE, robotCount)
dataFile.reset()
dataFile.addComment("# [x y meanVisitCount]")
data = self.meanTileVisits_[worldType][robotCount].getMeanGrid()
dataFile.setDataAs(data[:3], "iif")
dataFile.save(filename)
if self.meanFinalCoverage_[worldType][robotCount].hasData():
filename = self.getSaveFilename(worldDir, FINAL_COVERAGE_FILE, robotCount)
dataFile.reset()
dataFile.addComment("# [coverage timeStamp(usec)]")
data = self.meanFinalCoverage_[worldType][robotCount].getMean()
dataFile.setDataAs(data[:2], "fl")
dataFile.save(filename)
def write_stats(ranges, results, prefix="stats"):
attrs = list(ranges.keys())
df = DataFile(os.path.join(base_path, '%s_%s.dat' % (prefix, '_'.join(attrs))), ['entries', 'lookups', 'hits', 'total time', 'lookup time', 'scans'] + attr_names("iter time") + attr_names("iter thruput") + attr_names("file size") + attr_names("scan thruput") + attrs, overwrite=True)
for config, reps in results:
cfg = dict(config)
# TODO: avg. over the reps
iter_times = Entry("iter time")
iter_tps = Entry("iter thruput")
file_sizes = Entry("file size")
scan_tps = Entry("scan thruput")
for data in reps:
db_path, entries, lookups, hits, total_time, num_scans, scans_time, lookup_time, iter_time, iter_tp = data.split(", ")
iter_times.add(int(iter_time))
iter_tps.add(int(iter_tp))
file_sizes.add(int(os.path.getsize(db_path)))
scan_tps.add(float(scans_time)/float(num_scans))
for attr in ranges:
df[attr] = cfg[attr]
df['entries'] = entries
df['lookups'] = lookups
df['hits'] = hits
df['total time'] = total_time
df['lookup time'] = lookup_time
df['scans'] = num_scans
# df['file size'] = os.path.getsize(db_path)
for attr in attr_names("iter time"):
df[attr] = getattr(iter_times, attr.split()[-1])
for attr in attr_names("iter thruput"):
df[attr] = getattr(iter_tps, attr.split()[-1])
for attr in attr_names("file size"):
df[attr] = getattr(file_sizes, attr.split()[-1])
for attr in attr_names("scan thruput"):
df[attr] = getattr(scan_tps, attr.split()[-1])
print file_sizes
print iter_times
print iter_tps
print scan_tps
df.save()
df.close()
def exec_ranges(cmd,
args,
range_attrs,
num_reps=10,
pbs_queue=None,
pbs_name='storagesim',
config=None):
# range_attrs is in the form:
# range_val: [list of possible values]
#
# when several range_vals and value lists are given, all possible
# permutations are executed
base_path = args['base_dir']
if config != None:
df = DataFile(os.path.join(config, 'config_ranges.dat'),
list(range_attrs.keys()),
overwrite=True)
outputs = []
for attr_p in permutations(range_attrs):
# set the attrs accordingly for this exec
names = []
for attr, val in attr_p:
args[attr] = val
names.append('%s_%s' % (str(attr), str(val)))
if config != None:
df[attr] = val
if config != None:
df.save()
args['base_dir'] = os.path.join(base_path, '%s' % ('_'.join(names)))
if not os.path.exists(args['base_dir']):
os.mkdir(args['base_dir'])
outputs.append((attr_p,
exec_repetitions(cmd,
args,
num_reps=num_reps,
pbs_queue=pbs_queue,
pbs_name=pbs_name)))
if config != None:
df.close()
return outputs
def exec_ranges(cmd, args, range_attrs, num_reps=10, pbs_queue=None, pbs_name='storagesim', config=None):
# range_attrs is in the form:
# range_val: [list of possible values]
#
# when several range_vals and value lists are given, all possible
# permutations are executed
base_path = args['base_dir']
if config != None:
df = DataFile(os.path.join(config, 'config_ranges.dat'), list(range_attrs.keys()), overwrite=True)
outputs = []
for attr_p in permutations(range_attrs):
# set the attrs accordingly for this exec
names = []
for attr, val in attr_p:
args[attr] = val
names.append('%s_%s' % (str(attr), str(val)))
if config != None:
df[attr] = val
if config != None:
df.save()
args['base_dir'] = os.path.join(base_path, '%s' % ('_'.join(names)))
if not os.path.exists(args['base_dir']):
os.mkdir(args['base_dir'])
outputs.append((attr_p, exec_repetitions(cmd, args, num_reps=num_reps, pbs_queue=pbs_queue, pbs_name=pbs_name)))
if config != None:
df.close()
return outputs
def read_and_save_flash(self, filename, with_bootloader):
_df = DataFile([])
_df.data = self.read_all_flash(with_bootloader)
print 'Read', len(_df.data), 'bytes'
_df.save(filename)
#write 234000 256.21552423 us/byte
# 255.434597666 us/byte
#l = Loader('/dev/tty.wchusbserial14230', 57600)
#l = Loader('/dev/tty.wchusbserial14230', 230400)
l = Loader('/dev/tty.wchusbserial14220', 153600)
print l.dev.get_about()
l.read_and_save_flash('flash_with_loader.hex', True)
l.read_and_save_flash('flash_without_loader.hex', False)
l.write_flash(fw.data)
l.dev.start_app()
1 / 0
# df = DataFile([])
#
# df.load('flash.hex')
# df.save('flash2.hex')
dev = Bootloader('/dev/tty.wchusbserial14230', 57600)
print dev.sync(10)
print dev.get_about()
eeprom = dev.read_eeprom(0, 1024)
# flash = dev.read_flash(0x7000, 0x1000)
flash = dev.read_flash(0x0000, 32 * 1024)
# print_dump(flash)
print_dump(flash)
df = DataFile(flash)
df.save('flash.hex')
dev.start_app()
class ProjectData:
def __init__(self, project):
self.folder = os.path.dirname(project)
self.project_file = project
self.current_file = 0
self.current_label = 0
self.modified = False
self.bad_files = []
self.datafile = None
self.config = None
self.read_conf()
self.read_file()
def read(self):
if self.modified:
self.read_conf()
self.modified = False
self.read_file()
if self.current_label >= len(self.config["labels"]):
self.current_label = 0
def read_conf(self):
try:
with open(self.project_file) as in_file:
self.config = json.load(in_file)
except IOError:
logger.error("Unable to read {}: permission denied".format(
self.project_file))
exit(2)
def read_file(self):
if set(self.config["files"]) == set(self.bad_files):
dialogs.report_no_files()
exit(3)
current = self.config["files"][self.current_file]
file_path = os.path.join(self.folder, current)
if current in self.bad_files or not os.path.exists(file_path):
if not os.path.exists(file_path):
self.datafile = None
self.bad_files.append(current)
dialogs.notify_read_error(current)
self.next_file()
self.read_file()
return
try:
self.datafile = DataFile(file_path, self.config["labels"])
self.insert_header()
except (UnrecognizedFormatError, BadFileError):
self.datafile = None
self.bad_files.append(current)
dialogs.notify_read_error(current)
self.next_file()
self.read_file()
def insert_header(self):
header = self.datafile.get_data_header()
if str(header) not in self.config.keys():
self.config[str(header)] = {
"plot": [[i] for i in self.datafile.get_data_columns()],
"normalize": [],
"functions": []
}
self.modified = True
self.save_config()
def save_file(self):
self.datafile.save()
def save_config(self):
if self.modified:
try:
with open(self.project_file, 'w') as out_file:
json.dump(self.config, out_file)
except IOError:
logger.error("Unable to write {}: permission denied".format(
self.project_file))
exit(2)
self.modified = False
def next_label(self):
self.current_label = (self.current_label + 1) % len(
self.config["labels"])
def prev_label(self):
length = len(self.config["labels"])
self.current_label = (self.current_label - 1 + length) % length
def get_current_label(self):
label = self.config["labels"][self.current_label]
color = self.config["colors"][self.current_label]
return label, color
def get_label_color(self, label):
index = self.config["labels"].index(label)
return self.config["colors"][index]
def get_labels_info(self):
return self.config["labels"], self.config["colors"]
def set_labels_info(self, names, colors):
self.config["labels"] = names
self.config["colors"] = colors
self.modified = True
if self.current_label >= len(names):
self.current_label = 0
def get_plot_info(self):
header = self.datafile.get_data_header()
conf = self.config[str(header)]
return conf["plot"], conf["normalize"]
def set_plot_info(self, plot_set, normalize):
header = self.datafile.get_data_header()
conf = self.config[str(header)]
conf["plot"] = plot_set
conf["normalize"] = normalize
self.modified = True
def next_file(self):
self.current_file = (self.current_file + 1) % len(self.config["files"])
def prev_file(self):
self.current_file = (self.current_file - 1 + len(
self.config["files"])) % len(self.config["files"])
def get_functions(self):
header = self.datafile.get_data_header()
return self.config[str(header)]["functions"]
def add_function(self, fs):
header = self.datafile.get_data_header()
new_header = header + [fs.name]
self.config[str(new_header)] = self.config[str(header)]
self.config[str(new_header)]["functions"].append(fs.name)
self.datafile.add_function(fs)
self.modified = True
def remove_function(self, index):
header = self.datafile.get_data_header()
conf = self.config[str(header)]
f_name = conf["functions"][index]
f_col = self.datafile.get_data_columns()[header.index(f_name)]
for plot in conf["plot"]:
if f_col in plot:
plot.remove(f_col)
for i, col in enumerate(plot):
if col > f_col:
plot[i] = col - 1
self.datafile.remove_function(f_name)
del conf["functions"][index]
self.modified = True
class FilesData:
def __init__(self, files):
self.files_list = files
self.config_list = []
self.current_file = 0
self.current_label = 0
self.modified = False
self.bad_files = []
self.datafile = None
self.config = None
self.init()
self.read()
def init(self):
for file in self.files_list:
conf = file + ".json"
file_conf = conf if os.path.exists(conf) else None
self.config_list.append(file_conf)
# noinspection PyTypeChecker
def read(self):
conf_path = self.config_list[self.current_file]
if conf_path:
self.read_conf()
self.read_file()
else:
self.config = {
"labels": ["Label"],
"colors": ["#1f77b4"],
}
self.read_file()
self.config["plot"] = [[i]
for i in self.datafile.get_data_columns()]
self.config["normalize"] = []
self.config["functions"] = []
if self.current_label >= len(self.config["labels"]):
self.current_label = 0
self.modified = False
def read_conf(self):
conf_path = self.config_list[self.current_file]
try:
with open(conf_path) as in_file:
self.config = json.load(in_file)
except IOError:
logger.error(
"Unable to read {}: permission denied".format(conf_path))
exit(2)
def read_file(self):
if set(self.files_list) == set(self.bad_files):
dialogs.report_no_files()
exit(3)
current = self.files_list[self.current_file]
if current in self.bad_files:
self.next_file()
self.read_file()
return
try:
self.datafile = DataFile(current, self.config["labels"])
except (UnrecognizedFormatError, BadFileError):
self.datafile = None
self.bad_files.append(current)
dialogs.notify_read_error(os.path.basename(current))
self.next_file()
self.read_file()
def save_file(self):
self.datafile.save()
def save_config(self):
if self.modified:
conf_path = self.files_list[self.current_file] + ".json"
try:
with open(conf_path, 'w') as out_file:
json.dump(self.config, out_file)
except IOError:
logger.error(
"Unable to write {}: permission denied".format(conf_path))
exit(2)
self.config_list[self.current_file] = conf_path
self.modified = False
def next_label(self):
self.current_label = (self.current_label + 1) % len(
self.config["labels"])
def prev_label(self):
length = len(self.config["labels"])
self.current_label = (self.current_label - 1 + length) % length
def get_current_label(self):
label = self.config["labels"][self.current_label]
color = self.config["colors"][self.current_label]
return label, color
def get_label_color(self, label):
index = self.config["labels"].index(label)
return self.config["colors"][index]
def get_labels_info(self):
return self.config["labels"], self.config["colors"]
def set_labels_info(self, names, colors):
self.config["labels"] = names
self.config["colors"] = colors
self.modified = True
if self.current_label >= len(names):
self.current_label = 0
def get_plot_info(self):
return self.config["plot"], self.config["normalize"]
def set_plot_info(self, plot_set, normalize):
self.config["plot"] = plot_set
self.config["normalize"] = normalize
self.modified = True
def next_file(self):
self.current_file = (self.current_file + 1) % len(self.files_list)
def prev_file(self):
self.current_file = (self.current_file - 1 +
len(self.files_list)) % len(self.files_list)
def get_functions(self):
return self.config["functions"]
def add_function(self, fs):
self.datafile.add_function(fs)
self.config["functions"].append(fs.name)
self.modified = True
def remove_function(self, index):
f_name = self.config["functions"][index]
header = self.datafile.get_data_header()
f_col = self.datafile.get_data_columns()[header.index(f_name)]
for plot in self.config["plot"]:
if f_col in plot:
plot.remove(f_col)
for i, col in enumerate(plot):
if col > f_col:
plot[i] = col - 1
self.datafile.remove_function(f_name)
del self.config["functions"][index]
self.modified = True
