def libcore_profiler_main_profiler_test():
name = 'dos.library'
fd = read_lib_fd(name)
cfg = ConfigDict({
"enabled": True,
"libs": {
"names": [name],
"calls": True
},
"output": {
"file": None,
"append": False,
"dump": True
}
})
mp = MainProfiler()
prof = LibProfiler()
assert mp.parse_config(cfg)
assert mp.add_profiler(prof)
mp.setup()
assert prof.names == [name]
assert prof.add_calls
p = prof.create_profile(name, fd)
assert p
mp.shutdown()
def libcore_profiler_main_profiler_test():
name = 'dos.library'
fd = read_lib_fd(name)
cfg = ConfigDict({
"enabled": True,
"libs": {
"names": [name],
"calls": True
},
"output": {
"file": None,
"append": False,
"dump": True
}
})
mp = MainProfiler()
prof = LibProfiler()
assert mp.parse_config(cfg)
assert mp.add_profiler(prof)
mp.setup()
assert prof.names == [name]
assert prof.add_calls
p = prof.create_profile(name, fd)
assert p
mp.shutdown()
def libcore_profiler_profiler_set_get_data_test(): name = 'dos.library' fd = read_lib_fd(name) prof = LibProfiler(names=[name]) prof.setup() p = prof.create_profile(name, fd) data = prof.get_data() prof2 = LibProfiler() assert prof2.set_data(data) p2 = prof2.get_profile(name) assert p == p2
def libcore_profile_profiler_default_test():
name = 'dos.library'
fd = read_lib_fd(name)
prof = LibProfiler(names=[name])
prof.setup()
p = prof.create_profile(name, fd)
assert p
assert prof.create_profile('bla', fd) is None
assert prof.get_profile(name) == p
assert prof.get_num_libs() == 1
assert prof.get_all_lib_names() == [name]
prof.shutdown()
def __init__(self, machine, alloc,
profiler_cfg=None):
self.machine = machine
self.mem = machine.get_mem()
self.alloc = alloc
self.lib_reg = LibRegistry()
self.ctx_map = LibCtxMap(machine)
self.profiler = LibProfiler(profiler_cfg)
# tools
self._setup_creator()
# state
self.exec_lib = None
self.addr_vlib = {}
self.name_vlib = {}
def libcore_profiler_profiler_config_test():
name = 'dos.library'
fd = read_lib_fd(name)
prof = LibProfiler()
prof.parse_config(ConfigDict({
"names": [name],
"calls": True
}))
prof.setup()
p = prof.create_profile(name, fd)
data = prof.get_data()
prof2 = LibProfiler()
assert prof2.set_data(data)
p2 = prof2.get_profile(name)
assert p == p2
def libcore_create_lib_profile_test():
mem, traps, alloc, ctx = setup()
impl = VamosTestLibrary()
# create info for lib
date = datetime.date(2012, 11, 12)
info = LibInfo('vamostest.library', 42, 3, date)
# create lib
pc = LibProfilerConfig(profiling=True, all_libs=True)
profiler = LibProfiler(pc)
creator = LibCreator(alloc, traps, profiler=profiler)
lib = creator.create_lib(info, ctx, impl)
prof = profiler.get_profile('vamostest.library')
assert prof
# free lib
lib.free()
assert alloc.is_all_free()
def _load_profile(self, args):
# read file
try:
with open(args.input) as fh:
data = json.load(fh)
if "libs" in data:
data = ConfigDict(data["libs"])
self.profiler = LibProfiler()
self.profiler.set_data(data)
return True
else:
print("no 'libs' found in '%s'" % args.input)
return False
except IOError as e:
print("loading '%s' failed: %s" % (args.input, e))
return False
def gen_prof(add_samples=False):
name = 'vamostest.library'
fd = read_lib_fd(name)
cfg = LibProfilerConfig(profiling=True,
all_libs=True,
add_samples=add_samples)
p = LibProfiler(cfg)
lp = p.create_profile(name, fd)
assert lp
# get func
func_name = "Add"
func = fd.get_func_by_name(func_name)
fp = lp.get_func_prof(func.get_index())
fp.count(1.0)
fp.count(2.0)
fp.count(3.0)
return p, lp, fp
def libcore_create_lib_profile_test():
mem, traps, alloc, ctx = setup()
impl = VamosTestLibrary()
# create info for lib
date = datetime.date(2012, 11, 12)
info = LibInfo('vamostest.library', 42, 3, date)
# create lib
lib_profiler = LibProfiler(names=["all"])
creator = LibCreator(alloc, traps, lib_profiler=lib_profiler)
lib_profiler.setup()
lib = creator.create_lib(info, ctx, impl)
profiler = creator.get_profiler()
prof = profiler.get_profile('vamostest.library')
assert prof
assert lib.profile
# free lib
lib.free()
assert alloc.is_all_free()
def libcore_profiler_profiler_set_get_data_test(): name = 'dos.library' fd = read_lib_fd(name) prof = LibProfiler(names=[name]) prof.setup() p = prof.create_profile(name, fd) data = prof.get_data() prof2 = LibProfiler() assert prof2.set_data(data) p2 = prof2.get_profile(name) assert p == p2
def libcore_profile_profiler_test():
name = 'vamostest.library'
fd = read_lib_fd(name)
c = LibProfilerConfig(profiling=True, libs=[name])
p = LibProfiler(c)
assert not p.get_profile(name)
prof = p.create_profile(name, fd)
assert prof
assert prof == p.get_profile(name)
assert p.get_all_lib_names() == [name]
assert p.get_num_libs() == 1
def _load_profile(self, args):
# read file
try:
with open(args.input) as fh:
data = json.load(fh)
if 'libs' in data:
data = ConfigDict(data['libs'])
self.profiler = LibProfiler()
self.profiler.set_data(data)
return True
else:
print("no 'libs' found in '%s'" % args.input)
return False
except IOError as e:
print("loading '%s' failed: %s" % (args.input, e))
return False
def __init__(
self, machine, alloc, main_profiler=None, prof_names=None, prof_calls=False
):
self.machine = machine
self.mem = machine.get_mem()
self.alloc = alloc
self.lib_reg = LibRegistry()
self.ctx_map = {}
self.lib_profiler = LibProfiler(prof_names, prof_calls)
if main_profiler:
main_profiler.add_profiler(self.lib_profiler)
# tools
self._setup_creator()
# state
self.exec_lib = None
self.addr_vlib = {}
self.name_vlib = {}
self.ctx_extra_attr = {}
class LibProfilerTool(Tool):
def __init__(self):
Tool.__init__(self, "libprof", "library profile utilities")
self.profiler = None
def add_args(self, arg_parser):
sub = arg_parser.add_subparsers(dest="libprof_cmd")
# dump
parser = sub.add_parser("dump",
help="display library function profile info")
parser.add_argument("input", help="profile json file")
# missing
parser = sub.add_parser("missing",
help="display missing library functions")
parser.add_argument("input", help="profile json file")
# coverage
parser = sub.add_parser("coverage",
help="display library function coverage info")
parser.add_argument("input", help="profile json file")
parser.add_argument(
"-f",
"--functions",
action="store_true",
default=False,
help="show uncovered functions",
)
def setup(self, args):
return True
def shutdown(self):
pass
def run(self, args):
if not self._load_profile(args):
return 1
cmd = args.libprof_cmd
if cmd == "dump":
return self._do_dump()
elif cmd == "missing":
return self._do_missing()
elif cmd == "coverage":
return self._do_coverage(args)
else:
return 1
def _load_profile(self, args):
# read file
try:
with open(args.input) as fh:
data = json.load(fh)
if "libs" in data:
data = ConfigDict(data["libs"])
self.profiler = LibProfiler()
self.profiler.set_data(data)
return True
else:
print("no 'libs' found in '%s'" % args.input)
return False
except IOError as e:
print("loading '%s' failed: %s" % (args.input, e))
return False
def _do_dump(self):
self.profiler.dump()
return 0
def _do_missing(self):
p = self.profiler
for lib_name, lib_prof in p.get_all_libs():
print(lib_name)
for func_name, func_prof in lib_prof.get_all_funcs():
if func_prof.tag != LibImplScan.TAG_VALID:
print(" ", func_name)
def _do_coverage(self, args):
print("%-40s total valid called" % "library")
p = self.profiler
for lib_name, lib_prof in p.get_all_libs():
self._print_lib_cov(lib_name, lib_prof)
if args.functions:
self._print_funcs(lib_prof)
return 0
def _print_lib_cov(self, lib_name, lib_prof):
num_valid = 0
num_covered = 0
num_total = 0
for _, func_prof in lib_prof.get_all_funcs():
if func_prof.tag == LibImplScan.TAG_VALID:
num_valid += 1
if func_prof.num > 0:
num_covered += 1
num_total += 1
impl_ratio = 100.0 * num_valid / num_total
if num_valid > 0:
coverage = 100.0 * num_covered / num_valid
else:
coverage = 0.0
print("%-40s %4d %6.2f "
"%4d %6.2f %4d" % (lib_name, num_total, impl_ratio, num_valid,
coverage, num_covered))
def _print_funcs(self, lib_prof):
for name, func_prof in lib_prof.get_all_funcs():
if func_prof.tag == LibImplScan.TAG_VALID:
if func_prof.num == 0:
print(" ", name)
class VLibManager(object):
"""handle a set of vlib instances"""
def __init__(self, machine, alloc,
profiler_cfg=None):
self.machine = machine
self.mem = machine.get_mem()
self.alloc = alloc
self.lib_reg = LibRegistry()
self.ctx_map = LibCtxMap(machine)
self.profiler = LibProfiler(profiler_cfg)
# tools
self._setup_creator()
# state
self.exec_lib = None
self.addr_vlib = {}
self.name_vlib = {}
def _setup_creator(self):
# tools
log_missing = None
log_valid = None
if log_lib.isEnabledFor(logging.WARNING):
log_missing = log_lib
if log_lib.isEnabledFor(logging.INFO):
log_valid = log_lib
self.creator = LibCreator(self.alloc, self.machine.get_traps(),
log_missing=log_missing, log_valid=log_valid,
profiler=self.profiler)
def add_impl_cls(self, name, impl_cls):
self.lib_reg.add_lib_impl(name, impl_cls)
def add_ctx(self, name, ctx):
self.ctx_map.add_ctx(name, ctx)
def bootstrap_exec(self, exec_info=None,
version=0, revision=0):
"""setup exec library"""
if exec_info is None:
date = datetime.date(day=7, month=7, year=2007)
exec_info = LibInfo('exec.library', version, revision, date)
# make sure its an exec info
assert exec_info.get_name() == 'exec.library'
# create vlib
vlib = self._create_vlib(exec_info, False)
assert vlib
assert vlib.impl
# setup exec_lib
lib_base = vlib.get_library().get_addr()
self.exec_lib = ExecLibrary(self.mem, lib_base)
# store lib base
self.mem.w32(4, lib_base)
self.machine.set_zero_mem(0, lib_base)
# with exec_lib in place we can add exec's vlib
self._add_vlib(vlib)
# inc exec's open cnt so lib gets never expunged
self.exec_lib.lib_node.inc_open_cnt()
return vlib
def get_vlib_by_addr(self, addr):
"""return associated vlib for a base lib address"""
if addr in self.addr_vlib:
return self.addr_vlib[addr]
def get_vlib_by_name(self, name):
"""return associated vlib for a name"""
if name in self.name_vlib:
return self.name_vlib[name]
def get_profiler(self):
"""return the lib profiler"""
return self.profiler
def shutdown(self):
"""cleanup libs
try to expunge all libs and report still open ones
"""
log_libmgr.info("[vamos] +shutdown")
# dec exec's open cnt
self.exec_lib.lib_node.dec_open_cnt()
# now expunge all libs
left_libs = self.expunge_libs()
left_devs = self.expunge_devs()
log_libmgr.info("[vamos] +shutdown: left libs=%d, devs=%d",
left_libs, left_devs)
# finally shutdown profiler
self.profiler.shutdown()
return left_libs + left_devs
def expunge_libs(self):
"""expunge all unused vlibs
return number of libs _not_ expunged
"""
return self._expunge_list(self.exec_lib.lib_list)
def expunge_devs(self):
"""expunge all unused vlibs
return number of libs _not_ expunged
"""
return self._expunge_list(self.exec_lib.device_list)
def _expunge_list(self, node_list):
left_libs = 0
for node in node_list:
lib_base = node.get_addr()
# is it a vlib?
if lib_base in self.addr_vlib:
vlib = self.addr_vlib[lib_base]
if not self.expunge_lib(vlib):
left_libs += 1
return left_libs
def expunge_lib(self, vlib):
"""expunge a vlib"""
lib = vlib.get_library()
# still open?
if lib.open_cnt > 0:
return False
# vlib?
self._rem_vlib(vlib)
return True
def make_lib_name(self, name, version=0, revision=0, fake=False):
date = datetime.date(day=7, month=7, year=2007)
info = LibInfo(name, version, revision, date)
return self.make_lib(info, fake)
def make_lib(self, lib_info, fake=False):
vlib = self._create_vlib(lib_info, fake)
if vlib:
self._add_vlib(vlib)
return vlib
def open_lib_name(self, name, version=0, revision=0, fake=False):
vlib = self.get_vlib_by_name(name)
if not vlib:
vlib = self.make_lib_name(name, version, revision, fake)
if vlib:
vlib.open()
return vlib
def open_lib(self, lib_info, fake=False):
vlib = self.get_vlib_by_name(name)
if not vlib:
vlib = self.make_lib(lib_info, fake, is_dev)
if vlib:
vlib.open()
return vlib
def close_lib(self, vlib):
vlib.close()
return self.expunge_lib(vlib)
def _create_vlib(self, lib_info, fake):
# get lib ctx
name = lib_info.get_name()
ctx = self.ctx_map.get_ctx(name)
# get impl
if fake:
impl = None
else:
name = lib_info.get_name()
impl_cls = self.lib_reg.get_lib_impl(name)
if impl_cls:
impl = impl_cls()
# adjust version?
if lib_info.version == 0:
lib_info.version = impl.get_version()
else:
return None
# create lib
vlib = self.creator.create_lib(lib_info, ctx, impl)
# store vlib in context
ctx.vlib = vlib
return vlib
def _add_vlib(self, vlib):
addr = vlib.get_addr()
name = vlib.get_name()
# store internally
self.addr_vlib[addr] = vlib
self.name_vlib[name] = vlib
# add lib to exec lib list
lib = vlib.get_library()
if vlib.is_device():
self.exec_lib.device_list.enqueue(lib.node)
else:
self.exec_lib.lib_list.enqueue(lib.node)
def _rem_vlib(self, vlib):
addr = vlib.get_addr()
name = vlib.get_name()
# remove internally
del self.addr_vlib[addr]
del self.name_vlib[name]
# remove from exec lib list
lib = vlib.get_library()
lib.node.remove()
# free vlib
vlib.free()
def libcore_profiler_profiler_reuse_test():
name = 'dos.library'
fd = read_lib_fd(name)
prof = LibProfiler()
prof.parse_config(ConfigDict({
"names": [name],
"calls": True
}))
prof.setup()
p = prof.create_profile(name, fd)
assert p
f = p.get_func_by_name("Input")
assert f
idx = f.get_func_id()
assert p.get_func_by_index(idx) == f
f.count(1.0)
# store/restore
data = prof.get_data()
prof2 = LibProfiler()
assert prof2.set_data(data)
prof2.setup()
# reuse profile
p2 = prof2.create_profile(name, fd)
assert p == p2
f2 = p2.get_func_by_name("Input")
assert f2.get_num_calls() == 1
f2.count(1.0)
assert f2.get_num_calls() == 2
idx = f2.get_func_id()
assert p2.get_func_by_index(idx) == f2
def libcore_profile_profiler_default_test():
name = 'dos.library'
fd = read_lib_fd(name)
prof = LibProfiler(names=[name])
prof.setup()
p = prof.create_profile(name, fd)
assert p
assert prof.create_profile('bla', fd) is None
assert prof.get_profile(name) == p
assert prof.get_num_libs() == 1
assert prof.get_all_lib_names() == [name]
prof.shutdown()
def libcore_profiler_profiler_config_test():
name = 'dos.library'
fd = read_lib_fd(name)
prof = LibProfiler()
prof.parse_config(ConfigDict({
"names": [name],
"calls": True
}))
prof.setup()
p = prof.create_profile(name, fd)
data = prof.get_data()
prof2 = LibProfiler()
assert prof2.set_data(data)
p2 = prof2.get_profile(name)
assert p == p2
class LibProfilerTool(Tool):
def __init__(self):
Tool.__init__(self, "libprof", "library profile utilities")
self.profiler = None
def add_args(self, arg_parser):
sub = arg_parser.add_subparsers(dest='libprof_cmd')
# dump
parser = sub.add_parser('dump',
help='display library function profile info')
parser.add_argument('input', help='profile json file')
# missing
parser = sub.add_parser('missing',
help='display missing library functions')
parser.add_argument('input', help='profile json file')
# coverage
parser = sub.add_parser('coverage',
help='display library function coverage info')
parser.add_argument('input', help='profile json file')
parser.add_argument('-f', '--functions', action='store_true', default=False,
help='show uncovered functions')
def setup(self, args):
return True
def shutdown(self):
pass
def run(self, args):
if not self._load_profile(args):
return 1
cmd = args.libprof_cmd
if cmd == 'dump':
return self._do_dump()
elif cmd == 'missing':
return self._do_missing()
elif cmd == 'coverage':
return self._do_coverage(args)
else:
return 1
def _load_profile(self, args):
# read file
try:
with open(args.input) as fh:
data = json.load(fh)
if 'libs' in data:
data = ConfigDict(data['libs'])
self.profiler = LibProfiler()
self.profiler.set_data(data)
return True
else:
print("no 'libs' found in '%s'" % args.input)
return False
except IOError as e:
print("loading '%s' failed: %s" % (args.input, e))
return False
def _do_dump(self):
self.profiler.dump()
return 0
def _do_missing(self):
p = self.profiler
for lib_name, lib_prof in p.get_all_libs():
print(lib_name)
for func_name, func_prof in lib_prof.get_all_funcs():
if func_prof.tag != LibImplScan.TAG_VALID:
print(" ", func_name)
def _do_coverage(self, args):
print("%-40s total valid called" % 'library')
p = self.profiler
for lib_name, lib_prof in p.get_all_libs():
self._print_lib_cov(lib_name, lib_prof)
if args.functions:
self._print_funcs(lib_prof)
return 0
def _print_lib_cov(self, lib_name, lib_prof):
num_valid = 0
num_covered = 0
num_total = 0
for _, func_prof in lib_prof.get_all_funcs():
if func_prof.tag == LibImplScan.TAG_VALID:
num_valid += 1
if func_prof.num > 0:
num_covered += 1
num_total += 1
impl_ratio = 100.0 * num_valid / num_total
if num_valid > 0:
coverage = 100.0 * num_covered / num_valid
else:
coverage = 0.0
print("%-40s %4d %6.2f "
"%4d %6.2f %4d" % (lib_name, num_total, impl_ratio,
num_valid, coverage, num_covered))
def _print_funcs(self, lib_prof):
for name, func_prof in lib_prof.get_all_funcs():
if func_prof.tag == LibImplScan.TAG_VALID:
if func_prof.num == 0:
print(" ", name)
def libcore_profiler_profiler_reuse_test():
name = 'dos.library'
fd = read_lib_fd(name)
prof = LibProfiler()
prof.parse_config(ConfigDict({
"names": [name],
"calls": True
}))
prof.setup()
p = prof.create_profile(name, fd)
assert p
f = p.get_func_by_name("Input")
assert f
idx = f.get_func_id()
assert p.get_func_by_index(idx) == f
f.count(1.0)
# store/restore
data = prof.get_data()
prof2 = LibProfiler()
assert prof2.set_data(data)
prof2.setup()
# reuse profile
p2 = prof2.create_profile(name, fd)
assert p == p2
f2 = p2.get_func_by_name("Input")
assert f2.get_num_calls() == 1
f2.count(1.0)
assert f2.get_num_calls() == 2
idx = f2.get_func_id()
assert p2.get_func_by_index(idx) == f2