def __init__(self, ipython):
"""Public constructor."""
if VariableInspectorWindow.instance is not None:
raise Exception(
"""Only one instance of the Variable Inspector can exist at a
time. Call close() on the active instance before creating a new instance.
If you have lost the handle to the active instance, you can re-obtain it
via `VariableInspectorWindow.instance`.""")
VariableInspectorWindow.instance = self
self.closed = False
self.namespace = NamespaceMagics()
self.namespace.shell = ipython.kernel.shell
self._popout = widgets.PopupWidget()
self._popout.description = "Variable Inspector"
self._popout.button_text = self._popout.description
self._modal_body = widgets.ContainerWidget()
self._modal_body.set_css('overflow-y', 'scroll')
self._modal_body_label = widgets.HTMLWidget(value='Not hooked')
self._modal_body.children = [self._modal_body_label]
self._popout.children = [
self._modal_body,
]
self._ipython = ipython
self._ipython.register_post_execute(self._fill)
def __init__(self, backend='pytorch'):
""" Instantiate an object with parameters:
backend: 'cpu', 'pytorch' (default), ...
"""
self.backend_load(backend)
print("\n*** Starting experiment...")
self.running = True
self.reclaimed = False
self.start_time = time.time()
self.var_names_keep = []
# base-line
gc.collect()
gpu_clear_cache()
# grab the notebook var names during creation
ipython = get_ipython()
self.namespace = NamespaceMagics()
self.namespace.shell = ipython.kernel.shell
self.var_names_start = self.get_var_names()
#print(self.var_names_start)
self.gen_ram_used_start = gen_ram_used()
self.gpu_ram_used_start = gpu_ram_used()
#print(f"gpu used f{self.gpu_ram_used_start}" )
self.print_state()
print("\n") # extra vertical white space, to not mix with user's outputs
def __init__(self, method='filter'):
assert method in ['filter', 'choose']
self.method = method
self.strict = True
self.exclude_unsupported = True
self._jupyterlab_variableinspector_nms = NamespaceMagics()
self._jupyterlab_variableinspector_Jupyter = get_ipython()
self._jupyterlab_variableinspector_nms.shell = self._jupyterlab_variableinspector_Jupyter.kernel.shell
self.reset_all()
def _list_kernel_vars():
_nms = NamespaceMagics()
_Jupyter = get_ipython()
_nms.shell = _Jupyter.kernel.shell
variables = [
v for v in _nms.who_ls() if _get_var_type(v) in PYTHON_TYPE_TO_IOT_TYPE
]
return json.dumps([{
'varName': v,
'varType': PYTHON_TYPE_TO_IOT_TYPE[_get_var_type(v)]
} for v in variables])
def __init__(self):
self._sc = Sidecar(title='Variables')
get_ipython().user_ns_hidden['widgets'] = widgets
get_ipython().user_ns_hidden['NamespaceMagics'] = NamespaceMagics
self.closed = False
self.namespace = NamespaceMagics()
self.namespace.shell = get_ipython().kernel.shell
self._box = widgets.Box()
self._box.layout.overflow_y = 'scroll'
self._table = widgets.HTML(value='Not hooked')
self._box.children = [self._table]
self._ipython = get_ipython()
self._ipython.events.register('post_run_cell', self._fill)
class VariableInspectorWindow(object):
instance = None
def __init__(self, ipython):
"""Public constructor."""
if VariableInspectorWindow.instance is not None:
raise Exception(
"""Only one instance of the Variable Inspector can exist at a
time. Call close() on the active instance before creating a new instance.
If you have lost the handle to the active instance, you can re-obtain it
via `VariableInspectorWindow.instance`.""")
VariableInspectorWindow.instance = self
self.closed = False
self.namespace = NamespaceMagics()
self.namespace.shell = ipython.kernel.shell
self._popout = widgets.PopupWidget()
self._popout.description = "Variable Inspector"
self._popout.button_text = self._popout.description
self._modal_body = widgets.ContainerWidget()
self._modal_body.set_css('overflow-y', 'scroll')
self._modal_body_label = widgets.HTMLWidget(value='Not hooked')
self._modal_body.children = [self._modal_body_label]
self._popout.children = [
self._modal_body,
]
self._ipython = ipython
self._ipython.register_post_execute(self._fill)
def close(self):
"""Close and remove hooks."""
if not self.closed:
del self._ipython._post_execute[self._fill]
self._popout.close()
self.closed = True
VariableInspectorWindow.instance = None
def _fill(self):
"""Fill self with variable information."""
values = self.namespace.who_ls()
self._modal_body_label.value = '<table class="table table-bordered table-striped"><tr><th>Name</th><th>Type</th><th>Value</th></tr><tr><td>' + '</td></tr><tr><td>'.join(
[
'{0}</td><td>{1}</td><td>{2}'.format(v,
type(eval(v)).__name__,
str(eval(v)))
for v in values
]) + '</td></tr></table>'
def _ipython_display_(self):
"""Called when display() or pyout is used to display the Variable
Inspector."""
self._popout._ipython_display_()
self._popout.add_class('vbox')
self._modal_body.add_class('box-flex1')
def __init__(self,
exp_enable=True,
cl_enable=True,
cl_compact=False,
cl_gc_collect=True,
cl_set_seed=0):
""" Instantiate an object with parameters:
Parameters:
* exp_enable=False - run just the CellLogger if exp_enable=False, cl_enable=True
Cell logger Parameters: these are being passed to CellLogger (and the defaults)
* cl_enable=True - run the cell logger
* cl_compact=False - compact cell report
* cl_gc_collect=True - gc_collect at the end of each cell before mem measurement
* cl_set_seed=0 - set RNG seed before each cell is run to the provided value
"""
logger.debug(f"{self.__class__.__name__}::__init__: {self}")
if cl_enable:
self.cl = CellLogger(exp=self,
compact=cl_compact,
gc_collect=cl_gc_collect,
set_seed=cl_set_seed)
else:
self.cl = None
self.enable = exp_enable
self.running = False
if not self.enable: return
self.reclaimed = False
self.start_time = time.time()
self.var_names_keep = []
# grab the notebook var names during creation
ipython = get_ipython()
self.namespace = NamespaceMagics()
self.namespace.shell = ipython.kernel.shell
self.var_names_start = self.get_var_names()
def __init__(self, ipython):
"""Public constructor."""
if VariableInspectorWindow.instance is not None:
raise Exception("""Only one instance of the Variable Inspector can exist at a
time. Call close() on the active instance before creating a new instance.
If you have lost the handle to the active instance, you can re-obtain it
via `VariableInspectorWindow.instance`.""")
VariableInspectorWindow.instance = self
self.closed = False
self.namespace = NamespaceMagics()
self.namespace.shell = ipython.kernel.shell
self._box = widgets.Box()
self._box.layout.overflow_y = 'scroll'
self._table = widgets.HTML(value = 'Not hooked')
self._box.children = [self._table]
self._ipython = ipython
self._ipython.events.register('post_run_cell', self._fill)
class VariableInspector(object):
def __init__(self):
self._sc = Sidecar(title='Variables')
get_ipython().user_ns_hidden['widgets'] = widgets
get_ipython().user_ns_hidden['NamespaceMagics'] = NamespaceMagics
self.closed = False
self.namespace = NamespaceMagics()
self.namespace.shell = get_ipython().kernel.shell
self._box = widgets.Box()
self._box.layout.overflow_y = 'scroll'
self._table = widgets.HTML(value='Not hooked')
self._box.children = [self._table]
self._ipython = get_ipython()
self._ipython.events.register('post_run_cell', self._fill)
def close(self):
"""Close and remove hooks."""
if not self.closed:
self._ipython.events.unregister('post_run_cell', self._fill)
self._box.close()
self.closed = True
def _fill(self):
"""Fill self with variable information."""
types_to_exclude = ['module', 'function', 'builtin_function_or_method',
'instance', '_Feature', 'type', 'ufunc']
values = self.namespace.who_ls()
def eval(expr):
return self.namespace.shell.ev(expr)
var = [(v,
type(eval(v)).__name__,
str(_getsizeof(eval(v))),
str(_getshapeof(eval(v))) if _getshapeof(eval(v)) else '',
str(eval(v))[:200])
for v in values if (v not in ['_html', '_nms', 'NamespaceMagics', '_Jupyter']) & (type(eval(v)).__name__ not in types_to_exclude)]
self._table.value = '<div class="rendered_html jp-RenderedHTMLCommon"><table><thead><tr><th>Name</th><th>Type</th><th>Size</th><th>Shape</th><th>Value</th></tr></thead><tr><td>' + \
'</td></tr><tr><td>'.join(['{0}</td><td>{1}</td><td>{2}</td><td>{3}</td><td>{4}'.format(v1, v2, v3, v4, v5) for v1, v2, v3, v4, v5 in var]) + \
'</td></tr></table></div>'
def _ipython_display_(self):
"""Called when display() or pyout is used to display the Variable
Inspector."""
with self._sc:
self._box._ipython_display_()
def __init__(self, ipython,
regex_ignore=r'(VariableInspectorWindow|inspector)',
ignore_types=(types.ModuleType, types.FunctionType)):
"""Public constructor."""
if VariableInspectorWindow.instance is not None:
raise Exception("""Only one instance of the Variable Inspector can exist at a
time. Call close() on the active instance before creating a new instance.
If you have lost the handle to the active instance, you can re-obtain it
via `VariableInspectorWindow.instance`.""")
VariableInspectorWindow.instance = self
self.closed = False
self.namespace = NamespaceMagics()
self.namespace.shell = ipython.kernel.shell
self._box = widgets.Box()
self._box._dom_classes = ['inspector']
self._box.background_color = '#fff'
self._box.border_color = '#ccc'
self._box.border_width = 1
self._box.border_radius = 5
self._modal_body = widgets.VBox()
self._modal_body.overflow_y = 'scroll'
self._modal_body_label = widgets.HTML(value = 'Not hooked')
self._modal_body.children = [self._modal_body_label]
self._box.children = [
self._modal_body,
]
self._ipython = ipython
self._ipython.events.register('post_run_cell', self._fill)
self.regex_ignore = regex_ignore
self.ignore_types = ignore_types
def nbtutor(self, line, cell) -> None:
opts = magic_arguments.parse_argstring(self.nbtutor, line)
if opts.reset:
params = '-f' if opts.force else ''
NamespaceMagics(self.shell).reset(params)
debugger = Debugger(self.shell, cell, opts)
debugger.run_cell()
# FIXME: This pointless re-running the cell again via IPython is needed to get the
# "<ipython-input-{0}-{1}>" f_code.co_filename set and the code cached.
# I don't know enough about IPython to do this better inside the debugger.
self.shell.run_cell(cell)
if not debugger.code_error or opts.debug:
self.comm.send([x.to_dict() for x in debugger.trace_history])
class VariableInspectorWindow(object):
instance = None
def __init__(self, ipython):
"""Public constructor."""
if VariableInspectorWindow.instance is not None:
raise Exception("""Only one instance of the Variable Inspector can exist at a
time. Call close() on the active instance before creating a new instance.
If you have lost the handle to the active instance, you can re-obtain it
via `VariableInspectorWindow.instance`.""")
VariableInspectorWindow.instance = self
self.closed = False
self.namespace = NamespaceMagics()
self.namespace.shell = ipython.kernel.shell
self._box = widgets.Box()
self._box.layout.overflow_y = 'scroll'
self._table = widgets.HTML(value = 'Not hooked')
self._box.children = [self._table]
self._ipython = ipython
self._ipython.events.register('post_run_cell', self._fill)
def close(self):
"""Close and remove hooks."""
if not self.closed:
self._ipython.events.unregister('post_run_cell', self._fill)
self._box.close()
self.closed = True
VariableInspectorWindow.instance = None
def _fill(self):
"""Fill self with variable information."""
values = self.namespace.who_ls()
self._table.value = '<div class="rendered_html jp-RenderedHTMLCommon"><table><thead><tr><th>Name</th><th>Type</th><th>Value</th></tr></thead><tr><td>' + \
'</td></tr><tr><td>'.join(['{0}</td><td>{1}</td><td>{2}'.format(v, type(eval(v)).__name__, str(eval(v))) for v in values]) + \
'</td></tr></table></div>'
def _ipython_display_(self):
"""Called when display() or pyout is used to display the Variable
Inspector."""
self._box._ipython_display_()
def nbtutor(self, line, cell):
opts = magic_arguments.parse_argstring(self.nbtutor, line)
if opts.reset:
params = '-f' if opts.force else ''
NamespaceMagics(self.shell).reset(params)
if opts.nolies:
opts.inline = False
bdb = Bdb(self.shell, opts)
bdb.run_cell(cell)
self.shell.run_cell(cell)
# FIXME: This pointless re-running the cell again via ipython is needed
# to get the "<ipython-input-{0}-{1}>" f_code.co_filename set and the
# code cached. I don't know enough about the IPython API to do this
# better inside the debugger.
if not bdb.code_error or opts.debug:
self.comm.send(bdb.trace_history.clean())
class VariableInspectorWindow(object):
instance = None
def __init__(self, ipython,
regex_ignore=r'(VariableInspectorWindow|inspector)',
ignore_types=(types.ModuleType, types.FunctionType)):
"""Public constructor."""
if VariableInspectorWindow.instance is not None:
raise Exception("""Only one instance of the Variable Inspector can exist at a
time. Call close() on the active instance before creating a new instance.
If you have lost the handle to the active instance, you can re-obtain it
via `VariableInspectorWindow.instance`.""")
VariableInspectorWindow.instance = self
self.closed = False
self.namespace = NamespaceMagics()
self.namespace.shell = ipython.kernel.shell
self._box = widgets.Box()
self._box._dom_classes = ['inspector']
self._box.background_color = '#fff'
self._box.border_color = '#ccc'
self._box.border_width = 1
self._box.border_radius = 5
self._modal_body = widgets.VBox()
self._modal_body.overflow_y = 'scroll'
self._modal_body_label = widgets.HTML(value = 'Not hooked')
self._modal_body.children = [self._modal_body_label]
self._box.children = [
self._modal_body,
]
self._ipython = ipython
self._ipython.events.register('post_run_cell', self._fill)
self.regex_ignore = regex_ignore
self.ignore_types = ignore_types
def close(self):
"""Close and remove hooks."""
if not self.closed:
self._ipython.events.unregister('post_run_cell', self._fill)
self._box.close()
self.closed = True
VariableInspectorWindow.instance = None
def _fill(self):
"""Fill self with variable information."""
values = self.namespace.who_ls()
def get_type(var):
try:
return type(var).__name__ + ' ' + str(var.dtype)
except AttributeError:
return type(var).__name__
r = re.compile(self.regex_ignore)
self._modal_body_label.value = \
'<table class="table table-bordered table-striped"><tr><th>Name</th><th>Type</th><th>Value</th></tr><tr><td>' + \
'</td></tr><tr><td>'.join(
['{0}</td><td>{1}</td><td>{2}'.format(
v,
get_type(self._ipython.user_ns[v]),
str(self._ipython.user_ns[v]))
for v in values if not r.match(v) and not isinstance(self._ipython.user_ns[v], self.ignore_types)]) + \
'</td></tr></table>'
def _ipython_display_(self):
"""Called when display() or pyout is used to display the Variable
Inspector."""
self._box._ipython_display_()
def undock(self):
from IPython.display import HTML
from IPython.display import Javascript
return Javascript(
'''
$('div.inspector')
.detach()
.prependTo($('body'))
.css({
'z-index': 999,
position: 'fixed',
'box-shadow': '5px 5px 12px -3px black',
opacity: 0.9
})
.draggable();
''')
class VarHome:
"""
在jupyter中跟踪并管理变量，并可以一键保存和载入跟踪的变量
控制参数：
method:可选择'filter'或者'choose'，两种工作方式，'filter'会默认跟踪所有变量，'choose'会默认不跟踪所有变量
strict:是否跟踪模块或callable的变量
exclude_unsupported：是否屏蔽不支持的变量类型
方法：
vars：查看当前跟踪的变量的信息
clear_list：清空当前跟踪的变量列表
reset_all：清空所有列表
exclude_var：屏蔽变量
choose_var：跟踪变量
del_var：删除变量
"""
def __init__(self, method='filter'):
assert method in ['filter', 'choose']
self.method = method
self.strict = True
self.exclude_unsupported = True
self._jupyterlab_variableinspector_nms = NamespaceMagics()
self._jupyterlab_variableinspector_Jupyter = get_ipython()
self._jupyterlab_variableinspector_nms.shell = self._jupyterlab_variableinspector_Jupyter.kernel.shell
self.reset_all()
def __var_base(self):
# 筛选及获取变量的基本信息
need_list = [i for i in self.__update_var_list()]
self.__name_space = _main_module.__dict__.copy()
if self.strict == False:
self.__var_inf = {
i: get_var_inf(self.__name_space[i], strict=False).copy()
for i in need_list
}
else:
self.__var_inf = {
i: get_var_inf(self.__name_space[i], strict=True).copy()
for i in need_list
}
if self.exclude_unsupported == True:
self.__filter_unsupported()
if self.method == 'filter':
self.__var_inf = {
k: v
for k, v in self.__var_inf.items()
if (k not in self.__exclude_list)
}
elif self.method == 'choose':
self.__var_inf = {
k: v
for k, v in self.__var_inf.items() if (k in self.__choose_list)
}
def vars(self):
# 展示正在追踪的变量
self.__var_base()
show_df = pd.DataFrame(self.__var_inf).T
if len(show_df) > 0:
return show_df[['is_supported', 'type', 'size', 'memory usage']]
def __filter_unsupported(self):
# 排除不支持的变量类型
self.__var_inf = {
k: v
for k, v in self.__var_inf.items() if v['is_supported'] == True
}
def __update_var_list(self):
return self._jupyterlab_variableinspector_nms.who_ls()
def clear_list(self):
# 清空当前追踪列表
self.__var_base
for i in self.__var_inf.keys():
self.exclude_var(i)
def save_data(self, filename):
# 保存当前追踪的变量
self.__var_base()
data = {
i: self.__name_space[i]
for i in self.__var_inf.keys()
if self.__var_inf[i]['is_supported'] == True
}
save_dictionary(data, filename)
print(list(data.keys()))
return True
def load_data(self, filename):
# 读取变量
data = load_dictionary(filename)[0]
for k, v in data.items():
_main_module.__dict__[k] = v
print(list(data.keys()))
def exclude_var(self, var_name):
# 屏蔽变量
if var_name not in self.__exclude_list:
self.__exclude_list.append(var_name)
if var_name in self.__choose_list:
self.__choose_list.remove(var_name)
print('屏蔽变量：' + var_name)
else:
print('已存在')
def choose_var(self, var_name):
# 追踪变量
if var_name not in self.__choose_list and var_name in self.__update_var_list(
):
self.__choose_list.append(var_name)
if var_name in self.__exclude_list:
self.__exclude_list.remove(var_name)
print('选定变量：' + var_name)
else:
print('未能添加')
def reset_all(self):
# 清空所有状态
self.__exclude_list = []
self.__choose_list = []
def del_var(self, var_name):
# 删除变量
if var_name in _main_module.__dict__.keys():
_main_module.__dict__.pop(var_name)
class IPyExperiments():
"Create an experiment with time/memory checkpoints"
def __init__(self,
exp_enable=True,
cl_enable=True,
cl_compact=False,
cl_gc_collect=True,
cl_set_seed=0):
""" Instantiate an object with parameters:
Parameters:
* exp_enable=False - run just the CellLogger if exp_enable=False, cl_enable=True
Cell logger Parameters: these are being passed to CellLogger (and the defaults)
* cl_enable=True - run the cell logger
* cl_compact=False - compact cell report
* cl_gc_collect=True - gc_collect at the end of each cell before mem measurement
* cl_set_seed=0 - set RNG seed before each cell is run to the provided value
"""
logger.debug(f"{self.__class__.__name__}::__init__: {self}")
self.cl_enable = cl_enable
self.cl_kwargs = dict(compact=cl_compact,
gc_collect=cl_gc_collect,
set_seed=cl_set_seed)
self.enable = exp_enable
self.running = False
if not self.enable: return
self.reclaimed = False
self.start_time = time.time()
self.var_names_keep = []
# grab the notebook var names during creation
ipython = get_ipython()
self.namespace = NamespaceMagics()
self.namespace.shell = ipython.kernel.shell
self.var_names_start = self.get_var_names()
#print(self.var_names_start)
# The following doesn't work:
#
# we have to take a snapshot of all the variables and their references,
# so that when the experiment is over we can discover which variables were
# used in the scope of the experiment, including ones that were defined
# prior to the experiment (which would otherwise be missed if only
# variables names before and after are compared).
#self.var_start = {k:self.namespace.shell.user_ns[k] for k in self.var_names_start}
def backend_init(self):
pass
def start(self):
#print("Starting IPyExperiments")
# base-line
gc.collect()
self.gpu_clear_cache()
self.running = True
if self.enable:
self.cpu_ram_used_start = self.cpu_ram_used()
self.gpu_ram_used_start = self.gpu_ram_used()
#print(f"gpu used f{self.gpu_ram_used_start}")
self.print_state()
# XXX: perhaps prefix all the prints from exp with some |?
print(
"\n"
) # extra vertical white space, to not mix with user's outputs
# start the per cell sub-system
if self.cl_enable:
self.cl = CellLogger(exp=self, **self.cl_kwargs)
self.cl.start()
else:
self.cl = None
def __enter__(self):
return self
def __exit__(self, *exc):
logger.debug(f"{self.__class__.__name__}::__exit__: {self}")
self.__del__()
def keep_var_names(self, *args):
""" Pass a list of local variable **names** to not be deleted at the end of the experiment """
for x in args:
if not isinstance(x, str):
raise ValueError('expecting variable names as strings')
self.var_names_keep.extend(args)
def get_var_names(self):
""" Return a list of local variables created since the beginning of the experiment """
return self.namespace.who_ls()
# defined by subclasses
def cpu_ram(self):
return 0, 0, 0
def cpu_ram_total(self):
return 0
def cpu_ram_avail(self):
return 0
def cpu_ram_used(self):
return 0
def gpu_ram(self):
return 0, 0, 0
def gpu_ram_used(self):
return 0
def gpu_ram_avail(self):
return 0
def gpu_ram_used_fast(self, gpu_handle):
return 0
def gpu_clear_cache(self):
pass
def _available(self):
return self.cpu_ram_avail(), self.gpu_ram_avail()
def _consumed(self):
cpu_ram_cons = self.cpu_ram_used() - self.cpu_ram_used_start
gpu_ram_cons = self.gpu_ram_used() - self.gpu_ram_used_start
#print(f"gpu started with {self.gpu_ram_used_start}")
#print(f"gpu consumed {gpu_ram_cons}")
return cpu_ram_cons, gpu_ram_cons
def _reclaimed(self):
# return 0s, unless called from finish() after memory reclamation
if self.reclaimed:
cpu_ram_recl = self.cpu_ram_used_start + self.cpu_ram_cons - self.cpu_ram_used(
)
gpu_ram_recl = self.gpu_ram_used_start + self.gpu_ram_cons - self.gpu_ram_used(
)
else:
cpu_ram_recl = 0
gpu_ram_recl = 0
return cpu_ram_recl, gpu_ram_recl
def _data_format(self, cpu_ram_avail, cpu_ram_cons, cpu_ram_recl,
gpu_ram_avail, gpu_ram_cons, gpu_ram_recl):
if self.backend == 'cpu':
return IPyExperimentData(
IPyExperimentMemory(cpu_ram_cons, cpu_ram_recl, cpu_ram_avail),
IPyExperimentMemory(0, 0, 0))
else:
return IPyExperimentData(
IPyExperimentMemory(cpu_ram_cons, cpu_ram_recl, cpu_ram_avail),
IPyExperimentMemory(gpu_ram_cons, gpu_ram_recl, gpu_ram_avail))
@property
def data(self):
""" Return current data """
cpu_ram_avail, gpu_ram_avail = self._available()
cpu_ram_cons, gpu_ram_cons = self._consumed()
cpu_ram_recl, gpu_ram_recl = self._reclaimed()
return self._data_format(cpu_ram_avail, cpu_ram_cons, cpu_ram_recl,
gpu_ram_avail, gpu_ram_cons, gpu_ram_recl)
def print_state(self):
""" Print memory stats """
if 1: # align
cpu_ram_total, cpu_ram_free, cpu_ram_used = self.cpu_ram()
cpu_ram_util = cpu_ram_used / cpu_ram_total * 100 if cpu_ram_total else 100
vals = [cpu_ram_total, cpu_ram_free, cpu_ram_used]
if self.backend != 'cpu':
gpu_ram_total, gpu_ram_free, gpu_ram_used = self.gpu_ram()
gpu_ram_util = gpu_ram_used / gpu_ram_total * 100 if gpu_ram_total else 100
vals += [gpu_ram_total, gpu_ram_free, gpu_ram_used]
w = int2width(*map(b2mb, vals)) + 1 # some air
print("\n*** Current state:")
print(
f"RAM: {'Used':>{w}} {'Free':>{w}} {'Total':>{w}} {'Util':>{w}}"
)
if 1:
print(
f"CPU: {b2mb(cpu_ram_used):{w},.0f} {b2mb(cpu_ram_free):{w},.0f} {b2mb(cpu_ram_total):{w},.0f} MB {cpu_ram_util:6.2f}% "
)
if self.backend != 'cpu':
print(
f"GPU: {b2mb(gpu_ram_used):{w},.0f} {b2mb(gpu_ram_free):{w},.0f} {b2mb(gpu_ram_total):{w},.0f} MB {gpu_ram_util:6.2f}% "
)
def finish(self):
if self.cl:
logger.debug(self.__class__.__name__ + f"finish: 0 {self}")
self.cl.stop()
self.cl = None # free the CL object
self.running = False
if not self.enable: return
""" Finish the experiment, reclaim memory, return final stats """
print("\n" + self.__class__.__name__ + ": Finishing")
elapsed_time = int(time.time() - self.start_time)
print(
f"\n*** Experiment finished in {time.strftime('%H:%M:%S', time.gmtime(elapsed_time))} (elapsed wallclock time)"
)
# first take the final snapshot of consumed resources
cpu_ram_cons, gpu_ram_cons = self._consumed()
self.cpu_ram_cons = cpu_ram_cons
self.gpu_ram_cons = gpu_ram_cons
# get the new var names since constructor
var_names_cur = self.get_var_names()
#print(var_names_cur)
# XXX: this doesn't work, since some variables get modified during the
# experiment, but indirectly and therefore shouldn't be deleted.
# So the idea of comparing values before and after doesn't quite work.
#
# only newly introduced variables, or variables that have been re-used
# changed_vars = [k for k in var_names_cur
# if not (k in self.var_start and self.namespace.shell.user_ns[k] is self.var_start[k])]
# extract the var names added during the experiment and delete
# them, with the exception of those we were told to preserve
var_names_new = list(
set(var_names_cur) - set(self.var_names_start) -
set(self.var_names_keep))
var_names_deleted = []
var_names_failed_delete = []
for x in var_names_new:
# seems that some vars can disappear, so we need to check they are still there
if x in self.namespace.shell.user_ns:
# make sure not to delete objects of the same type as self (previous
# instances of the same)
if type(self.namespace.shell.user_ns[x]) != type(self):
# and even then it sometimes fails
try:
self.namespace.xdel(x)
var_names_deleted.append(x)
except:
#print(f"failed to delete {x}: xdel")
var_names_failed_delete.append(x)
else:
#print(f"failed to delete {x}, not in user_ns")
var_names_failed_delete.append(x)
if self.var_names_keep or var_names_deleted:
print("\n*** Newly defined local variables:")
if var_names_deleted:
print("Deleted:", ", ".join(sorted(var_names_deleted)))
if var_names_failed_delete:
print("Failed to delete:",
", ".join(sorted(var_names_failed_delete)))
if self.var_names_keep:
print("Kept: ", ", ".join(sorted(self.var_names_keep)))
# cleanup and reclamation
collected = gc.collect()
if collected:
print(
"\n*** Circular ref objects gc collected during the experiment:"
)
print(f"cleared {collected} objects (only temporary leakage)")
if len(gc.garbage):
print("\n*** Potential memory leaks during the experiment:")
print(f"uncollected gc.garbage of {len(gc.garbage)} objects")
# now we can attempt to reclaim GPU memory
self.gpu_clear_cache()
self.reclaimed = True
# now we can measure how much was reclaimed
cpu_ram_recl, gpu_ram_recl = self._reclaimed()
cpu_ram_pct = cpu_ram_recl / cpu_ram_cons if cpu_ram_cons else 1
gpu_ram_pct = gpu_ram_recl / gpu_ram_cons if gpu_ram_cons else 1
if 1: # align
vals = [cpu_ram_cons, cpu_ram_recl]
if self.backend != 'cpu':
vals += [gpu_ram_cons, gpu_ram_recl]
w = int2width(*map(b2mb, vals)) + 1 # some air
if w < 8: w = 8 # accommodate header width
print("\n*** Experiment memory:")
print(f"RAM: {'Consumed':>{w}} {'Reclaimed':>{w}}")
if 1:
print(
f"CPU: {b2mb(cpu_ram_cons):{w},.0f} {b2mb(cpu_ram_recl):{w},.0f} MB ({cpu_ram_pct*100:6.2f}%)"
)
if self.backend != 'cpu':
print(
f"GPU: {b2mb(gpu_ram_cons):{w},.0f} {b2mb(gpu_ram_recl):{w},.0f} MB ({gpu_ram_pct*100:6.2f}%)"
)
self.print_state()
print(
"\n") # extra vertical white space, to not mix with user's outputs
cpu_ram_avail, gpu_ram_avail = self._available()
return self._data_format(cpu_ram_avail, cpu_ram_cons, cpu_ram_recl,
gpu_ram_avail, gpu_ram_cons, gpu_ram_recl)
def __del__(self):
logger.debug(f"{self.__class__.__name__}::__del__: {self}")
# if explicit finish() wasn't called, do it on self-destruction
if self.running: self.finish()
def _deepnote_get_var_list():
import json
from sys import getsizeof
import numpy as np
import pandas as pd
from IPython import get_ipython
from IPython.core.magics.namespace import NamespaceMagics
MAX_CONTENT_LENGTH = 500
_nms = NamespaceMagics()
_Jupyter = get_ipython()
_nms.shell = _Jupyter.kernel.shell
def _getsizeof(x):
# return the size of variable x. Amended version of sys.getsizeof
# which also supports ndarray, Series and DataFrame
try:
if isinstance(x, (np.ndarray, pd.Series)):
return x.nbytes
elif isinstance(x, pd.DataFrame):
return x.memory_usage().sum()
else:
return getsizeof(x)
except:
None
def _getshapeof(x):
# returns the shape of x if it has one
# returns None otherwise - might want to return an empty string for an empty collum
try:
return x.shape
except AttributeError: # x does not have a shape
return None
def _getunderlyingdatatype(x):
# returns the underlying datatype of x if it has one
# returns None otherwise
try:
return x.dtype.name
except AttributeError: # x does not have an underlying dtype
return None
def _getcontentof(x):
try:
if type(x).__name__ == 'DataFrame':
colnames = ', '.join(x.columns.map(str))
content = "Column names: %s" % colnames
elif type(x).__name__ == 'Series':
content = "Series [%d rows]" % x.shape
elif type(x).__name__ == 'ndarray':
content = x.__repr__()
else:
if hasattr(x, '__len__') and len(x) > MAX_CONTENT_LENGTH:
content = str(x[:MAX_CONTENT_LENGTH]) + "…"
else:
content = str(x)
if len(content) > MAX_CONTENT_LENGTH:
return content[:MAX_CONTENT_LENGTH] + "…"
else:
return content
except:
return None
def _get_number_of_elements(x):
try:
return len(x)
except:
return None
def _get_number_of_columns(x):
try:
if isinstance(x, pd.DataFrame):
return len(x.columns)
else:
return None
except:
return None
def to_int(x):
# for JSON serialization purposes, we need to convert numpy integers to standard integers
return int(x) if x else None
def _get_dict_entry(var_name):
try:
v = eval(var_name)
shape = _getshapeof(v)
underlying_data_type = _getunderlyingdatatype(v)
return {'varName': var_name,
'varType': type(v).__name__,
'varSize': to_int(_getsizeof(v)),
'varShape': str(shape) if shape else '',
'varContent': _getcontentof(v) or '',
'varUnderlyingType': str(underlying_data_type) if underlying_data_type else '',
'numElements': to_int(_get_number_of_elements(v)),
'numColumns': to_int(_get_number_of_columns(v)) }
except LookupError as e:
return None
variables = _nms.who_ls()
variables = filter(lambda v: v not in ['_html', '_nms', 'NamespaceMagics', '_Jupyter'], variables)
variables = filter(lambda v: type(eval(v)).__name__ not in ['module', 'function', 'builtin_function_or_method', 'instance', '_Feature', 'type', 'ufunc'], variables)
variables_dic = {v: _get_dict_entry(v) for v in variables}
variables_dic = {k: v for k, v in variables_dic.items() if v is not None}
return json.dumps({'variables': variables_dic})
import json
from sys import getsizeof
from IPython import get_ipython
from IPython.core.magics.namespace import NamespaceMagics
_nms = NamespaceMagics()
_Jupyter = get_ipython()
_nms.shell = _Jupyter.kernel.shell
try:
import numpy as np
except ImportError:
pass
def _getsizeof(x):
# return the size of variable x. Amended version of sys.getsizeof
# which also supports ndarray, Series and DataFrame
if type(x).__name__ in ['ndarray', 'Series']:
return x.nbytes
elif type(x).__name__ == 'DataFrame':
return x.memory_usage().sum()
else:
return getsizeof(x)
def _getshapeof(x):
#returns the shape of x if it has one
#returns None otherwise - might want to return an empty string for an empty column
try:
return x.shape
except AttributeError: #x does not have a shape
return None
import json
from IPython import get_ipython
from IPython.core.magics.namespace import NamespaceMagics
_nms = NamespaceMagics()
_Jupyter = get_ipython()
_nms.shell = _Jupyter.kernel.shell
def dataframes_info():
values = _nms.who_ls()
info = {
v: (eval(v).columns.tolist())
for v in values if type(eval(v)).__name__ == 'DataFrame'
}
return json.dumps(info)
class IPyExperiments():
"Create an experiment with time/memory checkpoints"
def __init__(self, backend='pytorch'):
""" Instantiate an object with parameters:
backend: 'cpu', 'pytorch' (default), ...
"""
self.backend_load(backend)
print("\n*** Starting experiment...")
self.running = True
self.reclaimed = False
self.start_time = time.time()
self.var_names_keep = []
# base-line
gc.collect()
gpu_clear_cache()
# grab the notebook var names during creation
ipython = get_ipython()
self.namespace = NamespaceMagics()
self.namespace.shell = ipython.kernel.shell
self.var_names_start = self.get_var_names()
#print(self.var_names_start)
self.gen_ram_used_start = gen_ram_used()
self.gpu_ram_used_start = gpu_ram_used()
#print(f"gpu used f{self.gpu_ram_used_start}" )
self.print_state()
print("\n") # extra vertical white space, to not mix with user's outputs
def __enter__(self):
return self
def __exit__(self, *exc):
self.__del__()
# currently supporting:
# - cpu: no gpu backend
# - pytorch: (default)
#
# it can be easily expanded to support other backends if needed
#
# in order to add a new backend, we need:
# 1. import backend module
# 2. preload code that claims unreclaimable gpu memory
# 3. function that returns the current gpu id
# 4. function that releases the cache if any
def backend_load(self, backend='pytorch'):
""" Load one of the supported backends before running the first experiment, to set things up.
"""
global gpu_ram
global gpu_current_device, gpu_clear_cache
print(f"*** Loading backend: {backend}")
self.backend = backend
if backend == 'cpu':
# send 0's to any gpu inquiry
gpu_ram = gpu_ram_zeros
_, _, _ = gpu_ram() # check that all is ready to go
return
# gpu backends below
gpu_ram = gpu_ram_real
# initialize pynvml
pynvml.nvmlInit()
if backend == 'pytorch':
import torch, torch.cuda
# sanity check
if not torch.cuda.is_available():
raise Exception(f"torch.cuda.is_available() returns False; can't continue")
# force pytorch to load cuDNN and its kernels to claim unreclaimable memory
torch.ones((1, 1)).cuda()
gpu_current_device = torch.cuda.current_device
gpu_clear_cache = torch.cuda.empty_cache
_, _, _ = gpu_ram() # check that all is ready to go
return
raise ValueError(f"backend {backend} isn't yet supported; please submit a request at https://github.com/stas00/ipyexperiments/issues")
def keep_var_names(self, *args):
""" Pass a list of local variable **names** to not be deleted at the end of the experiment """
for x in args:
if not isinstance(x, str):
raise ValueError('expecting variable names as strings')
self.var_names_keep.extend(args)
def get_var_names(self):
""" Return a list of local variables created since the beginning of the experiment """
return self.namespace.who_ls()
def _available(self): return gen_ram_avail(), gpu_ram_avail()
def _consumed(self):
gen_ram_cons = gen_ram_used() - self.gen_ram_used_start
gpu_ram_cons = gpu_ram_used() - self.gpu_ram_used_start
#print(f"gpu started with {self.gpu_ram_used_start}")
#print(f"gpu consumed {gpu_ram_cons}")
return gen_ram_cons, gpu_ram_cons
def _reclaimed(self):
# return 0s, unless called from finish() after memory reclamation
if self.reclaimed:
gen_ram_recl = self.gen_ram_used_start + self.gen_ram_cons - gen_ram_used()
gpu_ram_recl = self.gpu_ram_used_start + self.gpu_ram_cons - gpu_ram_used()
else:
gen_ram_recl = 0
gpu_ram_recl = 0
return gen_ram_recl, gpu_ram_recl
def _format_stats(self, gen_ram_avail, gpu_ram_avail, gen_ram_cons, gpu_ram_cons, gen_ram_recl, gpu_ram_recl ):
cons = {'gen_ram': gen_ram_cons, 'gpu_ram': gpu_ram_cons }
recl = {'gen_ram': gen_ram_recl, 'gpu_ram': gpu_ram_recl }
avail = {'gen_ram': gen_ram_avail, 'gpu_ram': gpu_ram_avail}
return cons, recl, avail
def get_stats(self):
""" Return current stats """
gen_ram_avail, gpu_ram_avail = self._available()
gen_ram_cons, gpu_ram_cons = self._consumed()
gen_ram_recl, gpu_ram_recl = self._reclaimed()
return self._format_stats(gen_ram_avail, gpu_ram_avail, gen_ram_cons, gpu_ram_cons, gen_ram_recl, gpu_ram_recl)
def print_state(self):
""" Print memory stats (not exact due to pytorch memory caching) """
print("\n*** Current state:")
print("Gen RAM Free {0:>7s} | Proc size {1}".format(
hs(gen_ram_avail()), hs(gen_ram_used())))
if self.backend == 'cpu': return
gpu_ram_total, gpu_ram_free, gpu_ram_used = gpu_ram()
gpu_ram_util = gpu_ram_used/gpu_ram_free*100 if gpu_ram_free else 100
print("GPU RAM Free {0:>7s} | Used {1} | Util {2:2.1f}% | Total {3}".format(
hs(gpu_ram_free), hs(gpu_ram_used), gpu_ram_util, hs(gpu_ram_total)))
def finish(self):
""" Finish the experiment, reclaim memory, return final stats """
print("\n*** Finishing experiment...")
self.running = False
# first take the final snapshot of consumed resources
gen_ram_cons, gpu_ram_cons = self._consumed()
self.gen_ram_cons = gen_ram_cons
self.gpu_ram_cons = gpu_ram_cons
# get the new var names since constructor
var_names_cur = self.get_var_names()
#print(var_names_cur)
# extract the var names added during the experiment and delete
# them, with the exception of those we were told to preserve
var_names_new = list(set(var_names_cur) - set(self.var_names_start) - set(self.var_names_keep))
print("\n*** Deleting the following local variables:")
print(sorted(var_names_new))
for x in var_names_new: self.namespace.xdel(x)
if self.var_names_keep:
print("\n*** Keeping the following local variables:")
print(sorted(self.var_names_keep))
# cleanup and reclamation
collected = gc.collect()
if collected or len(gc.garbage):
print("\n*** Potential memory leaks during the experiment:")
if collected: print(f"cleared {collected} objects")
if len(gc.garbage): print(f"leaked garbage of {len(gc.garbage)} objects")
# now we can attempt to reclaim GPU memory
gpu_clear_cache()
self.reclaimed = True
# now we can measure how much was reclaimed
gen_ram_recl, gpu_ram_recl = self._reclaimed()
gen_ram_pct = gen_ram_recl/gen_ram_cons if gen_ram_cons else 1
gpu_ram_pct = gpu_ram_recl/gpu_ram_cons if gpu_ram_cons else 1
print("\n*** RAM consumed during the experiment:")
print(f"Gen: {hs(gen_ram_cons) }")
if self.backend != 'cpu':
print(f"GPU: {hs(gpu_ram_cons)}")
print("\n*** RAM reclaimed at the end of the experiment:")
print(f"Gen: {hs(gen_ram_recl)} ({gen_ram_pct*100:.2f}%)")
if self.backend != 'cpu':
print(f"GPU: {hs(gpu_ram_recl)} ({gpu_ram_pct*100:.2f}%)")
elapsed_time = int(time.time() - self.start_time)
print("\n*** Elapsed wallclock time:")
print(f"{time.strftime('%H:%M:%S', time.gmtime(elapsed_time))}")
self.print_state()
print("\n") # extra vertical white space, to not mix with user's outputs
gen_ram_avail, gpu_ram_avail = self._available()
return self._format_stats(gen_ram_avail, gpu_ram_avail, gen_ram_cons, gpu_ram_cons, gen_ram_recl, gpu_ram_recl)
def __del__(self):
# if explicit finish() wasn't called, do it on self-destruction
if self.running: self.finish()