def refresh(self: 'DataSymbol'):
self.fresher_ancestors = set()
self.defined_cell_num = cell_counter()
self.namespace_stale_symbols = set()
if self.safety.config.get('use_new_update_protocol', False):
return
self.required_cell_num = self.defined_cell_num
self._propagate_refresh_to_namespace_parents(set())
def _propagate_update_to_deps(self, dsym: 'DataSymbol'):
# print(self, 'propagate to child deps', self.children)
if dsym.should_mark_stale(self.updated_sym):
# if self.full_namespace_path == updated_dep.full_namespace_path:
# print('weird', self.full_namespace_path, self, updated_dep, self.obj_id, updated_dep.obj_id, self.cached_obj_id, updated_dep.cached_obj_id)
dsym.fresher_ancestors.add(self.updated_sym)
dsym.required_cell_num = cell_counter()
for child in dsym.children:
# print(self, 'prop to', child, self._get_children_to_skip())
self._propagate_update(child, child._get_obj())
def _propagate_refresh_to_namespace_parents(self, seen: 'Set[DataSymbol]'):
if self in seen:
return
# print('refresh propagate', self)
seen.add(self)
for self_alias in self.safety.aliases[self.obj_id]:
containing_scope: 'NamespaceScope' = cast(
'NamespaceScope', self_alias.containing_scope)
if not containing_scope.is_namespace_scope:
continue
# if containing_scope.max_defined_timestamp == cell_counter():
# return
containing_scope.max_defined_timestamp = cell_counter()
containing_namespace_obj_id = containing_scope.obj_id
# print('containing namespaces:', self.safety.aliases[containing_namespace_obj_id])
for alias in self.safety.aliases[containing_namespace_obj_id]:
alias.namespace_stale_symbols.discard(self)
if not alias.is_stale:
alias.defined_cell_num = cell_counter()
alias.fresher_ancestors = set()
# print('working on', alias, '; stale?', alias.is_stale, alias.namespace_stale_symbols)
alias._propagate_refresh_to_namespace_parents(seen)
def __init__(
self,
name: 'Union[str, int]',
symbol_type: 'DataSymbolType',
obj: 'Any',
containing_scope: 'Scope',
safety: 'NotebookSafety',
stmt_node: 'Optional[ast.AST]' = None,
parents: 'Optional[Set[DataSymbol]]' = None,
refresh_cached_obj=False,
):
# print(containing_scope, name, obj, is_subscript)
self.name = name
self.symbol_type = symbol_type
tombstone, obj_ref, has_weakref = self._update_obj_ref_inner(obj)
self._tombstone = tombstone
self._obj_ref = obj_ref
self._has_weakref = has_weakref
self.cached_obj_ref = None
self._cached_has_weakref = None
self.cached_obj_id = None
self.cached_obj_type = None
if refresh_cached_obj:
self._refresh_cached_obj()
self.containing_scope = containing_scope
self.safety = safety
self.stmt_node = self.update_stmt_node(stmt_node)
self._funcall_live_symbols = None
if parents is None:
parents = set()
self.parents: Set[DataSymbol] = parents
self.children: Set[DataSymbol] = set()
self.call_scope: 'Optional[Scope]' = None
if self.is_function:
self.call_scope = self.containing_scope.make_child_scope(self.name)
self.defined_cell_num = cell_counter()
# The notebook cell number this is required to have to not be considered stale
self.required_cell_num = self.defined_cell_num
self.fresher_ancestors: Set[DataSymbol] = set()
self.namespace_stale_symbols: Set[DataSymbol] = set()
# Will never be stale if no_warning is True
self.disable_warnings = False
def _propagate_staleness_to_deps(self,
dsym: 'DataSymbol',
skip_seen_check=False):
if not skip_seen_check and dsym in self.seen:
return
self.seen.add(dsym)
if dsym not in self.safety.updated_symbols:
if dsym.should_mark_stale(self.updated_sym):
dsym.fresher_ancestors.add(self.updated_sym)
dsym.required_cell_num = cell_counter()
self._propagate_staleness_to_namespace_parents(
dsym, skip_seen_check=True)
self._propagate_staleness_to_namespace_children(
dsym, skip_seen_check=True)
for child in self._non_class_to_instance_children(dsym):
# print('propagate from', dsym, 'to', child)
self._propagate_staleness_to_deps(child)
def safe_execute(self, cell: str, run_cell_func):
try:
cell = black.format_file_contents(cell,
fast=False,
mode=black.FileMode())
except: # noqa
pass
with save_number_of_currently_executing_cell():
self._last_execution_counter = cell_counter()
for line in cell.strip().split('\n'):
if _NB_MAGIC_PATTERN.search(line) is None:
break
else:
return run_cell_func(cell)
if self._active_cell_id is not None:
self._counters_by_cell_id[
self._active_cell_id] = self._last_execution_counter
self._active_cell_id = None
# Stage 1: Precheck.
if self._precheck_for_stale(
cell) and self.config.skip_unsafe_cells:
# FIXME: hack to increase cell number
# ideally we shouldn't show a cell number at all if we fail precheck since nothing executed
return run_cell_func('None')
def _backup():
# something went wrong silently (e.g. due to line magic); fall back to just executing the code
logger.warning(
'Something failed while attempting traced execution; '
'falling back to uninstrumented execution.')
return run_cell_func(cell)
# Stage 2: Trace / run the cell, updating dependencies as they are encountered.
try:
with self._tracing_context():
ret = run_cell_func(cell)
finally:
if self.trace_state.error_occurred:
ret = _backup()
return ret
def _collect_updated_symbols(self, dsym: 'DataSymbol', skip_aliases=False):
if skip_aliases:
aliases_to_consider = {dsym}
else:
aliases_to_consider = self.safety.aliases[dsym.obj_id]
for dsym_alias in aliases_to_consider:
if dsym_alias in self.seen:
continue
self.seen.add(dsym_alias)
containing_scope: 'NamespaceScope' = cast(
'NamespaceScope', dsym_alias.containing_scope)
if not containing_scope.is_namespace_scope:
continue
# TODO: figure out what this is for again
# self.safety.updated_scopes.add(containing_scope)
containing_scope.max_defined_timestamp = cell_counter()
containing_namespace_obj_id = containing_scope.obj_id
for alias in self.safety.aliases[containing_namespace_obj_id]:
alias.namespace_stale_symbols.discard(dsym)
self._collect_updated_symbols(alias)
def _precheck_for_stale(self, cell: str):
# Precheck process. First obtain the names that need to be checked. Then we check if their
# `defined_cell_num` is greater than or equal to required; if not we give a warning and return `True`.
try:
cell_ast = self._get_cell_ast(cell)
except SyntaxError:
return False
self.statement_cache[cell_counter()] = compute_lineno_to_stmt_mapping(
cell_ast)
stale_symbols, _, live_symbols, _ = self._precheck_stale_nodes(
cell_ast)
if self._last_refused_code is None or cell != self._last_refused_code:
self._prev_cell_stale_symbols = stale_symbols
if len(stale_symbols) > 0:
warning_counter = 0
for node in self._prev_cell_stale_symbols:
if warning_counter >= _MAX_WARNINGS:
logger.warning(
f'{len(self._prev_cell_stale_symbols) - warning_counter}'
' more nodes with stale dependencies skipped...')
break
_safety_warning(node)
warning_counter += 1
self.stale_dependency_detected = True
self._last_refused_code = cell
return True
else:
# Instead of breaking the dependency chain, simply refresh the nodes
# with stale deps to their required cell numbers
for node in self._prev_cell_stale_symbols:
node.defined_cell_num = node.required_cell_num
node.namespace_stale_symbols = set()
node.fresher_ancestors = set()
self._prev_cell_stale_symbols.clear()
self._last_refused_code = None
self._resync_symbols(live_symbols)
return False
def refresh(self):
self.max_defined_timestamp = cell_counter()
def _propagate_update(self, dsym: 'DataSymbol', new_parent_obj: 'Any', refresh=False):
# look at old obj_id and cur obj_id
# walk down namespace hierarchy from old obj_id, and track corresponding DCs from cur obj_id
# a few cases to consider:
# 1. mismatched obj ids or unavailable from new hierarchy:
# mark old dc as mutated AND stale, and propagate to old dc children
# 2. new / old DataSymbols have same obj ids:
# mark old dc as mutated, but NOT stale, and propagate to children
# Q: should old dc additionally be refreshed?
# Technically it should already be fresh, since if it's still a descendent of this namespace, we probably
# had to ref the namespace ancestor, which should have been caught by the checker if the descendent has
# some other stale ancestor. If not fresh, let's mark it so and log a warning about a potentially stale usage
if dsym._tombstone or dsym in self.seen:
return
self.seen.add(dsym)
# print('propagate update from', self)
new_id = None if new_parent_obj is NOT_FOUND else id(new_parent_obj)
if self.updated_sym is dsym:
old_parent_obj = dsym._get_cached_obj()
old_id = dsym.cached_obj_id
else:
old_parent_obj = dsym._get_obj()
old_id = dsym.obj_id
namespace = self.safety.namespaces.get(old_id, None)
if namespace is None and refresh: # if we are at a leaf
self._propagate_update_to_namespace_parents(dsym, refresh=refresh)
if namespace is not None and (old_id != new_id or self.mutated or not refresh):
for dc in namespace.all_data_symbols_this_indentation(exclude_class=True):
should_refresh = refresh # or updated_dep in set(namespace.all_data_symbols_this_indentation())
dc_in_self_namespace = False
if new_parent_obj is NOT_FOUND:
self._propagate_update(dc, NOT_FOUND, refresh=should_refresh)
else:
try:
obj = dsym._get_obj()
obj_attr_or_sub = retrieve_namespace_attr_or_sub(obj, dc.name, dc.is_subscript)
# print(dc, obj, obj_attr_or_sub, updated_dep, seen, parent_seen, refresh, mutated, old_id, new_id)
self._propagate_update(dc, obj_attr_or_sub, refresh=should_refresh)
dc_in_self_namespace = True
if new_parent_obj is not old_parent_obj and new_id is not None:
new_namespace = self.safety.namespaces.get(new_id, None)
if new_namespace is None:
new_namespace = namespace.shallow_clone(new_parent_obj)
self.safety.namespaces[new_id] = new_namespace
# TODO: handle class data cells properly;
# in fact; we still need to handle aliases of class data cells
if dc.name not in new_namespace.data_symbol_by_name(dc.is_subscript):
new_dc = dc.shallow_clone(obj_attr_or_sub, new_namespace, dc.symbol_type)
new_namespace.put(dc.name, new_dc)
self.safety.updated_symbols.add(new_dc)
except (KeyError, IndexError, AttributeError):
self._propagate_update(dc, NOT_FOUND, refresh=should_refresh)
if dc_in_self_namespace and dc.should_mark_stale(self.updated_sym):
# print(self, 'add', dc, 'to namespace stale symbols due to', updated_dep, self.defined_cell_num, dc.defined_cell_num, updated_dep.defined_cell_num, dc.fresher_ancestors)
dsym.namespace_stale_symbols.add(dc)
else:
dsym.namespace_stale_symbols.discard(dc)
# if mutated or self.cached_obj_id != self.obj_id:
# if (old_id != new_id or not refresh) and not mutated:
# self._propagate_update_to_deps(updated_dep, seen, parent_seen)
# elif mutated:
if old_id != new_id or not refresh or self.mutated:
to_skip = self._get_children_to_skip(dsym)
# self.seen |= to_skip
old_seen = self.seen
self.seen = self.seen | to_skip
self._propagate_update_to_deps(dsym)
self.seen = old_seen
# print(self, 'propagate+skip done')
if self.updated_sym is dsym:
return
if refresh:
for alias in self.safety.aliases[old_id]:
if alias.defined_cell_num < alias.required_cell_num < cell_counter():
logger.debug('possible stale usage of namespace descendent %s' % alias)
if len(alias.namespace_stale_symbols) > 0:
logger.debug('unexpected stale namespace symbols for symbol %s: %s' % (alias, alias.namespace_stale_symbols))
alias.namespace_stale_symbols.clear()
if old_id != new_id or self.mutated:
# TODO: better equality testing
# Doing equality testing properly requires that we still have a reference to old object around;
# we should be using weak refs, which complicates this.
# Depth also makes this challenging.
self._propagate_update_to_deps(alias)
# TODO: this will probably work, but will also unnecessarily propagate the refresh up to
# namespace parents. Need a mechanism to mark for refresh without parent propagation.
self.safety.updated_symbols.add(alias)
else:
# propagating to all aliases was causing problems
# see test `test_class_redef`
self._propagate_update_to_deps(dsym)
# for alias in self.safety.aliases[old_id]:
# # print('propagate', updated_dep, 'to', alias, 'via', self, updated_dep.defined_cell_num, alias.defined_cell_num, self.defined_cell_num)
# alias._propagate_update_to_deps(updated_dep, seen, parent_seen)
if namespace is None and dsym.should_mark_stale(self.updated_sym): # if we are at a leaf
# print('propagate', updated_dep, 'to', self, updated_dep.defined_cell_num, self.defined_cell_num)
self._propagate_update_to_namespace_parents(dsym, refresh=refresh)
def cell_counter(self):
if self.settings.store_history:
return cell_counter()
else:
return self._cell_counter