class RateLimitAggregator(AbstractContextManager): # type: ignore
"""
Runs the rate limits provided by the `rate_limit_params` configuration object.
It runs the rate limits in the order described by `rate_limit_params`.
"""
def __init__(self,
rate_limit_params: Sequence[RateLimitParameters]) -> None:
self.rate_limit_params = rate_limit_params
self.stack = ExitStack()
def __enter__(self) -> RateLimitStatsContainer:
stats = RateLimitStatsContainer()
for rate_limit_param in self.rate_limit_params:
child_stats = self.stack.enter_context(
rate_limit(rate_limit_param))
if child_stats:
stats.add_stats(rate_limit_param.rate_limit_name, child_stats)
return stats
def __exit__(
self,
exc_type: Optional[Type[BaseException]],
exc_val: Optional[BaseException],
exc_tb: Optional[TracebackType],
) -> None:
self.stack.pop_all().close()
class GitHubApi:
def __init__(self, auth_token=None, url_base=DEFAULT_URL_BASE):
self._establish_connection = partial(Connection, url_base,
auth_token=auth_token)
self._connection = None
self._exit_stack = ExitStack()
def __enter__(self):
self._connection = self._exit_stack.enter_context(
self._establish_connection()
)
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self._connection = None
self._exit_stack.pop_all()
self._exit_stack = None
def get_repository_api(self, fullname):
if not self._connection:
raise RuntimeError(
'GitHubApi should be used within context manager flow'
)
get_repo = self._connection.get(f'{self._connection.url_base}/repos/{fullname}')
return RepositoryApi(self._connection, get_repo.json())
class RateLimitAggregator(AbstractContextManager): # type: ignore
"""
Runs the rate limits provided by the `rate_limit_params` configuration object.
It runs the rate limits in the order described by `rate_limit_params`.
"""
def __init__(self,
rate_limit_params: Sequence[RateLimitParameters]) -> None:
self.rate_limit_params = rate_limit_params
self.stack = ExitStack()
def __enter__(self) -> RateLimitStatsContainer:
stats = RateLimitStatsContainer()
for rate_limit_param in self.rate_limit_params:
try:
child_stats = self.stack.enter_context(
rate_limit(rate_limit_param))
if child_stats:
stats.add_stats(rate_limit_param.rate_limit_name,
child_stats)
except RateLimitExceeded as e:
# If an exception occurs in one of the rate limiters, the __exit__ callbacks are not
# called since the error happened in the __enter__ method and not in the context
# block itself. In the case that one of the rate limiters caught a limit, we need
# these exit functions to be called so we can roll back any limits that were set
# earlier in the stack.
self.__exit__(*sys.exc_info())
_record_metrics(e, rate_limit_param)
raise e
return stats
def __exit__(
self,
exc_type: Optional[Type[BaseException]],
exc_val: Optional[BaseException],
exc_tb: Optional[TracebackType],
) -> None:
self.stack.pop_all().close()
class atomically_replaced_file:
def __init__(self, path, mode='w', buffering=-1):
"""Return a context manager supporting the atomic replacement of a file.
The context manager yields an open file object that has been
created in a mkdtemp-style temporary directory in the same
directory as the path. The temporary file will be renamed to
the target path (atomically if the platform allows it) if
there are no exceptions, and the temporary directory will
always be removed. Calling cancel() will prevent the
replacement.
The file object will have a name attribute containing the
file's path, and the mode and buffering arguments will be
handled exactly as with open(). The resulting permissions
will also match those produced by open().
E.g.::
with atomically_replaced_file('foo.txt', 'w') as f:
f.write('hello jack.')
"""
assert 'w' in mode
self.path = path
self.mode = mode
self.buffering = buffering
self.canceled = False
self.tmp_path = None
self.cleanup = ExitStack()
def __enter__(self):
with self.cleanup:
parent, name = os.path.split(self.path)
tmpdir = self.cleanup.enter_context(
temp_dir(dir=parent, prefix=name + b'-'))
self.tmp_path = tmpdir + b'/pending'
f = open(self.tmp_path, mode=self.mode, buffering=self.buffering)
f = self.cleanup.enter_context(f)
self.cleanup = self.cleanup.pop_all()
return f
def __exit__(self, exc_type, exc_value, traceback):
with self.cleanup:
if not (self.canceled or exc_type):
os.rename(self.tmp_path, self.path)
def cancel(self):
self.canceled = True
class Scope:
"""
A context manager that allows to register error and exit callbacks.
"""
_thread_locals = threading.local()
@frozen
class _ExitHandler:
callback: Callable[[], Any]
ignore_errors: bool = True
def __exit__(self, exc_type, exc_value, exc_traceback):
try:
self.callback()
except Exception:
if not self.ignore_errors:
raise
@frozen
class _ErrorHandler(_ExitHandler):
def __exit__(self, exc_type, exc_value, exc_traceback):
if exc_type:
return super().__exit__(exc_type=exc_type,
exc_value=exc_value,
exc_traceback=exc_traceback)
def __init__(self):
self._stack = ExitStack()
self.enabled = True
def on_error_do(self,
callback: Callable,
*args,
kwargs: Optional[Dict[str, Any]] = None,
ignore_errors: bool = False):
"""
Registers a function to be called on scope exit because of an error.
If ignore_errors is True, the errors from this function call
will be ignored.
"""
self._register_callback(self._ErrorHandler,
ignore_errors=ignore_errors,
callback=callback,
args=args,
kwargs=kwargs)
def on_exit_do(self,
callback: Callable,
*args,
kwargs: Optional[Dict[str, Any]] = None,
ignore_errors: bool = False):
"""
Registers a function to be called on scope exit.
"""
self._register_callback(self._ExitHandler,
ignore_errors=ignore_errors,
callback=callback,
args=args,
kwargs=kwargs)
def _register_callback(self,
handler_type,
callback: Callable,
args: Tuple[Any] = None,
kwargs: Dict[str, Any] = None,
ignore_errors: bool = False):
if args or kwargs:
callback = partial(callback, *args, **(kwargs or {}))
self._stack.push(handler_type(callback, ignore_errors=ignore_errors))
def add(self, cm: ContextManager[T]) -> T:
"""
Enters a context manager and adds it to the exit stack.
Returns: cm.__enter__() result
"""
return self._stack.enter_context(cm)
def enable(self):
self.enabled = True
def disable(self):
self.enabled = False
def close(self):
self.__exit__(None, None, None)
def __enter__(self) -> Scope:
return self
def __exit__(self, exc_type, exc_value, exc_traceback):
if not self.enabled:
return
self._stack.__exit__(exc_type, exc_value, exc_traceback)
self._stack.pop_all() # prevent issues on repetitive calls
@classmethod
def current(cls) -> Scope:
return cls._thread_locals.current
@contextmanager
def as_current(self):
previous = getattr(self._thread_locals, 'current', None)
self._thread_locals.current = self
try:
yield
finally:
self._thread_locals.current = previous
class State:
def __init__(self):
# Variables which manage state transitions.
self._next = deque()
self._debug_step = 0
# Manage all resources so they get cleaned up whenever the state
# machine exits for any reason.
self.resources = ExitStack()
def close(self):
# Transfer all resources to a new ExitStack, and release them from
# there. That way, if .close() gets called more than once, only the
# first call will release the resources, while subsequent ones will
# no-op.
self.resources.pop_all().close()
def __enter__(self):
return self
def __exit__(self, *exception):
self.close()
# Don't suppress any exceptions.
return False
def __del__(self):
self.close()
def __iter__(self):
return self
# We can't pickle the resources ExitStack, so if there's anything
# valuable in there, the subclass must override __getstate__() and
# __setstate__() as appropriate.
def __getstate__(self):
return dict(
state=[function.__name__ for function in self._next],
debug_step=self._debug_step,
)
def __setstate__(self, state):
self._next = deque()
for name in state['state']:
self._next.append(getattr(self, name))
self._debug_step = state['debug_step']
self.resources = ExitStack()
def _pop(self):
step = self._next.popleft()
# step could be a partial or a method.
name = getattr(step, 'func', step).__name__
log.debug('-> [{:2}] {}'.format(self._debug_step, name))
return step, name
def __next__(self):
try:
step, name = self._pop()
step()
self._debug_step += 1
except IndexError:
# Do not chain the exception.
self.close()
raise StopIteration from None
except:
log.exception(
'uncaught exception in state machine step: [{}] {}'.format(
self._debug_step, name))
self.close()
raise
def run_thru(self, stop_after):
"""Partially run the state machine.
Note that any resources maintained by this state machine are
*not* automatically cleaned up when .run_thru() completes,
unless an exception occurrs, because execution can be continued.
Call .close() explicitly to release the resources.
:param stop_after: Name or step number of the method to run the state
machine through. In other words, the state machine runs until the
specified step completes. Step numbers begin at 0.
"""
while True:
try:
step, name = self._pop()
except (StopIteration, IndexError):
# We're done.
break
try:
step()
except:
self.close()
raise
try:
if name == stop_after or self._debug_step == stop_after:
break
finally:
self._debug_step += 1
def run_until(self, stop_before):
"""Partially run the state machine.
Note that any resources maintained by this state machine are
*not* automatically cleaned up when .run_until() completes,
unless an exception occurs, because execution can be continued.
Call .close() explicitly to release the resources.
:param stop_before: Name or step number of method that the state
machine is run until the specified step is reached. Unlike
`run_thru()` the step is not run. Step numbers begin at 0.
"""
while True:
try:
step, name = self._pop()
except (StopIteration, IndexError):
# We're done.
break
if name == stop_before or self._debug_step == stop_before:
# Stop executing, but not before we push the last state back
# onto the deque. Otherwise, resuming the state machine would
# skip this step.
self._next.appendleft(step)
break
try:
step()
except:
self.close()
raise
self._debug_step += 1
class HLinkDB:
def __init__(self, filename):
self.closed = False
self._cleanup = ExitStack()
self._filename = filename
self._pending_save = None
# Map a "dev:ino" node to a list of paths associated with that node.
self._node_paths = pickle_load(filename) or {}
# Map a path to a "dev:ino" node (a reverse hard link index).
self._path_node = {}
for node, paths in self._node_paths.items():
for path in paths:
self._path_node[path] = node
def prepare_save(self):
""" Commit all of the relevant data to disk. Do as much work
as possible without actually making the changes visible."""
if self._pending_save:
raise Error('save of %r already in progress' % self._filename)
with self._cleanup:
if self._node_paths:
dir, name = os.path.split(self._filename)
self._pending_save = atomically_replaced_file(self._filename,
mode='wb',
buffering=65536)
with self._cleanup.enter_context(self._pending_save) as f:
pickle.dump(self._node_paths, f, 2)
else: # No data
self._cleanup.callback(lambda: unlink(self._filename))
self._cleanup = self._cleanup.pop_all()
def commit_save(self):
self.closed = True
if self._node_paths and not self._pending_save:
raise Error('cannot commit save of %r; no save prepared' %
self._filename)
self._cleanup.close()
self._pending_save = None
def abort_save(self):
self.closed = True
with self._cleanup:
if self._pending_save:
self._pending_save.cancel()
self._pending_save = None
def __enter__(self):
return self
def __exit__(self, type, value, traceback):
self.abort_save()
def __del__(self):
assert self.closed
def add_path(self, path, dev, ino):
# Assume path is new.
node = b'%d:%d' % (dev, ino)
self._path_node[path] = node
link_paths = self._node_paths.get(node)
if link_paths and path not in link_paths:
link_paths.append(path)
else:
self._node_paths[node] = [path]
def _del_node_path(self, node, path):
link_paths = self._node_paths[node]
link_paths.remove(path)
if not link_paths:
del self._node_paths[node]
def change_path(self, path, new_dev, new_ino):
prev_node = self._path_node.get(path)
if prev_node:
self._del_node_path(prev_node, path)
self.add_path(new_dev, new_ino, path)
def del_path(self, path):
# Path may not be in db (if updating a pre-hardlink support index).
node = self._path_node.get(path)
if node:
self._del_node_path(node, path)
del self._path_node[path]
def node_paths(self, dev, ino):
node = b'%d:%d' % (dev, ino)
return self._node_paths[node]
class State:
def __init__(self):
# Variables which manage state transitions.
self._next = deque()
self._debug_step = 1
# Manage all resources so they get cleaned up whenever the state
# machine exits for any reason.
self.resources = ExitStack()
def close(self):
# Transfer all resources to a new ExitStack, and release them from
# there. That way, if .close() gets called more than once, only the
# first call will release the resources, while subsequent ones will
# no-op.
self.resources.pop_all().close()
def __enter__(self):
return self
def __exit__(self, *exception):
self.close()
# Don't suppress any exceptions.
return False
def __del__(self):
self.close()
def __iter__(self):
return self
def _pop(self):
step = self._next.popleft()
# step could be a partial or a method.
name = getattr(step, 'func', step).__name__
log.debug('-> [{:2}] {}'.format(self._debug_step, name))
return step, name
def __next__(self):
try:
step, name = self._pop()
step()
self._debug_step += 1
except IndexError:
# Do not chain the exception.
self.close()
raise StopIteration from None
except:
log.exception('uncaught exception in state machine')
self.close()
raise
def run_thru(self, stop_after):
"""Partially run the state machine.
Note that any resources maintained by this state machine are
*not* automatically cleaned up when .run_thru() completes,
unless an exception occurrs, because execution can be continued.
Call .close() explicitly to release the resources.
:param stop_after: Name of method to run the state machine
through. In other words, the state machine runs until the
named method completes.
"""
while True:
try:
step, name = self._pop()
except (StopIteration, IndexError):
# We're done.
break
try:
step()
except:
self.close()
raise
self._debug_step += 1
if name == stop_after:
break
def run_until(self, stop_before):
"""Partially run the state machine.
Note that any resources maintained by this state machine are
*not* automatically cleaned up when .run_until() completes,
unless an exception occurs, because execution can be continued.
Call .close() explicitly to release the resources.
:param stop_before: Name of method that the state machine is run
until the method is reached. Unlike `run_thru()` the named
method is not run.
"""
while True:
try:
step, name = self._pop()
except (StopIteration, IndexError):
# We're done.
break
if name == stop_before:
# Stop executing, but not before we push the last state back
# onto the deque. Otherwise, resuming the state machine would
# skip this step.
self._next.appendleft(step)
break
try:
step()
except:
self.close()
raise
self._debug_step += 1
