def pipe(self, texts, as_tuples=False, n_threads=2, batch_size=1000,
disable=[]):
"""Process texts as a stream, and yield `Doc` objects in order.
Supports GIL-free multi-threading.
texts (iterator): A sequence of texts to process.
as_tuples (bool):
If set to True, inputs should be a sequence of
(text, context) tuples. Output will then be a sequence of
(doc, context) tuples. Defaults to False.
n_threads (int): The number of worker threads to use. If -1, OpenMP
will decide how many to use at run time. Default is 2.
batch_size (int): The number of texts to buffer.
disable (list): Names of the pipeline components to disable.
YIELDS (Doc): Documents in the order of the original text.
EXAMPLE:
>>> texts = [u'One document.', u'...', u'Lots of documents']
>>> for doc in nlp.pipe(texts, batch_size=50, n_threads=4):
>>> assert doc.is_parsed
"""
if as_tuples:
text_context1, text_context2 = itertools.tee(texts)
texts = (tc[0] for tc in text_context1)
contexts = (tc[1] for tc in text_context2)
docs = self.pipe(texts, n_threads=n_threads, batch_size=batch_size,
disable=disable)
for doc, context in izip(docs, contexts):
yield (doc, context)
return
docs = (self.make_doc(text) for text in texts)
for name, proc in self.pipeline:
if name in disable:
continue
if hasattr(proc, 'pipe'):
docs = proc.pipe(docs, n_threads=n_threads,
batch_size=batch_size)
else:
# Apply the function, but yield the doc
docs = _pipe(proc, docs)
# Track weakrefs of "recent" documents, so that we can see when they
# expire from memory. When they do, we know we don't need old strings.
# This way, we avoid maintaining an unbounded growth in string entries
# in the string store.
recent_refs = weakref.WeakSet()
old_refs = weakref.WeakSet()
# If there is anything that we have inside — after iterations we should
# carefully get it back.
original_strings_data = list(self.vocab.strings)
nr_seen = 0
for doc in docs:
yield doc
recent_refs.add(doc)
if nr_seen < 10000:
old_refs.add(doc)
nr_seen += 1
elif len(old_refs) == 0:
self.vocab.strings._cleanup_stale_strings()
nr_seen = 0
# We can't know which strings from the last batch have really expired.
# So we don't erase the strings — we just extend with the original
# content.
for string in original_strings_data:
self.vocab.strings.add(string)
def __init__(self, backend=None, engine_list=None):
"""
Initialize the main compiler engine and all compiler engines.
Sets 'next_engine'- and 'main_engine'-attributes of all compiler
engines and adds the back-end as the last engine.
Args:
backend (BasicEngine): Backend to send the circuit to.
engine_list (list<BasicEngine>): List of engines / backends to use
as compiler engines.
Example:
.. code-block:: python
from projectq import MainEngine
eng = MainEngine() # uses default setup and the Simulator
Alternatively, one can specify all compiler engines explicitly, e.g.,
Example:
.. code-block:: python
from projectq.cengines import (TagRemover, AutoReplacer,
LocalOptimizer,
DecompositionRuleSet)
from projectq.backends import Simulator
from projectq import MainEngine
rule_set = DecompositionRuleSet()
engines = [AutoReplacer(rule_set), TagRemover(),
LocalOptimizer(3)]
eng = MainEngine(Simulator(), engines)
"""
BasicEngine.__init__(self)
if backend is None:
backend = Simulator()
else: # Test that backend is BasicEngine object
if not isinstance(backend, BasicEngine):
raise UnsupportedEngineError(
"\nYou supplied a backend which is not supported,\n"
"i.e. not an instance of BasicEngine.\n"
"Did you forget the brackets to create an instance?\n"
"E.g. MainEngine(backend=Simulator) instead of \n"
" MainEngine(backend=Simulator())")
if engine_list is None:
try:
engine_list = projectq.default_engines()
except AttributeError:
from projectq.setups.default import default_engines
engine_list = default_engines()
else: # Test that engine list elements are all BasicEngine objects
if not isinstance(engine_list, list):
raise UnsupportedEngineError(
"\n The engine_list argument is not a list!\n")
for current_eng in engine_list:
if not isinstance(current_eng, BasicEngine):
raise UnsupportedEngineError(
"\nYou supplied an unsupported engine in engine_list,"
"\ni.e. not an instance of BasicEngine.\n"
"Did you forget the brackets to create an instance?\n"
"E.g. MainEngine(engine_list=[AutoReplacer]) instead "
"of\n MainEngine(engine_list=[AutoReplacer()])")
engine_list = engine_list + [backend]
self.backend = backend
# Test that user did not supply twice the same engine instance
num_different_engines = len(set([id(item) for item in engine_list]))
if len(engine_list) != num_different_engines:
raise UnsupportedEngineError(
"\n Error:\n You supplied twice the same engine as backend" +
" or item in engine_list. This doesn't work. Create two \n" +
" separate instances of a compiler engine if it is needed\n" +
" twice.\n")
self._qubit_idx = int(0)
for i in range(len(engine_list) - 1):
engine_list[i].next_engine = engine_list[i + 1]
engine_list[i].main_engine = self
engine_list[-1].main_engine = self
engine_list[-1].is_last_engine = True
self.next_engine = engine_list[0]
self.main_engine = self
self.active_qubits = weakref.WeakSet()
self._measurements = dict()
self.dirty_qubits = set()
# In order to terminate an example code without eng.flush or Measure
self._delfun = lambda x: x.flush(deallocate_qubits=True)
atexit.register(self._delfun, self)
class rpc(object):
""" Conveniently interact with a remote server
>>> remote = rpc(address) # doctest: +SKIP
>>> response = yield remote.add(x=10, y=20) # doctest: +SKIP
One rpc object can be reused for several interactions.
Additionally, this object creates and destroys many comms as necessary
and so is safe to use in multiple overlapping communications.
When done, close comms explicitly.
>>> remote.close_comms() # doctest: +SKIP
"""
active = weakref.WeakSet()
comms = ()
address = None
def __init__(self,
arg=None,
comm=None,
deserialize=True,
timeout=None,
connection_args=None,
serializers=None,
deserializers=None):
self.comms = {}
self.address = coerce_to_address(arg)
self.timeout = timeout
self.status = 'running'
self.deserialize = deserialize
self.serializers = serializers
self.deserializers = deserializers if deserializers is not None else serializers
self.connection_args = connection_args
rpc.active.add(self)
@gen.coroutine
def live_comm(self):
""" Get an open communication
Some comms to the ip/port target may be in current use by other
coroutines. We track this with the `comms` dict
:: {comm: True/False if open and ready for use}
This function produces an open communication, either by taking one
that we've already made or making a new one if they are all taken.
This also removes comms that have been closed.
When the caller is done with the stream they should set
self.comms[comm] = True
As is done in __getattr__ below.
"""
if self.status == 'closed':
raise RPCClosed("RPC Closed")
to_clear = set()
open = False
for comm, open in self.comms.items():
if comm.closed():
to_clear.add(comm)
if open:
break
for s in to_clear:
del self.comms[s]
if not open or comm.closed():
comm = yield connect(self.address,
self.timeout,
deserialize=self.deserialize,
connection_args=self.connection_args)
self.comms[comm] = False # mark as taken
raise gen.Return(comm)
def close_comms(self):
@gen.coroutine
def _close_comm(comm):
# Make sure we tell the peer to close
try:
yield comm.write({'op': 'close', 'reply': False})
yield comm.close()
except EnvironmentError:
comm.abort()
for comm in list(self.comms):
if comm and not comm.closed():
_close_comm(comm)
self.comms.clear()
def __getattr__(self, key):
@gen.coroutine
def send_recv_from_rpc(**kwargs):
if self.serializers is not None and kwargs.get(
'serializers') is None:
kwargs['serializers'] = self.serializers
if self.deserializers is not None and kwargs.get(
'deserializers') is None:
kwargs['deserializers'] = self.deserializers
try:
comm = yield self.live_comm()
result = yield send_recv(comm=comm, op=key, **kwargs)
except (RPCClosed, CommClosedError) as e:
raise e.__class__("%s: while trying to call remote method %r" %
(
e,
key,
))
self.comms[comm] = True # mark as open
raise gen.Return(result)
return send_recv_from_rpc
def close_rpc(self):
if self.status != 'closed':
rpc.active.discard(self)
self.status = 'closed'
self.close_comms()
def __enter__(self):
return self
def __exit__(self, *args):
self.close_rpc()
def __del__(self):
if self.status != 'closed':
rpc.active.discard(self)
self.status = 'closed'
still_open = [comm for comm in self.comms if not comm.closed()]
if still_open:
logger.warning("rpc object %s deleted with %d open comms",
self, len(still_open))
for comm in still_open:
comm.abort()
def __repr__(self):
return "<rpc to %r, %d comms>" % (self.address, len(self.comms))
class Comm(ABC):
"""
A message-oriented communication object, representing an established
communication channel. There should be only one reader and one
writer at a time: to manage current communications, even with a
single peer, you must create distinct ``Comm`` objects.
Messages are arbitrary Python objects. Concrete implementations
of this class can implement different serialization mechanisms
depending on the underlying transport's characteristics.
"""
_instances = weakref.WeakSet()
def __init__(self):
self._instances.add(self)
self.name = None
# XXX add set_close_callback()?
@abstractmethod
def read(self, deserializers=None):
"""
Read and return a message (a Python object).
This method is a coroutine.
Parameters
----------
deserializers : Optional[Dict[str, Tuple[Callable, Callable, bool]]]
An optional dict appropriate for distributed.protocol.deserialize.
See :ref:`serialization` for more.
"""
@abstractmethod
def write(self, msg, serializers=None, on_error=None):
"""
Write a message (a Python object).
This method is a coroutine.
Parameters
----------
msg :
on_error : Optional[str]
The behavior when serialization fails. See
``distributed.protocol.core.dumps`` for valid values.
"""
@abstractmethod
def close(self):
"""
Close the communication cleanly. This will attempt to flush
outgoing buffers before actually closing the underlying transport.
This method is a coroutine.
"""
@abstractmethod
def abort(self):
"""
Close the communication immediately and abruptly.
Useful in destructors or generators' ``finally`` blocks.
"""
@abstractmethod
def closed(self):
"""
Return whether the stream is closed.
"""
@abstractproperty
def local_address(self):
"""
The local address. For logging and debugging purposes only.
"""
@abstractproperty
def peer_address(self):
"""
The peer's address. For logging and debugging purposes only.
"""
@property
def extra_info(self):
"""
Return backend-specific information about the communication,
as a dict. Typically, this is information which is initialized
when the communication is established and doesn't vary afterwards.
"""
return {}
def __repr__(self):
clsname = self.__class__.__name__
if self.closed():
return "<closed %s>" % (clsname, )
else:
return "<%s %s local=%s remote=%s>" % (
clsname,
self.name or "",
self.local_address,
self.peer_address,
)
def __init__(self, plugin, objmap):
self._widgets = weakref.WeakSet()
self.preferences = plugin.preferences
InsertedObjectTypeExtension.__init__(self, plugin, objmap)
self.connectto(self.preferences, 'changed',
self.on_preferences_changed)
class LatexManager(object):
"""
The LatexManager opens an instance of the LaTeX application for
determining the metrics of text elements. The LaTeX environment can be
modified by setting fonts and/or a custem preamble in the rc parameters.
"""
_unclean_instances = weakref.WeakSet()
@staticmethod
def _build_latex_header():
latex_preamble = get_preamble()
latex_fontspec = get_fontspec()
# Create LaTeX header with some content, else LaTeX will load some
# math fonts later when we don't expect the additional output on stdout.
# TODO: is this sufficient?
latex_header = [
r"\documentclass{minimal}",
latex_preamble,
latex_fontspec,
r"\begin{document}",
r"text $math \mu$", # force latex to load fonts now
r"\typeout{pgf_backend_query_start}"
]
return "\n".join(latex_header)
@staticmethod
def _cleanup_remaining_instances():
unclean_instances = list(LatexManager._unclean_instances)
for latex_manager in unclean_instances:
latex_manager._cleanup()
def _stdin_writeln(self, s):
self.latex_stdin_utf8.write(s)
self.latex_stdin_utf8.write("\n")
self.latex_stdin_utf8.flush()
def _expect(self, s):
exp = s.encode("utf8")
buf = bytearray()
while True:
b = self.latex.stdout.read(1)
buf += b
if buf[-len(exp):] == exp:
break
if not len(b):
raise LatexError("LaTeX process halted", buf.decode("utf8"))
return buf.decode("utf8")
def _expect_prompt(self):
return self._expect("\n*")
def __init__(self):
# store references for __del__
self._os_path = os.path
self._shutil = shutil
self._debug = rcParams["pgf.debug"]
# create a tmp directory for running latex, remember to cleanup
self.tmpdir = tempfile.mkdtemp(prefix="mpl_pgf_lm_")
LatexManager._unclean_instances.add(self)
# test the LaTeX setup to ensure a clean startup of the subprocess
self.texcommand = get_texcommand()
self.latex_header = LatexManager._build_latex_header()
latex_end = "\n\\makeatletter\n\\@@end\n"
try:
latex = subprocess.Popen([str(self.texcommand), "-halt-on-error"],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
cwd=self.tmpdir)
except OSError as e:
if e.errno == errno.ENOENT:
raise RuntimeError(
"Latex command not found. "
"Install '%s' or change pgf.texsystem to the desired command."
% self.texcommand)
else:
raise RuntimeError("Error starting process '%s'" %
self.texcommand)
test_input = self.latex_header + latex_end
stdout, stderr = latex.communicate(test_input.encode("utf-8"))
if latex.returncode != 0:
raise LatexError(
"LaTeX returned an error, probably missing font or error in preamble:\n%s"
% stdout)
# open LaTeX process for real work
latex = subprocess.Popen([str(self.texcommand), "-halt-on-error"],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
cwd=self.tmpdir)
self.latex = latex
self.latex_stdin_utf8 = codecs.getwriter("utf8")(self.latex.stdin)
# write header with 'pgf_backend_query_start' token
self._stdin_writeln(self._build_latex_header())
# read all lines until our 'pgf_backend_query_start' token appears
self._expect("*pgf_backend_query_start")
self._expect_prompt()
# cache for strings already processed
self.str_cache = {}
def _cleanup(self):
if not self._os_path.isdir(self.tmpdir):
return
try:
self.latex.communicate()
self.latex_stdin_utf8.close()
self.latex.stdout.close()
except:
pass
try:
self._shutil.rmtree(self.tmpdir)
LatexManager._unclean_instances.discard(self)
except:
sys.stderr.write("error deleting tmp directory %s\n" % self.tmpdir)
def __del__(self):
if self._debug:
print("deleting LatexManager")
self._cleanup()
def get_width_height_descent(self, text, prop):
"""
Get the width, total height and descent for a text typesetted by the
current LaTeX environment.
"""
# apply font properties and define textbox
prop_cmds = _font_properties_str(prop)
textbox = "\\sbox0{%s %s}" % (prop_cmds, text)
# check cache
if textbox in self.str_cache:
return self.str_cache[textbox]
# send textbox to LaTeX and wait for prompt
self._stdin_writeln(textbox)
try:
self._expect_prompt()
except LatexError as e:
msg = "Error processing '%s'\nLaTeX Output:\n%s"
raise ValueError(msg % (text, e.latex_output))
# typeout width, height and text offset of the last textbox
self._stdin_writeln(r"\typeout{\the\wd0,\the\ht0,\the\dp0}")
# read answer from latex and advance to the next prompt
try:
answer = self._expect_prompt()
except LatexError as e:
msg = "Error processing '%s'\nLaTeX Output:\n%s"
raise ValueError(msg % (text, e.latex_output))
# parse metrics from the answer string
try:
width, height, offset = answer.splitlines()[0].split(",")
except:
msg = "Error processing '%s'\nLaTeX Output:\n%s" % (text, answer)
raise ValueError(msg)
w, h, o = float(width[:-2]), float(height[:-2]), float(offset[:-2])
# the height returned from LaTeX goes from base to top.
# the height matplotlib expects goes from bottom to top.
self.str_cache[textbox] = (w, h + o, o)
return w, h + o, o
guards.add(self)
try:
return func(self)
finally:
guards.remove(self)
return inner
#
# This setup uses SIGCHLD as a trigger to check on the runner process
# in order to detect the monitoredcommand's complete exit early instead
# of on the next polling interval. Because processes can be created
# and exited very rapidly, it includes a 16 millisecond debounce.
#
_commands = weakref.WeakSet()
_timeout_set = False
def _trigger_early_poll():
global _timeout_set
try:
# prevent changes to size during iteration
for command in set(_commands):
command.watch_children()
except Exception:
logger.exception("Signal handler exception")
finally:
_timeout_set = False
return False
def __init__(self, name, collections=None, capture_by_value=None):
"""Construct a new FuncGraph.
The graph will inherit its graph key, collections, seed, and distribution
strategy stack from the current context or graph.
Args:
name: the name of the function.
collections: a dictionary of collections this FuncGraph should start
with. If not specified (None), the FuncGraph will read (but not write
to) the outer graph's collections that are not whitelisted, and both
read and write to the outer graph's collections that are whitelisted.
The current whitelisted collections are the global variables, the
local variables, and the trainable variables.
Defaults to None.
capture_by_value: An optional boolean. If True, the func graph will
capture Variables by value instead of reference. By default inherit
from outer graphs, and failing that will default to False.
"""
super(FuncGraph, self).__init__()
self.name = name
self.inputs = []
self.outputs = []
self.control_outputs = []
self.control_captures = set()
self.structured_input_signature = None
self.structured_outputs = None
self._weak_variables = []
self._watched_variables = weakref.WeakSet()
self.outer_graph = ops.get_default_graph()
self.captures = py_collections.OrderedDict()
self.deferred_captures = py_collections.OrderedDict()
# Inherit capture-by-value from outer graph.
if capture_by_value is not None:
self.capture_by_value = capture_by_value
elif self.outer_graph is not None and isinstance(
self.outer_graph, FuncGraph):
self.capture_by_value = self.outer_graph.capture_by_value
else:
self.capture_by_value = False
self._building_function = True
# Map from resource tensor name to last op (in program order) which uses
# this tensor. Used to enforce that execution order matches program order
# for resource tensors.
self._last_op_using_resource_tensor = {}
graph = self.outer_graph
if context.executing_eagerly():
self.seed = context.global_seed()
# [for tf-data user migration from TF1.0 to 2.0] seed_used keep track of
# any None op_seed for random_op in the function, in which case we end up
# using function seed, which could be unintended behavior for the op.
self._seed_used = False
else:
self.seed = graph.seed
self._seed_used = False
# TODO(allenl): Figure out if we can remove colocation stack
# specialization (currently used in cond_v2), here and in the cache key.
self._colocation_stack = graph._colocation_stack.copy() # pylint: disable=protected-access
if collections is None:
for collection_name in graph.get_all_collection_keys():
if collection_name not in WHITELIST_COLLECTIONS:
self._collections[collection_name] = graph.get_collection(
collection_name)
for collection_name in WHITELIST_COLLECTIONS:
self._collections[collection_name] = graph.get_collection_ref(
collection_name)
else:
self._collections = collections
"""
while True:
for item in seq:
yield item
# Global variables to be shared across processes
_SHARED_SEQUENCES = {}
# We use a Value to provide unique id to different processes.
_SEQUENCE_COUNTER = None
# Because multiprocessing pools are inherently unsafe, starting from a clean
# state can be essential to avoiding deadlocks. In order to accomplish this, we
# need to be able to check on the status of Pools that we create.
_DATA_POOLS = weakref.WeakSet()
_WORKER_ID_QUEUE = None # Only created if needed.
_WORKER_IDS = set()
_FORCE_THREADPOOL = False
_FORCE_THREADPOOL_LOCK = threading.RLock()
def dont_use_multiprocessing_pool(f):
@functools.wraps(f)
def wrapped(*args, **kwargs):
with _FORCE_THREADPOOL_LOCK:
global _FORCE_THREADPOOL
old_force_threadpool, _FORCE_THREADPOOL = _FORCE_THREADPOOL, True
out = f(*args, **kwargs)
_FORCE_THREADPOOL = old_force_threadpool
return out
def __init__(self, manager, name):
self.manager = manager
self.name = name
self.objects = weakref.WeakSet()
@property
def exitcode(self):
return self._state.exitcode
@property
def name(self):
return self._name
@property
def daemon(self):
return self._process.daemon
@daemon.setter
def daemon(self, value):
self._process.daemon = value
_dangling = weakref.WeakSet()
@atexit.register
def _cleanup_dangling():
for proc in list(_dangling):
if proc.daemon and proc.is_alive():
try:
logger.warning("reaping stray process %s" % (proc,))
proc.terminate()
except OSError:
pass
self._stop_worker(w) for w in self.workers
if w.worker_address in workers
]
while workers & set(self.workers):
yield gen.sleep(0.01)
def __del__(self):
self.close()
def __enter__(self):
return self
def __exit__(self, *args):
self.close()
@property
def scheduler_address(self):
try:
return self.scheduler.address
except ValueError:
return '<unstarted>'
clusters_to_close = weakref.WeakSet()
@atexit.register
def close_clusters():
for cluster in clusters_to_close:
cluster.close()
if all(process.poll() is not None for process in processes):
return
for process in processes:
if process.poll() is None:
try:
process.kill()
except OSError:
pass
for process in processes:
process.wait()
# Hold a weakreference to each subprocess for cleanup during shutdown
worker_processes = weakref.WeakSet()
@atexit.register
def cleanup_processes():
cleanup(*list(worker_processes))
class TransformWorker(object):
def __init__(self,
args,
reload_process=False,
Response=webob.Response,
**kw):
self.args = args
self.kw = kw
def ApplyAttributes(self, attributes):
if self.default_fontsize is None:
self.default_fontsize = fontConst.DEFAULT_FONTSIZE
self.DECIMAL = '.'
Container.ApplyAttributes(self, attributes)
self.rightAlignedButtons = weakref.WeakSet()
self._clearButton = None
self.hinttext = ''
self.isTabStop = 1
self.integermode = None
self.floatmode = None
self.passwordchar = None
self.caretIndex = (0, 0)
self.selFrom = None
self.selTo = None
self.value = None
self.text = ''
self.suffix = ''
self.maxletters = None
self.historyMenu = None
self.historySaveLast = None
self.displayHistory = False
self.allowHistoryInnerMatches = False
self.maxHistoryShown = 5
self.numericControlsCont = None
self.updateNumericInputThread = None
self.draggedValue = None
self.OnChange = None
self.OnFocusLost = None
self.OnReturn = None
self.OnInsert = None
self.readonly = attributes.get('readonly', self.default_readonly)
self.fontStyle = attributes.get('fontStyle', self.default_fontStyle)
self.fontFamily = attributes.get('fontFamily', self.default_fontFamily)
self.fontPath = attributes.get('fontPath', self.default_fontPath)
self.fontsize = attributes.get('fontsize', self.default_fontsize)
self._textClipper = Container(name='_textClipper', parent=self, clipChildren=True, padding=(1, 0, 1, 0))
self._textClipper._OnSizeChange_NoBlock = self.OnClipperSizeChange
self.Prepare_()
self.autoselect = attributes.get('autoselect', self.default_autoselect)
self.adjustWidth = attributes.get('adjustWidth', self.default_adjustWidth)
self.dynamicHistoryWidth = attributes.get('dynamicHistoryWidth', self.default_dynamicHistoryWidth)
self.sr.text.shadow = self.sr.hinttext.shadow = attributes.get('shadow', None)
fontcolor = attributes.get('fontcolor', (1.0, 1.0, 1.0, 1.0))
if fontcolor is not None:
self.SetTextColor(fontcolor)
if attributes.get('ints', None):
self.IntMode(*attributes.ints)
elif attributes.get('floats', None):
self.FloatMode(*attributes.floats)
self.SetPasswordChar(attributes.get('passwordCharacter', self.default_passwordCharacter))
self.SetMaxLength(attributes.get('maxLength', self.default_maxLength))
self.SetLabel(attributes.get('label', self.default_label))
self.SetHintText(attributes.get('hinttext', self.default_hinttext))
self.SetValue(attributes.get('setvalue', self.default_setvalue))
self.height = 20
self.OnChange = attributes.get('OnChange', self.default_OnChange)
self.__OnSetFocus = attributes.get('OnSetFocus', self.default_OnSetFocus)
self.OnFocusLost = attributes.get('OnFocusLost', self.default_OnFocusLost)
self.OnReturn = attributes.get('OnReturn', self.default_OnReturn)
self.OnInsert = attributes.get('OnInsert', self.default_OnInsert)
OnAnyChar = attributes.get('OnAnyChar', self.default_OnAnyChar)
if OnAnyChar:
self.OnAnyChar = OnAnyChar
uicore.event.RegisterForTriuiEvents(uiconst.UI_MOUSEDOWN, self.OnGlobalMouseDownCallback)
"sw_ver": __version__,
"sw_sys": platform.system(),
}
#: maximum number of revokes to keep in memory.
REVOKES_MAX = 50000
#: how many seconds a revoke will be active before
#: being expired when the max limit has been exceeded.
REVOKE_EXPIRES = 10800
#: Mapping of reserved task_id->Request.
requests = {}
#: set of all reserved :class:`~celery.worker.request.Request`'s.
reserved_requests = weakref.WeakSet()
#: set of currently active :class:`~celery.worker.request.Request`'s.
active_requests = weakref.WeakSet()
#: count of tasks accepted by the worker, sorted by type.
total_count = Counter()
#: count of all tasks accepted by the worker
all_total_count = [0]
#: the list of currently revoked tasks. Persistent if ``statedb`` set.
revoked = LimitedSet(maxlen=REVOKES_MAX, expires=REVOKE_EXPIRES)
should_stop = None
should_terminate = None
class LatexManager:
"""
The LatexManager opens an instance of the LaTeX application for
determining the metrics of text elements. The LaTeX environment can be
modified by setting fonts and/or a custom preamble in `.rcParams`.
"""
_unclean_instances = weakref.WeakSet()
@staticmethod
def _build_latex_header():
latex_preamble = get_preamble()
latex_fontspec = get_fontspec()
# Create LaTeX header with some content, else LaTeX will load some math
# fonts later when we don't expect the additional output on stdout.
# TODO: is this sufficient?
latex_header = [
r"\documentclass{minimal}",
# Include TeX program name as a comment for cache invalidation.
# TeX does not allow this to be the first line.
rf"% !TeX program = {mpl.rcParams['pgf.texsystem']}",
# Test whether \includegraphics supports interpolate option.
r"\usepackage{graphicx}",
latex_preamble,
latex_fontspec,
r"\begin{document}",
r"text $math \mu$", # force latex to load fonts now
r"\typeout{pgf_backend_query_start}",
]
return "\n".join(latex_header)
@classmethod
def _get_cached_or_new(cls):
"""
Return the previous LatexManager if the header and tex system did not
change, or a new instance otherwise.
"""
return cls._get_cached_or_new_impl(cls._build_latex_header())
@classmethod
@functools.lru_cache(1)
def _get_cached_or_new_impl(cls, header): # Helper for _get_cached_or_new.
return cls()
@staticmethod
def _cleanup_remaining_instances():
unclean_instances = list(LatexManager._unclean_instances)
for latex_manager in unclean_instances:
latex_manager._cleanup()
def _stdin_writeln(self, s):
if self.latex is None:
self._setup_latex_process()
self.latex.stdin.write(s)
self.latex.stdin.write("\n")
self.latex.stdin.flush()
def _expect(self, s):
s = list(s)
chars = []
while True:
c = self.latex.stdout.read(1)
chars.append(c)
if chars[-len(s):] == s:
break
if not c:
self.latex.kill()
self.latex = None
raise LatexError("LaTeX process halted", "".join(chars))
return "".join(chars)
def _expect_prompt(self):
return self._expect("\n*")
def __init__(self):
# store references for __del__
self._os_path = os.path
self._shutil = shutil
# create a tmp directory for running latex, remember to cleanup
self.tmpdir = tempfile.mkdtemp(prefix="mpl_pgf_lm_")
LatexManager._unclean_instances.add(self)
# test the LaTeX setup to ensure a clean startup of the subprocess
self.texcommand = mpl.rcParams["pgf.texsystem"]
self.latex_header = LatexManager._build_latex_header()
latex_end = "\n\\makeatletter\n\\@@end\n"
try:
latex = subprocess.Popen([self.texcommand, "-halt-on-error"],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
encoding="utf-8",
cwd=self.tmpdir)
except FileNotFoundError as err:
raise RuntimeError(
f"{self.texcommand} not found. Install it or change "
f"rcParams['pgf.texsystem'] to an available TeX "
f"implementation.") from err
except OSError as err:
raise RuntimeError("Error starting process %r" %
self.texcommand) from err
test_input = self.latex_header + latex_end
stdout, stderr = latex.communicate(test_input)
if latex.returncode != 0:
raise LatexError("LaTeX returned an error, probably missing font "
"or error in preamble:\n%s" % stdout)
self.latex = None # Will be set up on first use.
self.str_cache = {} # cache for strings already processed
def _setup_latex_process(self):
# open LaTeX process for real work
self.latex = subprocess.Popen([self.texcommand, "-halt-on-error"],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
encoding="utf-8",
cwd=self.tmpdir)
# write header with 'pgf_backend_query_start' token
self._stdin_writeln(self._build_latex_header())
# read all lines until our 'pgf_backend_query_start' token appears
self._expect("*pgf_backend_query_start")
self._expect_prompt()
@cbook.deprecated("3.3")
def latex_stdin_utf8(self):
return self.latex.stdin
def _cleanup(self):
if not self._os_path.isdir(self.tmpdir):
return
try:
self.latex.communicate()
except Exception:
pass
try:
self._shutil.rmtree(self.tmpdir)
LatexManager._unclean_instances.discard(self)
except Exception:
sys.stderr.write("error deleting tmp directory %s\n" % self.tmpdir)
def __del__(self):
_log.debug("deleting LatexManager")
self._cleanup()
def get_width_height_descent(self, text, prop):
"""
Get the width, total height and descent for a text typeset by the
current LaTeX environment.
"""
# apply font properties and define textbox
prop_cmds = _font_properties_str(prop)
textbox = "\\sbox0{%s %s}" % (prop_cmds, text)
# check cache
if textbox in self.str_cache:
return self.str_cache[textbox]
# send textbox to LaTeX and wait for prompt
self._stdin_writeln(textbox)
try:
self._expect_prompt()
except LatexError as e:
raise ValueError("Error processing '{}'\nLaTeX Output:\n{}".format(
text, e.latex_output)) from e
# typeout width, height and text offset of the last textbox
self._stdin_writeln(r"\typeout{\the\wd0,\the\ht0,\the\dp0}")
# read answer from latex and advance to the next prompt
try:
answer = self._expect_prompt()
except LatexError as e:
raise ValueError("Error processing '{}'\nLaTeX Output:\n{}".format(
text, e.latex_output)) from e
# parse metrics from the answer string
try:
width, height, offset = answer.splitlines()[0].split(",")
except Exception as err:
raise ValueError("Error processing '{}'\nLaTeX Output:\n{}".format(
text, answer)) from err
w, h, o = float(width[:-2]), float(height[:-2]), float(offset[:-2])
# the height returned from LaTeX goes from base to top.
# the height matplotlib expects goes from bottom to top.
self.str_cache[textbox] = (w, h + o, o)
return w, h + o, o
import threading
import weakref
from celery.local import Proxy
from celery.utils.threads import LocalStack
__all__ = [
'set_default_app', 'get_current_app', 'get_current_task',
'get_current_worker_task', 'current_app', 'current_task'
]
#: Global default app used when no current app.
default_app = None
#: List of all app instances (weakrefs), must not be used directly.
_apps = weakref.WeakSet()
_task_join_will_block = False
def _set_task_join_will_block(blocks):
global _task_join_will_block
_task_join_will_block = blocks
def task_join_will_block():
return _task_join_will_block
class _TLS(threading.local):
#: Apps with the :attr:`~celery.app.base.BaseApp.set_as_current` attribute
class Node(object):
""" A wpantund OT NCP instance """
_VERBOSE = False # defines the default verbosity setting (can be changed per `Node`)
_SPEED_UP_FACTOR = 1 # defines the default time speed up factor
# path to `wpantund`, `wpanctl` and `ot-ncp-ftd` code
_WPANTUND = '/usr/local/sbin/wpantund'
_WPANCTL = '/usr/local/bin/wpanctl'
_OT_NCP_FTD = '../../examples/apps/ncp/ot-ncp-ftd'
_TUND_LOG_TO_FILE = True # determines if the wpantund logs are saved in file or sent to stdout
_TUND_LOG_FNAME = 'wpantund-logs' # name of wpantund log file (if # name of wpantund _TUND_LOG_TO_FILE is True)
# interface name
_INTFC_NAME_PREFIX = 'utun' if sys.platform == 'darwin' else 'wpan'
_START_INDEX = 4 if sys.platform == 'darwin' else 1
_cur_index = _START_INDEX
_all_nodes = weakref.WeakSet()
def __init__(self, verbose=_VERBOSE):
"""Creates a new `Node` instance"""
index = Node._cur_index
Node._cur_index += 1
self._index = index
self._interface_name = self._INTFC_NAME_PREFIX + str(index)
self._verbose = verbose
ncp_socket_path = 'system:{} {} {}'.format(self._OT_NCP_FTD, index,
self._SPEED_UP_FACTOR)
cmd = self._WPANTUND + \
' -o Config:NCP:SocketPath \"{}\"'.format(ncp_socket_path) + \
' -o Config:TUN:InterfaceName {}'.format(self._interface_name) + \
' -o Config:NCP:DriverName spinel' + \
' -o Daemon:SyslogMask \"all -debug\"'
if Node._TUND_LOG_TO_FILE:
self._tund_log_file = open(
self._TUND_LOG_FNAME + str(index) + '.log', 'wb')
else:
self._tund_log_file = None
if self._verbose:
_log('$ Node{}.__init__() cmd: {}'.format(index, cmd))
self._wpantund_process = subprocess.Popen(cmd,
shell=True,
stderr=self._tund_log_file)
self._wpanctl_cmd = self._WPANCTL + ' -I ' + self._interface_name + ' '
self._recvers = weakref.WeakValueDictionary(
) # map from local_port to `AsyncReceiver` object
Node._all_nodes.add(self)
def __del__(self):
self._wpantund_process.terminate()
self._tund_log_file.close()
def __repr__(self):
return 'Node (index={}, interface_name={})'.format(
self._index, self._interface_name)
@property
def index(self):
return self._index
@property
def interface_name(self):
return self._interface_name
@property
def tund_log_file(self):
return self._tund_log_file
#------------------------------------------------------------------------------------------------------------------
# Executing a `wpanctl` command
def wpanctl(self, cmd):
""" Runs a wpanctl command on the given wpantund/OT-NCP instance and returns the output """
if self._verbose:
_log('$ Node{}.wpanctl(\'{}\')'.format(self._index, cmd),
new_line=False)
result = subprocess.check_output(self._wpanctl_cmd + cmd,
shell=True,
stderr=subprocess.STDOUT)
if len(result) >= 1 and result[
-1] == '\n': # remove the last char if it is '\n',
result = result[:-1]
if self._verbose:
if '\n' in result:
_log(':')
for line in result.splitlines():
_log(' ' + line)
else:
_log(' -> \'{}\''.format(result))
return result
#------------------------------------------------------------------------------------------------------------------
# APIs matching `wpanctl` commands.
def get(self, prop_name, value_only=True):
return self.wpanctl('get ' + ('-v ' if value_only else '') + prop_name)
def set(self, prop_name, value, binary_data=False):
return self._update_prop('set', prop_name, value, binary_data)
def add(self, prop_name, value, binary_data=False):
return self._update_prop('add', prop_name, value, binary_data)
def remove(self, prop_name, value, binary_data=False):
return self._update_prop('remove', prop_name, value, binary_data)
def _update_prop(self, action, prop_name, value, binary_data):
return self.wpanctl(
action + ' ' + prop_name + ' ' + ('-d ' if binary_data else '') +
'-v ' + value) # use -v to handle values starting with `-`.
def reset(self):
return self.wpanctl('reset')
def status(self):
return self.wpanctl('status')
def leave(self):
return self.wpanctl('leave')
def form(self, name, channel=None, node_type=None):
return self.wpanctl(
'form \"' + name + '\"' +
(' -c {}'.format(channel) if channel is not None else '') +
(' -T {}'.format(node_type) if node_type is not None else ''))
def join(self,
name,
channel=None,
node_type=None,
panid=None,
xpanid=None):
return self.wpanctl(
'join \"' + name + '\"' +
(' -c {}'.format(channel) if channel is not None else '') +
(' -T {}'.format(node_type) if node_type is not None else '') +
(' -p {}'.format(panid) if panid is not None else '') +
(' -x {}'.format(xpanid) if xpanid is not None else ''))
def active_scan(self, channel=None):
return self.wpanctl(
'scan' + (' -c {}'.format(channel) if channel is not None else ''))
def energy_scan(self, channel=None):
return self.wpanctl('scan -e' + (
' -c {}'.format(channel) if channel is not None else ''))
def discover_scan(self,
channel=None,
joiner_only=False,
enable_filtering=False,
panid_filter=None):
return self.wpanctl(
'scan -d' +
(' -c {}'.format(channel) if channel is not None else '') +
(' -j' if joiner_only else '') +
(' -e' if enable_filtering else '') +
(' -p {}'.format(panid_filter) if panid_filter is not None else '')
)
def permit_join(self, duration_sec=None, port=None, udp=True, tcp=True):
if not udp and not tcp: # incorrect use!
return ''
traffic_type = ''
if udp and not tcp:
traffic_type = ' --udp'
if tcp and not udp:
traffic_type = ' --tcp'
if port is not None and duration_sec is None:
duration_sec = '240'
return self.wpanctl(
'permit-join' +
(' {}'.format(duration_sec) if duration_sec is not None else '') +
(' {}'.format(port) if port is not None else '') + traffic_type)
def config_gateway(self, prefix, default_route=False):
return self.wpanctl('config-gateway ' + prefix +
(' -d' if default_route else ''))
def add_route(self, route_prefix, prefix_len_in_bytes=None, priority=None):
"""route priority [(>0 for high, 0 for medium, <0 for low)]"""
return self.wpanctl(
'add-route ' + route_prefix +
(' -l {}'.format(prefix_len_in_bytes
) if prefix_len_in_bytes is not None else '') +
(' -p {}'.format(priority) if priority is not None else ''))
def remove_route(self,
route_prefix,
prefix_len_in_bytes=None,
priority=None):
"""route priority [(>0 for high, 0 for medium, <0 for low)]"""
return self.wpanctl(
'remove-route ' + route_prefix +
(' -l {}'.format(prefix_len_in_bytes
) if prefix_len_in_bytes is not None else '') +
(' -p {}'.format(priority) if priority is not None else ''))
#------------------------------------------------------------------------------------------------------------------
# Helper methods
def is_associated(self):
return self.get(WPAN_STATE) == STATE_ASSOCIATED
def join_node(self, node, node_type=JOIN_TYPE_ROUTER, should_set_key=True):
"""Join a network specified by another node, `node` should be a Node"""
if not node.is_associated():
return "{} is not associated".format(node)
name = node.get(WPAN_NAME)
panid = node.get(WPAN_PANID)
xpanid = node.get(WPAN_XPANID)
channel = node.get(WPAN_CHANNEL)
if should_set_key:
netkey = node.get(WPAN_KEY)
self.set(WPAN_KEY, netkey[1:-1], binary_data=True)
return self.join(name[1:-1],
channel=channel,
node_type=node_type,
panid=panid,
xpanid=xpanid)
def whitelist_node(self, node):
"""Adds a given node (of type `Node`) to the whitelist of `self` and enables whitelisting on `self`"""
self.add(WPAN_MAC_WHITELIST_ENTRIES, node.get(WPAN_EXT_ADDRESS)[1:-1])
self.set(WPAN_MAC_WHITELIST_ENABLED, '1')
def is_in_scan_result(self, scan_result):
"""Checks if node is in the scan results
`scan_result` must be an array of `ScanResult` object (see `parse_scan_result`).
"""
joinable = (self.get(WPAN_NETWORK_ALLOW_JOIN) == 'true')
panid = self.get(WPAN_PANID)
xpanid = self.get(WPAN_XPANID)[2:]
name = self.get(WPAN_NAME)[1:-1]
channel = self.get(WPAN_CHANNEL)
ext_address = self.get(WPAN_EXT_ADDRESS)[1:-1]
for item in scan_result:
if all([
item.network_name == name, item.panid == panid,
item.xpanid == xpanid, item.channel == channel,
item.ext_address == ext_address,
(item.type == ScanResult.TYPE_DISCOVERY_SCAN)
or (item.joinable == joinable)
]):
return True
return False
#------------------------------------------------------------------------------------------------------------------
# class methods
@classmethod
def init_all_nodes(cls, wait_time=15):
"""Issues a `wpanctl.leave` on all `Node` objects and waits for them to be ready"""
random.seed(12345)
time.sleep(0.5)
start_time = time.time()
for node in Node._all_nodes:
while True:
try:
node.leave()
except subprocess.CalledProcessError as e:
_log(' -> \'{}\' exit code: {}'.format(
e.output, e.returncode))
if time.time() - start_time > wait_time:
print 'Took too long to init all nodes ({}>{} sec)'.format(
time.time() - start_time, wait_time)
raise
except:
raise
else:
break
time.sleep(0.4)
@classmethod
def set_time_speedup_factor(cls, factor):
"""Sets up the time speed up factor - should be set before creating any `Node` objects"""
if len(Node._all_nodes) != 0:
raise Node._NodeError(
'set_time_speedup_factor() cannot be called after creating a `Node`'
)
Node._SPEED_UP_FACTOR = factor
#------------------------------------------------------------------------------------------------------------------
# IPv6 message Sender and Receiver class
class _NodeError(BaseException):
pass
def prepare_tx(self, src, dst, data=40, count=1):
"""Prepares an IPv6 msg transmission.
- `src` and `dst` can be either a string containing IPv6 address, or a tuple (ipv6 address as string, port),
if no port is given, a random port number is used.
- `data` can be either a string containing the message to be sent, or an int indicating size of the message (a
random message with the given length will be used).
- `count` gives number of times the message will be sent (default is 1).
Returns an `AsyncSender` object.
"""
if isinstance(src, tuple):
src_addr = src[0]
src_port = src[1]
else:
src_addr = src
src_port = random.randint(49152, 65535)
if isinstance(dst, tuple):
dst_addr = dst[0]
dst_port = dst[1]
else:
dst_addr = dst
dst_port = random.randint(49152, 65535)
if isinstance(data, int):
# create a random message with the given length.
all_chars = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789.,><?;:[]=-+)(*&^%$#@'
msg = ''.join(random.choice(all_chars) for _ in range(data))
else:
msg = data
return AsyncSender(self, src_addr, src_port, dst_addr, dst_port, msg,
count)
def prepare_rx(self, sender):
"""Prepare to receive messages from a sender (an `AsyncSender`)"""
local_port = sender.dst_port
if local_port in self._recvers:
receiver = self._recvers[local_port]
else:
receiver = AsyncReceiver(self, local_port)
self._recvers[local_port] = receiver
receiver._add_sender(sender.src_addr, sender.src_port, sender.msg,
sender.count)
return receiver
def _remove_recver(self, recvr):
# Removes a receiver from weak dictionary - called when the receiver is done and its scoket is closed
local_port = recvr.local_port
if local_port in self._recvers:
del self._recvers[local_port]
@staticmethod
def perform_async_tx_rx(timeout=20):
"""Called to perform all previously prepared async rx and tx operations"""
try:
start_time = time.time()
while asyncore.socket_map:
elapsed_time = time.time() - start_time
if elapsed_time > timeout:
print 'Performing aysnc tx/tx took too long ({}>{} sec)'.format(
elapsed_time, timeout)
raise Node._NodeError(
'perform_tx_rx timed out ({}>{} sec)'.format(
elapsed_time, timeout))
# perform a single asyncore loop
asyncore.loop(timeout=1, count=1)
except:
print 'Failed to perform async rx/tx'
raise
return ssl_context
else:
return None
g_request_id: int = 0
def next_request_id() -> int:
global g_request_id
g_request_id += 1
return g_request_id
# sleep tasks
g_sleep_tasks: weakref.WeakSet = weakref.WeakSet()
async def sleep(secs: float) -> Any:
'''
Interruptable sleep task
'''
loop = asyncio.get_event_loop()
task: asyncio.Task = loop.create_task(asyncio.sleep(secs))
g_sleep_tasks.add(task)
try:
return await task
except asyncio.CancelledError:
logging.debug('sleep task %s cancelled', task)
finally:
try:
class SpecCluster(Cluster):
""" Cluster that requires a full specification of workers
The SpecCluster class expects a full specification of the Scheduler and
Workers to use. It removes any handling of user inputs (like threads vs
processes, number of cores, and so on) and any handling of cluster resource
managers (like pods, jobs, and so on). Instead, it expects this
information to be passed in scheduler and worker specifications. This
class does handle all of the logic around asynchronously cleanly setting up
and tearing things down at the right times. Hopefully it can form a base
for other more user-centric classes.
Parameters
----------
workers: dict
A dictionary mapping names to worker classes and their specifications
See example below
scheduler: dict, optional
A similar mapping for a scheduler
worker: dict
A specification of a single worker.
This is used for any new workers that are created.
asynchronous: bool
If this is intended to be used directly within an event loop with
async/await
silence_logs: bool
Whether or not we should silence logging when setting up the cluster.
name: str, optional
A name to use when printing out the cluster, defaults to type name
Examples
--------
To create a SpecCluster you specify how to set up a Scheduler and Workers
>>> from dask.distributed import Scheduler, Worker, Nanny
>>> scheduler = {'cls': Scheduler, 'options': {"dashboard_address": ':8787'}}
>>> workers = {
... 'my-worker': {"cls": Worker, "options": {"nthreads": 1}},
... 'my-nanny': {"cls": Nanny, "options": {"nthreads": 2}},
... }
>>> cluster = SpecCluster(scheduler=scheduler, workers=workers)
The worker spec is stored as the ``.worker_spec`` attribute
>>> cluster.worker_spec
{
'my-worker': {"cls": Worker, "options": {"nthreads": 1}},
'my-nanny': {"cls": Nanny, "options": {"nthreads": 2}},
}
While the instantiation of this spec is stored in the ``.workers``
attribute
>>> cluster.workers
{
'my-worker': <Worker ...>
'my-nanny': <Nanny ...>
}
Should the spec change, we can await the cluster or call the
``._correct_state`` method to align the actual state to the specified
state.
We can also ``.scale(...)`` the cluster, which adds new workers of a given
form.
>>> worker = {'cls': Worker, 'options': {}}
>>> cluster = SpecCluster(scheduler=scheduler, worker=worker)
>>> cluster.worker_spec
{}
>>> cluster.scale(3)
>>> cluster.worker_spec
{
0: {'cls': Worker, 'options': {}},
1: {'cls': Worker, 'options': {}},
2: {'cls': Worker, 'options': {}},
}
Note that above we are using the standard ``Worker`` and ``Nanny`` classes,
however in practice other classes could be used that handle resource
management like ``KubernetesPod`` or ``SLURMJob``. The spec does not need
to conform to the expectations of the standard Dask Worker class. It just
needs to be called with the provided options, support ``__await__`` and
``close`` methods and the ``worker_address`` property..
Also note that uniformity of the specification is not required. Other API
could be added externally (in subclasses) that adds workers of different
specifications into the same dictionary.
If a single entry in the spec will generate multiple dask workers then
please provide a `"group"` element to the spec, that includes the suffixes
that will be added to each name (this should be handled by your worker
class).
>>> cluster.worker_spec
{
0: {"cls": MultiWorker, "options": {"processes": 3}, "group": ["-0", "-1", -2"]}
1: {"cls": MultiWorker, "options": {"processes": 2}, "group": ["-0", "-1"]}
}
These suffixes should correspond to the names used by the workers when
they deploy.
>>> [ws.name for ws in cluster.scheduler.workers.values()]
["0-0", "0-1", "0-2", "1-0", "1-1"]
"""
_instances = weakref.WeakSet()
def __init__(
self,
workers=None,
scheduler=None,
worker=None,
asynchronous=False,
loop=None,
security=None,
silence_logs=False,
name=None,
):
self._created = weakref.WeakSet()
self.scheduler_spec = copy.copy(scheduler)
self.worker_spec = copy.copy(workers) or {}
self.new_spec = copy.copy(worker)
self.workers = {}
self._i = 0
self.security = security or Security()
self.scheduler_comm = None
self._futures = set()
if silence_logs:
self._old_logging_level = silence_logging(level=silence_logs)
self._old_bokeh_logging_level = silence_logging(level=silence_logs,
root="bokeh")
self._loop_runner = LoopRunner(loop=loop, asynchronous=asynchronous)
self.loop = self._loop_runner.loop
self._instances.add(self)
self._correct_state_waiting = None
self._name = name or type(self).__name__
super().__init__(asynchronous=asynchronous)
if not self.asynchronous:
self._loop_runner.start()
self.sync(self._start)
self.sync(self._correct_state)
async def _start(self):
while self.status == "starting":
await asyncio.sleep(0.01)
if self.status == "running":
return
if self.status == "closed":
raise ValueError("Cluster is closed")
self._lock = asyncio.Lock()
if self.scheduler_spec is None:
try:
import distributed.dashboard # noqa: F401
except ImportError:
pass
else:
options = {"dashboard": True}
self.scheduler_spec = {"cls": Scheduler, "options": options}
cls = self.scheduler_spec["cls"]
if isinstance(cls, str):
cls = import_term(cls)
self.scheduler = cls(**self.scheduler_spec.get("options", {}))
self.status = "starting"
self.scheduler = await self.scheduler
self.scheduler_comm = rpc(
getattr(self.scheduler, "external_address", None)
or self.scheduler.address,
connection_args=self.security.get_connection_args("client"),
)
await super()._start()
def _correct_state(self):
if self._correct_state_waiting:
# If people call this frequently, we only want to run it once
return self._correct_state_waiting
else:
task = asyncio.ensure_future(self._correct_state_internal())
self._correct_state_waiting = task
return task
async def _correct_state_internal(self):
async with self._lock:
self._correct_state_waiting = None
pre = list(set(self.workers))
to_close = set(self.workers) - set(self.worker_spec)
if to_close:
if self.scheduler.status == SchedulerStatus.running:
await self.scheduler_comm.retire_workers(
workers=list(to_close))
tasks = [
self.workers[w].close() for w in to_close
if w in self.workers
]
await asyncio.wait(tasks)
for task in tasks: # for tornado gen.coroutine support
with suppress(RuntimeError):
await task
for name in to_close:
if name in self.workers:
del self.workers[name]
to_open = set(self.worker_spec) - set(self.workers)
workers = []
for name in to_open:
d = self.worker_spec[name]
cls, opts = d["cls"], d.get("options", {})
if "name" not in opts:
opts = opts.copy()
opts["name"] = name
if isinstance(cls, str):
cls = import_term(cls)
worker = cls(self.scheduler.address, **opts)
self._created.add(worker)
workers.append(worker)
if workers:
await asyncio.wait(workers)
for w in workers:
w._cluster = weakref.ref(self)
await w # for tornado gen.coroutine support
self.workers.update(dict(zip(to_open, workers)))
def _update_worker_status(self, op, msg):
if op == "remove":
name = self.scheduler_info["workers"][msg]["name"]
def f():
if (name in self.workers
and msg not in self.scheduler_info["workers"]
and not any(
d["name"] == name
for d in self.scheduler_info["workers"].values())):
self._futures.add(
asyncio.ensure_future(self.workers[name].close()))
del self.workers[name]
delay = parse_timedelta(
dask.config.get("distributed.deploy.lost-worker-timeout"))
asyncio.get_event_loop().call_later(delay, f)
super()._update_worker_status(op, msg)
def __await__(self):
async def _():
if self.status == "created":
await self._start()
await self.scheduler
await self._correct_state()
if self.workers:
await asyncio.wait(list(self.workers.values())
) # maybe there are more
return self
return _().__await__()
async def _close(self):
while self.status == "closing":
await asyncio.sleep(0.1)
if self.status == "closed":
return
self.status = "closing"
self.scale(0)
await self._correct_state()
for future in self._futures:
await future
async with self._lock:
with suppress(CommClosedError):
if self.scheduler_comm:
await self.scheduler_comm.close(close_workers=True)
else:
logger.warning("Cluster closed without starting up")
await self.scheduler.close()
for w in self._created:
assert w.status == WorkerStatus.closed, w.status
if hasattr(self, "_old_logging_level"):
silence_logging(self._old_logging_level)
if hasattr(self, "_old_bokeh_logging_level"):
silence_logging(self._old_bokeh_logging_level, root="bokeh")
await super()._close()
async def __aenter__(self):
await self
await self._correct_state()
assert self.status == "running"
return self
def __exit__(self, typ, value, traceback):
super().__exit__(typ, value, traceback)
self._loop_runner.stop()
def _threads_per_worker(self) -> int:
""" Return the number of threads per worker for new workers """
if not self.new_spec:
raise ValueError(
"To scale by cores= you must specify cores per worker")
for name in ["nthreads", "ncores", "threads", "cores"]:
with suppress(KeyError):
return self.new_spec["options"][name]
if not self.new_spec:
raise ValueError(
"To scale by cores= you must specify cores per worker")
def _memory_per_worker(self) -> int:
""" Return the memory limit per worker for new workers """
if not self.new_spec:
raise ValueError(
"to scale by memory= your worker definition must include a memory_limit definition"
)
for name in ["memory_limit", "memory"]:
with suppress(KeyError):
return parse_bytes(self.new_spec["options"][name])
raise ValueError(
"to use scale(memory=...) your worker definition must include a memory_limit definition"
)
def scale(self, n=0, memory=None, cores=None):
if memory is not None:
n = max(
n,
int(math.ceil(parse_bytes(memory) /
self._memory_per_worker())))
if cores is not None:
n = max(n, int(math.ceil(cores / self._threads_per_worker())))
if len(self.worker_spec) > n:
not_yet_launched = set(self.worker_spec) - {
v["name"]
for v in self.scheduler_info["workers"].values()
}
while len(self.worker_spec) > n and not_yet_launched:
del self.worker_spec[not_yet_launched.pop()]
while len(self.worker_spec) > n:
self.worker_spec.popitem()
if self.status not in ("closing", "closed"):
while len(self.worker_spec) < n:
self.worker_spec.update(self.new_worker_spec())
self.loop.add_callback(self._correct_state)
if self.asynchronous:
return NoOpAwaitable()
def new_worker_spec(self):
""" Return name and spec for the next worker
Returns
-------
d: dict mapping names to worker specs
See Also
--------
scale
"""
while self._i in self.worker_spec:
self._i += 1
return {self._i: self.new_spec}
@property
def _supports_scaling(self):
return not not self.new_spec
async def scale_down(self, workers):
# We may have groups, if so, map worker addresses to job names
if not all(w in self.worker_spec for w in workers):
mapping = {}
for name, spec in self.worker_spec.items():
if "group" in spec:
for suffix in spec["group"]:
mapping[str(name) + suffix] = name
else:
mapping[name] = name
workers = {mapping.get(w, w) for w in workers}
for w in workers:
if w in self.worker_spec:
del self.worker_spec[w]
await self
scale_up = scale # backwards compatibility
@property
def plan(self):
out = set()
for name, spec in self.worker_spec.items():
if "group" in spec:
out.update({str(name) + suffix for suffix in spec["group"]})
else:
out.add(name)
return out
@property
def requested(self):
out = set()
for name in self.workers:
try:
spec = self.worker_spec[name]
except KeyError:
continue
if "group" in spec:
out.update({str(name) + suffix for suffix in spec["group"]})
else:
out.add(name)
return out
def adapt(self,
*args,
minimum=0,
maximum=math.inf,
minimum_cores: int = None,
maximum_cores: int = None,
minimum_memory: str = None,
maximum_memory: str = None,
**kwargs) -> Adaptive:
""" Turn on adaptivity
This scales Dask clusters automatically based on scheduler activity.
Parameters
----------
minimum : int
Minimum number of workers
maximum : int
Maximum number of workers
minimum_cores : int
Minimum number of cores/threads to keep around in the cluster
maximum_cores : int
Maximum number of cores/threads to keep around in the cluster
minimum_memory : str
Minimum amount of memory to keep around in the cluster
Expressed as a string like "100 GiB"
maximum_memory : str
Maximum amount of memory to keep around in the cluster
Expressed as a string like "100 GiB"
Examples
--------
>>> cluster.adapt(minimum=0, maximum_memory="100 GiB", interval='500ms')
See Also
--------
dask.distributed.Adaptive : for more keyword arguments
"""
if minimum_cores is not None:
minimum = max(
minimum or 0,
math.ceil(minimum_cores / self._threads_per_worker()))
if minimum_memory is not None:
minimum = max(
minimum or 0,
math.ceil(
parse_bytes(minimum_memory) / self._memory_per_worker()),
)
if maximum_cores is not None:
maximum = min(
maximum,
math.floor(maximum_cores / self._threads_per_worker()))
if maximum_memory is not None:
maximum = min(
maximum,
math.floor(
parse_bytes(maximum_memory) / self._memory_per_worker()),
)
return super().adapt(*args, minimum=minimum, maximum=maximum, **kwargs)
class Nanny(ServerNode):
""" A process to manage worker processes
The nanny spins up Worker processes, watches then, and kills or restarts
them as necessary. It is necessary if you want to use the
``Client.restart`` method, or to restart the worker automatically if
it gets to the terminate fractiom of its memory limit.
The parameters for the Nanny are mostly the same as those for the Worker.
See Also
--------
Worker
"""
_instances = weakref.WeakSet()
process = None
status = None
def __init__(self,
scheduler_ip=None,
scheduler_port=None,
scheduler_file=None,
worker_port=0,
nthreads=None,
ncores=None,
loop=None,
local_dir=None,
local_directory=None,
services=None,
name=None,
memory_limit="auto",
reconnect=True,
validate=False,
quiet=False,
resources=None,
silence_logs=None,
death_timeout=None,
preload=None,
preload_argv=None,
preload_nanny=None,
preload_nanny_argv=None,
security=None,
contact_address=None,
listen_address=None,
worker_class=None,
env=None,
interface=None,
host=None,
port=None,
protocol=None,
config=None,
**worker_kwargs):
self._setup_logging(logger)
self.loop = loop or IOLoop.current()
self.security = security or Security()
assert isinstance(self.security, Security)
self.connection_args = self.security.get_connection_args("worker")
if scheduler_file:
cfg = json_load_robust(scheduler_file)
self.scheduler_addr = cfg["address"]
elif scheduler_ip is None and dask.config.get("scheduler-address"):
self.scheduler_addr = dask.config.get("scheduler-address")
elif scheduler_port is None:
self.scheduler_addr = coerce_to_address(scheduler_ip)
else:
self.scheduler_addr = coerce_to_address(
(scheduler_ip, scheduler_port))
if protocol is None:
protocol_address = self.scheduler_addr.split("://")
if len(protocol_address) == 2:
protocol = protocol_address[0]
if ncores is not None:
warnings.warn("the ncores= parameter has moved to nthreads=")
nthreads = ncores
self._given_worker_port = worker_port
self.nthreads = nthreads or CPU_COUNT
self.reconnect = reconnect
self.validate = validate
self.resources = resources
self.death_timeout = parse_timedelta(death_timeout)
self.preload = preload
if self.preload is None:
self.preload = dask.config.get("distributed.worker.preload")
self.preload_argv = preload_argv
if self.preload_argv is None:
self.preload_argv = dask.config.get(
"distributed.worker.preload-argv")
self.preload_nanny = preload_nanny
if self.preload_nanny is None:
self.preload_nanny = dask.config.get("distributed.nanny.preload")
self.preload_nanny_argv = preload_nanny_argv
if self.preload_nanny_argv is None:
self.preload_nanny_argv = dask.config.get(
"distributed.nanny.preload-argv")
self.Worker = Worker if worker_class is None else worker_class
self.env = env or {}
self.config = config or {}
worker_kwargs.update({
"port": worker_port,
"interface": interface,
"protocol": protocol,
"host": host,
})
self.worker_kwargs = worker_kwargs
self.contact_address = contact_address
self.memory_terminate_fraction = dask.config.get(
"distributed.worker.memory.terminate")
if local_dir is not None:
warnings.warn("The local_dir keyword has moved to local_directory")
local_directory = local_dir
if local_directory is None:
local_directory = dask.config.get(
"temporary-directory") or os.getcwd()
if not os.path.exists(local_directory):
os.makedirs(local_directory)
local_directory = os.path.join(local_directory,
"dask-worker-space")
self.local_directory = local_directory
self._preload_modules = preloading.on_creation(
self.preload_nanny, file_dir=self.local_directory)
self.services = services
self.name = name
self.quiet = quiet
self.auto_restart = True
self.memory_limit = parse_memory_limit(memory_limit, self.nthreads)
if silence_logs:
silence_logging(level=silence_logs)
self.silence_logs = silence_logs
handlers = {
"instantiate": self.instantiate,
"kill": self.kill,
"restart": self.restart,
# cannot call it 'close' on the rpc side for naming conflict
"get_logs": self.get_logs,
"terminate": self.close,
"close_gracefully": self.close_gracefully,
"run": self.run,
}
super(Nanny, self).__init__(handlers=handlers,
io_loop=self.loop,
connection_args=self.connection_args)
self.scheduler = self.rpc(self.scheduler_addr)
if self.memory_limit:
pc = PeriodicCallback(self.memory_monitor, 100, io_loop=self.loop)
self.periodic_callbacks["memory"] = pc
if (not host and not interface
and not self.scheduler_addr.startswith("inproc://")):
host = get_ip(get_address_host(self.scheduler.address))
self._start_address = address_from_user_args(
host=host,
port=port,
interface=interface,
protocol=protocol,
security=security,
)
self._listen_address = listen_address
Nanny._instances.add(self)
self.status = "init"
def __repr__(self):
return "<Nanny: %s, threads: %d>" % (self.worker_address,
self.nthreads)
async def _unregister(self, timeout=10):
if self.process is None:
return
worker_address = self.process.worker_address
if worker_address is None:
return
allowed_errors = (TimeoutError, CommClosedError, EnvironmentError,
RPCClosed)
with ignoring(allowed_errors):
await asyncio.wait_for(
self.scheduler.unregister(address=self.worker_address),
timeout)
@property
def worker_address(self):
return None if self.process is None else self.process.worker_address
@property
def worker_dir(self):
return None if self.process is None else self.process.worker_dir
@property
def local_dir(self):
""" For API compatibility with Nanny """
warnings.warn("The local_dir attribute has moved to local_directory")
return self.local_directory
async def start(self):
""" Start nanny, start local process, start watching """
await super().start()
await self.listen(self._start_address,
**self.security.get_listen_args("worker"))
self.ip = get_address_host(self.address)
await preloading.on_start(
self._preload_modules,
self,
argv=self.preload_nanny_argv,
)
logger.info(" Start Nanny at: %r", self.address)
response = await self.instantiate()
if response == "running":
assert self.worker_address
self.status = "running"
else:
await self.close()
self.start_periodic_callbacks()
return self
async def kill(self, comm=None, timeout=2):
""" Kill the local worker process
Blocks until both the process is down and the scheduler is properly
informed
"""
self.auto_restart = False
if self.process is None:
return "OK"
deadline = self.loop.time() + timeout
await self.process.kill(timeout=0.8 * (deadline - self.loop.time()))
async def instantiate(self, comm=None):
""" Start a local worker process
Blocks until the process is up and the scheduler is properly informed
"""
if self._listen_address:
start_arg = self._listen_address
else:
host = self.listener.bound_address[0]
start_arg = self.listener.prefix + unparse_host_port(
host, self._given_worker_port)
if self.process is None:
worker_kwargs = dict(
scheduler_ip=self.scheduler_addr,
nthreads=self.nthreads,
local_directory=self.local_directory,
services=self.services,
nanny=self.address,
name=self.name,
memory_limit=self.memory_limit,
reconnect=self.reconnect,
resources=self.resources,
validate=self.validate,
silence_logs=self.silence_logs,
death_timeout=self.death_timeout,
preload=self.preload,
preload_argv=self.preload_argv,
security=self.security,
contact_address=self.contact_address,
)
worker_kwargs.update(self.worker_kwargs)
self.process = WorkerProcess(
worker_kwargs=worker_kwargs,
worker_start_args=(start_arg, ),
silence_logs=self.silence_logs,
on_exit=self._on_exit_sync,
worker=self.Worker,
env=self.env,
config=self.config,
)
self.auto_restart = True
if self.death_timeout:
try:
result = await asyncio.wait_for(self.process.start(),
self.death_timeout)
except TimeoutError:
await self.close(timeout=self.death_timeout)
logger.error(
"Timed out connecting Nanny '%s' to scheduler '%s'",
self,
self.scheduler_addr,
)
raise
else:
result = await self.process.start()
return result
async def restart(self, comm=None, timeout=2, executor_wait=True):
start = time()
async def _():
if self.process is not None:
await self.kill()
await self.instantiate()
try:
await asyncio.wait_for(_(), timeout)
except TimeoutError:
logger.error("Restart timed out, returning before finished")
return "timed out"
else:
return "OK"
@property
def _psutil_process(self):
pid = self.process.process.pid
try:
proc = self._psutil_process_obj
except AttributeError:
self._psutil_process_obj = psutil.Process(pid)
if self._psutil_process_obj.pid != pid:
self._psutil_process_obj = psutil.Process(pid)
return self._psutil_process_obj
def memory_monitor(self):
""" Track worker's memory. Restart if it goes above terminate fraction """
if self.status != "running":
return
process = self.process.process
if process is None:
return
try:
proc = self._psutil_process
memory = proc.memory_info().rss
except (ProcessLookupError, psutil.NoSuchProcess, psutil.AccessDenied):
return
frac = memory / self.memory_limit
if self.memory_terminate_fraction and frac > self.memory_terminate_fraction:
logger.warning(
"Worker exceeded %d%% memory budget. Restarting",
100 * self.memory_terminate_fraction,
)
process.terminate()
def is_alive(self):
return self.process is not None and self.process.is_alive()
def run(self, *args, **kwargs):
return run(self, *args, **kwargs)
def _on_exit_sync(self, exitcode):
self.loop.add_callback(self._on_exit, exitcode)
async def _on_exit(self, exitcode):
if self.status not in ("closing", "closed"):
try:
await self.scheduler.unregister(address=self.worker_address)
except (EnvironmentError, CommClosedError):
if not self.reconnect:
await self.close()
return
try:
if self.status not in ("closing", "closed",
"closing-gracefully"):
if self.auto_restart:
logger.warning("Restarting worker")
await self.instantiate()
elif self.status == "closing-gracefully":
await self.close()
except Exception:
logger.error(
"Failed to restart worker after its process exited",
exc_info=True)
@property
def pid(self):
return self.process and self.process.pid
def _close(self, *args, **kwargs):
warnings.warn("Worker._close has moved to Worker.close", stacklevel=2)
return self.close(*args, **kwargs)
def close_gracefully(self, comm=None):
"""
A signal that we shouldn't try to restart workers if they go away
This is used as part of the cluster shutdown process.
"""
self.status = "closing-gracefully"
async def close(self, comm=None, timeout=5, report=None):
"""
Close the worker process, stop all comms.
"""
if self.status == "closing":
await self.finished()
assert self.status == "closed"
if self.status == "closed":
return "OK"
self.status = "closing"
logger.info("Closing Nanny at %r", self.address)
await preloading.on_teardown(self._preload_modules, self)
self.stop()
try:
if self.process is not None:
await self.kill(timeout=timeout)
except Exception:
pass
self.process = None
await self.rpc.close()
self.status = "closed"
if comm:
await comm.write("OK")
await ServerNode.close(self)
class WebSocket(_WebSocket):
# Class attributes
event_queue = None
instances = weakref.WeakSet()
origins = []
# Instance attributes
client_id = None
filter = None
request = None
def __init__(self, *args, **kwargs):
super(WebSocket, self).__init__(*args, **kwargs)
self.request = get_current_request()
def __new__(cls, *args, **kwargs):
instance = super(WebSocket, cls).__new__(cls, *args, **kwargs)
cls.instances.add(instance)
return instance
@classmethod
def start_reader(cls, request):
cls.event_queue = gevent.queue.Queue()
reader_id = 'stream-{}#ephemeral'.format(_random_id())
reader = request.get_queue_reader('annotations', reader_id)
reader.on_message.connect(cls.on_queue_message)
reader.start(block=False)
gevent.spawn(broadcast_from_queue, cls.event_queue, cls.instances)
@classmethod
def on_queue_message(cls, reader, message=None):
if message is not None:
cls.event_queue.put(message)
def opened(self):
transaction.commit() # Release the database transaction
if self.event_queue is None:
self.start_reader(self.request)
def send_annotations(self):
request = self.request
user = get_user(request)
annotations = Annotation.search_raw(query=self.query.query, user=user)
self.received = len(annotations)
packet = _annotation_packet(annotations, 'past')
data = json.dumps(packet)
self.send(data)
def received_message(self, msg):
transaction.begin()
try:
data = json.loads(msg.data)
msg_type = data.get('messageType', 'filter')
if msg_type == 'filter':
payload = data['filter']
self.offsetFrom = 0
# Let's try to validate the schema
validate(payload, filter_schema)
self.filter = FilterHandler(payload)
self.query = FilterToElasticFilter(payload, self.request)
self.offsetFrom = 0
elif msg_type == 'more_hits':
more_hits = data.get('moreHits', 10)
self.query.query['from'] = self.offsetFrom
self.query.query['size'] = more_hits
self.send_annotations()
self.offsetFrom += self.received
elif msg_type == 'client_id':
self.client_id = data.get('value')
except:
log.exception("Parsing filter: %s", msg)
transaction.abort()
self.close()
else:
transaction.commit()
def cluster(nworkers=2, nanny=False, worker_kwargs={}, active_rpc_timeout=1,
scheduler_kwargs={}):
ws = weakref.WeakSet()
old_globals = _globals.copy()
for name, level in logging_levels.items():
logging.getLogger(name).setLevel(level)
enable_proctitle_on_children()
with pristine_loop() as loop:
with check_active_rpc(loop, active_rpc_timeout):
if nanny:
_run_worker = run_nanny
else:
_run_worker = run_worker
# The scheduler queue will receive the scheduler's address
scheduler_q = mp_context.Queue()
# Launch scheduler
scheduler = mp_context.Process(target=run_scheduler,
args=(scheduler_q, nworkers + 1),
kwargs=scheduler_kwargs)
ws.add(scheduler)
scheduler.daemon = True
scheduler.start()
# Launch workers
workers = []
for i in range(nworkers):
q = mp_context.Queue()
fn = '_test_worker-%s' % uuid.uuid4()
kwargs = merge({'ncores': 1, 'local_dir': fn,
'memory_limit': TOTAL_MEMORY}, worker_kwargs)
proc = mp_context.Process(target=_run_worker,
args=(q, scheduler_q),
kwargs=kwargs)
ws.add(proc)
workers.append({'proc': proc, 'queue': q, 'dir': fn})
for worker in workers:
worker['proc'].start()
for worker in workers:
worker['address'] = worker['queue'].get()
saddr = scheduler_q.get()
start = time()
try:
with rpc(saddr) as s:
while True:
ncores = loop.run_sync(s.ncores)
if len(ncores) == nworkers:
break
if time() - start > 5:
raise Exception("Timeout on cluster creation")
# avoid sending processes down to function
yield {'address': saddr}, [{'address': w['address'],
'proc': weakref.ref(w['proc'])}
for w in workers]
finally:
logger.debug("Closing out test cluster")
loop.run_sync(lambda: disconnect_all([w['address'] for w in workers],
timeout=0.5))
loop.run_sync(lambda: disconnect(saddr, timeout=0.5))
scheduler.terminate()
scheduler_q.close()
scheduler_q._reader.close()
scheduler_q._writer.close()
for w in workers:
w['proc'].terminate()
w['queue'].close()
w['queue']._reader.close()
w['queue']._writer.close()
scheduler.join(2)
del scheduler
for proc in [w['proc'] for w in workers]:
proc.join(timeout=2)
with ignoring(UnboundLocalError):
del worker, w, proc
del workers[:]
for fn in glob('_test_worker-*'):
shutil.rmtree(fn)
_globals.clear()
_globals.update(old_globals)
assert not ws
class Ultrasonic(SensorBase):
"""Ultrasonic rangefinder control
The Ultrasonic rangefinder measures
absolute distance based on the round-trip time of a ping generated by
the controller. These sensors use two transducers, a speaker and a
microphone both tuned to the ultrasonic range. A common ultrasonic
sensor, the Daventech SRF04 requires a short pulse to be generated on
a digital channel. This causes the chirp to be emitted. A second line
becomes high as the ping is transmitted and goes low when the echo is
received. The time that the line is high determines the round trip
distance (time of flight).
.. not_implemented: initialize
"""
class Unit:
"""The units to return when PIDGet is called"""
kInches = 0
kMillimeters = 1
#: Time (sec) for the ping trigger pulse.
kPingTime = 10 * 1e-6
#: Priority that the ultrasonic round robin task runs.
kPriority = 90
#: Max time (ms) between readings.
kMaxUltrasonicTime = 0.1
kSpeedOfSoundInchesPerSec = 1130.0 * 12.0
PIDSourceType = PIDSource.PIDSourceType
_static_mutex = threading.RLock()
#: ultrasonic sensor list
sensors = weakref.WeakSet()
#: Automatic round robin mode
automaticEnabled = False
instances = 0
_thread = None
@staticmethod
def isAutomaticMode():
with Ultrasonic._static_mutex:
return Ultrasonic.automaticEnabled
@staticmethod
def ultrasonicChecker():
"""Background task that goes through the list of ultrasonic sensors
and pings each one in turn. The counter is configured to read the
timing of the returned echo pulse.
.. warning:: DANGER WILL ROBINSON, DANGER WILL ROBINSON: This code runs
as a task and assumes that none of the ultrasonic sensors will
change while it's running. If one does, then this will certainly
break. Make sure to disable automatic mode before changing
anything with the sensors!!
"""
while Ultrasonic.isAutomaticMode():
count = 0
for u in Ultrasonic.sensors:
if not Ultrasonic.isAutomaticMode():
return
if u is None:
continue
count += 1
if u.isEnabled():
# do the ping
u.pingChannel.pulse(Ultrasonic.kPingTime)
Timer.delay(.1) # wait for ping to return
if not count:
return
def __init__(self, pingChannel, echoChannel, units=Unit.kInches):
"""Create an instance of the Ultrasonic Sensor.
This is designed to supchannel the Daventech SRF04 and Vex ultrasonic
sensors.
:param pingChannel: The digital output channel that sends the pulse
to initiate the sensor sending the ping.
:param echoChannel: The digital input channel that receives the echo.
The length of time that the echo is high represents the round
trip time of the ping, and the distance.
:param units: The units returned in either kInches or kMillimeters
"""
# Convert to DigitalInput and DigitalOutput if necessary
self.pingAllocated = False
self.echoAllocated = False
if not hasattr(pingChannel, 'channel'):
from .digitaloutput import DigitalOutput
pingChannel = DigitalOutput(pingChannel)
self.pingAllocated = True
if not hasattr(echoChannel, 'channel'):
from .digitalinput import DigitalInput
echoChannel = DigitalInput(echoChannel)
self.echoAllocated = True
self.pingChannel = pingChannel
self.echoChannel = echoChannel
self.units = units
self.pidSource = self.PIDSourceType.kDisplacement
self.enabled = True # make it available for round robin scheduling
self.valueEntry = None
# set up counter for this sensor
self.counter = Counter(self.echoChannel)
self.counter.setMaxPeriod(1.0)
self.counter.setSemiPeriodMode(True)
self.counter.reset()
isAutomatic = Ultrasonic.isAutomaticMode()
self.setAutomaticMode(False)
Ultrasonic.sensors.add(self)
if isAutomatic:
self.setAutomaticMode(True)
Resource._add_global_resource(self)
Ultrasonic.instances += 1
hal.report(hal.UsageReporting.kResourceType_Ultrasonic,
Ultrasonic.instances)
LiveWindow.addSensor("Ultrasonic", self.echoChannel.getChannel(), self)
def free(self):
isAutomatic = Ultrasonic.isAutomaticMode()
self.setAutomaticMode(False)
try:
Ultrasonic.sensors.remove(self)
except (KeyError, ValueError):
pass
if isAutomatic and len(Ultrasonic.sensors):
self.setAutomaticMode(True)
if self.pingAllocated and self.pingChannel:
self.pingChannel.free()
self.pingChannel = None
if self.echoAllocated and self.echoChannel:
self.echoChannel.free()
self.echoChannel = None
if self.counter != None:
self.counter.free()
self.counter = None
LiveWindow.removeComponent(self)
super().free()
def setAutomaticMode(self, enabling):
"""Turn Automatic mode on/off. When in Automatic mode, all sensors
will fire in round robin, waiting a set time between each sensor.
:param enabling:
Set to true if round robin scheduling should start for all the
ultrasonic sensors. This scheduling method assures that the
sensors are non-interfering because no two sensors fire at the
same time. If another scheduling algorithm is preferred, it
can be implemented by pinging the sensors manually and waiting
for the results to come back.
:type enabling: bool
"""
enabling = bool(enabling)
if enabling == Ultrasonic.isAutomaticMode():
return # ignore the case of no change
with Ultrasonic._static_mutex:
Ultrasonic.automaticEnabled = enabling
if enabling:
# Clear all the counters so no data is valid. No synchronization is
# needed because the background task is stopped.
for u in Ultrasonic.sensors:
if u is not None:
u.counter.reset()
# Start round robin task
Ultrasonic._thread = threading.Thread(
target=Ultrasonic.ultrasonicChecker, name="ultrasonicChecker")
Ultrasonic.daemon = True
Ultrasonic._thread.start()
else:
# Wait for background task to stop running
Ultrasonic._thread.join()
Ultrasonic._thread = None
# Clear all the counters (data now invalid) since automatic mode is
# disabled. No synchronization is needed because the background task is
# stopped.
for u in Ultrasonic.sensors:
if u is not None:
u.counter.reset()
def ping(self):
"""Single ping to ultrasonic sensor. Send out a single ping to the
ultrasonic sensor. This only works if automatic (round robin) mode is
disabled. A single ping is sent out, and the counter should count the
semi-period when it comes in. The counter is reset to make the current
value invalid.
"""
# turn off automatic round robin if pinging single sensor
self.setAutomaticMode(False)
# reset the counter to zero (invalid data now)
self.counter.reset()
# do the ping to start getting a single range
self.pingChannel.pulse(Ultrasonic.kPingTime)
def isRangeValid(self):
"""Check if there is a valid range measurement. The ranges are
accumulated in a counter that will increment on each edge of the
echo (return) signal. If the count is not at least 2, then the range
has not yet been measured, and is invalid.
:returns: True if the range is valid
:rtype: bool
"""
return self.counter.get() > 1
def getRangeInches(self):
"""Get the range in inches from the ultrasonic sensor.
:returns: Range in inches of the target returned from the ultrasonic
sensor. If there is no valid value yet, i.e. at least one
measurement hasn't completed, then return 0.
:rtype: float
"""
if self.isRangeValid():
return self.counter.getPeriod() * \
Ultrasonic.kSpeedOfSoundInchesPerSec / 2.0
else:
return 0
def getRangeMM(self):
"""Get the range in millimeters from the ultrasonic sensor.
:returns: Range in millimeters of the target returned by the
ultrasonic sensor. If there is no valid value yet, i.e. at least
one measurement hasn't complted, then return 0.
:rtype: float
"""
return self.getRangeInches() * 25.4
def setPIDSourceType(self, pidSource):
"""Set which parameter you are using as a process
control variable.
:param pidSource: An enum to select the parameter.
:type pidSource: :class:`.PIDSource.PIDSourceType`
"""
if pidSource != self.PIDSourceType.kDisplacement:
raise ValueError(
"Only displacement PID is allowed for ultrasonics.")
self.pidSource = pidSource
def getPIDSourceType(self):
return self.pidSource
def pidGet(self):
"""Get the range in the current DistanceUnit (PIDSource interface).
:returns: The range in DistanceUnit
:rtype: float
"""
if self.units == self.Unit.kInches:
return self.getRangeInches()
elif self.units == self.Unit.kMillimeters:
return self.getRangeMM()
else:
return 0.0
def setDistanceUnits(self, units):
"""Set the current DistanceUnit that should be used for the
PIDSource interface.
:param units: The DistanceUnit that should be used.
"""
if units not in [self.Unit.kInches, self.Unit.kMillimeters]:
raise ValueError("Invalid units argument '%s'" % units)
self.units = units
def getDistanceUnits(self):
"""Get the current DistanceUnit that is used for the PIDSource
interface.
:returns: The type of DistanceUnit that is being used.
"""
return self.units
def isEnabled(self):
"""Is the ultrasonic enabled.
:returns: True if the ultrasonic is enabled
"""
return self.enabled
def setEnabled(self, enable):
"""Set if the ultrasonic is enabled.
:param enable: set to True to enable the ultrasonic
:type enable: bool
"""
self.enabled = bool(enable)
# Live Window code, only does anything if live window is activated.
def getSmartDashboardType(self):
return "Ultrasonic"
def initTable(self, subtable):
if subtable is not None:
self.valueEntry = subtable.getEntry("Value")
self.updateTable()
else:
self.valueEntry = None
def updateTable(self):
if self.valueEntry is not None:
self.valueEntry.setDouble(self.getRangeInches())
def startLiveWindowMode(self):
pass
def stopLiveWindowMode(self):
pass
def callback():
try:
futures._chain_future(ensure_future(coro, loop=loop), future)
except (SystemExit, KeyboardInterrupt):
raise
except BaseException as exc:
if future.set_running_or_notify_cancel():
future.set_exception(exc)
raise
loop.call_soon_threadsafe(callback)
return future
# WeakSet containing all alive tasks.
_all_tasks = weakref.WeakSet()
# Dictionary containing tasks that are currently active in
# all running event loops. {EventLoop: Task}
_current_tasks = {}
def _register_task(task):
"""Register a new task in asyncio as executed by loop."""
_all_tasks.add(task)
def _enter_task(loop, task):
current_task = _current_tasks.get(loop)
if current_task is not None:
raise RuntimeError(f"Cannot enter into task {task!r} while another "
def freeze(self):
self.next = frozenset(self.next)
# Assumption: All CFG nodes have identical life spans, because the graph
# owns them. Nodes should never be used outside the context of an existing
# graph.
self.prev = weakref.WeakSet(self.prev)
def add_to_rel_load_list(session, flush_context=None):
# keep track of new items to load relationships on during commit
session.info.setdefault('_load_rels',
weakref.WeakSet()).update(session.new)
import weakref
from multiprocessing import Process, Event
from time import sleep
from typing import Union
import yaml
from experimentor.core.meta import ExperimentorProcess
from experimentor.core.signal import Signal
from experimentor.core.subscriber import Subscriber
from experimentor.lib.log import get_logger
from experimentor.models.decorators import not_implemented
from experimentor.models.models import BaseModel
from experimentor.models.meta import MetaModel
_experiments = weakref.WeakSet() # Stores all the defined experiments
logger = get_logger(__name__)
class FormatPlaceholder:
def __init__(self, key):
self.key = key
def __format__(self, spec):
result = self.key
if spec:
result += ":" + spec
return "{" + result + "}"
class FormatDict(dict):
def __init__(self,
handlers,
stream_handlers=None,
connection_limit=512,
deserialize=True,
io_loop=None):
self.handlers = {
'identity': self.identity,
'connection_stream': self.handle_stream,
}
self.handlers.update(handlers)
self.stream_handlers = {}
self.stream_handlers.update(stream_handlers or {})
self.id = type(self).__name__ + '-' + str(uuid.uuid4())
self._address = None
self._listen_address = None
self._port = None
self._comms = {}
self.deserialize = deserialize
self.monitor = SystemMonitor()
self.counters = None
self.digests = None
self.events = None
self.event_counts = None
self._ongoing_coroutines = weakref.WeakSet()
self.listener = None
self.io_loop = io_loop or IOLoop.current()
self.loop = self.io_loop
# Statistics counters for various events
with ignoring(ImportError):
from .counter import Digest
self.digests = defaultdict(partial(Digest, loop=self.io_loop))
from .counter import Counter
self.counters = defaultdict(partial(Counter, loop=self.io_loop))
self.events = defaultdict(lambda: deque(maxlen=10000))
self.event_counts = defaultdict(lambda: 0)
self.periodic_callbacks = dict()
pc = PeriodicCallback(self.monitor.update, 500, io_loop=self.io_loop)
self.periodic_callbacks['monitor'] = pc
self._last_tick = time()
pc = PeriodicCallback(
self._measure_tick,
parse_timedelta(dask.config.get('distributed.admin.tick.interval'),
default='ms') * 1000,
io_loop=self.io_loop)
self.periodic_callbacks['tick'] = pc
self.thread_id = 0
@gen.coroutine
def set_thread_ident():
self.thread_id = get_thread_identity()
self.io_loop.add_callback(set_thread_ident)
self.__stopped = False
fun1 = FunTimes()
fun1.txaio = txaio.with_config(loop=asyncio.new_event_loop())
So `fun1` will run its futures on the newly-created event loop,
while `fun0` will work just as it did before this `with_config`
method was introduced (after 2.6.2).
"""
cfg = _Config()
if loop is not None:
cfg.loop = loop
return _AsyncioApi(cfg)
# logging should probably all be folded into _AsyncioApi as well
_stderr, _stdout = sys.stderr, sys.stdout
_loggers = weakref.WeakSet(
) # weak-ref's of each logger we've created before start_logging()
_log_level = 'info' # re-set by start_logging
_started_logging = False
_categories = {}
def add_log_categories(categories):
_categories.update(categories)
class FailedFuture(IFailedFuture):
"""
This provides an object with any features from Twisted's Failure
that we might need in Autobahn classes that use FutureMixin.
We need to encapsulate information from exceptions so that
