class Context:
def __init__(self, context):
self.__context = context
self.__statements = WeakSet()
def __hash__(self):
return hash(self.__context)
def __eq__(self, other):
try:
return str(other) == self.__context
except AttributeError:
return False
def __str__(self):
return str(self.__context)
def __repr__(self):
return f'Context({self.__context})'
def __len__(self):
return len(self.__statements)
def unwrap(self):
return self.__context
def add_statement(self, stmt):
self.__statements.add(stmt)
def remove_statement(self, stmt):
self.__statements.discard(stmt)
def statements(self):
return self.__statements
class LazyInvalidation(object):
def __enter__(self):
assert threading.current_thread() == MAIN_THREAD
assert not evaluation_stack
self._watchMap = WeakKeyDictionary()
self._watchable_objects = WeakSet()
global invalidation_strategy
invalidation_strategy._invalidate_all()
invalidation_strategy = self
def _watch_object(self, object):
if object.watcher is not None and object.watcher not in self._watchMap:
self._watchMap[object.watcher] = WeakWatchIntermediary(object, object.watcher)
def _add_dependency(self, object):
if evaluation_stack:
evaluation_stack[-1].deps.append(object)
def _unwatch_object(self, object):
object.invalidate()
self._watchable_objects.discard(object)
def __exit__(self, type, value, traceback):
global invalidation_strategy
invalidation_strategy = LazyConstants()
for intermediary in self._watchMap.itervalues():
intermediary.release()
self._watchMap.clear()
def _invalidate_all(self):
raise TypeError('Cannot nest lazy_invalidation contexts')
class Signal(object):
def __init__(self):
self._functions = WeakSet()
self._methods = WeakKeyDictionary()
def __call__(self, *args, **kargs):
# Call handler functions
to_be_removed = []
for func in self._functions.copy():
try:
func(*args, **kargs)
except RuntimeError:
Warning.warn(
'Signals func->RuntimeError: func "{}" will be removed.'.
format(func))
to_be_removed.append(func)
for remove in to_be_removed:
self._functions.discard(remove)
# Call handler methods
to_be_removed = []
emitters = self._methods.copy()
for obj, funcs in emitters.items():
msg_debug('obj is type "{}"'.format(type(obj)))
for func in funcs.copy():
try:
func(obj, *args, **kargs)
except RuntimeError:
warnings.warn(
'Signals methods->RuntimeError, obj.func "{}.{}" will be removed'
.format(obj, func))
to_be_removed.append((obj, func))
for obj, func in to_be_removed:
self._methods[obj].discard(func)
def connect(self, slot):
if inspect.ismethod(slot):
if slot.__self__ not in self._methods:
self._methods[slot.__self__] = set()
self._methods[slot.__self__].add(slot.__func__)
else:
self._functions.add(slot)
def disconnect(self, slot):
if inspect.ismethod(slot):
if slot.__self__ in self._methods:
self._methods[slot.__self__].remove(slot.__func__)
else:
if slot in self._functions:
self._functions.remove(slot)
def clear(self):
self._functions.clear()
self._methods.clear()
class PrivacyService(Service):
def __init__(self):
self._privacy_instances = WeakSet()
self._potential_vehicles_to_check = WeakSet()
@property
def privacy_instances(self):
return self._privacy_instances
def check_for_late_violators(self, sim):
for privacy in self.privacy_instances:
if not privacy.privacy_violators & PrivacyViolators.SIM:
continue
if not sim in privacy.violators:
if sim in privacy.late_violators:
continue
if privacy.is_sim_shoo_exempt(sim):
if not privacy.persistent_instance:
privacy.add_exempt_sim(sim)
if privacy.persistent_instance:
privacy.remove_sim_from_allowed_disallowed(sim)
if sim not in privacy.find_violating_sims():
continue
privacy.handle_late_violator(sim)
return True
else:
if privacy.persistent_instance:
privacy.remove_sim_from_allowed_disallowed(sim)
if sim not in privacy.find_violating_sims():
continue
privacy.handle_late_violator(sim)
return True
return False
def add_instance(self, instance):
self._privacy_instances.add(instance)
def remove_instance(self, instance):
self.privacy_instances.discard(instance)
def stop(self):
while self.privacy_instances:
instance = self.privacy_instances.pop()
instance.cleanup_privacy_instance()
self._potential_vehicles_to_check.clear()
def get_potential_vehicle_violators(self):
return self._potential_vehicles_to_check
def add_vehicle_to_monitor(self, vehicle):
self._potential_vehicles_to_check.add(vehicle)
def remove_vehicle_to_monitor(self, vehicle):
self._potential_vehicles_to_check.discard(vehicle)
class Signal(object):
def __init__(self):
self._functions = WeakSet()
self._methods = WeakKeyDictionary()
def __call__(self, *args, **kargs):
# Call handler functions
to_be_removed = []
for func in self._functions.copy():
try:
func(*args, **kargs)
except RuntimeError:
Warning.warn('Signals func->RuntimeError: func "{}" will be removed.'.format(func))
to_be_removed.append(func)
for remove in to_be_removed:
self._functions.discard(remove)
# Call handler methods
to_be_removed = []
emitters = self._methods.copy()
for obj, funcs in emitters.items():
msg_debug('obj is type "{}"'.format(type(obj)))
for func in funcs.copy():
try:
func(obj, *args, **kargs)
except RuntimeError:
warnings.warn('Signals methods->RuntimeError, obj.func "{}.{}" will be removed'.format(obj, func))
to_be_removed.append((obj, func))
for obj, func in to_be_removed:
self._methods[obj].discard(func)
def connect(self, slot):
if inspect.ismethod(slot):
if slot.__self__ not in self._methods:
self._methods[slot.__self__] = set()
self._methods[slot.__self__].add(slot.__func__)
else:
self._functions.add(slot)
def disconnect(self, slot):
if inspect.ismethod(slot):
if slot.__self__ in self._methods:
self._methods[slot.__self__].remove(slot.__func__)
else:
if slot in self._functions:
self._functions.remove(slot)
def clear(self):
self._functions.clear()
self._methods.clear()
class Observable:
def __init__(self):
self._observers = WeakSet()
def add_observer(self, observer):
self._observers.add(observer)
def remove_observer(self, observer):
self._observers.discard(observer)
def notify_observers(self):
for observer in self._observers:
observer.observable_update(self)
class ObservableEvent(object):
def __init__(self, weakref=False):
self.listeners = WeakSet() if weakref else set()
def __iadd__(self, ob):
self.listeners.add(ob)
return self
def __isub__(self, ob):
self.listeners.discard(ob)
return self
def notify(self, *a, **k):
for ob in list(self.listeners):
ob(*a, **k)
class ObservableEvent(object):
def __init__(self, weakref=False):
self.listeners = WeakSet() if weakref else set()
def __iadd__(self, ob):
self.listeners.add(ob)
return self
def __isub__(self, ob):
self.listeners.discard(ob)
return self
def notify(self, *a, **k):
for ob in list(self.listeners):
ob(*a, **k)
class CableBear (object):
def __init__ (self):
self._go_up_deferral = None
self._waiting = True
self.sessions = WeakSet()
core.listen_to_dependencies(self, components=["WebServer"])
def _all_dependencies_met (self):
core.WebServer.set_handler("/cable_bear/ws", TCPAnalysisSession, self)
def _handle_POXDesk_NewSession (self, event):
if self._waiting is False: return
self._waiting = False
event.session.sendmsg(dict(method="new_CableBear"))
log.debug("New POXDesk session -- starting CableBear frontend")
def _handle_POXDesk_EndSession (self, event):
if event.session in self.sessions:
self.sessions.discard(event.session)
if not self.sessions:
self._waiting = True
def _handle_core_GoingUpEvent (self, e):
if self._waiting:
self._go_up_deferral = e.get_deferral()
log.info("Waiting for CableBear frontend to connect")
def register_session (self, session):
if self.sessions: return False # One at a time
self.sessions.add(session)
return True
def unregister_session (self, session):
self.sessions.discard(session)
def add_record (self, msg):
bad = None
for s in self.sessions:
try:
s.sendmsg(msg)
except Exception:
log.exception("While trying to send to CableBear frontend")
if bad is None: bad = []
bad.append(s)
if bad:
for b in bad:
self.sessions.discard(b)
class AsyncoreLoop(object):
def __init__(self):
self._loop_lock = Lock()
self._started = False
self._shutdown = False
self._conns_lock = Lock()
self._conns = WeakSet()
def maybe_start(self):
should_start = False
did_acquire = False
try:
did_acquire = self._loop_lock.acquire(False)
if did_acquire and not self._started:
self._started = True
should_start = True
finally:
if did_acquire:
self._loop_lock.release()
if should_start:
thread = Thread(target=self._run_loop, name="cassandra_driver_event_loop")
thread.daemon = True
thread.start()
atexit.register(partial(self._cleanup, thread))
def _run_loop(self):
log.debug("Starting asyncore event loop")
with self._loop_lock:
while True:
try:
asyncore.loop(timeout=0.001, use_poll=True, count=1000)
except Exception:
log.debug("Asyncore event loop stopped unexepectedly", exc_info=True)
break
if self._shutdown:
break
with self._conns_lock:
if len(self._conns) == 0:
break
self._started = False
log.debug("Asyncore event loop ended")
def _cleanup(self, thread):
self._shutdown = True
log.debug("Waiting for event loop thread to join...")
thread.join()
log.debug("Event loop thread was joined")
def connection_created(self, connection):
with self._conns_lock:
self._conns.add(connection)
def connection_destroyed(self, connection):
with self._conns_lock:
self._conns.discard(connection)
class TestWeakSet(unittest.TestCase):
def setUp(self):
# need to keep references to them
self.items = [SomeClass(c) for c in ('a', 'b', 'c')]
self.items2 = [SomeClass(c) for c in ('x', 'y', 'z')]
self.letters = [SomeClass(c) for c in string.ascii_letters]
self.s = WeakSet(self.items)
self.d = dict.fromkeys(self.items)
self.obj = SomeClass('F')
self.fs = WeakSet([self.obj])
def test_methods(self):
weaksetmethods = dir(WeakSet)
for method in dir(set):
if method == 'test_c_api' or method.startswith('_'):
continue
self.assertIn(method, weaksetmethods,
"WeakSet missing method " + method)
def test_new_or_init(self):
self.assertRaises(TypeError, WeakSet, [], 2)
def test_len(self):
self.assertEqual(len(self.s), len(self.d))
self.assertEqual(len(self.fs), 1)
del self.obj
self.assertEqual(len(self.fs), 0)
def test_contains(self):
for c in self.letters:
self.assertEqual(c in self.s, c in self.d)
# 1 is not weakref'able, but that TypeError is caught by __contains__
self.assertNotIn(1, self.s)
self.assertIn(self.obj, self.fs)
del self.obj
self.assertNotIn(SomeClass('F'), self.fs)
def test_union(self):
u = self.s.union(self.items2)
for c in self.letters:
self.assertEqual(c in u, c in self.d or c in self.items2)
self.assertEqual(self.s, WeakSet(self.items))
self.assertEqual(type(u), WeakSet)
self.assertRaises(TypeError, self.s.union, [[]])
for C in set, frozenset, dict.fromkeys, list, tuple:
x = WeakSet(self.items + self.items2)
c = C(self.items2)
self.assertEqual(self.s.union(c), x)
def test_or(self):
i = self.s.union(self.items2)
self.assertEqual(self.s | set(self.items2), i)
self.assertEqual(self.s | frozenset(self.items2), i)
def test_intersection(self):
i = self.s.intersection(self.items2)
for c in self.letters:
self.assertEqual(c in i, c in self.d and c in self.items2)
self.assertEqual(self.s, WeakSet(self.items))
self.assertEqual(type(i), WeakSet)
for C in set, frozenset, dict.fromkeys, list, tuple:
x = WeakSet([])
self.assertEqual(self.s.intersection(C(self.items2)), x)
def test_isdisjoint(self):
self.assertTrue(self.s.isdisjoint(WeakSet(self.items2)))
self.assertTrue(not self.s.isdisjoint(WeakSet(self.letters)))
def test_and(self):
i = self.s.intersection(self.items2)
self.assertEqual(self.s & set(self.items2), i)
self.assertEqual(self.s & frozenset(self.items2), i)
def test_difference(self):
i = self.s.difference(self.items2)
for c in self.letters:
self.assertEqual(c in i, c in self.d and c not in self.items2)
self.assertEqual(self.s, WeakSet(self.items))
self.assertEqual(type(i), WeakSet)
self.assertRaises(TypeError, self.s.difference, [[]])
def test_sub(self):
i = self.s.difference(self.items2)
self.assertEqual(self.s - set(self.items2), i)
self.assertEqual(self.s - frozenset(self.items2), i)
def test_symmetric_difference(self):
i = self.s.symmetric_difference(self.items2)
for c in self.letters:
self.assertEqual(c in i, (c in self.d) ^ (c in self.items2))
self.assertEqual(self.s, WeakSet(self.items))
self.assertEqual(type(i), WeakSet)
self.assertRaises(TypeError, self.s.symmetric_difference, [[]])
def test_xor(self):
i = self.s.symmetric_difference(self.items2)
self.assertEqual(self.s ^ set(self.items2), i)
self.assertEqual(self.s ^ frozenset(self.items2), i)
def test_sub_and_super(self):
pl, ql, rl = map(lambda s: [SomeClass(c) for c in s], ['ab', 'abcde', 'def'])
p, q, r = map(WeakSet, (pl, ql, rl))
self.assertTrue(p < q)
self.assertTrue(p <= q)
self.assertTrue(q <= q)
self.assertTrue(q > p)
self.assertTrue(q >= p)
self.assertFalse(q < r)
self.assertFalse(q <= r)
self.assertFalse(q > r)
self.assertFalse(q >= r)
self.assertTrue(set('a').issubset('abc'))
self.assertTrue(set('abc').issuperset('a'))
self.assertFalse(set('a').issubset('cbs'))
self.assertFalse(set('cbs').issuperset('a'))
def test_gc(self):
# Create a nest of cycles to exercise overall ref count check
s = WeakSet(Foo() for i in range(1000))
for elem in s:
elem.cycle = s
elem.sub = elem
elem.set = WeakSet([elem])
def test_subclass_with_custom_hash(self):
# Bug #1257731
class H(WeakSet):
def __hash__(self):
return int(id(self) & 0x7fffffff)
s=H()
f=set()
f.add(s)
self.assertIn(s, f)
f.remove(s)
f.add(s)
f.discard(s)
def test_init(self):
s = WeakSet()
s.__init__(self.items)
self.assertEqual(s, self.s)
s.__init__(self.items2)
self.assertEqual(s, WeakSet(self.items2))
self.assertRaises(TypeError, s.__init__, s, 2);
self.assertRaises(TypeError, s.__init__, 1);
def test_constructor_identity(self):
s = WeakSet(self.items)
t = WeakSet(s)
self.assertNotEqual(id(s), id(t))
def test_hash(self):
self.assertRaises(TypeError, hash, self.s)
def test_clear(self):
self.s.clear()
self.assertEqual(self.s, WeakSet([]))
self.assertEqual(len(self.s), 0)
def test_copy(self):
dup = self.s.copy()
self.assertEqual(self.s, dup)
self.assertNotEqual(id(self.s), id(dup))
def test_add(self):
x = SomeClass('Q')
self.s.add(x)
self.assertIn(x, self.s)
dup = self.s.copy()
self.s.add(x)
self.assertEqual(self.s, dup)
self.assertRaises(TypeError, self.s.add, [])
self.fs.add(Foo())
self.assertTrue(len(self.fs) == 1)
self.fs.add(self.obj)
self.assertTrue(len(self.fs) == 1)
def test_remove(self):
x = SomeClass('a')
self.s.remove(x)
self.assertNotIn(x, self.s)
self.assertRaises(KeyError, self.s.remove, x)
self.assertRaises(TypeError, self.s.remove, [])
def test_discard(self):
a, q = SomeClass('a'), SomeClass('Q')
self.s.discard(a)
self.assertNotIn(a, self.s)
self.s.discard(q)
self.assertRaises(TypeError, self.s.discard, [])
def test_pop(self):
for i in range(len(self.s)):
elem = self.s.pop()
self.assertNotIn(elem, self.s)
self.assertRaises(KeyError, self.s.pop)
def test_update(self):
retval = self.s.update(self.items2)
self.assertEqual(retval, None)
for c in (self.items + self.items2):
self.assertIn(c, self.s)
self.assertRaises(TypeError, self.s.update, [[]])
def test_update_set(self):
self.s.update(set(self.items2))
for c in (self.items + self.items2):
self.assertIn(c, self.s)
def test_ior(self):
self.s |= set(self.items2)
for c in (self.items + self.items2):
self.assertIn(c, self.s)
def test_intersection_update(self):
retval = self.s.intersection_update(self.items2)
self.assertEqual(retval, None)
for c in (self.items + self.items2):
if c in self.items2 and c in self.items:
self.assertIn(c, self.s)
else:
self.assertNotIn(c, self.s)
self.assertRaises(TypeError, self.s.intersection_update, [[]])
def test_iand(self):
self.s &= set(self.items2)
for c in (self.items + self.items2):
if c in self.items2 and c in self.items:
self.assertIn(c, self.s)
else:
self.assertNotIn(c, self.s)
def test_difference_update(self):
retval = self.s.difference_update(self.items2)
self.assertEqual(retval, None)
for c in (self.items + self.items2):
if c in self.items and c not in self.items2:
self.assertIn(c, self.s)
else:
self.assertNotIn(c, self.s)
self.assertRaises(TypeError, self.s.difference_update, [[]])
self.assertRaises(TypeError, self.s.symmetric_difference_update, [[]])
def test_isub(self):
self.s -= set(self.items2)
for c in (self.items + self.items2):
if c in self.items and c not in self.items2:
self.assertIn(c, self.s)
else:
self.assertNotIn(c, self.s)
def test_symmetric_difference_update(self):
retval = self.s.symmetric_difference_update(self.items2)
self.assertEqual(retval, None)
for c in (self.items + self.items2):
if (c in self.items) ^ (c in self.items2):
self.assertIn(c, self.s)
else:
self.assertNotIn(c, self.s)
self.assertRaises(TypeError, self.s.symmetric_difference_update, [[]])
def test_ixor(self):
self.s ^= set(self.items2)
for c in (self.items + self.items2):
if (c in self.items) ^ (c in self.items2):
self.assertIn(c, self.s)
else:
self.assertNotIn(c, self.s)
def test_inplace_on_self(self):
t = self.s.copy()
t |= t
self.assertEqual(t, self.s)
t &= t
self.assertEqual(t, self.s)
t -= t
self.assertEqual(t, WeakSet())
t = self.s.copy()
t ^= t
self.assertEqual(t, WeakSet())
def test_eq(self):
# issue 5964
self.assertTrue(self.s == self.s)
self.assertTrue(self.s == WeakSet(self.items))
self.assertFalse(self.s == set(self.items))
self.assertFalse(self.s == list(self.items))
self.assertFalse(self.s == tuple(self.items))
self.assertFalse(self.s == 1)
def test_weak_destroy_while_iterating(self):
# Issue #7105: iterators shouldn't crash when a key is implicitly removed
# Create new items to be sure no-one else holds a reference
items = [SomeClass(c) for c in ('a', 'b', 'c')]
s = WeakSet(items)
it = iter(s)
next(it) # Trigger internal iteration
# Destroy an item
del items[-1]
gc.collect() # just in case
# We have removed either the first consumed items, or another one
self.assertIn(len(list(it)), [len(items), len(items) - 1])
del it
# The removal has been committed
self.assertEqual(len(s), len(items))
def test_weak_destroy_and_mutate_while_iterating(self):
# Issue #7105: iterators shouldn't crash when a key is implicitly removed
items = [SomeClass(c) for c in string.ascii_letters]
s = WeakSet(items)
@contextlib.contextmanager
def testcontext():
try:
it = iter(s)
next(it)
# Schedule an item for removal and recreate it
u = SomeClass(str(items.pop()))
gc.collect() # just in case
yield u
finally:
it = None # should commit all removals
with testcontext() as u:
self.assertNotIn(u, s)
with testcontext() as u:
self.assertRaises(KeyError, s.remove, u)
self.assertNotIn(u, s)
with testcontext() as u:
s.add(u)
self.assertIn(u, s)
t = s.copy()
with testcontext() as u:
s.update(t)
self.assertEqual(len(s), len(t))
with testcontext() as u:
s.clear()
self.assertEqual(len(s), 0)
class ROBroker(object):
"""
Helper class which simplifies handling of range observers.
Can be used as a mixin or standalone.
"""
implements(IRangeObservable)
__robservers = None
def addRangeObserver(self, rangeObserver):
assert IRangeObserver.providedBy(rangeObserver)
if self.__robservers == None:
self.__robservers = WeakSet()
self.__robservers.add(rangeObserver)
def removeRangeObserver(self, rangeObserver):
assert IRangeObserver.providedBy(rangeObserver)
if self.__robservers == None:
return
self.__robservers.discard(rangeObserver)
def notifyRangeWillChange(self, rangeFrom, rangeTo, srcobject=None):
"""
Call this to inform the respective observers of an impending range
change.
You must call `notifyRangeDidChange()` after you performed the
range change. The call is not stackable.
"""
if srcobject == None:
srcobject = self
self.__rangeFrom = rangeFrom
self.__rangeTo = rangeTo
self.__srcobject = srcobject
if self.__robservers != None:
for rangeObserver in set(self.__robservers):
rangeObserver.observedRangeWillChange(srcobject, rangeFrom, rangeTo)
def notifyRangeDidChange(self):
"""
Call this to inform the respective observers of a performed range
change.
You must have called `notifyRangeWillChange()` before.
"""
if self.__robservers != None:
for rangeObserver in set(self.__robservers):
rangeObserver.observedRangeDidChange(self.__srcobject, self.__rangeFrom, self.__rangeTo)
del self.__rangeFrom
del self.__rangeTo
del self.__srcobject
def notifyRangeWillIncrease(self, rangeFrom, rangeTo, srcobject=None):
"""
Call this to inform the respective observers of an impending range
increase.
You must call `notifyRangeDidIncrease()` after you performed the
range increase. The call is not stackable.
"""
if srcobject == None:
srcobject = self
self.__rangeFrom = rangeFrom
self.__rangeTo = rangeTo
self.__srcobject = srcobject
if self.__robservers != None:
for rangeObserver in set(self.__robservers):
rangeObserver.observedRangeWillIncrease(srcobject, rangeFrom, rangeTo)
def notifyRangeDidIncrease(self):
"""
Call this to inform the respective observers of a performed range
increase.
You must have called `notifyRangeWillIncrease()` before.
"""
if self.__robservers != None:
for rangeObserver in set(self.__robservers):
rangeObserver.observedRangeDidIncrease(self.__srcobject, self.__rangeFrom, self.__rangeTo)
del self.__rangeFrom
del self.__rangeTo
del self.__srcobject
def notifyRangeWillDecrease(self, rangeFrom, rangeTo, srcobject=None):
"""
Call this to inform the respective observers of an impending range
decrease.
You must call `notifyRangeDidDecrease()` after you performed the
range decrease. The call is not stackable.
"""
if srcobject == None:
srcobject = self
self.__rangeFrom = rangeFrom
self.__rangeTo = rangeTo
self.__srcobject = srcobject
if self.__robservers != None:
for rangeObserver in set(self.__robservers):
rangeObserver.observedRangeWillDecrease(srcobject, rangeFrom, rangeTo)
def notifyRangeDidDecrease(self):
"""
Call this to inform the respective observers of a performed range
decrease.
You must have called `notifyRangeWillDecrease()` before.
"""
if self.__robservers != None:
for rangeObserver in set(self.__robservers):
rangeObserver.observedRangeDidDecrease(self.__srcobject, self.__rangeFrom, self.__rangeTo)
del self.__rangeFrom
del self.__rangeTo
del self.__srcobject
class TileableData(SerializableWithKey, Tileable):
__slots__ = '__weakref__', '_siblings', '_cix', '_entities'
_no_copy_attrs_ = SerializableWithKey._no_copy_attrs_ | {'_cix'}
# required fields
_op = KeyField('op')
_shape = TupleField('shape',
ValueType.int64,
on_serialize=on_serialize_shape,
on_deserialize=on_deserialize_shape)
# optional fields
# `nsplits` means the sizes of chunks for each dimension
_nsplits = TupleField('nsplits',
ValueType.tuple(ValueType.uint64),
on_serialize=on_serialize_nsplits)
_extra_params = DictField('extra_params',
key_type=ValueType.string,
on_deserialize=AttributeDict)
def __init__(self, *args, **kwargs):
extras = AttributeDict(
(k, kwargs.pop(k)) for k in set(kwargs) - set(self.__slots__))
kwargs['_extra_params'] = kwargs.pop('_extra_params', extras)
if kwargs.get('_nsplits', None) is not None:
kwargs['_nsplits'] = tuple(tuple(s) for s in kwargs['_nsplits'])
super(TileableData, self).__init__(*args, **kwargs)
if hasattr(self, '_chunks') and self._chunks:
self._chunks = sorted(self._chunks, key=attrgetter('index'))
self._entities = WeakSet()
@property
def ndim(self):
return len(self.shape)
def __len__(self):
try:
return self.shape[0]
except IndexError:
if build_mode().is_build_mode:
return 0
raise TypeError('len() of unsized object')
@property
def shape(self):
if hasattr(self, '_shape') and self._shape is not None:
return self._shape
if hasattr(self, '_nsplits') and self._nsplits is not None:
self._shape = tuple(
builtins.sum(nsplit) for nsplit in self._nsplits)
return self._shape
def _update_shape(self, new_shape):
self._shape = new_shape
@property
def chunk_shape(self):
if hasattr(self, '_nsplits') and self._nsplits is not None:
return tuple(map(len, self._nsplits))
@property
def chunks(self):
return getattr(self, '_chunks', None)
@property
def op(self):
return getattr(self, '_op', None)
@property
def nsplits(self):
return getattr(self, '_nsplits', None)
@nsplits.setter
def nsplits(self, new_nsplits):
self._nsplits = new_nsplits
@property
def size(self):
return np.prod(self.shape).item()
@property
def inputs(self):
return self.op.inputs or []
@inputs.setter
def inputs(self, new_inputs):
self.op.inputs = new_inputs
@property
def params(self):
# params return the properties which useful to rebuild a new tileable object
return {'shape': self.shape}
@property
def extra_params(self):
return self._extra_params
@property
def cix(self):
if self.ndim == 0:
return ChunksIndexer(self)
try:
if getattr(self, '_cix', None) is None:
self._cix = ChunksIndexer(self)
return self._cix
except (TypeError, ValueError):
return ChunksIndexer(self)
@property
def entities(self):
return self._entities
def is_coarse(self):
return not hasattr(self, '_chunks') or self._chunks is None or len(
self._chunks) == 0
def to_coarse(self):
if self.is_coarse():
return self
new_entity = self.copy()
new_entity._obj_set('_id', self._id)
new_entity._chunks = None
if self.inputs is None or len(self.inputs) == 0:
new_entity.extra_params.update({'raw_chunk_size': self.nsplits})
return new_entity
def is_sparse(self):
return self.op.is_sparse()
issparse = is_sparse
@enter_build_mode
def attach(self, entity):
self._entities.add(entity)
@enter_build_mode
def detach(self, entity):
self._entities.discard(entity)
def tiles(self):
return handler.tiles(self)
def single_tiles(self):
return handler.single_tiles(self)
@kernel_mode
def build_graph(self,
graph=None,
cls=DAG,
tiled=False,
compose=True,
executed_keys=None):
from .utils import build_fetch
executed_keys = executed_keys or []
if tiled and self.is_coarse():
self.tiles()
graph = graph if graph is not None else cls()
keys = None
if tiled:
nodes = list(c.data for c in self.chunks)
keys = list(c.key for c in self.chunks)
else:
nodes = list(self.op.outputs)
visited = set()
while len(nodes) > 0:
node = nodes.pop()
# replace executed tensor/chunk by tensor/chunk with fetch op
if node.key in executed_keys:
node = build_fetch(node, coarse=True).data
visited.add(node)
if not graph.contains(node):
graph.add_node(node)
children = node.inputs or []
for c in children:
if c.key in executed_keys:
continue
if not graph.contains(c):
graph.add_node(c)
if not graph.has_successor(c, node):
graph.add_edge(c, node)
nodes.extend([
c for c in itertools.chain(
*[inp.op.outputs for inp in node.inputs or []])
if c not in visited
])
if tiled and compose:
graph.compose(keys=keys)
if not tiled and any(not n.is_coarse() for n in graph):
return self._to_coarse_graph(graph)
return graph
@staticmethod
def _to_coarse_graph(graph):
new_graph = type(graph)()
visited = dict()
for n in graph:
if n not in visited:
new_node = n.to_coarse()
visited[n] = new_node
new_graph.add_node(new_node)
for succ in graph.successors(n):
if succ not in visited:
new_node = succ.to_coarse()
visited[succ] = new_node
new_graph.add_node(new_node)
new_graph.add_edge(visited[n], visited[succ])
return new_graph
def visualize(self, graph_attrs=None, node_attrs=None, **kw):
from graphviz import Source
g = self.build_graph(**kw)
dot = g.to_dot(graph_attrs=graph_attrs, node_attrs=node_attrs)
return Source(dot)
def execute(self, session=None, **kw):
from .session import Session
if session is None:
session = Session.default_or_local()
return session.run(self, **kw)
def fetch(self, session=None, **kw):
from .session import Session
if session is None:
session = Session.default_or_local()
return session.fetch(self, **kw)
def _set_execute_session(self, session):
_cleaner.register(self, session)
_execute_session = property(fset=_set_execute_session)
class Window(object):
supported_drag_types = [const.DOC_ID_LIST_PBOARD_TYPE, ak.NSFilenamesPboardType]
app = WeakProperty()
def __init__(self, app, state=None):
self.app = app
self._current_editor = None
self.wc = WindowController(self)
self.state = state
self.command = CommandBar(self, app.text_commander)
self.projects = KVOList()
self.recent = self._suspended_recent = RecentItemStack(100)
self._recent_history = None
self.window_settings_loaded = False
self.no_document_undo_manager = UndoManager()
self.menu = self.make_context_menu()
self.dirty_editors = WeakSet()
def window_did_load(self):
wc = self.wc
wc.docsView.default_menu = self.menu
wc.docsView.setRefusesFirstResponder_(True)
wc.docsView.registerForDraggedTypes_(self.supported_drag_types)
wc.plusButton.setRefusesFirstResponder_(True)
wc.plusButton.setImage_(load_image(const.PLUS_BUTTON_IMAGE))
wc.propsViewButton.setRefusesFirstResponder_(True)
wc.propsViewButton.setImage_(load_image(const.PROPS_DOWN_BUTTON_IMAGE))
wc.propsViewButton.setAlternateImage_(load_image(const.PROPS_UP_BUTTON_IMAGE))
self._setstate(self._state)
self._state = None
if not self.projects:
self.new_project()
def make_context_menu(self):
def has_path(item):
return item and item.file_path
return Menu([
MenuItem("Copy Path", self.copy_path, is_enabled=has_path),
MenuItem("Close", self.close_item, "Command+w"),
])
def _setstate(self, state):
if state:
projects = state.get("projects")
if projects is None:
projects = state.get("project_serials", []) # legacy
for serial in projects:
proj = Project(self, serial=serial)
self.projects.append(proj)
for proj_index, doc_index in state.get("recent_items", []):
if proj_index < len(self.projects):
proj = self.projects[proj_index]
if doc_index == "<project>":
self.recent.push(proj.id)
elif doc_index < len(proj.editors):
doc = proj.editors[doc_index]
self.recent.push(doc.id)
if 'window_settings' in state:
self.window_settings = state['window_settings']
with self.suspend_recent_updates():
pass # focus recent
def __getstate__(self):
if self._state is not None:
return self._state
def iter_settings():
indexes = {}
serials = []
for i, project in enumerate(self.projects):
serial = project.serialize()
if serial:
serials.append(serial)
indexes[project.id] = [i, "<project>"]
for j, doc in enumerate(project.editors):
indexes[doc.id] = [i, j]
yield "projects", serials
rits = []
for ident in self.recent:
pair = indexes.get(ident)
if pair is not None:
rits.append(pair)
yield "recent_items", rits
yield "window_settings", self.window_settings
return {key: val for key, val in iter_settings() if val}
def __setstate__(self, state):
assert not hasattr(self, '_state'), 'can only be called once'
self._state = state
state = property(__getstate__, __setstate__)
def discard(self, item):
ident = None if item is None else item.id
recent = self.recent
update_current = item in self.selected_items or not self.selected_items
with self.suspend_recent_updates(update_current):
for project in list(self.projects):
pid = project.id
for editor in list(project.editors):
did = editor.id
if ident in (pid, did):
recent.discard(did)
assert editor.project is project, (editor.project, project)
editor.close()
if ident == pid:
recent.discard(pid)
self.projects.remove(project)
project.close()
def focus(self, value, offset=1):
"""Change the current document by navigating the tree or recent documents
:param value: One of the direction constants in
`editxt.constants` or an editor's file path. `NEXT` and
`PREVIOUS` select items in the recent editors stack. `UP` and
`DOWN` move up or down in the tree.
:param offset: The number of positions to move in direction.
:returns: True if a new editor was focused, otherwise false.
"""
def focus(ident):
for project in self.projects:
if project.id == ident:
self.current_editor = project
return True
else:
for editor in project.editors:
if editor.id == ident:
self.current_editor = editor
return True
return False
def get_item_in_tree(current, offset):
if current is not None:
items = []
index = 0
stop = sys.maxsize
for project in self.projects:
items.append(project)
if current.id == project.id:
stop = index + offset
if stop <= index:
break
index += 1
if project.expanded:
for editor in project.editors:
items.append(editor)
if current.id == editor.id:
stop = index + offset
if stop <= index:
break
index += 1
if 0 <= stop < len(items):
return items[stop]
return None
if isinstance(value, const.Constant):
if value == const.PREVIOUS or value == const.NEXT:
history = ((list(reversed(self.recent)) + [0])
if self._recent_history is None
else self._recent_history)
if value == const.PREVIOUS:
offset = offset + history[-1]
else:
offset = history[-1] - offset
if 0 <= offset < len(history) - 1:
ok = focus(history[offset])
if ok:
history[-1] = offset
self._recent_history = history
return ok
return False
if value == const.UP:
offset = -offset
editor = get_item_in_tree(self.current_editor, offset)
if editor is not None:
self.current_editor = editor
return True
if isinstance(value, (Editor, Project)):
return focus(value.id)
return False
@contextmanager
def suspend_recent_updates(self, update_current=True):
self.recent = RecentItemStack(1)
try:
yield
finally:
self.recent = recent = self._suspended_recent
if not update_current:
return
lookup = {}
for project in self.projects:
lookup[project.id] = project
lookup.update((e.id, e) for e in project.editors)
current = self.current_editor
current_id = None if current is None else current.id
if current_id in lookup and recent and current_id == recent[-1]:
return
while True:
ident = recent.pop()
if ident is None:
if self.projects:
for project in self.projects:
if project.expanded and project.editors:
self.current_editor = project.editors[0]
break
else:
self.current_editor = self.projects[0]
break
item = lookup.get(ident)
if item is not None:
self.current_editor = item
break
def do_menu_command(self, sender):
self.app.text_commander.do_menu_command(self.current_editor, sender)
def validate_menu_command(self, item):
return self.app.text_commander.is_menu_command_enabled(self.current_editor, item)
@property
def current_editor(self):
return self._current_editor
@current_editor.setter
def current_editor(self, editor):
self._current_editor = editor
self._recent_history = None
if editor is None:
self.wc.setup_current_editor(None)
self.selected_items = []
return
if self.wc.is_current_view(editor.main_view):
editor.focus()
else:
self.recent.push(editor.id)
if self.wc.setup_current_editor(editor):
if isinstance(editor, Editor) \
and self.find_project_with_editor(editor) is None:
self.insert_items([editor])
if not self.selected_items or editor is not self.selected_items[0]:
self.selected_items = [editor]
@property
def selected_items(self):
return self.wc.selected_items
@selected_items.setter
def selected_items(self, value):
self.wc.selected_items = value
def selected_editor_changed(self):
selected = self.selected_items
if selected and selected[0] is not self.current_editor:
self.current_editor = selected[0]
def on_dirty_status_changed(self, editor, dirty):
if dirty:
self.dirty_editors.add(editor)
else:
self.dirty_editors.discard(editor)
self.wc.on_dirty_status_changed(editor, self.is_dirty)
@property
def is_dirty(self):
return bool(self.dirty_editors)
def iter_editors_of_document(self, doc):
for project in self.projects:
for editor in project.iter_editors_of_document(doc):
yield editor
def should_select_item(self, outlineview, item):
return True
def open_documents(self):
editor = self.current_editor
if editor is not None and editor.dirname():
directory = editor.dirname()
else:
directory = os.path.expanduser("~")
self.wc.open_documents(directory, None, self.open_paths)
def save_as(self):
self.save(prompt=True)
def save(self, prompt=False):
editor = self.current_editor
if isinstance(editor, Editor):
editor.save(prompt=prompt)
def reload_current_document(self):
editor = self.current_editor
if isinstance(editor, Editor):
editor.document.reload_document()
def save_document_as(self, editor, save_with_path):
"""Prompt for path to save document
:param editor: The editor of the document to be saved.
:param save_with_path: A callback accepting a sinlge parameter (the
chosen file path) that does the work of actually saving the file.
Call with ``None`` to cancel the save operation.
"""
directory, filename = self._directory_and_filename(editor.file_path)
self.wc.save_document_as(directory, filename, save_with_path)
def prompt_to_overwrite(self, editor, save_with_path):
"""Prompt to overwrite the given editor's document's file path
:param editor: The editor of the document to be saved.
:param save_with_path: A callback accepting a sinlge parameter (the
chosen file path) that does the work of actually saving the file.
Call with ``None`` to cancel the save operation.
"""
def save_as():
self.save_document_as(editor, save_with_path)
if editor is None:
diff_with_original = None
else:
def diff_with_original():
from editxt.command.diff import diff
from editxt.command.parser import Options
diff(editor, Options(file=editor.file_path))
self.wc.prompt_to_overwrite(
editor.file_path, save_with_path, save_as, diff_with_original)
def prompt_to_close(self, editor, save_discard_or_cancel, save_as=True):
"""Prompt to see if the document can be closed
:param editor: The editor of the document to be closed.
:param save_discard_or_cancel: A callback to be called with the outcome
of the prompt: save (True), discard (False), or cancel (None).
:param save_as: Boolean, if true prompt to "save as" (with dialog),
otherwise prompt to save (without dialog).
"""
self.current_editor = editor
self.wc.prompt_to_close(editor.file_path, save_discard_or_cancel, save_as)
@staticmethod
def _directory_and_filename(path):
if isabs(path):
directory, filename = split(path)
while directory and directory != sep and not isdir(directory):
directory = dirname(directory)
else:
directory = None
filename = basename(path)
assert filename, path
if not directory:
# TODO editor.project.path or path of most recent document
# None -> directory used in the previous invocation of the panel
directory = None
return directory, filename
def new_project(self):
project = Project(self)
editor = project.create_editor()
self.projects.append(project)
self.current_editor = editor
return project
def toggle_properties_pane(self):
tree_rect = self.wc.docsScrollview.frame()
prop_rect = self.wc.propsView.frame()
if self.wc.propsViewButton.state() == ak.NSOnState:
# hide properties view
tree_rect.size.height += prop_rect.size.height - 1.0
tree_rect.origin.y = prop_rect.origin.y
prop_rect.size.height = 0.0
else:
# show properties view
tree_rect.size.height -= 115.0
if prop_rect.size.height > 0:
tree_rect.size.height += (prop_rect.size.height - 1.0)
tree_rect.origin.y = prop_rect.origin.y + 115.0
prop_rect.size.height = 116.0
self.wc.propsView.setHidden_(False)
resize_tree = fn.NSDictionary.dictionaryWithObjectsAndKeys_(
self.wc.docsScrollview, ak.NSViewAnimationTargetKey,
fn.NSValue.valueWithRect_(tree_rect), ak.NSViewAnimationEndFrameKey,
None,
)
resize_props = fn.NSDictionary.dictionaryWithObjectsAndKeys_(
self.wc.propsView, ak.NSViewAnimationTargetKey,
fn.NSValue.valueWithRect_(prop_rect), ak.NSViewAnimationEndFrameKey,
None,
)
anims = fn.NSArray.arrayWithObjects_(resize_tree, resize_props, None)
animation = ak.NSViewAnimation.alloc().initWithViewAnimations_(anims)
#animation.setAnimationBlockingMode_(NSAnimationBlocking)
animation.setDuration_(0.25)
animation.startAnimation()
def find_project_with_editor(self, editor):
for proj in self.projects:
for e in proj.editors:
if editor is e:
return proj
return None
def find_project_with_path(self, path):
for proj in self.projects:
p = proj.file_path
if p and os.path.exists(p) and os.path.samefile(p, path):
return proj
return None
def get_current_project(self, create=False):
item = self.current_editor
if item is not None:
return item if isinstance(item, Project) else item.project
if self.projects:
for project in self.projects:
if project.expanded:
return project
return self.projects[0]
if create:
proj = Project(self)
self.projects.append(proj)
return proj
return None
def tooltip_for_item(self, view, item):
it = view.realItemForOpaqueItem_(item)
null = it is None or it.file_path is None
return None if null else user_path(it.file_path)
def should_edit_item(self, col, item):
if col.isEditable():
obj = representedObject(item)
return isinstance(obj, Project) and obj.can_rename()
return False
def copy_path(self, item):
"""Copy item path to pasteboard
Put newline-delimited paths on pasteboard if there are multiple
items selected and the given item is one of them.
"""
selected = self.selected_items
if item not in selected:
selected = [item]
Pasteboard().text = "\n".join(item.file_path for item in selected)
def close_item(self, item):
"""Close editor or project
Close all selected items if there are multiple items selected
and the given item is one of them.
"""
def do_close(should_close):
if should_close:
for item in selected:
self.discard(item)
selected = self.selected_items
if item not in selected:
selected = [item]
self.app.async_interactive_close(selected, do_close)
def window_did_become_key(self, window):
editor = self.current_editor
if isinstance(editor, Editor):
# TODO refactor Editor to support check_for_external_changes()
editor.document.check_for_external_changes(window)
def should_close(self, do_close):
"""Determine if the window should be closed
If this returns false an interactive close loop will be started, which
may eventually result in the window being closed.
"""
def iter_dirty_editors():
app = self.app
for proj in self.projects:
wins = app.find_windows_with_project(proj)
if wins == [self]:
for editor in proj.dirty_editors():
doc_windows = app.iter_windows_with_editor_of_document(
editor.document)
if all(win is self for win in doc_windows):
yield editor
if next(iter_dirty_editors(), None) is None:
return True
def callback(should_close):
if should_close:
do_close()
self.app.async_interactive_close(iter_dirty_editors(), callback)
return False
def window_will_close(self):
self.app.discard_window(self)
def _get_window_settings(self):
return dict(
frame_string=str(self.wc.frame_string),
splitter_pos=self.wc.splitter_pos,
properties_hidden=self.wc.properties_hidden,
)
def _set_window_settings(self, settings):
fs = settings.get("frame_string")
if fs is not None:
self.wc.frame_string = fs
sp = settings.get("splitter_pos")
if sp is not None:
self.wc.splitter_pos = sp
self.wc.properties_hidden = settings.get("properties_hidden", False)
self.window_settings_loaded = True
window_settings = property(_get_window_settings, _set_window_settings)
def close(self):
wc = self.wc
if wc is not None:
self.window_settings_loaded = False
while self.projects:
self.projects.pop().close()
#wc.docsController.setContent_(None)
#self.wc = None
# drag/drop logic ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
def is_project_drag(self, info):
"""Return True if only projects are being dropped else False"""
pb = info.draggingPasteboard()
t = pb.availableTypeFromArray_(self.supported_drag_types)
if t == const.DOC_ID_LIST_PBOARD_TYPE:
id_list = pb.propertyListForType_(const.DOC_ID_LIST_PBOARD_TYPE)
items = self.iter_dropped_id_list(id_list)
return all(isinstance(item, Project) for item in items)
elif t == ak.NSFilenamesPboardType:
paths = pb.propertyListForType_(ak.NSFilenamesPboardType)
return all(Project.is_project_path(path) for path in paths)
return False
def get_id_path_pairs(self, items):
"""Get a list of (<item id>, <item path>) pairs for given items
:param items: A list of editors and/or projects.
:returns: A list of two-tuples (<item id>, <item path>). <item id> is
an opaque internal identifier for the document, and <item path> is
the file system path of the item or ``None`` if the item does not have
a path.
"""
def pair(item):
path = item.file_path
return (item.id, path if path and os.path.exists(path) else None)
return [pair(item) for item in items]
def validate_drop(self, outline_view, info, item, index):
if self.is_project_drag(info):
if item is not None:
obj = representedObject(item)
path = self.wc.docsController.indexPathForObject_(obj)
if path is not None:
index = path.indexAtPosition_(0)
outline_view.setDropItem_dropChildIndex_(None, index)
else:
return ak.NSDragOperationNone
elif index < 0:
outline_view.setDropItem_dropChildIndex_(None, len(self.projects))
return ak.NSDragOperationMove
else:
# text document drag
if item is not None:
obj = representedObject(item)
if isinstance(obj, Project):
if index < 0:
#outline_view.setDropItem_dropChildIndex_(item, 0)
# the following might be more correct, but is too confusing
outline_view.setDropItem_dropChildIndex_(item, len(obj.editors))
else:
return ak.NSDragOperationNone # document view cannot have children
else:
if index < 0:
# drop on listview background
last_proj_index = len(self.projects) - 1
if last_proj_index > -1:
# we have at least one project
path = fn.NSIndexPath.indexPathWithIndex_(last_proj_index)
node = self.wc.docsController.nodeAtArrangedIndexPath_(path)
proj = representedObject(node)
outline_view.setDropItem_dropChildIndex_(node, len(proj.editors))
else:
outline_view.setDropItem_dropChildIndex_(None, -1)
elif index == 0:
return ak.NSDragOperationNone # prevent drop above top project
op = info.draggingSourceOperationMask()
if op not in [ak.NSDragOperationCopy, ak.NSDragOperationGeneric]:
op = ak.NSDragOperationMove
return op
def accept_drop(self, view, pasteboard, parent=const.CURRENT, index=-1, action=const.MOVE):
"""Accept drop operation
:param view: The view on which the drop occurred.
:param pasteboard: NSPasteboard object.
:param parent: The parent item in the outline view.
:param index: The index in the outline view or parent item at which the
drop occurred.
:param action: The action to perform when dragging (see
``insert_items(..., action)``). Ignored if the items being dropped are
paths.
:returns: True if the drop was accepted, otherwise False.
"""
pb = pasteboard
t = pb.availableTypeFromArray_(self.supported_drag_types)
if t == const.DOC_ID_LIST_PBOARD_TYPE:
id_list = pb.propertyListForType_(const.DOC_ID_LIST_PBOARD_TYPE)
items = self.iter_dropped_id_list(id_list)
elif t == ak.NSFilenamesPboardType:
paths = pb.propertyListForType_(ak.NSFilenamesPboardType)
items = self.iter_dropped_paths(paths)
action = None
else:
assert t is None, t
return False
return bool(self.insert_items(items, parent, index, action))
def iter_dropped_id_list(self, id_list):
"""Iterate TextDocument objects referenced by pasteboard (if any)"""
if not id_list:
return
for ident in id_list:
item = self.app.find_item_with_id(ident)
if item is not None:
yield item
def open_url(self, url, link, focus=True):
"""Open file specified by URL
The URL must have two attributes:
- path : The path to the file. The first leading slash is
stripped, so absolute paths must have an extra slash.
- query : A query string from which an optional "goto" parameter
may be parsed. The goto parameter specifies a line or line +
selection (`line.sel_start.sel_length`) to goto/select after
opening the file.
:param url: Parsed URL. See `urllib.parse.urlparse` for structure.
:param link: The original URL string.
:param focus: Focus newly opened editor.
"""
path = unquote(url.path)
if path.startswith("/"):
path = path[1:]
editors = self.open_paths([path], focus=focus)
if editors:
assert len(editors) == 1, (link, editors)
query = parse_qs(url.query)
if "goto" in query:
goto = query["goto"][0]
try:
if "." in goto:
line, start, end = goto.split(".")
num = (int(line), int(start), int(end))
else:
num = int(goto)
except ValueError:
log.debug("invalid goto: %r (link: %s)", goto, link)
else:
editors[0].goto_line(num)
def open_paths(self, paths, focus=True):
return self.insert_items(self.iter_dropped_paths(paths), focus=focus)
def iter_dropped_paths(self, paths):
if not paths:
return
for path in paths:
if path is None or os.path.isfile(path) or not os.path.exists(path):
yield self.app.document_with_path(path)
# elif os.path.isdir(path):
# yield Project(self, name=os.path.dirname(path))
else:
log.info("cannot open path: %s", path)
def insert_items(self, items, project=const.CURRENT, index=-1, action=None,
focus=True):
"""Insert items into the document tree
:param items: An iterable of projects, editors, and/or documents.
:param project: The parent project into which items are being inserted.
Documents will be inserted in the current project if unspecified.
:param index: The index in the outline view or parent project at which
the item(s) should be inserted. Add after current if < 0 (default).
:param action: What to do with items that are already open in
this window:
- None : insert new item(s), but do not change existing item(s).
- MOVE : move existing item(s) to index.
- COPY : copy item(s) to index.
A file is considered to be "existing" if there is an editor
with the same path in the project where it is being
inserted. A project is considered to be "existing" if there
is a project with the same path in the window where it is
being inserted.
:param focus: Focus most recent newly opened editor if true (the
default).
:returns: A list of editors and projects that were inserted.
"""
if (project is not None and
project != const.CURRENT and
project.window is not self):
raise ValueError("project does not belong to this window")
inserted = []
focus_editor = None
with self.suspend_recent_updates(update_current=False):
pindex = index
if pindex < 0:
pindex = len(self.projects)
for is_project_group, group in groupby(items, self.is_project):
if is_project_group:
for item in group:
project, pindex = self._insert_project(item, pindex, action)
if project is not None:
inserted.append(project)
focus_editor = project
# Reset index since the project into which non-project
# items will be inserted has changed.
index = -1
else:
if project == const.CURRENT or project is None:
if index >= 0:
raise NotImplementedError
project = self.get_current_project(create=True)
inserts, focus_editor = project.insert_items(group, index, action)
inserted.extend(inserts)
if focus and focus_editor is not None:
self.current_editor = focus_editor
return inserted
def is_project(self, item):
"""Return true if item can be inserted as a project"""
# TODO return true if item is a directory path
return isinstance(item, Project)
def _insert_project(self, item, index, action):
if action != const.MOVE:
raise NotImplementedError('cannot copy project yet')
if item.window is self:
window = self
pindex = self.projects.index(item)
if pindex == index:
return None, index
if pindex - index <= 0:
index -= 1
else:
window = item.window
# BEGIN HACK crash on remove project with editors
editors = item.editors
tmp, editors[:] = list(editors), []
window.projects.remove(item) # this line should be all that's necessary
editors.extend(tmp)
# END HACK
item.window = self
self.projects.insert(index, item)
return item, index + 1
def show(self, sender):
self.wc.showWindow_(sender)
@property
def undo_manager(self):
editor = self.current_editor
if editor is None:
return self.no_document_undo_manager
return editor.undo_manager
class TileableData(SerializableWithKey, Tileable):
__slots__ = '__weakref__', '_siblings', '_cix', '_entities'
_no_copy_attrs_ = SerializableWithKey._no_copy_attrs_ | {'_cix'}
# required fields
_op = KeyField('op')
# optional fields
# `nsplits` means the sizes of chunks for each dimension
_nsplits = TupleField('nsplits', ValueType.tuple(ValueType.uint64),
on_serialize=on_serialize_nsplits)
_extra_params = DictField('extra_params', key_type=ValueType.string, on_deserialize=AttributeDict)
def __init__(self, *args, **kwargs):
extras = AttributeDict((k, kwargs.pop(k)) for k in set(kwargs) - set(self.__slots__))
kwargs['_extra_params'] = kwargs.pop('_extra_params', extras)
if kwargs.get('_nsplits', None) is not None:
kwargs['_nsplits'] = tuple(tuple(s) for s in kwargs['_nsplits'])
super(TileableData, self).__init__(*args, **kwargs)
if hasattr(self, '_chunks') and self._chunks:
self._chunks = sorted(self._chunks, key=attrgetter('index'))
self._entities = WeakSet()
@property
def chunk_shape(self):
if hasattr(self, '_nsplits') and self._nsplits is not None:
return tuple(map(len, self._nsplits))
@property
def chunks(self):
return getattr(self, '_chunks', None)
@property
def op(self):
return getattr(self, '_op', None)
@property
def nsplits(self):
return getattr(self, '_nsplits', None)
@nsplits.setter
def nsplits(self, new_nsplits):
self._nsplits = new_nsplits
@property
def inputs(self):
return self.op.inputs or []
@inputs.setter
def inputs(self, new_inputs):
self.op.inputs = new_inputs
@property
def params(self):
# params return the properties which useful to rebuild a new tileable object
return dict()
@property
def extra_params(self):
return self._extra_params
@property
def cix(self):
if self.ndim == 0:
return ChunksIndexer(self)
try:
if getattr(self, '_cix', None) is None:
self._cix = ChunksIndexer(self)
return self._cix
except (TypeError, ValueError):
return ChunksIndexer(self)
@property
def entities(self):
return self._entities
def is_coarse(self):
return not hasattr(self, '_chunks') or self._chunks is None or len(self._chunks) == 0
def is_sparse(self):
return self.op.is_sparse()
issparse = is_sparse
@enter_build_mode
def attach(self, entity):
self._entities.add(entity)
@enter_build_mode
def detach(self, entity):
self._entities.discard(entity)
def execute(self, session=None, **kw):
from .session import Session
if session is None:
session = Session.default_or_local()
return session.run(self, **kw)
def fetch(self, session=None, **kw):
from .session import Session
if session is None:
session = Session.default_or_local()
return session.fetch(self, **kw)
def _set_execute_session(self, session):
_cleaner.register(self, session)
_execute_session = property(fset=_set_execute_session)
class TestWeakSet(unittest.TestCase):
def setUp(self):
self.items = [ustr(c) for c in ('a', 'b', 'c')]
self.items2 = [ustr(c) for c in ('x', 'y', 'z')]
self.ab_items = [ustr(c) for c in 'ab']
self.abcde_items = [ustr(c) for c in 'abcde']
self.def_items = [ustr(c) for c in 'def']
self.ab_weakset = WeakSet(self.ab_items)
self.abcde_weakset = WeakSet(self.abcde_items)
self.def_weakset = WeakSet(self.def_items)
self.letters = [ustr(c) for c in string.ascii_letters]
self.s = WeakSet(self.items)
self.d = dict.fromkeys(self.items)
self.obj = ustr('F')
self.fs = WeakSet([self.obj])
def test_methods(self):
weaksetmethods = dir(WeakSet)
for method in dir(set):
if method == 'test_c_api' or method.startswith('_'):
continue
self.assertIn(method, weaksetmethods,
'WeakSet missing method ' + method)
def test_new_or_init(self):
self.assertRaises(TypeError, WeakSet, [], 2)
def test_len(self):
self.assertEqual(len(self.s), len(self.d))
self.assertEqual(len(self.fs), 1)
del self.obj
self.assertEqual(len(self.fs), 0)
def test_contains(self):
for c in self.letters:
self.assertEqual(c in self.s, c in self.d)
self.assertNotIn(1, self.s)
self.assertIn(self.obj, self.fs)
del self.obj
self.assertNotIn(ustr('F'), self.fs)
def test_union(self):
u = self.s.union(self.items2)
for c in self.letters:
self.assertEqual(c in u, c in self.d or c in self.items2)
self.assertEqual(self.s, WeakSet(self.items))
self.assertEqual(type(u), WeakSet)
self.assertRaises(TypeError, self.s.union, [[]])
for C in (set, frozenset, dict.fromkeys, list, tuple):
x = WeakSet(self.items + self.items2)
c = C(self.items2)
self.assertEqual(self.s.union(c), x)
del c
self.assertEqual(len(u), len(self.items) + len(self.items2))
self.items2.pop()
gc.collect()
self.assertEqual(len(u), len(self.items) + len(self.items2))
def test_or(self):
i = self.s.union(self.items2)
self.assertEqual(self.s | set(self.items2), i)
self.assertEqual(self.s | frozenset(self.items2), i)
def test_intersection(self):
s = WeakSet(self.letters)
i = s.intersection(self.items2)
for c in self.letters:
self.assertEqual(c in i, c in self.items2 and c in self.letters)
self.assertEqual(s, WeakSet(self.letters))
self.assertEqual(type(i), WeakSet)
for C in (set, frozenset, dict.fromkeys, list, tuple):
x = WeakSet([])
self.assertEqual(i.intersection(C(self.items)), x)
self.assertEqual(len(i), len(self.items2))
self.items2.pop()
gc.collect()
self.assertEqual(len(i), len(self.items2))
def test_isdisjoint(self):
self.assertTrue(self.s.isdisjoint(WeakSet(self.items2)))
self.assertTrue(not self.s.isdisjoint(WeakSet(self.letters)))
def test_and(self):
i = self.s.intersection(self.items2)
self.assertEqual(self.s & set(self.items2), i)
self.assertEqual(self.s & frozenset(self.items2), i)
def test_difference(self):
i = self.s.difference(self.items2)
for c in self.letters:
self.assertEqual(c in i, c in self.d and c not in self.items2)
self.assertEqual(self.s, WeakSet(self.items))
self.assertEqual(type(i), WeakSet)
self.assertRaises(TypeError, self.s.difference, [[]])
def test_sub(self):
i = self.s.difference(self.items2)
self.assertEqual(self.s - set(self.items2), i)
self.assertEqual(self.s - frozenset(self.items2), i)
def test_symmetric_difference(self):
i = self.s.symmetric_difference(self.items2)
for c in self.letters:
self.assertEqual(c in i, (c in self.d) ^ (c in self.items2))
self.assertEqual(self.s, WeakSet(self.items))
self.assertEqual(type(i), WeakSet)
self.assertRaises(TypeError, self.s.symmetric_difference, [[]])
self.assertEqual(len(i), len(self.items) + len(self.items2))
self.items2.pop()
gc.collect()
self.assertEqual(len(i), len(self.items) + len(self.items2))
def test_xor(self):
i = self.s.symmetric_difference(self.items2)
self.assertEqual(self.s ^ set(self.items2), i)
self.assertEqual(self.s ^ frozenset(self.items2), i)
def test_sub_and_super(self):
self.assertTrue(self.ab_weakset <= self.abcde_weakset)
self.assertTrue(self.abcde_weakset <= self.abcde_weakset)
self.assertTrue(self.abcde_weakset >= self.ab_weakset)
self.assertFalse(self.abcde_weakset <= self.def_weakset)
self.assertFalse(self.abcde_weakset >= self.def_weakset)
self.assertTrue(set('a').issubset('abc'))
self.assertTrue(set('abc').issuperset('a'))
self.assertFalse(set('a').issubset('cbs'))
self.assertFalse(set('cbs').issuperset('a'))
def test_lt(self):
self.assertTrue(self.ab_weakset < self.abcde_weakset)
self.assertFalse(self.abcde_weakset < self.def_weakset)
self.assertFalse(self.ab_weakset < self.ab_weakset)
self.assertFalse(WeakSet() < WeakSet())
def test_gt(self):
self.assertTrue(self.abcde_weakset > self.ab_weakset)
self.assertFalse(self.abcde_weakset > self.def_weakset)
self.assertFalse(self.ab_weakset > self.ab_weakset)
self.assertFalse(WeakSet() > WeakSet())
def test_gc(self):
s = WeakSet(Foo() for i in range(1000))
for elem in s:
elem.cycle = s
elem.sub = elem
elem.set = WeakSet([elem])
def test_subclass_with_custom_hash(self):
class H(WeakSet):
def __hash__(self):
return int(id(self) & 2147483647)
s = H()
f = set()
f.add(s)
self.assertIn(s, f)
f.remove(s)
f.add(s)
f.discard(s)
def test_init(self):
s = WeakSet()
s.__init__(self.items)
self.assertEqual(s, self.s)
s.__init__(self.items2)
self.assertEqual(s, WeakSet(self.items2))
self.assertRaises(TypeError, s.__init__, s, 2)
self.assertRaises(TypeError, s.__init__, 1)
def test_constructor_identity(self):
s = WeakSet(self.items)
t = WeakSet(s)
self.assertNotEqual(id(s), id(t))
def test_hash(self):
self.assertRaises(TypeError, hash, self.s)
def test_clear(self):
self.s.clear()
self.assertEqual(self.s, WeakSet([]))
self.assertEqual(len(self.s), 0)
def test_copy(self):
dup = self.s.copy()
self.assertEqual(self.s, dup)
self.assertNotEqual(id(self.s), id(dup))
def test_add(self):
x = ustr('Q')
self.s.add(x)
self.assertIn(x, self.s)
dup = self.s.copy()
self.s.add(x)
self.assertEqual(self.s, dup)
self.assertRaises(TypeError, self.s.add, [])
self.fs.add(Foo())
self.assertTrue(len(self.fs) == 1)
self.fs.add(self.obj)
self.assertTrue(len(self.fs) == 1)
def test_remove(self):
x = ustr('a')
self.s.remove(x)
self.assertNotIn(x, self.s)
self.assertRaises(KeyError, self.s.remove, x)
self.assertRaises(TypeError, self.s.remove, [])
def test_discard(self):
a, q = ustr('a'), ustr('Q')
self.s.discard(a)
self.assertNotIn(a, self.s)
self.s.discard(q)
self.assertRaises(TypeError, self.s.discard, [])
def test_pop(self):
for i in range(len(self.s)):
elem = self.s.pop()
self.assertNotIn(elem, self.s)
self.assertRaises(KeyError, self.s.pop)
def test_update(self):
retval = self.s.update(self.items2)
self.assertEqual(retval, None)
for c in (self.items + self.items2):
self.assertIn(c, self.s)
self.assertRaises(TypeError, self.s.update, [[]])
def test_update_set(self):
self.s.update(set(self.items2))
for c in (self.items + self.items2):
self.assertIn(c, self.s)
def test_ior(self):
self.s |= set(self.items2)
for c in (self.items + self.items2):
self.assertIn(c, self.s)
def test_intersection_update(self):
retval = self.s.intersection_update(self.items2)
self.assertEqual(retval, None)
for c in (self.items + self.items2):
if c in self.items2 and c in self.items:
self.assertIn(c, self.s)
else:
self.assertNotIn(c, self.s)
self.assertRaises(TypeError, self.s.intersection_update, [[]])
def test_iand(self):
self.s &= set(self.items2)
for c in (self.items + self.items2):
if c in self.items2 and c in self.items:
self.assertIn(c, self.s)
else:
self.assertNotIn(c, self.s)
def test_difference_update(self):
retval = self.s.difference_update(self.items2)
self.assertEqual(retval, None)
for c in (self.items + self.items2):
if c in self.items and c not in self.items2:
self.assertIn(c, self.s)
else:
self.assertNotIn(c, self.s)
self.assertRaises(TypeError, self.s.difference_update, [[]])
self.assertRaises(TypeError, self.s.symmetric_difference_update, [[]])
def test_isub(self):
self.s -= set(self.items2)
for c in (self.items + self.items2):
if c in self.items and c not in self.items2:
self.assertIn(c, self.s)
else:
self.assertNotIn(c, self.s)
def test_symmetric_difference_update(self):
retval = self.s.symmetric_difference_update(self.items2)
self.assertEqual(retval, None)
for c in (self.items + self.items2):
if (c in self.items) ^ (c in self.items2):
self.assertIn(c, self.s)
else:
self.assertNotIn(c, self.s)
self.assertRaises(TypeError, self.s.symmetric_difference_update, [[]])
def test_ixor(self):
self.s ^= set(self.items2)
for c in (self.items + self.items2):
if (c in self.items) ^ (c in self.items2):
self.assertIn(c, self.s)
else:
self.assertNotIn(c, self.s)
def test_inplace_on_self(self):
t = self.s.copy()
t |= t
self.assertEqual(t, self.s)
t &= t
self.assertEqual(t, self.s)
t -= t
self.assertEqual(t, WeakSet())
t = self.s.copy()
t ^= t
self.assertEqual(t, WeakSet())
def test_eq(self):
self.assertTrue(self.s == self.s)
self.assertTrue(self.s == WeakSet(self.items))
self.assertFalse(self.s == set(self.items))
self.assertFalse(self.s == list(self.items))
self.assertFalse(self.s == tuple(self.items))
self.assertFalse(self.s == WeakSet([Foo]))
self.assertFalse(self.s == 1)
def test_ne(self):
self.assertTrue(self.s != set(self.items))
s1 = WeakSet()
s2 = WeakSet()
self.assertFalse(s1 != s2)
def test_weak_destroy_while_iterating(self):
items = [ustr(c) for c in ('a', 'b', 'c')]
s = WeakSet(items)
it = iter(s)
next(it)
del items[-1]
gc.collect()
self.assertIn(len(list(it)), [len(items), len(items) - 1])
del it
self.assertEqual(len(s), len(items))
def test_weak_destroy_and_mutate_while_iterating(self):
items = [ustr(c) for c in string.ascii_letters]
s = WeakSet(items)
@contextlib.contextmanager
def testcontext():
try:
it = iter(s)
yielded = ustr(str(next(it)))
u = ustr(str(items.pop()))
if yielded == u:
next(it)
gc.collect()
yield u
finally:
it = None
with testcontext() as u:
self.assertNotIn(u, s)
with testcontext() as u:
self.assertRaises(KeyError, s.remove, u)
self.assertNotIn(u, s)
with testcontext() as u:
s.add(u)
self.assertIn(u, s)
t = s.copy()
with testcontext() as u:
s.update(t)
self.assertEqual(len(s), len(t))
with testcontext() as u:
s.clear()
self.assertEqual(len(s), 0)
def test_len_cycles(self):
N = 20
items = [RefCycle() for i in range(N)]
s = WeakSet(items)
del items
it = iter(s)
try:
next(it)
except StopIteration:
pass
gc.collect()
n1 = len(s)
del it
gc.collect()
n2 = len(s)
self.assertIn(n1, (0, 1))
self.assertEqual(n2, 0)
def test_len_race(self):
self.addCleanup(gc.set_threshold, *gc.get_threshold())
for th in range(1, 100):
N = 20
gc.collect(0)
gc.set_threshold(th, th, th)
items = [RefCycle() for i in range(N)]
s = WeakSet(items)
del items
it = iter(s)
try:
next(it)
except StopIteration:
pass
n1 = len(s)
del it
n2 = len(s)
self.assertGreaterEqual(n1, 0)
self.assertLessEqual(n1, N)
self.assertGreaterEqual(n2, 0)
self.assertLessEqual(n2, n1)
class Manager:
def __init__(self):
self.listeners = {}
self.cell_aliases = {}
self.cell_rev_aliases = {}
self.macro_listeners = {}
self.observers = {}
self.registrar_listeners = WeakKeyDictionary()
self.rev_registrar_listeners = WeakKeyDictionary()
self.pin_to_cells = {}
self.cells = WeakValueDictionary()
self.cell_to_output_pin = WeakKeyDictionary()
self._childids = WeakValueDictionary()
self.registrar_items = []
self.unstable_workers = WeakSet()
super().__init__()
def set_stable(self, worker, value):
assert value in (True, False), value
if not value:
#print("UNSTABLE", worker)
self.unstable_workers.add(worker)
else:
#print("STABLE", worker)
self.unstable_workers.discard(worker)
def add_cell_alias(self, source, target):
from .cell import Cell
assert isinstance(source, Cell)
assert isinstance(target, Cell)
assert source is not target
cell_id = self.get_cell_id(source)
target_ref = weakref.ref(target)
try:
aliases = self.cell_aliases[cell_id]
if target_ref not in aliases:
aliases.append(target_ref)
except KeyError:
self.cell_aliases[cell_id] = [target_ref]
if cell_id not in self.cells:
self.cells[cell_id] = source
#reverse alias
cell_id = self.get_cell_id(target)
source_ref = weakref.ref(source)
try:
rev_aliases = self.cell_rev_aliases[cell_id]
if source_ref not in rev_aliases:
rev_aliases.append(source_ref)
except KeyError:
self.cell_rev_aliases[cell_id] = [source_ref]
if cell_id not in self.cells:
self.cells[cell_id] = target
def add_registrar_item(self, registrar_name, dtype, data, data_name):
item = registrar_name, dtype, data, data_name
for curr_item in self.registrar_items:
if data_name is None:
exists = (curr_item[:3] == item[:3])
else:
exists = (curr_item[:2] == item[:2]) and \
curr_item[3] == data_name
if exists:
raise ValueError("Registrar item already exists")
self.registrar_items.append(item)
def remove_registrar_item(self, registrar_name, dtype, data, data_name):
item = registrar_name, dtype, data, data_name
self.registrar_items.remove(item)
def add_listener(self, cell, input_pin):
cell_id = self.get_cell_id(cell)
pin_ref = weakref.ref(input_pin)
try:
listeners = self.listeners[cell_id]
assert pin_ref not in listeners
# TODO: tolerate (silently ignore) a connection that exists already?
listeners.append(pin_ref)
except KeyError:
self.listeners[cell_id] = [pin_ref]
try:
curr_pin_to_cells = self.pin_to_cells[input_pin.get_pin_id()]
assert cell_id not in curr_pin_to_cells
# TODO: tolerate (append) multiple inputs?
curr_pin_to_cells.append(cell_id)
except KeyError:
self.pin_to_cells[input_pin.get_pin_id()] = [cell_id]
if cell_id not in self.cells:
self.cells[cell_id] = cell
def _remove_listener(self, cell_id, input_pin, worker):
input_pin_id = input_pin.get_pin_id()
l = self.listeners[cell_id]
l[:] = [ref for ref in l if ref().get_pin_id() != input_pin_id]
if not len(l):
self.listeners.pop(cell_id)
cell = self.cells.get(cell_id, None)
if cell is not None:
cell._on_disconnect(input_pin, worker, False)
def remove_listener(self, cell, input_pin):
worker = input_pin.worker_ref()
input_pin_id = input_pin.get_pin_id()
cell_ids = self.pin_to_cells.pop(input_pin_id, [])
cell_id = self.get_cell_id(cell)
self._remove_listener(cell_id, input_pin, worker)
def remove_listeners_pin(self, input_pin):
worker = input_pin.worker_ref()
cell_ids = self.pin_to_cells.pop(input_pin.get_pin_id(), [])
for cell_id in cell_ids:
self._remove_listener(cell_id, input_pin, worker)
def remove_aliases(self, cell):
cell_id = self.get_cell_id(cell)
cell_ref = weakref.ref(cell)
targets = self.cell_aliases.pop(cell_id, [])
for target_ref in targets:
target = target_ref()
if target is None:
continue
target._on_disconnect(cell, None, incoming=True)
target_id = self.get_cell_id(target)
r = self.cell_rev_aliases[target_id]
r[:] = [rr for rr in r if rr is not cell_ref]
if not len(r):
self.cell_rev_aliases.pop(target_id)
#rev_aliases
targets = self.cell_rev_aliases.pop(cell_id, [])
for target_ref in targets:
target = target_ref()
if target is None:
continue
target_id = self.get_cell_id(target)
r = self.cell_aliases[target_id]
r[:] = [rr for rr in r if rr is not cell_ref]
if not len(r):
self.cell_aliases.pop(target_id)
def remove_listeners_cell(self, cell):
cell_id = self.get_cell_id(cell)
listeners = self.listeners.pop(cell_id, [])
for listener in listeners:
pin = listener()
if pin is None:
continue
pin_id = pin.get_pin_id()
if pin_id not in self.pin_to_cells:
continue
self.pin_to_cells[pin_id][:] = \
[c for c in self.pin_to_cells[pin_id] if c != cell_id ]
def add_macro_listener(self, cell, macro_object, macro_arg):
cell_id = self.get_cell_id(cell)
m = (macro_object, macro_arg)
try:
macro_listeners = self.macro_listeners[cell_id]
assert m not in macro_listeners
macro_listeners.append(m)
except KeyError:
self.macro_listeners[cell_id] = [m]
if cell_id not in self.cells:
self.cells[cell_id] = cell
def remove_macro_listener(self, cell, macro_object, macro_arg):
cell_id = self.get_cell_id(cell)
m = (macro_object, macro_arg)
if cell_id in self.macro_listeners:
l = self.macro_listeners[cell_id]
if m in l:
l.remove(m)
def remove_macro_listeners_cell(self, cell):
cell_id = self.get_cell_id(cell)
listeners = self.macro_listeners.pop(cell_id, [])
def add_registrar_listener(self, registrar, key, target, namespace_name):
if registrar not in self.registrar_listeners:
self.registrar_listeners[registrar] = {}
d = self.registrar_listeners[registrar]
if key not in d:
d[key] = []
d[key].append((weakref.ref(target), namespace_name))
if target not in self.rev_registrar_listeners:
self.rev_registrar_listeners[target] = {}
r = self.rev_registrar_listeners[target]
if key not in r:
r[key] = []
r[key].append(weakref.ref(registrar))
def remove_registrar_listeners(self, target):
if target not in self.rev_registrar_listeners:
return
rev = self.rev_registrar_listeners.pop(target)
for key in rev:
for registrar_ref in rev[key]:
registrar = registrar_ref()
if registrar not in self.registrar_listeners:
continue
r = self.registrar_listeners[registrar]
t = r[key]
t[:] = [tt for tt in t if tt[0]() is not None and tt[0]() is not target]
if not len(t):
r.pop(key)
if not len(r):
self.registrar_listeners.pop(registrar)
def add_observer(self, cell, observer):
cell_id = self.get_cell_id(cell)
obs_ref = weakref.ref(observer)
try:
observers = self.observers[cell_id]
assert obs_ref not in observers
observers.append(obs_ref)
except KeyError:
self.observers[cell_id] = [obs_ref]
if cell_id not in self.cells:
self.cells[cell_id] = cell
def remove_observer(self, cell, observer):
cell_id = self.get_cell_id(cell)
obs_ref = weakref.ref(observer)
if cell_id in self.observers:
l = self.observers[cell_id]
if obs_ref in l:
l.remove(obs_ref)
def remove_observers_cell(self, cell):
cell_id = self.get_cell_id(cell)
listeners = self.observers.pop(cell_id, [])
def _update(self, cell, dtype, value, *,
worker=None, only_last=False):
import threading
assert threading.current_thread() is threading.main_thread()
from .cell import Signal
cell_id = self.get_cell_id(cell)
observers = self.observers.get(cell_id, [])
for observer in observers:
obs = observer()
if obs is not None:
obs(value)
macro_listeners = self.macro_listeners.get(cell_id, [])
if not only_last:
for macro_object, macro_arg in macro_listeners:
try:
updated = macro_object.update_cell(macro_arg)
except Exception:
#TODO: proper logging
import traceback
traceback.print_exc()
aliases = self.cell_aliases.get(cell_id, [])
for target_cell_ref in aliases:
target_cell = target_cell_ref()
if target_cell is not None:
if isinstance(target_cell, Signal):
#print("cell-cell alias", cell, "=>", target_cell)
self._update(target_cell, None, None,
worker=worker, only_last=only_last)
else:
value2 = value
if dtype is not None and \
(dtype == "cson" or dtype[0] == "cson") and \
target_cell.dtype is not None and \
(target_cell.dtype == "json" or target_cell.dtype[0] == "json"):
if isinstance(value, (str, bytes)):
value2 = cson2json(value)
target_cell._update(value2, propagate=True)
listeners = self.listeners.get(cell_id, [])
if only_last:
listeners = listeners[-1:]
resource_name0 = None
if cell.resource is not None:
resource_name0 = cell.resource.filepath
for input_pin_ref in listeners:
input_pin = input_pin_ref()
if input_pin is None:
continue #TODO: error?
if worker is not None and input_pin.worker_ref() is worker:
continue
value2 = value
if dtype is not None and \
(dtype == "cson" or dtype[0] == "cson") and \
input_pin.dtype is not None and \
(input_pin.dtype == "json" or input_pin.dtype[0] == "json"):
if isinstance(value, (str, bytes)):
value2 = cson2json(value)
resource_name = "pin: " + str(input_pin)
if resource_name0 is not None:
resource_name = resource_name0 + " in " + resource_name
input_pin.receive_update(value2, resource_name)
def update_from_code(self, cell, only_last=False):
import seamless
if cell.dtype == "array":
value = TransportedArray(cell._data, cell._store)
else:
value = cell._data
#print("manager.update_from_code", cell, head(value))
self._update(cell, cell.dtype, value, only_last=only_last)
from .. import run_work
from .macro import get_macro_mode
if not get_macro_mode():
run_work()
def update_from_worker(self, cell_id, value, worker, *, preliminary):
import seamless
from .cell import Signal
cell = self.cells.get(cell_id, None)
if cell is None or cell._destroyed:
return #cell has died...
#print("manager.update_from_worker", cell, head(value), worker)
if isinstance(cell, Signal):
assert value is None
self._update(cell, None, None, worker=worker)
else:
changed = cell._update(value,propagate=False,
preliminary=preliminary)
if changed:
if cell.dtype == "array":
value = TransportedArray(value, cell._store)
self._update(cell, cell.dtype, value, worker=worker)
def update_registrar_key(self, registrar, key):
from .worker import Worker
from .macro import MacroObject
if registrar not in self.registrar_listeners:
return
d = self.registrar_listeners[registrar]
if key not in d:
return
for t in list(d[key]):
target = t[0]()
if target is None:
continue
if isinstance(target, Worker):
namespace_name = t[1]
target.receive_registrar_update(registrar.name, key, namespace_name)
elif isinstance(target, MacroObject):
target.update_cell((registrar.name, key))
else:
raise TypeError(target)
@classmethod
def get_cell_id(cls, cell):
return id(cell)
def disconnect(self, source, target):
from .transformer import Transformer
from .cell import Cell, CellLike
from .context import Context
from .worker import EditPinBase, ExportedEditPin, \
InputPinBase, ExportedInputPin, OutputPinBase, ExportedOutputPin
if isinstance(source, EditPinBase):
source, target = target, source
if isinstance(source, CellLike) and source._like_cell:
if isinstance(target, ExportedInputPin):
target = target.get_pin()
if isinstance(source, Context):
assert "_output" in source._pins
source = source._pins["_output"]
self.remove_listener(source, target)
worker = target.worker_ref()
if worker is not None:
source._on_disconnect(target, worker, incoming = False)
elif isinstance(source, OutputPinBase):
if isinstance(target, Context):
assert "_input" in target._pins
target = target._pins["_input"]
if isinstance(source, ExportedOutputPin):
source = source.get_pin()
cell_id = self.get_cell_id(target)
ok = False
if cell_id in self.cells and \
target in self.cell_to_output_pin:
if cell_id not in source._cell_ids:
ok = False
else:
for ref in self.cell_to_output_pin[target]:
if ref() is source:
self.cell_to_output_pin[target].remove(ref)
source._cell_ids.remove(cell_id)
ok = True
if not ok:
raise ValueError("Connection does not exist")
if target not in self.cell_to_output_pin:
self.cell_to_output_pin[target] = []
self.cell_to_output_pin[target].append(weakref.ref(source))
worker = source.worker_ref()
if worker is not None:
if isinstance(worker, Transformer):
worker._on_disconnect_output()
target._on_disconnect(source, worker, incoming = True)
else:
raise TypeError
def connect(self, source, target):
from .transformer import Transformer
from .cell import Cell, CellLike
from .context import Context
from .worker import EditPinBase, ExportedEditPin, \
InputPinBase, ExportedInputPin, OutputPinBase, ExportedOutputPin
if isinstance(source, EditPinBase):
source, target = target, source
if isinstance(source, CellLike) and source._like_cell:
assert isinstance(target, (InputPinBase, EditPinBase, CellLike))
assert source._get_manager() is self
assert target._get_manager() is self
if isinstance(target, ExportedInputPin):
target = target.get_pin()
if isinstance(target, Cell):
self.add_cell_alias(source, target)
target._on_connect(source, None, incoming = True)
if source._status == Cell.StatusFlags.OK:
value = source._data
if source.dtype is not None and \
(source.dtype == "cson" or source.dtype[0] == "cson") and \
target.dtype is not None and \
(target.dtype == "json" or target.dtype[0] == "json"):
if isinstance(value, (str, bytes)):
value = cson2json(value)
target._update(value,propagate=True)
return
assert not isinstance(target, Context) #TODO?
worker = target.worker_ref()
assert worker is not None #weakref may not be dead
source._on_connect(target, worker, incoming = False)
self.add_listener(source, target)
if source._status == Cell.StatusFlags.OK:
self.update_from_code(source, only_last=True)
else:
if isinstance(target, EditPinBase) and target.last_value is not None:
self.update_from_worker(
self.get_cell_id(source),
target.last_value,
worker, preliminary=False
)
elif isinstance(source, OutputPinBase):
assert isinstance(target, CellLike) and target._like_cell
if isinstance(target, Context):
assert "_input" in target._pins
target = target._pins["_input"]
if isinstance(source, ExportedOutputPin):
source = source.get_pin()
worker = source.worker_ref()
assert worker is not None #weakref may not be dead
target._on_connect(source, worker, incoming = True)
cell_id = self.get_cell_id(target)
if cell_id not in self.cells:
self.cells[cell_id] = target
if cell_id not in source._cell_ids:
source._cell_ids.append(cell_id)
if target not in self.cell_to_output_pin:
self.cell_to_output_pin[target] = []
self.cell_to_output_pin[target].append(weakref.ref(source))
if isinstance(worker, Transformer):
worker._on_connect_output()
elif source.last_value is not None:
self.update_from_worker(
cell_id,
source.last_value,
worker,
preliminary=False
)
else:
raise TypeError
class MountManager:
_running = False
_last_run = None
_stop = False
_mounting = False
def __init__(self, latency):
self.latency = latency
self.mounts = WeakKeyDictionary()
self.contexts = WeakSet()
self.lock = RLock()
self.cell_updates = deque()
self._tick = Event()
self.stash = None
self.paths = WeakKeyDictionary()
@property
def reorganizing(self):
return self.stash is not None
@contextmanager
def reorganize(self, context):
if context is None:
self.stash = NoStash
yield
self.stash = None
return
if self.stash is not None:
assert context._part_of2(self.stash.context)
yield
return
with self.lock:
self.stash = MountManagerStash(self, context)
try:
self.stash.activate()
yield
#print("reorganize success")
self.stash.join()
except Exception as e:
#print("reorganize failure")
self.stash.undo()
raise e
finally:
self.stash = None
def add_mount(self, cell, path, mode, authority, persistent, **kwargs):
root = cell._root()
if root not in self.paths:
paths = set()
self.paths[root] = paths
else:
paths = self.paths[root]
assert path not in paths, path
#print("add mount", path, cell)
paths.add(path)
self.mounts[cell] = MountItem(self, cell, path, mode, authority, persistent, **kwargs)
if self.stash is None or self.stash is NoStash:
try:
self._mounting = True
self.mounts[cell].init()
finally:
self._mounting = False
def add_link(self, link, path, persistent):
paths = self.paths[link._root()]
assert path not in paths, path
#print("add link", path, link)
paths.add(path)
self.mounts[link] = LinkItem(link, path, persistent)
if self.stash is None or self.stash is NoStash:
self.mounts[link].init()
def unmount(self, cell_or_link, from_del=False):
#print("UNMOUNT", cell_or_link, cell_or_link._mount)
assert not is_dummy_mount(cell_or_link._mount), cell_or_link
root = cell_or_link._root()
if from_del and (cell_or_link not in self.mounts or root not in self.paths):
return
paths = self.paths[root]
path = cell_or_link._mount["path"]
assert path in paths
paths.remove(path)
assert cell_or_link in self.mounts, (cell_or_link, path) #... but path is in paths
mountitem = self.mounts.pop(cell_or_link)
mountitem.destroy()
def unmount_context(self, context, from_del=False):
#print("unmount context", context)
self.contexts.discard(context) # may or may not exist, especially at __del__ time
mount = context._mount
"""context._mount is authoritative!
If context is destroyed while an unmount is undesired,
(because of stash replacement)
context._mount MUST have been set to None!
"""
if context._root() is context:
self.paths.pop(context, None)
if mount is None:
return
assert not is_dummy_mount(mount), context
try:
paths = self.paths[context._root()]
except KeyError:
return
try:
paths.remove(mount["path"])
except KeyError:
pass
if mount["persistent"] == False:
dirpath = mount["path"].replace("/", os.sep)
try:
#print("rmdir", dirpath)
os.rmdir(dirpath)
except:
print("Error: cannot remove directory %s" % dirpath)
def add_context(self, context, path, as_parent):
#print("add context", path, context, as_parent, context._mount["persistent"])
paths = self.paths[context._root()]
if not as_parent:
assert path not in paths, path
paths.add(path)
self.contexts.add(context)
else:
if path in paths:
assert context in self.contexts, (path, context)
if self.stash is None or self.stash is NoStash:
self._check_context(context, as_parent)
else:
self.stash.context_as_parent[context] = as_parent
def _check_context(self, context, as_parent):
mount = context._mount
assert not is_dummy_mount(mount), context
dirpath = mount["path"].replace("/", os.sep)
persistent, authority = mount["persistent"], mount["authority"]
if os.path.exists(dirpath):
if authority == "cell" and not as_parent:
print("Warning: Directory path '%s' already exists" % dirpath) #TODO: log warning
else:
if authority == "file-strict":
raise Exception("Directory path '%s' does not exist, but authority is 'file-strict'" % dirpath)
os.mkdir(dirpath)
def add_cell_update(self, cell):
#print("add_cell_update", cell, self.reorganizing, self.mounting)
if self.reorganizing or self._mounting:
return
assert cell in self.mounts, (cell, hex(id(cell)))
self.cell_updates.append(cell)
def _run(self):
for cell, mount_item in list(self.mounts.items()):
if isinstance(cell, Link):
continue
if cell in self.cell_updates:
continue
try:
mount_item.conditional_read()
except Exception:
exc = traceback.format_exc()
if exc != mount_item.last_exc:
print(exc)
mount_item.last_exc = exc
while 1:
try:
cell = self.cell_updates.popleft()
except IndexError:
break
mount_item = self.mounts.get(cell)
if mount_item is None: #cell was deleted
continue
try:
mount_item.conditional_write(with_none=True)
except Exception:
exc = traceback.format_exc()
if exc != mount_item.last_exc:
print(exc)
mount_item.last_exc = exc
self._tick.set()
def run(self):
try:
self._running = True
while not self._stop:
t = time.time()
self._run()
while time.time() - t < self.latency:
if not self._tick.is_set():
break
time.sleep(0.01)
finally:
self._running = False
def start(self):
self._stop = False
t = self.thread = Thread(target=self.run)
t.setDaemon(True)
t.start()
def stop(self, wait=False, waiting_loop_period=0.01):
self._stop = True
if wait:
while self._running:
time.sleep(waiting_loop_period)
def tick(self):
"""Waits until one iteration of the run() loop has finished"""
if self._running:
self._tick.clear()
self._tick.wait()
def destroy(self):
for path in list(self.mounts.keys()):
self.unmount(path)
for context in sorted(self.contexts,key=lambda l:-len(l.path)):
self.unmount_context(context)
class Context(ContextBase, AttributeSetter):
"""Create a zmq Context
A zmq Context creates sockets via its ``ctx.socket`` method.
"""
sockopts = None
_instance = None
_instance_lock = Lock()
_instance_pid = None
_shadow = False
_sockets = None
def __init__(self, io_threads=1, **kwargs):
super(Context, self).__init__(io_threads=io_threads, **kwargs)
if kwargs.get('shadow', False):
self._shadow = True
else:
self._shadow = False
self.sockopts = {}
self._sockets = WeakSet()
def __del__(self):
"""deleting a Context should terminate it, without trying non-threadsafe destroy"""
# Calling locals() here conceals issue #1167 on Windows CPython 3.5.4.
locals()
if not self._shadow and not _exiting:
self.term()
def __enter__(self):
return self
def __exit__(self, *args, **kwargs):
self.term()
def __copy__(self, memo=None):
"""Copying a Context creates a shadow copy"""
return self.__class__.shadow(self.underlying)
__deepcopy__ = __copy__
@classmethod
def shadow(cls, address):
"""Shadow an existing libzmq context
address is the integer address of the libzmq context
or an FFI pointer to it.
.. versionadded:: 14.1
"""
from zmq.utils.interop import cast_int_addr
address = cast_int_addr(address)
return cls(shadow=address)
@classmethod
def shadow_pyczmq(cls, ctx):
"""Shadow an existing pyczmq context
ctx is the FFI `zctx_t *` pointer
.. versionadded:: 14.1
"""
from pyczmq import zctx # type: ignore
from zmq.utils.interop import cast_int_addr
underlying = zctx.underlying(ctx)
address = cast_int_addr(underlying)
return cls(shadow=address)
# static method copied from tornado IOLoop.instance
@classmethod
def instance(cls: Type[T], io_threads=1) -> T:
"""Returns a global Context instance.
Most single-threaded applications have a single, global Context.
Use this method instead of passing around Context instances
throughout your code.
A common pattern for classes that depend on Contexts is to use
a default argument to enable programs with multiple Contexts
but not require the argument for simpler applications::
class MyClass(object):
def __init__(self, context=None):
self.context = context or Context.instance()
.. versionchanged:: 18.1
When called in a subprocess after forking,
a new global instance is created instead of inheriting
a Context that won't work from the parent process.
"""
if (cls._instance is None or cls._instance_pid != os.getpid()
or cls._instance.closed):
with cls._instance_lock:
if (cls._instance is None or cls._instance_pid != os.getpid()
or cls._instance.closed):
cls._instance = cls(io_threads=io_threads)
cls._instance_pid = os.getpid()
return cls._instance
def term(self):
"""Close or terminate the context.
Context termination is performed in the following steps:
- Any blocking operations currently in progress on sockets open within context shall
raise :class:`zmq.ContextTerminated`.
With the exception of socket.close(), any further operations on sockets open within this context
shall raise :class:`zmq.ContextTerminated`.
- After interrupting all blocking calls, term shall block until the following conditions are satisfied:
- All sockets open within context have been closed.
- For each socket within context, all messages sent on the socket have either been
physically transferred to a network peer,
or the socket's linger period set with the zmq.LINGER socket option has expired.
For further details regarding socket linger behaviour refer to libzmq documentation for ZMQ_LINGER.
This can be called to close the context by hand. If this is not called,
the context will automatically be closed when it is garbage collected.
"""
return super(Context, self).term()
# -------------------------------------------------------------------------
# Hooks for ctxopt completion
# -------------------------------------------------------------------------
def __dir__(self):
keys = dir(self.__class__)
for collection in (ctx_opt_names, ):
keys.extend(collection)
return keys
# -------------------------------------------------------------------------
# Creating Sockets
# -------------------------------------------------------------------------
def _add_socket(self, socket):
self._sockets.add(socket)
def _rm_socket(self, socket):
if self._sockets:
self._sockets.discard(socket)
def destroy(self, linger=None):
"""Close all sockets associated with this context and then terminate
the context.
.. warning::
destroy involves calling ``zmq_close()``, which is **NOT** threadsafe.
If there are active sockets in other threads, this must not be called.
Parameters
----------
linger : int, optional
If specified, set LINGER on sockets prior to closing them.
"""
if self.closed:
return
sockets = self._sockets
self._sockets = WeakSet()
for s in sockets:
if s and not s.closed:
if linger is not None:
s.setsockopt(LINGER, linger)
s.close()
self.term()
@property
def _socket_class(self):
return Socket
def socket(self, socket_type, **kwargs):
"""Create a Socket associated with this Context.
Parameters
----------
socket_type : int
The socket type, which can be any of the 0MQ socket types:
REQ, REP, PUB, SUB, PAIR, DEALER, ROUTER, PULL, PUSH, etc.
kwargs:
will be passed to the __init__ method of the socket class.
"""
if self.closed:
raise ZMQError(ENOTSUP)
s = self._socket_class(self, socket_type, **kwargs)
for opt, value in self.sockopts.items():
try:
s.setsockopt(opt, value)
except ZMQError:
# ignore ZMQErrors, which are likely for socket options
# that do not apply to a particular socket type, e.g.
# SUBSCRIBE for non-SUB sockets.
pass
self._add_socket(s)
return s
def setsockopt(self, opt, value):
"""set default socket options for new sockets created by this Context
.. versionadded:: 13.0
"""
self.sockopts[opt] = value
def getsockopt(self, opt):
"""get default socket options for new sockets created by this Context
.. versionadded:: 13.0
"""
return self.sockopts[opt]
def _set_attr_opt(self, name, opt, value):
"""set default sockopts as attributes"""
if name in constants.ctx_opt_names:
return self.set(opt, value)
else:
self.sockopts[opt] = value
def _get_attr_opt(self, name, opt):
"""get default sockopts as attributes"""
if name in constants.ctx_opt_names:
return self.get(opt)
else:
if opt not in self.sockopts:
raise AttributeError(name)
else:
return self.sockopts[opt]
def __delattr__(self, key):
"""delete default sockopts as attributes"""
key = key.upper()
try:
opt = getattr(constants, key)
except AttributeError:
raise AttributeError("no such socket option: %s" % key)
else:
if opt not in self.sockopts:
raise AttributeError(key)
else:
del self.sockopts[opt]
class Character(PyfaBase, EveItemWrapper):
"""
"Core" character class. It should be used for managing characters
(e.g. in character editor). Fits will use proxy twin of the character.
We cannot use core because different fits carry different attributes
on character and all child entities (like skills, on-character
implants, and so on).
Pyfa model children:
.RestrictedSet(skills)
"""
__tablename__ = 'characters'
id = Column('character_id', Integer, primary_key=True)
alias = Column(String)
def __init__(self, alias='', source=None):
self.__generic_init()
# Use default source, unless specified otherwise
if source is None:
source = SourceManager.default
self.source = source
self.alias = alias
@reconstructor
def _dbinit(self):
self.__generic_init()
# Use default source for all reconstructed characters
self.source = SourceManager.default
# Restore entities which are stored on character
for skill in self._skills:
self.skill.add(skill)
def __generic_init(self):
char_type_id = Type.character_static
EveItemWrapper.__init__(self, char_type_id)
# Attributes which store objects hidden behind properties
self.__source = None
# Set with fits which are loaded and use this character
self.__loaded_proxies = WeakSet()
self.skills = SkillCoreSet(self)
self._eos_fit = EosFit()
# EVE item wrapper methods
@property
def _source(self):
return self.source
@property
def _eos_item(self):
return self._eos_fit.character
@property
def _src_children(self):
return get_src_children(chain(
self.skills,
))
# Miscellanea public stuff
@property
def source(self):
return self.__source
@source.setter
def source(self, new_source):
# Attempt to fetch source from source manager if passed object
# is not instance of source class
if not isinstance(new_source, Source) and new_source is not None:
new_source = SourceManager.get(new_source)
old_source = self.source
# Do not update anything if sources are the same
if new_source is old_source:
return
self.__source = new_source
# Update eos model with new data
self._eos_fit.source = getattr(new_source, 'eos', None)
# Update pyfa model with new data
# Unlike fit, character represents EVE item, thus we need
# to update it too
self._update_source()
for src_child in self._src_children:
src_child._update_source()
persist = pyfa_persist
abandon = pyfa_abandon
def validate(self):
self._eos_fit.validate()
# Auxiliary methods
def _proxy_iter(self):
"""
Safe iterator over related character proxies, avoids issues
with set size getting changed by GC during iteration.
"""
for char_proxy in tuple(self.__loaded_proxies):
yield char_proxy
def _link_proxy(self, char_proxy):
"""Create connection between character core and proxy"""
self.__loaded_proxies.add(char_proxy)
def _unlink_proxy(self, char_proxy):
"""Remove connection between character core and proxy"""
self.__loaded_proxies.discard(char_proxy)
def __repr__(self):
spec = ['id']
return make_repr_str(self, spec)
class TestWeakSet(unittest.TestCase):
def setUp(self):
# need to keep references to them
self.items = [ustr(c) for c in ('a', 'b', 'c')]
self.items2 = [ustr(c) for c in ('x', 'y', 'z')]
self.ab_items = [ustr(c) for c in 'ab']
self.abcde_items = [ustr(c) for c in 'abcde']
self.def_items = [ustr(c) for c in 'def']
self.ab_weakset = WeakSet(self.ab_items)
self.abcde_weakset = WeakSet(self.abcde_items)
self.def_weakset = WeakSet(self.def_items)
self.letters = [ustr(c) for c in string.ascii_letters]
self.s = WeakSet(self.items)
self.d = dict.fromkeys(self.items)
self.obj = ustr('F')
self.fs = WeakSet([self.obj])
def test_methods(self):
weaksetmethods = dir(WeakSet)
for method in dir(set):
if method == 'test_c_api' or method.startswith('_'):
continue
self.assertIn(method, weaksetmethods,
"WeakSet missing method " + method)
def test_new_or_init(self):
self.assertRaises(TypeError, WeakSet, [], 2)
def test_len(self):
self.assertEqual(len(self.s), len(self.d))
self.assertEqual(len(self.fs), 1)
del self.obj
support.gc_collect()
self.assertEqual(len(self.fs), 0)
def test_contains(self):
for c in self.letters:
self.assertEqual(c in self.s, c in self.d)
# 1 is not weakref'able, but that TypeError is caught by __contains__
self.assertNotIn(1, self.s)
self.assertIn(self.obj, self.fs)
del self.obj
support.gc_collect()
self.assertNotIn(ustr('F'), self.fs)
def test_union(self):
u = self.s.union(self.items2)
for c in self.letters:
self.assertEqual(c in u, c in self.d or c in self.items2)
self.assertEqual(self.s, WeakSet(self.items))
self.assertEqual(type(u), WeakSet)
self.assertRaises(TypeError, self.s.union, [[]])
for C in set, frozenset, dict.fromkeys, list, tuple:
x = WeakSet(self.items + self.items2)
c = C(self.items2)
self.assertEqual(self.s.union(c), x)
del c
self.assertEqual(len(u), len(self.items) + len(self.items2))
self.items2.pop()
gc.collect()
self.assertEqual(len(u), len(self.items) + len(self.items2))
def test_or(self):
i = self.s.union(self.items2)
self.assertEqual(self.s | set(self.items2), i)
self.assertEqual(self.s | frozenset(self.items2), i)
def test_intersection(self):
s = WeakSet(self.letters)
i = s.intersection(self.items2)
for c in self.letters:
self.assertEqual(c in i, c in self.items2 and c in self.letters)
self.assertEqual(s, WeakSet(self.letters))
self.assertEqual(type(i), WeakSet)
for C in set, frozenset, dict.fromkeys, list, tuple:
x = WeakSet([])
self.assertEqual(i.intersection(C(self.items)), x)
self.assertEqual(len(i), len(self.items2))
self.items2.pop()
gc.collect()
self.assertEqual(len(i), len(self.items2))
def test_isdisjoint(self):
self.assertTrue(self.s.isdisjoint(WeakSet(self.items2)))
self.assertTrue(not self.s.isdisjoint(WeakSet(self.letters)))
def test_and(self):
i = self.s.intersection(self.items2)
self.assertEqual(self.s & set(self.items2), i)
self.assertEqual(self.s & frozenset(self.items2), i)
def test_difference(self):
i = self.s.difference(self.items2)
for c in self.letters:
self.assertEqual(c in i, c in self.d and c not in self.items2)
self.assertEqual(self.s, WeakSet(self.items))
self.assertEqual(type(i), WeakSet)
self.assertRaises(TypeError, self.s.difference, [[]])
def test_sub(self):
i = self.s.difference(self.items2)
self.assertEqual(self.s - set(self.items2), i)
self.assertEqual(self.s - frozenset(self.items2), i)
def test_symmetric_difference(self):
i = self.s.symmetric_difference(self.items2)
for c in self.letters:
self.assertEqual(c in i, (c in self.d) ^ (c in self.items2))
self.assertEqual(self.s, WeakSet(self.items))
self.assertEqual(type(i), WeakSet)
self.assertRaises(TypeError, self.s.symmetric_difference, [[]])
self.assertEqual(len(i), len(self.items) + len(self.items2))
self.items2.pop()
gc.collect()
self.assertEqual(len(i), len(self.items) + len(self.items2))
def test_xor(self):
i = self.s.symmetric_difference(self.items2)
self.assertEqual(self.s ^ set(self.items2), i)
self.assertEqual(self.s ^ frozenset(self.items2), i)
def test_sub_and_super(self):
self.assertTrue(self.ab_weakset <= self.abcde_weakset)
self.assertTrue(self.abcde_weakset <= self.abcde_weakset)
self.assertTrue(self.abcde_weakset >= self.ab_weakset)
self.assertFalse(self.abcde_weakset <= self.def_weakset)
self.assertFalse(self.abcde_weakset >= self.def_weakset)
self.assertTrue(set('a').issubset('abc'))
self.assertTrue(set('abc').issuperset('a'))
self.assertFalse(set('a').issubset('cbs'))
self.assertFalse(set('cbs').issuperset('a'))
def test_lt(self):
self.assertTrue(self.ab_weakset < self.abcde_weakset)
self.assertFalse(self.abcde_weakset < self.def_weakset)
self.assertFalse(self.ab_weakset < self.ab_weakset)
self.assertFalse(WeakSet() < WeakSet())
def test_gt(self):
self.assertTrue(self.abcde_weakset > self.ab_weakset)
self.assertFalse(self.abcde_weakset > self.def_weakset)
self.assertFalse(self.ab_weakset > self.ab_weakset)
self.assertFalse(WeakSet() > WeakSet())
def test_gc(self):
# Create a nest of cycles to exercise overall ref count check
s = WeakSet(Foo() for i in range(1000))
for elem in s:
elem.cycle = s
elem.sub = elem
elem.set = WeakSet([elem])
def test_subclass_with_custom_hash(self):
# Bug #1257731
class H(WeakSet):
def __hash__(self):
return int(id(self) & 0x7fffffff)
s=H()
f=set()
f.add(s)
self.assertIn(s, f)
f.remove(s)
f.add(s)
f.discard(s)
def test_init(self):
s = WeakSet()
s.__init__(self.items)
self.assertEqual(s, self.s)
s.__init__(self.items2)
self.assertEqual(s, WeakSet(self.items2))
self.assertRaises(TypeError, s.__init__, s, 2);
self.assertRaises(TypeError, s.__init__, 1);
def test_constructor_identity(self):
s = WeakSet(self.items)
t = WeakSet(s)
self.assertNotEqual(id(s), id(t))
def test_hash(self):
self.assertRaises(TypeError, hash, self.s)
def test_clear(self):
self.s.clear()
self.assertEqual(self.s, WeakSet([]))
self.assertEqual(len(self.s), 0)
def test_copy(self):
dup = self.s.copy()
self.assertEqual(self.s, dup)
self.assertNotEqual(id(self.s), id(dup))
def test_add(self):
x = ustr('Q')
self.s.add(x)
self.assertIn(x, self.s)
dup = self.s.copy()
self.s.add(x)
self.assertEqual(self.s, dup)
self.assertRaises(TypeError, self.s.add, [])
self.fs.add(Foo())
support.gc_collect()
self.assertTrue(len(self.fs) == 1)
self.fs.add(self.obj)
self.assertTrue(len(self.fs) == 1)
def test_remove(self):
x = ustr('a')
self.s.remove(x)
self.assertNotIn(x, self.s)
self.assertRaises(KeyError, self.s.remove, x)
self.assertRaises(TypeError, self.s.remove, [])
def test_discard(self):
a, q = ustr('a'), ustr('Q')
self.s.discard(a)
self.assertNotIn(a, self.s)
self.s.discard(q)
self.assertRaises(TypeError, self.s.discard, [])
def test_pop(self):
for i in range(len(self.s)):
elem = self.s.pop()
self.assertNotIn(elem, self.s)
self.assertRaises(KeyError, self.s.pop)
def test_update(self):
retval = self.s.update(self.items2)
self.assertEqual(retval, None)
for c in (self.items + self.items2):
self.assertIn(c, self.s)
self.assertRaises(TypeError, self.s.update, [[]])
def test_update_set(self):
self.s.update(set(self.items2))
for c in (self.items + self.items2):
self.assertIn(c, self.s)
def test_ior(self):
self.s |= set(self.items2)
for c in (self.items + self.items2):
self.assertIn(c, self.s)
def test_intersection_update(self):
retval = self.s.intersection_update(self.items2)
self.assertEqual(retval, None)
for c in (self.items + self.items2):
if c in self.items2 and c in self.items:
self.assertIn(c, self.s)
else:
self.assertNotIn(c, self.s)
self.assertRaises(TypeError, self.s.intersection_update, [[]])
def test_iand(self):
self.s &= set(self.items2)
for c in (self.items + self.items2):
if c in self.items2 and c in self.items:
self.assertIn(c, self.s)
else:
self.assertNotIn(c, self.s)
def test_difference_update(self):
retval = self.s.difference_update(self.items2)
self.assertEqual(retval, None)
for c in (self.items + self.items2):
if c in self.items and c not in self.items2:
self.assertIn(c, self.s)
else:
self.assertNotIn(c, self.s)
self.assertRaises(TypeError, self.s.difference_update, [[]])
self.assertRaises(TypeError, self.s.symmetric_difference_update, [[]])
def test_isub(self):
self.s -= set(self.items2)
for c in (self.items + self.items2):
if c in self.items and c not in self.items2:
self.assertIn(c, self.s)
else:
self.assertNotIn(c, self.s)
def test_symmetric_difference_update(self):
retval = self.s.symmetric_difference_update(self.items2)
self.assertEqual(retval, None)
for c in (self.items + self.items2):
if (c in self.items) ^ (c in self.items2):
self.assertIn(c, self.s)
else:
self.assertNotIn(c, self.s)
self.assertRaises(TypeError, self.s.symmetric_difference_update, [[]])
def test_ixor(self):
self.s ^= set(self.items2)
for c in (self.items + self.items2):
if (c in self.items) ^ (c in self.items2):
self.assertIn(c, self.s)
else:
self.assertNotIn(c, self.s)
def test_inplace_on_self(self):
t = self.s.copy()
t |= t
self.assertEqual(t, self.s)
t &= t
self.assertEqual(t, self.s)
t -= t
self.assertEqual(t, WeakSet())
t = self.s.copy()
t ^= t
self.assertEqual(t, WeakSet())
def test_eq(self):
# issue 5964
self.assertTrue(self.s == self.s)
self.assertTrue(self.s == WeakSet(self.items))
self.assertFalse(self.s == set(self.items))
self.assertFalse(self.s == list(self.items))
self.assertFalse(self.s == tuple(self.items))
self.assertFalse(self.s == WeakSet([Foo]))
self.assertFalse(self.s == 1)
def test_ne(self):
self.assertTrue(self.s != set(self.items))
s1 = WeakSet()
s2 = WeakSet()
self.assertFalse(s1 != s2)
def test_weak_destroy_while_iterating(self):
# Issue #7105: iterators shouldn't crash when a key is implicitly removed
# Create new items to be sure no-one else holds a reference
items = [ustr(c) for c in ('a', 'b', 'c')]
s = WeakSet(items)
it = iter(s)
next(it) # Trigger internal iteration
# Destroy an item
del items[-1]
gc.collect() # just in case
# We have removed either the first consumed items, or another one
self.assertIn(len(list(it)), [len(items), len(items) - 1])
del it
# The removal has been committed
self.assertEqual(len(s), len(items))
def test_weak_destroy_and_mutate_while_iterating(self):
# Issue #7105: iterators shouldn't crash when a key is implicitly removed
items = [ustr(c) for c in string.ascii_letters]
s = WeakSet(items)
@contextlib.contextmanager
def testcontext():
try:
it = iter(s)
# Start iterator
yielded = ustr(str(next(it)))
# Schedule an item for removal and recreate it
u = ustr(str(items.pop()))
if yielded == u:
# The iterator still has a reference to the removed item,
# advance it (issue #20006).
next(it)
gc.collect() # just in case
yield u
finally:
it = None # should commit all removals
with testcontext() as u:
self.assertNotIn(u, s)
with testcontext() as u:
self.assertRaises(KeyError, s.remove, u)
self.assertNotIn(u, s)
with testcontext() as u:
s.add(u)
self.assertIn(u, s)
t = s.copy()
with testcontext() as u:
s.update(t)
self.assertEqual(len(s), len(t))
with testcontext() as u:
s.clear()
self.assertEqual(len(s), 0)
def test_len_cycles(self):
N = 20
items = [RefCycle() for i in range(N)]
s = WeakSet(items)
del items
it = iter(s)
try:
next(it)
except StopIteration:
pass
gc.collect()
n1 = len(s)
del it
gc.collect()
gc.collect()
n2 = len(s)
# one item may be kept alive inside the iterator
self.assertIn(n1, (0, 1))
self.assertEqual(n2, 0)
@support.impl_detail("PyPy has no cyclic collection", pypy=False)
def test_len_race(self):
# Extended sanity checks for len() in the face of cyclic collection
self.addCleanup(gc.set_threshold, *gc.get_threshold())
for th in range(1, 100):
N = 20
gc.collect(0)
gc.set_threshold(th, th, th)
items = [RefCycle() for i in range(N)]
s = WeakSet(items)
del items
# All items will be collected at next garbage collection pass
it = iter(s)
try:
next(it)
except StopIteration:
pass
n1 = len(s)
del it
n2 = len(s)
self.assertGreaterEqual(n1, 0)
self.assertLessEqual(n1, N)
self.assertGreaterEqual(n2, 0)
self.assertLessEqual(n2, n1)
class TestWeakSet(unittest.TestCase):
def setUp(self):
# need to keep references to them
self.items = [ustr(c) for c in ('a', 'b', 'c')]
self.items2 = [ustr(c) for c in ('x', 'y', 'z')]
self.letters = [ustr(c) for c in string.ascii_letters]
self.s = WeakSet(self.items)
self.d = dict.fromkeys(self.items)
self.obj = ustr('F')
self.fs = WeakSet([self.obj])
def test_methods(self):
weaksetmethods = dir(WeakSet)
for method in dir(set):
if method == 'test_c_api' or method.startswith('_'):
continue
self.assert_(method in weaksetmethods,
"WeakSet missing method " + method)
def test_new_or_init(self):
self.assertRaises(TypeError, WeakSet, [], 2)
def test_len(self):
self.assertEqual(len(self.s), len(self.d))
self.assertEqual(len(self.fs), 1)
del self.obj
self.assertEqual(len(self.fs), 0)
def test_contains(self):
for c in self.letters:
self.assertEqual(c in self.s, c in self.d)
self.assertRaises(TypeError, self.s.__contains__, [[]])
self.assert_(self.obj in self.fs)
del self.obj
self.assert_(ustr('F') not in self.fs)
def test_union(self):
u = self.s.union(self.items2)
for c in self.letters:
self.assertEqual(c in u, c in self.d or c in self.items2)
self.assertEqual(self.s, WeakSet(self.items))
self.assertEqual(type(u), WeakSet)
self.assertRaises(TypeError, self.s.union, [[]])
for C in set, frozenset, dict.fromkeys, list, tuple:
x = WeakSet(self.items + self.items2)
c = C(self.items2)
self.assertEqual(self.s.union(c), x)
def test_or(self):
i = self.s.union(self.items2)
self.assertEqual(self.s | set(self.items2), i)
self.assertEqual(self.s | frozenset(self.items2), i)
def test_intersection(self):
i = self.s.intersection(self.items2)
for c in self.letters:
self.assertEqual(c in i, c in self.d and c in self.items2)
self.assertEqual(self.s, WeakSet(self.items))
self.assertEqual(type(i), WeakSet)
for C in set, frozenset, dict.fromkeys, list, tuple:
x = WeakSet([])
self.assertEqual(self.s.intersection(C(self.items2)), x)
def test_isdisjoint(self):
self.assert_(self.s.isdisjoint(WeakSet(self.items2)))
self.assert_(not self.s.isdisjoint(WeakSet(self.letters)))
def test_and(self):
i = self.s.intersection(self.items2)
self.assertEqual(self.s & set(self.items2), i)
self.assertEqual(self.s & frozenset(self.items2), i)
def test_difference(self):
i = self.s.difference(self.items2)
for c in self.letters:
self.assertEqual(c in i, c in self.d and c not in self.items2)
self.assertEqual(self.s, WeakSet(self.items))
self.assertEqual(type(i), WeakSet)
self.assertRaises(TypeError, self.s.difference, [[]])
def test_sub(self):
i = self.s.difference(self.items2)
self.assertEqual(self.s - set(self.items2), i)
self.assertEqual(self.s - frozenset(self.items2), i)
def test_symmetric_difference(self):
i = self.s.symmetric_difference(self.items2)
for c in self.letters:
self.assertEqual(c in i, (c in self.d) ^ (c in self.items2))
self.assertEqual(self.s, WeakSet(self.items))
self.assertEqual(type(i), WeakSet)
self.assertRaises(TypeError, self.s.symmetric_difference, [[]])
def test_xor(self):
i = self.s.symmetric_difference(self.items2)
self.assertEqual(self.s ^ set(self.items2), i)
self.assertEqual(self.s ^ frozenset(self.items2), i)
def test_sub_and_super(self):
pl, ql, rl = map(lambda s: [ustr(c) for c in s], ['ab', 'abcde', 'def'])
p, q, r = map(WeakSet, (pl, ql, rl))
self.assert_(p < q)
self.assert_(p <= q)
self.assert_(q <= q)
self.assert_(q > p)
self.assert_(q >= p)
self.failIf(q < r)
self.failIf(q <= r)
self.failIf(q > r)
self.failIf(q >= r)
self.assert_(set('a').issubset('abc'))
self.assert_(set('abc').issuperset('a'))
self.failIf(set('a').issubset('cbs'))
self.failIf(set('cbs').issuperset('a'))
def test_gc(self):
# Create a nest of cycles to exercise overall ref count check
class A:
pass
s = set(A() for i in range(1000))
for elem in s:
elem.cycle = s
elem.sub = elem
elem.set = set([elem])
def test_subclass_with_custom_hash(self):
# Bug #1257731
class H(WeakSet):
def __hash__(self):
return int(id(self) & 0x7fffffff)
s=H()
f=set()
f.add(s)
self.assert_(s in f)
f.remove(s)
f.add(s)
f.discard(s)
def test_init(self):
s = WeakSet()
s.__init__(self.items)
self.assertEqual(s, self.s)
s.__init__(self.items2)
self.assertEqual(s, WeakSet(self.items2))
self.assertRaises(TypeError, s.__init__, s, 2);
self.assertRaises(TypeError, s.__init__, 1);
def test_constructor_identity(self):
s = WeakSet(self.items)
t = WeakSet(s)
self.assertNotEqual(id(s), id(t))
def test_set_literal(self):
s = set([1,2,3])
t = {1,2,3}
self.assertEqual(s, t)
def test_hash(self):
self.assertRaises(TypeError, hash, self.s)
def test_clear(self):
self.s.clear()
self.assertEqual(self.s, set())
self.assertEqual(len(self.s), 0)
def test_copy(self):
dup = self.s.copy()
self.assertEqual(self.s, dup)
self.assertNotEqual(id(self.s), id(dup))
def test_add(self):
x = ustr('Q')
self.s.add(x)
self.assert_(x in self.s)
dup = self.s.copy()
self.s.add(x)
self.assertEqual(self.s, dup)
self.assertRaises(TypeError, self.s.add, [])
self.fs.add(Foo())
self.assert_(len(self.fs) == 1)
self.fs.add(self.obj)
self.assert_(len(self.fs) == 1)
def test_remove(self):
x = ustr('a')
self.s.remove(x)
self.assert_(x not in self.s)
self.assertRaises(KeyError, self.s.remove, x)
self.assertRaises(TypeError, self.s.remove, [])
def test_discard(self):
a, q = ustr('a'), ustr('Q')
self.s.discard(a)
self.assert_(a not in self.s)
self.s.discard(q)
self.assertRaises(TypeError, self.s.discard, [])
def test_pop(self):
for i in range(len(self.s)):
elem = self.s.pop()
self.assert_(elem not in self.s)
self.assertRaises(KeyError, self.s.pop)
def test_update(self):
retval = self.s.update(self.items2)
self.assertEqual(retval, None)
for c in (self.items + self.items2):
self.assert_(c in self.s)
self.assertRaises(TypeError, self.s.update, [[]])
def test_update_set(self):
self.s.update(set(self.items2))
for c in (self.items + self.items2):
self.assert_(c in self.s)
def test_ior(self):
self.s |= set(self.items2)
for c in (self.items + self.items2):
self.assert_(c in self.s)
def test_intersection_update(self):
retval = self.s.intersection_update(self.items2)
self.assertEqual(retval, None)
for c in (self.items + self.items2):
if c in self.items2 and c in self.items:
self.assert_(c in self.s)
else:
self.assert_(c not in self.s)
self.assertRaises(TypeError, self.s.intersection_update, [[]])
def test_iand(self):
self.s &= set(self.items2)
for c in (self.items + self.items2):
if c in self.items2 and c in self.items:
self.assert_(c in self.s)
else:
self.assert_(c not in self.s)
def test_difference_update(self):
retval = self.s.difference_update(self.items2)
self.assertEqual(retval, None)
for c in (self.items + self.items2):
if c in self.items and c not in self.items2:
self.assert_(c in self.s)
else:
self.assert_(c not in self.s)
self.assertRaises(TypeError, self.s.difference_update, [[]])
self.assertRaises(TypeError, self.s.symmetric_difference_update, [[]])
def test_isub(self):
self.s -= set(self.items2)
for c in (self.items + self.items2):
if c in self.items and c not in self.items2:
self.assert_(c in self.s)
else:
self.assert_(c not in self.s)
def test_symmetric_difference_update(self):
retval = self.s.symmetric_difference_update(self.items2)
self.assertEqual(retval, None)
for c in (self.items + self.items2):
if (c in self.items) ^ (c in self.items2):
self.assert_(c in self.s)
else:
self.assert_(c not in self.s)
self.assertRaises(TypeError, self.s.symmetric_difference_update, [[]])
def test_ixor(self):
self.s ^= set(self.items2)
for c in (self.items + self.items2):
if (c in self.items) ^ (c in self.items2):
self.assert_(c in self.s)
else:
self.assert_(c not in self.s)
def test_inplace_on_self(self):
t = self.s.copy()
t |= t
self.assertEqual(t, self.s)
t &= t
self.assertEqual(t, self.s)
t -= t
self.assertEqual(t, WeakSet())
t = self.s.copy()
t ^= t
self.assertEqual(t, WeakSet())
class TestWeakSet(unittest.TestCase):
def setUp(self):
# need to keep references to them
self.items = [ustr(c) for c in ('a', 'b', 'c')]
self.items2 = [ustr(c) for c in ('x', 'y', 'z')]
self.ab_items = [ustr(c) for c in 'ab']
self.abcde_items = [ustr(c) for c in 'abcde']
self.def_items = [ustr(c) for c in 'def']
self.ab_weakset = WeakSet(self.ab_items)
self.abcde_weakset = WeakSet(self.abcde_items)
self.def_weakset = WeakSet(self.def_items)
self.letters = [ustr(c) for c in string.ascii_letters]
self.s = WeakSet(self.items)
self.d = dict.fromkeys(self.items)
self.obj = ustr('F')
self.fs = WeakSet([self.obj])
def test_methods(self):
weaksetmethods = dir(WeakSet)
for method in dir(set):
if method == 'test_c_api' or method.startswith('_'):
continue
self.assertIn(method, weaksetmethods,
"WeakSet missing method " + method)
def test_new_or_init(self):
self.assertRaises(TypeError, WeakSet, [], 2)
@support.impl_detail("finalization", graalvm=False)
def test_len(self):
self.assertEqual(len(self.s), len(self.d))
self.assertEqual(len(self.fs), 1)
del self.obj
self.assertEqual(len(self.fs), 0)
@support.impl_detail("finalization", graalvm=False)
def test_contains(self):
for c in self.letters:
self.assertEqual(c in self.s, c in self.d)
# 1 is not weakref'able, but that TypeError is caught by __contains__
self.assertNotIn(1, self.s)
self.assertIn(self.obj, self.fs)
del self.obj
self.assertNotIn(ustr('F'), self.fs)
@support.impl_detail("finalization", graalvm=False)
def test_union(self):
u = self.s.union(self.items2)
for c in self.letters:
self.assertEqual(c in u, c in self.d or c in self.items2)
self.assertEqual(self.s, WeakSet(self.items))
self.assertEqual(type(u), WeakSet)
self.assertRaises(TypeError, self.s.union, [[]])
for C in set, frozenset, dict.fromkeys, list, tuple:
x = WeakSet(self.items + self.items2)
c = C(self.items2)
self.assertEqual(self.s.union(c), x)
del c
self.assertEqual(len(u), len(self.items) + len(self.items2))
self.items2.pop()
gc.collect()
self.assertEqual(len(u), len(self.items) + len(self.items2))
def test_or(self):
i = self.s.union(self.items2)
self.assertEqual(self.s | set(self.items2), i)
self.assertEqual(self.s | frozenset(self.items2), i)
@support.impl_detail("finalization", graalvm=False)
def test_intersection(self):
s = WeakSet(self.letters)
i = s.intersection(self.items2)
for c in self.letters:
self.assertEqual(c in i, c in self.items2 and c in self.letters)
self.assertEqual(s, WeakSet(self.letters))
self.assertEqual(type(i), WeakSet)
for C in set, frozenset, dict.fromkeys, list, tuple:
x = WeakSet([])
self.assertEqual(i.intersection(C(self.items)), x)
self.assertEqual(len(i), len(self.items2))
self.items2.pop()
gc.collect()
self.assertEqual(len(i), len(self.items2))
def test_isdisjoint(self):
self.assertTrue(self.s.isdisjoint(WeakSet(self.items2)))
self.assertTrue(not self.s.isdisjoint(WeakSet(self.letters)))
def test_and(self):
i = self.s.intersection(self.items2)
self.assertEqual(self.s & set(self.items2), i)
self.assertEqual(self.s & frozenset(self.items2), i)
def test_difference(self):
i = self.s.difference(self.items2)
for c in self.letters:
self.assertEqual(c in i, c in self.d and c not in self.items2)
self.assertEqual(self.s, WeakSet(self.items))
self.assertEqual(type(i), WeakSet)
self.assertRaises(TypeError, self.s.difference, [[]])
def test_sub(self):
i = self.s.difference(self.items2)
self.assertEqual(self.s - set(self.items2), i)
self.assertEqual(self.s - frozenset(self.items2), i)
@support.impl_detail("finalization", graalvm=False)
def test_symmetric_difference(self):
i = self.s.symmetric_difference(self.items2)
for c in self.letters:
self.assertEqual(c in i, (c in self.d) ^ (c in self.items2))
self.assertEqual(self.s, WeakSet(self.items))
self.assertEqual(type(i), WeakSet)
self.assertRaises(TypeError, self.s.symmetric_difference, [[]])
self.assertEqual(len(i), len(self.items) + len(self.items2))
self.items2.pop()
gc.collect()
self.assertEqual(len(i), len(self.items) + len(self.items2))
def test_xor(self):
i = self.s.symmetric_difference(self.items2)
self.assertEqual(self.s ^ set(self.items2), i)
self.assertEqual(self.s ^ frozenset(self.items2), i)
def test_sub_and_super(self):
self.assertTrue(self.ab_weakset <= self.abcde_weakset)
self.assertTrue(self.abcde_weakset <= self.abcde_weakset)
self.assertTrue(self.abcde_weakset >= self.ab_weakset)
self.assertFalse(self.abcde_weakset <= self.def_weakset)
self.assertFalse(self.abcde_weakset >= self.def_weakset)
self.assertTrue(set('a').issubset('abc'))
self.assertTrue(set('abc').issuperset('a'))
self.assertFalse(set('a').issubset('cbs'))
self.assertFalse(set('cbs').issuperset('a'))
def test_lt(self):
self.assertTrue(self.ab_weakset < self.abcde_weakset)
self.assertFalse(self.abcde_weakset < self.def_weakset)
self.assertFalse(self.ab_weakset < self.ab_weakset)
self.assertFalse(WeakSet() < WeakSet())
def test_gt(self):
self.assertTrue(self.abcde_weakset > self.ab_weakset)
self.assertFalse(self.abcde_weakset > self.def_weakset)
self.assertFalse(self.ab_weakset > self.ab_weakset)
self.assertFalse(WeakSet() > WeakSet())
def test_gc(self):
# Create a nest of cycles to exercise overall ref count check
s = WeakSet(Foo() for i in range(1000))
for elem in s:
elem.cycle = s
elem.sub = elem
elem.set = WeakSet([elem])
def test_subclass_with_custom_hash(self):
# Bug #1257731
class H(WeakSet):
def __hash__(self):
return int(id(self) & 0x7fffffff)
s=H()
f=set()
f.add(s)
self.assertIn(s, f)
f.remove(s)
f.add(s)
f.discard(s)
def test_init(self):
s = WeakSet()
s.__init__(self.items)
self.assertEqual(s, self.s)
s.__init__(self.items2)
self.assertEqual(s, WeakSet(self.items2))
self.assertRaises(TypeError, s.__init__, s, 2);
self.assertRaises(TypeError, s.__init__, 1);
def test_constructor_identity(self):
s = WeakSet(self.items)
t = WeakSet(s)
self.assertNotEqual(id(s), id(t))
def test_hash(self):
self.assertRaises(TypeError, hash, self.s)
def test_clear(self):
self.s.clear()
self.assertEqual(self.s, WeakSet([]))
self.assertEqual(len(self.s), 0)
def test_copy(self):
dup = self.s.copy()
self.assertEqual(self.s, dup)
self.assertNotEqual(id(self.s), id(dup))
@support.impl_detail("refcounting", graalvm=False)
def test_add(self):
x = ustr('Q')
self.s.add(x)
self.assertIn(x, self.s)
dup = self.s.copy()
self.s.add(x)
self.assertEqual(self.s, dup)
self.assertRaises(TypeError, self.s.add, [])
self.fs.add(Foo())
self.assertTrue(len(self.fs) == 1)
self.fs.add(self.obj)
self.assertTrue(len(self.fs) == 1)
def test_remove(self):
x = ustr('a')
self.s.remove(x)
self.assertNotIn(x, self.s)
self.assertRaises(KeyError, self.s.remove, x)
self.assertRaises(TypeError, self.s.remove, [])
def test_discard(self):
a, q = ustr('a'), ustr('Q')
self.s.discard(a)
self.assertNotIn(a, self.s)
self.s.discard(q)
self.assertRaises(TypeError, self.s.discard, [])
def test_pop(self):
for i in range(len(self.s)):
elem = self.s.pop()
self.assertNotIn(elem, self.s)
self.assertRaises(KeyError, self.s.pop)
def test_update(self):
retval = self.s.update(self.items2)
self.assertEqual(retval, None)
for c in (self.items + self.items2):
self.assertIn(c, self.s)
self.assertRaises(TypeError, self.s.update, [[]])
def test_update_set(self):
self.s.update(set(self.items2))
for c in (self.items + self.items2):
self.assertIn(c, self.s)
def test_ior(self):
self.s |= set(self.items2)
for c in (self.items + self.items2):
self.assertIn(c, self.s)
def test_intersection_update(self):
retval = self.s.intersection_update(self.items2)
self.assertEqual(retval, None)
for c in (self.items + self.items2):
if c in self.items2 and c in self.items:
self.assertIn(c, self.s)
else:
self.assertNotIn(c, self.s)
self.assertRaises(TypeError, self.s.intersection_update, [[]])
def test_iand(self):
self.s &= set(self.items2)
for c in (self.items + self.items2):
if c in self.items2 and c in self.items:
self.assertIn(c, self.s)
else:
self.assertNotIn(c, self.s)
def test_difference_update(self):
retval = self.s.difference_update(self.items2)
self.assertEqual(retval, None)
for c in (self.items + self.items2):
if c in self.items and c not in self.items2:
self.assertIn(c, self.s)
else:
self.assertNotIn(c, self.s)
self.assertRaises(TypeError, self.s.difference_update, [[]])
self.assertRaises(TypeError, self.s.symmetric_difference_update, [[]])
def test_isub(self):
self.s -= set(self.items2)
for c in (self.items + self.items2):
if c in self.items and c not in self.items2:
self.assertIn(c, self.s)
else:
self.assertNotIn(c, self.s)
def test_symmetric_difference_update(self):
retval = self.s.symmetric_difference_update(self.items2)
self.assertEqual(retval, None)
for c in (self.items + self.items2):
if (c in self.items) ^ (c in self.items2):
self.assertIn(c, self.s)
else:
self.assertNotIn(c, self.s)
self.assertRaises(TypeError, self.s.symmetric_difference_update, [[]])
def test_ixor(self):
self.s ^= set(self.items2)
for c in (self.items + self.items2):
if (c in self.items) ^ (c in self.items2):
self.assertIn(c, self.s)
else:
self.assertNotIn(c, self.s)
def test_inplace_on_self(self):
t = self.s.copy()
t |= t
self.assertEqual(t, self.s)
t &= t
self.assertEqual(t, self.s)
t -= t
self.assertEqual(t, WeakSet())
t = self.s.copy()
t ^= t
self.assertEqual(t, WeakSet())
def test_eq(self):
# issue 5964
self.assertTrue(self.s == self.s)
self.assertTrue(self.s == WeakSet(self.items))
self.assertFalse(self.s == set(self.items))
self.assertFalse(self.s == list(self.items))
self.assertFalse(self.s == tuple(self.items))
self.assertFalse(self.s == WeakSet([Foo]))
self.assertFalse(self.s == 1)
def test_ne(self):
self.assertTrue(self.s != set(self.items))
s1 = WeakSet()
s2 = WeakSet()
self.assertFalse(s1 != s2)
@support.impl_detail("finalization", graalvm=False)
def test_weak_destroy_while_iterating(self):
# Issue #7105: iterators shouldn't crash when a key is implicitly removed
# Create new items to be sure no-one else holds a reference
items = [ustr(c) for c in ('a', 'b', 'c')]
s = WeakSet(items)
it = iter(s)
next(it) # Trigger internal iteration
# Destroy an item
del items[-1]
gc.collect() # just in case
# We have removed either the first consumed items, or another one
self.assertIn(len(list(it)), [len(items), len(items) - 1])
del it
# The removal has been committed
self.assertEqual(len(s), len(items))
@support.impl_detail("finalization", graalvm=False)
def test_weak_destroy_and_mutate_while_iterating(self):
# Issue #7105: iterators shouldn't crash when a key is implicitly removed
items = [ustr(c) for c in string.ascii_letters]
s = WeakSet(items)
@contextlib.contextmanager
def testcontext():
try:
it = iter(s)
# Start iterator
yielded = ustr(str(next(it)))
# Schedule an item for removal and recreate it
u = ustr(str(items.pop()))
if yielded == u:
# The iterator still has a reference to the removed item,
# advance it (issue #20006).
next(it)
gc.collect() # just in case
yield u
finally:
it = None # should commit all removals
with testcontext() as u:
self.assertNotIn(u, s)
with testcontext() as u:
self.assertRaises(KeyError, s.remove, u)
self.assertNotIn(u, s)
with testcontext() as u:
s.add(u)
self.assertIn(u, s)
t = s.copy()
with testcontext() as u:
s.update(t)
self.assertEqual(len(s), len(t))
with testcontext() as u:
s.clear()
self.assertEqual(len(s), 0)
@support.impl_detail("finalization", graalvm=False)
def test_len_cycles(self):
N = 20
items = [RefCycle() for i in range(N)]
s = WeakSet(items)
del items
it = iter(s)
try:
next(it)
except StopIteration:
pass
gc.collect()
n1 = len(s)
del it
gc.collect()
n2 = len(s)
# one item may be kept alive inside the iterator
self.assertIn(n1, (0, 1))
self.assertEqual(n2, 0)
@support.impl_detail("finalization", graalvm=False)
def test_len_race(self):
# Extended sanity checks for len() in the face of cyclic collection
self.addCleanup(gc.set_threshold, *gc.get_threshold())
for th in range(1, 100):
N = 20
gc.collect(0)
gc.set_threshold(th, th, th)
items = [RefCycle() for i in range(N)]
s = WeakSet(items)
del items
# All items will be collected at next garbage collection pass
it = iter(s)
try:
next(it)
except StopIteration:
pass
n1 = len(s)
del it
n2 = len(s)
self.assertGreaterEqual(n1, 0)
self.assertLessEqual(n1, N)
self.assertGreaterEqual(n2, 0)
self.assertLessEqual(n2, n1)
def test_repr(self):
assert repr(self.s) == repr(self.s.data)
class TestWeakSet(unittest.TestCase):
def setUp(self):
# need to keep references to them
self.items = [SomeClass(c) for c in ('a', 'b', 'c')]
self.items2 = [SomeClass(c) for c in ('x', 'y', 'z')]
self.letters = [SomeClass(c) for c in string.ascii_letters]
self.s = WeakSet(self.items)
self.d = dict.fromkeys(self.items)
self.obj = SomeClass('F')
self.fs = WeakSet([self.obj])
def test_methods(self):
weaksetmethods = dir(WeakSet)
for method in dir(set):
if method == 'test_c_api' or method.startswith('_'):
continue
self.assertIn(method, weaksetmethods,
"WeakSet missing method " + method)
def test_new_or_init(self):
self.assertRaises(TypeError, WeakSet, [], 2)
def test_len(self):
self.assertEqual(len(self.s), len(self.d))
self.assertEqual(len(self.fs), 1)
del self.obj
self.assertEqual(len(self.fs), 0)
def test_contains(self):
for c in self.letters:
self.assertEqual(c in self.s, c in self.d)
# 1 is not weakref'able, but that TypeError is caught by __contains__
self.assertNotIn(1, self.s)
self.assertIn(self.obj, self.fs)
del self.obj
self.assertNotIn(SomeClass('F'), self.fs)
def test_union(self):
u = self.s.union(self.items2)
for c in self.letters:
self.assertEqual(c in u, c in self.d or c in self.items2)
self.assertEqual(self.s, WeakSet(self.items))
self.assertEqual(type(u), WeakSet)
self.assertRaises(TypeError, self.s.union, [[]])
for C in set, frozenset, dict.fromkeys, list, tuple:
x = WeakSet(self.items + self.items2)
c = C(self.items2)
self.assertEqual(self.s.union(c), x)
def test_or(self):
i = self.s.union(self.items2)
self.assertEqual(self.s | set(self.items2), i)
self.assertEqual(self.s | frozenset(self.items2), i)
def test_intersection(self):
i = self.s.intersection(self.items2)
for c in self.letters:
self.assertEqual(c in i, c in self.d and c in self.items2)
self.assertEqual(self.s, WeakSet(self.items))
self.assertEqual(type(i), WeakSet)
for C in set, frozenset, dict.fromkeys, list, tuple:
x = WeakSet([])
self.assertEqual(self.s.intersection(C(self.items2)), x)
def test_isdisjoint(self):
self.assertTrue(self.s.isdisjoint(WeakSet(self.items2)))
self.assertTrue(not self.s.isdisjoint(WeakSet(self.letters)))
def test_and(self):
i = self.s.intersection(self.items2)
self.assertEqual(self.s & set(self.items2), i)
self.assertEqual(self.s & frozenset(self.items2), i)
def test_difference(self):
i = self.s.difference(self.items2)
for c in self.letters:
self.assertEqual(c in i, c in self.d and c not in self.items2)
self.assertEqual(self.s, WeakSet(self.items))
self.assertEqual(type(i), WeakSet)
self.assertRaises(TypeError, self.s.difference, [[]])
def test_sub(self):
i = self.s.difference(self.items2)
self.assertEqual(self.s - set(self.items2), i)
self.assertEqual(self.s - frozenset(self.items2), i)
def test_symmetric_difference(self):
i = self.s.symmetric_difference(self.items2)
for c in self.letters:
self.assertEqual(c in i, (c in self.d) ^ (c in self.items2))
self.assertEqual(self.s, WeakSet(self.items))
self.assertEqual(type(i), WeakSet)
self.assertRaises(TypeError, self.s.symmetric_difference, [[]])
def test_xor(self):
i = self.s.symmetric_difference(self.items2)
self.assertEqual(self.s ^ set(self.items2), i)
self.assertEqual(self.s ^ frozenset(self.items2), i)
def test_sub_and_super(self):
pl, ql, rl = map(lambda s: [SomeClass(c) for c in s], ['ab', 'abcde', 'def'])
p, q, r = map(WeakSet, (pl, ql, rl))
self.assertTrue(p < q)
self.assertTrue(p <= q)
self.assertTrue(q <= q)
self.assertTrue(q > p)
self.assertTrue(q >= p)
self.assertFalse(q < r)
self.assertFalse(q <= r)
self.assertFalse(q > r)
self.assertFalse(q >= r)
self.assertTrue(set('a').issubset('abc'))
self.assertTrue(set('abc').issuperset('a'))
self.assertFalse(set('a').issubset('cbs'))
self.assertFalse(set('cbs').issuperset('a'))
def test_gc(self):
# Create a nest of cycles to exercise overall ref count check
s = WeakSet(Foo() for i in range(1000))
for elem in s:
elem.cycle = s
elem.sub = elem
elem.set = WeakSet([elem])
def test_subclass_with_custom_hash(self):
# Bug #1257731
class H(WeakSet):
def __hash__(self):
return int(id(self) & 0x7fffffff)
s=H()
f=set()
f.add(s)
self.assertIn(s, f)
f.remove(s)
f.add(s)
f.discard(s)
def test_init(self):
s = WeakSet()
s.__init__(self.items)
self.assertEqual(s, self.s)
s.__init__(self.items2)
self.assertEqual(s, WeakSet(self.items2))
self.assertRaises(TypeError, s.__init__, s, 2);
self.assertRaises(TypeError, s.__init__, 1);
def test_constructor_identity(self):
s = WeakSet(self.items)
t = WeakSet(s)
self.assertNotEqual(id(s), id(t))
def test_hash(self):
self.assertRaises(TypeError, hash, self.s)
def test_clear(self):
self.s.clear()
self.assertEqual(self.s, WeakSet([]))
self.assertEqual(len(self.s), 0)
def test_copy(self):
dup = self.s.copy()
self.assertEqual(self.s, dup)
self.assertNotEqual(id(self.s), id(dup))
def test_add(self):
x = SomeClass('Q')
self.s.add(x)
self.assertIn(x, self.s)
dup = self.s.copy()
self.s.add(x)
self.assertEqual(self.s, dup)
self.assertRaises(TypeError, self.s.add, [])
self.fs.add(Foo())
self.assertTrue(len(self.fs) == 1)
self.fs.add(self.obj)
self.assertTrue(len(self.fs) == 1)
def test_remove(self):
x = SomeClass('a')
self.s.remove(x)
self.assertNotIn(x, self.s)
self.assertRaises(KeyError, self.s.remove, x)
self.assertRaises(TypeError, self.s.remove, [])
def test_discard(self):
a, q = SomeClass('a'), SomeClass('Q')
self.s.discard(a)
self.assertNotIn(a, self.s)
self.s.discard(q)
self.assertRaises(TypeError, self.s.discard, [])
def test_pop(self):
for i in range(len(self.s)):
elem = self.s.pop()
self.assertNotIn(elem, self.s)
self.assertRaises(KeyError, self.s.pop)
def test_update(self):
retval = self.s.update(self.items2)
self.assertEqual(retval, None)
for c in (self.items + self.items2):
self.assertIn(c, self.s)
self.assertRaises(TypeError, self.s.update, [[]])
def test_update_set(self):
self.s.update(set(self.items2))
for c in (self.items + self.items2):
self.assertIn(c, self.s)
def test_ior(self):
self.s |= set(self.items2)
for c in (self.items + self.items2):
self.assertIn(c, self.s)
def test_intersection_update(self):
retval = self.s.intersection_update(self.items2)
self.assertEqual(retval, None)
for c in (self.items + self.items2):
if c in self.items2 and c in self.items:
self.assertIn(c, self.s)
else:
self.assertNotIn(c, self.s)
self.assertRaises(TypeError, self.s.intersection_update, [[]])
def test_iand(self):
self.s &= set(self.items2)
for c in (self.items + self.items2):
if c in self.items2 and c in self.items:
self.assertIn(c, self.s)
else:
self.assertNotIn(c, self.s)
def test_difference_update(self):
retval = self.s.difference_update(self.items2)
self.assertEqual(retval, None)
for c in (self.items + self.items2):
if c in self.items and c not in self.items2:
self.assertIn(c, self.s)
else:
self.assertNotIn(c, self.s)
self.assertRaises(TypeError, self.s.difference_update, [[]])
self.assertRaises(TypeError, self.s.symmetric_difference_update, [[]])
def test_isub(self):
self.s -= set(self.items2)
for c in (self.items + self.items2):
if c in self.items and c not in self.items2:
self.assertIn(c, self.s)
else:
self.assertNotIn(c, self.s)
def test_symmetric_difference_update(self):
retval = self.s.symmetric_difference_update(self.items2)
self.assertEqual(retval, None)
for c in (self.items + self.items2):
if (c in self.items) ^ (c in self.items2):
self.assertIn(c, self.s)
else:
self.assertNotIn(c, self.s)
self.assertRaises(TypeError, self.s.symmetric_difference_update, [[]])
def test_ixor(self):
self.s ^= set(self.items2)
for c in (self.items + self.items2):
if (c in self.items) ^ (c in self.items2):
self.assertIn(c, self.s)
else:
self.assertNotIn(c, self.s)
def test_inplace_on_self(self):
t = self.s.copy()
t |= t
self.assertEqual(t, self.s)
t &= t
self.assertEqual(t, self.s)
t -= t
self.assertEqual(t, WeakSet())
t = self.s.copy()
t ^= t
self.assertEqual(t, WeakSet())
def test_eq(self):
# issue 5964
self.assertTrue(self.s == self.s)
self.assertTrue(self.s == WeakSet(self.items))
self.assertFalse(self.s == set(self.items))
self.assertFalse(self.s == list(self.items))
self.assertFalse(self.s == tuple(self.items))
self.assertFalse(self.s == 1)
def test_weak_destroy_while_iterating(self):
# Issue #7105: iterators shouldn't crash when a key is implicitly removed
# Create new items to be sure no-one else holds a reference
items = [SomeClass(c) for c in ('a', 'b', 'c')]
s = WeakSet(items)
it = iter(s)
next(it) # Trigger internal iteration
# Destroy an item
del items[-1]
gc.collect() # just in case
# We have removed either the first consumed items, or another one
self.assertIn(len(list(it)), [len(items), len(items) - 1])
del it
# The removal has been committed
self.assertEqual(len(s), len(items))
def test_weak_destroy_and_mutate_while_iterating(self):
# Issue #7105: iterators shouldn't crash when a key is implicitly removed
items = [SomeClass(c) for c in string.ascii_letters]
s = WeakSet(items)
@contextlib.contextmanager
def testcontext():
try:
it = iter(s)
next(it)
# Schedule an item for removal and recreate it
u = SomeClass(str(items.pop()))
gc.collect() # just in case
yield u
finally:
it = None # should commit all removals
with testcontext() as u:
self.assertNotIn(u, s)
with testcontext() as u:
self.assertRaises(KeyError, s.remove, u)
self.assertNotIn(u, s)
with testcontext() as u:
s.add(u)
self.assertIn(u, s)
t = s.copy()
with testcontext() as u:
s.update(t)
self.assertEqual(len(s), len(t))
with testcontext() as u:
s.clear()
self.assertEqual(len(s), 0)
class ObjectRoutingComponent(FootprintToggleMixin, Component, HasTunableFactory, AutoFactoryInit, component_name=types.OBJECT_ROUTING_COMPONENT):
FACTORY_TUNABLES = {'routing_behavior_map': TunableMapping(description='\n A mapping of states to behavior. When the object enters a state, its\n corresponding routing behavior is started.\n ', key_type=TunableReference(manager=services.get_instance_manager(sims4.resources.Types.OBJECT_STATE), class_restrictions='ObjectStateValue'), value_type=OptionalTunable(tunable=ObjectRoutingBehavior.TunableReference(), enabled_by_default=True, enabled_name='Start_Behavior', disabled_name='Stop_All_Behavior', disabled_value=UNSET)), 'privacy_rules': OptionalTunable(description='\n If enabled, this object will care about privacy regions.\n ', tunable=TunableTuple(description='\n Privacy rules for this object.\n ', on_enter=TunableTuple(description='\n Tuning for when this object is considered a violator of\n privacy.\n ', loot_list=TunableList(description='\n A list of loot operations to apply when the object\n enters a privacy region.\n ', tunable=LootActions.TunableReference(pack_safe=True))))), 'tracking_category': TunableEnumEntry(description='\n Used to classify routing objects for the purpose of putting them\n into buckets for the object routing service to restrict the number\n of simultaneously-active objects.\n ', tunable_type=ObjectRoutingBehaviorTrackingCategory, default=ObjectRoutingBehaviorTrackingCategory.NONE)}
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self._running_behavior = None
self._idle_element = None
self._previous_parent_ref = None
self._pending_running_behavior = None
self._privacy_violations = WeakSet()
@property
def previous_parent(self):
if self._previous_parent_ref is not None:
return self._previous_parent_ref()
def _setup(self):
master_controller = services.get_master_controller()
master_controller.add_sim(self.owner)
if self.privacy_rules:
privacy_service = services.privacy_service()
privacy_service.add_vehicle_to_monitor(self.owner)
self.owner.routing_component.on_sim_added()
self.add_callbacks()
def on_add(self, *_, **__):
zone = services.current_zone()
if not zone.is_zone_loading:
self._setup()
def on_finalize_load(self):
self._setup()
def on_remove(self):
self.remove_callbacks()
self.owner.routing_component.on_sim_removed()
master_controller = services.get_master_controller()
master_controller.remove_sim(self.owner)
if self.privacy_rules:
privacy_service = services.privacy_service()
privacy_service.remove_vehicle_to_monitor(self.owner)
def add_callbacks(self):
if self.privacy_rules:
self.owner.register_on_location_changed(self._check_privacy)
self.register_routing_event_callbacks()
def remove_callbacks(self):
if self.owner.is_on_location_changed_callback_registered(self._check_privacy):
self.owner.unregister_on_location_changed(self._check_privacy)
self.unregister_routing_event_callbacks()
def handle_privacy_violation(self, privacy):
if not self.privacy_rules:
return
resolver = SingleObjectResolver(self.owner)
loots = LootOperationList(resolver, self.privacy_rules.on_enter.loot_list)
loots.apply_operations()
if privacy not in self._privacy_violations:
self._privacy_violations.add(privacy)
def violates_privacy(self, privacy):
if not self.privacy_rules:
return False
elif not privacy.vehicle_violates_privacy(self.owner):
return False
return True
def _check_privacy(self, _, old_location, new_location):
if not self.privacy_rules:
return
for privacy in services.privacy_service().privacy_instances:
if not privacy.privacy_violators & PrivacyViolators.VEHICLES:
continue
new_violation = privacy not in self._privacy_violations
violates_privacy = self.violates_privacy(privacy)
if new_violation:
if violates_privacy:
self.handle_privacy_violation(privacy)
if not violates_privacy:
self._privacy_violations.discard(privacy)
elif not violates_privacy:
self._privacy_violations.discard(privacy)
def on_state_changed(self, state, old_value, new_value, from_init):
if new_value is old_value:
return
if new_value not in self.routing_behavior_map:
return
self._stop_runnning_behavior()
routing_behavior_type = self.routing_behavior_map[new_value]
if routing_behavior_type is UNSET:
return
routing_behavior = routing_behavior_type(self.owner)
self._set_running_behavior(routing_behavior)
self._cancel_idle_behavior()
def on_location_changed(self, old_location):
parent = self.owner.parent
if parent is not None:
self._previous_parent_ref = parent.ref()
def component_reset(self, reset_reason):
if self._running_behavior is not None:
self._pending_running_behavior = type(self._running_behavior)
self._running_behavior.trigger_hard_stop()
self._set_running_behavior(None)
services.get_master_controller().on_reset_sim(self.owner, reset_reason)
def post_component_reset(self):
if self._pending_running_behavior is not None:
routing_behavior = self._pending_running_behavior(self.owner)
self._set_running_behavior(routing_behavior)
self._pending_running_behavior = None
self._cancel_idle_behavior()
def _cancel_idle_behavior(self):
if self._idle_element is not None:
self._idle_element.trigger_soft_stop()
self._idle_element = None
def _set_running_behavior(self, new_behavior):
if new_behavior == self._running_behavior:
return
self._running_behavior = new_behavior
if self.tracking_category and self.tracking_category is not ObjectRoutingBehaviorTrackingCategory.NONE:
routing_service = services.get_object_routing_service()
if routing_service:
if new_behavior:
routing_service.on_routing_start(self.owner, self.tracking_category, new_behavior)
else:
routing_service.on_routing_stop(self.owner, self.tracking_category)
@componentmethod
def get_idle_element(self):
self._idle_element = soft_sleep_forever()
return (self._idle_element, self._cancel_idle_behavior)
@componentmethod
def get_next_work(self):
if self._running_behavior is None or self.owner.has_work_locks:
return WorkRequest()
work_request = WorkRequest(work_element=self._running_behavior, required_sims=(self.owner,))
return work_request
@componentmethod
def get_next_work_priority(self):
return PriorityExtended.SubLow
@componentmethod
def on_requested_as_resource(self, other_work):
if not any(resource.is_sim for resource in other_work.resources):
return
self.restart_running_behavior()
def restart_running_behavior(self):
routing_behavior_type = type(self._running_behavior) if self._running_behavior is not None else None
self._stop_runnning_behavior()
if routing_behavior_type is not None:
routing_behavior = routing_behavior_type(self.owner)
self._set_running_behavior(routing_behavior)
def _stop_runnning_behavior(self):
if self._running_behavior is not None:
self._running_behavior.trigger_soft_stop()
self._set_running_behavior(None)
@componentmethod
def get_participant(self, participant_type=ParticipantType.Actor, **kwargs):
participants = self.get_participants(participant_type=participant_type, **kwargs)
if not participants:
return
if len(participants) > 1:
raise ValueError('Too many participants returned for {}!'.format(participant_type))
return next(iter(participants))
@componentmethod
def get_participants(self, participant_type, **kwargs):
if participant_type is ParticipantType.Actor:
obj = self._running_behavior._obj if self._running_behavior else None
return (obj,)
elif participant_type is ParticipantType.Object:
target = self._running_behavior.get_target() if self._running_behavior else None
return (target,)
class AsyncoreLoop(object):
def __init__(self):
self._pid = os.getpid()
self._loop_lock = Lock()
self._started = False
self._shutdown = False
self._conns_lock = Lock()
self._conns = WeakSet()
self._thread = None
atexit.register(partial(_cleanup, weakref.ref(self)))
def maybe_start(self):
should_start = False
did_acquire = False
try:
did_acquire = self._loop_lock.acquire(False)
if did_acquire and not self._started:
self._started = True
should_start = True
finally:
if did_acquire:
self._loop_lock.release()
if should_start:
self._thread = Thread(target=self._run_loop, name="cassandra_driver_event_loop")
self._thread.daemon = True
self._thread.start()
def _run_loop(self):
log.debug("Starting asyncore event loop")
with self._loop_lock:
while True:
try:
asyncore.loop(timeout=0.001, use_poll=True, count=1000)
except Exception:
log.debug("Asyncore event loop stopped unexepectedly", exc_info=True)
break
if self._shutdown:
break
with self._conns_lock:
if len(self._conns) == 0:
break
self._started = False
log.debug("Asyncore event loop ended")
def _cleanup(self):
self._shutdown = True
if not self._thread:
return
log.debug("Waiting for event loop thread to join...")
self._thread.join(timeout=1.0)
if self._thread.is_alive():
log.warning(
"Event loop thread could not be joined, so shutdown may not be clean. "
"Please call Cluster.shutdown() to avoid this.")
log.debug("Event loop thread was joined")
def connection_created(self, connection):
with self._conns_lock:
self._conns.add(connection)
def connection_destroyed(self, connection):
with self._conns_lock:
self._conns.discard(connection)
class TileableData(EntityData, Tileable, _ExecutableMixin):
__slots__ = '_cix', '_entities', '_executed_sessions'
_no_copy_attrs_ = SerializableWithKey._no_copy_attrs_ | {'_cix'}
# optional fields
# `nsplits` means the sizes of chunks for each dimension
_nsplits = TupleField('nsplits',
ValueType.tuple(ValueType.uint64),
on_serialize=on_serialize_nsplits)
def __init__(self, *args, **kwargs):
if kwargs.get('_nsplits', None) is not None:
kwargs['_nsplits'] = tuple(tuple(s) for s in kwargs['_nsplits'])
super().__init__(*args, **kwargs)
if hasattr(self, '_chunks') and self._chunks:
self._chunks = sorted(self._chunks, key=attrgetter('index'))
self._entities = WeakSet()
self._executed_sessions = []
@property
def chunk_shape(self):
if hasattr(self, '_nsplits') and self._nsplits is not None:
return tuple(map(len, self._nsplits))
@property
def chunks(self) -> List["Chunk"]:
return getattr(self, '_chunks', None)
@property
def nsplits(self):
return getattr(self, '_nsplits', None)
@nsplits.setter
def nsplits(self, new_nsplits):
self._nsplits = new_nsplits
@property
def params(self) -> dict:
# params return the properties which useful to rebuild a new tileable object
return dict()
@property
def cix(self):
if self.ndim == 0:
return ChunksIndexer(self)
try:
if getattr(self, '_cix', None) is None:
self._cix = ChunksIndexer(self)
return self._cix
except (TypeError, ValueError):
return ChunksIndexer(self)
@property
def entities(self):
return self._entities
def is_coarse(self):
return not hasattr(self, '_chunks') or self._chunks is None or len(
self._chunks) == 0
@enter_mode(build=True)
def attach(self, entity):
self._entities.add(entity)
@enter_mode(build=True)
def detach(self, entity):
self._entities.discard(entity)
class TestWeakSet(unittest.TestCase):
def setUp(self):
# need to keep references to them
self.items = [ustr(c) for c in ("a", "b", "c")]
self.items2 = [ustr(c) for c in ("x", "y", "z")]
self.letters = [ustr(c) for c in string.ascii_letters]
self.s = WeakSet(self.items)
self.d = dict.fromkeys(self.items)
self.obj = ustr("F")
self.fs = WeakSet([self.obj])
def test_methods(self):
weaksetmethods = dir(WeakSet)
for method in dir(set):
if method == "test_c_api" or method.startswith("_"):
continue
self.assert_(method in weaksetmethods, "WeakSet missing method " + method)
def test_new_or_init(self):
self.assertRaises(TypeError, WeakSet, [], 2)
def test_len(self):
self.assertEqual(len(self.s), len(self.d))
self.assertEqual(len(self.fs), 1)
del self.obj
self.assertEqual(len(self.fs), 0)
def test_contains(self):
for c in self.letters:
self.assertEqual(c in self.s, c in self.d)
self.assertRaises(TypeError, self.s.__contains__, [[]])
self.assert_(self.obj in self.fs)
del self.obj
self.assert_(ustr("F") not in self.fs)
def test_union(self):
u = self.s.union(self.items2)
for c in self.letters:
self.assertEqual(c in u, c in self.d or c in self.items2)
self.assertEqual(self.s, WeakSet(self.items))
self.assertEqual(type(u), WeakSet)
self.assertRaises(TypeError, self.s.union, [[]])
for C in set, frozenset, dict.fromkeys, list, tuple:
x = WeakSet(self.items + self.items2)
c = C(self.items2)
self.assertEqual(self.s.union(c), x)
def test_or(self):
i = self.s.union(self.items2)
self.assertEqual(self.s | set(self.items2), i)
self.assertEqual(self.s | frozenset(self.items2), i)
def test_intersection(self):
i = self.s.intersection(self.items2)
for c in self.letters:
self.assertEqual(c in i, c in self.d and c in self.items2)
self.assertEqual(self.s, WeakSet(self.items))
self.assertEqual(type(i), WeakSet)
for C in set, frozenset, dict.fromkeys, list, tuple:
x = WeakSet([])
self.assertEqual(self.s.intersection(C(self.items2)), x)
def test_isdisjoint(self):
self.assert_(self.s.isdisjoint(WeakSet(self.items2)))
self.assert_(not self.s.isdisjoint(WeakSet(self.letters)))
def test_and(self):
i = self.s.intersection(self.items2)
self.assertEqual(self.s & set(self.items2), i)
self.assertEqual(self.s & frozenset(self.items2), i)
def test_difference(self):
i = self.s.difference(self.items2)
for c in self.letters:
self.assertEqual(c in i, c in self.d and c not in self.items2)
self.assertEqual(self.s, WeakSet(self.items))
self.assertEqual(type(i), WeakSet)
self.assertRaises(TypeError, self.s.difference, [[]])
def test_sub(self):
i = self.s.difference(self.items2)
self.assertEqual(self.s - set(self.items2), i)
self.assertEqual(self.s - frozenset(self.items2), i)
def test_symmetric_difference(self):
i = self.s.symmetric_difference(self.items2)
for c in self.letters:
self.assertEqual(c in i, (c in self.d) ^ (c in self.items2))
self.assertEqual(self.s, WeakSet(self.items))
self.assertEqual(type(i), WeakSet)
self.assertRaises(TypeError, self.s.symmetric_difference, [[]])
def test_xor(self):
i = self.s.symmetric_difference(self.items2)
self.assertEqual(self.s ^ set(self.items2), i)
self.assertEqual(self.s ^ frozenset(self.items2), i)
def test_sub_and_super(self):
pl, ql, rl = map(lambda s: [ustr(c) for c in s], ["ab", "abcde", "def"])
p, q, r = map(WeakSet, (pl, ql, rl))
self.assert_(p < q)
self.assert_(p <= q)
self.assert_(q <= q)
self.assert_(q > p)
self.assert_(q >= p)
self.failIf(q < r)
self.failIf(q <= r)
self.failIf(q > r)
self.failIf(q >= r)
self.assert_(set("a").issubset("abc"))
self.assert_(set("abc").issuperset("a"))
self.failIf(set("a").issubset("cbs"))
self.failIf(set("cbs").issuperset("a"))
def test_gc(self):
# Create a nest of cycles to exercise overall ref count check
s = WeakSet(Foo() for i in range(1000))
for elem in s:
elem.cycle = s
elem.sub = elem
elem.set = WeakSet([elem])
def test_subclass_with_custom_hash(self):
# Bug #1257731
class H(WeakSet):
def __hash__(self):
return int(id(self) & 0x7FFFFFFF)
s = H()
f = set()
f.add(s)
self.assert_(s in f)
f.remove(s)
f.add(s)
f.discard(s)
def test_init(self):
s = WeakSet()
s.__init__(self.items)
self.assertEqual(s, self.s)
s.__init__(self.items2)
self.assertEqual(s, WeakSet(self.items2))
self.assertRaises(TypeError, s.__init__, s, 2)
self.assertRaises(TypeError, s.__init__, 1)
def test_constructor_identity(self):
s = WeakSet(self.items)
t = WeakSet(s)
self.assertNotEqual(id(s), id(t))
def test_hash(self):
self.assertRaises(TypeError, hash, self.s)
def test_clear(self):
self.s.clear()
self.assertEqual(self.s, WeakSet([]))
self.assertEqual(len(self.s), 0)
def test_copy(self):
dup = self.s.copy()
self.assertEqual(self.s, dup)
self.assertNotEqual(id(self.s), id(dup))
def test_add(self):
x = ustr("Q")
self.s.add(x)
self.assert_(x in self.s)
dup = self.s.copy()
self.s.add(x)
self.assertEqual(self.s, dup)
self.assertRaises(TypeError, self.s.add, [])
self.fs.add(Foo())
self.assert_(len(self.fs) == 1)
self.fs.add(self.obj)
self.assert_(len(self.fs) == 1)
def test_remove(self):
x = ustr("a")
self.s.remove(x)
self.assert_(x not in self.s)
self.assertRaises(KeyError, self.s.remove, x)
self.assertRaises(TypeError, self.s.remove, [])
def test_discard(self):
a, q = ustr("a"), ustr("Q")
self.s.discard(a)
self.assert_(a not in self.s)
self.s.discard(q)
self.assertRaises(TypeError, self.s.discard, [])
def test_pop(self):
for i in range(len(self.s)):
elem = self.s.pop()
self.assert_(elem not in self.s)
self.assertRaises(KeyError, self.s.pop)
def test_update(self):
retval = self.s.update(self.items2)
self.assertEqual(retval, None)
for c in self.items + self.items2:
self.assert_(c in self.s)
self.assertRaises(TypeError, self.s.update, [[]])
def test_update_set(self):
self.s.update(set(self.items2))
for c in self.items + self.items2:
self.assert_(c in self.s)
def test_ior(self):
self.s |= set(self.items2)
for c in self.items + self.items2:
self.assert_(c in self.s)
def test_intersection_update(self):
retval = self.s.intersection_update(self.items2)
self.assertEqual(retval, None)
for c in self.items + self.items2:
if c in self.items2 and c in self.items:
self.assert_(c in self.s)
else:
self.assert_(c not in self.s)
self.assertRaises(TypeError, self.s.intersection_update, [[]])
def test_iand(self):
self.s &= set(self.items2)
for c in self.items + self.items2:
if c in self.items2 and c in self.items:
self.assert_(c in self.s)
else:
self.assert_(c not in self.s)
def test_difference_update(self):
retval = self.s.difference_update(self.items2)
self.assertEqual(retval, None)
for c in self.items + self.items2:
if c in self.items and c not in self.items2:
self.assert_(c in self.s)
else:
self.assert_(c not in self.s)
self.assertRaises(TypeError, self.s.difference_update, [[]])
self.assertRaises(TypeError, self.s.symmetric_difference_update, [[]])
def test_isub(self):
self.s -= set(self.items2)
for c in self.items + self.items2:
if c in self.items and c not in self.items2:
self.assert_(c in self.s)
else:
self.assert_(c not in self.s)
def test_symmetric_difference_update(self):
retval = self.s.symmetric_difference_update(self.items2)
self.assertEqual(retval, None)
for c in self.items + self.items2:
if (c in self.items) ^ (c in self.items2):
self.assert_(c in self.s)
else:
self.assert_(c not in self.s)
self.assertRaises(TypeError, self.s.symmetric_difference_update, [[]])
def test_ixor(self):
self.s ^= set(self.items2)
for c in self.items + self.items2:
if (c in self.items) ^ (c in self.items2):
self.assert_(c in self.s)
else:
self.assert_(c not in self.s)
def test_inplace_on_self(self):
t = self.s.copy()
t |= t
self.assertEqual(t, self.s)
t &= t
self.assertEqual(t, self.s)
t -= t
self.assertEqual(t, WeakSet())
t = self.s.copy()
t ^= t
self.assertEqual(t, WeakSet())
def test_eq(self):
# issue 5964
self.assertTrue(self.s == self.s)
self.assertTrue(self.s == WeakSet(self.items))
self.assertFalse(self.s == set(self.items))
self.assertFalse(self.s == list(self.items))
self.assertFalse(self.s == tuple(self.items))
self.assertFalse(self.s == WeakSet([Foo]))
self.assertFalse(self.s == 1)
class TestWeakSet(unittest.TestCase):
def setUp(self):
# need to keep references to them
self.items = [SomeClass(c) for c in ('a', 'b', 'c')]
self.items2 = [SomeClass(c) for c in ('x', 'y', 'z')]
self.letters = [SomeClass(c) for c in string.ascii_letters]
self.ab_items = [SomeClass(c) for c in 'ab']
self.abcde_items = [SomeClass(c) for c in 'abcde']
self.def_items = [SomeClass(c) for c in 'def']
self.ab_weakset = WeakSet(self.ab_items)
self.abcde_weakset = WeakSet(self.abcde_items)
self.def_weakset = WeakSet(self.def_items)
self.s = WeakSet(self.items)
self.d = dict.fromkeys(self.items)
self.obj = SomeClass('F')
self.fs = WeakSet([self.obj])
def test_methods(self):
weaksetmethods = dir(WeakSet)
for method in dir(set):
if method == 'test_c_api' or method.startswith('_'):
continue
self.assertIn(method, weaksetmethods,
"WeakSet missing method " + method)
def test_new_or_init(self):
self.assertRaises(TypeError, WeakSet, [], 2)
def test_len(self):
self.assertEqual(len(self.s), len(self.d))
self.assertEqual(len(self.fs), 1)
del self.obj
test_support.gc_collect()
# len of weak collections is eventually consistent on
# Jython. In practice this does not matter because of the
# nature of weaksets - we cannot rely on what happens in the
# reaper thread and how it interacts with gc
self.assertIn(len(self.fs), (0, 1))
def test_contains(self):
for c in self.letters:
self.assertEqual(c in self.s, c in self.d)
# 1 is not weakref'able, but that TypeError is caught by __contains__
self.assertNotIn(1, self.s)
self.assertIn(self.obj, self.fs)
del self.obj
test_support.gc_collect()
self.assertNotIn(SomeClass('F'), self.fs)
def test_union(self):
u = self.s.union(self.items2)
for c in self.letters:
self.assertEqual(c in u, c in self.d or c in self.items2)
self.assertEqual(self.s, WeakSet(self.items))
self.assertEqual(type(u), WeakSet)
self.assertRaises(TypeError, self.s.union, [[]])
for C in set, frozenset, dict.fromkeys, list, tuple:
x = WeakSet(self.items + self.items2)
c = C(self.items2)
self.assertEqual(self.s.union(c), x)
del c
test_support.gc_collect()
self.assertEqual(len(list(u)), len(list(self.items)) + len(list(self.items2)))
self.items2.pop()
test_support.gc_collect()
self.assertEqual(len(list(u)), len(list(self.items)) + len(list(self.items2)))
def test_or(self):
i = self.s.union(self.items2)
self.assertEqual(self.s | set(self.items2), i)
self.assertEqual(self.s | frozenset(self.items2), i)
def test_intersection(self):
s = WeakSet(self.letters)
i = s.intersection(self.items2)
for c in self.letters:
self.assertEqual(c in i, c in self.items2 and c in self.letters)
self.assertEqual(s, WeakSet(self.letters))
self.assertEqual(type(i), WeakSet)
for C in set, frozenset, dict.fromkeys, list, tuple:
x = WeakSet([])
self.assertEqual(i.intersection(C(self.items)), x)
self.assertEqual(len(i), len(self.items2))
self.items2.pop()
test_support.gc_collect()
self.assertEqual(len(list(i)), len(list(self.items2)))
def test_isdisjoint(self):
self.assertTrue(self.s.isdisjoint(WeakSet(self.items2)))
self.assertTrue(not self.s.isdisjoint(WeakSet(self.letters)))
def test_and(self):
i = self.s.intersection(self.items2)
self.assertEqual(self.s & set(self.items2), i)
self.assertEqual(self.s & frozenset(self.items2), i)
def test_difference(self):
i = self.s.difference(self.items2)
for c in self.letters:
self.assertEqual(c in i, c in self.d and c not in self.items2)
self.assertEqual(self.s, WeakSet(self.items))
self.assertEqual(type(i), WeakSet)
self.assertRaises(TypeError, self.s.difference, [[]])
def test_sub(self):
i = self.s.difference(self.items2)
self.assertEqual(self.s - set(self.items2), i)
self.assertEqual(self.s - frozenset(self.items2), i)
def test_symmetric_difference(self):
i = self.s.symmetric_difference(self.items2)
for c in self.letters:
self.assertEqual(c in i, (c in self.d) ^ (c in self.items2))
self.assertEqual(self.s, WeakSet(self.items))
self.assertEqual(type(i), WeakSet)
self.assertRaises(TypeError, self.s.symmetric_difference, [[]])
self.assertEqual(len(i), len(self.items) + len(self.items2))
self.items2.pop()
test_support.gc_collect()
self.assertEqual(len(list(i)), len(list(self.items)) + len(list(self.items2)))
def test_xor(self):
i = self.s.symmetric_difference(self.items2)
self.assertEqual(self.s ^ set(self.items2), i)
self.assertEqual(self.s ^ frozenset(self.items2), i)
def test_sub_and_super(self):
self.assertTrue(self.ab_weakset <= self.abcde_weakset)
self.assertTrue(self.abcde_weakset <= self.abcde_weakset)
self.assertTrue(self.abcde_weakset >= self.ab_weakset)
self.assertFalse(self.abcde_weakset <= self.def_weakset)
self.assertFalse(self.abcde_weakset >= self.def_weakset)
self.assertTrue(set('a').issubset('abc'))
self.assertTrue(set('abc').issuperset('a'))
self.assertFalse(set('a').issubset('cbs'))
self.assertFalse(set('cbs').issuperset('a'))
def test_lt(self):
self.assertTrue(self.ab_weakset < self.abcde_weakset)
self.assertFalse(self.abcde_weakset < self.def_weakset)
self.assertFalse(self.ab_weakset < self.ab_weakset)
self.assertFalse(WeakSet() < WeakSet())
def test_gt(self):
self.assertTrue(self.abcde_weakset > self.ab_weakset)
self.assertFalse(self.abcde_weakset > self.def_weakset)
self.assertFalse(self.ab_weakset > self.ab_weakset)
self.assertFalse(WeakSet() > WeakSet())
def test_gc(self):
# Create a nest of cycles to exercise overall ref count check
s = WeakSet(Foo() for i in range(1000))
for elem in s:
elem.cycle = s
elem.sub = elem
elem.set = WeakSet([elem])
def test_subclass_with_custom_hash(self):
# Bug #1257731
class H(WeakSet):
def __hash__(self):
return int(id(self) & 0x7fffffff)
s=H()
f=set()
f.add(s)
self.assertIn(s, f)
f.remove(s)
f.add(s)
f.discard(s)
def test_init(self):
s = WeakSet()
s.__init__(self.items)
self.assertEqual(s, self.s)
s.__init__(self.items2)
self.assertEqual(s, WeakSet(self.items2))
self.assertRaises(TypeError, s.__init__, s, 2);
self.assertRaises(TypeError, s.__init__, 1);
def test_constructor_identity(self):
s = WeakSet(self.items)
t = WeakSet(s)
self.assertNotEqual(id(s), id(t))
def test_hash(self):
self.assertRaises(TypeError, hash, self.s)
def test_clear(self):
self.s.clear()
self.assertEqual(self.s, WeakSet([]))
self.assertEqual(len(self.s), 0)
def test_copy(self):
dup = self.s.copy()
self.assertEqual(self.s, dup)
self.assertNotEqual(id(self.s), id(dup))
def test_add(self):
x = SomeClass('Q')
self.s.add(x)
self.assertIn(x, self.s)
dup = self.s.copy()
self.s.add(x)
self.assertEqual(self.s, dup)
if not test_support.is_jython: # Jython/JVM can weakly reference list and other objects
self.assertRaises(TypeError, self.s.add, [])
self.fs.add(Foo())
test_support.gc_collect() # CPython assumes Foo() went out of scope and was collected, so ensure the same
self.assertEqual(len(list(self.fs)), 1)
self.fs.add(self.obj)
self.assertEqual(len(list(self.fs)), 1)
def test_remove(self):
x = SomeClass('a')
self.s.remove(x)
self.assertNotIn(x, self.s)
self.assertRaises(KeyError, self.s.remove, x)
if not test_support.is_jython: # Jython/JVM can weakly reference list and other objects
self.assertRaises(TypeError, self.s.remove, [])
def test_discard(self):
a, q = SomeClass('a'), SomeClass('Q')
self.s.discard(a)
self.assertNotIn(a, self.s)
self.s.discard(q)
if not test_support.is_jython: # Jython/JVM can weakly reference list and other objects
self.assertRaises(TypeError, self.s.discard, [])
def test_pop(self):
for i in range(len(self.s)):
elem = self.s.pop()
self.assertNotIn(elem, self.s)
self.assertRaises(KeyError, self.s.pop)
def test_update(self):
retval = self.s.update(self.items2)
self.assertEqual(retval, None)
for c in (self.items + self.items2):
self.assertIn(c, self.s)
self.assertRaises(TypeError, self.s.update, [[]])
def test_update_set(self):
self.s.update(set(self.items2))
for c in (self.items + self.items2):
self.assertIn(c, self.s)
def test_ior(self):
self.s |= set(self.items2)
for c in (self.items + self.items2):
self.assertIn(c, self.s)
def test_intersection_update(self):
retval = self.s.intersection_update(self.items2)
self.assertEqual(retval, None)
for c in (self.items + self.items2):
if c in self.items2 and c in self.items:
self.assertIn(c, self.s)
else:
self.assertNotIn(c, self.s)
self.assertRaises(TypeError, self.s.intersection_update, [[]])
def test_iand(self):
self.s &= set(self.items2)
for c in (self.items + self.items2):
if c in self.items2 and c in self.items:
self.assertIn(c, self.s)
else:
self.assertNotIn(c, self.s)
def test_difference_update(self):
retval = self.s.difference_update(self.items2)
self.assertEqual(retval, None)
for c in (self.items + self.items2):
if c in self.items and c not in self.items2:
self.assertIn(c, self.s)
else:
self.assertNotIn(c, self.s)
if not test_support.is_jython: # Jython/JVM can weakly reference list and other objects
self.assertRaises(TypeError, self.s.difference_update, [[]])
self.assertRaises(TypeError, self.s.symmetric_difference_update, [[]])
def test_isub(self):
self.s -= set(self.items2)
for c in (self.items + self.items2):
if c in self.items and c not in self.items2:
self.assertIn(c, self.s)
else:
self.assertNotIn(c, self.s)
def test_symmetric_difference_update(self):
retval = self.s.symmetric_difference_update(self.items2)
self.assertEqual(retval, None)
for c in (self.items + self.items2):
if (c in self.items) ^ (c in self.items2):
self.assertIn(c, self.s)
else:
self.assertNotIn(c, self.s)
self.assertRaises(TypeError, self.s.symmetric_difference_update, [[]])
def test_ixor(self):
self.s ^= set(self.items2)
for c in (self.items + self.items2):
if (c in self.items) ^ (c in self.items2):
self.assertIn(c, self.s)
else:
self.assertNotIn(c, self.s)
def test_inplace_on_self(self):
t = self.s.copy()
t |= t
self.assertEqual(t, self.s)
t &= t
self.assertEqual(t, self.s)
t -= t
self.assertEqual(t, WeakSet())
t = self.s.copy()
t ^= t
self.assertEqual(t, WeakSet())
def test_eq(self):
# issue 5964 (http://bugs.python.org/issue5964)
self.assertEqual(self.s, self.s)
self.assertEqual(self.s, WeakSet(self.items))
# Jython diverges here in the next test because it constructs
# WeakSet as a subclass of set; this seems to be the proper
# thing to do given what is the typical comparison
self.assertEqual(self.s, set(self.items))
self.assertNotEqual(self.s, list(self.items))
self.assertNotEqual(self.s, tuple(self.items))
self.assertNotEqual(self.s, 1)
def test_weak_destroy_while_iterating(self):
# Issue #7105: iterators shouldn't crash when a key is implicitly removed
# Create new items to be sure no-one else holds a reference
items = [SomeClass(c) for c in ('a', 'b', 'c')]
s = WeakSet(items)
it = iter(s)
next(it) # Trigger internal iteration
# Destroy an item
del items[-1]
test_support.gc_collect() # just in case
# We have removed either the first consumed items, or another one
self.assertIn(len(list(it)), [len(items), len(items) - 1])
del it
extra_collect()
# The removal has been committed
self.assertEqual(len(s), len(items))
def test_weak_destroy_and_mutate_while_iterating(self):
# Issue #7105: iterators shouldn't crash when a key is implicitly removed
items = [SomeClass(c) for c in string.ascii_letters]
s = WeakSet(items)
@contextlib.contextmanager
def testcontext():
try:
it = iter(s)
next(it)
# Schedule an item for removal and recreate it
u = SomeClass(str(items.pop()))
test_support.gc_collect() # just in case
yield u
finally:
it = None # should commit all removals
test_support.gc_collect()
with testcontext() as u:
self.assertNotIn(u, s)
with testcontext() as u:
self.assertRaises(KeyError, s.remove, u)
self.assertNotIn(u, s)
with testcontext() as u:
s.add(u)
self.assertIn(u, s)
t = s.copy()
with testcontext() as u:
s.update(t)
self.assertEqual(len(s), len(t))
with testcontext() as u:
s.clear()
self.assertEqual(len(s), 0)
def test_len_cycles(self):
N = 20
items = [RefCycle() for i in range(N)]
s = WeakSet(items)
del items
# do some gc
test_support.gc_collect()
it = iter(s)
try:
next(it)
except StopIteration:
pass
# do some gc
test_support.gc_collect()
n1 = len(s)
del it
# do some gc
test_support.gc_collect()
n2 = len(s)
# one item may be kept alive inside the iterator
self.assertIn(n1, (0, 1))
self.assertEqual(n2, 0)
@unittest.skipIf(test_support.is_jython, "GarbageCollection not deterministic in Jython")
def test_len_race(self):
# Extended sanity checks for len() in the face of cyclic collection
self.addCleanup(gc.set_threshold, *gc.get_threshold())
for th in range(1, 100):
N = 20
gc.collect(0)
gc.set_threshold(th, th, th)
items = [RefCycle() for i in range(N)]
s = WeakSet(items)
del items
# All items will be collected at next garbage collection pass
it = iter(s)
try:
next(it)
except StopIteration:
pass
n1 = len(s)
del it
n2 = len(s)
self.assertGreaterEqual(n1, 0)
self.assertLessEqual(n1, N)
self.assertGreaterEqual(n2, 0)
self.assertLessEqual(n2, n1)
class TileableData(EntityData, _ExecutableMixin):
__slots__ = "_cix", "_entities", "_executed_sessions"
_no_copy_attrs_ = Base._no_copy_attrs_ | {"_cix"}
# optional fields
# `nsplits` means the sizes of chunks for each dimension
_nsplits = TupleField(
"nsplits",
FieldTypes.tuple(FieldTypes.uint64),
on_serialize=on_serialize_nsplits,
)
def __init__(self: TileableType, *args, **kwargs):
if kwargs.get("_nsplits", None) is not None:
kwargs["_nsplits"] = tuple(tuple(s) for s in kwargs["_nsplits"])
super().__init__(*args, **kwargs)
try:
chunks = self._chunks
if chunks:
self._chunks = sorted(chunks, key=attrgetter("index"))
except AttributeError: # pragma: no cover
pass
self._entities = WeakSet()
self._executed_sessions = []
@property
def chunk_shape(self):
if hasattr(self, "_nsplits") and self._nsplits is not None:
return tuple(map(len, self._nsplits))
@property
def chunks(self) -> List[Chunk]:
return getattr(self, "_chunks", None)
@property
def nsplits(self):
return getattr(self, "_nsplits", None)
@nsplits.setter
def nsplits(self, new_nsplits):
self._nsplits = new_nsplits
@property
def params(self) -> dict:
# params return the properties which useful to rebuild a new tileable object
return dict()
@property
def cix(self):
if self.ndim == 0:
return _ChunksIndexer(self)
try:
if getattr(self, "_cix", None) is None:
self._cix = _ChunksIndexer(self)
return self._cix
except (TypeError, ValueError):
return _ChunksIndexer(self)
@property
def entities(self):
return self._entities
def is_coarse(self):
if not hasattr(self, "_chunks"):
return True
if not self._chunks:
return True
return False
def attach(self, entity):
self._entities.add(entity)
def detach(self, entity):
self._entities.discard(entity)
class WindowManager(Manager):
DIAGNOSTIC_PHANTOM_KEY = "lsp_diagnostic_phantom"
def __init__(
self,
window: sublime.Window,
workspace: ProjectFolders,
configs: WindowConfigManager,
) -> None:
self._window = window
self._configs = configs
self._sessions = WeakSet() # type: WeakSet[Session]
self._workspace = workspace
self._pending_listeners = deque() # type: Deque[AbstractViewListener]
self._listeners = WeakSet() # type: WeakSet[AbstractViewListener]
self._new_listener = None # type: Optional[AbstractViewListener]
self._new_session = None # type: Optional[Session]
self._diagnostic_phantom_set = None # type: Optional[sublime.PhantomSet]
self._panel_code_phantoms = None # type: Optional[sublime.PhantomSet]
self.total_error_count = 0
self.total_warning_count = 0
sublime.set_timeout(
functools.partial(self._update_panel_main_thread,
_NO_DIAGNOSTICS_PLACEHOLDER, []))
def get_config_manager(self) -> WindowConfigManager:
return self._configs
def on_load_project_async(self) -> None:
self.update_workspace_folders_async()
self._configs.update()
def on_post_save_project_async(self) -> None:
self.on_load_project_async()
def update_workspace_folders_async(self) -> None:
if self._workspace.update():
workspace_folders = self._workspace.get_workspace_folders()
for session in self._sessions:
session.update_folders(workspace_folders)
def enable_config_async(self, config_name: str) -> None:
self._configs.enable_config(config_name)
def disable_config_async(self, config_name: str) -> None:
self._configs.disable_config(config_name)
def open_location_async(self,
location: Location,
session_name: Optional[str],
view: sublime.View,
flags: int = 0,
group: int = -1) -> Promise[bool]:
for session in self.sessions(view):
if session_name is None or session_name == session.config.name:
return session.open_location_async(location, flags, group)
return Promise.resolve(False)
def register_listener_async(self, listener: AbstractViewListener) -> None:
set_diagnostics_count(listener.view, self.total_error_count,
self.total_warning_count)
# Update workspace folders in case the user have changed those since window was created.
# There is no currently no notification in ST that would notify about folder changes.
self.update_workspace_folders_async()
self._pending_listeners.appendleft(listener)
if self._new_listener is None:
self._dequeue_listener_async()
def unregister_listener_async(self,
listener: AbstractViewListener) -> None:
self._listeners.discard(listener)
def listeners(self) -> Generator[AbstractViewListener, None, None]:
yield from self._listeners
def listener_for_view(
self, view: sublime.View) -> Optional[AbstractViewListener]:
for listener in self.listeners():
if listener.view == view:
return listener
return None
def _dequeue_listener_async(self) -> None:
listener = None # type: Optional[AbstractViewListener]
if self._new_listener is not None:
listener = self._new_listener
# debug("re-checking listener", listener)
self._new_listener = None
else:
try:
listener = self._pending_listeners.pop()
if not listener.view.is_valid():
# debug("listener", listener, "is no longer valid")
return self._dequeue_listener_async()
# debug("adding new pending listener", listener)
self._listeners.add(listener)
except IndexError:
# We have handled all pending listeners.
self._new_session = None
return
if self._new_session:
self._sessions.add(self._new_session)
self._publish_sessions_to_listener_async(listener)
if self._new_session:
if not any(self._new_session.session_views_async()):
self._sessions.discard(self._new_session)
self._new_session.end_async()
self._new_session = None
config = self._needed_config(listener.view)
if config:
# debug("found new config for listener", listener)
self._new_listener = listener
self.start_async(config, listener.view)
else:
# debug("no new config found for listener", listener)
self._new_listener = None
self._dequeue_listener_async()
def _publish_sessions_to_listener_async(
self, listener: AbstractViewListener) -> None:
inside_workspace = self._workspace.contains(listener.view)
scheme = urllib.parse.urlparse(listener.get_uri()).scheme
for session in self._sessions:
if session.can_handle(listener.view,
scheme,
capability=None,
inside_workspace=inside_workspace):
# debug("registering session", session.config.name, "to listener", listener)
try:
listener.on_session_initialized_async(session)
except Exception as ex:
message = "failed to register session {} to listener {}".format(
session.config.name, listener)
exception_log(message, ex)
def window(self) -> sublime.Window:
return self._window
def sessions(
self,
view: sublime.View,
capability: Optional[str] = None
) -> Generator[Session, None, None]:
inside_workspace = self._workspace.contains(view)
sessions = list(self._sessions)
uri = view.settings().get("lsp_uri")
if not isinstance(uri, str):
return
scheme = urllib.parse.urlparse(uri).scheme
for session in sessions:
if session.can_handle(view, scheme, capability, inside_workspace):
yield session
def get_session(self, config_name: str,
file_path: str) -> Optional[Session]:
return self._find_session(config_name, file_path)
def _can_start_config(self, config_name: str, file_path: str) -> bool:
return not bool(self._find_session(config_name, file_path))
def _find_session(self, config_name: str,
file_path: str) -> Optional[Session]:
inside = self._workspace.contains(file_path)
for session in self._sessions:
if session.config.name == config_name and session.handles_path(
file_path, inside):
return session
return None
def _needed_config(self, view: sublime.View) -> Optional[ClientConfig]:
configs = self._configs.match_view(view)
handled = False
file_name = view.file_name()
inside = self._workspace.contains(view)
for config in configs:
handled = False
for session in self._sessions:
if config.name == session.config.name and session.handles_path(
file_name, inside):
handled = True
break
if not handled:
return config
return None
def start_async(self, config: ClientConfig,
initiating_view: sublime.View) -> None:
config = ClientConfig.from_config(config, {})
file_path = initiating_view.file_name() or ''
if not self._can_start_config(config.name, file_path):
# debug('Already starting on this window:', config.name)
return
try:
workspace_folders = sorted_workspace_folders(
self._workspace.folders, file_path)
plugin_class = get_plugin(config.name)
variables = extract_variables(self._window)
cwd = None # type: Optional[str]
if plugin_class is not None:
if plugin_class.needs_update_or_installation():
config.set_view_status(initiating_view, "installing...")
plugin_class.install_or_update()
additional_variables = plugin_class.additional_variables()
if isinstance(additional_variables, dict):
variables.update(additional_variables)
cannot_start_reason = plugin_class.can_start(
self._window, initiating_view, workspace_folders, config)
if cannot_start_reason:
config.erase_view_status(initiating_view)
message = "cannot start {}: {}".format(
config.name, cannot_start_reason)
self._configs.disable_config(config.name,
only_for_session=True)
# Continue with handling pending listeners
self._new_session = None
sublime.set_timeout_async(self._dequeue_listener_async)
return self._window.status_message(message)
cwd = plugin_class.on_pre_start(self._window, initiating_view,
workspace_folders, config)
config.set_view_status(initiating_view, "starting...")
session = Session(self, self._create_logger(config.name),
workspace_folders, config, plugin_class)
if cwd:
transport_cwd = cwd # type: Optional[str]
else:
transport_cwd = workspace_folders[
0].path if workspace_folders else None
transport_config = config.resolve_transport_config(variables)
transport = create_transport(transport_config, transport_cwd,
session)
if plugin_class:
plugin_class.on_post_start(self._window, initiating_view,
workspace_folders, config)
config.set_view_status(initiating_view, "initialize")
session.initialize_async(variables=variables,
transport=transport,
working_directory=cwd,
init_callback=functools.partial(
self._on_post_session_initialize,
initiating_view))
self._new_session = session
except Exception as e:
message = "".join((
"Failed to start {0} - disabling for this window for the duration of the current session.\n",
"Re-enable by running \"LSP: Enable Language Server In Project\" from the Command Palette.",
"\n\n--- Error: ---\n{1}")).format(config.name, str(e))
exception_log(
"Unable to start subprocess for {}".format(config.name), e)
if isinstance(e, CalledProcessError):
print("Server output:\n{}".format(
e.output.decode('utf-8', 'replace')))
self._configs.disable_config(config.name, only_for_session=True)
config.erase_view_status(initiating_view)
sublime.message_dialog(message)
# Continue with handling pending listeners
self._new_session = None
sublime.set_timeout_async(self._dequeue_listener_async)
def _on_post_session_initialize(self,
initiating_view: sublime.View,
session: Session,
is_error: bool = False) -> None:
if is_error:
session.config.erase_view_status(initiating_view)
self._new_listener = None
self._new_session = None
else:
sublime.set_timeout_async(self._dequeue_listener_async)
def _create_logger(self, config_name: str) -> Logger:
logger_map = {
"panel": PanelLogger,
"remote": RemoteLogger,
}
loggers = []
for logger_type in userprefs().log_server:
if logger_type not in logger_map:
debug(
"Invalid logger type ({}) specified for log_server settings"
.format(logger_type))
continue
loggers.append(logger_map[logger_type])
if len(loggers) == 0:
return RouterLogger() # logs nothing
elif len(loggers) == 1:
return loggers[0](self, config_name)
else:
router_logger = RouterLogger()
for logger in loggers:
router_logger.append(logger(self, config_name))
return router_logger
def handle_message_request(self, session: Session, params: Any,
request_id: Any) -> None:
view = self._window.active_view()
if view:
MessageRequestHandler(view, session, request_id, params,
session.config.name).show()
def restart_sessions_async(self) -> None:
self._end_sessions_async()
listeners = list(self._listeners)
self._listeners.clear()
for listener in listeners:
self.register_listener_async(listener)
def _end_sessions_async(self) -> None:
for session in self._sessions:
session.end_async()
self._sessions.clear()
def end_config_sessions_async(self, config_name: str) -> None:
sessions = list(self._sessions)
for session in sessions:
if session.config.name == config_name:
session.end_async()
self._sessions.discard(session)
def get_project_path(self, file_path: str) -> Optional[str]:
candidate = None # type: Optional[str]
for folder in self._workspace.folders:
if file_path.startswith(folder):
if candidate is None or len(folder) > len(candidate):
candidate = folder
return candidate
def should_present_diagnostics(self, uri: DocumentUri) -> Optional[str]:
scheme, path = parse_uri(uri)
if scheme != "file":
return None
if not self._workspace.contains(path):
return "not inside window folders"
view = self._window.active_view()
if not view:
return None
settings = view.settings()
if matches_pattern(path, settings.get("binary_file_patterns")):
return "matches a pattern in binary_file_patterns"
if matches_pattern(path, settings.get("file_exclude_patterns")):
return "matches a pattern in file_exclude_patterns"
if matches_pattern(path, settings.get("folder_exclude_patterns")):
return "matches a pattern in folder_exclude_patterns"
return None
def on_post_exit_async(self, session: Session, exit_code: int,
exception: Optional[Exception]) -> None:
self._sessions.discard(session)
for listener in self._listeners:
listener.on_session_shutdown_async(session)
if exit_code != 0 or exception:
config = session.config
msg = "".join((
"{0} exited with status code {1}. ",
"Do you want to restart it? If you choose Cancel, it will be disabled for this window for the ",
"duration of the current session. ",
"Re-enable by running \"LSP: Enable Language Server In Project\" from the Command Palette."
)).format(config.name, exit_code)
if exception:
msg += "\n\n--- Error: ---\n{}".format(str(exception))
if sublime.ok_cancel_dialog(msg, "Restart {}".format(config.name)):
for listener in self._listeners:
self.register_listener_async(listener)
else:
self._configs.disable_config(config.name,
only_for_session=True)
def plugin_unloaded(self) -> None:
"""
This is called **from the main thread** when the plugin unloads. In that case we must destroy all sessions
from the main thread. That could lead to some dict/list being mutated while iterated over, so be careful
"""
self._end_sessions_async()
def handle_server_message(self, server_name: str, message: str) -> None:
sublime.set_timeout(
lambda: log_server_message(self._window, server_name, message))
def handle_log_message(self, session: Session, params: Any) -> None:
self.handle_server_message(session.config.name,
extract_message(params))
def handle_stderr_log(self, session: Session, message: str) -> None:
self.handle_server_message(session.config.name, message)
def handle_show_message(self, session: Session, params: Any) -> None:
sublime.status_message("{}: {}".format(session.config.name,
extract_message(params)))
def update_diagnostics_panel_async(self) -> None:
to_render = [] # type: List[str]
self.total_error_count = 0
self.total_warning_count = 0
listeners = list(self._listeners)
prephantoms = [] # type: List[Tuple[int, int, str, str]]
row = 0
contributions = OrderedDict(
) # type: OrderedDict[str, List[Tuple[str, Optional[int], Optional[str], Optional[str]]]]
for session in self._sessions:
local_errors, local_warnings = session.diagnostics_manager.sum_total_errors_and_warnings_async(
)
self.total_error_count += local_errors
self.total_warning_count += local_warnings
for path, contribution in session.diagnostics_manager.diagnostics_panel_contributions_async(
):
seen = path in contributions
contributions.setdefault(path, []).extend(contribution)
if not seen:
contributions.move_to_end(path)
for path, contribution in contributions.items():
to_render.append("{}:".format(path))
row += 1
for content, offset, code, href in contribution:
to_render.append(content)
if offset is not None and code is not None and href is not None:
prephantoms.append((row, offset, code, href))
row += content.count("\n") + 1
to_render.append("") # add spacing between filenames
row += 1
for listener in listeners:
set_diagnostics_count(listener.view, self.total_error_count,
self.total_warning_count)
characters = "\n".join(to_render)
if not characters:
characters = _NO_DIAGNOSTICS_PLACEHOLDER
sublime.set_timeout(
functools.partial(self._update_panel_main_thread, characters,
prephantoms))
def _update_panel_main_thread(
self, characters: str, prephantoms: List[Tuple[int, int, str,
str]]) -> None:
panel = ensure_diagnostics_panel(self._window)
if not panel or not panel.is_valid():
return
panel.run_command("lsp_update_panel", {"characters": characters})
if self._panel_code_phantoms is None:
self._panel_code_phantoms = sublime.PhantomSet(panel, "hrefs")
phantoms = [] # type: List[sublime.Phantom]
for row, col, code, href in prephantoms:
point = panel.text_point(row, col)
region = sublime.Region(point, point)
phantoms.append(
sublime.Phantom(region, make_link(href, code),
sublime.LAYOUT_INLINE))
self._panel_code_phantoms.update(phantoms)
def show_diagnostics_panel_async(self) -> None:
if self._window.active_panel() is None:
self._window.run_command("show_panel",
{"panel": "output.diagnostics"})
class AsyncoreLoop(object):
def __init__(self):
self._pid = os.getpid()
self._loop_lock = Lock()
self._started = False
self._shutdown = False
self._conns_lock = Lock()
self._conns = WeakSet()
self._thread = None
atexit.register(partial(_cleanup, weakref.ref(self)))
def maybe_start(self):
should_start = False
did_acquire = False
try:
did_acquire = self._loop_lock.acquire(False)
if did_acquire and not self._started:
self._started = True
should_start = True
finally:
if did_acquire:
self._loop_lock.release()
if should_start:
self._thread = Thread(target=self._run_loop, name="cassandra_driver_event_loop")
self._thread.daemon = True
self._thread.start()
def _run_loop(self):
log.debug("Starting asyncore event loop")
with self._loop_lock:
while True:
try:
asyncore.loop(timeout=0.001, use_poll=True, count=1000)
except Exception:
log.debug("Asyncore event loop stopped unexepectedly", exc_info=True)
break
if self._shutdown:
break
with self._conns_lock:
if len(self._conns) == 0:
break
self._started = False
log.debug("Asyncore event loop ended")
def _cleanup(self):
self._shutdown = True
if not self._thread:
return
log.debug("Waiting for event loop thread to join...")
self._thread.join(timeout=1.0)
if self._thread.is_alive():
log.warning(
"Event loop thread could not be joined, so shutdown may not be clean. "
"Please call Cluster.shutdown() to avoid this.")
log.debug("Event loop thread was joined")
def connection_created(self, connection):
with self._conns_lock:
self._conns.add(connection)
def connection_destroyed(self, connection):
with self._conns_lock:
self._conns.discard(connection)
class Privacy(LineOfSight):
_PRIVACY_SURFACE_BLOCKING_FOOTPRINT_COST = 100000
_PRIVACY_DISCOURAGEMENT_COST = routing.get_default_discouragement_cost()
_SHOO_CONSTRAINT_RADIUS = Tunable(description='\n The radius of the constraint a Shooed Sim will attempt to route to.\n ', tunable_type=float, default=2.5)
_UNAVAILABLE_TOOLTIP = TunableLocalizedStringFactory(description='\n Tooltip displayed when an object is not accessible due to being inside\n a privacy region.\n ')
_EMBARRASSED_AFFORDANCE = TunableReference(description='\n The affordance a Sim will play when getting embarrassed by walking in\n on a privacy situation.\n ', manager=services.get_instance_manager(Types.INTERACTION))
def __init__(self, *, interaction=None, tests=None, shoo_exempt_tests=None, max_line_of_sight_radius=None, map_divisions=None, simplification_ratio=None, boundary_epsilon=None, facing_offset=None, routing_surface_only=None, shoo_constraint_radius=None, unavailable_tooltip=None, embarrassed_affordance=None, reserved_surface_space=None, vehicle_tests=None, central_object=None, post_route_affordance=None, add_to_privacy_service=True, privacy_cost_override=None, additional_exit_offsets=None, persistent_instance=False, privacy_violators=None):
super().__init__(max_line_of_sight_radius, map_divisions, simplification_ratio, boundary_epsilon)
logger.assert_raise(bool(interaction) != bool(central_object), 'Privacy must define either one of interaction or central object, and never both.')
self._max_line_of_sight_radius = max_line_of_sight_radius
self._interaction = interaction
self._tests = tests
self._shoo_exempt_tests = shoo_exempt_tests
self._privacy_constraints = []
self._allowed_sims = WeakSet()
self._disallowed_sims = WeakSet()
self._violators = WeakSet()
self._late_violators = WeakSet()
self._exempt_sims = WeakSet()
self.is_active = False
self.has_shooed = False
self.central_object = central_object
self.additional_exit_offsets = additional_exit_offsets
self._multi_surface = True
self.persistent_instance = persistent_instance
self._routing_surface_only = routing_surface_only
self._shoo_constraint_radius = shoo_constraint_radius
self._unavailable_tooltip = unavailable_tooltip
self._embarrassed_affordance = embarrassed_affordance
self._reserved_surface_space = reserved_surface_space
self._post_route_affordance = post_route_affordance
self._privacy_cost_override = privacy_cost_override
self.privacy_violators = privacy_violators
self._vehicle_tests = vehicle_tests
self._pushed_interactions = []
if add_to_privacy_service:
self.add_privacy()
@property
def shoo_constraint_radius(self):
return self._shoo_constraint_radius or self._SHOO_CONSTRAINT_RADIUS
@property
def unavailable_tooltip(self):
return self._unavailable_toolip or self._UNAVAILABLE_TOOLTIP
@property
def embarrassed_affordance(self):
return self._embarrassed_affordance or self._EMBARRASSED_AFFORDANCE
@property
def privacy_discouragement_cost(self):
return self._privacy_cost_override or self._PRIVACY_DISCOURAGEMENT_COST
@property
def interaction(self):
return self._interaction
@property
def is_active(self) -> bool:
return self._is_active
@is_active.setter
def is_active(self, value):
self._is_active = value
def _is_sim_allowed(self, sim):
if self._tests:
resolver = SingleSimResolver(sim.sim_info) if self._interaction is None else self._interaction.get_resolver(target=sim)
if self._tests and self._tests.run_tests(resolver):
return True
elif self._interaction is not None and self._interaction.can_sim_violate_privacy(sim):
return True
if self._interaction is not None and self._interaction.can_sim_violate_privacy(sim):
return True
return False
def evaluate_sim(self, sim):
if self._is_sim_allowed(sim):
self._allowed_sims.add(sim)
return True
self._disallowed_sims.add(sim)
return False
def build_privacy(self, target=None):
self.is_active = True
if self.central_object is None:
target_object = self._interaction.get_participant(ParticipantType.Object)
target_object = None if target_object.is_sim else target_object
self.central_object = target_object or (target or self._interaction.sim)
if self._routing_surface_only:
allow_object_routing_surface = True
routing_surface = self.central_object.provided_routing_surface
if routing_surface is None:
return False
else:
allow_object_routing_surface = False
routing_surface = self.central_object.routing_surface
self.generate(self.central_object.position, routing_surface, allow_object_routing_surface=allow_object_routing_surface)
for poly in self.constraint.geometry.polygon:
self._privacy_constraints.append(PolygonFootprint(poly, routing_surface=routing_surface, cost=self.privacy_discouragement_cost, footprint_type=FootprintType.FOOTPRINT_TYPE_PATH, enabled=True))
if self._reserved_surface_space is not None and target is not None:
reserved_space = self._reserved_surface_space.reserved_space
try:
polygon = _generate_single_poly_rectangle_points(target.position, target.part_owner.orientation.transform_vector(sims4.math.Vector3.Z_AXIS()), target.part_owner.orientation.transform_vector(sims4.math.Vector3.X_AXIS()), reserved_space.left, reserved_space.right, reserved_space.front, reserved_space.back)
except AttributeError as exc:
raise AttributeError('Interaction: {} is trying to reserve surface space with sim as target. Exception:{}'.format(self._interaction, exc))
routing_surface = self.central_object.provided_routing_surface
if routing_surface is None:
routing_surface = target.routing_surface
footprint_cost = self.privacy_discouragement_cost if self._reserved_surface_space.allow_routing else self._PRIVACY_SURFACE_BLOCKING_FOOTPRINT_COST
self._privacy_constraints.append(PolygonFootprint(polygon, routing_surface=routing_surface, cost=footprint_cost, footprint_type=FootprintType.FOOTPRINT_TYPE_PATH, enabled=True))
if self.privacy_violators & PrivacyViolators.SIM:
if self._interaction is not None:
self._allowed_sims.update(self._interaction.get_participants(ParticipantType.AllSims))
for sim in services.sim_info_manager().instanced_sims_gen():
if sim not in self._allowed_sims:
self.evaluate_sim(sim)
violating_sims = self.find_violating_sims()
self._exempt_sims = set([s for s in violating_sims if self.is_sim_shoo_exempt(s)])
self._cancel_unavailable_interactions(violating_sims)
self._add_overrides_and_constraints_if_needed(violating_sims)
if self.privacy_violators & PrivacyViolators.VEHICLES:
violating_vehicles = self.find_violating_vehicles()
for vehicle in violating_vehicles:
vehicle.objectrouting_component.handle_privacy_violation(self)
return True
def cleanup_privacy_instance(self):
if self.is_active:
self.is_active = False
for sim in self._allowed_sims:
self.remove_override_for_sim(sim)
for sim in self._late_violators:
self.remove_override_for_sim(sim)
del self._privacy_constraints[:]
self._allowed_sims.clear()
self._disallowed_sims.clear()
self._violators.clear()
self._late_violators.clear()
self._exempt_sims.clear()
self._cancel_pushed_interactions()
def add_privacy(self):
services.privacy_service().add_instance(self)
def remove_privacy(self):
self.cleanup_privacy_instance()
services.privacy_service().remove_instance(self)
def intersects_with_object(self, obj):
if obj.routing_surface != self.central_object.routing_surface:
return False
delta = obj.position - self.central_object.position
distance = delta.magnitude_2d_squared()
if distance > self.max_line_of_sight_radius*self.max_line_of_sight_radius:
return False
object_footprint = obj.footprint_polygon
if object_footprint is None:
object_footprint = sims4.geometry.CompoundPolygon([sims4.geometry.Polygon([obj.position])])
return self.constraint.geometry.polygon.intersects(object_footprint)
def vehicle_violates_privacy(self, vehicle):
if vehicle.objectrouting_component is None:
return False
if self._vehicle_tests is not None:
resolver = SingleObjectResolver(vehicle)
if self._vehicle_tests.run_tests(resolver):
return False
elif not self.intersects_with_object(vehicle):
return False
elif not self.intersects_with_object(vehicle):
return False
return True
def find_violating_vehicles(self):
violators = []
privacy_service = services.privacy_service()
for vehicle in privacy_service.get_potential_vehicle_violators():
if self.vehicle_violates_privacy(vehicle):
violators.append(vehicle)
return violators
def find_violating_sims(self, consider_exempt=True):
if not self.is_active:
return []
if not self.privacy_violators & PrivacyViolators.SIM:
return []
check_all_surfaces_on_level = not self._routing_surface_only
nearby_sims = placement.get_nearby_sims_gen(self.central_object.position, self._routing_surface, radius=self.max_line_of_sight_radius, exclude=self._allowed_sims, only_sim_position=True, check_all_surfaces_on_level=check_all_surfaces_on_level)
violators = []
for sim in nearby_sims:
if consider_exempt and sim in self._exempt_sims:
continue
if any(sim_primitive.is_traversing_portal() for sim_primitive in sim.primitives if isinstance(sim_primitive, FollowPath)):
continue
if sim not in self._disallowed_sims and self.evaluate_sim(sim):
continue
if sims4.geometry.test_point_in_compound_polygon(sim.position, self.constraint.geometry.polygon):
violators.append(sim)
return violators
def is_sim_shoo_exempt(self, sim):
if sim in self._exempt_sims:
return True
if self.central_object.provided_routing_surface == sim.location.routing_surface:
return False
elif self._shoo_exempt_tests:
resolver = SingleSimResolver(sim.sim_info)
if self._shoo_exempt_tests.run_tests(resolver):
return True
return False
def add_exempt_sim(self, sim):
self._exempt_sims.add(sim)
def _add_overrides_and_constraints_if_needed(self, violating_sims):
for sim in self._allowed_sims:
self.add_override_for_sim(sim)
for sim in violating_sims:
self._violators.add(sim)
if sim in self._exempt_sims:
continue
liabilities = ((SHOO_LIABILITY, ShooLiability(self, sim)),)
result = self._route_sim_away(sim, liabilities=liabilities)
if result:
self._pushed_interactions.append(result.interaction)
def _cancel_unavailable_interactions(self, violating_sims):
for sim in violating_sims:
if sim in self._exempt_sims:
continue
interactions_to_cancel = set()
if sim.queue.running is not None:
interactions_to_cancel.add(sim.queue.running)
for interaction in sim.si_state:
if interaction.is_super:
if interaction.target is not None:
if sim.locked_from_obj_by_privacy(interaction.target):
interactions_to_cancel.add(interaction)
for interaction in sim.queue:
if interaction.target is not None and sim.locked_from_obj_by_privacy(interaction.target):
interactions_to_cancel.add(interaction)
elif interaction.target is not None:
break
for interaction in interactions_to_cancel:
interaction.cancel(FinishingType.INTERACTION_INCOMPATIBILITY, cancel_reason_msg='Canceled due to incompatibility with privacy instance.')
def _route_sim_away(self, sim, liabilities=()):
context = InteractionContext(sim, InteractionContext.SOURCE_SCRIPT, Priority.High, insert_strategy=QueueInsertStrategy.NEXT)
from interactions.utils.satisfy_constraint_interaction import BuildAndForceSatisfyShooConstraintInteraction
result = sim.push_super_affordance(BuildAndForceSatisfyShooConstraintInteraction, None, context, liabilities=liabilities, privacy_inst=self, name_override='BuildShooFromPrivacy')
if result:
if self._post_route_affordance is not None:
def route_away_callback(_):
post_route_context = context.clone_for_continuation(result.interaction)
sim.push_super_affordance(self._post_route_affordance, None, post_route_context)
result.interaction.register_on_finishing_callback(route_away_callback)
else:
logger.debug('Failed to push BuildAndForceSatisfyShooConstraintInteraction on Sim {} to route them out of a privacy area. Result: {}', sim, result, owner='tastle')
if self.interaction is not None:
self.interaction.cancel(FinishingType.TRANSITION_FAILURE, cancel_reason_msg='Failed to shoo Sims away.')
return result
def _cancel_pushed_interactions(self):
for interaction in self._pushed_interactions:
interaction.cancel(FinishingType.AUTO_EXIT, cancel_reason_msg='Privacy finished and is cleaning up.')
self._pushed_interactions.clear()
def handle_late_violator(self, sim):
self._cancel_unavailable_interactions((sim,))
self.add_override_for_sim(sim)
liabilities = ((LATE_SHOO_LIABILITY, LateShooLiability(self, sim)),)
result = self._route_sim_away(sim, liabilities=liabilities)
if not result:
return
if not self._violators:
context = InteractionContext(sim, InteractionContext.SOURCE_SCRIPT, Priority.High, insert_strategy=QueueInsertStrategy.NEXT)
if self.interaction is None:
result = sim.push_super_affordance(self.embarrassed_affordance, sim, context)
else:
result = sim.push_super_affordance(self.embarrassed_affordance, self.interaction.get_participant(ParticipantType.Actor), context)
if not result and not services.sim_spawner_service().sim_is_leaving(sim):
logger.warn('Failed to push the embarrassed affordance on Sim {}. Interaction {}. Result {}. Context {} ', sim, self.interaction, result, context, owner='tastle')
return
self._late_violators.add(sim)
def add_override_for_sim(self, sim):
for footprint in self._privacy_constraints:
sim.routing_context.ignore_footprint_contour(footprint.footprint_id)
def remove_override_for_sim(self, sim):
for footprint in self._privacy_constraints:
sim.routing_context.remove_footprint_contour_override(footprint.footprint_id)
@property
def allowed_sims(self):
return self._allowed_sims
@property
def disallowed_sims(self):
return self._disallowed_sims
def remove_sim_from_allowed_disallowed(self, sim):
if sim in self._allowed_sims:
self._allowed_sims.remove(sim)
if sim in self._disallowed_sims:
self._disallowed_sims.remove(sim)
@property
def violators(self):
return self._violators
def remove_violator(self, sim):
self.remove_override_for_sim(sim)
self._violators.discard(sim)
@property
def late_violators(self):
return self._late_violators
def remove_late_violator(self, sim):
self.remove_override_for_sim(sim)
self._late_violators.discard(sim)
class Consumer(Service, ConsumerT):
"""Base Consumer."""
app: AppT
logger = logger
#: Tuple of exception types that may be raised when the
#: underlying consumer driver is stopped.
consumer_stopped_errors: ClassVar[Tuple[Type[BaseException], ...]] = ()
# Mapping of TP to list of gap in offsets.
_gap: MutableMapping[TP, List[int]]
# Mapping of TP to list of acked offsets.
_acked: MutableMapping[TP, List[int]]
#: Fast lookup to see if tp+offset was acked.
_acked_index: MutableMapping[TP, Set[int]]
#: Keeps track of the currently read offset in each TP
_read_offset: MutableMapping[TP, Optional[int]]
#: Keeps track of the currently committed offset in each TP.
_committed_offset: MutableMapping[TP, Optional[int]]
#: The consumer.wait_empty() method will set this to be notified
#: when something acks a message.
_waiting_for_ack: Optional[asyncio.Future] = None
#: Used by .commit to ensure only one thread is comitting at a time.
#: Other thread starting to commit while a commit is already active,
#: will wait for the original request to finish, and do nothing.
_commit_fut: Optional[asyncio.Future] = None
#: Set of unacked messages: that is messages that we started processing
#: and that we MUST attempt to complete processing of, before
#: shutting down or resuming a rebalance.
_unacked_messages: MutableSet[Message]
#: Time of last record batch received.
#: Set only when not set, and reset by commit() so actually
#: tracks how long it ago it was since we received a record that
#: was never committed.
_last_batch: Counter[TP]
#: Time of when the consumer was started.
_time_start: float
# How often to poll and track log end offsets.
_end_offset_monitor_interval: float
_commit_every: Optional[int]
_n_acked: int = 0
_active_partitions: Optional[Set[TP]]
_paused_partitions: Set[TP]
flow_active: bool = True
can_resume_flow: Event
def __init__(self,
transport: TransportT,
callback: ConsumerCallback,
on_partitions_revoked: PartitionsRevokedCallback,
on_partitions_assigned: PartitionsAssignedCallback,
*,
commit_interval: float = None,
commit_livelock_soft_timeout: float = None,
loop: asyncio.AbstractEventLoop = None,
**kwargs: Any) -> None:
assert callback is not None
self.transport = transport
self.app = self.transport.app
self.in_transaction = self.app.in_transaction
self.callback = callback
self._on_message_in = self.app.sensors.on_message_in
self._on_partitions_revoked = on_partitions_revoked
self._on_partitions_assigned = on_partitions_assigned
self._commit_every = self.app.conf.broker_commit_every
self.scheduler = self.app.conf.ConsumerScheduler()
self.commit_interval = (commit_interval
or self.app.conf.broker_commit_interval)
self.commit_livelock_soft_timeout = (
commit_livelock_soft_timeout
or self.app.conf.broker_commit_livelock_soft_timeout)
self._gap = defaultdict(list)
self._acked = defaultdict(list)
self._acked_index = defaultdict(set)
self._read_offset = defaultdict(lambda: None)
self._committed_offset = defaultdict(lambda: None)
self._unacked_messages = WeakSet()
self._waiting_for_ack = None
self._time_start = monotonic()
self._last_batch = Counter()
self._end_offset_monitor_interval = self.commit_interval * 2
self.randomly_assigned_topics = set()
self.can_resume_flow = Event()
self._reset_state()
super().__init__(loop=loop or self.transport.loop, **kwargs)
self.transactions = self.transport.create_transaction_manager(
consumer=self,
producer=self.app.producer,
beacon=self.beacon,
loop=self.loop,
)
def on_init_dependencies(self) -> Iterable[ServiceT]:
"""Return list of services this consumer depends on."""
# We start the TransactionManager only if
# processing_guarantee='exactly_once'
if self.in_transaction:
return [self.transactions]
return []
def _reset_state(self) -> None:
self._active_partitions = None
self._paused_partitions = set()
self.can_resume_flow.clear()
self.flow_active = True
self._last_batch.clear()
self._time_start = monotonic()
async def on_restart(self) -> None:
"""Call when the consumer is restarted."""
self._reset_state()
self.on_init()
def _get_active_partitions(self) -> Set[TP]:
tps = self._active_partitions
if tps is None:
return self._set_active_tps(self.assignment())
assert all(isinstance(x, TP) for x in tps)
return tps
def _set_active_tps(self, tps: Set[TP]) -> Set[TP]:
xtps = self._active_partitions = ensure_TPset(tps) # copy
xtps.difference_update(self._paused_partitions)
return xtps
@abc.abstractmethod
async def _commit(self, offsets: Mapping[TP,
int]) -> bool: # pragma: no cover
...
async def perform_seek(self) -> None:
"""Seek all partitions to their current committed position."""
read_offset = self._read_offset
_committed_offsets = await self.seek_to_committed()
read_offset.update({
tp: offset if offset is not None and offset >= 0 else None
for tp, offset in _committed_offsets.items()
})
committed_offsets = {
ensure_TP(tp): offset if offset else None
for tp, offset in _committed_offsets.items() if offset is not None
}
self._committed_offset.update(committed_offsets)
@abc.abstractmethod
async def seek_to_committed(self) -> Mapping[TP, int]:
"""Seek all partitions to their committed offsets."""
...
async def seek(self, partition: TP, offset: int) -> None:
"""Seek partition to specific offset."""
self.log.dev('SEEK %r -> %r', partition, offset)
# reset livelock detection
self._last_batch.pop(partition, None)
await self._seek(partition, offset)
# set new read offset so we will reread messages
self._read_offset[ensure_TP(partition)] = offset if offset else None
@abc.abstractmethod
async def _seek(self, partition: TP, offset: int) -> None:
...
def stop_flow(self) -> None:
"""Block consumer from processing any more messages."""
self.flow_active = False
self.can_resume_flow.clear()
def resume_flow(self) -> None:
"""Allow consumer to process messages."""
self.flow_active = True
self.can_resume_flow.set()
def pause_partitions(self, tps: Iterable[TP]) -> None:
"""Pause fetching from partitions."""
tpset = ensure_TPset(tps)
self._get_active_partitions().difference_update(tpset)
self._paused_partitions.update(tpset)
def resume_partitions(self, tps: Iterable[TP]) -> None:
"""Resume fetching from partitions."""
tpset = ensure_TPset(tps)
self._get_active_partitions().update(tps)
self._paused_partitions.difference_update(tpset)
@abc.abstractmethod
def _new_topicpartition(self, topic: str,
partition: int) -> TP: # pragma: no cover
...
def _is_changelog_tp(self, tp: TP) -> bool:
return tp.topic in self.app.tables.changelog_topics
@Service.transitions_to(CONSUMER_PARTITIONS_REVOKED)
async def on_partitions_revoked(self, revoked: Set[TP]) -> None:
"""Call during rebalancing when partitions are being revoked."""
# NOTE:
# The ConsumerRebalanceListener is responsible for calling
# app.on_rebalance_start(), and this must have happened
# before we get to this point (see aiokafka implementation).
span = self.app._start_span_from_rebalancing('on_partitions_revoked')
T = traced_from_parent_span(span)
with span:
# see comment in on_partitions_assigned
# remove revoked partitions from active + paused tps.
if self._active_partitions is not None:
self._active_partitions.difference_update(revoked)
self._paused_partitions.difference_update(revoked)
await T(self._on_partitions_revoked, partitions=revoked)(revoked)
@Service.transitions_to(CONSUMER_PARTITIONS_ASSIGNED)
async def on_partitions_assigned(self, assigned: Set[TP]) -> None:
"""Call during rebalancing when partitions are being assigned."""
span = self.app._start_span_from_rebalancing('on_partitions_assigned')
T = traced_from_parent_span(span)
with span:
# remove recently revoked tps from set of paused tps.
self._paused_partitions.intersection_update(assigned)
# cache set of assigned partitions
self._set_active_tps(assigned)
# start callback chain of assigned callbacks.
# need to copy set at this point, since we cannot have
# the callbacks mutate our active list.
self._last_batch.clear()
await T(self._on_partitions_assigned,
partitions=assigned)(assigned)
self.app.on_rebalance_return()
@abc.abstractmethod
async def _getmany(self, active_partitions: Optional[Set[TP]],
timeout: float) -> RecordMap:
...
async def getmany(self,
timeout: float) -> AsyncIterator[Tuple[TP, Message]]:
"""Fetch batch of messages from server."""
# records' contain mapping from TP to list of messages.
# if there are two agents, consuming from topics t1 and t2,
# normal order of iteration would be to process each
# tp in the dict:
# for tp. messages in records.items():
# for message in messages:
# yield tp, message
#
# The problem with this, is if we have prefetched 16k records
# for one partition, the other partitions won't even start processing
# before those 16k records are completed.
#
# So we try round-robin between the tps instead:
#
# iterators: Dict[TP, Iterator] = {
# tp: iter(messages)
# for tp, messages in records.items()
# }
# while iterators:
# for tp, messages in iterators.items():
# yield tp, next(messages)
# # remove from iterators if empty.
#
# The problem with this implementation is that
# the records mapping is ordered by TP, so records.keys()
# will look like this:
#
# TP(topic='bar', partition=0)
# TP(topic='bar', partition=1)
# TP(topic='bar', partition=2)
# TP(topic='bar', partition=3)
# TP(topic='foo', partition=0)
# TP(topic='foo', partition=1)
# TP(topic='foo', partition=2)
# TP(topic='foo', partition=3)
#
# If there are 100 partitions for each topic,
# it will process 100 items in the first topic, then 100 items
# in the other topic, but even worse if partition counts
# vary greatly, t1 has 1000 partitions and t2
# has 1 partition, then t2 will end up being starved most of the time.
#
# We solve this by going round-robin through each topic.
records, active_partitions = await self._wait_next_records(timeout)
if records is None or self.should_stop:
return
records_it = self.scheduler.iterate(records)
to_message = self._to_message # localize
if self.flow_active:
for tp, record in records_it:
if not self.flow_active:
break
if active_partitions is None or tp in active_partitions:
highwater_mark = self.highwater(tp)
self.app.monitor.track_tp_end_offset(tp, highwater_mark)
# convert timestamp to seconds from int milliseconds.
yield tp, to_message(tp, record)
async def _wait_next_records(
self,
timeout: float) -> Tuple[Optional[RecordMap], Optional[Set[TP]]]:
if not self.flow_active:
await self.wait(self.can_resume_flow)
# Implementation for the Fetcher service.
is_client_only = self.app.client_only
active_partitions: Optional[Set[TP]]
if is_client_only:
active_partitions = None
else:
active_partitions = self._get_active_partitions()
records: RecordMap = {}
if is_client_only or active_partitions:
# Fetch records only if active partitions to avoid the risk of
# fetching all partitions in the beginning when none of the
# partitions is paused/resumed.
records = await self._getmany(
active_partitions=active_partitions,
timeout=timeout,
)
else:
# We should still release to the event loop
await self.sleep(1)
return records, active_partitions
@abc.abstractmethod
def _to_message(self, tp: TP, record: Any) -> ConsumerMessage:
...
def track_message(self, message: Message) -> None:
"""Track message and mark it as pending ack."""
# add to set of pending messages that must be acked for graceful
# shutdown. This is called by transport.Conductor,
# before delivering messages to streams.
self._unacked_messages.add(message)
# call sensors
self._on_message_in(message.tp, message.offset, message)
def ack(self, message: Message) -> bool:
"""Mark message as being acknowledged by stream."""
if not message.acked:
message.acked = True
tp = message.tp
offset = message.offset
if self.app.topics.acks_enabled_for(message.topic):
committed = self._committed_offset[tp]
try:
if committed is None or offset > committed:
acked_index = self._acked_index[tp]
if offset not in acked_index:
self._unacked_messages.discard(message)
acked_index.add(offset)
acked_for_tp = self._acked[tp]
acked_for_tp.append(offset)
self._n_acked += 1
return True
finally:
notify(self._waiting_for_ack)
return False
async def _wait_for_ack(self, timeout: float) -> None:
# arm future so that `ack()` can wake us up
self._waiting_for_ack = asyncio.Future(loop=self.loop)
try:
# wait for `ack()` to wake us up
await asyncio.wait_for(self._waiting_for_ack,
loop=self.loop,
timeout=1)
except (asyncio.TimeoutError,
asyncio.CancelledError): # pragma: no cover
pass
finally:
self._waiting_for_ack = None
@Service.transitions_to(CONSUMER_WAIT_EMPTY)
async def wait_empty(self) -> None:
"""Wait for all messages that started processing to be acked."""
wait_count = 0
T = traced_from_parent_span()
while not self.should_stop and self._unacked_messages:
wait_count += 1
if not wait_count % 10: # pragma: no cover
remaining = [(m.refcount, m) for m in self._unacked_messages]
self.log.warning('wait_empty: Waiting for %r tasks', remaining)
self.log.dev('STILL WAITING FOR ALL STREAMS TO FINISH')
self.log.dev('WAITING FOR %r EVENTS', len(self._unacked_messages))
gc.collect()
await T(self.commit)()
if not self._unacked_messages:
break
await T(self._wait_for_ack)(timeout=1)
self.log.dev('COMMITTING AGAIN AFTER STREAMS DONE')
await T(self.commit_and_end_transactions)()
async def commit_and_end_transactions(self) -> None:
"""Commit all safe offsets and end transaction."""
await self.commit(start_new_transaction=False)
async def on_stop(self) -> None:
"""Call when consumer is stopping."""
if self.app.conf.stream_wait_empty:
await self.wait_empty()
else:
await self.commit_and_end_transactions()
self._last_batch.clear()
@Service.task
async def _commit_handler(self) -> None:
interval = self.commit_interval
await self.sleep(interval)
async for sleep_time in self.itertimer(interval, name='commit'):
await self.commit()
@Service.task
async def _commit_livelock_detector(self) -> None: # pragma: no cover
soft_timeout = self.commit_livelock_soft_timeout
interval: float = self.commit_interval * 2.5
acks_enabled_for = self.app.topics.acks_enabled_for
await self.sleep(interval)
async for sleep_time in self.itertimer(interval, name='livelock'):
for tp, last_batch_time in self._last_batch.items():
if last_batch_time and acks_enabled_for(tp.topic):
s_since_batch = monotonic() - last_batch_time
if s_since_batch > soft_timeout:
self.log.warning(
'Possible livelock: '
'COMMIT OFFSET NOT ADVANCING FOR %r', tp)
async def commit(self,
topics: TPorTopicSet = None,
start_new_transaction: bool = True) -> bool:
"""Maybe commit the offset for all or specific topics.
Arguments:
topics: Set containing topics and/or TopicPartitions to commit.
"""
if self.app.client_only:
# client only cannot commit as consumer does not have group_id
return False
if await self.maybe_wait_for_commit_to_finish():
# original commit finished, return False as we did not commit
return False
self._commit_fut = asyncio.Future(loop=self.loop)
try:
return await self.force_commit(
topics,
start_new_transaction=start_new_transaction,
)
finally:
# set commit_fut to None so that next call will commit.
fut, self._commit_fut = self._commit_fut, None
# notify followers that the commit is done.
notify(fut)
async def maybe_wait_for_commit_to_finish(self) -> bool:
"""Wait for any existing commit operation to finish."""
# Only one coroutine allowed to commit at a time,
# and other coroutines should wait for the original commit to finish
# then do nothing.
if self._commit_fut is not None:
# something is already committing so wait for that future.
try:
await self._commit_fut
except asyncio.CancelledError:
# if future is cancelled we have to start new commit
pass
else:
return True
return False
@Service.transitions_to(CONSUMER_COMMITTING)
async def force_commit(self,
topics: TPorTopicSet = None,
start_new_transaction: bool = True) -> bool:
"""Force offset commit."""
sensor_state = self.app.sensors.on_commit_initiated(self)
# Go over the ack list in each topic/partition
commit_tps = list(self._filter_tps_with_pending_acks(topics))
did_commit = await self._commit_tps(
commit_tps, start_new_transaction=start_new_transaction)
self.app.sensors.on_commit_completed(self, sensor_state)
return did_commit
async def _commit_tps(self, tps: Iterable[TP],
start_new_transaction: bool) -> bool:
commit_offsets = self._filter_committable_offsets(tps)
if commit_offsets:
try:
# send all messages attached to the new offset
await self._handle_attached(commit_offsets)
except ProducerSendError as exc:
await self.crash(exc)
else:
return await self._commit_offsets(
commit_offsets,
start_new_transaction=start_new_transaction)
return False
def _filter_committable_offsets(self, tps: Iterable[TP]) -> Dict[TP, int]:
commit_offsets = {}
for tp in tps:
# Find the latest offset we can commit in this tp
offset = self._new_offset(tp)
# check if we can commit to this offset
if offset is not None and self._should_commit(tp, offset):
commit_offsets[tp] = offset
return commit_offsets
async def _handle_attached(self, commit_offsets: Mapping[TP, int]) -> None:
for tp, offset in commit_offsets.items():
app = cast(_App, self.app)
attachments = app._attachments
producer = app.producer
# Start publishing the messages and return a list of pending
# futures.
pending = await attachments.publish_for_tp_offset(tp, offset)
# then we wait for either
# 1) all the attached messages to be published, or
# 2) the producer crashing
#
# If the producer crashes we will not be able to send any messages
# and it only crashes when there's an irrecoverable error.
#
# If we cannot commit it means the events will be processed again,
# so conforms to at-least-once semantics.
if pending:
await producer.wait_many(pending)
async def _commit_offsets(self,
offsets: Mapping[TP, int],
start_new_transaction: bool = True) -> bool:
table = terminal.logtable(
[(str(tp), str(offset)) for tp, offset in offsets.items()],
title='Commit Offsets',
headers=['TP', 'Offset'],
)
self.log.dev('COMMITTING OFFSETS:\n%s', table)
assignment = self.assignment()
committable_offsets: Dict[TP, int] = {}
revoked: Dict[TP, int] = {}
for tp, offset in offsets.items():
if tp in assignment:
committable_offsets[tp] = offset
else:
revoked[tp] = offset
if revoked:
self.log.info(
'Discarded commit for revoked partitions that '
'will be eventually processed again: %r',
revoked,
)
if not committable_offsets:
return False
with flight_recorder(self.log, timeout=300.0) as on_timeout:
did_commit = False
on_timeout.info('+consumer.commit()')
if self.in_transaction:
did_commit = await self.transactions.commit(
committable_offsets,
start_new_transaction=start_new_transaction,
)
else:
did_commit = await self._commit(committable_offsets)
on_timeout.info('-consumer.commit()')
if did_commit:
on_timeout.info('+tables.on_commit')
self.app.tables.on_commit(committable_offsets)
on_timeout.info('-tables.on_commit')
self._committed_offset.update(committable_offsets)
self.app.monitor.on_tp_commit(committable_offsets)
for tp in offsets:
self._last_batch.pop(tp, None)
return did_commit
def _filter_tps_with_pending_acks(self,
topics: TPorTopicSet = None
) -> Iterator[TP]:
return (tp for tp in self._acked
if topics is None or tp in topics or tp.topic in topics)
def _should_commit(self, tp: TP, offset: int) -> bool:
committed = self._committed_offset[tp]
return committed is None or bool(offset) and offset > committed
def _new_offset(self, tp: TP) -> Optional[int]:
# get the new offset for this tp, by going through
# its list of acked messages.
acked = self._acked[tp]
# We iterate over it until we find a gap
# then return the offset before that.
# For example if acked[tp] is:
# 1 2 3 4 5 6 7 8 9
# the return value will be: 9
# If acked[tp] is:
# 34 35 36 40 41 42 43 44
# ^--- gap
# the return value will be: 36
if acked:
max_offset = max(acked)
gap_for_tp = self._gap[tp]
if gap_for_tp:
gap_index = next(
(i for i, x in enumerate(gap_for_tp) if x > max_offset),
len(gap_for_tp))
gaps = gap_for_tp[:gap_index]
acked.extend(gaps)
gap_for_tp[:gap_index] = []
acked.sort()
# Note: acked is always kept sorted.
# find first list of consecutive numbers
batch = next(consecutive_numbers(acked))
# remove them from the list to clean up.
acked[:len(batch) - 1] = []
self._acked_index[tp].difference_update(batch)
# return the highest commit offset
return batch[-1]
return None
async def on_task_error(self, exc: BaseException) -> None:
"""Call when processing a message failed."""
await self.commit()
def _add_gap(self, tp: TP, offset_from: int, offset_to: int) -> None:
committed = self._committed_offset[tp]
gap_for_tp = self._gap[tp]
for offset in range(offset_from, offset_to):
if committed is None or offset > committed:
gap_for_tp.append(offset)
async def _drain_messages(self,
fetcher: ServiceT) -> None: # pragma: no cover
# This is the background thread started by Fetcher, used to
# constantly read messages using Consumer.getmany.
# It takes Fetcher as argument, because we must be able to
# stop it using `await Fetcher.stop()`.
callback = self.callback
getmany = self.getmany
consumer_should_stop = self._stopped.is_set
fetcher_should_stop = fetcher._stopped.is_set
get_read_offset = self._read_offset.__getitem__
set_read_offset = self._read_offset.__setitem__
get_commit_offset = self._committed_offset.__getitem__
flag_consumer_fetching = CONSUMER_FETCHING
set_flag = self.diag.set_flag
unset_flag = self.diag.unset_flag
commit_every = self._commit_every
acks_enabled_for = self.app.topics.acks_enabled_for
try:
while not (consumer_should_stop() or fetcher_should_stop()):
set_flag(flag_consumer_fetching)
ait = cast(AsyncIterator, getmany(timeout=5.0))
last_batch = self._last_batch
# Sleeping because sometimes getmany is called in a loop
# never releasing to the event loop
await self.sleep(0)
if not self.should_stop:
async for tp, message in ait:
offset = message.offset
r_offset = get_read_offset(tp)
committed_offset = get_commit_offset(tp)
if committed_offset != r_offset:
last_batch[tp] = monotonic()
if r_offset is None or offset > r_offset:
gap = offset - (r_offset or 0)
# We have a gap in income messages
if gap > 1 and r_offset:
acks_enabled = acks_enabled_for(message.topic)
if acks_enabled:
self._add_gap(tp, r_offset + 1, offset)
if commit_every is not None:
if self._n_acked >= commit_every:
self._n_acked = 0
await self.commit()
await callback(message)
set_read_offset(tp, offset)
else:
self.log.dev('DROPPED MESSAGE ROFF %r: k=%r v=%r',
offset, message.key, message.value)
unset_flag(flag_consumer_fetching)
except self.consumer_stopped_errors:
if self.transport.app.should_stop:
# we're already stopping so ignore
self.log.info('Broker stopped consumer, shutting down...')
return
raise
except asyncio.CancelledError:
if self.transport.app.should_stop:
# we're already stopping so ignore
self.log.info('Consumer shutting down for user cancel.')
return
raise
except Exception as exc:
self.log.exception('Drain messages raised: %r', exc)
raise
finally:
unset_flag(flag_consumer_fetching)
def close(self) -> None:
"""Close consumer for graceful shutdown."""
...
@property
def unacked(self) -> Set[Message]:
"""Return the set of currently unacknowledged messages."""
return cast(Set[Message], self._unacked_messages)
class Consumer(Service, ConsumerT):
"""Base Consumer."""
app: AppT
logger = logger
#: Tuple of exception types that may be raised when the
#: underlying consumer driver is stopped.
consumer_stopped_errors: ClassVar[Tuple[Type[BaseException], ...]] = ()
# Mapping of TP to list of acked offsets.
_acked: MutableMapping[TP, List[int]]
#: Fast lookup to see if tp+offset was acked.
_acked_index: MutableMapping[TP, Set[int]]
#: Keeps track of the currently read offset in each TP
_read_offset: MutableMapping[TP, Optional[int]]
#: Keeps track of the currently commited offset in each TP.
_committed_offset: MutableMapping[TP, Optional[int]]
#: The consumer.wait_empty() method will set this to be notified
#: when something acks a message.
_waiting_for_ack: Optional[asyncio.Future] = None
#: Used by .commit to ensure only one thread is comitting at a time.
#: Other thread starting to commit while a commit is already active,
#: will wait for the original request to finish, and do nothing.
_commit_fut: Optional[asyncio.Future] = None
#: Set of unacked messages: that is messages that we started processing
#: and that we MUST attempt to complete processing of, before
#: shutting down or resuming a rebalance.
_unacked_messages: MutableSet[Message]
#: Time of last record batch received.
#: Set only when not set, and reset by commit() so actually
#: tracks how long it ago it was since we received a record that
#: was never committed.
_last_batch: Optional[float]
#: Time of when the consumer was started.
_time_start: float
# How often to poll and track log end offsets.
_end_offset_monitor_interval: float
_commit_every: Optional[int]
_n_acked: int = 0
def __init__(self,
transport: TransportT,
callback: ConsumerCallback,
on_partitions_revoked: PartitionsRevokedCallback,
on_partitions_assigned: PartitionsAssignedCallback,
*,
commit_interval: float = None,
commit_livelock_soft_timeout: float = None,
loop: asyncio.AbstractEventLoop = None,
**kwargs: Any) -> None:
assert callback is not None
self.transport = transport
self.app = self.transport.app
self.callback = callback
self._on_message_in = self.app.sensors.on_message_in
self._on_partitions_revoked = on_partitions_revoked
self._on_partitions_assigned = on_partitions_assigned
self._commit_every = self.app.conf.broker_commit_every
self.commit_interval = (commit_interval
or self.app.conf.broker_commit_interval)
self.commit_livelock_soft_timeout = (
commit_livelock_soft_timeout
or self.app.conf.broker_commit_livelock_soft_timeout)
self._acked = defaultdict(list)
self._acked_index = defaultdict(set)
self._read_offset = defaultdict(lambda: None)
self._committed_offset = defaultdict(lambda: None)
self._unacked_messages = WeakSet()
self._waiting_for_ack = None
self._time_start = monotonic()
self._last_batch = None
self._end_offset_monitor_interval = self.commit_interval * 2
self.randomly_assigned_topics = set()
super().__init__(loop=loop or self.transport.loop, **kwargs)
@abc.abstractmethod
async def _commit(
self,
offsets: Mapping[TP, Tuple[int, str]]) -> bool: # pragma: no cover
...
@abc.abstractmethod
def _new_topicpartition(self, topic: str,
partition: int) -> TP: # pragma: no cover
...
def _is_changelog_tp(self, tp: TP) -> bool:
return tp.topic in self.app.tables.changelog_topics
@Service.transitions_to(CONSUMER_PARTITIONS_ASSIGNED)
async def on_partitions_assigned(self, assigned: Set[TP]) -> None:
await self._on_partitions_assigned(assigned)
@Service.transitions_to(CONSUMER_PARTITIONS_REVOKED)
async def on_partitions_revoked(self, revoked: Set[TP]) -> None:
await self._on_partitions_revoked(revoked)
def track_message(self, message: Message) -> None:
# add to set of pending messages that must be acked for graceful
# shutdown. This is called by transport.Conductor,
# before delivering messages to streams.
self._unacked_messages.add(message)
# call sensors
self._on_message_in(message.tp, message.offset, message)
def ack(self, message: Message) -> bool:
if not message.acked:
message.acked = True
tp = message.tp
offset = message.offset
if self.app.topics.acks_enabled_for(message.topic):
committed = self._committed_offset[tp]
try:
if committed is None or offset > committed:
acked_index = self._acked_index[tp]
if offset not in acked_index:
self._unacked_messages.discard(message)
acked_index.add(offset)
acked_for_tp = self._acked[tp]
acked_for_tp.append(offset)
self._n_acked += 1
return True
finally:
notify(self._waiting_for_ack)
return False
@Service.transitions_to(CONSUMER_WAIT_EMPTY)
async def wait_empty(self) -> None:
"""Wait for all messages that started processing to be acked."""
wait_count = 0
while not self.should_stop and self._unacked_messages:
wait_count += 1
if not wait_count % 100_000: # pragma: no cover
remaining = [(m.refcount, m) for m in self._unacked_messages]
self.log.warn(f'Waiting for {remaining}')
self.log.dev('STILL WAITING FOR ALL STREAMS TO FINISH')
self.log.dev('WAITING FOR %r EVENTS', len(self._unacked_messages))
gc.collect()
await self.commit()
if not self._unacked_messages:
break
# arm future so that `ack()` can wake us up
self._waiting_for_ack = asyncio.Future(loop=self.loop)
try:
# wait for `ack()` to wake us up
await asyncio.wait_for(self._waiting_for_ack,
loop=self.loop,
timeout=1)
except (asyncio.TimeoutError,
asyncio.CancelledError): # pragma: no cover
pass
finally:
self._waiting_for_ack = None
self.log.dev('COMMITTING AGAIN AFTER STREAMS DONE')
await self.commit()
async def on_stop(self) -> None:
if self.app.conf.stream_wait_empty:
await self.wait_empty()
self._last_batch = None
@Service.task
async def _commit_handler(self) -> None:
await self.sleep(self.commit_interval)
while not self.should_stop:
await self.commit()
await self.sleep(self.commit_interval)
@Service.task
async def _commit_livelock_detector(self) -> None: # pragma: no cover
soft_timeout = self.commit_livelock_soft_timeout
interval: float = self.commit_interval * 2.5
await self.sleep(interval)
while not self.should_stop:
if self._last_batch is not None:
s_since_batch = monotonic() - self._last_batch
if s_since_batch > soft_timeout:
self.log.warn(
'Possible livelock: COMMIT OFFSET NOT ADVANCING')
await self.sleep(interval)
async def commit(self,
topics: TPorTopicSet = None) -> bool: # pragma: no cover
"""Maybe commit the offset for all or specific topics.
Arguments:
topics: Set containing topics and/or TopicPartitions to commit.
"""
if await self.maybe_wait_for_commit_to_finish():
# original commit finished, return False as we did not commit
return False
self._commit_fut = asyncio.Future(loop=self.loop)
try:
return await self.force_commit(topics)
finally:
# set commit_fut to None so that next call will commit.
fut, self._commit_fut = self._commit_fut, None
# notify followers that the commit is done.
if fut is not None and not fut.done():
fut.set_result(None)
async def maybe_wait_for_commit_to_finish(self) -> bool:
# Only one coroutine allowed to commit at a time,
# and other coroutines should wait for the original commit to finish
# then do nothing.
if self._commit_fut is not None:
# something is already committing so wait for that future.
try:
await self._commit_fut
except asyncio.CancelledError:
# if future is cancelled we have to start new commit
pass
else:
return True
return False
@Service.transitions_to(CONSUMER_COMMITTING)
async def force_commit(self, topics: TPorTopicSet = None) -> bool:
sensor_state = self.app.sensors.on_commit_initiated(self)
# Go over the ack list in each topic/partition
commit_tps = list(self._filter_tps_with_pending_acks(topics))
did_commit = await self._commit_tps(commit_tps)
self.app.sensors.on_commit_completed(self, sensor_state)
return did_commit
async def _commit_tps(self, tps: Iterable[TP]) -> bool:
commit_offsets = self._filter_committable_offsets(tps)
if commit_offsets:
try:
# send all messages attached to the new offset
await self._handle_attached(commit_offsets)
except ProducerSendError as exc:
await self.crash(exc)
else:
return await self._commit_offsets(commit_offsets)
return False
def _filter_committable_offsets(self, tps: Iterable[TP]) -> Dict[TP, int]:
commit_offsets = {}
for tp in tps:
# Find the latest offset we can commit in this tp
offset = self._new_offset(tp)
# check if we can commit to this offset
if offset is not None and self._should_commit(tp, offset):
commit_offsets[tp] = offset
return commit_offsets
async def _handle_attached(self, commit_offsets: Mapping[TP, int]) -> None:
for tp, offset in commit_offsets.items():
app = cast(App, self.app)
attachments = app._attachments
producer = app.producer
# Start publishing the messages and return a list of pending
# futures.
pending = await attachments.publish_for_tp_offset(tp, offset)
# then we wait for either
# 1) all the attached messages to be published, or
# 2) the producer crashing
#
# If the producer crashes we will not be able to send any messages
# and it only crashes when there's an irrecoverable error.
#
# If we cannot commit it means the events will be processed again,
# so conforms to at-least-once semantics.
if pending:
await producer.wait_many(pending)
async def _commit_offsets(self, commit_offsets: Mapping[TP, int]) -> bool:
meta = ''
return await self._commit(
{tp: (offset, meta)
for tp, offset in commit_offsets.items()})
def _filter_tps_with_pending_acks(self,
topics: TPorTopicSet = None
) -> Iterator[TP]:
return (tp for tp in self._acked
if topics is None or tp in topics or tp.topic in topics)
def _should_commit(self, tp: TP, offset: int) -> bool:
committed = self._committed_offset[tp]
return committed is None or bool(offset) and offset > committed
def _new_offset(self, tp: TP) -> Optional[int]:
# get the new offset for this tp, by going through
# its list of acked messages.
acked = self._acked[tp]
# We iterate over it until we find a gap
# then return the offset before that.
# For example if acked[tp] is:
# 1 2 3 4 5 6 7 8 9
# the return value will be: 9
# If acked[tp] is:
# 34 35 36 40 41 42 43 44
# ^--- gap
# the return value will be: 36
if acked:
acked.sort()
# Note: acked is always kept sorted.
# find first list of consecutive numbers
batch = next(consecutive_numbers(acked))
# remove them from the list to clean up.
acked[:len(batch)] = []
self._acked_index[tp].difference_update(batch)
# return the highest commit offset
return batch[-1]
return None
async def on_task_error(self, exc: BaseException) -> None:
await self.commit()
async def _drain_messages(self,
fetcher: ServiceT) -> None: # pragma: no cover
# This is the background thread started by Fetcher, used to
# constantly read messages using Consumer.getmany.
# It takes Fetcher as argument, because we must be able to
# stop it using `await Fetcher.stop()`.
callback = self.callback
getmany = self.getmany
consumer_should_stop = self._stopped.is_set
fetcher_should_stop = fetcher._stopped.is_set
get_read_offset = self._read_offset.__getitem__
set_read_offset = self._read_offset.__setitem__
flag_consumer_fetching = CONSUMER_FETCHING
set_flag = self.diag.set_flag
unset_flag = self.diag.unset_flag
commit_every = self._commit_every
try:
while not (consumer_should_stop() or fetcher_should_stop()):
set_flag(flag_consumer_fetching)
ait = cast(AsyncIterator, getmany(timeout=5.0))
# Sleeping because sometimes getmany is called in a loop
# never releasing to the event loop
await self.sleep(0)
if not self.should_stop:
async for tp, message in ait:
offset = message.offset
r_offset = get_read_offset(tp)
if r_offset is None or offset > r_offset:
if commit_every is not None:
if self._n_acked >= commit_every:
self._n_acked = 0
await self.commit()
await callback(message)
set_read_offset(tp, offset)
else:
self.log.dev('DROPPED MESSAGE ROFF %r: k=%r v=%r',
offset, message.key, message.value)
unset_flag(flag_consumer_fetching)
except self.consumer_stopped_errors:
if self.transport.app.should_stop:
# we're already stopping so ignore
self.log.info('Broker stopped consumer, shutting down...')
return
raise
except asyncio.CancelledError:
if self.transport.app.should_stop:
# we're already stopping so ignore
self.log.info('Consumer shutting down for user cancel.')
return
raise
except Exception as exc:
self.log.exception('Drain messages raised: %r', exc)
raise
finally:
unset_flag(flag_consumer_fetching)
def close(self) -> None:
...
@property
def unacked(self) -> Set[Message]:
return cast(Set[Message], self._unacked_messages)
class ConnectionPool(object):
"""A pool of :class:`psycopg2:connection` objects.
.. attribute:: minconn
The minimum number of connections to keep in the pool. By default one
connection is opened when the pool is created.
.. attribute:: maxconn
The maximum number of connections in the pool. By default the pool will
attempt to open as many connections as requested.
.. attribute:: idle_timeout
How many seconds to keep an idle connection before closing it. The
default value causes idle connections to be closed after 10 minutes
(approximately, it depends on :meth:`putconn` being called).
.. attribute:: connect_kwargs
The keyword arguments to pass to :func:`psycopg2.connect`. If the `dsn`
argument isn't specified, then it's set to an empty string by default.
The following attributes are internal, they're documented here to provide
insight into how the pool works.
.. attribute:: connections_in_use
The set of connections that have been checked out of the pool through
:meth:`getconn`. Type: :class:`weakref.WeakSet`.
.. attribute:: idle_connections
The pool of unused connections, last in first out.
Type: :class:`collections.deque`.
.. attribute:: return_times
A timestamp is stored in this dict when a connection is added to
:attr:`.idle_connections`. That timestamp is used in :meth:`getconn` to
compute how long the connection stayed idle in the pool.
Type: :class:`dict`.
This class provides two main methods (:meth:`getconn` and :meth:`putconn`),
plus another one that you probably don't need (:meth:`clear`).
"""
__slots__ = ('minconn', 'maxconn', 'idle_timeout', 'connect_kwargs',
'idle_connections', 'connections_in_use', 'return_times',
'__dict__')
def __init__(self,
minconn=1,
maxconn=float('inf'),
idle_timeout=600,
**connect_kwargs):
self.minconn = minconn
self.maxconn = maxconn
self.idle_timeout = idle_timeout
connect_kwargs.setdefault('dsn', '')
self.connect_kwargs = connect_kwargs
self.connections_in_use = WeakSet()
self.idle_connections = deque()
self.return_times = {}
for i in range(self.minconn):
self._connect()
def _connect(self, for_immediate_use=False):
"""Open a new connection.
"""
conn = psycopg2._connect(**self.connect_kwargs)
if for_immediate_use:
self.connections_in_use.add(conn)
else:
self.return_times[conn] = uptime()
self.idle_connections.append(conn)
return conn
def getconn(self):
"""Get a connection from the pool.
If there is no idle connection available, then a new one is opened;
unless there are already :attr:`.maxconn` connections open, then a
:class:`PoolError` exception is raised.
Any connection that is broken, or has been idle for more than
:attr:`.idle_timeout` seconds, is closed and discarded.
"""
while True:
try:
# Attempt to take an idle connection from the pool.
conn = self.idle_connections.pop()
except IndexError:
# We don't have any idle connection available, open a new one.
if len(self.connections_in_use) >= self.maxconn:
raise PoolError("connection pool exhausted")
conn = self._connect(for_immediate_use=True)
else:
# Close and discard the connection if it's broken or too old.
idle_since = self.return_times.pop(conn, 0)
close = (conn.info.transaction_status !=
_ext.TRANSACTION_STATUS_IDLE
or self.idle_timeout and idle_since <
(uptime() - self.idle_timeout))
if close:
conn.close()
continue
break
return conn
def putconn(self, conn):
"""Return a connection to the pool.
You should always return a connection to the pool, even if you've closed
it. That being said, the pool only holds weak references to connections
returned by :meth:`getconn`, so they should be garbage collected even if
you fail to return them.
"""
self.connections_in_use.discard(conn)
# Determine if the connection should be kept or discarded.
current_time = uptime()
if self.idle_timeout == 0 and len(
self.idle_connections) >= self.minconn:
conn.close()
else:
status = conn.info.transaction_status
if status == _ext.TRANSACTION_STATUS_UNKNOWN:
# The connection is broken, discard it.
conn.close()
else:
if status != _ext.TRANSACTION_STATUS_IDLE:
# The connection is still in a transaction, roll it back.
conn.rollback()
self.return_times[conn] = current_time
self.idle_connections.append(conn)
# Clean up the idle connections.
if self.idle_timeout:
# We cap the number of iterations to ensure that we don't end up in
# an infinite loop.
for i in range(len(self.idle_connections)):
try:
conn = self.idle_connections[0]
except IndexError:
break
return_time = self.return_times.get(conn)
if return_time is None:
# The connection's return time is missing, give up.
break
if return_time < (current_time - self.idle_timeout):
# This connection has been idle too long, attempt to drop it.
try:
popped_conn = self.idle_connections.popleft()
except IndexError:
# Another thread removed this connection from the queue.
continue
if popped_conn == conn:
# Okay, we can close and discard this connection.
self.return_times.pop(conn, None)
conn.close()
else:
# We got a different connection, put it back.
self.idle_connections.appendleft(popped_conn)
continue
else:
# The leftmost connection isn't too old, so we can assume
# that the other ones aren't either.
break
# Open new connections if we've dropped below minconn.
while (len(self.idle_connections) +
len(self.connections_in_use)) < self.minconn:
self._connect()
def clear(self):
"""Close and discard all idle connections in the pool (regardless of the
values of :attr:`.minconn` and :attr:`.idle_timeout`).
This method could be useful if you have periods of high activity that
result in many connections being opened, followed by prolonged periods
with zero activity (no calls to :meth:`getconn` or :meth:`putconn`),
*and* you care about closing those extraneous connections during the
inactivity period. It's up to you to call this method in that case.
Alternatively you may want to run a cron task to `close idle connections
from the server <https://stackoverflow.com/a/30769511/>`_.
"""
for conn in list(self.idle_connections):
try:
self.idle_connections.remove(conn)
except ValueError:
continue
self.return_times.pop(conn, None)
conn.close()
class SwitchboardManager(gobject.GObject):
"""Switchboard management
@undocumented: do_get_property, do_set_property
@group Handlers: _handle_*, _default_handler, _error_handler"""
__gsignals__ = {
"handler-created": (gobject.SIGNAL_RUN_FIRST,
gobject.TYPE_NONE,
(object, object))
}
def __init__(self, client):
"""Initializer
@param client: the main Client instance"""
gobject.GObject.__init__(self)
self._client = weakref.proxy(client)
self._reset()
self._handlers_class = set()
self._client._protocol.connect("switchboard-invitation-received",
self._ns_switchboard_invite)
def _reset(self):
self._switchboards = {}
self._orphaned_switchboards = set()
self._requested_switchboards = {}
self._pending_switchboards = {}
self._orphaned_handlers = WeakSet()
def close(self):
for switchboard in self._orphaned_switchboards:
switchboard.leave()
for switchboard in self._pending_switchboards:
switchboard.leave()
for switchboard in self._switchboards:
switchboard.leave()
self._reset()
def register_handler_class(self, handler_class, *extra_arguments):
self._handlers_class.add((handler_class, extra_arguments))
def register_handler(self, handler):
self._orphaned_handlers.add(handler)
def request_switchboard(self, handler, priority=99):
handler_participants = handler.total_participants
participants = ", ".join(map(lambda c: c.account, handler_participants))
logger.info("Requesting switchboard for participant(s) %s" % participants)
# If the Handler was orphan, then it is no more
self._orphaned_handlers.discard(handler)
# Check already open switchboards
for switchboard in self._switchboards.keys():
switchboard_participants = set(switchboard.participants.values())
if handler_participants == switchboard_participants and \
(switchboard.state == msnp.ProtocolState.OPEN or \
switchboard.state == msnp.ProtocolState.OPENING):
logger.info("Using already opened switchboard %s" %
switchboard.session_id)
self._switchboards[switchboard].add(handler)
handler._switchboard = switchboard
return
# Check Orphaned switchboards
for switchboard in list(self._orphaned_switchboards):
switchboard_participants = set(switchboard.participants.values())
if handler_participants == switchboard_participants and \
(switchboard.state == msnp.ProtocolState.OPEN or \
switchboard.state == msnp.ProtocolState.OPENING):
logger.info("Using orphaned switchboard %s" %
switchboard.session_id)
self._switchboards[switchboard] = set([handler]) #FIXME: WeakSet ?
self._orphaned_switchboards.discard(switchboard)
handler._switchboard = switchboard
return
# Check pending switchboards
for switchboard, handlers in self._pending_switchboards.iteritems():
pending_handler = handlers.pop()
handlers.add(pending_handler)
switchboard_participants = pending_handler.total_participants
if handler_participants == switchboard_participants:
self._pending_switchboards[switchboard].add(handler)
logger.info("Using pending switchboard")
return
# Check switchboards being requested for same participants
if participants in self._requested_switchboards:
self._requested_switchboards[participants].add(handler)
logger.info("Using already requested switchboard for same contacts")
return
logger.info("Requesting new switchboard")
self._requested_switchboards[participants] = set([handler])
self._client._protocol.request_switchboard(priority,
(self._ns_switchboard_request_response, participants))
def close_handler(self, handler):
logger.info("Closing switchboard handler %s" % repr(handler))
self._orphaned_handlers.discard(handler)
handler._on_closed()
for switchboard in self._switchboards.keys():
handlers = self._switchboards[switchboard]
handlers.discard(handler)
if len(handlers) == 0:
switchboard.leave()
del self._switchboards[switchboard]
self._orphaned_switchboards.add(switchboard)
for switchboard in self._pending_switchboards.keys():
handlers = self._pending_switchboards[switchboard]
handlers.discard(handler)
if len(handlers) == 0:
del self._pending_switchboards[switchboard]
self._orphaned_switchboards.add(switchboard)
def _ns_switchboard_request_response(self, session, participants):
switchboard = self._build_switchboard(session)
handlers = self._requested_switchboards.pop(participants, set())
self._pending_switchboards[switchboard] = handlers
def _ns_switchboard_invite(self, protocol, session, inviter):
switchboard = self._build_switchboard(session)
self._orphaned_switchboards.add(switchboard)
def _build_switchboard(self, session):
server, session_id, key = session
client = self._client
proxies = client._proxies
transport_class = client._transport_class
transport = transport_class(server, ServerType.SWITCHBOARD, proxies)
switchboard = msnp.SwitchboardProtocol(client, transport,
session_id, key, proxies)
switchboard.connect("notify::state", self._sb_state_changed)
switchboard.connect("message-received", self._sb_message_received)
transport.establish_connection()
return switchboard
def _sb_state_changed(self, switchboard, param_spec):
state = switchboard.state
if state == msnp.ProtocolState.OPEN:
self._switchboards[switchboard] = set() #FIXME: WeakSet ?
# Requested switchboards
if switchboard in self._pending_switchboards:
handlers = self._pending_switchboards[switchboard]
while True:
try:
handler = handlers.pop()
self._switchboards[switchboard].add(handler)
handler._switchboard = switchboard
except KeyError:
break
del self._pending_switchboards[switchboard]
# Orphaned Handlers
for handler in list(self._orphaned_handlers):
switchboard_participants = set(switchboard.participants.values())
handler_participants = handler.total_participants
if handler_participants == switchboard_participants:
self._switchboards[switchboard].add(handler)
self._orphaned_handlers.discard(handler)
self._orphaned_switchboards.discard(switchboard)
handler._switchboard = switchboard
# no one wants it, it is an orphan
if len(self._switchboards[switchboard]) == 0:
del self._switchboards[switchboard]
self._orphaned_switchboards.add(switchboard)
elif state == msnp.ProtocolState.CLOSED:
if switchboard in self._switchboards.keys():
for handler in self._switchboards[switchboard]:
self._orphaned_handlers.add(handler)
handler._on_switchboard_closed()
del self._switchboards[switchboard]
self._orphaned_switchboards.discard(switchboard)
def _sb_message_received(self, switchboard, message):
switchboard_participants = set(switchboard.participants.values())
# Get current handlers for this switchboard
if switchboard in self._switchboards.keys():
handlers = self._switchboards[switchboard]
handlers_class = [type(handler) for handler in handlers]
elif switchboard in list(self._orphaned_switchboards):
handlers = set() #FIXME: WeakSet ?
handlers_class = []
self._switchboards[switchboard] = handlers
else:
logger.warning("Message received on unknown switchboard")
return
# Signal message to existing handlers
for handler in list(handlers):
if not handler._can_handle_message(message, handler):
continue
handler._on_message_received(message)
return
# Create first handler that could handle this message
for handler_class, extra_args in self._handlers_class:
if not handler_class._can_handle_message(message):
continue
handler = handler_class.handle_message(self._client,
message, *extra_args)
if handler is None:
continue
self._orphaned_handlers.discard(handler)
self._orphaned_switchboards.discard(switchboard)
handlers.add(handler)
handler._switchboard = switchboard
self.emit("handler-created", handler_class, handler)
handler._on_message_received(message)
return