def setUp(self):
# Set a very small check interval, this will make it more likely
# that the interpreter crashes when threading is done incorrectly.
if sys.version_info[:2] >= (3, 2):
self._int = sys.getswitchinterval()
sys.setswitchinterval(0.0000001)
else:
self._int = sys.getcheckinterval()
sys.setcheckinterval(1)
def test_trashcan_threads(self):
# Issue #13992: trashcan mechanism should be thread-safe
NESTING = 60
N_THREADS = 2
def sleeper_gen():
"""A generator that releases the GIL when closed or dealloc'ed."""
try:
yield
finally:
time.sleep(0.000001)
class C(list):
# Appending to a list is atomic, which avoids the use of a lock.
inits = []
dels = []
def __init__(self, alist):
self[:] = alist
C.inits.append(None)
def __del__(self):
# This __del__ is called by subtype_dealloc().
C.dels.append(None)
# `g` will release the GIL when garbage-collected. This
# helps assert subtype_dealloc's behaviour when threads
# switch in the middle of it.
g = sleeper_gen()
next(g)
# Now that __del__ is finished, subtype_dealloc will proceed
# to call list_dealloc, which also uses the trashcan mechanism.
def make_nested():
"""Create a sufficiently nested container object so that the
trashcan mechanism is invoked when deallocating it."""
x = C([])
for i in range(NESTING):
x = [C([x])]
del x
def run_thread():
"""Exercise make_nested() in a loop."""
while not exit:
make_nested()
old_switchinterval = sys.getswitchinterval()
sys.setswitchinterval(1e-5)
try:
exit = []
threads = []
for i in range(N_THREADS):
t = threading.Thread(target=run_thread)
threads.append(t)
with start_threads(threads, lambda: exit.append(1)):
time.sleep(1.0)
finally:
sys.setswitchinterval(old_switchinterval)
gc.collect()
self.assertEqual(len(C.inits), len(C.dels))
def tidy_up():
"""
Put the bytecode back to how it was. Good as new.
"""
if sys.version_info[0] < 3:
sys.setcheckinterval(old_check_interval)
else:
sys.setswitchinterval(old_check_interval)
for i in range(3):
pb[where + i][0] = orig_bytes[i]
def test_main():
old_switchinterval = None
try:
old_switchinterval = sys.getswitchinterval()
sys.setswitchinterval(1e-5)
except AttributeError:
pass
try:
run_unittest(ThreadedImportTests)
finally:
if old_switchinterval is not None:
sys.setswitchinterval(old_switchinterval)
def release_priority_execution(p):
sys.setcheckinterval(100)
if sys.version_info > (3, 0, 0):
sys.setswitchinterval(0.005)
try:
if platform.system() == "Windows":
p.nice(psutil.NORMAL_PRIORITY_CLASS)
else:
p.nice(5)
except psutil.AccessDenied:
pass
gc.enable()
def test_thread_safety_during_modification(self):
hosts = range(100)
policy = RoundRobinPolicy()
policy.populate(None, hosts)
errors = []
def check_query_plan():
try:
for i in xrange(100):
list(policy.make_query_plan())
except Exception as exc:
errors.append(exc)
def host_up():
for i in xrange(1000):
policy.on_up(randint(0, 99))
def host_down():
for i in xrange(1000):
policy.on_down(randint(0, 99))
threads = []
for i in range(5):
threads.append(Thread(target=check_query_plan))
threads.append(Thread(target=host_up))
threads.append(Thread(target=host_down))
# make the GIL switch after every instruction, maximizing
# the chance of race conditions
check = six.PY2 or '__pypy__' in sys.builtin_module_names
if check:
original_interval = sys.getcheckinterval()
else:
original_interval = sys.getswitchinterval()
try:
if check:
sys.setcheckinterval(0)
else:
sys.setswitchinterval(0.0001)
map(lambda t: t.start(), threads)
map(lambda t: t.join(), threads)
finally:
if check:
sys.setcheckinterval(original_interval)
else:
sys.setswitchinterval(original_interval)
if errors:
self.fail("Saw errors: %s" % (errors,))
def test_pending_calls_race(self):
# Issue #14406: multi-threaded race condition when waiting on all
# futures.
event = threading.Event()
def future_func():
event.wait()
oldswitchinterval = sys.getswitchinterval()
sys.setswitchinterval(1e-6)
try:
fs = {self.executor.submit(future_func) for i in range(100)}
event.set()
futures.wait(fs, return_when=futures.ALL_COMPLETED)
finally:
sys.setswitchinterval(oldswitchinterval)
def test_switchinterval(self):
self.assertRaises(TypeError, sys.setswitchinterval)
self.assertRaises(TypeError, sys.setswitchinterval, "a")
self.assertRaises(ValueError, sys.setswitchinterval, -1.0)
self.assertRaises(ValueError, sys.setswitchinterval, 0.0)
orig = sys.getswitchinterval()
# sanity check
self.assertTrue(orig < 0.5, orig)
try:
for n in 0.00001, 0.05, 3.0, orig:
sys.setswitchinterval(n)
self.assertAlmostEqual(sys.getswitchinterval(), n)
finally:
sys.setswitchinterval(orig)
def test_main():
old_switchinterval = None
# Issue #15599: FreeBSD/KVM cannot handle gil_interval == 1.
new_switchinterval = 0.00001 if 'freebsd' in sys.platform else 0.00000001
try:
old_switchinterval = sys.getswitchinterval()
sys.setswitchinterval(new_switchinterval)
except AttributeError:
pass
try:
run_unittest(ThreadedImportTests)
finally:
if old_switchinterval is not None:
sys.setswitchinterval(old_switchinterval)
def setUp(self):
"""
Reduce the CPython check interval so that thread switches happen much
more often, hopefully exercising more possible race conditions. Also,
delay actual test startup until the reactor has been started.
"""
if _PY3:
if getattr(sys, 'getswitchinterval', None) is not None:
self.addCleanup(sys.setswitchinterval, sys.getswitchinterval())
sys.setswitchinterval(0.0000001)
else:
if getattr(sys, 'getcheckinterval', None) is not None:
self.addCleanup(sys.setcheckinterval, sys.getcheckinterval())
sys.setcheckinterval(7)
def test_enumerate_after_join(self):
# Try hard to trigger #1703448: a thread is still returned in
# threading.enumerate() after it has been join()ed.
enum = threading.enumerate
old_interval = sys.getswitchinterval()
try:
for i in range(1, 100):
sys.setswitchinterval(i * 0.0002)
t = threading.Thread(target=lambda: None)
t.start()
t.join()
l = enum()
self.assertNotIn(t, l, "#1703448 triggered after %d trials: %s" % (i, l))
finally:
sys.setswitchinterval(old_interval)
def request_priority_execution():
gc.disable()
sys.setcheckinterval(999999999)
if sys.version_info > (3, 0, 0):
sys.setswitchinterval(1000)
p = psutil.Process(os.getpid())
try:
if platform.system() == "Windows":
p.nice(psutil.HIGH_PRIORITY_CLASS)
else:
p.nice(10)
except psutil.AccessDenied:
pass
return p
def test_lru_cache_threaded(self):
n, m = 5, 11
class C:
def orig(self, x, y):
return 3 * x + y
instance = C()
f = per_instance_lru_cache(maxsize=n*m)(C.orig)
hits, misses, maxsize, currsize = f.cache_info(instance)
self.assertEqual(currsize, 0)
start = threading.Event()
def full(k):
start.wait(10)
for _ in range(m):
self.assertEqual(f(instance, k, 0), instance.orig(k, 0))
def clear():
start.wait(10)
for _ in range(2*m):
f.cache_clear(instance)
orig_si = sys.getswitchinterval()
sys.setswitchinterval(1e-6)
try:
# create n threads in order to fill cache
threads = [threading.Thread(target=full, args=[k]) for k in range(n)]
with support.start_threads(threads):
start.set()
hits, misses, maxsize, currsize = f.cache_info(instance)
# XXX: Why can be not equal?
self.assertLessEqual(misses, n)
self.assertLessEqual(hits, m*n - misses)
self.assertEqual(currsize, n)
# create n threads in order to fill cache and 1 to clear it
threads = [threading.Thread(target=clear)]
threads += [threading.Thread(target=full, args=[k]) for k in range(n)]
start.clear()
with support.start_threads(threads):
start.set()
finally:
sys.setswitchinterval(orig_si)
def test_enumerate_after_join(self):
# Try hard to trigger #1703448: a thread is still returned in
# threading.enumerate() after it has been join()ed.
enum = threading.enumerate
old_interval = sys.getswitchinterval()
try:
for i in range(1, 100):
sys.setswitchinterval(i * 0.0002)
t = threading.Thread(target=lambda: None)
t.start()
t.join()
l = enum()
self.assertNotIn(
t, l, "#1703448 triggered after %d trials: %s" % (i, l))
finally:
sys.setswitchinterval(old_interval)
def test_pending_calls_race(self):
# Issue #14406: multi-threaded race condition when waiting on all
# futures.
event = threading.Event()
def future_func():
event.wait()
oldswitchinterval = sys.getswitchinterval()
sys.setswitchinterval(1e-6)
try:
fs = {self.executor.submit(future_func) for i in range(100)}
event.set()
futures.wait(fs, return_when=futures.ALL_COMPLETED)
finally:
sys.setswitchinterval(oldswitchinterval)
def __init__(self):
self.interval = 1/60.0
sys.setcheckinterval(1000)
try:
sys.setswitchinterval(self.interval)
except AttributeError:
# sys.setswitchinterval is only python3
pass
self.taskQueue = asynctaskqueue.AsyncTaskQueue()
self.pendingTasks = []
self.threads = []
self.timer = TimerCallback(callback=self._onTimer, targetFps=1/self.interval)
# call timer.start here to initialize the QTimer now on the main thread
self.timer.start()
def test_tasklet_with_schedule(self):
# make sure that we get enough tick counting
hold = sys.getswitchinterval()
self.addCleanup(sys.setswitchinterval, hold)
sys.setswitchinterval(0.001)
n1 = self.run_tasklets(runtask)
n2 = self.run_tasklets(runtask2)
sys.setswitchinterval(hold)
if self.verbose:
print()
print(20 * "*", "runtask:", n1, "runtask2:", n2)
if not self.softSchedule:
self.assertGreater(n1, n2)
else:
self.assertLess(n1, n2)
async def test_concurrent_access(
session_data: typing.Dict[str, typing.Any],
session_source: typing.IO[bytes],
fs_backend: FSBackend,
event_loop: asyncio.AbstractEventLoop,
):
"""Test concurrent access of mutltple backends to the same data source."""
executor = futures.ThreadPoolExecutor(max_workers=2)
# Prepare new session backend
path = Path(session_source.name)
new_fs_backend = await FSBackend.create(path.name, event_loop)
# Prepare modified session data
testing_key = list(session_data.keys())[0]
await new_fs_backend.delete(testing_key)
# Try to run multiple tasks as quick as possible
# Configuring a thread execution
sw_interval = sys.getswitchinterval()
target_interval = 1e-12
# Greatly improve the chance of an operation being interrupted
# by thread switch, thus testing synchronization effectively.
# Feel free to tweak the parameters below to see their impact.
# see: https://gist.github.com/mRcfps/0af4f1cb29ffe27cf3aa05d542ac742a
sys.setswitchinterval(target_interval)
# Run coroutine functions in two separate threads
coros = [
# Try to load unmodified data
fs_backend.load(),
# Try to save modified data
new_fs_backend.save(),
new_fs_backend.load(),
]
await asyncio.gather(*coros, loop=event_loop)
# Check concurrent access
assert testing_key in fs_backend
assert testing_key not in new_fs_backend
# Update fs_backend state
await fs_backend.load()
assert testing_key not in fs_backend
sys.setswitchinterval(sw_interval)
def test_threaded():
go = threading.Event()
item = CachedCostItemWait(go)
num_threads = 3
orig_si = sys.getswitchinterval()
sys.setswitchinterval(1e-6)
try:
tpr = concurrent.futures.ThreadPoolExecutor(max_workers=num_threads, thread_name_prefix="test")
futures = [tpr.submit(lambda: item.cost) for _ in range(num_threads)]
_, not_done = concurrent.futures.wait(futures)
# "Threads not stopped"
assert len(not_done) == 0
finally:
sys.setswitchinterval(orig_si)
assert item.cost == 2
def run(self):
"""Run the sampler to make a measurement of the current stack frames.
"""
measure_tool = self.get_measure_tool()
# Muck with switch interval to prevent thread context switching while
# trying to capture profiling information for safety
switch_interval = sys.getswitchinterval()
try:
logging.debug('Process sampling')
sys.setswitchinterval(10000)
for dummy_frame_id, frame in (sys._current_frames( # pylint: disable=protected-access
).items()):
self.my_db.record(measure_tool(frame))
finally:
sys.setswitchinterval(switch_interval)
logging.debug('Switch interval now %.2f', sys.getswitchinterval())
def __init__(self,
graph: DeltaGraph,
lvl: int = logging.ERROR,
msg_lvl: int = logging.ERROR,
switchinterval: float = None,
queue_size: int = 16,
queue_interval: float = 1.0):
self.log = make_logger(lvl, "DeltaPySimulator")
self.msg_log = MessageLog(msg_lvl)
self.set_excepthook()
# speed optimization
if switchinterval is not None:
sys.setswitchinterval(switchinterval)
self.queue_size = queue_size
self.queue_interval = queue_interval
# the graph
self.graph = graph
self.graph.do_automatic_splitting()
self.graph.check()
self.add_message_log()
# i/o queues
self.in_queues: Dict[str, Dict[str, DeltaQueue]] = {
node.name: {}
for node in self.graph.nodes
}
self.out_queues: Dict[str, Dict[str, DeltaQueue]] = {
node.name: {}
for node in self.graph.nodes
}
for node in self.graph.nodes:
self._create_io_queues(node)
# Signal to stop child threads
self.sig_stop = threading.Event()
for node in self.graph.nodes:
node.set_communications(self)
# child threads for node's workers
self.threads: Dict[str, threading.Thread] = {}
self.running = False
def frequent_thread_switches():
"""Make concurrency bugs more likely to manifest."""
interval = None
if not sys.platform.startswith('java'):
if hasattr(sys, 'getswitchinterval'):
interval = sys.getswitchinterval()
sys.setswitchinterval(1e-6)
else:
interval = sys.getcheckinterval()
sys.setcheckinterval(1)
try:
yield
finally:
if not sys.platform.startswith('java'):
if hasattr(sys, 'setswitchinterval'):
sys.setswitchinterval(interval)
else:
sys.setcheckinterval(interval)
def frequent_thread_switches():
"""Make concurrency bugs more likely to manifest."""
interval = None
if not sys.platform.startswith('java'):
if hasattr(sys, 'getswitchinterval'):
interval = sys.getswitchinterval()
sys.setswitchinterval(1e-6)
else:
interval = sys.getcheckinterval()
sys.setcheckinterval(1)
try:
yield
finally:
if not sys.platform.startswith('java'):
if hasattr(sys, 'setswitchinterval'):
sys.setswitchinterval(interval)
else:
sys.setcheckinterval(interval)
def reconfig():
""" Set global PVM configs """
cfg = {}
cfg['sys.platform'] = sys.platform
cfg['sys.maxsize'] = sys.maxsize
cfg['sys.path'] = sys.path
cfg['sys.excepthook'] = sys.excepthook
cfg['old sys.switchinterval'] = sys.getswitchinterval()
sys.setswitchinterval(LONGER_CHECK_INTERVAL)
cfg['new sys.switchinterval'] = sys.getswitchinterval()
cfg['old sys.recursionlimit'] = sys.getrecursionlimit()
sys.setrecursionlimit(BIGGER_RECURSION_LIMIT)
cfg['new sys.recursionlimit'] = sys.getrecursionlimit()
cfg['old gc.threshold'] = str(gc.get_threshold())
gc.set_threshold(*LOWER_GC_THRESHOLD)
cfg['new gc.threshold'] = str(gc.get_threshold())
sys._clear_type_cache()
cfg['sys._clear_type_cache'] = True
return cfg
def adjust_balance_concurrently(self, account):
def transact():
account.deposit(5)
time.sleep(0.001)
account.withdraw(5)
# Greatly improve the chance of an operation being interrupted
# by thread switch, thus testing synchronization effectively
try:
sys.setswitchinterval(1e-12)
except AttributeError:
# For Python 2 compatibility
sys.setcheckinterval(1)
threads = [threading.Thread(target=transact) for _ in range(1000)]
for thread in threads:
thread.start()
for thread in threads:
thread.join()
def setup_gil():
"""
Set extremely long GIL release interval to let threads naturally progress
through CPU-heavy sequences without forcing the wake of another thread that
may contend trying to run the same CPU-heavy code. For the new-style work,
this drops runtime ~33% and involuntary context switches by >80%,
essentially making threads cooperatively scheduled.
"""
try:
# Python 2.
sys.setcheckinterval(100000)
except AttributeError:
pass
try:
# Python 3.
sys.setswitchinterval(10)
except AttributeError:
pass
def adjust_balance_concurrently(self, account):
def transact():
account.deposit(5)
time.sleep(0.001)
account.withdraw(5)
# Greatly improve the chance of an operation being interrupted
# by thread switch, thus testing synchronization effectively
try:
sys.setswitchinterval(1e-12)
except AttributeError:
# For Python 2 compatibility
sys.setcheckinterval(1)
threads = [threading.Thread(target=transact) for _ in range(1000)]
for thread in threads:
thread.start()
for thread in threads:
thread.join()
def test_is_alive_after_fork(self):
# Try hard to trigger #18418: is_alive() could sometimes be True on
# threads that vanished after a fork.
old_interval = sys.getswitchinterval()
self.addCleanup(sys.setswitchinterval, old_interval)
# Make the bug more likely to manifest.
sys.setswitchinterval(1e-6)
for i in range(20):
t = threading.Thread(target=lambda: None)
t.start()
self.addCleanup(t.join)
pid = os.fork()
if pid == 0:
os._exit(1 if t.is_alive() else 0)
else:
pid, status = os.waitpid(pid, 0)
self.assertEqual(0, status)
def test_is_alive_after_fork(self):
# Try hard to trigger #18418: is_alive() could sometimes be True on
# threads that vanished after a fork.
old_interval = sys.getswitchinterval()
self.addCleanup(sys.setswitchinterval, old_interval)
# Make the bug more likely to manifest.
sys.setswitchinterval(1e-6)
for i in range(20):
t = threading.Thread(target=lambda: None)
t.start()
self.addCleanup(t.join)
pid = os.fork()
if pid == 0:
os._exit(1 if t.is_alive() else 0)
else:
pid, status = os.waitpid(pid, 0)
self.assertEqual(0, status)
def setup_gil():
"""
Set extremely long GIL release interval to let threads naturally progress
through CPU-heavy sequences without forcing the wake of another thread that
may contend trying to run the same CPU-heavy code. For the new-style work,
this drops runtime ~33% and involuntary context switches by >80%,
essentially making threads cooperatively scheduled.
"""
try:
# Python 2.
sys.setcheckinterval(100000)
except AttributeError:
pass
try:
# Python 3.
sys.setswitchinterval(10)
except AttributeError:
pass
def lazy_client_trial(reset, target, test, get_client, use_greenlets):
"""Test concurrent operations on a lazily-connecting client.
`reset` takes a collection and resets it for the next trial.
`target` takes a lazily-connecting collection and an index from
0 to NTHREADS, and performs some operation, e.g. an insert.
`test` takes the lazily-connecting collection and asserts a
post-condition to prove `target` succeeded.
"""
if use_greenlets and not has_gevent:
raise SkipTest('Gevent not installed')
collection = MongoClient(host, port).pymongo_test.test
# Make concurrency bugs more likely to manifest.
interval = None
if not sys.platform.startswith('java'):
if sys.version_info >= (3, 2):
interval = sys.getswitchinterval()
sys.setswitchinterval(1e-6)
else:
interval = sys.getcheckinterval()
sys.setcheckinterval(1)
try:
for i in range(NTRIALS):
reset(collection)
lazy_client = get_client(_connect=False,
use_greenlets=use_greenlets)
lazy_collection = lazy_client.pymongo_test.test
run_threads(lazy_collection, target, use_greenlets)
test(lazy_collection)
finally:
if not sys.platform.startswith('java'):
if sys.version_info >= (3, 2):
sys.setswitchinterval(interval)
else:
sys.setcheckinterval(interval)
def lazy_client_trial(reset, target, test, get_client, use_greenlets):
"""Test concurrent operations on a lazily-connecting client.
`reset` takes a collection and resets it for the next trial.
`target` takes a lazily-connecting collection and an index from
0 to NTHREADS, and performs some operation, e.g. an insert.
`test` takes the lazily-connecting collection and asserts a
post-condition to prove `target` succeeded.
"""
if use_greenlets and not has_gevent:
raise SkipTest('Gevent not installed')
collection = MongoClient(host, port).pymongo_test.test
# Make concurrency bugs more likely to manifest.
interval = None
if not sys.platform.startswith('java'):
if sys.version_info >= (3, 2):
interval = sys.getswitchinterval()
sys.setswitchinterval(1e-6)
else:
interval = sys.getcheckinterval()
sys.setcheckinterval(1)
try:
for i in range(NTRIALS):
reset(collection)
lazy_client = get_client(
_connect=False, use_greenlets=use_greenlets)
lazy_collection = lazy_client.pymongo_test.test
run_threads(lazy_collection, target, use_greenlets)
test(lazy_collection)
finally:
if not sys.platform.startswith('java'):
if sys.version_info >= (3, 2):
sys.setswitchinterval(interval)
else:
sys.setcheckinterval(interval)
def _inject_jump(self, where, dest):
"""
Monkeypatch bytecode at ``where`` to force it to jump to ``dest``.
Returns function which puts things back to how they were.
"""
# We're about to do dangerous things to a function's code content.
# We can't make a lock to prevent the interpreter from using those
# bytes, so the best we can do is to set the check interval to be high
# and just pray that this keeps other threads at bay.
if sys.version_info[0] < 3:
old_check_interval = sys.getcheckinterval()
sys.setcheckinterval(2**20)
else:
old_check_interval = sys.getswitchinterval()
sys.setswitchinterval(1000)
pb = ctypes.pointer(self.ob_sval)
orig_bytes = [pb[where + i][0] for i in range(3)]
v = struct.pack("<BH", opcode.opmap["JUMP_ABSOLUTE"], dest)
# Overwrite code to cause it to jump to the target
if sys.version_info[0] < 3:
for i in range(3):
pb[where + i][0] = ord(v[i])
else:
for i in range(3):
pb[where + i][0] = v[i]
def tidy_up():
"""
Put the bytecode back to how it was. Good as new.
"""
if sys.version_info[0] < 3:
sys.setcheckinterval(old_check_interval)
else:
sys.setswitchinterval(old_check_interval)
for i in range(3):
pb[where + i][0] = orig_bytes[i]
return tidy_up
def pretest_posttest():
"""Fixture for all tests ensuring environment cleanup"""
try:
sys.setswitchinterval(1)
except AttributeError:
sys.setcheckinterval(100) # deprecated
if getattr(tqdm, "_instances", False):
n = len(tqdm._instances)
if n:
tqdm._instances.clear()
raise EnvironmentError(
"{0} `tqdm` instances still in existence PRE-test".format(n))
yield
if getattr(tqdm, "_instances", False):
n = len(tqdm._instances)
if n:
tqdm._instances.clear()
raise EnvironmentError(
"{0} `tqdm` instances still in existence POST-test".format(n))
def init(self):
""" Initialize parameter viewer
This function created all the GUI elements.
"""
for ii, iname in enumerate(self._instrumentnames):
instr = self._instruments[ii]
pp = instr.parameters
ppnames = sorted(instr.parameters.keys())
ppnames = [
p for p in ppnames if hasattr(instr.parameters[p], 'get')
]
gatesroot = QtWidgets.QTreeWidgetItem(self, [iname])
for parameter_name in ppnames:
# hack to make this semi thread-safe
si = min(sys.getswitchinterval(), 0.1)
# hack to make this semi thread-safe
sys.setswitchinterval(100)
sys.setswitchinterval(si) # hack to make this semi thread-safe
box = QtWidgets.QDoubleSpinBox()
# do not emit signals when still editing
box.setKeyboardTracking(False)
box.setMinimum(-10000)
box.setMaximum(10000)
box.setSingleStep(5)
v = ''
A = QtWidgets.QTreeWidgetItem(gatesroot, [parameter_name, v])
self._itemsdict[iname][parameter_name] = A
if hasattr(pp[parameter_name], 'set'):
qq = A
self.setItemWidget(qq, 1, box)
self._itemsdict[iname][parameter_name] = box
box.valueChanged.connect(
partial(self._valueChanged, iname, parameter_name))
self.setSortingEnabled(True)
self.expandAll()
def start(options):
"""Main entry point for the application"""
# Set the global figure manager in matplotlib. Very important this happens first.
from workbench.plotting.config import init_mpl_gcf
init_mpl_gcf()
# cleanup static resources at exit
atexit.register(qCleanupResources)
# fix/validate arguments
if options.script is not None:
# convert into absolute path
options.script = os.path.abspath(os.path.expanduser(options.script))
if not os.path.exists(options.script):
print('script "{}" does not exist'.format(options.script))
options.script = None
app = initialize()
# the default sys check interval leads to long lags
# when request scripts to be aborted
setswitchinterval(SYSCHECK_INTERVAL)
exit_value = 0
try:
exit_value = start_workbench(app, options)
except BaseException:
# We count this as a crash
import traceback
# This is type of thing we want to capture and have reports
# about. Prints to stderr as we can't really count on anything
# else
traceback.print_exc(file=ORIGINAL_STDERR)
try:
print_file_path = os.path.join(ConfigService.getAppDataDirectory(),
STACKTRACE_FILE)
with open(print_file_path, 'w') as print_file:
traceback.print_exc(file=print_file)
except OSError:
pass
exit_value = -1
finally:
ORIGINAL_SYS_EXIT(exit_value)
def trial(self, reset, target, test):
"""Test concurrent operations on a lazily-connecting client.
`reset` takes a collection and resets it for the next trial.
`target` takes a lazily-connecting collection and an index from
0 to nthreads, and performs some operation, e.g. an insert.
`test` takes a collection and asserts a post-condition to prove
`target` succeeded.
"""
if self.use_greenlets and not has_gevent:
raise SkipTest('Gevent not installed')
collection = self._get_client().pymongo_test.test
# Make concurrency bugs more likely to manifest.
if not sys.platform.startswith('java'):
if PY3:
self.interval = sys.getswitchinterval()
sys.setswitchinterval(1e-6)
else:
self.interval = sys.getcheckinterval()
sys.setcheckinterval(1)
try:
for i in range(self.ntrials):
reset(collection)
lazy_client = self._get_client(
_connect=False, use_greenlets=self.use_greenlets)
lazy_collection = lazy_client.pymongo_test.test
self.run_threads(lazy_collection, target)
test(collection)
finally:
if not sys.platform.startswith('java'):
if PY3:
sys.setswitchinterval(self.interval)
else:
sys.setcheckinterval(self.interval)
def updatedata(self, force_update=False):
""" Update data in viewer using station.snapshow """
self._update_counter = self._update_counter + 1
logging.debug('ParameterViewer: update values')
for instrument_name in self._instrumentnames:
instr = self._instruments[self._instrumentnames.index(
instrument_name)]
parameters = {}
try:
parameters = instr.parameters
except AttributeError as ex:
# instrument was removed
print('instrument was removed, stopping ParameterViewer')
# logging.exception(ex)
self._timer.stop()
parameter_names = sorted(parameters.keys())
si = sys.getswitchinterval()
for parameter_name in parameter_names:
# hack to make this semi thread-safe
for field_name in self._fields:
if field_name == 'Value':
sys.setswitchinterval(100)
value = parameters[parameter_name].get_latest()
sys.setswitchinterval(si)
self.update_field_signal.emit(instrument_name,
parameter_name,
field_name, value,
force_update)
else:
if self._update_counter % 20 == 1 or 1:
sys.setswitchinterval(100)
value = getattr(parameters[parameter_name],
field_name, '')
sys.setswitchinterval(si)
self.update_field_signal.emit(
instrument_name, parameter_name, field_name,
value, force_update)
for callback_function in self.callbacklist:
try:
callback_function()
except Exception as ex:
logging.debug('update function failed')
logging.exception(str(ex))
def trial(self, reset, target, test):
"""Test concurrent operations on a lazily-connecting client.
`reset` takes a collection and resets it for the next trial.
`target` takes a lazily-connecting collection and an index from
0 to nthreads, and performs some operation, e.g. an insert.
`test` takes a collection and asserts a post-condition to prove
`target` succeeded.
"""
if self.use_greenlets and not has_gevent:
raise SkipTest("Gevent not installed")
collection = self._get_client().pymongo_test.test
# Make concurrency bugs more likely to manifest.
if not sys.platform.startswith("java"):
if PY3:
self.interval = sys.getswitchinterval()
sys.setswitchinterval(1e-6)
else:
self.interval = sys.getcheckinterval()
sys.setcheckinterval(1)
try:
for i in range(self.ntrials):
reset(collection)
lazy_client = self._get_client(_connect=False, use_greenlets=self.use_greenlets)
lazy_collection = lazy_client.pymongo_test.test
self.run_threads(lazy_collection, target)
test(collection)
finally:
if not sys.platform.startswith("java"):
if PY3:
sys.setswitchinterval(self.interval)
else:
sys.setcheckinterval(self.interval)
def adjust_balance_concurrently(self):
def transact():
self.account.deposit(5)
time.sleep(0.001)
self.account.withdraw(5)
try:
sys.setswitchinterval(1e-12)
except AttributeError:
# For Python 2 compatibility
sys.setcheckinterval(1)
threads = []
for _ in range(1000):
t = threading.Thread(target=transact)
threads.append(t)
t.start()
for thread in threads:
thread.join()
self.assertEqual(self.account.get_balance(), 1000)
def initialise_qapp_and_launch_workbench(command_line_options):
# Set the global figure manager in matplotlib. Very important this happens first.
from workbench.plotting.config import init_mpl_gcf
init_mpl_gcf()
# cleanup static resources at exit
atexit.register(qCleanupResources)
# fix/validate arguments
if command_line_options.script is not None:
# convert into absolute path
command_line_options.script = os.path.abspath(os.path.expanduser(command_line_options.script))
if not os.path.exists(command_line_options.script):
print('script "{}" does not exist'.format(command_line_options.script))
command_line_options.script = None
app = initialize()
# the default sys check interval leads to long lags
# when request scripts to be aborted
setswitchinterval(SYSCHECK_INTERVAL)
create_and_launch_workbench(app, command_line_options)
def setUpClass(cls):
# Increase switching interval to lure out race conditions a bit ...
sys.setswitchinterval(1e-6)
config = bndl.conf
config['bndl.compute.worker_count'] = 0
config['bndl.net.listen_addresses'] = 'tcp://127.0.0.1:0'
config.update(cls.config)
cls.ctx = create_ctx(config)
cls.node_count = 0 if not cls.worker_count else cls.worker_count // 2 + 1
cls.supervisors = []
for i in range(cls.worker_count):
args = ('--listen-addresses', 'tcp://127.0.0.%s:0' % (i // 2 + 1),
'--seeds', cls.ctx.node.addresses[0])
superv = WorkerSupervisor(args, process_count=1)
superv.start()
cls.supervisors.append(superv)
for _ in range(2):
cls.ctx.await_workers(cls.worker_count, 120, 120)
assert cls.ctx.worker_count == cls.worker_count, \
'%s != %s' % (cls.ctx.worker_count, cls.worker_count)
def adjust_balance_concurrently(account):
def transact():
account.deposit(5)
time.sleep(0.001)
account.withdraw(5)
# Greatly improve the chance of an operation being interrupted
# by thread switch, thus testing synchronization effectively.
# Feel free to tweak the parameters below to see their impact.
try:
sys.setswitchinterval(1e-12)
except AttributeError:
# Python 2 compatible
sys.setcheckinterval(1)
threads = []
for _ in range(1000):
t = threading.Thread(target=transact)
threads.append(t)
t.start()
for thread in threads:
thread.join()
def main():
sys.setswitchinterval(5e-4)
parser = ArgumentParser(
description="Run OSC controller from configuration file",
epilog="See the configuration files under the"
"`config` directory for example options."
)
parser.add_argument(
'-c', '--config',
default=_DEFAULT_CONFIG,
help="Location of the config file",
)
parser.add_argument(
'-v', '--verbose',
action="count",
help="Enable verbose logging. "
"Repeat up to three times for more logging"
)
args = parser.parse_args()
set_up_logging(verbosity_level=args.verbose)
with configure_controller(args.config) as c:
c.run()
def main():
"""Parse options and run script."""
try:
options, args = getopt.getopt(
sys.argv[1:],
'hvsacuo:p:n:t:i:',
['help']
)
options = dict(options)
except getopt.GetoptError:
print_usage()
return 2
if '-h' in options or '--help' in options:
print_usage()
return
#
# Prevent timing single connection async calls since
# this combination will simply generate a SYN flood,
# and is not a practical use case.
#
if '-a' in options and '-c' in options:
print_usage()
return
if '-v' in options:
level = logging.DEBUG
verbose = True
else:
level = logging.WARNING
verbose = False
logging.basicConfig(
level=level,
format=LOGGING_FORMAT,
stream=sys.stderr
)
if '-i' in options:
interval = float(options.get('-i'))
sys.setswitchinterval(interval)
host = options.get('-o', '127.0.0.1')
port = int(options.get('-p', rfoo.DEFAULT_PORT))
t0 = time.time()
try:
if '-s' in options:
logging.warning('Start as server.')
rfoo.start_server(host=host, port=port, handler=rfoo.ExampleHandler)
return
logging.warning('Start as client.')
if len(args) > 0:
data = 'x' * int(args[0])
else:
data = 'x'
n = int(options.get('-n', 1))
t = int(options.get('-t', 1))
m = int(n / t)
if '-a' in options:
gate = rfoo.Notifier
else:
gate = rfoo.Proxy
def client():
#
# Time connection setup/teardown.
#
if '-c' in options:
for i in range(m):
connection = rfoo.connect(host=host, port=port)
r = rfoo.Proxy(connection).echo(data)
if level == logging.DEBUG:
logging.debug('Received %r from proxy.', r)
connection.close()
#
# Time with dummy connection (no network).
#
elif '-u' in options:
handler = rfoo.ExampleHandler()
dummy = DummyConnection(handler)
echo = gate(dummy).echo
for i in range(m):
r = echo(data)
if level == logging.DEBUG:
logging.debug('Received %r from proxy.', r)
#
# Time calls synched / asynch (notifications).
#
else:
connection = rfoo.connect(host=host, port=port)
echo = gate(connection).echo
for i in range(m):
r = echo(data)
#if level == logging.DEBUG:
# logging.debug('Received %r from proxy.', r)
logging.warning('Received %r from proxy.', r)
if t == 1:
client()
return
threads = [threading.Thread(target=client) for i in range(t)]
t0 = time.time()
for t in threads:
t.start()
for t in threads:
t.join()
finally:
logging.warning('Running time, %f seconds.', time.time() - t0)
def _main():
try:
sys.setswitchinterval(1000.0) # we have only a single python thread
except AttributeError:
pass # python compiled without threading
if is_macos:
ensure_osx_locale()
try:
locale.setlocale(locale.LC_ALL, '')
except Exception:
if not is_macos:
raise
print('Failed to set locale with LANG:',
os.environ.get('LANG'),
file=sys.stderr)
os.environ.pop('LANG')
try:
locale.setlocale(locale.LC_ALL, '')
except Exception:
print('Failed to set locale with no LANG, ignoring',
file=sys.stderr)
# Ensure kitty is in PATH
rpath = os.path.dirname(kitty_exe())
items = frozenset(os.environ['PATH'].split(os.pathsep))
if rpath and rpath not in items:
os.environ['PATH'] += os.pathsep + rpath
args = sys.argv[1:]
if is_macos and os.environ.pop('KITTY_LAUNCHED_BY_LAUNCH_SERVICES',
None) == '1':
os.chdir(os.path.expanduser('~'))
args = macos_cmdline()
try:
cwd_ok = os.path.isdir(os.getcwd())
except Exception:
cwd_ok = False
if not cwd_ok:
os.chdir(os.path.expanduser('~'))
args, rest = parse_args(args=args)
args.args = rest
if args.debug_config:
init_glfw(args.debug_keyboard) # needed for parsing native keysyms
create_opts(args, debug_config=True)
return
if getattr(args, 'detach', False):
detach()
if args.replay_commands:
from kitty.client import main
main(args.replay_commands)
return
if args.single_instance:
is_first = single_instance(args.instance_group)
if not is_first:
talk_to_instance(args)
return
init_glfw(args.debug_keyboard) # needed for parsing native keysyms
opts = create_opts(args)
if opts.editor != '.':
os.environ['EDITOR'] = opts.editor
try:
with setup_profiling(args):
# Avoid needing to launch threads to reap zombies
run_app(opts, args)
finally:
glfw_terminate()
reader = None
print(len(agent.readaheads))
agent = None
print(len(pool.agents))
fbak = None
@cli.command()
def fstest():
fbak = Fruitbak(confdir = Path('/dev/shm/conf'))
data = "derp".encode()
hashfunc = fbak.hashfunc
fbak.pool.agent().put_chunk(hashfunc(data), data)
del hashfunc
del fbak
sleep(1)
@cli.command()
def listchunks():
fbak = Fruitbak(confdir = Path('/dev/shm/conf'))
for hash in fbak.pool.agent().lister():
print(hash)
def xyzzy(full, full_set):
"""Nothing happens"""
if __name__ == '__main__':
# up from default 0.005s
setswitchinterval(1)
cli()
def tearDown(self):
if self.old_switchinterval is not None:
sys.setswitchinterval(self.old_switchinterval)
# hopefully given how nothing else is running in the process, this count
# will eventually be the same roughly on each iteration and be roughly the
# same for each average collected offset.
# gc.enable()
# gc.collect(0)
# gc.collect(1)
# gc.collect(2)
# gc.disable()
if os.name != "nt":
print("Making highest priority, os.SCHED_RR")
try:
pid = os.getpid()
niceValue = os.nice(-20)
sys.setswitchinterval(0.5)
print("sys.getswitchinterval", sys.getswitchinterval())
os.sched_setaffinity(pid, [(os.cpu_count() or 1) - 1])
os.sched_setscheduler(pid, os.SCHED_RR, os.sched_param(1))
print("sched_getscheduler", os.sched_getscheduler(pid))
print("sched_getparam", os.sched_getparam(pid))
print("sched_getaffinity", os.sched_getaffinity(pid))
print("sched_getprioritymax", os.sched_get_priority_max(0))
print("sched_getprioritymin", os.sched_get_priority_min(0))
print("sched_rr_getinterval", os.sched_rr_get_interval(pid))
print("nice", os.nice(0))
except PermissionError:
print("run as root to make top OS priority for more accurate results.")
else:
print("lol windows good luck")
def setUp(self):
self._original_switch_interval = getswitchinterval()
setswitchinterval(1)
def main():
"""Parse options and run script."""
try:
options, args = getopt.getopt(sys.argv[1:], 'hvsacuo:p:n:t:i:',
['help'])
options = dict(options)
except getopt.GetoptError:
print_usage()
return 2
if '-h' in options or '--help' in options:
print_usage()
return
#
# Prevent timing single connection async calls since
# this combination will simply generate a SYN flood,
# and is not a practical use case.
#
if '-a' in options and '-c' in options:
print_usage()
return
if '-v' in options:
level = logging.DEBUG
verbose = True
else:
level = logging.WARNING
verbose = False
logging.basicConfig(level=level, format=LOGGING_FORMAT, stream=sys.stderr)
if '-i' in options:
interval = float(options.get('-i'))
sys.setswitchinterval(interval)
host = options.get('-o', '127.0.0.1')
port = int(options.get('-p', rfoo.DEFAULT_PORT))
t0 = time.time()
try:
if '-s' in options:
logging.warning('Start as server.')
rfoo.start_server(host=host,
port=port,
handler=rfoo.ExampleHandler)
return
logging.warning('Start as client.')
if len(args) > 0:
data = 'x' * int(args[0])
else:
data = 'x'
n = int(options.get('-n', 1))
t = int(options.get('-t', 1))
m = int(n / t)
if '-a' in options:
gate = rfoo.Notifier
else:
gate = rfoo.Proxy
def client():
#
# Time connection setup/teardown.
#
if '-c' in options:
for i in range(m):
connection = rfoo.connect(host=host, port=port)
r = rfoo.Proxy(connection).echo(data)
if level == logging.DEBUG:
logging.debug('Received %r from proxy.', r)
connection.close()
#
# Time with dummy connection (no network).
#
elif '-u' in options:
handler = rfoo.ExampleHandler()
dummy = DummyConnection(handler)
echo = gate(dummy).echo
for i in range(m):
r = echo(data)
if level == logging.DEBUG:
logging.debug('Received %r from proxy.', r)
#
# Time calls synched / asynch (notifications).
#
else:
connection = rfoo.connect(host=host, port=port)
echo = gate(connection).echo
for i in range(m):
r = echo(data)
#if level == logging.DEBUG:
# logging.debug('Received %r from proxy.', r)
logging.warning('Received %r from proxy.', r)
if t == 1:
client()
return
threads = [threading.Thread(target=client) for i in range(t)]
t0 = time.time()
for t in threads:
t.start()
for t in threads:
t.join()
finally:
logging.warning('Running time, %f seconds.', time.time() - t0)
MAX_ITER = 5000
Counter = type("Counter", (object,), {'count': 0})
def increment_loop(counter):
for _ in range(MAX_ITER):
counter.count = counter.count + 1
def main():
counter = Counter()
t1 = threading.Thread(target=increment_loop, args=(counter,))
t2 = threading.Thread(target=increment_loop, args=(counter,))
t2.start()
t1.start()
t1.join()
t2.join()
print("Counter value '{}'".format(counter.count))
if __name__ == '__main__':
# This forces to have more context switching to reproduce a data race in
# Python 2 and Python 3 respectively.
import sys
if sys.version_info.major == 2:
sys.setcheckinterval(20)
else:
# No more ticks in Python 3, it's time based.
# 0.05 millisecond, 1000 times smaller than the default (5 milliseconds)
sys.setswitchinterval(5e-5)
main()
def run():
''' Start the execution '''
try:
Z80.execute()
except KeyboardInterrupt:
return
video.init()
Z80.Z80(3.5) # MhZ
load_rom(ROMFILE)
Z80.reset()
Z80.ports.port_out(254, 0xff) # white border on startup
sys.setswitchinterval(255) # we don't use threads, kind of speed up
SNADIR = '../Perfect_SNA/'
# ok
# load.load_sna(SNADIR + 'Action Reflex.sna')
# load.load_sna(SNADIR + 'Ball Breaker 1.sna')
# load.load_sna(SNADIR + 'Ball Breaker 2.sna')
# load.load_sna(SNADIR + 'Batty.sna')
# load.load_sna(SNADIR + 'Bomb Jack.sna')
# load.load_sna(SNADIR + 'Bruce Lee.sna')
# load.load_sna(SNADIR + 'Capitan Trueno 1.sna')
# load.load_sna(SNADIR + 'Cybernoid 1.sna')
# load.load_sna(SNADIR + 'Cybernoid 2.sna')
# load.load_sna(SNADIR + 'Cyclone.sna')
# load.load_sna(SNADIR + 'Eric And The Floaters.sna')
def tearDown(self):
if sys.version_info[:2] >= (3, 2):
sys.setswitchinterval(self._int)
else:
sys.setcheckinterval(self._int)
def setUp(self):
try:
self.old_switchinterval = sys.getswitchinterval()
sys.setswitchinterval(0.000001)
except AttributeError:
self.old_switchinterval = None
def tearDown(self):
if self.old_switchinterval is not None:
sys.setswitchinterval(self.old_switchinterval)
def tearDown(self):
setswitchinterval(self._original_switch_interval)
def main():
usage = "usage: %prog [-h|--help] [options]"
parser = OptionParser(usage=usage)
parser.add_option("-t", "--throughput",
action="store_true", dest="throughput", default=False,
help="run throughput tests")
parser.add_option("-l", "--latency",
action="store_true", dest="latency", default=False,
help="run latency tests")
parser.add_option("-b", "--bandwidth",
action="store_true", dest="bandwidth", default=False,
help="run I/O bandwidth tests")
parser.add_option("-i", "--interval",
action="store", type="int", dest="check_interval", default=None,
help="sys.setcheckinterval() value")
parser.add_option("-I", "--switch-interval",
action="store", type="float", dest="switch_interval", default=None,
help="sys.setswitchinterval() value")
parser.add_option("-n", "--num-threads",
action="store", type="int", dest="nthreads", default=4,
help="max number of threads in tests")
# Hidden option to run the pinging and bandwidth clients
parser.add_option("", "--latclient",
action="store", dest="latclient", default=None,
help=SUPPRESS_HELP)
parser.add_option("", "--bwclient",
action="store", dest="bwclient", default=None,
help=SUPPRESS_HELP)
options, args = parser.parse_args()
if args:
parser.error("unexpected arguments")
if options.latclient:
kwargs = eval(options.latclient)
latency_client(**kwargs)
return
if options.bwclient:
kwargs = eval(options.bwclient)
bandwidth_client(**kwargs)
return
if not options.throughput and not options.latency and not options.bandwidth:
options.throughput = options.latency = options.bandwidth = True
if options.check_interval:
sys.setcheckinterval(options.check_interval)
if options.switch_interval:
sys.setswitchinterval(options.switch_interval)
print("== %s %s (%s) ==" % (
platform.python_implementation(),
platform.python_version(),
platform.python_build()[0],
))
# Processor identification often has repeated spaces
cpu = ' '.join(platform.processor().split())
print("== %s %s on '%s' ==" % (
platform.machine(),
platform.system(),
cpu,
))
print()
if options.throughput:
print("--- Throughput ---")
print()
run_throughput_tests(options.nthreads)
if options.latency:
print("--- Latency ---")
print()
run_latency_tests(options.nthreads)
if options.bandwidth:
print("--- I/O bandwidth ---")
print()
run_bandwidth_tests(options.nthreads)
import time
import os
if six.PY2:
import thread
import Queue
else:
import _thread as thread
import queue as Queue
# NOTE: See http://bugs.python.org/issue7946
# we cannot effectively use threading for loading files/network/etc.
# without setting the switchinterval down on python 3 due to the new
# GIL implementation
_swival = 0.000001
sys.setswitchinterval(_swival)
import threading
import traceback
class TaskError(Exception):
"""Exception generated for task errors"""
pass
class TaskTimeout(TaskError):
"""Exception generated when timing out waiting on a task"""
pass
class UserTaskException(Exception):
pass
