def replacement_process_join(self,
timeout: float = -1) -> None: # type: ignore
from multiprocessing.process import _children
# print(multiprocessing.process.active_children())
self._check_closed()
assert self._parent_pid == os.getpid(), "can only join a child process"
assert self._popen is not None, "can only join a started process"
tident = threading.get_ident()
if timeout < 0:
interval = sys.getswitchinterval()
else:
interval = min(timeout, sys.getswitchinterval())
start_time = scalene.get_wallclock_time()
while True:
scalene.set_thread_sleeping(tident)
res = self._popen.wait(timeout)
if res is not None:
_children.discard(self)
return
# print(multiprocessing.process.active_children())
scalene.reset_thread_sleeping(tident)
if interval != -1:
end_time = scalene.get_wallclock_time()
if end_time - start_time >= interval:
_children.discard(self)
return
def replacement_process_join(self, timeout: float = -1) -> None: # type: ignore
"""
A drop-in replacement for multiprocessing.Process.join
that periodically yields to handle signals
"""
# print(multiprocessing.process.active_children())
self._check_closed()
assert self._parent_pid == os.getpid(), "can only join a child process"
assert self._popen is not None, "can only join a started process"
tident = threading.get_ident()
if timeout < 0:
interval = sys.getswitchinterval()
else:
interval = min(timeout, sys.getswitchinterval())
start_time = scalene.get_wallclock_time()
while True:
scalene.set_thread_sleeping(tident)
res = self._popen.wait(interval)
if res is not None:
from multiprocessing.process import _children
_children.discard(self)
return
scalene.reset_thread_sleeping(tident)
# I think that this should be timeout--
# Interval is the sleep time per-tic,
# but timeout determines whether it returns
if timeout != -1:
end_time = scalene.get_wallclock_time()
if end_time - start_time >= timeout:
from multiprocessing.process import _children
_children.discard(self)
return
def acquire(self, blocking: bool = True, timeout: float = -1) -> bool:
tident = threading.get_ident()
if blocking == 0:
blocking = False
start_time = time.perf_counter()
if blocking:
if timeout < 0:
interval = sys.getswitchinterval()
else:
interval = min(timeout, sys.getswitchinterval())
else:
interval = -1
while True:
scalene.set_thread_sleeping(tident)
acquired_lock = self.__lock.acquire(blocking, interval)
scalene.reset_thread_sleeping(tident)
if acquired_lock:
return True
if not blocking:
return False
# If a timeout was specified, check to see if it's expired.
if timeout != -1:
end_time = time.perf_counter()
if end_time - start_time >= timeout:
return False
def test_setswitchinterval(self):
import sys
raises(TypeError, sys.setswitchinterval)
orig = sys.getswitchinterval()
for n in 1e-6, 0.1, orig: # orig last to restore starting state
sys.setswitchinterval(n)
assert sys.getswitchinterval() == n
raises(ValueError, sys.setswitchinterval, 0.0)
def test_switchinterval(self):
self.assertRaises(TypeError, sys.setswitchinterval)
self.assertRaises(TypeError, sys.setswitchinterval, 'a')
self.assertRaises(ValueError, sys.setswitchinterval, -1.0)
self.assertRaises(ValueError, sys.setswitchinterval, 0.0)
orig = sys.getswitchinterval()
self.assertTrue(orig < 0.5, orig)
try:
for n in (1e-05, 0.05, 3.0, orig):
sys.setswitchinterval(n)
self.assertAlmostEqual(sys.getswitchinterval(), n)
finally:
sys.setswitchinterval(orig)
def test_switchinterval(self):
self.assertRaises(TypeError, sys.setswitchinterval)
self.assertRaises(TypeError, sys.setswitchinterval, "a")
self.assertRaises(ValueError, sys.setswitchinterval, -1.0)
self.assertRaises(ValueError, sys.setswitchinterval, 0.0)
orig = sys.getswitchinterval()
# sanity check
self.assertTrue(orig < 0.5, orig)
try:
for n in 0.00001, 0.05, 3.0, orig:
sys.setswitchinterval(n)
self.assertAlmostEqual(sys.getswitchinterval(), n)
finally:
sys.setswitchinterval(orig)
def test_switchinterval(self):
self.assertRaises(TypeError, sys.setswitchinterval)
self.assertRaises(TypeError, sys.setswitchinterval, "a")
self.assertRaises(ValueError, sys.setswitchinterval, -1.0)
self.assertRaises(ValueError, sys.setswitchinterval, 0.0)
orig = sys.getswitchinterval()
# sanity check
self.assertTrue(orig < 0.5, orig)
try:
for n in 0.00001, 0.05, 3.0, orig:
sys.setswitchinterval(n)
self.assertAlmostEqual(sys.getswitchinterval(), n)
finally:
sys.setswitchinterval(orig)
def acquire(self, blocking: bool = True, timeout: float = None):
if blocking:
if timeout is None:
timeout = sys.getswitchinterval()
else:
timeout = min(timeout, sys.getswitchinterval())
else:
timeout = None
while True:
acquired = super().acquire(blocking, timeout)
if acquired:
return True
if not blocking:
return False
def __init__(self):
print('looking at some sys functions')
print(sys.api_version)
print(sys.argv)
print(sys.base_exec_prefix)
print(sys.base_prefix)
print(sys.builtin_module_names)
print(sys.byteorder)
print(sys.copyright)
print(sys.dllhandle)
print(sys.exc_info())
# print(sys.exc_traceback)
print(sys.executable)
print(sys.path)
print(sys.maxsize)
print(sys.platform)
print(sys.flags)
print(sys.float_info)
print(sys.float_repr_style)
print(sys._framework)
print(sys.getdefaultencoding())
print(sys.getwindowsversion())
print(sys.getallocatedblocks())
print(sys.getfilesystemencodeerrors())
# print(sys.getcheckinterval())
print(sys.getprofile())
print(sys.getrecursionlimit())
print(sys.getswitchinterval())
print(sys.gettrace())
print(sys._git)
print('\n')
print(sys.getsizeof(self))
def setupUI(self):
"""
Thread switching interval can be set from here
"""
sys.setswitchinterval(1)
print(sys.getswitchinterval())
self.btn1 = QPushButton('Button1', self)
self.btn2 = QPushButton('Button2', self)
self.btn3 = QPushButton('Start log', self)
self.btn4 = QPushButton('Stop log', self)
self.resize(300, 100)
self.btn2.move(0, 60)
self.btn3.move(120, 60)
self.btn4.move(120, 0)
self.btn1.clicked.connect(lambda v: self.execute1("1"))
self.btn2.clicked.connect(lambda v: self.execute2("2"))
self.btn3.clicked.connect(self.startLog)
self.btn4.clicked.connect(self.stopLog)
logging.basicConfig(filename="log.txt",
format='%(asctime)s - %(message)s',
datefmt='%d-%b-%y %H:%M:%S',
level=logging.INFO)
logging.warning("Started")
def test_trashcan_threads(self):
# Issue #13992: trashcan mechanism should be thread-safe
NESTING = 60
N_THREADS = 2
def sleeper_gen():
"""A generator that releases the GIL when closed or dealloc'ed."""
try:
yield
finally:
time.sleep(0.000001)
class C(list):
# Appending to a list is atomic, which avoids the use of a lock.
inits = []
dels = []
def __init__(self, alist):
self[:] = alist
C.inits.append(None)
def __del__(self):
# This __del__ is called by subtype_dealloc().
C.dels.append(None)
# `g` will release the GIL when garbage-collected. This
# helps assert subtype_dealloc's behaviour when threads
# switch in the middle of it.
g = sleeper_gen()
next(g)
# Now that __del__ is finished, subtype_dealloc will proceed
# to call list_dealloc, which also uses the trashcan mechanism.
def make_nested():
"""Create a sufficiently nested container object so that the
trashcan mechanism is invoked when deallocating it."""
x = C([])
for i in range(NESTING):
x = [C([x])]
del x
def run_thread():
"""Exercise make_nested() in a loop."""
while not exit:
make_nested()
old_switchinterval = sys.getswitchinterval()
sys.setswitchinterval(1e-5)
try:
exit = []
threads = []
for i in range(N_THREADS):
t = threading.Thread(target=run_thread)
threads.append(t)
with threading_helper.start_threads(threads,
lambda: exit.append(1)):
time.sleep(1.0)
finally:
sys.setswitchinterval(old_switchinterval)
gc.collect()
self.assertEqual(len(C.inits), len(C.dels))
def __init__(
self,
callback: Callable[[Real], None],
frequency: Real,
spread: Real,
center: Real,
update_frequency: Real = 60.0,
waveform: Callable[[Real], Real] = math.sin,
):
super().__init__()
self._callback = callback
self._frequency = frequency
self._spread = spread
self._center = center
self._update_frequency = update_frequency
self._waveform = waveform
self._phase_offset = 0.0
self._previous_frequency = frequency
self._is_stopped = True
self._executor = ThreadPoolExecutor(max_workers=1)
self._run_future = None
self._stop_signal = False
self._time_error = 0.0
self._run_lock = threading.Lock()
# The switch interval should not be changed after this
self._thread_switch_interval = sys.getswitchinterval()
def updatedata(self, force_update=False):
""" Update data in viewer using station.snapshow """
logging.debug('ParameterViewer: update values')
for iname in self._instrumentnames:
instr = self._instruments[self._instrumentnames.index(iname)]
try:
pp = instr.parameters
except AttributeError as ex:
# instrument was removed
print('instrument was removed, stopping ParameterViewer')
self._timer.stop()
ppnames = sorted(instr.parameters.keys())
si = sys.getswitchinterval()
for parameter_name in ppnames:
# hack to make this semi thread-safe
sys.setswitchinterval(100)
value = pp[parameter_name].get_latest()
sys.setswitchinterval(si)
self.update_field.emit(iname, parameter_name, value,
force_update)
for f in self.callbacklist:
try:
f()
except Exception as e:
logging.debug('update function failed')
logging.debug(str(e))
def select(self, timeout: float = -1):
tident = threading.get_ident()
start_time = scalene.get_wallclock_time()
if timeout < 0:
interval = sys.getswitchinterval()
else:
interval = min(timeout, sys.getswitchinterval())
while True:
scalene.set_thread_sleeping(tident)
selected = super().select(interval)
scalene.reset_thread_sleeping(tident)
if selected:
return selected
end_time = scalene.get_wallclock_time()
if timeout != -1:
if end_time - start_time >= timeout:
return None
def setUp(self):
"""
Reduce the CPython check interval so that thread switches happen much
more often, hopefully exercising more possible race conditions. Also,
delay actual test startup until the reactor has been started.
"""
self.addCleanup(sys.setswitchinterval, sys.getswitchinterval())
sys.setswitchinterval(0.0000001)
def about_python():
"""Show information about the python install"""
if parameters["Python"]:
print("[Python]")
print("sysconfig.get_python_version()={}".format(
sysconfig.get_python_version()))
if sys_type() == "Windows":
print("sys.winver={}".format(sys.winver))
printm("sys.version", sys.version)
print("sys.version_info={}".format(sys.version_info))
print("sys.hexversion={}".format(sys.hexversion))
print("sys.implementation={}".format(sys.implementation))
print("platform.python_build()={}".format(platform.python_build()))
print("platform.python_branch()={}".format(platform.python_branch()))
print("platform.python_implementation()={}".format(
platform.python_implementation()))
print("platform.python_revision()={}".format(
platform.python_revision()))
print("platform.python_version()={}".format(platform.python_version()))
print("platform.python_version_tuple()={}".format(
platform.python_version_tuple()))
printm("sys.copyright", sys.copyright)
print()
print("[Python/Config]")
print("sys.base_prefix={}".format(sys.base_prefix))
print("sys.executable={}".format(sys.executable))
print("sys.flags={}".format(sys.flags))
printm("sys.builtin_module_names", sys.builtin_module_names)
printm("sys.modules", sys.modules)
print("sys.path={}".format(sys.path))
python_version = platform.python_version_tuple()
if python_version[0] == 3 and python_version[1] >= 9:
printm("sys.platlibdir", sys.platlibdir) # Python 3.9+
print("sys.getrecursionlimit()={}".format(sys.getrecursionlimit()))
print("sys.getswitchinterval()={}".format(sys.getswitchinterval()))
print("sys.thread_info={}".format(sys.thread_info))
print("platform.python_compiler()={}".format(
platform.python_compiler()))
if sys_type() == "Unix":
print("platform.libc_ver()={}".format(platform.libc_ver()))
print("sys.api_version={}".format(sys.api_version))
print()
print("[Python/Math]")
print("sys.int_info={}".format(sys.int_info))
print("sys.maxsize={}".format(sys.maxsize))
print("sys.float_info={}".format(sys.float_info))
print()
print("[Python/Unicode]")
print("sys.getdefaultencoding()={}".format(sys.getdefaultencoding()))
print("sys.getfilesystemencoding()={}".format(
sys.getfilesystemencoding()))
print("unicodedata.unidata_version={}".format(
unicodedata.unidata_version))
print("sys.maxunicode={}".format(sys.maxunicode))
print()
def setUp(self):
# Set a very small check interval, this will make it more likely
# that the interpreter crashes when threading is done incorrectly.
if sys.version_info[:2] >= (3, 2):
self._int = sys.getswitchinterval()
sys.setswitchinterval(0.000_000_1)
else:
self._int = sys.getcheckinterval()
sys.setcheckinterval(1)
def run(self):
"""Run the sampler to make a measurement of the current stack frames.
"""
measure_tool = self.get_measure_tool()
# Muck with switch interval to prevent thread context switching while
# trying to capture profiling information for safety
switch_interval = sys.getswitchinterval()
try:
logging.debug('Process sampling')
sys.setswitchinterval(10000)
for dummy_frame_id, frame in (sys._current_frames( # pylint: disable=protected-access
).items()):
self.my_db.record(measure_tool(frame))
finally:
sys.setswitchinterval(switch_interval)
logging.debug('Switch interval now %.2f', sys.getswitchinterval())
def __enter__(self) -> bool:
timeout = sys.getswitchinterval()
tident = threading.get_ident()
while True:
scalene.set_thread_sleeping(tident)
acquired = self._semlock.acquire(timeout=timeout) # type: ignore
scalene.reset_thread_sleeping(tident)
if acquired:
return True
def test_trashcan_threads(self):
# Issue #13992: trashcan mechanism should be thread-safe
NESTING = 60
N_THREADS = 2
def sleeper_gen():
"""A generator that releases the GIL when closed or dealloc'ed."""
try:
yield
finally:
time.sleep(0.000001)
class C(list):
# Appending to a list is atomic, which avoids the use of a lock.
inits = []
dels = []
def __init__(self, alist):
self[:] = alist
C.inits.append(None)
def __del__(self):
# This __del__ is called by subtype_dealloc().
C.dels.append(None)
# `g` will release the GIL when garbage-collected. This
# helps assert subtype_dealloc's behaviour when threads
# switch in the middle of it.
g = sleeper_gen()
next(g)
# Now that __del__ is finished, subtype_dealloc will proceed
# to call list_dealloc, which also uses the trashcan mechanism.
def make_nested():
"""Create a sufficiently nested container object so that the
trashcan mechanism is invoked when deallocating it."""
x = C([])
for i in range(NESTING):
x = [C([x])]
del x
def run_thread():
"""Exercise make_nested() in a loop."""
while not exit:
make_nested()
old_switchinterval = sys.getswitchinterval()
sys.setswitchinterval(1e-5)
try:
exit = []
threads = []
for i in range(N_THREADS):
t = threading.Thread(target=run_thread)
threads.append(t)
with start_threads(threads, lambda: exit.append(1)):
time.sleep(1.0)
finally:
sys.setswitchinterval(old_switchinterval)
gc.collect()
self.assertEqual(len(C.inits), len(C.dels))
def setUpModule():
thread_info = threading_helper.threading_setup()
unittest.addModuleCleanup(threading_helper.threading_cleanup, *thread_info)
try:
old_switchinterval = sys.getswitchinterval()
unittest.addModuleCleanup(sys.setswitchinterval, old_switchinterval)
sys.setswitchinterval(1e-5)
except AttributeError:
pass
def setUp(self):
# Set a very small check interval, this will make it more likely
# that the interpreter crashes when threading is done incorrectly.
if sys.version_info[:2] >= (3, 2):
self._int = sys.getswitchinterval()
sys.setswitchinterval(0.0000001)
else:
self._int = sys.getcheckinterval()
sys.setcheckinterval(1)
def fast_thread_switching():
"""Fixture that reduces thread switching interval.
This makes it easier to provoke race conditions.
"""
old = sys.getswitchinterval()
sys.setswitchinterval(1e-6)
yield
sys.setswitchinterval(old)
def reconfig():
""" Set global PVM configs """
cfg = {}
cfg['sys.platform'] = sys.platform
cfg['sys.maxsize'] = sys.maxsize
cfg['sys.path'] = sys.path
cfg['sys.excepthook'] = sys.excepthook
cfg['old sys.switchinterval'] = sys.getswitchinterval()
sys.setswitchinterval(LONGER_CHECK_INTERVAL)
cfg['new sys.switchinterval'] = sys.getswitchinterval()
cfg['old sys.recursionlimit'] = sys.getrecursionlimit()
sys.setrecursionlimit(BIGGER_RECURSION_LIMIT)
cfg['new sys.recursionlimit'] = sys.getrecursionlimit()
cfg['old gc.threshold'] = str(gc.get_threshold())
gc.set_threshold(*LOWER_GC_THRESHOLD)
cfg['new gc.threshold'] = str(gc.get_threshold())
sys._clear_type_cache()
cfg['sys._clear_type_cache'] = True
return cfg
def select(
self, timeout: Optional[float] = -1
) -> List[Tuple[selectors.SelectorKey, int]]:
tident = threading.get_ident()
start_time = scalene.get_wallclock_time()
if not timeout or timeout < 0:
interval = sys.getswitchinterval()
else:
interval = min(timeout, sys.getswitchinterval())
while True:
scalene.set_thread_sleeping(tident)
selected = super().select(interval)
scalene.reset_thread_sleeping(tident)
if selected:
return selected
end_time = scalene.get_wallclock_time()
if timeout and timeout != -1:
if end_time - start_time >= timeout:
return [] # None
def thread_join_replacement(self, timeout=None):
"""We will replace threading.Thread.join with this method which always periodically yields."""
start_time = time.perf_counter()
interval = sys.getswitchinterval()
while self.is_alive():
Scalene.original_thread_join(
self, interval) # Scalene.last_signal_interval * 100)
# If a timeout was specified, check to see if it's expired.
if timeout:
if time.perf_counter() - start_time >= timeout:
return
def test_main():
old_switchinterval = None
try:
old_switchinterval = sys.getswitchinterval()
sys.setswitchinterval(1e-5)
except AttributeError:
pass
try:
run_unittest(ThreadedImportTests)
finally:
if old_switchinterval is not None:
sys.setswitchinterval(old_switchinterval)
def test_main():
old_switchinterval = None
try:
old_switchinterval = sys.getswitchinterval()
sys.setswitchinterval(1e-5)
except AttributeError:
pass
try:
run_unittest(ThreadedImportTests)
finally:
if old_switchinterval is not None:
sys.setswitchinterval(old_switchinterval)
def test_pending_calls_race(self):
event = threading.Event()
def future_func():
event.wait()
oldswitchinterval = sys.getswitchinterval()
sys.setswitchinterval(1e-06)
try:
fs = {self.executor.submit(future_func) for i in range(100)}
event.set()
futures.wait(fs, return_when=futures.ALL_COMPLETED)
finally:
sys.setswitchinterval(oldswitchinterval)
def thread_join_replacement(self, timeout=None):
"""We replace threading.Thread.join with this method which always
periodically yields."""
start_time = Scalene.gettime()
interval = sys.getswitchinterval()
while self.is_alive():
Scalene.__original_thread_join(self, interval)
# If a timeout was specified, check to see if it's expired.
if timeout:
end_time = Scalene.gettime()
if end_time - start_time >= timeout:
return None
return None
def _create_gui(self):
""" Initialize parameter viewer GUI
This function creates all the GUI elements.
"""
for ii, instrument_name in enumerate(self._instrumentnames):
instr = self._instruments[ii]
parameters = instr.parameters
parameter_names = sorted(instr.parameters.keys())
parameter_names = [
p for p in parameter_names
if hasattr(instr.parameters[p], 'get')
]
gatesroot = QtWidgets.QTreeWidgetItem(self, [instrument_name])
self._itemsdict[instrument_name]['_treewidgetitem'] = gatesroot
for parameter_name in parameter_names:
# hack to make this semi thread-safe
si = min(sys.getswitchinterval(), 0.1)
# hack to make this semi thread-safe
sys.setswitchinterval(100)
sys.setswitchinterval(si) # hack to make this semi thread-safe
box = QtWidgets.QDoubleSpinBox()
# do not emit signals when still editing
box.setKeyboardTracking(False)
initial_values = [parameter_name] + [''] * len(self._fields)
widget = QtWidgets.QTreeWidgetItem(gatesroot, initial_values)
self._itemsdict[instrument_name][parameter_name] = {
'widget': widget,
'double_box': None
}
px = parameters[parameter_name].get()
if hasattr(parameters[parameter_name], 'set') and isfloat(px):
self.setItemWidget(widget, 1, box)
self._itemsdict[instrument_name][parameter_name][
'double_box'] = box
box.valueChanged.connect(
partial(self._valueChanged, instrument_name,
parameter_name))
self.set_column_sizehints(self._default_sizehints)
self.set_parameter_properties(step_size=5,
minimum_value=-1e20,
maximum_value=1e20)
self.setSortingEnabled(True)
self.expandAll()
def test_thread_safety_during_modification(self):
hosts = range(100)
policy = RoundRobinPolicy()
policy.populate(None, hosts)
errors = []
def check_query_plan():
try:
for i in xrange(100):
list(policy.make_query_plan())
except Exception as exc:
errors.append(exc)
def host_up():
for i in xrange(1000):
policy.on_up(randint(0, 99))
def host_down():
for i in xrange(1000):
policy.on_down(randint(0, 99))
threads = []
for i in range(5):
threads.append(Thread(target=check_query_plan))
threads.append(Thread(target=host_up))
threads.append(Thread(target=host_down))
# make the GIL switch after every instruction, maximizing
# the chance of race conditions
check = six.PY2 or '__pypy__' in sys.builtin_module_names
if check:
original_interval = sys.getcheckinterval()
else:
original_interval = sys.getswitchinterval()
try:
if check:
sys.setcheckinterval(0)
else:
sys.setswitchinterval(0.0001)
map(lambda t: t.start(), threads)
map(lambda t: t.join(), threads)
finally:
if check:
sys.setcheckinterval(original_interval)
else:
sys.setswitchinterval(original_interval)
if errors:
self.fail("Saw errors: %s" % (errors, ))
def test_trashcan_threads(self):
NESTING = 60
N_THREADS = 2
def sleeper_gen():
"""A generator that releases the GIL when closed or dealloc'ed."""
try:
yield
finally:
time.sleep(1e-06)
class C(list):
inits = []
dels = []
def __init__(self, alist):
self[:] = alist
C.inits.append(None)
def __del__(self):
C.dels.append(None)
g = sleeper_gen()
next(g)
def make_nested():
"""Create a sufficiently nested container object so that the
trashcan mechanism is invoked when deallocating it."""
x = C([])
for i in range(NESTING):
x = [C([x])]
del x
def run_thread():
"""Exercise make_nested() in a loop."""
while not exit:
make_nested()
old_switchinterval = sys.getswitchinterval()
sys.setswitchinterval(1e-05)
try:
exit = []
threads = []
for i in range(N_THREADS):
t = threading.Thread(target=run_thread)
threads.append(t)
with start_threads(threads, lambda: exit.append(1)):
time.sleep(1.0)
finally:
sys.setswitchinterval(old_switchinterval)
gc.collect()
self.assertEqual(len(C.inits), len(C.dels))
def test_thread_safety_during_modification(self):
hosts = range(100)
policy = RoundRobinPolicy()
policy.populate(None, hosts)
errors = []
def check_query_plan():
try:
for i in xrange(100):
list(policy.make_query_plan())
except Exception as exc:
errors.append(exc)
def host_up():
for i in xrange(1000):
policy.on_up(randint(0, 99))
def host_down():
for i in xrange(1000):
policy.on_down(randint(0, 99))
threads = []
for i in range(5):
threads.append(Thread(target=check_query_plan))
threads.append(Thread(target=host_up))
threads.append(Thread(target=host_down))
# make the GIL switch after every instruction, maximizing
# the chance of race conditions
check = six.PY2 or '__pypy__' in sys.builtin_module_names
if check:
original_interval = sys.getcheckinterval()
else:
original_interval = sys.getswitchinterval()
try:
if check:
sys.setcheckinterval(0)
else:
sys.setswitchinterval(0.0001)
map(lambda t: t.start(), threads)
map(lambda t: t.join(), threads)
finally:
if check:
sys.setcheckinterval(original_interval)
else:
sys.setswitchinterval(original_interval)
if errors:
self.fail("Saw errors: %s" % (errors,))
def setUp(self):
"""
Reduce the CPython check interval so that thread switches happen much
more often, hopefully exercising more possible race conditions. Also,
delay actual test startup until the reactor has been started.
"""
if _PY3:
if getattr(sys, 'getswitchinterval', None) is not None:
self.addCleanup(sys.setswitchinterval, sys.getswitchinterval())
sys.setswitchinterval(0.0000001)
else:
if getattr(sys, 'getcheckinterval', None) is not None:
self.addCleanup(sys.setcheckinterval, sys.getcheckinterval())
sys.setcheckinterval(7)
def updatedata(self, force_update=False):
""" Update data in viewer using station.snapshow """
self._update_counter = self._update_counter + 1
logging.debug('ParameterViewer: update values')
for instrument_name in self._instrumentnames:
instr = self._instruments[self._instrumentnames.index(
instrument_name)]
parameters = {}
try:
parameters = instr.parameters
except AttributeError as ex:
# instrument was removed
print('instrument was removed, stopping ParameterViewer')
# logging.exception(ex)
self._timer.stop()
parameter_names = sorted(parameters.keys())
si = sys.getswitchinterval()
for parameter_name in parameter_names:
# hack to make this semi thread-safe
for field_name in self._fields:
if field_name == 'Value':
sys.setswitchinterval(100)
value = parameters[parameter_name].get_latest()
sys.setswitchinterval(si)
self.update_field_signal.emit(instrument_name,
parameter_name,
field_name, value,
force_update)
else:
if self._update_counter % 20 == 1 or 1:
sys.setswitchinterval(100)
value = getattr(parameters[parameter_name],
field_name, '')
sys.setswitchinterval(si)
self.update_field_signal.emit(
instrument_name, parameter_name, field_name,
value, force_update)
for callback_function in self.callbacklist:
try:
callback_function()
except Exception as ex:
logging.debug('update function failed')
logging.exception(str(ex))
def test_main():
old_switchinterval = None
# Issue #15599: FreeBSD/KVM cannot handle gil_interval == 1.
new_switchinterval = 0.00001 if 'freebsd' in sys.platform else 0.00000001
try:
old_switchinterval = sys.getswitchinterval()
sys.setswitchinterval(new_switchinterval)
except AttributeError:
pass
try:
run_unittest(ThreadedImportTests)
finally:
if old_switchinterval is not None:
sys.setswitchinterval(old_switchinterval)
def test_pending_calls_race(self):
# Issue #14406: multi-threaded race condition when waiting on all
# futures.
event = threading.Event()
def future_func():
event.wait()
oldswitchinterval = sys.getswitchinterval()
sys.setswitchinterval(1e-6)
try:
fs = {self.executor.submit(future_func) for i in range(100)}
event.set()
futures.wait(fs, return_when=futures.ALL_COMPLETED)
finally:
sys.setswitchinterval(oldswitchinterval)
def test_enumerate_after_join(self):
# Try hard to trigger #1703448: a thread is still returned in
# threading.enumerate() after it has been join()ed.
enum = threading.enumerate
old_interval = sys.getswitchinterval()
try:
for i in range(1, 100):
sys.setswitchinterval(i * 0.0002)
t = threading.Thread(target=lambda: None)
t.start()
t.join()
l = enum()
self.assertNotIn(t, l, "#1703448 triggered after %d trials: %s" % (i, l))
finally:
sys.setswitchinterval(old_interval)
def test_lru_cache_threaded(self):
n, m = 5, 11
class C:
def orig(self, x, y):
return 3 * x + y
instance = C()
f = per_instance_lru_cache(maxsize=n*m)(C.orig)
hits, misses, maxsize, currsize = f.cache_info(instance)
self.assertEqual(currsize, 0)
start = threading.Event()
def full(k):
start.wait(10)
for _ in range(m):
self.assertEqual(f(instance, k, 0), instance.orig(k, 0))
def clear():
start.wait(10)
for _ in range(2*m):
f.cache_clear(instance)
orig_si = sys.getswitchinterval()
sys.setswitchinterval(1e-6)
try:
# create n threads in order to fill cache
threads = [threading.Thread(target=full, args=[k]) for k in range(n)]
with support.start_threads(threads):
start.set()
hits, misses, maxsize, currsize = f.cache_info(instance)
# XXX: Why can be not equal?
self.assertLessEqual(misses, n)
self.assertLessEqual(hits, m*n - misses)
self.assertEqual(currsize, n)
# create n threads in order to fill cache and 1 to clear it
threads = [threading.Thread(target=clear)]
threads += [threading.Thread(target=full, args=[k]) for k in range(n)]
start.clear()
with support.start_threads(threads):
start.set()
finally:
sys.setswitchinterval(orig_si)
def run_threads():
interval = sys.getswitchinterval()
print('interval = {:0.3f}'.format(interval))
q = Queue()
threads = [
threading.Thread(target=show_thread,
name='T{}'.format(i),
args=(q,))
for i in range(3)
]
for t in threads:
t.setDaemon(True)
t.start()
for t in threads:
t.join()
while not q.empty():
print(q.get(), end=' ')
print()
def test_is_alive_after_fork(self):
# Try hard to trigger #18418: is_alive() could sometimes be True on
# threads that vanished after a fork.
old_interval = sys.getswitchinterval()
self.addCleanup(sys.setswitchinterval, old_interval)
# Make the bug more likely to manifest.
sys.setswitchinterval(1e-6)
for i in range(20):
t = threading.Thread(target=lambda: None)
t.start()
self.addCleanup(t.join)
pid = os.fork()
if pid == 0:
os._exit(1 if t.is_alive() else 0)
else:
pid, status = os.waitpid(pid, 0)
self.assertEqual(0, status)
def frequent_thread_switches():
"""Make concurrency bugs more likely to manifest."""
interval = None
if not sys.platform.startswith('java'):
if hasattr(sys, 'getswitchinterval'):
interval = sys.getswitchinterval()
sys.setswitchinterval(1e-6)
else:
interval = sys.getcheckinterval()
sys.setcheckinterval(1)
try:
yield
finally:
if not sys.platform.startswith('java'):
if hasattr(sys, 'setswitchinterval'):
sys.setswitchinterval(interval)
else:
sys.setcheckinterval(interval)
def _inject_jump(self, where, dest):
"""
Monkeypatch bytecode at ``where`` to force it to jump to ``dest``.
Returns function which puts things back to how they were.
"""
# We're about to do dangerous things to a function's code content.
# We can't make a lock to prevent the interpreter from using those
# bytes, so the best we can do is to set the check interval to be high
# and just pray that this keeps other threads at bay.
if sys.version_info[0] < 3:
old_check_interval = sys.getcheckinterval()
sys.setcheckinterval(2**20)
else:
old_check_interval = sys.getswitchinterval()
sys.setswitchinterval(1000)
pb = ctypes.pointer(self.ob_sval)
orig_bytes = [pb[where + i][0] for i in range(3)]
v = struct.pack("<BH", opcode.opmap["JUMP_ABSOLUTE"], dest)
# Overwrite code to cause it to jump to the target
if sys.version_info[0] < 3:
for i in range(3):
pb[where + i][0] = ord(v[i])
else:
for i in range(3):
pb[where + i][0] = v[i]
def tidy_up():
"""
Put the bytecode back to how it was. Good as new.
"""
if sys.version_info[0] < 3:
sys.setcheckinterval(old_check_interval)
else:
sys.setswitchinterval(old_check_interval)
for i in range(3):
pb[where + i][0] = orig_bytes[i]
return tidy_up
def lazy_client_trial(reset, target, test, get_client, use_greenlets):
"""Test concurrent operations on a lazily-connecting client.
`reset` takes a collection and resets it for the next trial.
`target` takes a lazily-connecting collection and an index from
0 to NTHREADS, and performs some operation, e.g. an insert.
`test` takes the lazily-connecting collection and asserts a
post-condition to prove `target` succeeded.
"""
if use_greenlets and not has_gevent:
raise SkipTest('Gevent not installed')
collection = MongoClient(host, port).pymongo_test.test
# Make concurrency bugs more likely to manifest.
interval = None
if not sys.platform.startswith('java'):
if sys.version_info >= (3, 2):
interval = sys.getswitchinterval()
sys.setswitchinterval(1e-6)
else:
interval = sys.getcheckinterval()
sys.setcheckinterval(1)
try:
for i in range(NTRIALS):
reset(collection)
lazy_client = get_client(
_connect=False, use_greenlets=use_greenlets)
lazy_collection = lazy_client.pymongo_test.test
run_threads(lazy_collection, target, use_greenlets)
test(lazy_collection)
finally:
if not sys.platform.startswith('java'):
if sys.version_info >= (3, 2):
sys.setswitchinterval(interval)
else:
sys.setcheckinterval(interval)
def trial(self, reset, target, test):
"""Test concurrent operations on a lazily-connecting client.
`reset` takes a collection and resets it for the next trial.
`target` takes a lazily-connecting collection and an index from
0 to nthreads, and performs some operation, e.g. an insert.
`test` takes a collection and asserts a post-condition to prove
`target` succeeded.
"""
if self.use_greenlets and not has_gevent:
raise SkipTest("Gevent not installed")
collection = self._get_client().pymongo_test.test
# Make concurrency bugs more likely to manifest.
if not sys.platform.startswith("java"):
if PY3:
self.interval = sys.getswitchinterval()
sys.setswitchinterval(1e-6)
else:
self.interval = sys.getcheckinterval()
sys.setcheckinterval(1)
try:
for i in range(self.ntrials):
reset(collection)
lazy_client = self._get_client(_connect=False, use_greenlets=self.use_greenlets)
lazy_collection = lazy_client.pymongo_test.test
self.run_threads(lazy_collection, target)
test(collection)
finally:
if not sys.platform.startswith("java"):
if PY3:
sys.setswitchinterval(self.interval)
else:
sys.setcheckinterval(self.interval)
def run():
print('sys.executable', sys.executable)
print('sys.argv', sys.argv)
print('sys.path', sys.path)
print('sys.prefix', sys.prefix)
print('sys.exec_prefix', sys.exec_prefix)
print('sys.platform', sys.platform)
print('sys.version', sys.version)
print('sys.version_info', sys.version_info)
print('sys.hexversion', sys.hexversion)
print('sys.api_version', sys.api_version)
print('sys.byteorder', sys.byteorder)
print('sys.builtin_module_names', sys.builtin_module_names)
print('sys.dont_write_bytecode', sys.dont_write_bytecode)
print('sys.flags', sys.flags)
print('sys.int_info', sys.int_info)
print('sys.float_info', sys.float_info)
print('sys.maxsize', sys.maxsize)
print('sys.maxunicode', sys.maxunicode)
print('sys.getdefaultencoding()', sys.getdefaultencoding())
print('sys.getfilesystemencoding()', sys.getfilesystemencoding())
print('sys.getswitchinterval()', sys.getswitchinterval())
def setUp(self):
try:
self.old_switchinterval = sys.getswitchinterval()
sys.setswitchinterval(0.000001)
except AttributeError:
self.old_switchinterval = None
def setUp(self):
self._original_switch_interval = getswitchinterval()
setswitchinterval(1)
