def test_repeating(self, exclusive):
PERIOD = 0.1
TIMES = 3
invocations = [0, 0]
invoked = threading.Event()
def _work():
invocations[0] += 1
invocations[1] = monotonic_time()
if invocations[0] == TIMES:
invoked.set()
op = periodic.Operation(_work,
period=PERIOD,
scheduler=self.sched,
executor=self.exc,
exclusive=exclusive)
op.start()
invoked.wait(PERIOD * TIMES + PERIOD)
# depending on timing, _work may be triggered one more time.
# nothing prevents this, although is unlikely.
# we don't care of this case
op.stop()
self.assertTrue(invoked.is_set())
self.assertTrue(TIMES <= invocations[0] <= TIMES + 1)
def test_repeating_exclusive_with_pool_exhausted(self):
PERIOD = 0.1
TRIES_BEFORE_SUCCESS = 2
exc = _RecoveringExecutor(tries_before_success=TRIES_BEFORE_SUCCESS)
attempts = [0]
done = threading.Event()
def _work():
attempts[0] = exc.attempts
logging.info('_work invoked after %d attempts', attempts[0])
done.set()
op = periodic.Operation(_work, period=PERIOD,
scheduler=self.sched,
executor=exc,
exclusive=True)
op.start()
timeout = 2 # seconds
# We intentionally using a timeout much longer than actually needed
# the timeout should be >= PERIOD * (TRIES_BEFORE_SUCCESS + 1).
# We use larger value to reduce the chance of false failures
# on overloaded CI workers.
self.assertTrue(done.wait(timeout))
self.assertEqual(attempts[0], TRIES_BEFORE_SUCCESS + 1)
op.stop()
def test_repeating_after_block(self):
PERIOD = 0.1
TIMES = 5
BLOCK_AT = 2
invocations = [0, 0]
executions = [0, 0]
done = threading.Event()
def _work():
invocations[0] += 1
invocations[1] = monotonic_time()
if invocations[0] == BLOCK_AT:
# must be > (PERIOD * TIMES) ~= forever
time.sleep(10 * PERIOD * TIMES)
executions[0] += 1
executions[1] = monotonic_time()
if invocations[0] == TIMES:
done.set()
op = periodic.Operation(_work,
period=PERIOD,
scheduler=self.sched,
executor=self.exc)
op.start()
done.wait(PERIOD * TIMES + PERIOD)
# depending on timing, _work may be triggered one more time.
# nothing prevents this, although is unlikely.
# we don't care of this case
op.stop()
self.assertTrue(done.is_set())
self.assertTrue(executions[1] >= invocations[1])
self.assertTrue(TIMES <= invocations[0] <= TIMES + 1)
# one execution never completed
self.assertEqual(executions[0], invocations[0] - 1)
def start(self):
if not config.getboolean('guest_agent', 'enable_qga_poller'):
self.log.info('Not starting QEMU-GA poller. It is disabled in'
' configuration')
return
def per_vm_operation(job, period):
disp = periodic.VmDispatcher(self._cif.getVMs, self._executor,
lambda vm: job(vm, self),
_TASK_TIMEOUT)
return periodic.Operation(disp,
period,
self._scheduler,
timeout=_TASK_TIMEOUT,
executor=self._executor)
self._operations = [
periodic.Operation(self._cleanup,
config.getint('guest_agent', 'cleanup_period'),
self._scheduler,
executor=self._executor),
# Monitor what QEMU-GA offers
per_vm_operation(CapabilityCheck,
config.getint('guest_agent', 'qga_info_period')),
]
self.log.info("Starting QEMU-GA poller")
self._executor.start()
for op in self._operations:
op.start()
def test_dump_executor_state_on_resource_exhausted(self):
PERIOD = 0.1
MAX_TASKS = 20 # random value
log = fakelib.FakeLogger()
exc = executor.Executor(name="test.Executor",
# intentional we just want to clog the queue
workers_count=0,
max_tasks=MAX_TASKS,
scheduler=self.sched, # unused
max_workers=0,
log=log)
exc.start()
op = periodic.Operation(lambda: None,
period=PERIOD,
scheduler=self.sched,
executor=exc,
timeout=None,
exclusive=False)
with MonkeyPatchScope([
(throttledlog, '_logger', log),
]):
# the first dispatch is done here
op.start()
for _ in range(MAX_TASKS - 1):
op._dispatch()
# this will trigger the exception, and the dump
op._dispatch()
level, message, args = log.messages[-1]
self.assertTrue(message.startswith('executor state:'))
def test_invalid_period(self):
op = periodic.Operation(lambda: None,
period=0,
scheduler=self.sched,
executor=self.exc)
with pytest.raises(periodic.InvalidValue):
op.start()
def per_vm_operation(job, period):
disp = periodic.VmDispatcher(
self._cif.getVMs, self._executor,
lambda vm: job(vm, self),
_TASK_TIMEOUT)
return periodic.Operation(
disp, period, self._scheduler, timeout=_TASK_TIMEOUT,
executor=self._executor)
def test_start_with_invalid_operation(self):
"""
periodic.start() should swallow any error that
periodic.Operation.start() may raise, and keep starting
the other operations after the failed one.
"""
lock = threading.Lock()
done = threading.Event()
def _work():
with lock:
self.tasks -= 1
if not self.tasks:
done.set()
ops = [
periodic.Operation(_work,
period=1.0,
scheduler=self.sched,
executor=self.exc),
# will raise periodic.InvalidValue
periodic.Operation(lambda: None, period=0,
scheduler=self.sched,
executor=self.exc),
periodic.Operation(_work,
period=1.0,
scheduler=self.sched,
executor=self.exc),
]
with MonkeyPatchScope([
(periodic, 'config',
make_config([('sampling', 'enable', 'false')])),
(periodic, '_create', lambda cif, sched: ops),
]):
# Don't assume operations are started in order,
# we just know all of them will be start()ed.
# See the documentation of periodic.start()
periodic.start(fake.ClientIF(), self.sched)
done.wait(0.5)
self.assertTrue(done.is_set())
def test_start_twice(self):
def _work():
pass
op = periodic.Operation(_work,
period=1.0,
scheduler=self.sched,
executor=self.exc)
op.start()
self.assertRaises(AssertionError, op.start)
def test_start(self):
invoked = threading.Event()
def _work():
invoked.set()
op = periodic.Operation(_work, period=1.0,
scheduler=self.sched,
executor=self.exc)
op.start()
invoked.wait(0.5)
self.assertTrue(invoked.is_set())
def start(self):
if not config.getboolean('guest_agent', 'enable_qga_poller'):
self.log.info('Not starting QEMU-GA poller. It is disabled in'
' configuration')
return
def per_vm_operation(job, period):
disp = periodic.VmDispatcher(self._cif.getVMs, self._executor,
lambda vm: job(vm, self),
_TASK_TIMEOUT)
return periodic.Operation(disp,
period,
self._scheduler,
timeout=_TASK_TIMEOUT,
executor=self._executor)
self._operations = [
periodic.Operation(self._cleanup,
config.getint('guest_agent', 'cleanup_period'),
self._scheduler,
executor=self._executor),
# Monitor what QEMU-GA offers
per_vm_operation(CapabilityCheck,
config.getint('guest_agent', 'qga_info_period')),
# Basic system information
per_vm_operation(
SystemInfoCheck,
config.getint('guest_agent', 'qga_sysinfo_period')),
per_vm_operation(
NetworkInterfacesCheck,
config.getint('guest_agent', 'qga_sysinfo_period')),
# List of active users
per_vm_operation(
ActiveUsersCheck,
config.getint('guest_agent', 'qga_active_users_period')),
# Filesystem info and disk mapping
per_vm_operation(
DiskInfoCheck,
config.getint('guest_agent', 'qga_disk_info_period')),
]
self.log.info("Starting QEMU-GA poller")
self._executor.start()
for op in self._operations:
op.start()
def start(self):
if not config.getboolean('guest_agent', 'enable_qga_poller'):
self.log.info('Not starting QEMU-GA poller. It is disabled in'
' configuration')
return
self._operation = periodic.Operation(
self._poller,
config.getint('guest_agent', 'qga_polling_period'),
self._scheduler,
timeout=_TASK_TIMEOUT,
executor=self._executor,
exclusive=True)
self.log.info("Starting QEMU-GA poller")
self._executor.start()
self._operation.start()
def test_repeating_if_raises(self):
PERIOD = 0.1
TIMES = 5
def _work():
pass
exc = _FakeExecutor(fail=True, max_attempts=TIMES)
op = periodic.Operation(_work, period=PERIOD,
scheduler=self.sched,
executor=exc)
op.start()
completed = exc.done.wait(PERIOD * TIMES + PERIOD)
# depending on timing, _work may be triggered one more time.
# nothing prevents this, although is unlikely.
# we don't care of this case
op.stop()
self.assertTrue(completed)
self.assertTrue(TIMES <= exc.attempts <= TIMES + 1)
def test_repeating_exclusive_operation(self):
PERIOD = 0.2
executions = [0]
ready = threading.Event()
done = threading.Event()
log = logging.getLogger('test')
def _work():
n = executions[0]
executions[0] += 1
log.info('BEGIN _work() n=%d', n)
if n == 0:
# block just the first time
# we intentionally don't set done
# to emulate lost worker
log.info('waiting for readiness...')
ready.wait()
log.info('ready!')
else:
done.set()
log.info('done!')
log.info('END')
op = periodic.Operation(_work,
period=PERIOD,
scheduler=self.sched,
executor=self.exc,
timeout=None,
exclusive=True)
op.start()
self.assertFalse(done.wait(PERIOD * 4))
# we just wait "long enough" to make sure we cross at least one
# timeout threshold.
ready.set()
completed = done.wait(PERIOD * 2) # guard against races
op.stop()
self.assertTrue(completed)
# op.stop() doesn't guarantee the immediate termination, so _work()
# can run one extra time
self.assertGreaterEqual(executions[0], 2)
def test_stop(self):
PERIOD = 0.1
invocations = [0]
def _work():
invocations[0] = monotonic_time()
op = periodic.Operation(_work, period=PERIOD,
scheduler=self.sched,
executor=self.exc)
op.start()
time.sleep(PERIOD * 2)
# avoid pathological case on which nothing ever runs
self.assertTrue(invocations[0] > 0)
op.stop()
# cooldown. Let's try to avoid scheduler mistakes.
time.sleep(PERIOD)
stop = monotonic_time()
self.assertTrue(stop > invocations[0])
def test_invalid_period(self):
op = periodic.Operation(lambda: None, period=0,
scheduler=self.sched,
executor=self.exc)
self.assertRaises(periodic.InvalidValue, op.start)
