def TestFilterContinue():
"""Tests that the CONTINUE filter has no effect"""
# Add a filter that just passes to the next filter.
uuid_cont = stdout_of([
"gnt-filter", "add",
'--predicates=[["jobid", [">", "id", "watermark"]]]',
"--action=CONTINUE",
"--priority=0",
])
# Add a filter that rejects all new jobs.
uuid_reject = stdout_of([
"gnt-filter", "add",
'--predicates=[["jobid", [">", "id", "watermark"]]]',
"--action=REJECT",
"--priority=1",
])
# Newly queued jobs must now fail.
AssertCommand(["gnt-debug", "delay", "0.01"], fail=True)
# Delete the rejecting filter.
AssertCommand(["gnt-filter", "delete", uuid_reject])
# Newly queued jobs must now succeed.
AssertCommand(["gnt-debug", "delay", "0.01"])
# Clean up.
AssertCommand(["gnt-filter", "delete", uuid_cont])
def TestFilterWatermark():
"""Tests that the filter watermark is set correctly"""
# Check what the current highest job ID is
highest_jid1 = int(stdout_of(
["gnt-debug", "delay", "--print-jobid", "0.01"]
))
# Add the filter; this sets the watermark
uuid = stdout_of(["gnt-filter", "add"])
# Check what the current highest job ID is
highest_jid2 = int(stdout_of(
["gnt-debug", "delay", "--print-jobid", "0.01"]
))
info_out = stdout_of(["gnt-filter", "info", uuid])
# The second line of gnt-filter info shows the watermark.
watermark = int(
info_out.split('\n')[1].strip().lower().split("watermark: ")[1]
)
# The atermark must be at least as high as the JID of the job we started
# just before the creation, and must be lower than the JID of any job
# created afterwards.
assert highest_jid1 <= watermark < highest_jid2, \
"Watermark not in range: %d <= %d < %d" % (highest_jid1, watermark,
highest_jid2)
# Clean up.
AssertCommand(["gnt-filter", "delete", uuid])
def TestFilterReasonChain():
"""Tests that filters are processed in the right order and that the
"reason" predicate works.
"""
# Add a filter chain that pauses all new jobs apart from those with a
# specific reason.
# Accept all jobs that have the "allow this" reason.
uuid1 = stdout_of([
"gnt-filter",
"add",
'--predicates=[["reason", ["=", "reason", "allow this"]]]',
"--action=ACCEPT",
# Default priority 0
])
# Reject those that haven't (but make the one above run first).
uuid2 = stdout_of([
"gnt-filter",
"add",
'--predicates=[["jobid", [">", "id", "watermark"]]]',
"--action=REJECT",
"--priority=1",
])
# This job must now go into queued status.
AssertCommand(["gnt-debug", "delay", "0.01"], fail=True)
AssertCommand(["gnt-debug", "delay", "--reason=allow this", "0.01"])
# Clean up.
AssertCommand(["gnt-filter", "delete", uuid1])
AssertCommand(["gnt-filter", "delete", uuid2])
def TestFilterReasonChain():
"""Tests that filters are processed in the right order and that the
"reason" predicate works.
"""
# Add a filter chain that pauses all new jobs apart from those with a
# specific reason.
# Accept all jobs that have the "allow this" reason.
uuid1 = stdout_of([
"gnt-filter",
"add",
'--predicates=[["reason", ["=", "reason", "allow this"]]]',
"--action=ACCEPT",
# Default priority 0
])
# Reject those that haven't (but make the one above run first).
uuid2 = stdout_of([
"gnt-filter",
"add",
'--predicates=[["jobid", [">", "id", "watermark"]]]',
"--action=REJECT",
"--priority=1",
])
# This job must now go into queued status.
AssertCommand(["gnt-debug", "delay", "0.01"], fail=True)
AssertCommand(["gnt-debug", "delay", "--reason=allow this", "0.01"])
# Clean up.
AssertCommand(["gnt-filter", "delete", uuid1])
AssertCommand(["gnt-filter", "delete", uuid2])
def TestFilterContinue():
"""Tests that the CONTINUE filter has no effect"""
# Add a filter that just passes to the next filter.
uuid_cont = stdout_of([
"gnt-filter",
"add",
'--predicates=[["jobid", [">", "id", "watermark"]]]',
"--action=CONTINUE",
"--priority=0",
])
# Add a filter that rejects all new jobs.
uuid_reject = stdout_of([
"gnt-filter",
"add",
'--predicates=[["jobid", [">", "id", "watermark"]]]',
"--action=REJECT",
"--priority=1",
])
# Newly queued jobs must now fail.
AssertCommand(["gnt-debug", "delay", "0.01"], fail=True)
# Delete the rejecting filter.
AssertCommand(["gnt-filter", "delete", uuid_reject])
# Newly queued jobs must now succeed.
AssertCommand(["gnt-debug", "delay", "0.01"])
# Clean up.
AssertCommand(["gnt-filter", "delete", uuid_cont])
def TestAdHocReasonRateLimit():
"""Tests that ad-hoc rate limiting using --reason="rate-limit:n:..." works.
"""
# Make sure our test is not constrained by "max-running-jobs"
# (simply set it to the default).
AssertCommand(["gnt-cluster", "modify", "--max-running-jobs=20"])
AssertCommand(["gnt-cluster", "modify", "--max-tracked-jobs=25"])
# Only the first 2 jobs must be scheduled.
jid1 = int(stdout_of([
"gnt-debug", "delay", "--print-jobid", "--submit",
"--reason=rate-limit:2:hello", "200",
]))
jid2 = int(stdout_of([
"gnt-debug", "delay", "--print-jobid", "--submit",
"--reason=rate-limit:2:hello", "200",
]))
jid3 = int(stdout_of([
"gnt-debug", "delay", "--print-jobid", "--submit",
"--reason=rate-limit:2:hello", "200",
]))
time.sleep(5) # give the scheduler some time to notice
AssertIn(GetJobStatus(jid1), ["running", "waiting"],
msg="Job should not be rate-limited")
AssertIn(GetJobStatus(jid2), ["running", "waiting"],
msg="Job should not be rate-limited")
AssertEqual(GetJobStatus(jid3), "queued", msg="Job should be rate-limited")
# Clean up.
KillWaitJobs([jid1, jid2, jid3])
def TestFilterWatermark():
"""Tests that the filter watermark is set correctly"""
# Check what the current highest job ID is
highest_jid1 = int(
stdout_of(["gnt-debug", "delay", "--print-jobid", "0.01"]))
# Add the filter; this sets the watermark
uuid = stdout_of(["gnt-filter", "add"])
# Check what the current highest job ID is
highest_jid2 = int(
stdout_of(["gnt-debug", "delay", "--print-jobid", "0.01"]))
info_out = stdout_of(["gnt-filter", "info", uuid])
# The second line of gnt-filter info shows the watermark.
watermark = int(
info_out.split('\n')[1].strip().lower().split("watermark: ")[1])
# The atermark must be at least as high as the JID of the job we started
# just before the creation, and must be lower than the JID of any job
# created afterwards.
assert highest_jid1 <= watermark < highest_jid2, \
"Watermark not in range: %d <= %d < %d" % (highest_jid1, watermark,
highest_jid2)
# Clean up.
AssertCommand(["gnt-filter", "delete", uuid])
def TestFilterAcceptPause():
"""Tests that the PAUSE filter allows scheduling, but prevents starting,
and that the ACCEPT filter immediately allows starting.
"""
AssertCommand(["gnt-cluster", "watcher", "pause", "600"])
# Add a filter chain that pauses all new jobs apart from those with a
# specific reason.
# When the pausing filter is deleted, paused jobs must be continued.
# Accept all jobs that have the "allow this" reason.
uuid1 = stdout_of([
"gnt-filter",
"add",
'--predicates=[["reason", ["=", "reason", "allow this"]]]',
"--action=ACCEPT",
# Default priority 0
])
# Pause those that haven't (but make the one above run first).
uuid2 = stdout_of([
"gnt-filter",
"add",
'--predicates=[["jobid", [">", "id", "watermark"]]]',
"--action=PAUSE",
"--priority=1",
])
# This job must now go into queued status.
jid1 = int(stdout_of([
"gnt-debug", "delay", "--submit", "--print-jobid", "0.01",
]))
# This job should run and finish.
jid2 = int(stdout_of([
"gnt-debug", "delay", "--submit", "--print-jobid", "--reason=allow this",
"0.01",
]))
time.sleep(5) # give some time to get queued
AssertStatusRetry(jid1, "queued") # job should be paused
AssertStatusRetry(jid2, "success") # job should not be paused
# Delete the filters.
AssertCommand(["gnt-filter", "delete", uuid1])
AssertCommand(["gnt-filter", "delete", uuid2])
# Now the paused job should run through.
time.sleep(5)
AssertStatusRetry(jid1, "success")
AssertCommand(["gnt-cluster", "watcher", "continue"])
def TestFilterAddRemove(): """gnt-filter add/delete""" uuid1 = stdout_of(["gnt-filter", "add", "--reason", "reason1"]) TestFilterList() TestFilterListFields() uuid2 = stdout_of(["gnt-filter", "list", "--no-headers", "--output=uuid"]) AssertEqual(uuid1, uuid2) AssertCommand(["gnt-filter", "delete", uuid1]) TestFilterList()
def TestFilterAddRemove():
"""gnt-filter add/delete"""
uuid1 = stdout_of(["gnt-filter", "add", "--reason", "reason1"])
TestFilterList()
TestFilterListFields()
uuid2 = stdout_of(["gnt-filter", "list", "--no-headers", "--output=uuid"])
AssertEqual(uuid1, uuid2)
AssertCommand(["gnt-filter", "delete", uuid1])
TestFilterList()
def TestAdHocReasonRateLimit():
"""Tests that ad-hoc rate limiting using --reason="rate-limit:n:..." works.
"""
# Make sure our test is not constrained by "max-running-jobs"
# (simply set it to the default).
AssertCommand(["gnt-cluster", "modify", "--max-running-jobs=20"])
AssertCommand(["gnt-cluster", "modify", "--max-tracked-jobs=25"])
# Only the first 2 jobs must be scheduled.
jid1 = int(
stdout_of([
"gnt-debug",
"delay",
"--print-jobid",
"--submit",
"--reason=rate-limit:2:hello",
"200",
]))
jid2 = int(
stdout_of([
"gnt-debug",
"delay",
"--print-jobid",
"--submit",
"--reason=rate-limit:2:hello",
"200",
]))
jid3 = int(
stdout_of([
"gnt-debug",
"delay",
"--print-jobid",
"--submit",
"--reason=rate-limit:2:hello",
"200",
]))
time.sleep(5) # give the scheduler some time to notice
AssertIn(GetJobStatus(jid1), ["running", "waiting"],
msg="Job should not be rate-limited")
AssertIn(GetJobStatus(jid2), ["running", "waiting"],
msg="Job should not be rate-limited")
AssertEqual(GetJobStatus(jid3), "queued", msg="Job should be rate-limited")
# Clean up.
KillWaitJobs([jid1, jid2, jid3])
def fn():
out = stdout_of([
"gnt-job", "list", "--output=status", "--no-headers", "--filter",
'"REPAIR_COMMAND" in summary'
])
if 'success' not in out:
raise retry.RetryAgain()
def fn():
out = stdout_of([
"gnt-job", "list", "--output=status", "--no-headers", "--filter",
'"%s(%s)" in summary' % (move_type, inst.name)
])
if 'success' not in out:
raise retry.RetryAgain()
def fn():
out = stdout_of([
"gnt-node", "list", "--output=name", "--no-headers", "--filter",
"drained"
])
if node.primary not in out:
raise retry.RetryAgain()
def fn():
out = stdout_of([
"gnt-instance", "list", "--output=status", "--no-headers",
"--filter",
"name == \"%s\"" % inst.name
])
if "running" not in out:
raise retry.RetryAgain()
def fn():
out = stdout_of(["gnt-job",
"list",
"--output=status",
"--no-headers",
"--filter",
'"%s(%s)" in summary' % (move_type, inst.name)
])
if 'success' not in out:
raise retry.RetryAgain()
def fn():
out = stdout_of(["gnt-job",
"list",
"--output=status",
"--no-headers",
"--filter",
'"REPAIR_COMMAND" in summary'
])
if 'success' not in out:
raise retry.RetryAgain()
def fn():
out = stdout_of(["gnt-node",
"list",
"--output=name",
"--no-headers",
"--filter",
"drained"
])
if node.primary not in out:
raise retry.RetryAgain()
def fn():
out = stdout_of(["gnt-instance",
"list",
"--output=status",
"--no-headers",
"--filter",
"name == \"%s\"" % inst.name
])
if "running" not in out:
raise retry.RetryAgain()
def GetJobStatus(job_id):
"""Queries the status of the job by parsing output of gnt-job info.
@type job_id: int
@param job_id: ID of the job to query
@return: status of the job as lower-case string
"""
out = stdout_of(["gnt-job", "info", str(job_id)])
# The second line of gnt-job info shows the status.
return out.split('\n')[1].strip().lower().split("status: ")[1]
def GetJobStatus(job_id):
"""Queries the status of the job by parsing output of gnt-job info.
@type job_id: int
@param job_id: ID of the job to query
@return: status of the job as lower-case string
"""
out = stdout_of(["gnt-job", "info", str(job_id)])
# The second line of gnt-job info shows the status.
return out.split('\n')[1].strip().lower().split("status: ")[1]
def TestFilterRateLimit():
"""Tests that the RATE_LIMIT filter does reject new jobs when all
rate-limiting buckets are taken.
"""
# Make sure our test is not constrained by "max-running-jobs"
# (simply set it to the default).
AssertCommand(["gnt-cluster", "modify", "--max-running-jobs=20"])
AssertCommand(["gnt-cluster", "modify", "--max-tracked-jobs=25"])
AssertCommand(["gnt-cluster", "watcher", "pause", "600"])
# Add a filter that rejects all new jobs.
uuid = stdout_of([
"gnt-filter", "add",
'--predicates=[["jobid", [">", "id", "watermark"]]]',
"--action=RATE_LIMIT 2",
])
# Now only the first 2 jobs must be scheduled.
jid1 = int(stdout_of([
"gnt-debug", "delay", "--print-jobid", "--submit", "200"
]))
jid2 = int(stdout_of([
"gnt-debug", "delay", "--print-jobid", "--submit", "200"
]))
jid3 = int(stdout_of([
"gnt-debug", "delay", "--print-jobid", "--submit", "200"
]))
time.sleep(5) # give the scheduler some time to notice
AssertIn(GetJobStatus(jid1), ["running", "waiting"],
msg="Job should not be rate-limited")
AssertIn(GetJobStatus(jid2), ["running", "waiting"],
msg="Job should not be rate-limited")
AssertEqual(GetJobStatus(jid3), "queued", msg="Job should be rate-limited")
# Clean up.
AssertCommand(["gnt-filter", "delete", uuid])
KillWaitJobs([jid1, jid2, jid3])
AssertCommand(["gnt-cluster", "watcher", "continue"])
def TestFilterRateLimit():
"""Tests that the RATE_LIMIT filter does reject new jobs when all
rate-limiting buckets are taken.
"""
# Make sure our test is not constrained by "max-running-jobs"
# (simply set it to the default).
AssertCommand(["gnt-cluster", "modify", "--max-running-jobs=20"])
AssertCommand(["gnt-cluster", "modify", "--max-tracked-jobs=25"])
AssertCommand(["gnt-cluster", "watcher", "pause", "600"])
# Add a filter that rejects all new jobs.
uuid = stdout_of([
"gnt-filter",
"add",
'--predicates=[["jobid", [">", "id", "watermark"]]]',
"--action=RATE_LIMIT 2",
])
# Now only the first 2 jobs must be scheduled.
jid1 = int(
stdout_of(["gnt-debug", "delay", "--print-jobid", "--submit", "200"]))
jid2 = int(
stdout_of(["gnt-debug", "delay", "--print-jobid", "--submit", "200"]))
jid3 = int(
stdout_of(["gnt-debug", "delay", "--print-jobid", "--submit", "200"]))
time.sleep(5) # give the scheduler some time to notice
AssertIn(GetJobStatus(jid1), ["running", "waiting"],
msg="Job should not be rate-limited")
AssertIn(GetJobStatus(jid2), ["running", "waiting"],
msg="Job should not be rate-limited")
AssertEqual(GetJobStatus(jid3), "queued", msg="Job should be rate-limited")
# Clean up.
AssertCommand(["gnt-filter", "delete", uuid])
KillWaitJobs([jid1, jid2, jid3])
AssertCommand(["gnt-cluster", "watcher", "continue"])
def TestFilterReject():
"""Tests that the REJECT filter does reject new jobs and that the
"jobid" predicate works.
"""
# Add a filter that rejects all new jobs.
uuid = stdout_of([
"gnt-filter", "add",
'--predicates=[["jobid", [">", "id", "watermark"]]]',
"--action=REJECT",
])
# Newly queued jobs must now fail.
AssertCommand(["gnt-debug", "delay", "0.01"], fail=True)
# Clean up.
AssertCommand(["gnt-filter", "delete", uuid])
def TestFilterOpCode():
"""Tests that filtering with the "opcode" predicate works"""
# Check that delay jobs work fine.
AssertCommand(["gnt-debug", "delay", "0.01"])
# Add a filter that rejects all new delay jobs.
uuid = stdout_of([
"gnt-filter", "add",
'--predicates=[["opcode", ["=", "OP_ID", "OP_TEST_DELAY"]]]',
"--action=REJECT",
])
# Newly queued delay jobs must now fail.
AssertCommand(["gnt-debug", "delay", "0.01"], fail=True)
# Clean up.
AssertCommand(["gnt-filter", "delete", uuid])
def TestFilterReject():
"""Tests that the REJECT filter does reject new jobs and that the
"jobid" predicate works.
"""
# Add a filter that rejects all new jobs.
uuid = stdout_of([
"gnt-filter",
"add",
'--predicates=[["jobid", [">", "id", "watermark"]]]',
"--action=REJECT",
])
# Newly queued jobs must now fail.
AssertCommand(["gnt-debug", "delay", "0.01"], fail=True)
# Clean up.
AssertCommand(["gnt-filter", "delete", uuid])
def TestFilterOpCode():
"""Tests that filtering with the "opcode" predicate works"""
# Check that delay jobs work fine.
AssertCommand(["gnt-debug", "delay", "0.01"])
# Add a filter that rejects all new delay jobs.
uuid = stdout_of([
"gnt-filter",
"add",
'--predicates=[["opcode", ["=", "OP_ID", "OP_TEST_DELAY"]]]',
"--action=REJECT",
])
# Newly queued delay jobs must now fail.
AssertCommand(["gnt-debug", "delay", "0.01"], fail=True)
# Clean up.
AssertCommand(["gnt-filter", "delete", uuid])
def TestFilterAcceptPause():
"""Tests that the PAUSE filter allows scheduling, but prevents starting,
and that the ACCEPT filter immediately allows starting.
"""
AssertCommand(["gnt-cluster", "watcher", "pause", "600"])
# Add a filter chain that pauses all new jobs apart from those with a
# specific reason.
# When the pausing filter is deleted, paused jobs must be continued.
# Accept all jobs that have the "allow this" reason.
uuid1 = stdout_of([
"gnt-filter",
"add",
'--predicates=[["reason", ["=", "reason", "allow this"]]]',
"--action=ACCEPT",
# Default priority 0
])
# Pause those that haven't (but make the one above run first).
uuid2 = stdout_of([
"gnt-filter",
"add",
'--predicates=[["jobid", [">", "id", "watermark"]]]',
"--action=PAUSE",
"--priority=1",
])
# This job must now go into queued status.
jid1 = int(
stdout_of([
"gnt-debug",
"delay",
"--submit",
"--print-jobid",
"0.01",
]))
# This job should run and finish.
jid2 = int(
stdout_of([
"gnt-debug",
"delay",
"--submit",
"--print-jobid",
"--reason=allow this",
"0.01",
]))
time.sleep(5) # give some time to get queued
AssertStatusRetry(jid1, "queued") # job should be paused
AssertStatusRetry(jid2, "success") # job should not be paused
# Delete the filters.
AssertCommand(["gnt-filter", "delete", uuid1])
AssertCommand(["gnt-filter", "delete", uuid2])
# Now the paused job should run through.
time.sleep(5)
AssertStatusRetry(jid1, "success")
AssertCommand(["gnt-cluster", "watcher", "continue"])
def _GetMaintTags(node):
tags = stdout_of(["gnt-node", "list-tags", node.primary]).split()
return [t for t in tags if t.startswith('maintd:repairready:')]
def _GetMaintTags(node):
tags = stdout_of(["gnt-node",
"list-tags",
node.primary
]).split()
return [t for t in tags if t.startswith('maintd:repairready:')]
def RunWithLocks(fn, locks, timeout, block, *args, **kwargs):
""" Runs the given function, acquiring a set of locks beforehand.
@type fn: function
@param fn: The function to invoke.
@type locks: dict of string to list of string
@param locks: The locks to acquire, per lock category.
@type timeout: number
@param timeout: The number of seconds the locks should be held before
expiring.
@type block: bool
@param block: Whether the test should block when locks are used or not.
This function allows a set of locks to be acquired in preparation for a QA
test, to try and see if the function can run in parallel with other
operations.
Locks are acquired by invoking a gnt-debug delay operation which can be
interrupted as needed. The QA test is then run in a separate thread, with the
current thread observing jobs waiting for locks. When a job is spotted waiting
for a lock held by the started delay operation, this is noted, and the delay
is interrupted, allowing the QA test to continue.
A default timeout is not provided by design - the test creator must make a
good conservative estimate.
"""
if filter(lambda l_type: l_type not in AVAILABLE_LOCKS, locks):
raise qa_error.Error("Attempted to acquire locks that cannot yet be "
"acquired in the course of a QA test.")
# The watcher may interfere by issuing its own jobs - therefore pause it
# also reject all its jobs and wait for any running jobs to finish.
AssertCommand(["gnt-cluster", "watcher", "pause", "12h"])
filter_uuid = stdout_of([
"gnt-filter", "add",
'--predicates=[["reason", ["=", "source", "gnt:watcher"]]]',
"--action=REJECT"
])
while stdout_of(["gnt-job", "list", "--no-header", "--running"]) != "":
time.sleep(1)
# Find out the lock names prior to starting the delay function
lock_name_map = _FindLockNames(locks)
blocking_owned_locks = []
test_blocked = False
termination_socket = _StartDelayFunction(locks, timeout)
delay_fn_terminated = False
try:
qa_thread = QAThread(fn, args, kwargs)
qa_thread.start()
while qa_thread.isAlive():
blocking_locks = _GetBlockingLocks()
blocking_owned_locks = \
set(blocking_locks).intersection(set(lock_name_map))
if blocking_owned_locks:
# Set the flag first - if the termination attempt fails, we do not want
# to redo it in the finally block
delay_fn_terminated = True
_TerminateDelayFunction(termination_socket)
test_blocked = True
break
time.sleep(5) # Set arbitrarily
# The thread should be either finished or unblocked at this point
qa_thread.join()
# Raise any errors that might have occured in the thread
qa_thread.reraise()
finally:
if not delay_fn_terminated:
_TerminateDelayFunction(termination_socket)
blocking_lock_names = ", ".join(
map(lock_name_map.get, blocking_owned_locks))
if not block and test_blocked:
raise qa_error.Error("QA test succeded, but was blocked by locks: %s" %
blocking_lock_names)
elif block and not test_blocked:
raise qa_error.Error("QA test succeded, but was not blocked as it was "
"expected to by locks: %s" % blocking_lock_names)
else:
pass
# Revive the watcher
AssertCommand(["gnt-filter", "delete", filter_uuid])
AssertCommand(["gnt-cluster", "watcher", "continue"])
def RunWithLocks(fn, locks, timeout, block, *args, **kwargs):
""" Runs the given function, acquiring a set of locks beforehand.
@type fn: function
@param fn: The function to invoke.
@type locks: dict of string to list of string
@param locks: The locks to acquire, per lock category.
@type timeout: number
@param timeout: The number of seconds the locks should be held before
expiring.
@type block: bool
@param block: Whether the test should block when locks are used or not.
This function allows a set of locks to be acquired in preparation for a QA
test, to try and see if the function can run in parallel with other
operations.
Locks are acquired by invoking a gnt-debug delay operation which can be
interrupted as needed. The QA test is then run in a separate thread, with the
current thread observing jobs waiting for locks. When a job is spotted waiting
for a lock held by the started delay operation, this is noted, and the delay
is interrupted, allowing the QA test to continue.
A default timeout is not provided by design - the test creator must make a
good conservative estimate.
"""
if filter(lambda l_type: l_type not in AVAILABLE_LOCKS, locks):
raise qa_error.Error("Attempted to acquire locks that cannot yet be "
"acquired in the course of a QA test.")
# The watcher may interfere by issuing its own jobs - therefore pause it
# also reject all its jobs and wait for any running jobs to finish.
AssertCommand(["gnt-cluster", "watcher", "pause", "12h"])
filter_uuid = stdout_of([
"gnt-filter", "add",
'--predicates=[["reason", ["=", "source", "gnt:watcher"]]]',
"--action=REJECT"
])
while stdout_of(["gnt-job", "list", "--no-header", "--running"]) != "":
time.sleep(1)
# Find out the lock names prior to starting the delay function
lock_name_map = _FindLockNames(locks)
blocking_owned_locks = []
test_blocked = False
termination_socket = _StartDelayFunction(locks, timeout)
delay_fn_terminated = False
try:
qa_thread = QAThread(fn, args, kwargs)
qa_thread.start()
while qa_thread.isAlive():
blocking_locks = _GetBlockingLocks()
blocking_owned_locks = \
set(blocking_locks).intersection(set(lock_name_map))
if blocking_owned_locks:
# Set the flag first - if the termination attempt fails, we do not want
# to redo it in the finally block
delay_fn_terminated = True
_TerminateDelayFunction(termination_socket)
test_blocked = True
break
time.sleep(5) # Set arbitrarily
# The thread should be either finished or unblocked at this point
qa_thread.join()
# Raise any errors that might have occured in the thread
qa_thread.reraise()
finally:
if not delay_fn_terminated:
_TerminateDelayFunction(termination_socket)
blocking_lock_names = ", ".join(map(lock_name_map.get, blocking_owned_locks))
if not block and test_blocked:
raise qa_error.Error("QA test succeded, but was blocked by locks: %s" %
blocking_lock_names)
elif block and not test_blocked:
raise qa_error.Error("QA test succeded, but was not blocked as it was "
"expected to by locks: %s" % blocking_lock_names)
else:
pass
# Revive the watcher
AssertCommand(["gnt-filter", "delete", filter_uuid])
AssertCommand(["gnt-cluster", "watcher", "continue"])
