def TestFilterContinue(): """Tests that the CONTINUE filter has no effect""" # Add a filter that just passes to the next filter. uuid_cont = stdout_of([ "gnt-filter", "add", '--predicates=[["jobid", [">", "id", "watermark"]]]', "--action=CONTINUE", "--priority=0", ]) # Add a filter that rejects all new jobs. uuid_reject = stdout_of([ "gnt-filter", "add", '--predicates=[["jobid", [">", "id", "watermark"]]]', "--action=REJECT", "--priority=1", ]) # Newly queued jobs must now fail. AssertCommand(["gnt-debug", "delay", "0.01"], fail=True) # Delete the rejecting filter. AssertCommand(["gnt-filter", "delete", uuid_reject]) # Newly queued jobs must now succeed. AssertCommand(["gnt-debug", "delay", "0.01"]) # Clean up. AssertCommand(["gnt-filter", "delete", uuid_cont])
def TestFilterWatermark(): """Tests that the filter watermark is set correctly""" # Check what the current highest job ID is highest_jid1 = int(stdout_of( ["gnt-debug", "delay", "--print-jobid", "0.01"] )) # Add the filter; this sets the watermark uuid = stdout_of(["gnt-filter", "add"]) # Check what the current highest job ID is highest_jid2 = int(stdout_of( ["gnt-debug", "delay", "--print-jobid", "0.01"] )) info_out = stdout_of(["gnt-filter", "info", uuid]) # The second line of gnt-filter info shows the watermark. watermark = int( info_out.split('\n')[1].strip().lower().split("watermark: ")[1] ) # The atermark must be at least as high as the JID of the job we started # just before the creation, and must be lower than the JID of any job # created afterwards. assert highest_jid1 <= watermark < highest_jid2, \ "Watermark not in range: %d <= %d < %d" % (highest_jid1, watermark, highest_jid2) # Clean up. AssertCommand(["gnt-filter", "delete", uuid])
def TestFilterReasonChain(): """Tests that filters are processed in the right order and that the "reason" predicate works. """ # Add a filter chain that pauses all new jobs apart from those with a # specific reason. # Accept all jobs that have the "allow this" reason. uuid1 = stdout_of([ "gnt-filter", "add", '--predicates=[["reason", ["=", "reason", "allow this"]]]', "--action=ACCEPT", # Default priority 0 ]) # Reject those that haven't (but make the one above run first). uuid2 = stdout_of([ "gnt-filter", "add", '--predicates=[["jobid", [">", "id", "watermark"]]]', "--action=REJECT", "--priority=1", ]) # This job must now go into queued status. AssertCommand(["gnt-debug", "delay", "0.01"], fail=True) AssertCommand(["gnt-debug", "delay", "--reason=allow this", "0.01"]) # Clean up. AssertCommand(["gnt-filter", "delete", uuid1]) AssertCommand(["gnt-filter", "delete", uuid2])
def TestFilterReasonChain(): """Tests that filters are processed in the right order and that the "reason" predicate works. """ # Add a filter chain that pauses all new jobs apart from those with a # specific reason. # Accept all jobs that have the "allow this" reason. uuid1 = stdout_of([ "gnt-filter", "add", '--predicates=[["reason", ["=", "reason", "allow this"]]]', "--action=ACCEPT", # Default priority 0 ]) # Reject those that haven't (but make the one above run first). uuid2 = stdout_of([ "gnt-filter", "add", '--predicates=[["jobid", [">", "id", "watermark"]]]', "--action=REJECT", "--priority=1", ]) # This job must now go into queued status. AssertCommand(["gnt-debug", "delay", "0.01"], fail=True) AssertCommand(["gnt-debug", "delay", "--reason=allow this", "0.01"]) # Clean up. AssertCommand(["gnt-filter", "delete", uuid1]) AssertCommand(["gnt-filter", "delete", uuid2])
def TestFilterContinue(): """Tests that the CONTINUE filter has no effect""" # Add a filter that just passes to the next filter. uuid_cont = stdout_of([ "gnt-filter", "add", '--predicates=[["jobid", [">", "id", "watermark"]]]', "--action=CONTINUE", "--priority=0", ]) # Add a filter that rejects all new jobs. uuid_reject = stdout_of([ "gnt-filter", "add", '--predicates=[["jobid", [">", "id", "watermark"]]]', "--action=REJECT", "--priority=1", ]) # Newly queued jobs must now fail. AssertCommand(["gnt-debug", "delay", "0.01"], fail=True) # Delete the rejecting filter. AssertCommand(["gnt-filter", "delete", uuid_reject]) # Newly queued jobs must now succeed. AssertCommand(["gnt-debug", "delay", "0.01"]) # Clean up. AssertCommand(["gnt-filter", "delete", uuid_cont])
def TestAdHocReasonRateLimit(): """Tests that ad-hoc rate limiting using --reason="rate-limit:n:..." works. """ # Make sure our test is not constrained by "max-running-jobs" # (simply set it to the default). AssertCommand(["gnt-cluster", "modify", "--max-running-jobs=20"]) AssertCommand(["gnt-cluster", "modify", "--max-tracked-jobs=25"]) # Only the first 2 jobs must be scheduled. jid1 = int(stdout_of([ "gnt-debug", "delay", "--print-jobid", "--submit", "--reason=rate-limit:2:hello", "200", ])) jid2 = int(stdout_of([ "gnt-debug", "delay", "--print-jobid", "--submit", "--reason=rate-limit:2:hello", "200", ])) jid3 = int(stdout_of([ "gnt-debug", "delay", "--print-jobid", "--submit", "--reason=rate-limit:2:hello", "200", ])) time.sleep(5) # give the scheduler some time to notice AssertIn(GetJobStatus(jid1), ["running", "waiting"], msg="Job should not be rate-limited") AssertIn(GetJobStatus(jid2), ["running", "waiting"], msg="Job should not be rate-limited") AssertEqual(GetJobStatus(jid3), "queued", msg="Job should be rate-limited") # Clean up. KillWaitJobs([jid1, jid2, jid3])
def TestFilterWatermark(): """Tests that the filter watermark is set correctly""" # Check what the current highest job ID is highest_jid1 = int( stdout_of(["gnt-debug", "delay", "--print-jobid", "0.01"])) # Add the filter; this sets the watermark uuid = stdout_of(["gnt-filter", "add"]) # Check what the current highest job ID is highest_jid2 = int( stdout_of(["gnt-debug", "delay", "--print-jobid", "0.01"])) info_out = stdout_of(["gnt-filter", "info", uuid]) # The second line of gnt-filter info shows the watermark. watermark = int( info_out.split('\n')[1].strip().lower().split("watermark: ")[1]) # The atermark must be at least as high as the JID of the job we started # just before the creation, and must be lower than the JID of any job # created afterwards. assert highest_jid1 <= watermark < highest_jid2, \ "Watermark not in range: %d <= %d < %d" % (highest_jid1, watermark, highest_jid2) # Clean up. AssertCommand(["gnt-filter", "delete", uuid])
def TestFilterAcceptPause(): """Tests that the PAUSE filter allows scheduling, but prevents starting, and that the ACCEPT filter immediately allows starting. """ AssertCommand(["gnt-cluster", "watcher", "pause", "600"]) # Add a filter chain that pauses all new jobs apart from those with a # specific reason. # When the pausing filter is deleted, paused jobs must be continued. # Accept all jobs that have the "allow this" reason. uuid1 = stdout_of([ "gnt-filter", "add", '--predicates=[["reason", ["=", "reason", "allow this"]]]', "--action=ACCEPT", # Default priority 0 ]) # Pause those that haven't (but make the one above run first). uuid2 = stdout_of([ "gnt-filter", "add", '--predicates=[["jobid", [">", "id", "watermark"]]]', "--action=PAUSE", "--priority=1", ]) # This job must now go into queued status. jid1 = int(stdout_of([ "gnt-debug", "delay", "--submit", "--print-jobid", "0.01", ])) # This job should run and finish. jid2 = int(stdout_of([ "gnt-debug", "delay", "--submit", "--print-jobid", "--reason=allow this", "0.01", ])) time.sleep(5) # give some time to get queued AssertStatusRetry(jid1, "queued") # job should be paused AssertStatusRetry(jid2, "success") # job should not be paused # Delete the filters. AssertCommand(["gnt-filter", "delete", uuid1]) AssertCommand(["gnt-filter", "delete", uuid2]) # Now the paused job should run through. time.sleep(5) AssertStatusRetry(jid1, "success") AssertCommand(["gnt-cluster", "watcher", "continue"])
def TestFilterAddRemove(): """gnt-filter add/delete""" uuid1 = stdout_of(["gnt-filter", "add", "--reason", "reason1"]) TestFilterList() TestFilterListFields() uuid2 = stdout_of(["gnt-filter", "list", "--no-headers", "--output=uuid"]) AssertEqual(uuid1, uuid2) AssertCommand(["gnt-filter", "delete", uuid1]) TestFilterList()
def TestFilterAddRemove(): """gnt-filter add/delete""" uuid1 = stdout_of(["gnt-filter", "add", "--reason", "reason1"]) TestFilterList() TestFilterListFields() uuid2 = stdout_of(["gnt-filter", "list", "--no-headers", "--output=uuid"]) AssertEqual(uuid1, uuid2) AssertCommand(["gnt-filter", "delete", uuid1]) TestFilterList()
def TestAdHocReasonRateLimit(): """Tests that ad-hoc rate limiting using --reason="rate-limit:n:..." works. """ # Make sure our test is not constrained by "max-running-jobs" # (simply set it to the default). AssertCommand(["gnt-cluster", "modify", "--max-running-jobs=20"]) AssertCommand(["gnt-cluster", "modify", "--max-tracked-jobs=25"]) # Only the first 2 jobs must be scheduled. jid1 = int( stdout_of([ "gnt-debug", "delay", "--print-jobid", "--submit", "--reason=rate-limit:2:hello", "200", ])) jid2 = int( stdout_of([ "gnt-debug", "delay", "--print-jobid", "--submit", "--reason=rate-limit:2:hello", "200", ])) jid3 = int( stdout_of([ "gnt-debug", "delay", "--print-jobid", "--submit", "--reason=rate-limit:2:hello", "200", ])) time.sleep(5) # give the scheduler some time to notice AssertIn(GetJobStatus(jid1), ["running", "waiting"], msg="Job should not be rate-limited") AssertIn(GetJobStatus(jid2), ["running", "waiting"], msg="Job should not be rate-limited") AssertEqual(GetJobStatus(jid3), "queued", msg="Job should be rate-limited") # Clean up. KillWaitJobs([jid1, jid2, jid3])
def fn(): out = stdout_of([ "gnt-job", "list", "--output=status", "--no-headers", "--filter", '"REPAIR_COMMAND" in summary' ]) if 'success' not in out: raise retry.RetryAgain()
def fn(): out = stdout_of([ "gnt-job", "list", "--output=status", "--no-headers", "--filter", '"%s(%s)" in summary' % (move_type, inst.name) ]) if 'success' not in out: raise retry.RetryAgain()
def fn(): out = stdout_of([ "gnt-node", "list", "--output=name", "--no-headers", "--filter", "drained" ]) if node.primary not in out: raise retry.RetryAgain()
def fn(): out = stdout_of([ "gnt-instance", "list", "--output=status", "--no-headers", "--filter", "name == \"%s\"" % inst.name ]) if "running" not in out: raise retry.RetryAgain()
def fn(): out = stdout_of(["gnt-job", "list", "--output=status", "--no-headers", "--filter", '"%s(%s)" in summary' % (move_type, inst.name) ]) if 'success' not in out: raise retry.RetryAgain()
def fn(): out = stdout_of(["gnt-job", "list", "--output=status", "--no-headers", "--filter", '"REPAIR_COMMAND" in summary' ]) if 'success' not in out: raise retry.RetryAgain()
def fn(): out = stdout_of(["gnt-node", "list", "--output=name", "--no-headers", "--filter", "drained" ]) if node.primary not in out: raise retry.RetryAgain()
def fn(): out = stdout_of(["gnt-instance", "list", "--output=status", "--no-headers", "--filter", "name == \"%s\"" % inst.name ]) if "running" not in out: raise retry.RetryAgain()
def GetJobStatus(job_id): """Queries the status of the job by parsing output of gnt-job info. @type job_id: int @param job_id: ID of the job to query @return: status of the job as lower-case string """ out = stdout_of(["gnt-job", "info", str(job_id)]) # The second line of gnt-job info shows the status. return out.split('\n')[1].strip().lower().split("status: ")[1]
def GetJobStatus(job_id): """Queries the status of the job by parsing output of gnt-job info. @type job_id: int @param job_id: ID of the job to query @return: status of the job as lower-case string """ out = stdout_of(["gnt-job", "info", str(job_id)]) # The second line of gnt-job info shows the status. return out.split('\n')[1].strip().lower().split("status: ")[1]
def TestFilterRateLimit(): """Tests that the RATE_LIMIT filter does reject new jobs when all rate-limiting buckets are taken. """ # Make sure our test is not constrained by "max-running-jobs" # (simply set it to the default). AssertCommand(["gnt-cluster", "modify", "--max-running-jobs=20"]) AssertCommand(["gnt-cluster", "modify", "--max-tracked-jobs=25"]) AssertCommand(["gnt-cluster", "watcher", "pause", "600"]) # Add a filter that rejects all new jobs. uuid = stdout_of([ "gnt-filter", "add", '--predicates=[["jobid", [">", "id", "watermark"]]]', "--action=RATE_LIMIT 2", ]) # Now only the first 2 jobs must be scheduled. jid1 = int(stdout_of([ "gnt-debug", "delay", "--print-jobid", "--submit", "200" ])) jid2 = int(stdout_of([ "gnt-debug", "delay", "--print-jobid", "--submit", "200" ])) jid3 = int(stdout_of([ "gnt-debug", "delay", "--print-jobid", "--submit", "200" ])) time.sleep(5) # give the scheduler some time to notice AssertIn(GetJobStatus(jid1), ["running", "waiting"], msg="Job should not be rate-limited") AssertIn(GetJobStatus(jid2), ["running", "waiting"], msg="Job should not be rate-limited") AssertEqual(GetJobStatus(jid3), "queued", msg="Job should be rate-limited") # Clean up. AssertCommand(["gnt-filter", "delete", uuid]) KillWaitJobs([jid1, jid2, jid3]) AssertCommand(["gnt-cluster", "watcher", "continue"])
def TestFilterRateLimit(): """Tests that the RATE_LIMIT filter does reject new jobs when all rate-limiting buckets are taken. """ # Make sure our test is not constrained by "max-running-jobs" # (simply set it to the default). AssertCommand(["gnt-cluster", "modify", "--max-running-jobs=20"]) AssertCommand(["gnt-cluster", "modify", "--max-tracked-jobs=25"]) AssertCommand(["gnt-cluster", "watcher", "pause", "600"]) # Add a filter that rejects all new jobs. uuid = stdout_of([ "gnt-filter", "add", '--predicates=[["jobid", [">", "id", "watermark"]]]', "--action=RATE_LIMIT 2", ]) # Now only the first 2 jobs must be scheduled. jid1 = int( stdout_of(["gnt-debug", "delay", "--print-jobid", "--submit", "200"])) jid2 = int( stdout_of(["gnt-debug", "delay", "--print-jobid", "--submit", "200"])) jid3 = int( stdout_of(["gnt-debug", "delay", "--print-jobid", "--submit", "200"])) time.sleep(5) # give the scheduler some time to notice AssertIn(GetJobStatus(jid1), ["running", "waiting"], msg="Job should not be rate-limited") AssertIn(GetJobStatus(jid2), ["running", "waiting"], msg="Job should not be rate-limited") AssertEqual(GetJobStatus(jid3), "queued", msg="Job should be rate-limited") # Clean up. AssertCommand(["gnt-filter", "delete", uuid]) KillWaitJobs([jid1, jid2, jid3]) AssertCommand(["gnt-cluster", "watcher", "continue"])
def TestFilterReject(): """Tests that the REJECT filter does reject new jobs and that the "jobid" predicate works. """ # Add a filter that rejects all new jobs. uuid = stdout_of([ "gnt-filter", "add", '--predicates=[["jobid", [">", "id", "watermark"]]]', "--action=REJECT", ]) # Newly queued jobs must now fail. AssertCommand(["gnt-debug", "delay", "0.01"], fail=True) # Clean up. AssertCommand(["gnt-filter", "delete", uuid])
def TestFilterOpCode(): """Tests that filtering with the "opcode" predicate works""" # Check that delay jobs work fine. AssertCommand(["gnt-debug", "delay", "0.01"]) # Add a filter that rejects all new delay jobs. uuid = stdout_of([ "gnt-filter", "add", '--predicates=[["opcode", ["=", "OP_ID", "OP_TEST_DELAY"]]]', "--action=REJECT", ]) # Newly queued delay jobs must now fail. AssertCommand(["gnt-debug", "delay", "0.01"], fail=True) # Clean up. AssertCommand(["gnt-filter", "delete", uuid])
def TestFilterReject(): """Tests that the REJECT filter does reject new jobs and that the "jobid" predicate works. """ # Add a filter that rejects all new jobs. uuid = stdout_of([ "gnt-filter", "add", '--predicates=[["jobid", [">", "id", "watermark"]]]', "--action=REJECT", ]) # Newly queued jobs must now fail. AssertCommand(["gnt-debug", "delay", "0.01"], fail=True) # Clean up. AssertCommand(["gnt-filter", "delete", uuid])
def TestFilterOpCode(): """Tests that filtering with the "opcode" predicate works""" # Check that delay jobs work fine. AssertCommand(["gnt-debug", "delay", "0.01"]) # Add a filter that rejects all new delay jobs. uuid = stdout_of([ "gnt-filter", "add", '--predicates=[["opcode", ["=", "OP_ID", "OP_TEST_DELAY"]]]', "--action=REJECT", ]) # Newly queued delay jobs must now fail. AssertCommand(["gnt-debug", "delay", "0.01"], fail=True) # Clean up. AssertCommand(["gnt-filter", "delete", uuid])
def TestFilterAcceptPause(): """Tests that the PAUSE filter allows scheduling, but prevents starting, and that the ACCEPT filter immediately allows starting. """ AssertCommand(["gnt-cluster", "watcher", "pause", "600"]) # Add a filter chain that pauses all new jobs apart from those with a # specific reason. # When the pausing filter is deleted, paused jobs must be continued. # Accept all jobs that have the "allow this" reason. uuid1 = stdout_of([ "gnt-filter", "add", '--predicates=[["reason", ["=", "reason", "allow this"]]]', "--action=ACCEPT", # Default priority 0 ]) # Pause those that haven't (but make the one above run first). uuid2 = stdout_of([ "gnt-filter", "add", '--predicates=[["jobid", [">", "id", "watermark"]]]', "--action=PAUSE", "--priority=1", ]) # This job must now go into queued status. jid1 = int( stdout_of([ "gnt-debug", "delay", "--submit", "--print-jobid", "0.01", ])) # This job should run and finish. jid2 = int( stdout_of([ "gnt-debug", "delay", "--submit", "--print-jobid", "--reason=allow this", "0.01", ])) time.sleep(5) # give some time to get queued AssertStatusRetry(jid1, "queued") # job should be paused AssertStatusRetry(jid2, "success") # job should not be paused # Delete the filters. AssertCommand(["gnt-filter", "delete", uuid1]) AssertCommand(["gnt-filter", "delete", uuid2]) # Now the paused job should run through. time.sleep(5) AssertStatusRetry(jid1, "success") AssertCommand(["gnt-cluster", "watcher", "continue"])
def _GetMaintTags(node): tags = stdout_of(["gnt-node", "list-tags", node.primary]).split() return [t for t in tags if t.startswith('maintd:repairready:')]
def _GetMaintTags(node): tags = stdout_of(["gnt-node", "list-tags", node.primary ]).split() return [t for t in tags if t.startswith('maintd:repairready:')]
def RunWithLocks(fn, locks, timeout, block, *args, **kwargs): """ Runs the given function, acquiring a set of locks beforehand. @type fn: function @param fn: The function to invoke. @type locks: dict of string to list of string @param locks: The locks to acquire, per lock category. @type timeout: number @param timeout: The number of seconds the locks should be held before expiring. @type block: bool @param block: Whether the test should block when locks are used or not. This function allows a set of locks to be acquired in preparation for a QA test, to try and see if the function can run in parallel with other operations. Locks are acquired by invoking a gnt-debug delay operation which can be interrupted as needed. The QA test is then run in a separate thread, with the current thread observing jobs waiting for locks. When a job is spotted waiting for a lock held by the started delay operation, this is noted, and the delay is interrupted, allowing the QA test to continue. A default timeout is not provided by design - the test creator must make a good conservative estimate. """ if filter(lambda l_type: l_type not in AVAILABLE_LOCKS, locks): raise qa_error.Error("Attempted to acquire locks that cannot yet be " "acquired in the course of a QA test.") # The watcher may interfere by issuing its own jobs - therefore pause it # also reject all its jobs and wait for any running jobs to finish. AssertCommand(["gnt-cluster", "watcher", "pause", "12h"]) filter_uuid = stdout_of([ "gnt-filter", "add", '--predicates=[["reason", ["=", "source", "gnt:watcher"]]]', "--action=REJECT" ]) while stdout_of(["gnt-job", "list", "--no-header", "--running"]) != "": time.sleep(1) # Find out the lock names prior to starting the delay function lock_name_map = _FindLockNames(locks) blocking_owned_locks = [] test_blocked = False termination_socket = _StartDelayFunction(locks, timeout) delay_fn_terminated = False try: qa_thread = QAThread(fn, args, kwargs) qa_thread.start() while qa_thread.isAlive(): blocking_locks = _GetBlockingLocks() blocking_owned_locks = \ set(blocking_locks).intersection(set(lock_name_map)) if blocking_owned_locks: # Set the flag first - if the termination attempt fails, we do not want # to redo it in the finally block delay_fn_terminated = True _TerminateDelayFunction(termination_socket) test_blocked = True break time.sleep(5) # Set arbitrarily # The thread should be either finished or unblocked at this point qa_thread.join() # Raise any errors that might have occured in the thread qa_thread.reraise() finally: if not delay_fn_terminated: _TerminateDelayFunction(termination_socket) blocking_lock_names = ", ".join( map(lock_name_map.get, blocking_owned_locks)) if not block and test_blocked: raise qa_error.Error("QA test succeded, but was blocked by locks: %s" % blocking_lock_names) elif block and not test_blocked: raise qa_error.Error("QA test succeded, but was not blocked as it was " "expected to by locks: %s" % blocking_lock_names) else: pass # Revive the watcher AssertCommand(["gnt-filter", "delete", filter_uuid]) AssertCommand(["gnt-cluster", "watcher", "continue"])
def RunWithLocks(fn, locks, timeout, block, *args, **kwargs): """ Runs the given function, acquiring a set of locks beforehand. @type fn: function @param fn: The function to invoke. @type locks: dict of string to list of string @param locks: The locks to acquire, per lock category. @type timeout: number @param timeout: The number of seconds the locks should be held before expiring. @type block: bool @param block: Whether the test should block when locks are used or not. This function allows a set of locks to be acquired in preparation for a QA test, to try and see if the function can run in parallel with other operations. Locks are acquired by invoking a gnt-debug delay operation which can be interrupted as needed. The QA test is then run in a separate thread, with the current thread observing jobs waiting for locks. When a job is spotted waiting for a lock held by the started delay operation, this is noted, and the delay is interrupted, allowing the QA test to continue. A default timeout is not provided by design - the test creator must make a good conservative estimate. """ if filter(lambda l_type: l_type not in AVAILABLE_LOCKS, locks): raise qa_error.Error("Attempted to acquire locks that cannot yet be " "acquired in the course of a QA test.") # The watcher may interfere by issuing its own jobs - therefore pause it # also reject all its jobs and wait for any running jobs to finish. AssertCommand(["gnt-cluster", "watcher", "pause", "12h"]) filter_uuid = stdout_of([ "gnt-filter", "add", '--predicates=[["reason", ["=", "source", "gnt:watcher"]]]', "--action=REJECT" ]) while stdout_of(["gnt-job", "list", "--no-header", "--running"]) != "": time.sleep(1) # Find out the lock names prior to starting the delay function lock_name_map = _FindLockNames(locks) blocking_owned_locks = [] test_blocked = False termination_socket = _StartDelayFunction(locks, timeout) delay_fn_terminated = False try: qa_thread = QAThread(fn, args, kwargs) qa_thread.start() while qa_thread.isAlive(): blocking_locks = _GetBlockingLocks() blocking_owned_locks = \ set(blocking_locks).intersection(set(lock_name_map)) if blocking_owned_locks: # Set the flag first - if the termination attempt fails, we do not want # to redo it in the finally block delay_fn_terminated = True _TerminateDelayFunction(termination_socket) test_blocked = True break time.sleep(5) # Set arbitrarily # The thread should be either finished or unblocked at this point qa_thread.join() # Raise any errors that might have occured in the thread qa_thread.reraise() finally: if not delay_fn_terminated: _TerminateDelayFunction(termination_socket) blocking_lock_names = ", ".join(map(lock_name_map.get, blocking_owned_locks)) if not block and test_blocked: raise qa_error.Error("QA test succeded, but was blocked by locks: %s" % blocking_lock_names) elif block and not test_blocked: raise qa_error.Error("QA test succeded, but was not blocked as it was " "expected to by locks: %s" % blocking_lock_names) else: pass # Revive the watcher AssertCommand(["gnt-filter", "delete", filter_uuid]) AssertCommand(["gnt-cluster", "watcher", "continue"])