def test_revert_task(add_recovery):
with task_manager() as tm:
# Create a task
c = Callable(hang_timeout=WAIT_TIMEOUT)
t = task.Task(id="task-id", abort_callback=c.finish)
# Add recovery to task
r = add_recovery(t, "fakerecovery", ["arg1", "arg2", "arg3"])
# Run the task
t.prepare(tm.scheduleJob, "tag", None, t, "job", c)
c.wait_until_running()
assert "task-id" in tm.getAllTasks()
# Finish the running task
c.finish()
assert t.wait(timeout=WAIT_TIMEOUT), "Task is not finished"
# Revert the task and run recovery rollback
assert r.args is None
tm.revertTask("task-id")
assert r.args == ("arg1", "arg2", "arg3")
# Check that task is in a recovered state
t.getState() == "recovered"
def test_stop_clear_task(add_recovery):
with task_manager() as tm:
# Create a task
c = Callable(hang_timeout=WAIT_TIMEOUT)
t = task.Task(id="task-id", abort_callback=c.finish)
# Add recovery to task
r = add_recovery(t, "fakerecovery", "arg")
# Run the task
t.prepare(tm.scheduleJob, "tag", None, t, "job", c)
c.wait_until_running()
assert "task-id" in tm.getAllTasks()
# Abort the task
assert r.args is None
tm.stopTask("task-id")
# Wait for task to finish
assert t.wait(timeout=WAIT_TIMEOUT), "Task is not finished"
assert c.is_finished()
# Assert that recovery was run
assert r.args == ("arg", )
# Check that task is in a recovered state
t.getState() == "recovered"
# Clear the task from the manager list
tm.clearTask("task-id")
assert "task-id" not in tm.getAllTasks()
def wrapper(*args, **kwargs):
try:
ctask = task.Task(id=None, name=name)
try:
response = self.STATUS_OK.copy()
result = ctask.prepare(func, *args, **kwargs)
if type(result) == dict:
response.update(result)
return response
except se.GeneralException as e:
# Match api.method format
self.log.error("FINISH %s error=%s", name, e)
return e.response()
except BaseException as e:
# Match api.method format
self.log.exception("FINISH %s error=%s", name, e)
defaultException = ctask.defaultException
if (defaultException
and hasattr(defaultException, "response")):
resp = defaultException.response()
defaultExceptionInfo = (resp['status']['code'],
resp['status']['message'])
return se.generateResponse(e, defaultExceptionInfo)
return se.generateResponse(e)
except:
try:
# We should never reach this
self.log.exception(
"Unhandled exception (name=%s, args=%s, kwargs=%s)",
name, args, kwargs)
finally:
return self.STATUS_ERROR.copy()
def runTask(args):
if type(args) == tuple:
cmd = args[0]
args = args[1:]
else:
cmd = args
args = None
ctask = task.Task(id=None, name=cmd)
ctask.prepare(cmd, *args)
def fake_hsm(monkeypatch):
"""
Create fake hsm instance for testing connection verbs.
Monkeypatch the hsm and storageServer to allow testing the flows without
attempting real connections.
"""
monkeypatch.setattr(hsm, 'sdCache', FakeStorageDomainCache())
monkeypatch.setattr(hsm.vars, 'task', task.Task("fake-task-id"))
monkeypatch.setattr(storageServer, 'ConnectionFactory',
FakeConnectionFactory())
return FakeConnectHSM()
def test_persistent_job(tmpdir, add_recovery):
store = str(tmpdir)
# Simulate SPM starting a persistent job and fencing out
with task_manager() as tm:
# Create a task
c = Callable(hang_timeout=WAIT_TIMEOUT)
t = task.Task(id="task-id", abort_callback=c.finish)
# Add recovery for task
r = add_recovery(t, "fakerecovery", ["arg1", "arg2", "arg3"])
# Simulate async call for a task job
t.prepare(tm.scheduleJob, "tag", store, t, "job", c)
c.wait_until_running()
assert "task-id" in tm.getAllTasks()
# Prevent storing the state to simulate SPM fencing
# with an unexpected shutdown
t.store = None
# Simulate another SPM recovering the stored task
with task_manager() as tm:
tm.loadDumpedTasks(store)
# Start recovery
assert r.args is None
tm.recoverDumpedTasks()
# Wait for recovery to finish
t = tm._getTask("task-id")
assert t.wait(timeout=WAIT_TIMEOUT), "Task is not finished"
# Check that recovery was called
assert r.args == ("arg1", "arg2", "arg3")
# Check that task is in a recovered state
t.getState() == "recovered"
def fake_task(monkeypatch):
"""
Create fake task, expected in various places in the code. In the real code
a task is created for every HSM public call by the dispatcher.
"""
monkeypatch.setattr(threadlocal.vars, 'task', task.Task("fake-task-id"))
