def test_revert_task(add_recovery): with task_manager() as tm: # Create a task c = Callable(hang_timeout=WAIT_TIMEOUT) t = task.Task(id="task-id", abort_callback=c.finish) # Add recovery to task r = add_recovery(t, "fakerecovery", ["arg1", "arg2", "arg3"]) # Run the task t.prepare(tm.scheduleJob, "tag", None, t, "job", c) c.wait_until_running() assert "task-id" in tm.getAllTasks() # Finish the running task c.finish() assert t.wait(timeout=WAIT_TIMEOUT), "Task is not finished" # Revert the task and run recovery rollback assert r.args is None tm.revertTask("task-id") assert r.args == ("arg1", "arg2", "arg3") # Check that task is in a recovered state t.getState() == "recovered"
def test_stop_clear_task(add_recovery): with task_manager() as tm: # Create a task c = Callable(hang_timeout=WAIT_TIMEOUT) t = task.Task(id="task-id", abort_callback=c.finish) # Add recovery to task r = add_recovery(t, "fakerecovery", "arg") # Run the task t.prepare(tm.scheduleJob, "tag", None, t, "job", c) c.wait_until_running() assert "task-id" in tm.getAllTasks() # Abort the task assert r.args is None tm.stopTask("task-id") # Wait for task to finish assert t.wait(timeout=WAIT_TIMEOUT), "Task is not finished" assert c.is_finished() # Assert that recovery was run assert r.args == ("arg", ) # Check that task is in a recovered state t.getState() == "recovered" # Clear the task from the manager list tm.clearTask("task-id") assert "task-id" not in tm.getAllTasks()
def wrapper(*args, **kwargs): try: ctask = task.Task(id=None, name=name) try: response = self.STATUS_OK.copy() result = ctask.prepare(func, *args, **kwargs) if type(result) == dict: response.update(result) return response except se.GeneralException as e: # Match api.method format self.log.error("FINISH %s error=%s", name, e) return e.response() except BaseException as e: # Match api.method format self.log.exception("FINISH %s error=%s", name, e) defaultException = ctask.defaultException if (defaultException and hasattr(defaultException, "response")): resp = defaultException.response() defaultExceptionInfo = (resp['status']['code'], resp['status']['message']) return se.generateResponse(e, defaultExceptionInfo) return se.generateResponse(e) except: try: # We should never reach this self.log.exception( "Unhandled exception (name=%s, args=%s, kwargs=%s)", name, args, kwargs) finally: return self.STATUS_ERROR.copy()
def runTask(args): if type(args) == tuple: cmd = args[0] args = args[1:] else: cmd = args args = None ctask = task.Task(id=None, name=cmd) ctask.prepare(cmd, *args)
def fake_hsm(monkeypatch): """ Create fake hsm instance for testing connection verbs. Monkeypatch the hsm and storageServer to allow testing the flows without attempting real connections. """ monkeypatch.setattr(hsm, 'sdCache', FakeStorageDomainCache()) monkeypatch.setattr(hsm.vars, 'task', task.Task("fake-task-id")) monkeypatch.setattr(storageServer, 'ConnectionFactory', FakeConnectionFactory()) return FakeConnectHSM()
def test_persistent_job(tmpdir, add_recovery): store = str(tmpdir) # Simulate SPM starting a persistent job and fencing out with task_manager() as tm: # Create a task c = Callable(hang_timeout=WAIT_TIMEOUT) t = task.Task(id="task-id", abort_callback=c.finish) # Add recovery for task r = add_recovery(t, "fakerecovery", ["arg1", "arg2", "arg3"]) # Simulate async call for a task job t.prepare(tm.scheduleJob, "tag", store, t, "job", c) c.wait_until_running() assert "task-id" in tm.getAllTasks() # Prevent storing the state to simulate SPM fencing # with an unexpected shutdown t.store = None # Simulate another SPM recovering the stored task with task_manager() as tm: tm.loadDumpedTasks(store) # Start recovery assert r.args is None tm.recoverDumpedTasks() # Wait for recovery to finish t = tm._getTask("task-id") assert t.wait(timeout=WAIT_TIMEOUT), "Task is not finished" # Check that recovery was called assert r.args == ("arg1", "arg2", "arg3") # Check that task is in a recovered state t.getState() == "recovered"
def fake_task(monkeypatch): """ Create fake task, expected in various places in the code. In the real code a task is created for every HSM public call by the dispatcher. """ monkeypatch.setattr(threadlocal.vars, 'task', task.Task("fake-task-id"))