def test_persistence(): """ Tests persistence of a managers jobs. """ staging_directory = tempfile.mkdtemp() try: app = Bunch( staging_directory=staging_directory, persistence_directory=staging_directory, authorizer=get_authorizer(None), dependency_manager=TestDependencyManager(), job_metrics=Bunch(default_job_instrumenter=NULL_JOB_INSTRUMENTER), ) assert not exists(join(staging_directory, "queued_jobs")) queue1 = StatefulManagerProxy( QueueManager('test', app, num_concurrent_jobs=0)) job_id = queue1.setup_job('4', 'tool1', '1.0.0') touch_file = join(staging_directory, 'ran') queue1.launch(job_id, 'touch %s' % touch_file) time.sleep(.4) assert (not (exists(touch_file))) queue1.shutdown() queue2 = StatefulManagerProxy( QueueManager('test', app, num_concurrent_jobs=1)) time.sleep(1) assert exists(touch_file) finally: shutil.rmtree(staging_directory) try: queue2.shutdown() except: pass
def _setup_job_with_unexecuted_preprocessing_directive(app): staging_directory = app.staging_directory queue1 = DoesntPreprocessStatefulManagerProxy(QueueManager('test', app, num_concurrent_jobs=0)) job_id = queue1.setup_job(TEST_JOB_ID, 'tool1', '1.0.0') action = {"name": TEST_STAGED_FILE, "type": "input", "action": {"action_type": "message", "contents": "cow file"}} remote_staging = {"setup": [action]} touch_file = join(staging_directory, TEST_COMMAND_TOUCH_FILE) queue1.preprocess_and_launch(job_id, {"command_line": "touch '%s'" % touch_file, "remote_staging": remote_staging}) queue1.shutdown()
def _setup_manager_that_executes(app): queue2 = StatefulManagerProxy(QueueManager('test', app, num_concurrent_jobs=1)) try: queue2.recover_active_jobs() time.sleep(1) finally: try: queue2.shutdown() except Exception: pass
def _setup_manager_that_preprocesses(app): # Setup a manager that will preprocess the job but won't execute it. # Now start a real stateful manager proxy and watch the file get staged. queue2 = StatefulManagerProxy(QueueManager('test', app, num_concurrent_jobs=0)) try: queue2.recover_active_jobs() time.sleep(1) finally: try: queue2.shutdown() except Exception: pass
def test_launched_job_recovery(): """Tests persistence and recovery of launched managers jobs.""" with _app() as app: staging_directory = app.staging_directory queue1 = StatefulManagerProxy(QueueManager('test', app, num_concurrent_jobs=0)) job_id = queue1.setup_job(TEST_JOB_ID, 'tool1', '1.0.0') touch_file = join(staging_directory, TEST_COMMAND_TOUCH_FILE) queue1.preprocess_and_launch(job_id, {"command_line": 'touch %s' % touch_file}) time.sleep(.4) assert not exists(touch_file) queue1.shutdown() _setup_manager_that_executes(app) assert exists(touch_file)
def _set_manager(self, **kwds): self.manager = QueueManager('_default_', self.app, **kwds)