def test_basic_bursting(self): ws = worker.WorkerService() assert ws.is_bursting is False # Initial count of all workers( w0 = len(ws.get_all_workers(worker.QUEUE_DEFAULT)) d = Dispatcher() [d.dispatch_task(jobs.test_sleep, args=(5.2, )) for _ in range(9)] for _ in range(11): print(ws.is_bursting, len(ws.get_all_workers(worker.QUEUE_DEFAULT)), w0) if ws.is_bursting and len(ws.get_all_workers( worker.QUEUE_DEFAULT)) > w0: print('--- break') break pprint.pprint(ws.query()) time.sleep(1) else: assert False, "Expected to find worker bursting" # Wait for all BG tasks to finish. for i in range(6): if ws.is_bursting: time.sleep(1) # Assert the count of workers goes back to the original amount # when bursting is done. assert len(ws.get_all_workers(worker.QUEUE_DEFAULT)) == w0 assert ws.is_bursting is False
def mutate_and_get_payload(cls, root, info, owner, labbook_name, pull_only=False, override_method="abort", client_mutation_id=None): # Load LabBook username = get_logged_in_username() lb = InventoryManager().load_labbook(username, owner, labbook_name, author=get_logged_in_author()) # Extract valid Bearer token token = None if hasattr(info.context.headers, 'environ'): if "HTTP_AUTHORIZATION" in info.context.headers.environ: token = parse_token( info.context.headers.environ["HTTP_AUTHORIZATION"]) if not token: raise ValueError( "Authorization header not provided. " "Must have a valid session to query for collaborators") default_remote = lb.client_config.config['git']['default_remote'] admin_service = None for remote in lb.client_config.config['git']['remotes']: if default_remote == remote: admin_service = lb.client_config.config['git']['remotes'][ remote]['admin_service'] break if not admin_service: raise ValueError('admin_service could not be found') # Configure git creds mgr = GitLabManager(default_remote, admin_service, access_token=token) mgr.configure_git_credentials(default_remote, username) override = MergeOverride(override_method) job_metadata = {'method': 'sync_labbook', 'labbook': lb.key} job_kwargs = { 'repository': lb, 'pull_only': pull_only, 'username': username, 'override': override } dispatcher = Dispatcher() job_key = dispatcher.dispatch_task(jobs.sync_repository, kwargs=job_kwargs, metadata=job_metadata) logger.info( f"Syncing LabBook {lb.root_dir} in background job with key {job_key.key_str}" ) return SyncLabbook(job_key=job_key.key_str)
def mutate_and_get_payload(cls, root, info, dataset_owner, dataset_name, labbook_owner=None, labbook_name=None, client_mutation_id=None): logged_in_user = get_logged_in_username() # Schedule Job to clear file cache if dataset is no longer in use job_metadata = {'method': 'verify_dataset_contents'} job_kwargs = { 'logged_in_username': logged_in_user, 'access_token': flask.g.access_token, 'id_token': flask.g.id_token, 'dataset_owner': dataset_owner, 'dataset_name': dataset_name, 'labbook_owner': labbook_owner, 'labbook_name': labbook_name } dispatcher = Dispatcher() job_key = dispatcher.dispatch_task(jobs.verify_dataset_contents, metadata=job_metadata, kwargs=job_kwargs) logger.info( f"Dispatched verify_dataset_contents({dataset_owner}/{dataset_name}) to Job {job_key}" ) return VerifyDataset(background_job_key=job_key)
def test_query_failed_task(self, fixture_working_dir): """Test listing labbooks""" d = Dispatcher() job_id = d.dispatch_task(jobs.test_exit_fail) time.sleep(1) query = """ { jobStatus(jobId: "%s") { result status jobMetadata failureMessage startedAt finishedAt } } """ % job_id r = fixture_working_dir[2].execute(query) assert 'errors' not in r assert r['data']['jobStatus']['result'] is None assert r['data']['jobStatus']['status'] == 'failed' assert r['data']['jobStatus']['failureMessage'] == \ 'Exception: Intentional Exception from job `test_exit_fail`' assert r['data']['jobStatus']['startedAt'] is not None assert r['data']['jobStatus']['finishedAt'] is not None # Assert the following dict is empty assert not json.loads(r['data']['jobStatus']['jobMetadata'])
def test_query_finished_task(self, fixture_working_dir): """Test listing labbooks""" d = Dispatcher() job_id = d.dispatch_task(jobs.test_exit_success) time.sleep(1) query = """ { jobStatus(jobId: "%s") { result status jobMetadata failureMessage startedAt finishedAt } } """ % job_id.key_str r = fixture_working_dir[2].execute(query) assert 'errors' not in r assert int(r['data']['jobStatus']['result']) == 0 assert r['data']['jobStatus']['status'] == 'finished' assert r['data']['jobStatus']['startedAt'] is not None assert r['data']['jobStatus']['failureMessage'] is None assert r['data']['jobStatus']['finishedAt'] assert r['data']['jobStatus']['jobMetadata'] == '{}'
def mutate_and_get_payload(cls, root, info, owner, labbook_name, no_cache=False, client_mutation_id=None): username = get_logged_in_username() if BuildImage.get_container_status(labbook_name, owner, username): raise ValueError(f'Cannot build image for running container {owner}/{labbook_name}') lb = InventoryManager().load_labbook(username, owner, labbook_name, author=get_logged_in_author()) # Generate Dockerfile # TODO BVB - Move to build_image ?? ib = ImageBuilder(lb) ib.assemble_dockerfile(write=True) # Kick off building in a background thread d = Dispatcher() build_kwargs = { 'path': lb.root_dir, 'username': username, 'nocache': no_cache } metadata = {'labbook': lb.key, 'method': 'build_image'} res = d.dispatch_task(jobs.build_labbook_image, kwargs=build_kwargs, metadata=metadata) return BuildImage(environment=Environment(owner=owner, name=labbook_name), background_job_key=res.key_str)
def process_linked_datasets(labbook: LabBook, logged_in_username: str) -> None: """Method to update or init any linked dataset submodule references, clean up lingering files, and schedule jobs to auto-import if needed Args: labbook: the labbook to analyze logged_in_username: the current logged in username Returns: """ im = InventoryManager(config_file=labbook.client_config.config_file) # Update linked datasets inside the Project or clean them out if needed im.update_linked_datasets(labbook, logged_in_username, init=True) # Check for linked datasets, and schedule auto-imports d = Dispatcher() datasets = im.get_linked_datasets(labbook) for ds in datasets: kwargs = { 'logged_in_username': logged_in_username, 'dataset_owner': ds.namespace, 'dataset_name': ds.name, 'remote_url': ds.remote, } metadata = { 'dataset': f"{logged_in_username}|{ds.namespace}|{ds.name}", 'method': 'dataset_jobs.check_and_import_dataset' } d.dispatch_task( gtmcore.dispatcher.dataset_jobs.check_and_import_dataset, kwargs=kwargs, metadata=metadata)
def mutate_and_get_payload(cls, root, info, owner, labbook_name, confirm, client_mutation_id=None): username = get_logged_in_username() lb = InventoryManager().load_labbook(username, owner, labbook_name, author=get_logged_in_author()) if confirm: logger.info(f"Deleting {str(lb)}...") try: lb, stopped = ContainerOperations.stop_container( labbook=lb, username=username) except OSError as e: logger.warning(e) lb, docker_removed = ContainerOperations.delete_image( labbook=lb, username=username) if not docker_removed: raise ValueError( f'Cannot delete docker image for {str(lb)} - unable to delete Project from disk' ) datasets_to_schedule = InventoryManager().delete_labbook( username, owner, labbook_name) # Schedule jobs to clean the file cache for any linked datasets (if no other references exist) for cleanup_job in datasets_to_schedule: # Schedule Job to clear file cache if dataset is no longer in use job_metadata = {'method': 'clean_dataset_file_cache'} job_kwargs = { 'logged_in_username': username, 'dataset_owner': cleanup_job.namespace, 'dataset_name': cleanup_job.name, 'cache_location': cleanup_job.cache_root } dispatcher = Dispatcher() job_key = dispatcher.dispatch_task( jobs.clean_dataset_file_cache, metadata=job_metadata, kwargs=job_kwargs) logger.info( f"Dispatched clean_dataset_file_cache({ cleanup_job.namespace}/{cleanup_job.name})" f" to Job {job_key}") # Verify Delete worked if os.path.exists(lb.root_dir): logger.error( f'Deleted {str(lb)} but root directory {lb.root_dir} still exists!' ) return DeleteLabbook(success=False) else: return DeleteLabbook(success=True) else: logger.info(f"Dry run in deleting {str(lb)} -- not deleted.") return DeleteLabbook(success=False)
def test_failing_task(self): d = Dispatcher() job_ref = d.dispatch_task(bg_jobs.test_exit_fail) time.sleep(1) res = d.query_task(job_ref) assert res assert res.status == 'failed' assert res.failure_message == 'Exception: Intentional Exception from job `test_exit_fail`'
def test_query_failed_tasks(self): d = Dispatcher() job_ref = d.dispatch_task(bg_jobs.test_exit_fail) time.sleep(1) assert job_ref in [j.job_key for j in d.failed_jobs] assert job_ref not in [j.job_key for j in d.finished_jobs] t = d.query_task(job_ref) assert t.failure_message == 'Exception: Intentional Exception from job `test_exit_fail`'
def test_fail_dependent_job(self): d = Dispatcher() job_ref_1 = d.dispatch_task(bg_jobs.test_exit_fail) job_ref_2 = d.dispatch_task(bg_jobs.test_exit_success, dependent_job=job_ref_1) time.sleep(3) assert d.query_task(job_ref_1).status == 'failed' assert d.query_task(job_ref_2).status == 'deferred'
def mutate_and_get_payload(cls, root, info, dataset_owner, dataset_name, labbook_name=None, labbook_owner=None, all_keys=None, keys=None, client_mutation_id=None): logged_in_username = get_logged_in_username() lb = None im = InventoryManager() if labbook_name: # This is a linked dataset, load repo from the Project lb = im.load_labbook(logged_in_username, labbook_owner, labbook_name) dataset_dir = os.path.join(lb.root_dir, '.gigantum', 'datasets', dataset_owner, dataset_name) ds = im.load_dataset_from_directory(dataset_dir) else: # this is a normal dataset. Load repo from working dir ds = im.load_dataset(logged_in_username, dataset_owner, dataset_name) d = Dispatcher() dl_kwargs = { 'logged_in_username': logged_in_username, 'access_token': flask.g.access_token, 'id_token': flask.g.id_token, 'dataset_owner': dataset_owner, 'dataset_name': dataset_name, 'labbook_owner': labbook_owner, 'labbook_name': labbook_name, 'all_keys': all_keys, 'keys': keys } # Gen unique keys for tracking jobs lb_key = f"{logged_in_username}|{labbook_owner}|{labbook_name}" if lb else None ds_key = f"{logged_in_username}|{dataset_owner}|{dataset_name}" if lb_key: ds_key = f"{lb_key}|LINKED|{ds_key}" metadata = { 'dataset': ds_key, 'labbook': lb_key, 'method': 'download_dataset_files' } res = d.dispatch_task(jobs.download_dataset_files, kwargs=dl_kwargs, metadata=metadata) return DownloadDatasetFiles(background_job_key=res.key_str)
def _loader(self): self.job_key = self.id d = Dispatcher() q = d.query_task(JobKey(self.job_key)) self.status = q.status self.job_metadata = json.dumps(q.meta) self.failure_message = q.failure_message self.started_at = q.started_at self.finished_at = q.finished_at self.result = q.result
def test_simple_task(self): d = Dispatcher() job_ref = d.dispatch_task(bg_jobs.test_exit_success) time.sleep(1) res = d.query_task(job_ref) assert res assert res.status == 'finished' assert res.result == 0 assert res.failure_message is None assert res.finished_at is not None
def test_simple_scheduler(self): # Run a simple tasks that increments the integer contained in a file. d = Dispatcher() path = "/tmp/labmanager-unit-test-{}".format(os.getpid()) if os.path.exists(path): os.remove(path) d.schedule_task(bg_jobs.test_incr, args=(path, ), repeat=3, interval=2) time.sleep(8) try: with open(path) as fp: assert json.load(fp)['amt'] == 3 except Exception as e: raise e
def temporary_worker(): """A pytest fixture that creates a temporary directory and a config file to match. Deletes directory after test""" def run_worker(): with rq.Connection(): qs = 'labmanager_unittests' w = rq.Worker(qs) w.work() # This task is used to kill the worker. Sometimes if tests fail the worker runs forever and # holds up the entire process. This gives each test 25 seconds to run before killing the worker # and forcing the test to fail. def watch_proc(p): count = 0 while count < 15: count = count + 1 time.sleep(1) try: p.terminate() except: pass worker_proc = multiprocessing.Process(target=run_worker) worker_proc.start() watchdog_thread = threading.Thread(target=watch_proc, args=(worker_proc, )) watchdog_thread.start() dispatcher = Dispatcher('labmanager_unittests') yield worker_proc, dispatcher worker_proc.terminate()
def mutate_and_get_payload(cls, root, info, owner, dataset_name, transaction_id, cancel=False, rollback=False, client_mutation_id=None): logged_in_username = get_logged_in_username() logged_in_author = get_logged_in_author() ds = InventoryManager().load_dataset(logged_in_username, owner, dataset_name, author=logged_in_author) if cancel and rollback: # TODO: Add ability to reset raise ValueError("Currently cannot rollback a canceled upload.") # logger.warning(f"Cancelled tx {transaction_id}, doing git reset") else: logger.info( f"Done batch upload {transaction_id}, cancelled={cancel}") if cancel: logger.warning("Sweeping aborted batch upload.") d = Dispatcher() job_kwargs = { 'logged_in_username': logged_in_username, 'logged_in_email': logged_in_author.email, 'dataset_owner': owner, 'dataset_name': dataset_name, 'dispatcher': Dispatcher } # Gen unique keys for tracking jobs metadata = { 'dataset': f"{logged_in_username}|{owner}|{dataset_name}", 'method': 'complete_dataset_upload_transaction' } res = d.dispatch_task( dataset_jobs.complete_dataset_upload_transaction, kwargs=job_kwargs, metadata=metadata) return CompleteDatasetUploadTransaction(background_job_key=res.key_str)
def stop_dev_env_monitors(dev_env_key: str, redis_conn: redis.Redis, labbook_name: str) -> None: """Method to stop a dev env monitor and all related activity monitors Args: dev_env_key(str): Key in redis containing the dev env monitor info redis_conn(redis.Redis): The redis instance to the state db labbook_name(str): The name of the related lab book - used only for logging / user messaging purposes Returns: """ # Unschedule dev env monitor logger.info(f"Stopping dev env monitor {dev_env_key}") d = Dispatcher() process_id = redis_conn.hget(dev_env_key, "process_id") if process_id: logger.info("Dev Tool process id to stop: `{}` ".format(process_id)) d.unschedule_task(JobKey(process_id.decode())) _, dev_env_name = dev_env_key.rsplit(":", 1) logger.info( "Stopped dev tool monitor `{}` for lab book `{}`. PID {}".format( dev_env_name, labbook_name, process_id)) # Remove dev env monitor key redis_conn.delete(dev_env_key) # Make sure the monitor is unscheduled so it doesn't start activity monitors again time.sleep(2) else: logger.info( "Shutting down container with no Dev Tool monitoring processes to stop." ) # Get all related activity monitor keys activity_monitor_keys = redis_conn.keys( "{}:activity_monitor*".format(dev_env_key)) logger.info(f"Signaling {activity_monitor_keys} for shutdown.") # Signal all activity monitors to exit for am in activity_monitor_keys: # Set run flag in redis redis_conn.hset(am.decode(), "run", False) logger.info( "Signaled activity monitor for lab book `{}` to stop".format( labbook_name))
def test_unschedule_task(self): d = Dispatcher() path = "/tmp/labmanager-unit-test-{}".format(os.getpid()) if os.path.exists(path): os.remove(path) future_t = datetime.datetime.utcnow() + datetime.timedelta(seconds=5) jr = d.schedule_task(bg_jobs.test_incr, scheduled_time=future_t, args=(path, ), repeat=4, interval=1) time.sleep(2) n = d.unschedule_task(jr) assert n, "Task should have been cancelled, instead it was not found." time.sleep(5) assert not os.path.exists(path=path)
def test_abort(self): d = Dispatcher() job_ref_1 = d.dispatch_task(bg_jobs.test_sleep, args=(3, )) time.sleep(1.2) assert d.query_task(job_ref_1).status == 'started' workers = rq.Worker.all(connection=d._redis_conn) wk = [w for w in workers if w.state == 'busy'] assert len(wk) == 1, "There must be precisely one busy worker" job_pid = wk[0].get_current_job().meta['pid'] d.abort_task(job_ref_1) time.sleep(0.1) j = d.query_task(job_ref_1) # There should be no result, cause it was cancelled assert j.result is None # RQ should identify the task as failed assert j.status == "failed" # Assert the JOB pid is gone with pytest.raises(OSError): os.kill(int(job_pid), 0) # Now assert the worker pid is still alive (so it can be assigned something else) worker_pid = wk[0].pid try: os.kill(int(worker_pid), 0) assert True, "Worker process is still hanging around." except OSError: assert False, "Worker process is killed"
def mutate_and_get_payload(cls, root, info, owner, labbook_name, remote_url, client_mutation_id=None): username = get_logged_in_username() logger.info(f"Importing remote labbook from {remote_url}") lb = LabBook(author=get_logged_in_author()) default_remote = lb.client_config.config['git']['default_remote'] admin_service = None for remote in lb.client_config.config['git']['remotes']: if default_remote == remote: admin_service = lb.client_config.config['git']['remotes'][ remote]['admin_service'] break # Extract valid Bearer token if hasattr(info.context, 'headers' ) and "HTTP_AUTHORIZATION" in info.context.headers.environ: token = parse_token( info.context.headers.environ["HTTP_AUTHORIZATION"]) else: raise ValueError( "Authorization header not provided. Must have a valid session to query for collaborators" ) gl_mgr = GitLabManager(default_remote, admin_service=admin_service, access_token=token) gl_mgr.configure_git_credentials(default_remote, username) job_metadata = {'method': 'import_labbook_from_remote'} job_kwargs = {'remote_url': remote_url, 'username': username} dispatcher = Dispatcher() job_key = dispatcher.dispatch_task(jobs.import_labbook_from_remote, metadata=job_metadata, kwargs=job_kwargs) logger.info( f"Dispatched import_labbook_from_remote({remote_url}) to Job {job_key}" ) return ImportRemoteLabbook(job_key=job_key.key_str)
def test_run_only_once(self): # Assert that this method only gets called once. d = Dispatcher() path = "/tmp/labmanager-unit-test-{}".format(os.getpid()) if os.path.exists(path): os.remove(path) future_t = datetime.datetime.utcnow() + datetime.timedelta(seconds=1) jr = d.schedule_task(bg_jobs.test_incr, scheduled_time=future_t, args=(path, ), repeat=0) time.sleep(4) with open(path) as fp: assert json.load(fp)['amt'] == 1
def test_get_background_jobs_basics(self, fixture_working_dir_env_repo_scoped): d = Dispatcher() time.sleep(0.25) t1 = d.dispatch_task(jobs.test_exit_fail).key_str t2 = d.dispatch_task(jobs.test_exit_success).key_str t3 = d.dispatch_task(jobs.test_sleep, args=(5,)).key_str query = """ { backgroundJobs { edges { node { id jobKey failureMessage status result } } } } """ time.sleep(1) try: time1 = time.time() result = fixture_working_dir_env_repo_scoped[2].execute(query) time2 = time.time() tdiff = time2 - time1 assert tdiff < 0.5, "Query should not take more than a few millis (took {}s)".format(tdiff) assert any([t1 == x['node']['jobKey'] and 'failed' == x['node']['status'] and 'Exception: ' in x['node']['failureMessage'] for x in result['data']['backgroundJobs']['edges']]) assert any([t2 == x['node']['jobKey'] and "finished" == x['node']['status'] and x['node']['failureMessage'] is None for x in result['data']['backgroundJobs']['edges']]) assert any([t3 == x['node']['jobKey'] and "started" == x['node']['status'] and x['node']['failureMessage'] is None for x in result['data']['backgroundJobs']['edges']]) finally: time.sleep(2)
def mutate_and_process_upload(cls, info, upload_file_path, upload_filename, **kwargs): if not upload_file_path: logger.error('No file uploaded') raise ValueError('No file uploaded') username = get_logged_in_username() job_metadata = {'method': 'import_dataset_from_zip'} job_kwargs = { 'archive_path': upload_file_path, 'username': username, 'owner': username } dispatcher = Dispatcher() job_key = dispatcher.dispatch_task(jobs.import_dataset_from_zip, kwargs=job_kwargs, metadata=job_metadata) return ImportDataset(import_job_key=job_key.key_str)
def dispatcher_mock(self, function_ref, kwargs, metadata): assert kwargs['logged_in_username'] == 'other-test-user2' assert kwargs['dataset_owner'] == 'testuser' assert kwargs['dataset_name'] == 'test-ds' # Inject mocked config file kwargs['config_file'] = mock_config_file[0] # Stop patching so job gets scheduled for real dispatcher_patch.stop() # Call same method as in mutation d = Dispatcher() res = d.dispatch_task( gtmcore.dispatcher.dataset_jobs.check_and_import_dataset, kwargs=kwargs, metadata=metadata) return res
def mutate_and_get_payload(cls, root, info, dataset_owner, dataset_name, labbook_name=None, labbook_owner=None, all_keys=None, keys=None, client_mutation_id=None): logged_in_username = get_logged_in_username() d = Dispatcher() dl_kwargs = { 'logged_in_username': logged_in_username, 'access_token': flask.g.access_token, 'id_token': flask.g.id_token, 'dataset_owner': dataset_owner, 'dataset_name': dataset_name, 'labbook_owner': labbook_owner, 'labbook_name': labbook_name, 'all_keys': all_keys, 'keys': keys } # Gen unique keys for tracking jobs lb_key = f"{logged_in_username}|{labbook_owner}|{labbook_name}" if labbook_owner else None ds_key = f"{logged_in_username}|{dataset_owner}|{dataset_name}" if lb_key: ds_key = f"{lb_key}|LINKED|{ds_key}" metadata = { 'dataset': ds_key, 'labbook': lb_key, 'method': 'download_dataset_files' } res = d.dispatch_task(dataset_jobs.download_dataset_files, kwargs=dl_kwargs, metadata=metadata, persist=True) return DownloadDatasetFiles(background_job_key=res.key_str)
def test_unschedule_midway_through(self): d = Dispatcher() path = "/tmp/labmanager-unit-test-{}".format(os.getpid()) if os.path.exists(path): os.remove(path) future_t = None # i.e., start right now. jr = d.schedule_task(bg_jobs.test_incr, scheduled_time=future_t, args=(path, ), repeat=6, interval=5) time.sleep(8) n = d.unschedule_task(jr) assert n, "Task should have been cancelled, instead it was not found." time.sleep(5) with open(path) as fp: assert json.load(fp)['amt'] in [2]
def test_schedule_with_repeat_is_zero(self): # When repeat is zero, it should run only once. d = Dispatcher() path = "/tmp/labmanager-unit-test-{}".format(os.getpid()) if os.path.exists(path): os.remove(path) jr = d.schedule_task(bg_jobs.test_incr, args=(path, ), repeat=0, interval=4) time.sleep(6) n = d.unschedule_task(jr) time.sleep(5) with open(path) as fp: assert json.load(fp)['amt'] in [ 1 ], "When repeat=0, the task should run only once."
def dispatcher_mock(self, function_ref, kwargs, metadata): assert kwargs['logged_in_username'] == 'default' assert kwargs['logged_in_email'] == '*****@*****.**' assert kwargs['dataset_owner'] == 'default' assert kwargs['dataset_name'] == 'dataset1' # Inject mocked config file kwargs['config_file'] = mock_create_dataset[0] # Stop patching so job gets scheduled for real dispatcher_patch.stop() # Call same method as in mutation d = Dispatcher() kwargs['dispatcher'] = Dispatcher res = d.dispatch_task(gtmcore.dispatcher.dataset_jobs. complete_dataset_upload_transaction, kwargs=kwargs, metadata=metadata) return res
def mutate_and_get_payload(cls, root, info, owner, labbook_name, client_mutation_id=None): username = get_logged_in_username() lb = InventoryManager().load_labbook(username, owner, labbook_name) d = Dispatcher() lb_jobs = d.get_jobs_for_labbook(lb.key) jobs = [ j for j in d.get_jobs_for_labbook(lb.key) if j.meta.get('method') == 'build_image' and j.status == 'started' ] if len(jobs) == 1: d.abort_task(jobs[0].job_key) ContainerOperations.delete_image(lb, username=username) return CancelBuild(build_stopped=True, message="Stopped build") elif len(jobs) == 0: logger.warning(f"No build_image tasks found for {str(lb)}") return CancelBuild(build_stopped=False, message="No build task found") else: logger.warning(f"Multiple build jobs found for {str(lb)}") return CancelBuild(build_stopped=False, message="Multiple builds found")