def test_basic_bursting(self):
ws = worker.WorkerService()
assert ws.is_bursting is False
# Initial count of all workers(
w0 = len(ws.get_all_workers(worker.QUEUE_DEFAULT))
d = Dispatcher()
[d.dispatch_task(jobs.test_sleep, args=(5.2, )) for _ in range(9)]
for _ in range(11):
print(ws.is_bursting,
len(ws.get_all_workers(worker.QUEUE_DEFAULT)), w0)
if ws.is_bursting and len(ws.get_all_workers(
worker.QUEUE_DEFAULT)) > w0:
print('--- break')
break
pprint.pprint(ws.query())
time.sleep(1)
else:
assert False, "Expected to find worker bursting"
# Wait for all BG tasks to finish.
for i in range(6):
if ws.is_bursting:
time.sleep(1)
# Assert the count of workers goes back to the original amount
# when bursting is done.
assert len(ws.get_all_workers(worker.QUEUE_DEFAULT)) == w0
assert ws.is_bursting is False
def mutate_and_get_payload(cls,
root,
info,
owner,
labbook_name,
pull_only=False,
override_method="abort",
client_mutation_id=None):
# Load LabBook
username = get_logged_in_username()
lb = InventoryManager().load_labbook(username,
owner,
labbook_name,
author=get_logged_in_author())
# Extract valid Bearer token
token = None
if hasattr(info.context.headers, 'environ'):
if "HTTP_AUTHORIZATION" in info.context.headers.environ:
token = parse_token(
info.context.headers.environ["HTTP_AUTHORIZATION"])
if not token:
raise ValueError(
"Authorization header not provided. "
"Must have a valid session to query for collaborators")
default_remote = lb.client_config.config['git']['default_remote']
admin_service = None
for remote in lb.client_config.config['git']['remotes']:
if default_remote == remote:
admin_service = lb.client_config.config['git']['remotes'][
remote]['admin_service']
break
if not admin_service:
raise ValueError('admin_service could not be found')
# Configure git creds
mgr = GitLabManager(default_remote, admin_service, access_token=token)
mgr.configure_git_credentials(default_remote, username)
override = MergeOverride(override_method)
job_metadata = {'method': 'sync_labbook', 'labbook': lb.key}
job_kwargs = {
'repository': lb,
'pull_only': pull_only,
'username': username,
'override': override
}
dispatcher = Dispatcher()
job_key = dispatcher.dispatch_task(jobs.sync_repository,
kwargs=job_kwargs,
metadata=job_metadata)
logger.info(
f"Syncing LabBook {lb.root_dir} in background job with key {job_key.key_str}"
)
return SyncLabbook(job_key=job_key.key_str)
def mutate_and_get_payload(cls,
root,
info,
dataset_owner,
dataset_name,
labbook_owner=None,
labbook_name=None,
client_mutation_id=None):
logged_in_user = get_logged_in_username()
# Schedule Job to clear file cache if dataset is no longer in use
job_metadata = {'method': 'verify_dataset_contents'}
job_kwargs = {
'logged_in_username': logged_in_user,
'access_token': flask.g.access_token,
'id_token': flask.g.id_token,
'dataset_owner': dataset_owner,
'dataset_name': dataset_name,
'labbook_owner': labbook_owner,
'labbook_name': labbook_name
}
dispatcher = Dispatcher()
job_key = dispatcher.dispatch_task(jobs.verify_dataset_contents,
metadata=job_metadata,
kwargs=job_kwargs)
logger.info(
f"Dispatched verify_dataset_contents({dataset_owner}/{dataset_name}) to Job {job_key}"
)
return VerifyDataset(background_job_key=job_key)
def test_query_failed_task(self, fixture_working_dir):
"""Test listing labbooks"""
d = Dispatcher()
job_id = d.dispatch_task(jobs.test_exit_fail)
time.sleep(1)
query = """
{
jobStatus(jobId: "%s") {
result
status
jobMetadata
failureMessage
startedAt
finishedAt
}
}
""" % job_id
r = fixture_working_dir[2].execute(query)
assert 'errors' not in r
assert r['data']['jobStatus']['result'] is None
assert r['data']['jobStatus']['status'] == 'failed'
assert r['data']['jobStatus']['failureMessage'] == \
'Exception: Intentional Exception from job `test_exit_fail`'
assert r['data']['jobStatus']['startedAt'] is not None
assert r['data']['jobStatus']['finishedAt'] is not None
# Assert the following dict is empty
assert not json.loads(r['data']['jobStatus']['jobMetadata'])
def test_query_finished_task(self, fixture_working_dir):
"""Test listing labbooks"""
d = Dispatcher()
job_id = d.dispatch_task(jobs.test_exit_success)
time.sleep(1)
query = """
{
jobStatus(jobId: "%s") {
result
status
jobMetadata
failureMessage
startedAt
finishedAt
}
}
""" % job_id.key_str
r = fixture_working_dir[2].execute(query)
assert 'errors' not in r
assert int(r['data']['jobStatus']['result']) == 0
assert r['data']['jobStatus']['status'] == 'finished'
assert r['data']['jobStatus']['startedAt'] is not None
assert r['data']['jobStatus']['failureMessage'] is None
assert r['data']['jobStatus']['finishedAt']
assert r['data']['jobStatus']['jobMetadata'] == '{}'
def mutate_and_get_payload(cls, root, info, owner, labbook_name, no_cache=False, client_mutation_id=None):
username = get_logged_in_username()
if BuildImage.get_container_status(labbook_name, owner, username):
raise ValueError(f'Cannot build image for running container {owner}/{labbook_name}')
lb = InventoryManager().load_labbook(username, owner, labbook_name,
author=get_logged_in_author())
# Generate Dockerfile
# TODO BVB - Move to build_image ??
ib = ImageBuilder(lb)
ib.assemble_dockerfile(write=True)
# Kick off building in a background thread
d = Dispatcher()
build_kwargs = {
'path': lb.root_dir,
'username': username,
'nocache': no_cache
}
metadata = {'labbook': lb.key,
'method': 'build_image'}
res = d.dispatch_task(jobs.build_labbook_image, kwargs=build_kwargs, metadata=metadata)
return BuildImage(environment=Environment(owner=owner, name=labbook_name),
background_job_key=res.key_str)
def process_linked_datasets(labbook: LabBook, logged_in_username: str) -> None:
"""Method to update or init any linked dataset submodule references, clean up lingering files, and schedule
jobs to auto-import if needed
Args:
labbook: the labbook to analyze
logged_in_username: the current logged in username
Returns:
"""
im = InventoryManager(config_file=labbook.client_config.config_file)
# Update linked datasets inside the Project or clean them out if needed
im.update_linked_datasets(labbook, logged_in_username, init=True)
# Check for linked datasets, and schedule auto-imports
d = Dispatcher()
datasets = im.get_linked_datasets(labbook)
for ds in datasets:
kwargs = {
'logged_in_username': logged_in_username,
'dataset_owner': ds.namespace,
'dataset_name': ds.name,
'remote_url': ds.remote,
}
metadata = {
'dataset': f"{logged_in_username}|{ds.namespace}|{ds.name}",
'method': 'dataset_jobs.check_and_import_dataset'
}
d.dispatch_task(
gtmcore.dispatcher.dataset_jobs.check_and_import_dataset,
kwargs=kwargs,
metadata=metadata)
def mutate_and_get_payload(cls,
root,
info,
owner,
labbook_name,
confirm,
client_mutation_id=None):
username = get_logged_in_username()
lb = InventoryManager().load_labbook(username,
owner,
labbook_name,
author=get_logged_in_author())
if confirm:
logger.info(f"Deleting {str(lb)}...")
try:
lb, stopped = ContainerOperations.stop_container(
labbook=lb, username=username)
except OSError as e:
logger.warning(e)
lb, docker_removed = ContainerOperations.delete_image(
labbook=lb, username=username)
if not docker_removed:
raise ValueError(
f'Cannot delete docker image for {str(lb)} - unable to delete Project from disk'
)
datasets_to_schedule = InventoryManager().delete_labbook(
username, owner, labbook_name)
# Schedule jobs to clean the file cache for any linked datasets (if no other references exist)
for cleanup_job in datasets_to_schedule:
# Schedule Job to clear file cache if dataset is no longer in use
job_metadata = {'method': 'clean_dataset_file_cache'}
job_kwargs = {
'logged_in_username': username,
'dataset_owner': cleanup_job.namespace,
'dataset_name': cleanup_job.name,
'cache_location': cleanup_job.cache_root
}
dispatcher = Dispatcher()
job_key = dispatcher.dispatch_task(
jobs.clean_dataset_file_cache,
metadata=job_metadata,
kwargs=job_kwargs)
logger.info(
f"Dispatched clean_dataset_file_cache({ cleanup_job.namespace}/{cleanup_job.name})"
f" to Job {job_key}")
# Verify Delete worked
if os.path.exists(lb.root_dir):
logger.error(
f'Deleted {str(lb)} but root directory {lb.root_dir} still exists!'
)
return DeleteLabbook(success=False)
else:
return DeleteLabbook(success=True)
else:
logger.info(f"Dry run in deleting {str(lb)} -- not deleted.")
return DeleteLabbook(success=False)
def test_failing_task(self):
d = Dispatcher()
job_ref = d.dispatch_task(bg_jobs.test_exit_fail)
time.sleep(1)
res = d.query_task(job_ref)
assert res
assert res.status == 'failed'
assert res.failure_message == 'Exception: Intentional Exception from job `test_exit_fail`'
def test_query_failed_tasks(self):
d = Dispatcher()
job_ref = d.dispatch_task(bg_jobs.test_exit_fail)
time.sleep(1)
assert job_ref in [j.job_key for j in d.failed_jobs]
assert job_ref not in [j.job_key for j in d.finished_jobs]
t = d.query_task(job_ref)
assert t.failure_message == 'Exception: Intentional Exception from job `test_exit_fail`'
def test_fail_dependent_job(self):
d = Dispatcher()
job_ref_1 = d.dispatch_task(bg_jobs.test_exit_fail)
job_ref_2 = d.dispatch_task(bg_jobs.test_exit_success,
dependent_job=job_ref_1)
time.sleep(3)
assert d.query_task(job_ref_1).status == 'failed'
assert d.query_task(job_ref_2).status == 'deferred'
def mutate_and_get_payload(cls,
root,
info,
dataset_owner,
dataset_name,
labbook_name=None,
labbook_owner=None,
all_keys=None,
keys=None,
client_mutation_id=None):
logged_in_username = get_logged_in_username()
lb = None
im = InventoryManager()
if labbook_name:
# This is a linked dataset, load repo from the Project
lb = im.load_labbook(logged_in_username, labbook_owner,
labbook_name)
dataset_dir = os.path.join(lb.root_dir, '.gigantum', 'datasets',
dataset_owner, dataset_name)
ds = im.load_dataset_from_directory(dataset_dir)
else:
# this is a normal dataset. Load repo from working dir
ds = im.load_dataset(logged_in_username, dataset_owner,
dataset_name)
d = Dispatcher()
dl_kwargs = {
'logged_in_username': logged_in_username,
'access_token': flask.g.access_token,
'id_token': flask.g.id_token,
'dataset_owner': dataset_owner,
'dataset_name': dataset_name,
'labbook_owner': labbook_owner,
'labbook_name': labbook_name,
'all_keys': all_keys,
'keys': keys
}
# Gen unique keys for tracking jobs
lb_key = f"{logged_in_username}|{labbook_owner}|{labbook_name}" if lb else None
ds_key = f"{logged_in_username}|{dataset_owner}|{dataset_name}"
if lb_key:
ds_key = f"{lb_key}|LINKED|{ds_key}"
metadata = {
'dataset': ds_key,
'labbook': lb_key,
'method': 'download_dataset_files'
}
res = d.dispatch_task(jobs.download_dataset_files,
kwargs=dl_kwargs,
metadata=metadata)
return DownloadDatasetFiles(background_job_key=res.key_str)
def _loader(self):
self.job_key = self.id
d = Dispatcher()
q = d.query_task(JobKey(self.job_key))
self.status = q.status
self.job_metadata = json.dumps(q.meta)
self.failure_message = q.failure_message
self.started_at = q.started_at
self.finished_at = q.finished_at
self.result = q.result
def test_simple_task(self):
d = Dispatcher()
job_ref = d.dispatch_task(bg_jobs.test_exit_success)
time.sleep(1)
res = d.query_task(job_ref)
assert res
assert res.status == 'finished'
assert res.result == 0
assert res.failure_message is None
assert res.finished_at is not None
def test_simple_scheduler(self):
# Run a simple tasks that increments the integer contained in a file.
d = Dispatcher()
path = "/tmp/labmanager-unit-test-{}".format(os.getpid())
if os.path.exists(path):
os.remove(path)
d.schedule_task(bg_jobs.test_incr, args=(path, ), repeat=3, interval=2)
time.sleep(8)
try:
with open(path) as fp:
assert json.load(fp)['amt'] == 3
except Exception as e:
raise e
def temporary_worker():
"""A pytest fixture that creates a temporary directory and a config file to match. Deletes directory after test"""
def run_worker():
with rq.Connection():
qs = 'labmanager_unittests'
w = rq.Worker(qs)
w.work()
# This task is used to kill the worker. Sometimes if tests fail the worker runs forever and
# holds up the entire process. This gives each test 25 seconds to run before killing the worker
# and forcing the test to fail.
def watch_proc(p):
count = 0
while count < 15:
count = count + 1
time.sleep(1)
try:
p.terminate()
except:
pass
worker_proc = multiprocessing.Process(target=run_worker)
worker_proc.start()
watchdog_thread = threading.Thread(target=watch_proc, args=(worker_proc, ))
watchdog_thread.start()
dispatcher = Dispatcher('labmanager_unittests')
yield worker_proc, dispatcher
worker_proc.terminate()
def mutate_and_get_payload(cls,
root,
info,
owner,
dataset_name,
transaction_id,
cancel=False,
rollback=False,
client_mutation_id=None):
logged_in_username = get_logged_in_username()
logged_in_author = get_logged_in_author()
ds = InventoryManager().load_dataset(logged_in_username,
owner,
dataset_name,
author=logged_in_author)
if cancel and rollback:
# TODO: Add ability to reset
raise ValueError("Currently cannot rollback a canceled upload.")
# logger.warning(f"Cancelled tx {transaction_id}, doing git reset")
else:
logger.info(
f"Done batch upload {transaction_id}, cancelled={cancel}")
if cancel:
logger.warning("Sweeping aborted batch upload.")
d = Dispatcher()
job_kwargs = {
'logged_in_username': logged_in_username,
'logged_in_email': logged_in_author.email,
'dataset_owner': owner,
'dataset_name': dataset_name,
'dispatcher': Dispatcher
}
# Gen unique keys for tracking jobs
metadata = {
'dataset': f"{logged_in_username}|{owner}|{dataset_name}",
'method': 'complete_dataset_upload_transaction'
}
res = d.dispatch_task(
dataset_jobs.complete_dataset_upload_transaction,
kwargs=job_kwargs,
metadata=metadata)
return CompleteDatasetUploadTransaction(background_job_key=res.key_str)
def stop_dev_env_monitors(dev_env_key: str, redis_conn: redis.Redis,
labbook_name: str) -> None:
"""Method to stop a dev env monitor and all related activity monitors
Args:
dev_env_key(str): Key in redis containing the dev env monitor info
redis_conn(redis.Redis): The redis instance to the state db
labbook_name(str): The name of the related lab book - used only for logging / user messaging purposes
Returns:
"""
# Unschedule dev env monitor
logger.info(f"Stopping dev env monitor {dev_env_key}")
d = Dispatcher()
process_id = redis_conn.hget(dev_env_key, "process_id")
if process_id:
logger.info("Dev Tool process id to stop: `{}` ".format(process_id))
d.unschedule_task(JobKey(process_id.decode()))
_, dev_env_name = dev_env_key.rsplit(":", 1)
logger.info(
"Stopped dev tool monitor `{}` for lab book `{}`. PID {}".format(
dev_env_name, labbook_name, process_id))
# Remove dev env monitor key
redis_conn.delete(dev_env_key)
# Make sure the monitor is unscheduled so it doesn't start activity monitors again
time.sleep(2)
else:
logger.info(
"Shutting down container with no Dev Tool monitoring processes to stop."
)
# Get all related activity monitor keys
activity_monitor_keys = redis_conn.keys(
"{}:activity_monitor*".format(dev_env_key))
logger.info(f"Signaling {activity_monitor_keys} for shutdown.")
# Signal all activity monitors to exit
for am in activity_monitor_keys:
# Set run flag in redis
redis_conn.hset(am.decode(), "run", False)
logger.info(
"Signaled activity monitor for lab book `{}` to stop".format(
labbook_name))
def test_unschedule_task(self):
d = Dispatcher()
path = "/tmp/labmanager-unit-test-{}".format(os.getpid())
if os.path.exists(path):
os.remove(path)
future_t = datetime.datetime.utcnow() + datetime.timedelta(seconds=5)
jr = d.schedule_task(bg_jobs.test_incr,
scheduled_time=future_t,
args=(path, ),
repeat=4,
interval=1)
time.sleep(2)
n = d.unschedule_task(jr)
assert n, "Task should have been cancelled, instead it was not found."
time.sleep(5)
assert not os.path.exists(path=path)
def test_abort(self):
d = Dispatcher()
job_ref_1 = d.dispatch_task(bg_jobs.test_sleep, args=(3, ))
time.sleep(1.2)
assert d.query_task(job_ref_1).status == 'started'
workers = rq.Worker.all(connection=d._redis_conn)
wk = [w for w in workers if w.state == 'busy']
assert len(wk) == 1, "There must be precisely one busy worker"
job_pid = wk[0].get_current_job().meta['pid']
d.abort_task(job_ref_1)
time.sleep(0.1)
j = d.query_task(job_ref_1)
# There should be no result, cause it was cancelled
assert j.result is None
# RQ should identify the task as failed
assert j.status == "failed"
# Assert the JOB pid is gone
with pytest.raises(OSError):
os.kill(int(job_pid), 0)
# Now assert the worker pid is still alive (so it can be assigned something else)
worker_pid = wk[0].pid
try:
os.kill(int(worker_pid), 0)
assert True, "Worker process is still hanging around."
except OSError:
assert False, "Worker process is killed"
def mutate_and_get_payload(cls,
root,
info,
owner,
labbook_name,
remote_url,
client_mutation_id=None):
username = get_logged_in_username()
logger.info(f"Importing remote labbook from {remote_url}")
lb = LabBook(author=get_logged_in_author())
default_remote = lb.client_config.config['git']['default_remote']
admin_service = None
for remote in lb.client_config.config['git']['remotes']:
if default_remote == remote:
admin_service = lb.client_config.config['git']['remotes'][
remote]['admin_service']
break
# Extract valid Bearer token
if hasattr(info.context, 'headers'
) and "HTTP_AUTHORIZATION" in info.context.headers.environ:
token = parse_token(
info.context.headers.environ["HTTP_AUTHORIZATION"])
else:
raise ValueError(
"Authorization header not provided. Must have a valid session to query for collaborators"
)
gl_mgr = GitLabManager(default_remote,
admin_service=admin_service,
access_token=token)
gl_mgr.configure_git_credentials(default_remote, username)
job_metadata = {'method': 'import_labbook_from_remote'}
job_kwargs = {'remote_url': remote_url, 'username': username}
dispatcher = Dispatcher()
job_key = dispatcher.dispatch_task(jobs.import_labbook_from_remote,
metadata=job_metadata,
kwargs=job_kwargs)
logger.info(
f"Dispatched import_labbook_from_remote({remote_url}) to Job {job_key}"
)
return ImportRemoteLabbook(job_key=job_key.key_str)
def test_run_only_once(self):
# Assert that this method only gets called once.
d = Dispatcher()
path = "/tmp/labmanager-unit-test-{}".format(os.getpid())
if os.path.exists(path):
os.remove(path)
future_t = datetime.datetime.utcnow() + datetime.timedelta(seconds=1)
jr = d.schedule_task(bg_jobs.test_incr,
scheduled_time=future_t,
args=(path, ),
repeat=0)
time.sleep(4)
with open(path) as fp:
assert json.load(fp)['amt'] == 1
def test_get_background_jobs_basics(self, fixture_working_dir_env_repo_scoped):
d = Dispatcher()
time.sleep(0.25)
t1 = d.dispatch_task(jobs.test_exit_fail).key_str
t2 = d.dispatch_task(jobs.test_exit_success).key_str
t3 = d.dispatch_task(jobs.test_sleep, args=(5,)).key_str
query = """
{
backgroundJobs {
edges {
node {
id
jobKey
failureMessage
status
result
}
}
}
}
"""
time.sleep(1)
try:
time1 = time.time()
result = fixture_working_dir_env_repo_scoped[2].execute(query)
time2 = time.time()
tdiff = time2 - time1
assert tdiff < 0.5, "Query should not take more than a few millis (took {}s)".format(tdiff)
assert any([t1 == x['node']['jobKey']
and 'failed' == x['node']['status']
and 'Exception: ' in x['node']['failureMessage']
for x in result['data']['backgroundJobs']['edges']])
assert any([t2 == x['node']['jobKey'] and "finished" == x['node']['status']
and x['node']['failureMessage'] is None
for x in result['data']['backgroundJobs']['edges']])
assert any([t3 == x['node']['jobKey'] and "started" == x['node']['status']
and x['node']['failureMessage'] is None
for x in result['data']['backgroundJobs']['edges']])
finally:
time.sleep(2)
def mutate_and_process_upload(cls, info, upload_file_path, upload_filename,
**kwargs):
if not upload_file_path:
logger.error('No file uploaded')
raise ValueError('No file uploaded')
username = get_logged_in_username()
job_metadata = {'method': 'import_dataset_from_zip'}
job_kwargs = {
'archive_path': upload_file_path,
'username': username,
'owner': username
}
dispatcher = Dispatcher()
job_key = dispatcher.dispatch_task(jobs.import_dataset_from_zip,
kwargs=job_kwargs,
metadata=job_metadata)
return ImportDataset(import_job_key=job_key.key_str)
def dispatcher_mock(self, function_ref, kwargs, metadata):
assert kwargs['logged_in_username'] == 'other-test-user2'
assert kwargs['dataset_owner'] == 'testuser'
assert kwargs['dataset_name'] == 'test-ds'
# Inject mocked config file
kwargs['config_file'] = mock_config_file[0]
# Stop patching so job gets scheduled for real
dispatcher_patch.stop()
# Call same method as in mutation
d = Dispatcher()
res = d.dispatch_task(
gtmcore.dispatcher.dataset_jobs.check_and_import_dataset,
kwargs=kwargs,
metadata=metadata)
return res
def mutate_and_get_payload(cls,
root,
info,
dataset_owner,
dataset_name,
labbook_name=None,
labbook_owner=None,
all_keys=None,
keys=None,
client_mutation_id=None):
logged_in_username = get_logged_in_username()
d = Dispatcher()
dl_kwargs = {
'logged_in_username': logged_in_username,
'access_token': flask.g.access_token,
'id_token': flask.g.id_token,
'dataset_owner': dataset_owner,
'dataset_name': dataset_name,
'labbook_owner': labbook_owner,
'labbook_name': labbook_name,
'all_keys': all_keys,
'keys': keys
}
# Gen unique keys for tracking jobs
lb_key = f"{logged_in_username}|{labbook_owner}|{labbook_name}" if labbook_owner else None
ds_key = f"{logged_in_username}|{dataset_owner}|{dataset_name}"
if lb_key:
ds_key = f"{lb_key}|LINKED|{ds_key}"
metadata = {
'dataset': ds_key,
'labbook': lb_key,
'method': 'download_dataset_files'
}
res = d.dispatch_task(dataset_jobs.download_dataset_files,
kwargs=dl_kwargs,
metadata=metadata,
persist=True)
return DownloadDatasetFiles(background_job_key=res.key_str)
def test_unschedule_midway_through(self):
d = Dispatcher()
path = "/tmp/labmanager-unit-test-{}".format(os.getpid())
if os.path.exists(path):
os.remove(path)
future_t = None # i.e., start right now.
jr = d.schedule_task(bg_jobs.test_incr,
scheduled_time=future_t,
args=(path, ),
repeat=6,
interval=5)
time.sleep(8)
n = d.unschedule_task(jr)
assert n, "Task should have been cancelled, instead it was not found."
time.sleep(5)
with open(path) as fp:
assert json.load(fp)['amt'] in [2]
def test_schedule_with_repeat_is_zero(self):
# When repeat is zero, it should run only once.
d = Dispatcher()
path = "/tmp/labmanager-unit-test-{}".format(os.getpid())
if os.path.exists(path):
os.remove(path)
jr = d.schedule_task(bg_jobs.test_incr,
args=(path, ),
repeat=0,
interval=4)
time.sleep(6)
n = d.unschedule_task(jr)
time.sleep(5)
with open(path) as fp:
assert json.load(fp)['amt'] in [
1
], "When repeat=0, the task should run only once."
def dispatcher_mock(self, function_ref, kwargs, metadata):
assert kwargs['logged_in_username'] == 'default'
assert kwargs['logged_in_email'] == '*****@*****.**'
assert kwargs['dataset_owner'] == 'default'
assert kwargs['dataset_name'] == 'dataset1'
# Inject mocked config file
kwargs['config_file'] = mock_create_dataset[0]
# Stop patching so job gets scheduled for real
dispatcher_patch.stop()
# Call same method as in mutation
d = Dispatcher()
kwargs['dispatcher'] = Dispatcher
res = d.dispatch_task(gtmcore.dispatcher.dataset_jobs.
complete_dataset_upload_transaction,
kwargs=kwargs,
metadata=metadata)
return res
def mutate_and_get_payload(cls,
root,
info,
owner,
labbook_name,
client_mutation_id=None):
username = get_logged_in_username()
lb = InventoryManager().load_labbook(username, owner, labbook_name)
d = Dispatcher()
lb_jobs = d.get_jobs_for_labbook(lb.key)
jobs = [
j for j in d.get_jobs_for_labbook(lb.key)
if j.meta.get('method') == 'build_image' and j.status == 'started'
]
if len(jobs) == 1:
d.abort_task(jobs[0].job_key)
ContainerOperations.delete_image(lb, username=username)
return CancelBuild(build_stopped=True, message="Stopped build")
elif len(jobs) == 0:
logger.warning(f"No build_image tasks found for {str(lb)}")
return CancelBuild(build_stopped=False,
message="No build task found")
else:
logger.warning(f"Multiple build jobs found for {str(lb)}")
return CancelBuild(build_stopped=False,
message="Multiple builds found")
