def test_remote_worker(self):
experiment_name = 'test_remote_worker_' + str(uuid.uuid4())
queue_name = experiment_name
logger = logging.getLogger('test_remote_worker')
logger.setLevel(10)
pw = subprocess.Popen(
['studio-start-remote-worker',
'--queue=' + queue_name,
'--single-run',
'--timeout=30',
'--image=peterzhokhoff/studioml'],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT
)
stubtest_worker(
self,
experiment_name=experiment_name,
runner_args=['--queue=' + queue_name, '--force-git'],
config_name='test_config_http_client.yaml',
test_script='tf_hello_world.py',
script_args=['arg0'],
expected_output='[ 2. 6.]',
queue=PubsubQueue(queue_name))
workerout, _ = pw.communicate()
if workerout:
logger.debug("studio-start-remote-worker output: \n" + workerout)
def test_remote_worker_co(self):
logger = logs.getLogger('test_remote_worker_co')
logger.setLevel(10)
tmpfile = os.path.join(tempfile.gettempdir(), str(uuid.uuid4()))
random_str = str(uuid.uuid4())
with open(tmpfile, 'w') as f:
f.write(random_str)
experiment_name = 'test_remote_worker_co_' + str(uuid.uuid4())
queue_name = experiment_name
pw = subprocess.Popen([
'studio-start-remote-worker', '--queue=' + queue_name,
'--single-run', '--no-cache', '--image=peterzhokhoff/studioml'
],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
stubtest_worker(self,
experiment_name=experiment_name,
runner_args=[
'--capture-once=' + tmpfile + ':f',
'--queue=' + queue_name, '--force-git'
],
config_name='test_config_http_client.yaml',
test_script='art_hello_world.py',
script_args=[],
expected_output=random_str,
queue=PubsubQueue(queue_name))
workerout, _ = pw.communicate()
logger.debug('studio-start-remote-worker output: \n' + str(workerout))
os.remove(tmpfile)
def test_args_conflict(self):
stubtest_worker(self,
experiment_name='test_runner_conflict_' +
str(uuid.uuid4()),
runner_args=['--verbose=debug'],
config_name='test_config.yaml',
test_script='conflicting_args.py',
script_args=['--experiment', 'aaa'],
expected_output='Experiment key = aaa')
def test_worker(self):
experiment_name = 'test_ec2_worker_' + str(uuid.uuid4())
stubtest_worker(
self,
experiment_name=experiment_name,
runner_args=['--cloud=ec2', '--force-git', '--gpus=1',
'--cloud-timeout=-1'],
config_name='test_config.yaml',
test_script='tf_hello_world.py',
script_args=['arg0'],
expected_output='[ 2. 6.]',
)
def test_worker_spot(self):
experiment_name = 'test_ec2_worker_' + str(uuid.uuid4())
stubtest_worker(
self,
experiment_name=experiment_name,
runner_args=[
'--cloud=ec2spot',
'--force-git',
'--bid=50%',
'--cloud-timeout=120',
],
config_name='test_config_http_client.yaml',
test_script='tf_hello_world.py',
script_args=['arg0'],
expected_output='[ 2. 6.]',
)
def test_remote_worker_c(self):
tmpfile = os.path.join(tempfile.gettempdir(),
str(uuid.uuid4()))
logger = logging.getLogger('test_remote_worker_c')
logger.setLevel(10)
experiment_name = "test_remote_worker_c_" + str(uuid.uuid4())
random_str1 = str(uuid.uuid4())
with open(tmpfile, 'w') as f:
f.write(random_str1)
random_str2 = str(uuid.uuid4())
queue_name = experiment_name
pw = subprocess.Popen(
['studio-start-remote-worker',
'--queue=' + queue_name,
'--single-run',
'--image=peterzhokhoff/studioml'],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
db = stubtest_worker(
self,
experiment_name=experiment_name,
runner_args=[
'--capture=' + tmpfile + ':f',
'--queue=' + queue_name,
'--force-git'],
config_name='test_config_http_client.yaml',
test_script='art_hello_world.py',
script_args=[random_str2],
expected_output=random_str1,
queue=PubsubQueue(queue_name),
delete_when_done=False)
workerout, _ = pw.communicate()
if workerout:
logger.debug("studio-start-remote-worker output: \n" + workerout)
os.remove(tmpfile)
tmppath = os.path.join(tempfile.gettempdir(), str(uuid.uuid4()))
if os.path.exists(tmppath):
os.remove(tmppath)
db.get_artifact(
db.get_experiment(experiment_name).artifacts['f'],
tmppath,
only_newer=False
)
with open(tmppath, 'r') as f:
self.assertEquals(f.read(), random_str2)
os.remove(tmppath)
db.delete_experiment(experiment_name)
def test_worker(self):
experiment_name = 'test_gcloud_worker_' + str(uuid.uuid4())
with stubtest_worker(
self,
experiment_name=experiment_name,
runner_args=[
'--cloud=gcloud', '--force-git', '--cloud-timeout=120'
],
config_name='test_config_http_client.yaml',
test_script='tf_hello_world.py',
script_args=['arg0'],
expected_output='[ 2. 6.]',
):
pass
def test_worker_spot_container(self):
experiment_name = 'test_gcloud_spot_simg_' + str(uuid.uuid4())
with stubtest_worker(self,
experiment_name=experiment_name,
runner_args=[
'--cloud=gcspot', '--force-git',
'--cloud-timeout=120',
'--container=shub://vsoch/hello-world'
],
config_name='test_config_http_client.yaml',
test_script='',
script_args=[],
expected_output='RaawwWWWWWRRRR!!',
test_workspace=False):
pass
def test_worker(self):
experiment_name = 'test_ec2_worker_' + str(uuid.uuid4())
with stubtest_worker(
self,
experiment_name=experiment_name,
runner_args=[
'--cloud=ec2', '--force-git', '--gpus=1',
'--cloud-timeout=120', '--ssh-keypair=peterz-k1'
],
config_name='test_config_http_client.yaml',
test_script='tf_hello_world.py',
script_args=['arg0'],
expected_output='[ 2.0 6.0 ]',
):
pass
def test_baked_image(self):
'''
create a docker image with baked in credentials
and run a remote worker tests with it
'''
logger = logging.getLogger('test_baked_image')
logger.setLevel(logging.DEBUG)
# check if docker is installed
dockertestp = subprocess.Popen(['docker'],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
dockertestout, _ = dockertestp.communicate()
if dockertestout:
logger.info("docker test output: \n" + dockertestout)
if dockertestp.returncode != 0:
logger.error("docker is not installed (correctly)")
return
image = 'test_image' + str(uuid.uuid4())
addcredsp = subprocess.Popen(
[
'studio-add-credentials',
'--tag=' + image,
'--base-image=peterzhokhoff/studioml'],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
addcredsout, _ = addcredsp.communicate()
if addcredsout:
logger.info('studio-add-credentials output: \n' + addcredsout)
if addcredsp.returncode != 0:
logger.error("studio-add-credentials failed.")
self.assertTrue(False)
experiment_name = 'test_remote_worker_baked' + str(uuid.uuid4())
queue_name = experiment_name
logger = logging.getLogger('test_baked_image')
logger.setLevel(10)
pw = subprocess.Popen(
['studio-start-remote-worker',
'--queue=' + queue_name,
'--single-run',
'--timeout=30',
'--image=' + image],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
stubtest_worker(
self,
experiment_name=experiment_name,
runner_args=['--queue=' + queue_name, '--force-git'],
config_name='test_config_http_client.yaml',
test_script='tf_hello_world.py',
script_args=['arg0'],
expected_output='[ 2. 6.]',
queue=PubsubQueue(queue_name))
workerout, _ = pw.communicate()
if workerout:
logger.debug("studio-start-remote-worker output: \n" + workerout)
rmip = subprocess.Popen(['docker', 'rmi', image],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
rmiout, _ = rmip.communicate()
if rmiout:
logger.info('docker rmi output: \n' + rmiout)
