def test_remote_worker(self): experiment_name = 'test_remote_worker_' + str(uuid.uuid4()) queue_name = experiment_name logger = logging.getLogger('test_remote_worker') logger.setLevel(10) pw = subprocess.Popen( ['studio-start-remote-worker', '--queue=' + queue_name, '--single-run', '--timeout=30', '--image=peterzhokhoff/studioml'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT ) stubtest_worker( self, experiment_name=experiment_name, runner_args=['--queue=' + queue_name, '--force-git'], config_name='test_config_http_client.yaml', test_script='tf_hello_world.py', script_args=['arg0'], expected_output='[ 2. 6.]', queue=PubsubQueue(queue_name)) workerout, _ = pw.communicate() if workerout: logger.debug("studio-start-remote-worker output: \n" + workerout)
def test_remote_worker_co(self): logger = logs.getLogger('test_remote_worker_co') logger.setLevel(10) tmpfile = os.path.join(tempfile.gettempdir(), str(uuid.uuid4())) random_str = str(uuid.uuid4()) with open(tmpfile, 'w') as f: f.write(random_str) experiment_name = 'test_remote_worker_co_' + str(uuid.uuid4()) queue_name = experiment_name pw = subprocess.Popen([ 'studio-start-remote-worker', '--queue=' + queue_name, '--single-run', '--no-cache', '--image=peterzhokhoff/studioml' ], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) stubtest_worker(self, experiment_name=experiment_name, runner_args=[ '--capture-once=' + tmpfile + ':f', '--queue=' + queue_name, '--force-git' ], config_name='test_config_http_client.yaml', test_script='art_hello_world.py', script_args=[], expected_output=random_str, queue=PubsubQueue(queue_name)) workerout, _ = pw.communicate() logger.debug('studio-start-remote-worker output: \n' + str(workerout)) os.remove(tmpfile)
def test_args_conflict(self): stubtest_worker(self, experiment_name='test_runner_conflict_' + str(uuid.uuid4()), runner_args=['--verbose=debug'], config_name='test_config.yaml', test_script='conflicting_args.py', script_args=['--experiment', 'aaa'], expected_output='Experiment key = aaa')
def test_worker(self): experiment_name = 'test_ec2_worker_' + str(uuid.uuid4()) stubtest_worker( self, experiment_name=experiment_name, runner_args=['--cloud=ec2', '--force-git', '--gpus=1', '--cloud-timeout=-1'], config_name='test_config.yaml', test_script='tf_hello_world.py', script_args=['arg0'], expected_output='[ 2. 6.]', )
def test_worker_spot(self): experiment_name = 'test_ec2_worker_' + str(uuid.uuid4()) stubtest_worker( self, experiment_name=experiment_name, runner_args=[ '--cloud=ec2spot', '--force-git', '--bid=50%', '--cloud-timeout=120', ], config_name='test_config_http_client.yaml', test_script='tf_hello_world.py', script_args=['arg0'], expected_output='[ 2. 6.]', )
def test_remote_worker_c(self): tmpfile = os.path.join(tempfile.gettempdir(), str(uuid.uuid4())) logger = logging.getLogger('test_remote_worker_c') logger.setLevel(10) experiment_name = "test_remote_worker_c_" + str(uuid.uuid4()) random_str1 = str(uuid.uuid4()) with open(tmpfile, 'w') as f: f.write(random_str1) random_str2 = str(uuid.uuid4()) queue_name = experiment_name pw = subprocess.Popen( ['studio-start-remote-worker', '--queue=' + queue_name, '--single-run', '--image=peterzhokhoff/studioml'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) db = stubtest_worker( self, experiment_name=experiment_name, runner_args=[ '--capture=' + tmpfile + ':f', '--queue=' + queue_name, '--force-git'], config_name='test_config_http_client.yaml', test_script='art_hello_world.py', script_args=[random_str2], expected_output=random_str1, queue=PubsubQueue(queue_name), delete_when_done=False) workerout, _ = pw.communicate() if workerout: logger.debug("studio-start-remote-worker output: \n" + workerout) os.remove(tmpfile) tmppath = os.path.join(tempfile.gettempdir(), str(uuid.uuid4())) if os.path.exists(tmppath): os.remove(tmppath) db.get_artifact( db.get_experiment(experiment_name).artifacts['f'], tmppath, only_newer=False ) with open(tmppath, 'r') as f: self.assertEquals(f.read(), random_str2) os.remove(tmppath) db.delete_experiment(experiment_name)
def test_worker(self): experiment_name = 'test_gcloud_worker_' + str(uuid.uuid4()) with stubtest_worker( self, experiment_name=experiment_name, runner_args=[ '--cloud=gcloud', '--force-git', '--cloud-timeout=120' ], config_name='test_config_http_client.yaml', test_script='tf_hello_world.py', script_args=['arg0'], expected_output='[ 2. 6.]', ): pass
def test_worker_spot_container(self): experiment_name = 'test_gcloud_spot_simg_' + str(uuid.uuid4()) with stubtest_worker(self, experiment_name=experiment_name, runner_args=[ '--cloud=gcspot', '--force-git', '--cloud-timeout=120', '--container=shub://vsoch/hello-world' ], config_name='test_config_http_client.yaml', test_script='', script_args=[], expected_output='RaawwWWWWWRRRR!!', test_workspace=False): pass
def test_worker(self): experiment_name = 'test_ec2_worker_' + str(uuid.uuid4()) with stubtest_worker( self, experiment_name=experiment_name, runner_args=[ '--cloud=ec2', '--force-git', '--gpus=1', '--cloud-timeout=120', '--ssh-keypair=peterz-k1' ], config_name='test_config_http_client.yaml', test_script='tf_hello_world.py', script_args=['arg0'], expected_output='[ 2.0 6.0 ]', ): pass
def test_baked_image(self): ''' create a docker image with baked in credentials and run a remote worker tests with it ''' logger = logging.getLogger('test_baked_image') logger.setLevel(logging.DEBUG) # check if docker is installed dockertestp = subprocess.Popen(['docker'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) dockertestout, _ = dockertestp.communicate() if dockertestout: logger.info("docker test output: \n" + dockertestout) if dockertestp.returncode != 0: logger.error("docker is not installed (correctly)") return image = 'test_image' + str(uuid.uuid4()) addcredsp = subprocess.Popen( [ 'studio-add-credentials', '--tag=' + image, '--base-image=peterzhokhoff/studioml'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) addcredsout, _ = addcredsp.communicate() if addcredsout: logger.info('studio-add-credentials output: \n' + addcredsout) if addcredsp.returncode != 0: logger.error("studio-add-credentials failed.") self.assertTrue(False) experiment_name = 'test_remote_worker_baked' + str(uuid.uuid4()) queue_name = experiment_name logger = logging.getLogger('test_baked_image') logger.setLevel(10) pw = subprocess.Popen( ['studio-start-remote-worker', '--queue=' + queue_name, '--single-run', '--timeout=30', '--image=' + image], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) stubtest_worker( self, experiment_name=experiment_name, runner_args=['--queue=' + queue_name, '--force-git'], config_name='test_config_http_client.yaml', test_script='tf_hello_world.py', script_args=['arg0'], expected_output='[ 2. 6.]', queue=PubsubQueue(queue_name)) workerout, _ = pw.communicate() if workerout: logger.debug("studio-start-remote-worker output: \n" + workerout) rmip = subprocess.Popen(['docker', 'rmi', image], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) rmiout, _ = rmip.communicate() if rmiout: logger.info('docker rmi output: \n' + rmiout)