def create_instance(instance_name, cpu=1, mem=4): tf.logging.info("Creating instance %s", instance_name) out = cloud.shell_output(CREATE_INSTANCE, instance_name=instance_name, cpu=cpu, mem=mem) return out.split("\n")[1:-1][0].split()[8]
def tar_and_copy_t2t(train_dir): """Tar Tensor2Tensor and cp to train_dir.""" tf.logging.info("Tarring and pushing local Tensor2Tensor package.") output = text_encoder.native_to_unicode( cloud.shell_output("pip show tensor2tensor")).split("\n") assert output[1].startswith("Version") assert output[7].startswith("Location") t2t_version = output[1].split(":")[1].strip() t2t_dir = output[7].split(":")[1].strip() # A local installation cloned from GitHub will have a setup.py file and a docs # folder is_local_t2t = all([ tf.gfile.Exists(os.path.join(t2t_dir, fname)) for fname in ["setup.py", "docs/cloud_mlengine.md"] ]) if is_local_t2t: tf.logging.info("Found local T2T installation. Tarring directory %s", t2t_dir) else: # PyPI installation # Create a folder with just a setup.py file pointing to the right version tf.logging.info( "Found PyPI T2T installation. Launching tensor2tensor==%s", t2t_version) t2t_dir = os.path.join(tempfile.gettempdir(), "tensor2tensor_tmp") shutil.rmtree(t2t_dir, ignore_errors=True) os.mkdir(t2t_dir) setup_fname = os.path.join(t2t_dir, "setup.py") setup_file_str = get_setup_file( name="DummyT2TPackage", packages=["tensor2tensor==%s" % t2t_version]) with tf.gfile.Open(setup_fname, "w") as f: f.write(setup_file_str) t2t_tar = _tar_and_copy(t2t_dir, train_dir) return t2t_tar
def tar_and_copy_t2t(train_dir): """Tar Tensor2Tensor and cp to train_dir.""" tf.logging.info('Tarring and pushing local Tensor2Tensor package.') output = cloud.shell_output('pip show tensor2tensor').decode( 'utf-8').split('\n') assert output[1].startswith('Version') assert output[7].startswith('Location') t2t_version = output[1].split(':')[1].strip() t2t_dir = output[7].split(':')[1].strip() # A local installation cloned from GitHub will have a setup.py file and a docs # folder is_local_t2t = all([ tf.gfile.Exists(os.path.join(t2t_dir, fname)) for fname in ['setup.py', 'docs/cloud_mlengine.md'] ]) if is_local_t2t: tf.logging.info('Found local T2T installation. Tarring directory %s', t2t_dir) else: # PyPI installation # Create a folder with just a setup.py file pointing to the right version tf.logging.info( 'Found PyPI T2T installation. Launching tensor2tensor==%s', t2t_version) t2t_dir = os.path.join(tempfile.gettempdir(), 'tensor2tensor_tmp') shutil.rmtree(t2t_dir, ignore_errors=True) os.mkdir(t2t_dir) setup_fname = os.path.join(t2t_dir, 'setup.py') setup_file_str = get_setup_file( name='DummyT2TPackage', packages=['tensor2tensor==%s' % t2t_version]) with tf.gfile.Open(setup_fname, 'w') as f: f.write(setup_file_str) t2t_tar = _tar_and_copy(t2t_dir, train_dir) return t2t_tar
def list_vm_names_and_ips(): list_out = cloud.shell_output(cloud.Gcloud.LIST_VM) lines = [l.split() for l in list_out.split("\n")[1:-1]] names_and_ips = [(l[0].strip(), l[-2].strip()) for l in lines] return names_and_ips
def default_zone(): return cloud.shell_output(DEFAULT_ZONE).strip()