def create_instance(instance_name, cpu=1, mem=4):
tf.logging.info("Creating instance %s", instance_name)
out = cloud.shell_output(CREATE_INSTANCE,
instance_name=instance_name,
cpu=cpu,
mem=mem)
return out.split("\n")[1:-1][0].split()[8]
def tar_and_copy_t2t(train_dir):
"""Tar Tensor2Tensor and cp to train_dir."""
tf.logging.info("Tarring and pushing local Tensor2Tensor package.")
output = text_encoder.native_to_unicode(
cloud.shell_output("pip show tensor2tensor")).split("\n")
assert output[1].startswith("Version")
assert output[7].startswith("Location")
t2t_version = output[1].split(":")[1].strip()
t2t_dir = output[7].split(":")[1].strip()
# A local installation cloned from GitHub will have a setup.py file and a docs
# folder
is_local_t2t = all([
tf.gfile.Exists(os.path.join(t2t_dir, fname))
for fname in ["setup.py", "docs/cloud_mlengine.md"]
])
if is_local_t2t:
tf.logging.info("Found local T2T installation. Tarring directory %s",
t2t_dir)
else:
# PyPI installation
# Create a folder with just a setup.py file pointing to the right version
tf.logging.info(
"Found PyPI T2T installation. Launching tensor2tensor==%s",
t2t_version)
t2t_dir = os.path.join(tempfile.gettempdir(), "tensor2tensor_tmp")
shutil.rmtree(t2t_dir, ignore_errors=True)
os.mkdir(t2t_dir)
setup_fname = os.path.join(t2t_dir, "setup.py")
setup_file_str = get_setup_file(
name="DummyT2TPackage",
packages=["tensor2tensor==%s" % t2t_version])
with tf.gfile.Open(setup_fname, "w") as f:
f.write(setup_file_str)
t2t_tar = _tar_and_copy(t2t_dir, train_dir)
return t2t_tar
def tar_and_copy_t2t(train_dir):
"""Tar Tensor2Tensor and cp to train_dir."""
tf.logging.info('Tarring and pushing local Tensor2Tensor package.')
output = cloud.shell_output('pip show tensor2tensor').decode(
'utf-8').split('\n')
assert output[1].startswith('Version')
assert output[7].startswith('Location')
t2t_version = output[1].split(':')[1].strip()
t2t_dir = output[7].split(':')[1].strip()
# A local installation cloned from GitHub will have a setup.py file and a docs
# folder
is_local_t2t = all([
tf.gfile.Exists(os.path.join(t2t_dir, fname))
for fname in ['setup.py', 'docs/cloud_mlengine.md']
])
if is_local_t2t:
tf.logging.info('Found local T2T installation. Tarring directory %s',
t2t_dir)
else:
# PyPI installation
# Create a folder with just a setup.py file pointing to the right version
tf.logging.info(
'Found PyPI T2T installation. Launching tensor2tensor==%s',
t2t_version)
t2t_dir = os.path.join(tempfile.gettempdir(), 'tensor2tensor_tmp')
shutil.rmtree(t2t_dir, ignore_errors=True)
os.mkdir(t2t_dir)
setup_fname = os.path.join(t2t_dir, 'setup.py')
setup_file_str = get_setup_file(
name='DummyT2TPackage',
packages=['tensor2tensor==%s' % t2t_version])
with tf.gfile.Open(setup_fname, 'w') as f:
f.write(setup_file_str)
t2t_tar = _tar_and_copy(t2t_dir, train_dir)
return t2t_tar
def list_vm_names_and_ips():
list_out = cloud.shell_output(cloud.Gcloud.LIST_VM)
lines = [l.split() for l in list_out.split("\n")[1:-1]]
names_and_ips = [(l[0].strip(), l[-2].strip()) for l in lines]
return names_and_ips
def default_zone():
return cloud.shell_output(DEFAULT_ZONE).strip()
def list_vm_names_and_ips():
list_out = cloud.shell_output(cloud.Gcloud.LIST_VM)
lines = [l.split() for l in list_out.split("\n")[1:-1]]
names_and_ips = [(l[0].strip(), l[-2].strip()) for l in lines]
return names_and_ips
def create_instance(instance_name, cpu=1, mem=4):
tf.logging.info("Creating instance %s", instance_name)
out = cloud.shell_output(CREATE_INSTANCE, instance_name=instance_name,
cpu=cpu, mem=mem)
return out.split("\n")[1:-1][0].split()[8]
def default_zone(): return cloud.shell_output(DEFAULT_ZONE).strip()
