def cp(src, dst):
"""Copy file from a persistent storage"""
platform = CONFIG["platform"]
parts_src = parse_file_path(src)
parts_dst = parse_file_path(dst)
if parts_src[0] and parts_dst[0]:
raise ValueError(
"Can't copy from persistent storage to persistent storage")
if parts_src[0]:
volume = parts_src[0]
elif parts_dst[0]:
volume = parts_dst[0]
else:
raise ValueError("Must copy/to from a persistent volume")
k8s_backend = get_backend(platform)
job_spec = core.JobSpec(
image="alpine:3.10",
name="fiber-cp",
command=["sleep", "60"],
volumes={volume: {
"mode": "rw",
"bind": "/persistent"
}},
)
job = k8s_backend.create_job(job_spec)
pod_name = job.data.metadata.name
print("launched pod: {}".format(pod_name))
exitcode = os.system(
"kubectl wait --for=condition=Ready pod/{}".format(pod_name))
"""
status = k8s_backend.get_job_status(job)
while status == ProcessStatus.INITIAL:
print("Waiting for pod {} to be up".format(pod_name))
time.sleep(1)
if status != ProcessStatus.STARTED:
raise RuntimeError("Tempory pod failed: {}".format(pod_name))
"""
if parts_src[0]:
new_src = "{}:{}".format(pod_name, parts_src[1])
new_dst = dst
elif parts_dst[0]:
new_src = src
new_dst = "{}:{}".format(pod_name, parts_dst[1])
cmd = "kubectl cp {} {}".format(new_src, new_dst)
os.system(cmd)
def run(attach, build, gpu, cpu, memory, volume, args):
"""Run a command on a kubernetes cluster with fiber."""
platform = CONFIG["platform"]
print('Running "{}" on Kubernetes cluster ({}) '.format(
" ".join(args), platform))
if platform == "gcp":
builder = GCPImageBuilder()
elif platform == "aws":
registry = CONFIG["docker_registry"]
if not registry:
registry = input(
"What docker registry do you plan to use? "
"AWS registry format: "
'"[aws_account_id].dkr.ecr.[region].amazonaws.com"\n> ')
builder = AWSImageBuilder(registry)
else:
raise ValueError('Unknow platform "{}"'.format(platform))
full_image_name = builder.build()
"""
if build:
dockerfile = select_docker_file(files)
build_docker_image(dockerfile, image_base_name, full_image_name, info)
"""
# run this to refresh access tokens
exitcode = os.system("kubectl get po > /dev/null")
job_name = os.path.basename(os.getcwd())
k8s_backend = get_backend(platform)
job_spec = core.JobSpec(
image=full_image_name,
name=job_name,
command=args,
)
if gpu:
job_spec.gpu = gpu
if cpu:
job_spec.cpu = cpu
if memory:
job_spec.mem = memory
if volume:
volumes = {volume: {"mode": "rw", "bind": "/persistent"}}
job_spec.volumes = volumes
job = k8s_backend.create_job(job_spec)
pod_name = job.data.metadata.name
exitcode = 0
print("Created pod: {}".format(pod_name))
if attach:
# wait until job is running
"""
os.system(
"kubectl wait --for=condition=Ready pod/{}".format(pod_name)
)
"""
exitcode = os.system("kubectl logs -f {}".format(pod_name))
if exitcode != 0:
return exitcode
return 0
def run(attach, build, gpu, cpu, memory, volume, args):
"""Run a command on a kubernetes cluster with fiber."""
print("Running \"{}\" on Kubernetes cluster".format(" ".join(args)))
files = find_docker_files()
n = len(files)
if n == 0:
print("No docker files found")
return 1
cwd = os.path.basename(os.getcwd())
image_base_name = cwd
full_image_name = get_docker_registry_image_name(image_base_name)
if build:
dockerfile = select_docker_file(files)
build_docker_image(dockerfile, image_base_name, full_image_name)
# run this to refresh access tokens
exitcode = os.system("kubectl get po > /dev/null")
k8s_backend = Backend(incluster=False)
job_spec = core.JobSpec(
image=full_image_name,
name=image_base_name,
command=args,
)
if gpu:
job_spec.gpu = gpu
if cpu:
job_spec.cpu = cpu
if memory:
job_spec.mem = memory
if volume:
volumes = {volume: {"mode": "rw", "bind": "/persistent"}}
job_spec.volumes = volumes
job = k8s_backend.create_job(job_spec)
pod_name = job.data.metadata.name
exitcode = 0
print("Created pod: {}".format(pod_name))
if attach:
# wait until job is running
"""
os.system(
"kubectl wait --for=condition=Ready pod/{}".format(pod_name)
)
"""
exitcode = os.system("kubectl logs -f {}".format(pod_name))
if exitcode != 0:
return exitcode
return 0