def build_operator_image(root_dir, registry, project=None, should_push=True):
"""Build the main docker image for the TfJob CRD.
Args:
root_dir: Root directory of the repository.
registry: The registry to use.
project: If set it will be built using GCB.
Returns:
build_info: Dictionary containing information about the build.
"""
context_dir = tempfile.mkdtemp(prefix="tmpTfJobCrdContext")
logging.info("context_dir: %s", context_dir)
if not os.path.exists(context_dir):
os.makedirs(context_dir)
# Build the go binaries
go_path = os.environ["GOPATH"]
targets = [
"github.com/tensorflow/k8s/cmd/tf_operator",
"github.com/tensorflow/k8s/test/e2e",
]
for t in targets:
util.run(["go", "install", t])
# List of paths to copy relative to root.
sources = [
"images/tf_operator/Dockerfile",
os.path.join(go_path, "bin/tf_operator"),
os.path.join(go_path, "bin/e2e"),
"grpc_tensorflow_server/grpc_tensorflow_server.py"
]
for s in sources:
src_path = os.path.join(root_dir, s)
dest_path = os.path.join(context_dir, os.path.basename(s))
if os.path.exists(dest_path):
os.unlink(dest_path)
if os.path.isdir(src_path):
shutil.copytree(src_path, dest_path)
else:
shutil.copyfile(src_path, dest_path)
image_base = registry + "/tf_operator"
n = datetime.datetime.now()
commit = build_and_push_image.GetGitHash(root_dir)
image = (image_base + ":" + n.strftime("v%Y%m%d") + "-" + commit)
latest_image = image_base + ":latest"
if project:
util.run([
"gcloud", "container", "builds", "submit", context_dir,
"--tag=" + image, "--project=" + project
])
# Add the latest tag.
util.run([
"gcloud", "container", "images", "add-tag", "--quiet", image,
latest_image
])
else:
util.run(["docker", "build", "-t", image, context_dir])
logging.info("Built image: %s", image)
util.run(["docker", "tag", image, latest_image])
if should_push:
util.run(["gcloud", "docker", "--", "push", image])
logging.info("Pushed image: %s", image)
util.run(["gcloud", "docker", "--", "push", latest_image])
logging.info("Pushed image: %s", latest_image)
output = {
"image": image,
"commit": commit,
}
return output
def build_operator_image(root_dir,
registry,
project=None,
should_push=True,
version_tag=None):
"""Build the main docker image for the TFJob CRD.
Args:
root_dir: Root directory of the repository.
registry: The registry to use.
project: If set it will be built using GCB.
should_push: Should push the image to the registry, Defaule is True.
version_tag: Optional tag for the version. If not specified derive
the tag from the git hash.
Returns:
build_info: Dictionary containing information about the build.
"""
context_dir = tempfile.mkdtemp(prefix="tmpTFJobCrdContext")
logging.info("context_dir: %s", context_dir)
if not os.path.exists(context_dir):
os.makedirs(context_dir)
# Build the go binaries
go_path = os.environ["GOPATH"]
commit = build_and_push_image.GetGitHash(root_dir)
targets = [
"github.com/kubeflow/tf-operator/cmd/tf-operator",
"github.com/kubeflow/tf-operator/cmd/tf-operator.v2",
"github.com/kubeflow/tf-operator/cmd/tf-operator.v1beta1",
"github.com/kubeflow/tf-operator/test/e2e",
"github.com/kubeflow/tf-operator/dashboard/backend",
]
for t in targets:
if t in [
"github.com/kubeflow/tf-operator/cmd/tf-operator",
"github.com/kubeflow/tf-operator/cmd/tf-operator.v2",
"github.com/kubeflow/tf-operator/cmd/tf-operator.v1beta1"
]:
util.run([
"go", "install", "-ldflags",
"-X github.com/kubeflow/tf-operator/pkg/version.GitSHA={}".
format(commit), t
])
util.run(["go", "install", t])
# Dashboard's frontend:
# Resolving dashboard's front-end dependencies
util.run(
["yarn", "--cwd", "{}/dashboard/frontend".format(root_dir), "install"])
# Building dashboard's front-end
util.run(
["yarn", "--cwd", "{}/dashboard/frontend".format(root_dir), "build"])
# If the release is not done from a Linux machine
# we need to grab the artefacts from /bin/linux_amd64
bin_path = "bin"
if platform.system() != "Linux":
bin_path += "/linux_amd64"
# List of paths to copy relative to root.
sources = [
"build/images/tf_operator/Dockerfile",
"examples/tf_sample/tf_smoke.py",
os.path.join(go_path, bin_path, "tf-operator"),
os.path.join(go_path, bin_path, "tf-operator.v2"),
os.path.join(go_path, bin_path, "tf-operator.v1beta1"),
os.path.join(go_path, bin_path, "e2e"),
os.path.join(go_path, bin_path, "backend"), "dashboard/frontend/build"
]
for s in sources:
src_path = os.path.join(root_dir, s)
dest_path = os.path.join(context_dir, os.path.basename(s))
if os.path.exists(dest_path):
os.unlink(dest_path)
if os.path.isdir(src_path):
shutil.copytree(src_path, dest_path)
else:
shutil.copyfile(src_path, dest_path)
image_base = registry + "/tf_operator"
if not version_tag:
logging.info("No version tag specified; computing tag automatically.")
n = datetime.datetime.now()
version_tag = n.strftime("v%Y%m%d") + "-" + commit
logging.info("Using version tag: %s", version_tag)
image = image_base + ":" + version_tag
latest_image = image_base + ":latest"
if project:
util.run([
"gcloud", "container", "builds", "submit", context_dir,
"--tag=" + image, "--project=" + project
])
# Add the latest tag.
util.run([
"gcloud", "container", "images", "add-tag", "--quiet", image,
latest_image
])
else:
util.run(["docker", "build", "-t", image, context_dir])
logging.info("Built image: %s", image)
util.run(["docker", "tag", image, latest_image])
if should_push:
_push_image(image, latest_image)
output = {
"image": image,
"commit": commit,
}
return output