def main() -> int:
"""
Utility to create and publish the Docker cache to Docker Hub
:return:
"""
# We need to be in the same directory than the script so the commands in the dockerfiles work as
# expected. But the script can be invoked from a different path
base = os.path.split(os.path.realpath(__file__))[0]
os.chdir(base)
logging.getLogger().setLevel(logging.DEBUG)
logging.getLogger('botocore').setLevel(logging.INFO)
logging.getLogger('boto3').setLevel(logging.INFO)
logging.getLogger('urllib3').setLevel(logging.INFO)
logging.getLogger('s3transfer').setLevel(logging.INFO)
def script_name() -> str:
return os.path.split(sys.argv[0])[1]
logging.basicConfig(format='{}: %(asctime)-15s %(message)s'.format(script_name()))
parser = argparse.ArgumentParser(description="Utility for preserving and loading Docker cache", epilog="")
parser.add_argument("--docker-registry",
help="Docker hub registry name",
type=str,
required=True)
args = parser.parse_args()
platforms = build_util.get_platforms()
try:
_login_dockerhub()
return build_save_containers(platforms=platforms, registry=args.docker_registry, load_cache=True)
finally:
_logout_dockerhub()
def main() -> int:
# We need to be in the same directory than the script so the commands in the dockerfiles work as
# expected. But the script can be invoked from a different path
base = os.path.split(os.path.realpath(__file__))[0]
os.chdir(base)
logging.getLogger().setLevel(logging.DEBUG)
logging.getLogger('botocore').setLevel(logging.INFO)
logging.getLogger('boto3').setLevel(logging.INFO)
logging.getLogger('urllib3').setLevel(logging.INFO)
logging.getLogger('s3transfer').setLevel(logging.INFO)
def script_name() -> str:
return os.path.split(sys.argv[0])[1]
logging.basicConfig(
format='{}: %(asctime)-15s %(message)s'.format(script_name()))
parser = argparse.ArgumentParser(
description="Utility for preserving and loading Docker cache",
epilog="")
parser.add_argument(
"--docker-cache-bucket",
help="S3 docker cache bucket, e.g. mxnet-ci-docker-cache",
type=str,
required=True)
args = parser.parse_args()
platforms = build_util.get_platforms()
_get_aws_session() # Init AWS credentials
return build_save_containers(platforms=platforms,
bucket=args.docker_cache_bucket)
def main() -> int:
# We need to be in the same directory than the script so the commands in the dockerfiles work as
# expected. But the script can be invoked from a different path
base = os.path.split(os.path.realpath(__file__))[0]
os.chdir(base)
logging.getLogger().setLevel(logging.DEBUG)
logging.getLogger('botocore').setLevel(logging.INFO)
logging.getLogger('boto3').setLevel(logging.INFO)
logging.getLogger('urllib3').setLevel(logging.INFO)
logging.getLogger('s3transfer').setLevel(logging.INFO)
def script_name() -> str:
return os.path.split(sys.argv[0])[1]
logging.basicConfig(format='{}: %(asctime)-15s %(message)s'.format(script_name()))
parser = argparse.ArgumentParser(description="Utility for preserving and loading Docker cache",epilog="")
parser.add_argument("--docker-cache-bucket",
help="S3 docker cache bucket, e.g. mxnet-ci-docker-cache",
type=str,
required=True)
args = parser.parse_args()
platforms = build_util.get_platforms()
_get_aws_session() # Init AWS credentials
return build_save_containers(platforms=platforms, bucket=args.docker_cache_bucket)
def main() -> int:
"""
Utility to create and publish the Docker cache to Docker Hub
:return:
"""
# We need to be in the same directory than the script so the commands in the dockerfiles work as
# expected. But the script can be invoked from a different path
base = os.path.split(os.path.realpath(__file__))[0]
os.chdir(base)
logging.getLogger().setLevel(logging.DEBUG)
logging.getLogger('botocore').setLevel(logging.INFO)
logging.getLogger('boto3').setLevel(logging.INFO)
logging.getLogger('urllib3').setLevel(logging.INFO)
logging.getLogger('s3transfer').setLevel(logging.INFO)
def script_name() -> str:
return os.path.split(sys.argv[0])[1]
logging.basicConfig(
format='{}: %(asctime)-15s %(message)s'.format(script_name()))
parser = argparse.ArgumentParser(
description="Utility for preserving and loading Docker cache",
epilog="")
parser.add_argument("--docker-registry",
help="Docker hub registry name",
type=str,
required=True)
parser.add_argument("--no-publish",
help="Only build but don't publish. Used for testing.",
action='store_true')
args = parser.parse_args()
platforms = build_util.get_platforms(legacy_only=True)
secret_name = os.environ['DOCKERHUB_SECRET_NAME']
endpoint_url = os.environ['DOCKERHUB_SECRET_ENDPOINT_URL']
region_name = os.environ['DOCKERHUB_SECRET_ENDPOINT_REGION']
try:
if not args.no_publish:
login_dockerhub(secret_name, endpoint_url, region_name)
return build_save_containers(platforms=platforms,
registry=args.docker_registry,
load_cache=True,
no_publish=args.no_publish)
finally:
logout_dockerhub()
return_text += "<p>Source RPMS: \n"
for i in external_src_rpms:
i = os.path.basename(i)
NAME = rpm_utils.rpm_query(
'NAME', '../external_packages/%s/%s' % (conf_obj.name, i))
return_text += "<a href='%s/%s/%s'>%s</a> \n" % (
external_url_path, conf_obj.name, i, NAME)
return_text += "</p>\n"
return return_text
distros = build.get_platforms()
# Command line options
try:
opts, remaining_args = getopt.getopt(sys.argv[1:], "",
["skip_zip", "platforms="])
(bundle, output_dir, webroot_path, package_src_dir,
hostname_url) = remaining_args
except:
print "Usage: ./mk-distro-index.py [--skip_zip] <bundle name> <output webdir> <webroot_path> <package source dir> <hostname_url>"
print " --platforms: comma separated list of platforms (distros) to include"
sys.exit(1)
skip_zip = False
for option, value in opts:
if option == "--skip_zip":
import packaging
import build
import utils
import config
import datastore
import string
import distutils
include_packages = False
include_zip = False
fail_on_missing = True
skip_installers = False
skip_obs_repos = False
validated = False
config.sd_latest_build_distros = build.get_platforms()
try:
opts, remaining_args = getopt.getopt(sys.argv[1:], "", [
"include_packages", "include_zip", "skip_missing", "skip_installers",
"skip_obs_repos", "platforms=", "validated"
])
for option, value in opts:
if option == "--include_packages":
include_packages = True
if option == "--include_zip":
include_zip = True
if option == "--skip_missing":
fail_on_missing = False
if option == "--skip_installers":
skip_installers = True
import sys
import re
import shutil
import getopt
import glob
sys.path.append("../pyutils")
import utils
import packaging
import build
source_basepath = ""
package_basepath = ""
archive_basepath = ""
platforms = build.get_platforms()
try:
opts, remaining_args = getopt.getopt(sys.argv[1:], "", [ "source_basepath=", "package_basepath=", "archive_basepath=", "platforms=", ])
for option, value in opts:
if option == "--source_basepath":
source_basepath = value
if option == "--package_basepath":
package_basepath = value
if option == "--archive_basepath":
archive_basepath = value
if option == "--platforms":
platforms = value.split(",")
(HEAD_or_RELEASE, num_builds) = remaining_args
import packaging import build import utils import config import datastore import string import distutils include_packages = False include_zip = False fail_on_missing=True skip_installers = False skip_obs_repos = False validated = False config.sd_latest_build_distros = build.get_platforms() try: opts, remaining_args = getopt.getopt(sys.argv[1:], "", [ "include_packages", "include_zip", "skip_missing", "skip_installers", "skip_obs_repos", "platforms=", "validated" ]) for option, value in opts: if option == "--include_packages": include_packages = True if option == "--include_zip": include_zip = True if option == "--skip_missing": fail_on_missing = False if option == "--skip_installers": skip_installers = True if option == "--skip_obs_repos": skip_obs_repos = True if option == "--platforms":
# For the source RPMS
if external_src_rpms:
return_text += "<p>Source RPMS: \n"
for i in external_src_rpms:
i = os.path.basename(i)
NAME = rpm_utils.rpm_query('NAME', '../external_packages/%s/%s' % (conf_obj.name, i) )
return_text += "<a href='%s/%s/%s'>%s</a> \n" % (external_url_path, conf_obj.name, i, NAME)
return_text += "</p>\n"
return return_text
distros = build.get_platforms()
# Command line options
try:
opts, remaining_args = getopt.getopt(sys.argv[1:], "", [ "skip_zip", "platforms=" ])
(bundle, output_dir, webroot_path, package_src_dir, hostname_url) = remaining_args
except:
print "Usage: ./mk-distro-index.py [--skip_zip] <bundle name> <output webdir> <webroot_path> <package source dir> <hostname_url>"
print " --platforms: comma separated list of platforms (distros) to include"
sys.exit(1)
skip_zip = False
for option, value in opts:
if option == "--skip_zip":
skip_zip = True
# override distro list
