def do_lab(lab_dir, lab, role, source_reg, dest_reg, force, logger):
docker_dir = os.path.join(labdir, lab, 'dockerfiles')
if not os.path.isdir(docker_dir):
return
df_list = [f for f in os.listdir(docker_dir) if os.path.isfile(os.path.join(docker_dir, f))]
for df in df_list:
if df.endswith('.swp'):
continue
try:
parts = df.split('.')
image = '%s.%s.%s' % (parts[1], parts[2], role)
except:
print('could not get image from %s' % df);
continue
local_created, local_user, version, tag, base = InspectLocalReg.inspectLocal(image, logger, dest_reg)
if local_created is not None:
with_reg = '%s/%s' % (source_reg, image)
remote_created, remote_user, version, tag = InspectRemoteReg.inspectRemote(with_reg, logger)
if force or local_created is None or remote_created > local_created:
cmd = 'docker pull %s/%s' % (source_reg, image)
#print cmd
os.system(cmd)
cmd = 'docker tag %s/%s %s/%s' % (source_reg, image, dest_reg, image)
#print cmd
os.system(cmd)
cmd = 'docker push %s/%s' % (dest_reg, image)
#print cmd
os.system(cmd)
else:
print('local registry for %s is up to date.' % image)
def updateLab(labdir, lab, role, remote_reg, local_reg, logger, no_copy):
''' push local lab containers to remote, i.e., as part of a release '''
docker_dir = os.path.join(labdir, lab, 'dockerfiles')
if not os.path.isdir(docker_dir):
return
df_list = [f for f in os.listdir(docker_dir) if os.path.isfile(os.path.join(docker_dir, f))]
for df in df_list:
if df.endswith('.swp'):
continue
try:
parts = df.split('.')
image = '%s.%s.%s' % (parts[1], parts[2], role)
container = parts[2]
except:
print('could not get image from %s' % df);
continue
local_created, local_user, local_version, tag, base = InspectLocalReg.inspectLocal(image, logger, local_reg, no_pull=True)
if local_created is not None:
with_reg = '%s/%s' % (remote_reg, image)
remote_created, remote_user, remote_version, tag = InspectRemoteReg.inspectRemote(with_reg, logger, no_pull=True)
if local_created != remote_created:
print('DIFFERENT: %s:%s local created/version %s/%s remote: %s/%s' % (lab, container, local_created,
local_version, remote_created, remote_version))
logger.debug('DIFFERENT: %s:%s local created/version %s/%s remote: %s/%s' % (lab, container, local_created,
local_version, remote_created, remote_version))
if not no_copy:
pull_push(image, local_reg, remote_reg)
else:
logger.debug('updateLab, no diff for %s' % image)
else:
print('ERROR, no local info for image %s' % image)
exit(1)
def do_lab(lab_dir, lab, role, registry):
framework_version = labutils.framework_version
docker_dir = os.path.join(lab_dir, lab, 'dockerfiles')
if not os.path.isdir(docker_dir):
print('%s not a directory' % docker_dir)
return
print('LAB: %s' % lab)
df_list = [
f for f in os.listdir(docker_dir)
if os.path.isfile(os.path.join(docker_dir, f))
]
for df in df_list:
if df.endswith('.swp'):
continue
dfile_path = os.path.join(docker_dir, df)
image_base = VersionInfo.getFrom(dfile_path, registry)
print('image base from %s is %s' % (dfile_path, image_base))
base_id = VersionInfo.getImageId(image_base)
try:
parts = df.split('.')
image = '%s.%s.%s' % (parts[1], parts[2], role)
except:
print('could not get image from %s' % df)
continue
print('image: %s expect base %s' % (image, image_base))
created, user, version, use_tag, base = InspectLocalReg.inspectLocal(
image, registry, True)
print('created: %s user: %s version: %s tag %s base %s' %
(created, user, version, use_tag, base))
if not base.startswith(image_base):
cmd = './relabel.sh %s %s %s %s %s' % (registry, framework_version,
image, image_base, base_id)
print("MISMATCH")
print(cmd)
os.system(cmd)
def checkDates(image, source_reg, dest_reg, no_copy, lab, logger):
dest_created, dest_user, dest_version, tag, base = InspectLocalReg.inspectLocal(image, logger, dest_reg, no_pull=True)
if dest_created is not None:
with_reg = '%s/%s' % (source_reg, image)
source_created, source_user, source_version, tag, base = InspectLocalReg.inspectLocal(image, logger, source_reg, no_pull=True)
if source_created != dest_created:
print('DIFFERENT: %s:%s source created/version %s/%s destination: %s/%s' % (lab, image, source_created,
source_version, dest_created, dest_version))
logger.debug('DIFFERENT: %s:%s source created/version %s/%s destination: %s/%s' % (lab, image, source_created,
source_version, dest_created, dest_version))
if not no_copy:
pull_push(image, source_reg, dest_reg)
else:
print('%s not in %s, would add it' % (image, dest_reg))
if not no_copy:
pull_push(image, source_reg, dest_reg)
def getDates(image, reg, lab, logger):
if reg == 'oabuoun':
with_reg = '%s/%s' % (reg, image)
created, user, version, tag = InspectRemoteReg.inspectRemote(with_reg, logger, no_pull=True)
else:
created, user, version, tag, base = InspectLocalReg.inspectLocal(image, logger, reg, no_pull=True)
if created is not None:
print('%s %s %s' % (lab, image, created))
def refreshLab(labdir, lab, role, remote_reg, local_reg, logger, no_copy):
''' force local to match remote '''
logger.debug('Refresh containers for lab %s' % lab)
docker_dir = os.path.join(labdir, lab, 'dockerfiles')
if not os.path.isdir(docker_dir):
logger.debug('No docker file for %s, bail' % lab)
return
df_list = [
f for f in os.listdir(docker_dir)
if os.path.isfile(os.path.join(docker_dir, f))
]
for df in df_list:
if df.endswith('.swp'):
continue
try:
parts = df.split('.')
image = '%s.%s.%s' % (parts[1], parts[2], role)
container = parts[2]
except:
print('could not get image from %s' % df)
continue
with_reg = '%s/%s' % (remote_reg, image)
remote_created, remote_user, remote_version, tag = InspectRemoteReg.inspectRemote(
with_reg, logger, no_pull=True)
logger.debug('%s %s' % (with_reg, remote_created))
if remote_created is not None:
local_created, local_user, local_version, tag, base = InspectLocalReg.inspectLocal(
image, logger, local_reg, no_pull=True)
logger.debug('%s %s' % (image, local_created))
if local_created != remote_created:
print(
'DIFFERENT: %s:%s local created/version %s/%s remote: %s/%s'
% (lab, container, local_created, local_version,
remote_created, remote_version))
logger.debug(
'DIFFERENT: %s:%s local created/version %s/%s remote: %s/%s'
% (lab, container, local_created, local_version,
remote_created, remote_version))
if not no_copy:
pull_push(image, remote_reg, local_reg)
else:
logger.debug('refreshLab, no diff for %s' % image)
else:
print('ERROR, no remote info for image %s' % image)
exit(1)
def doUpdateOrRefresh(local_registry, remote_registry, args, lgr):
''' either push local images to remote registry (an update, which is the default), or pull remote images into local registry (refresh). '''
if not args.quiet and not args.no_copy:
if not args.refresh:
msg = 'The will push images from the %s registry to the %s registry. Continue? (y/n)' % (
local_registry, remote_registry)
else:
msg = 'The will push images from the %s registry to the %s registry. Continue? (y/n)' % (
remote_registry, local_registry)
confirm = str(input(msg)).lower().strip()
if confirm != 'y':
print('aborting')
exit(1)
if not args.refresh and not args.no_copy:
os.system('docker login')
ldir = os.getenv('LABTAINER_DIR')
if ldir is None:
print('LABTAINER_DIR not defined.')
exit(1)
labdir = os.path.join(ldir, 'labs')
if args.lab is not None:
if not args.refresh:
updateLab(labdir, args.lab, 'student', remote_registry,
local_registry, lgr, args.no_copy)
else:
refreshLab(labdir, args.lab, 'student', remote_registry,
local_registry, lgr, args.no_copy)
else:
lgr.debug('Do all images')
grader = 'labtainer.grader'
local_created, local_user, local_version, tag, base = InspectLocalReg.inspectLocal(
grader, lgr, local_registry)
if local_created is not None:
with_reg = '%s/%s' % (remote_registry, grader)
remote_created, remote_user, remote_version, tag = InspectRemoteReg.inspectRemote(
with_reg, lgr, no_pull=True)
lgr.debug('%s local: %s remote: %s' %
(grader, local_created, remote_created))
if local_created != remote_created:
print('DIFFERENT: %s local created %s remote: %s' %
(grader, local_created, remote_created))
if not args.no_copy:
if not args.refresh:
pull_push(grader, local_registry, remote_registry)
else:
pull_push(grader, remote_registry, local_registry)
else:
print('No %s image on docker hub!' % grader)
lgr.debug('No %s image on docker hub!' % grader)
exit(1)
skip = []
with open('skip-labs') as fh:
for line in fh:
f = os.path.basename(line).strip()
print('will skip [%s]' % f)
skip.append(f)
#lab_list = os.listdir(labdir)
mycwd = os.getcwd()
os.chdir(labdir)
cmd = 'git ls-files ./ | cut -d/ -f1 | uniq'
child = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE)
output = child.communicate()
lab_list = output[0].decode('utf-8').strip().splitlines()
os.chdir(mycwd)
#lab_list = [x[0] for x in os.walk(labdir)]
for lab in sorted(lab_list):
if lab not in skip:
if not args.refresh:
updateLab(labdir, lab, 'student', remote_registry,
local_registry, lgr, args.no_copy)
else:
refreshLab(labdir, lab, 'student', remote_registry,
local_registry, lgr, args.no_copy)
if not args.no_copy:
if not args.refresh:
print(
'Comparing base images in %s to %s, and replacing content of %s if different'
% (local_registry, remote_registry, remote_registry))
else:
print(
'Comparing base images in %s to %s, and replacing content of %s if different'
% (local_registry, remote_registry, local_registry))
base_names = LabtainerBase.getBaseList()
for base in base_names:
with_registry = '%s/%s' % (remote_registry, base)
print(base)
remote_created, remote_user = RemoteBase.inspectRemote(
with_registry, lgr)
local_created, local_user = LocalBase.inspectLocal(
base, lgr, local_registry)
if remote_created != local_created:
print('Difference in %s, local: %s remote: %s' %
(base, local_created, remote_created))
if not args.no_copy:
if not args.refresh:
pull_push(base, local_registry, remote_registry)
else:
pull_push(base, remote_registry, local_registry)
headless_images = [
'labtainer.master.base', 'labtainer.master.headless'
]
for base in headless_images:
with_registry = '%s/%s' % (remote_registry, base)
print(base)
remote_created, remote_user = RemoteBase.inspectRemote(
with_registry, lgr)
local_created, local_user = LocalBase.inspectLocal(
base, lgr, local_registry)
if remote_created != local_created:
print('Difference in %s, local: %s remote: %s' %
(base, local_created, remote_created))
if not args.no_copy:
if not args.refresh:
pull_push(base, local_registry, remote_registry)
else:
pull_push(base, remote_registry, local_registry)
def main():
src_path = '../'
labtainer_config_file = os.path.join(src_path, 'config', 'labtainer.config')
logger = LabtainerLogging.LabtainerLogging("/tmp/labtainer-publish.log", 'publish', labtainer_config_file)
labutils.logger = logger
parser = argparse.ArgumentParser(description='Build the images labs and publish to a registry')
parser.add_argument('-l', '--lab', action='store', help='build and publish just this lab')
parser.add_argument('-s', '--start', action='store', help='all labs starting with this one')
parser.add_argument('-d', '--default_registry', action='store_true', default=False, help='build and publish with default registry -- instead of the typical test registry')
parser.add_argument('-f', '--force', action='store_true', default=False, help='force rebuild of all images')
parser.add_argument('-n', '--no_build', action='store_true', default=False, help='Do not rebuild, just report on what would be built')
parser.add_argument('-q', '--quiet', action='store_true', default=False, help='Do not prompt user for ok')
args = parser.parse_args()
if not args.default_registry:
if os.getenv('TEST_REGISTRY') is None:
#print('use putenv to set it')
os.putenv("TEST_REGISTRY", "TRUE")
''' why does putenv not set the value? '''
os.environ['TEST_REGISTRY'] = 'TRUE'
else:
#print('exists, set it true')
os.environ['TEST_REGISTRY'] = 'TRUE'
branch, test_registry = registry.getBranchRegistry()
print('Using test registry %s' % test_registry)
ok = InspectLocalReg.checkRegistryExists(test_registry, logger)
if not ok:
print('Default is to use a test registry, which does not seem to exist. Use -d option to force publishing directly to Docker Hub')
exit(1)
else:
if os.getenv('TEST_REGISTRY') is not None:
print('Request to use default registry, but TEST_REGISTRY is set. Unset that first.')
exit(1)
skip_labs = 'skip-labs'
skip = []
with open(skip_labs) as fh:
for line in fh:
f = os.path.basename(line).strip()
print('adding [%s]' % f)
skip.append(f)
labsdir = os.path.abspath(os.path.join(src_path, 'labs'))
labtainer_config = ParseLabtainerConfig.ParseLabtainerConfig(labtainer_config_file, logger)
default_registry = labtainer_config.default_registry
if args.lab is not None:
logger.debug('Doing just one lab %s labsdir %s' % (args.lab, labsdir))
DoLab(args.lab, labsdir, args.force, logger, False, args.default_registry, default_registry, no_build=args.no_build)
else:
# do them all. warn of incomplete git
mycwd = os.getcwd()
os.chdir(labsdir)
command = 'git status -s'
ps = subprocess.Popen(shlex.split(command), True, stdout=subprocess.PIPE,stderr=subprocess.PIPE)
grep_command = 'grep -E "^M|^D|^A"'
ps_grep = subprocess.Popen(shlex.split(grep_command), stdin=ps.stdout,stdout=subprocess.PIPE, stderr=subprocess.PIPE)
ps.stdout.close()
output = ps_grep.communicate()
if len(output[0]) > 0:
for line in output[0].decode('utf-8').splitlines():
print(line.strip())
if not args.quiet:
dumb = input("any key to continue")
# Do login here and now so we don't wait for lab to build before prompt
if args.default_registry:
os.system('docker login')
#cmd = 'svn ls https://tor.ern.nps.edu/svn/proj/labtainer/trunk/labs'
cmd = 'git ls-files ./ | cut -d/ -f1 | uniq'
child = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE)
output = child.communicate()
lab_list = output[0].decode('utf-8').strip().splitlines()
for lab in sorted(lab_list):
#lab = lab[:len(lab)-1]
lab = lab.strip()
if args.start is not None and lab < args.start:
continue
if lab not in skip:
print('Lab: %s' % lab)
lab_dir = os.path.join(labsdir, lab)
os.chdir(lab_dir)
cmd = 'git checkout ./'
os.system(cmd)
os.chdir(mycwd)
DoLab(lab, labsdir, args.force, logger, False, args.default_registry, default_registry, no_build=args.no_build)