help='include metasploitable and kali images')
args = parser.parse_args()
lab_config_file = os.path.join(f'{labtainer_dir}/config', 'labtainer.config')
labutils.logger = LabtainerLogging.LabtainerLogging(
"pull.log", 'pull-all', f'{labtainer_dir}/config/labtainer.config')
logger = labutils.logger
labtainer_config = ParseLabtainerConfig.ParseLabtainerConfig(
lab_config_file, logger)
test_registry = False
if not args.test_registry:
env = os.getenv('TEST_REGISTRY')
if env is not None and env.lower() == 'true':
test_registry = True
if args.test_registry or test_registry:
branch, use_registry = registry.getBranchRegistry()
else:
use_registry = labtainer_config.default_registry
print('registry is: %s' % use_registry)
config_list = [
'base', 'network', 'firefox', 'wireshark', 'java', 'centos', 'lamp',
'netmon', 'tap'
]
if args.metasploit:
config_list.append('metasploitable')
config_list.append('kali')
for config in config_list:
image_name = '%s/labtainer.%s' % (use_registry, config)
local_created, local_user, local_version = labutils.inspectImage(
image_name)
if args.force or local_created is None:
def DoRebuildLab(lab_path,
force_build=False,
just_container=None,
start_config=None,
labtainer_config=None,
run_container=None,
servers=None,
clone_count=None,
no_pull=False,
no_build=False,
use_cache=True,
local_build=False):
retval = []
labname = os.path.basename(lab_path)
labutils.isValidLab(lab_path)
if start_config is None:
labtainer_config, start_config = labutils.GetBothConfigs(
lab_path, labutils.logger, servers, clone_count)
host_home_xfer = labtainer_config.host_home_xfer
build_student = 'bin/buildImage.sh'
LABS_DIR = os.path.abspath('../../labs')
didfix = False
''' hackey assumption about running from labtainers-student or labtainers-instructor '''
container_bin = './lab_bin'
for name, container in start_config.containers.items():
labutils.logger.debug('this container name %s just_container %s' %
(name, just_container))
if just_container is not None and just_container != name:
continue
elif just_container == name:
force_build = True
print('Force build of %s' % just_container)
mycontainer_name = container.full_name
mycontainer_image_name = container.image_name
if container.registry == labtainer_config.test_registry:
branch, container_registry = registry.getBranchRegistry()
base_registry = container_registry
else:
container_registry = container.registry
base_registry = container.base_registry
clone_names = labutils.GetContainerCloneNames(container)
for clone_full in clone_names:
cmd = 'docker rm %s' % clone_full
ps = subprocess.Popen(shlex.split(cmd),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
output = ps.communicate()
#labutils.logger.debug("Command was (%s)" % cmd)
if len(output[1]) > 0:
if 'running container' in output[1].decode('utf-8'):
labutils.logger.debug("Error from command %s was '%s'" %
(cmd, output[1].decode('utf-8')))
exit(1)
else:
labutils.logger.debug(
"non-fatal Error from command %s was '%s'" %
(cmd, output[1].decode('utf-8')))
if container.from_image is not None:
labutils.logger.debug('skip image taken from %s' %
container.from_image)
continue
force_this_build = force_build
if container.no_pull == 'YES':
no_pull = True
labutils.logger.debug('force_this_build: %r no_pull %r' %
(force_this_build, no_pull))
image_info = labutils.imageInfo(mycontainer_image_name,
container_registry,
base_registry,
labtainer_config,
is_rebuild=True,
no_pull=no_pull,
local_build=local_build)
if not force_this_build and image_info is None:
if not local_build:
labutils.logger.debug(
'Image %s exists nowhere, so force the build' %
mycontainer_image_name)
print('Image %s exists nowhere, so force the build' %
mycontainer_image_name)
else:
labutils.logger.debug(
'Image %s does not exist locally. Local build requested, so force the build'
% mycontainer_image_name)
print(
'Image %s does not exist locally. Local build requested, so force the build'
% mycontainer_image_name)
force_this_build = True
container_dir = os.path.join(lab_path, name)
try:
os.mkdir(os.path.join(container_dir, 'home_tar'))
except:
pass
try:
os.mkdir(os.path.join(container_dir, 'sys_tar'))
except:
pass
removeStrays(container_dir, name, labname)
''' make sure big files have been copied before checking tars '''
BigFiles.BigFiles(lab_path)
BigExternal.BigExternal(lab_path, labutils.logger)
''' create sys_tar and home_tar before checking build dependencies '''
CheckTars.CheckTars(container_dir, name, labutils.logger)
if force_this_build or CheckBuild(
lab_path, mycontainer_image_name, image_info, mycontainer_name,
name, True, container_bin, start_config, base_registry,
container.user, local_build):
if no_build:
labutils.logger.debug("Would (but won't) rebuild %s" %
(mycontainer_name))
print("Would (but won't) rebuild %s" % (mycontainer_name))
return retval
labutils.logger.debug(
"Will rebuild %s, force_this_build: %s apt_source %s" %
(mycontainer_name, force_this_build,
labtainer_config.apt_source))
#Check if the container's Dockerfile exists
dfile = '%s/%s/dockerfiles/Dockerfile.%s.%s.student' % (
LABS_DIR, labname, labname, name)
if not os.path.isfile(dfile):
labutils.logger.error(
"Dockerfile.%s.%s.student is missing from labs/%s/dockerfiles."
% (labname, name, labname))
exit(1)
if local_build:
ts, thebase = BaseImageTime(dfile, base_registry)
if ts == 0:
labutils.logger.debug(
'No base found %s, look for local base' % thebase)
ts, thebase = BaseImageTime(dfile, None)
if ts == 0:
labutils.logger.error(
'No local image for %s and local build requested. Try "docker pull %s/%s"'
% (thebase, base_registry, the_base))
exit(1)
else:
labutils.logger.debug(
'Using local version of base %s' % thebase)
base_registry = 'LOCAL'
else:
labutils.logger.debug('got ts, base %s' % thebase)
retval.append(
RegistryInfo(name, container.image_name, container_registry,
base_registry))
if os.path.isfile(build_student):
cmd = '%s %s %s %s %s %s %s %s %s %s %s %s' % (
build_student, labname, name, container.user,
container.password, True, LABS_DIR,
labtainer_config.apt_source, base_registry,
labutils.framework_version, str(local_build),
str(use_cache))
else:
labutils.logger.error("no image rebuild script\n")
exit(1)
labutils.logger.debug('cmd is %s' % cmd)
ps = subprocess.Popen(shlex.split(cmd),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
output = ps.communicate()
labutils.logger.debug('build_image back from communicate')
#fatal_error = CheckBuildError(ps.stdout, labname, name)
fatal_error = CheckBuildError(output[0].decode('utf-8'), labname,
name)
if not fatal_error:
labutils.logger.debug('not fatal, do stderr')
fatal_error = CheckBuildError(output[1].decode('utf-8'),
labname, name)
#fatal_error = CheckBuildError(ps.stderr, labname, name)
else:
labutils.logger.debug('fatal, do stderr')
CheckBuildError(output[1].decode('utf-8'), labname, name)
#CheckBuildError(ps.stderr, labname, name)
labutils.logger.debug('done checkerror fatal %r' % fatal_error)
if ps.returncode != 0:
labutils.logger.error(
'Problem building %s, check log at $LABTAINER_DIR/logs/docker_build.log'
% labname)
exit(1)
if fatal_error:
exit(1)
return retval
def main():
src_path = '../'
labtainer_config_file = os.path.join(src_path, 'config', 'labtainer.config')
logger = LabtainerLogging.LabtainerLogging("/tmp/labtainer-publish.log", 'publish', labtainer_config_file)
labutils.logger = logger
parser = argparse.ArgumentParser(description='Build the images labs and publish to a registry')
parser.add_argument('-l', '--lab', action='store', help='build and publish just this lab')
parser.add_argument('-s', '--start', action='store', help='all labs starting with this one')
parser.add_argument('-d', '--default_registry', action='store_true', default=False, help='build and publish with default registry -- instead of the typical test registry')
parser.add_argument('-f', '--force', action='store_true', default=False, help='force rebuild of all images')
parser.add_argument('-n', '--no_build', action='store_true', default=False, help='Do not rebuild, just report on what would be built')
parser.add_argument('-q', '--quiet', action='store_true', default=False, help='Do not prompt user for ok')
args = parser.parse_args()
if not args.default_registry:
if os.getenv('TEST_REGISTRY') is None:
#print('use putenv to set it')
os.putenv("TEST_REGISTRY", "TRUE")
''' why does putenv not set the value? '''
os.environ['TEST_REGISTRY'] = 'TRUE'
else:
#print('exists, set it true')
os.environ['TEST_REGISTRY'] = 'TRUE'
branch, test_registry = registry.getBranchRegistry()
print('Using test registry %s' % test_registry)
ok = InspectLocalReg.checkRegistryExists(test_registry, logger)
if not ok:
print('Default is to use a test registry, which does not seem to exist. Use -d option to force publishing directly to Docker Hub')
exit(1)
else:
if os.getenv('TEST_REGISTRY') is not None:
print('Request to use default registry, but TEST_REGISTRY is set. Unset that first.')
exit(1)
skip_labs = 'skip-labs'
skip = []
with open(skip_labs) as fh:
for line in fh:
f = os.path.basename(line).strip()
print('adding [%s]' % f)
skip.append(f)
labsdir = os.path.abspath(os.path.join(src_path, 'labs'))
labtainer_config = ParseLabtainerConfig.ParseLabtainerConfig(labtainer_config_file, logger)
default_registry = labtainer_config.default_registry
if args.lab is not None:
logger.debug('Doing just one lab %s labsdir %s' % (args.lab, labsdir))
DoLab(args.lab, labsdir, args.force, logger, False, args.default_registry, default_registry, no_build=args.no_build)
else:
# do them all. warn of incomplete git
mycwd = os.getcwd()
os.chdir(labsdir)
command = 'git status -s'
ps = subprocess.Popen(shlex.split(command), True, stdout=subprocess.PIPE,stderr=subprocess.PIPE)
grep_command = 'grep -E "^M|^D|^A"'
ps_grep = subprocess.Popen(shlex.split(grep_command), stdin=ps.stdout,stdout=subprocess.PIPE, stderr=subprocess.PIPE)
ps.stdout.close()
output = ps_grep.communicate()
if len(output[0]) > 0:
for line in output[0].decode('utf-8').splitlines():
print(line.strip())
if not args.quiet:
dumb = input("any key to continue")
# Do login here and now so we don't wait for lab to build before prompt
if args.default_registry:
os.system('docker login')
#cmd = 'svn ls https://tor.ern.nps.edu/svn/proj/labtainer/trunk/labs'
cmd = 'git ls-files ./ | cut -d/ -f1 | uniq'
child = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE)
output = child.communicate()
lab_list = output[0].decode('utf-8').strip().splitlines()
for lab in sorted(lab_list):
#lab = lab[:len(lab)-1]
lab = lab.strip()
if args.start is not None and lab < args.start:
continue
if lab not in skip:
print('Lab: %s' % lab)
lab_dir = os.path.join(labsdir, lab)
os.chdir(lab_dir)
cmd = 'git checkout ./'
os.system(cmd)
os.chdir(mycwd)
DoLab(lab, labsdir, args.force, logger, False, args.default_registry, default_registry, no_build=args.no_build)
logger = LabtainerLogging.LabtainerLogging("labtainer-publish.log", 'publish', labtainer_config_file)
labtainer_config = ParseLabtainerConfig.ParseLabtainerConfig(labtainer_config_file, logger)
labutils.logger = logger
labdir = os.path.join(os.getenv('LABTAINER_DIR'), 'labs')
mycwd = os.getcwd()
os.chdir(labdir)
cmd = 'git ls-files ./ | cut -d/ -f1 | uniq'
child = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE)
output = child.communicate()
lab_list = output[0].decode('utf-8').strip().splitlines()
os.chdir(mycwd)
if args.dockerhub:
registry = labtainer_config.default_registry
print('Get registry dates from Docker Hub')
elif not args.premaster:
branch, registry = registry.getBranchRegistry()
print('Get registry dates for branch %s from %s' % (branch, registry))
else:
registry = labtainer_config.test_registry
print('Get registry dates for PREMASTER from %s' % (registry))
if args.lab == None:
skip_labs = 'skip-labs'
skip = []
with open(skip_labs) as fh:
for line in fh:
f = os.path.basename(line).strip()
skip.append(f)
for lab in sorted(lab_list):
lab = lab.strip()
if lab not in skip:
doLab(labdir, lab, 'student', registry, logger)
cmd = 'git ls-files ./ | cut -d/ -f1 | uniq'
child = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE)
output = child.communicate()
lab_list = output[0].decode('utf-8').strip().splitlines()
os.chdir(mycwd)
for lab in sorted(lab_list):
lab = lab.strip()
if lab not in skip:
doLab(labdir, lab, 'student', source_registry, dest_registry, lgr, no_copy)
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='Compare a source registry with a destination registry, and update the destination so they match')
parser.add_argument('-n', '--no_copy', action='store_true', default=False, help='Do not modify registry, just report differences')
parser.add_argument('-l', '--lab', action='store', help='only check this lab')
parser.add_argument('-q', '--quiet', action='store_true', default=False, help='Do not prompt for confirmation.')
args = parser.parse_args()
config_file = os.path.join(os.getenv('LABTAINER_DIR'), 'config', 'labtainer.config')
labtainer_config = ParseLabtainerConfig.ParseLabtainerConfig(config_file, None)
lgr = LabtainerLogging.LabtainerLogging("refresh_branch.log", 'none', config_file)
''' source is the premaster mirror '''
source_registry = labtainer_config.test_registry
branch, dest_registry = registry.getBranchRegistry()
if dest_registry is None:
print('No registry found for branch %s' % branch)
exit(1)
updateRegistry(source_registry, dest_registry, lgr, args.lab, args.no_copy, args.quiet)