def build_image(client, path, challenge_name, step_name, service_name, filename, no_cache: bool, registry_info: RegistryInfo) -> BuildResult: d = datetime.datetime.now() username = get_dockerhub_username() from duckietown_challenges.utils import tag_from_date if username.lower() != username: msg = f'Are you sure that the DockerHub username is not lowercase? You gave "{username}".' dtslogger.warning(msg) username = username.lower() br = BuildResult( repository=('%s-%s-%s' % (challenge_name, step_name, service_name)).lower(), organization=username, registry=registry_info.registry, tag=tag_from_date(d), digest=None) complete = get_complete_tag(br) cmd = ['docker', 'build', '--pull', '-t', complete, '-f', filename] if no_cache: cmd.append('--no-cache') cmd.append(path) dtslogger.debug('$ %s' % " ".join(cmd)) subprocess.check_call(cmd) image = client.images.get(complete) repo_digests = image.attrs.get('RepoDigests', []) if repo_digests: msg = 'Already found repo digest: %s' % repo_digests dtslogger.info(msg) else: dtslogger.info('Image not present on registry. Need to push.') cmd = ['docker', 'push', complete] dtslogger.debug('$ %s' % " ".join(cmd)) subprocess.check_call(cmd) image = client.images.get(complete) dtslogger.info('image id: %s' % image.id) dtslogger.info('complete: %s' % get_complete_tag(br)) repo_digests = image.attrs.get('RepoDigests', []) if not repo_digests: msg = 'Could not find any repo digests (push not succeeded?)' raise Exception(msg) dtslogger.info('RepoDigests: %s' % repo_digests) _, digest = repo_digests[0].split('@') # br.digest = image.id br.digest = digest br = parse_complete_tag(get_complete_tag(br)) return br
def get_environment_clean(): env = {} # add other environment env.update(os.environ) # remove DOCKER_HOST if present if 'DOCKER_HOST' in env: r = env['DOCKER_HOST'] msg = 'I will IGNORE the DOCKER_HOST variable that is currently set to %r' % r dtslogger.warning(msg) env.pop('DOCKER_HOST') return env
def check_user_in_docker_group(): # first, let's see if there exists a group "docker" group_names = [g.gr_name for g in grp.getgrall()] G = 'docker' if G not in group_names: msg = 'No group %s defined.' % G dtslogger.warning(msg) else: group_id = grp.getgrnam(G).gr_gid my_groups = os.getgroups() if group_id not in my_groups: msg = 'My groups are %s and "%s" group is %s ' % (my_groups, G, group_id) msg += '\n\nNote that when you add a user to a group, you need to login in and out.' dtslogger.debug(msg)
def ipfs_available(): if os.path.exists('/ipfs'): fn = '/ipfs/QmYwAPJzv5CZsnA625s3Xf2nemtYgPpHdWEz79ojWnPbdG/readme' try: d = open(fn).read() except: msg = f'Could not open an IPFS file: {traceback.format_exc()}' dtslogger.warning(msg) return False if 'Hello' in d: return True else: dtslogger.warning(d) return False else: return False
def validate_user_data(user_data_yaml): if 'VARIABLE' in user_data_yaml: msg = 'Invalid user_data_yaml:\n' + user_data_yaml msg += '\n\nThe above contains VARIABLE' raise Exception(msg) try: import requests except ImportError: msg = 'Skipping validation because "requests" not installed.' dtslogger.warning(msg) else: url = 'https://validate.core-os.net/validate' r = requests.put(url, data=user_data_yaml) info = json.loads(r.content) result = info['result'] nerrors = 0 for x in result: kind = x['kind'] line = x['line'] message = x['message'] m = 'Invalid at line %s: %s' % (line, message) m += '| %s' % user_data_yaml.split('\n')[line - 1] if kind == 'error': dtslogger.error(m) nerrors += 1 else: ignore = [ 'bootcmd', 'package_upgrade', 'runcmd', 'ssh_pwauth', 'sudo', 'chpasswd', 'lock_passwd', 'plain_text_passwd' ] show = False for i in ignore: if 'unrecognized key "%s"' % i in m: break else: show = True if show: dtslogger.warning(m) if nerrors: msg = 'There are %d errors: exiting' % nerrors raise Exception(msg)
def check_docker_environment(): username = getpass.getuser() from . import dtslogger dtslogger.debug('Checking docker environment for user %s' % username) check_executable_exists('docker') check_user_in_docker_group() # # if on_linux(): # # if username != 'root': # check_user_in_docker_group() # # print('checked groups') # else: # dtslogger.debug('skipping env check because not on Linux') try: import docker except Exception as e: msg = 'Could not import package docker:\n%s' % e msg += '\n\nYou need to install the package' raise InvalidEnvironment(msg) if 'DOCKER_HOST' in os.environ: msg = 'Note that the variable DOCKER_HOST is set to "%s"' % os.environ[ 'DOCKER_HOST'] dtslogger.warning(msg) try: client = docker.from_env() containers = client.containers.list(filters=dict(status='running')) # dtslogger.debug(json.dumps(client.info(), indent=4)) except Exception as e: msg = 'I cannot communicate with Docker:\n%s' % e msg += '\n\nMake sure the docker service is running.' raise InvalidEnvironment(msg) return client
def command(shell, args): # configure arguments parser = argparse.ArgumentParser() parser.add_argument('-C', '--workdir', default=None, help="Directory containing the project to push") parser.add_argument('-a', '--arch', default=DEFAULT_ARCH, help="Target architecture for the image to push") parser.add_argument( '-H', '--machine', default=DEFAULT_MACHINE, help="Docker socket or hostname from where to push the image") parser.add_argument( '-f', '--force', default=False, action='store_true', help="Whether to force the push when the git index is not clean") parsed, _ = parser.parse_known_args(args=args) # --- code_dir = parsed.workdir if parsed.workdir else os.getcwd() dtslogger.info('Project workspace: {}'.format(code_dir)) # show info about project shell.include.devel.info.command(shell, args) # get info about current repo repo_info = shell.include.devel.info.get_repo_info(code_dir) repo = repo_info['REPOSITORY'] branch = repo_info['BRANCH'] nmodified = repo_info['INDEX_NUM_MODIFIED'] nadded = repo_info['INDEX_NUM_ADDED'] # check if the index is clean if nmodified + nadded > 0: dtslogger.warning( 'Your index is not clean (some files are not committed).') dtslogger.warning( 'If you know what you are doing, use --force to force the execution of the command.' ) if not parsed.force: exit(1) dtslogger.warning('Forced!') # create defaults default_tag = "duckietown/%s:%s" % (repo, branch) tag = "duckietown/%s:%s-%s" % (repo, branch, parsed.arch) tags = [tag] + ([default_tag] if parsed.arch == DEFAULT_ARCH else []) for t in tags: # push image dtslogger.info("Pushing image {}...".format(t)) _run_cmd(['docker', '-H=%s' % parsed.machine, 'push', t])
def command(shell, args): check_docker_environment() home = os.path.expanduser('~') prog = 'dts challenges evaluator' parser = argparse.ArgumentParser(prog=prog, usage=usage) group = parser.add_argument_group('Basic') group.add_argument('--submission', type=int, default=None, help='Run a specific submission.') group.add_argument( '--reset', dest='reset', action='store_true', default=False, help='(needs --submission) Re-evaluate the specific submission.') group = parser.add_argument_group('Advanced') group.add_argument('--no-watchtower', dest='no_watchtower', action='store_true', default=False, help="Disable starting of watchtower") group.add_argument('--no-pull', dest='no_pull', action='store_true', default=False, help="Disable pulling of container") group.add_argument('--image', help="Evaluator image to run", default='duckietown/dt-challenges-evaluator:v3') group.add_argument('--name', default=None, help='Name for this evaluator') group.add_argument("--features", default=None, help="Pretend to be what you are not.") dtslogger.debug('args: %s' % args) parsed = parser.parse_args(args) machine_id = socket.gethostname() if parsed.name is None: container_name = '%s-%s' % (socket.gethostname(), os.getpid()) else: container_name = parsed.name import docker client = docker.from_env() command = ['dt-challenges-evaluator'] if parsed.submission: command += ['--submission', str(parsed.submission)] if parsed.reset: command += ['--reset'] else: command += ['--continuous'] command += ['--name', container_name] command += ['--machine-id', machine_id] if parsed.features: dtslogger.debug('Passing features %r' % parsed.features) command += ['--features', parsed.features] volumes = { '/var/run/docker.sock': { 'bind': '/var/run/docker.sock', 'mode': 'rw' }, os.path.join(home, '.dt-shell'): { 'bind': '/root/.dt-shell', 'mode': 'ro' }, '/tmp': { 'bind': '/tmp', 'mode': 'rw' } } env = {} UID = os.getuid() USERNAME = getpass.getuser() extra_environment = dict(username=USERNAME, uid=UID) env.update(extra_environment) if not parsed.no_watchtower: ensure_watchtower_active(client) url = get_duckietown_server_url() dtslogger.info('The server URL is: %s' % url) if 'localhost' in url: h = socket.gethostname() replacement = h + '.local' dtslogger.warning( 'There is "localhost" inside, so I will try to change it to %r' % replacement) dtslogger.warning( 'This is because Docker cannot see the host as "localhost".') url = url.replace("localhost", replacement) dtslogger.warning('The new url is: %s' % url) dtslogger.warning( 'This will be passed to the evaluator in the Docker container.' ) env['DTSERVER'] = url image = parsed.image name, tag = image.split(':') if not parsed.no_pull: dtslogger.info('Updating container %s' % image) client.images.pull(name, tag) try: container = client.containers.get(container_name) except: pass else: dtslogger.error('stopping previous %s' % container_name) container.stop() dtslogger.error('removing') container.remove() dtslogger.info('Starting container %s with %s' % (container_name, image)) dtslogger.info('Container command: %s' % " ".join(command)) client.containers.run(image, command=command, volumes=volumes, environment=env, network_mode='host', detach=True, name=container_name, tty=True) while True: try: container = client.containers.get(container_name) except Exception as e: msg = 'Cannot get container %s: %s' % (container_name, e) dtslogger.error(msg) dtslogger.info('Will wait.') time.sleep(5) continue dtslogger.info('status: %s' % container.status) if container.status == 'exited': msg = 'The container exited.' logs = '' for c in container.logs(stdout=True, stderr=True, stream=True): logs += c dtslogger.error(msg) tf = 'evaluator.log' with open(tf, 'w') as f: f.write(logs) msg = 'Logs saved at %s' % (tf) dtslogger.info(msg) break try: for c in container.logs(stdout=True, stderr=True, stream=True, follow=True): sys.stdout.write(c) time.sleep(3) except Exception as e: dtslogger.error(e) dtslogger.info('Will try to re-attach to container.') time.sleep(3) except KeyboardInterrupt: dtslogger.info('Received CTRL-C. Stopping container...') container.stop() dtslogger.info('Removing container') container.remove() dtslogger.info('Container removed.') break
def command(shell, args): parser = argparse.ArgumentParser() parser.add_argument('--hostname', default='duckiebot') parser.add_argument('--linux-username', default='duckie') parser.add_argument('--linux-password', default='quackquack') parser.add_argument('--wifi-ssid', dest="wifissid", default='duckietown') parser.add_argument('--wifi-password', dest="wifipass", default='quackquack') parsed = parser.parse_args(args=args) check_docker_environment() if not is_valid_hostname(parsed.hostname): msg = 'This is not a valid hostname: %r.' % parsed.hostname raise Exception(msg) if '-' in parsed.hostname: msg = 'Cannot use the hostname %r. It cannot contain "-" because of a ROS limitation. ' % parsed.hostname raise Exception(msg) if len(parsed.hostname) < 3: msg = 'This hostname is too short. Choose something more descriptive.' raise Exception(msg) MIN_AVAILABLE_GB = 0.0 try: import psutil except ImportError: msg = 'Skipping disk check because psutil not installed.' dtslogger.info(msg) else: disk = psutil.disk_usage(os.getcwd()) disk_available_gb = disk.free / (1024 * 1024 * 1024.0) if disk_available_gb < MIN_AVAILABLE_GB: msg = 'This procedure requires that you have at least %f GB of memory.' % MIN_AVAILABLE_GB msg += '\nYou only have %f GB available.' % disk_available_gb raise Exception(msg) p = platform.system().lower() if 'darwin' in p: msg = 'This procedure cannot be run on Mac. You need an Ubuntu machine.' raise Exception(msg) this = dirname(realpath(__file__)) script_files = realpath(join(this, '..', 'init_sd_card.scripts')) script_file = join(script_files, 'init_sd_card.sh') if not os.path.exists(script_file): msg = 'Could not find script %s' % script_file raise Exception(msg) ssh_key_pri = join(script_files, 'DT18_key_00') ssh_key_pub = join(script_files, 'DT18_key_00.pub') for f in [ssh_key_pri, ssh_key_pub]: if not os.path.exists(f): msg = 'Could not find file %s' % f raise Exception(msg) script_cmd = '/bin/bash %s' % script_file token = shell.get_dt1_token() env = dict() env['DUCKIE_TOKEN'] = token env['IDENTITY_FILE'] = ssh_key_pub env['WIFISSID'] = parsed.wifissid env['WIFIPASS'] = parsed.wifipass env['HOST_NAME'] = parsed.hostname env['DTS_USERNAME'] = parsed.linux_username env['PASSWORD'] = parsed.linux_password # add other environment env.update(os.environ) if 'DOCKER_HOST' in env: r = env['DOCKER_HOST'] msg = 'I will IGNORE the DOCKER_HOST variable that is currently set to %r' % r dtslogger.warning(msg) env.pop('DOCKER_HOST') ssh_dir = os.path.expanduser('~/.ssh') if not os.path.exists(ssh_dir): os.makedirs(ssh_dir) ssh_key_pri_copied = os.path.join(ssh_dir, 'DT18_key_00') ssh_key_pub_copied = os.path.join(ssh_dir, 'DT18_key_00.pub') if not os.path.exists(ssh_key_pri_copied): shutil.copy(ssh_key_pri, ssh_key_pri_copied) if not os.path.exists(ssh_key_pub_copied): shutil.copy(ssh_key_pub, ssh_key_pub_copied) ssh_config = os.path.join(ssh_dir, 'config') if not os.path.exists(ssh_config): msg = ('Could not find ssh config file %s' % ssh_config) dtslogger.info(msg) current = "" else: current = open(ssh_config).read() bit0 = """ # --- init_sd_card generated --- # Use the key for all hosts IdentityFile $IDENTITY Host $HOSTNAME User $DTS_USERNAME Hostname $HOSTNAME.local IdentityFile $IDENTITY StrictHostKeyChecking no # ------------------------------ """ subs = dict(HOSTNAME=parsed.hostname, IDENTITY=ssh_key_pri_copied, DTS_USERNAME=parsed.linux_username) bit = Template(bit0).substitute(**subs) if not bit in current: dtslogger.info('Updating ~/.ssh/config with: ' + bit) with open(ssh_config, 'a') as f: f.write(bit) else: dtslogger.info('Configuration already found in ~/.ssh/config') ret = subprocess.call(script_cmd, shell=True, env=env, stdin=sys.stdin, stderr=sys.stderr, stdout=sys.stdout) if ret == 0: dtslogger.info('Done!') else: msg = ( 'An error occurred while initializing the SD card, please check and try again (%s).' % ret) raise Exception(msg)
def command(shell, args): check_docker_environment() home = os.path.expanduser('~') prog = 'dts challenges evaluator' parser = argparse.ArgumentParser(prog=prog, usage=usage) group = parser.add_argument_group('Basic') group.add_argument('--submission', type=int, default=None, help='Run a specific submission.') group.add_argument('--reset', dest='reset', action='store_true', default=False, help='(needs --submission) Re-evaluate the specific submission.') group = parser.add_argument_group('Advanced') group.add_argument('--no-watchtower', dest='no_watchtower', action='store_true', default=False, help="Disable starting of watchtower") group.add_argument('--no-pull', dest='no_pull', action='store_true', default=False, help="Disable pulling of containers") group.add_argument('--no-upload', dest='no_upload', action='store_true', default=False, help="Disable upload of artifacts") group.add_argument('--no-delete', dest='no_delete', action='store_true', default=False, help="Does not erase temporary files in /tmp/duckietown") group.add_argument('--image', help="Evaluator image to run", default='duckietown/dt-challenges-evaluator:v4') group.add_argument('--name', default=None, help='Name for this evaluator') group.add_argument("--features", default=None, help="Pretend to be what you are not.") group.add_argument("--ipfs", action='store_true', default=False, help="Run with IPFS available") group.add_argument("--one", action='store_true', default=False, help="Only run 1 submission") # dtslogger.debug('args: %s' % args) parsed = parser.parse_args(args) machine_id = socket.gethostname() if parsed.name is None: container_name = '%s-%s' % (socket.gethostname(), os.getpid()) else: container_name = parsed.name client = check_docker_environment() command = ['dt-challenges-evaluator'] if parsed.submission: command += ['--submission', str(parsed.submission)] if parsed.reset: command += ['--reset'] else: if not parsed.one: command += ['--continuous'] command += ['--name', container_name] command += ['--machine-id', machine_id] if parsed.no_upload: command += ['--no-upload'] if parsed.no_pull: command += ['--no-pull'] if parsed.no_delete: command += ['--no-delete'] if parsed.one: command += ['--one'] if parsed.features: dtslogger.debug('Passing features %r' % parsed.features) command += ['--features', parsed.features] mounts = [] volumes = { '/var/run/docker.sock': {'bind': '/var/run/docker.sock', 'mode': 'rw'}, os.path.join(home, '.dt-shell'): {'bind': '/root/.dt-shell', 'mode': 'ro'}, '/tmp': {'bind': '/tmp', 'mode': 'rw'} } if parsed.ipfs: if not ipfs_available(): msg = 'IPFS not available/mounted correctly.' raise UserError(msg) command += ['--ipfs'] # volumes['/ipfs'] = {'bind': '/ipfs', 'mode': 'ro'} from docker.types import Mount mount = Mount(type='bind', source='/ipfs', target='/ipfs', read_only=True) mounts.append(mount) env = {} UID = os.getuid() USERNAME = getpass.getuser() extra_environment = dict(username=USERNAME, uid=UID) env.update(extra_environment) if not parsed.no_watchtower: ensure_watchtower_active(client) from duckietown_challenges.rest import get_duckietown_server_url url = get_duckietown_server_url() dtslogger.info('The server URL is: %s' % url) if 'localhost' in url: h = socket.gethostname() replacement = h + '.local' dtslogger.warning('There is "localhost" inside, so I will try to change it to %r' % replacement) dtslogger.warning('This is because Docker cannot see the host as "localhost".') url = url.replace("localhost", replacement) dtslogger.warning('The new url is: %s' % url) dtslogger.warning('This will be passed to the evaluator in the Docker container.') env['DTSERVER'] = url image = parsed.image dtslogger.info('Using evaluator image %s' % image) name, tag = image.split(':') if not parsed.no_pull: dtslogger.info('Updating container %s' % image) client.images.pull(name, tag) # noinspection PyBroadException try: container = client.containers.get(container_name) except: pass else: dtslogger.error('stopping previous %s' % container_name) container.stop() dtslogger.error('removing') container.remove() dtslogger.info('Starting container %s with %s' % (container_name, image)) dtslogger.info('Container command: %s' % " ".join(command)) # add all the groups import grp group_add = [g.gr_gid for g in grp.getgrall() if USERNAME in g.gr_mem] client.containers.run(image, group_add=group_add, command=command, volumes=volumes, environment=env, mounts=mounts, network_mode='host', detach=True, name=container_name, tty=True) last_log_timestamp = None while True: try: container = client.containers.get(container_name) except Exception as e: msg = 'Cannot get container %s: %s' % (container_name, e) dtslogger.error(msg) dtslogger.info('Will wait.') time.sleep(5) continue dtslogger.info('status: %s' % container.status) if container.status == 'exited': logs = '' for c in container.logs(stdout=True, stderr=True, stream=True, since=last_log_timestamp): logs += c.decode('utf-8') last_log_timestamp = datetime.datetime.now() tf = 'evaluator.log' with open(tf, 'w') as f: f.write(logs) msg = 'The container exited.' msg += '\nLogs saved at %s' % tf dtslogger.info(msg) break try: if last_log_timestamp is not None: print('since: %s' % last_log_timestamp.isoformat()) for c0 in container.logs(stdout=True, stderr=True, stream=True, # follow=True, since=last_log_timestamp, tail=0): c: bytes = c0 try: s = c.decode('utf-8') except: s = c.decode('utf-8', errors='replace') sys.stdout.write(s) last_log_timestamp = datetime.datetime.now() time.sleep(3) except KeyboardInterrupt: dtslogger.info('Received CTRL-C. Stopping container...') container.stop() dtslogger.info('Removing container') container.remove() dtslogger.info('Container removed.') break except BaseException: s = traceback.format_exc() if 'Read timed out' in s: dtslogger.debug('(reattaching)') else: dtslogger.error(s) dtslogger.info('Will try to re-attach to container.') time.sleep(3)
def command(shell, args): # configure arguments parser = argparse.ArgumentParser() parser.add_argument('-C', '--workdir', default=None, help="Directory containing the project to build") parser.add_argument('-a', '--arch', default=DEFAULT_ARCH, choices=set(CANONICAL_ARCH.values()), help="Target architecture for the image to build") parser.add_argument( '-H', '--machine', default=DEFAULT_MACHINE, help="Docker socket or hostname where to build the image") parser.add_argument( '--pull', default=False, action='store_true', help="Whether to pull the latest base image used by the Dockerfile" ) parser.add_argument('--no-cache', default=False, action='store_true', help="Whether to use the Docker cache") parser.add_argument( '--no-multiarch', default=False, action='store_true', help="Whether to disable multiarch support (based on bin_fmt)") parser.add_argument( '-f', '--force', default=False, action='store_true', help="Whether to force the build when the git index is not clean") parser.add_argument('--push', default=False, action='store_true', help="Whether to push the resulting image") parser.add_argument( '--rm', default=False, action='store_true', help= "Whether to remove the images once the build succeded (after pushing)" ) parser.add_argument( '--loop', default=False, action='store_true', help= "(Experimental) Whether to reuse the same base image to speed up the build process" ) parser.add_argument( '--ignore-watchtower', default=False, action='store_true', help="Whether to ignore a running Docker watchtower") parsed, _ = parser.parse_known_args(args=args) # --- code_dir = parsed.workdir if parsed.workdir else os.getcwd() dtslogger.info('Project workspace: {}'.format(code_dir)) # show info about project shell.include.devel.info.command(shell, args) # get info about current repo repo_info = shell.include.devel.info.get_repo_info(code_dir) repo = repo_info['REPOSITORY'] branch = repo_info['BRANCH'] nmodified = repo_info['INDEX_NUM_MODIFIED'] nadded = repo_info['INDEX_NUM_ADDED'] # check if the index is clean if nmodified + nadded > 0: dtslogger.warning( 'Your index is not clean (some files are not committed).') dtslogger.warning( 'If you know what you are doing, use --force (-f) to force the execution of the command.' ) if not parsed.force: exit(1) dtslogger.warning('Forced!') # create defaults default_tag = "duckietown/%s:%s" % (repo, branch) tag = "%s-%s" % (default_tag, parsed.arch) # get info about docker endpoint dtslogger.info('Retrieving info about Docker endpoint...') epoint = _run_cmd([ 'docker', '-H=%s' % parsed.machine, 'info', '--format', '{{json .}}' ], get_output=True, print_output=False) epoint = json.loads(epoint[0]) if 'ServerErrors' in epoint: dtslogger.error('\n'.join(epoint['ServerErrors'])) return epoint['MemTotal'] = _sizeof_fmt(epoint['MemTotal']) print(DOCKER_INFO.format(**epoint)) # check if there is a watchtower instance running on the endpoint if shell.include.devel.watchtower.is_running(parsed.machine): dtslogger.warning( 'An instance of a Docker watchtower was found running on the Docker endpoint.' ) dtslogger.warning( 'Building new images next to an active watchtower might (sure it will) create race conditions.' ) dtslogger.warning('Solutions:') dtslogger.warning( ' - Recommended: Use the command `dts devel watchtower stop [options]` to stop the watchtower.' ) dtslogger.warning( ' - NOT Recommended: Use the flag `--ignore-watchtower` to ignore this warning and continue.' ) if not parsed.ignore_watchtower: exit(2) dtslogger.warning('Ignored!') # print info about multiarch msg = 'Building an image for {} on {}.'.format(parsed.arch, epoint['Architecture']) dtslogger.info(msg) # register bin_fmt in the target machine (if needed) if not parsed.no_multiarch: if epoint['Architecture'] not in ARCH_MAP[CANONICAL_ARCH[ parsed.arch]]: dtslogger.info('Configuring machine for multiarch builds...') try: _run_cmd([ 'docker', '-H=%s' % parsed.machine, 'run', '--rm', '--privileged', 'multiarch/qemu-user-static:register', '--reset' ], True) dtslogger.info('Multiarch Enabled!') except: msg = 'Multiarch cannot be enabled on the target machine. This might create issues.' dtslogger.warning(msg) else: msg = 'Building an image for {} on {}. Multiarch not needed!'.format( parsed.arch, epoint['Architecture']) dtslogger.info(msg) # define labels buildlabels = [] # define build args buildargs = ['--build-arg', 'ARCH={}'.format(parsed.arch)] # loop mode (Experimental) if parsed.loop: buildargs += ['--build-arg', 'BASE_IMAGE={}'.format(repo)] buildargs += [ '--build-arg', 'BASE_TAG={}-{}'.format(branch, parsed.arch) ] buildlabels += ['--label', 'LOOP=1'] # --- msg = "WARNING: Experimental mode 'loop' is enabled!. Use with caution" dtslogger.warn(msg) # build buildlog = _run_cmd([ 'docker', '-H=%s' % parsed.machine, 'build', '--pull=%d' % int(parsed.pull), '--no-cache=%d' % int(parsed.no_cache), '-t', tag] + \ buildlabels + \ buildargs + [ code_dir ], True, True) # get image history historylog = _run_cmd([ 'docker', '-H=%s' % parsed.machine, 'history', '-H=false', '--format', '{{.ID}}:{{.Size}}', tag ], True) historylog = [l.split(':') for l in historylog if len(l.strip()) > 0] # run docker image analysis ImageAnalyzer.process(buildlog, historylog, codens=100) # image tagging if parsed.arch == DEFAULT_ARCH: dtslogger.info("Tagging image {} as {}.".format(tag, default_tag)) _run_cmd( ['docker', '-H=%s' % parsed.machine, 'tag', tag, default_tag]) # perform push (if needed) if parsed.push: if not parsed.loop: shell.include.devel.push.command(shell, args) else: msg = "Forbidden: You cannot push an image when using the experimental mode `--loop`." dtslogger.warn(msg) # perform remove (if needed) if parsed.rm: shell.include.devel.clean.command(shell, args)
def command(shell, args): prog = 'dts challenges evaluate' parser = argparse.ArgumentParser(prog=prog, usage=usage) group = parser.add_argument_group('Basic') group.add_argument('--no-cache', action='store_true', default=False, help="") group.add_argument('--no-build', action='store_true', default=False, help="") group.add_argument('--no-pull', action='store_true', default=False, help="") group.add_argument('--challenge', help="Specific challenge to evaluate") group.add_argument('--image', help="Evaluator image to run", default='duckietown/dt-challenges-evaluator:v4') group.add_argument('--shell', action='store_true', default=False, help="Runs a shell in the container") group.add_argument('--output', help="", default='output') group.add_argument('--visualize', help="Visualize the evaluation", action='store_true', default=False) parser.add_argument('--impersonate', type=str, default=None) group.add_argument('-C', dest='change', default=None) parsed = parser.parse_args(args) if parsed.change: os.chdir(parsed.change) client = check_docker_environment() command = ['dt-challenges-evaluate-local'] if parsed.no_cache: command.append('--no-cache') if parsed.no_build: command.append('--no-build') if parsed.challenge: command.extend(['--challenge', parsed.challenge]) if parsed.impersonate: command.extend(['--impersonate', parsed.impersonate]) output_rp = os.path.realpath(parsed.output) command.extend(['--output', parsed.output]) # # if parsed.features: # dtslogger.debug('Passing features %r' % parsed.features) # command += ['--features', parsed.features] # fake_dir = '/submission' tmpdir = '/tmp' UID = os.getuid() USERNAME = getpass.getuser() dir_home_guest = '/fake-home/%s' % USERNAME # os.path.expanduser('~') dir_fake_home_host = os.path.join(tmpdir, 'fake-%s-home' % USERNAME) if not os.path.exists(dir_fake_home_host): os.makedirs(dir_fake_home_host) dir_fake_home_guest = dir_home_guest dir_dtshell_host = os.path.join(os.path.expanduser('~'), '.dt-shell') dir_dtshell_guest = os.path.join(dir_fake_home_guest, '.dt-shell') dir_tmpdir_host = '/tmp' dir_tmpdir_guest = '/tmp' volumes = { '/var/run/docker.sock': { 'bind': '/var/run/docker.sock', 'mode': 'rw' } } d = os.path.join(os.getcwd(), parsed.output) if not os.path.exists(d): os.makedirs(d) volumes[output_rp] = {'bind': d, 'mode': 'rw'} volumes[os.getcwd()] = {'bind': os.getcwd(), 'mode': 'ro'} volumes[dir_tmpdir_host] = {'bind': dir_tmpdir_guest, 'mode': 'rw'} volumes[dir_dtshell_host] = {'bind': dir_dtshell_guest, 'mode': 'ro'} volumes[dir_fake_home_host] = { 'bind': dir_fake_home_guest, 'mode': 'rw' } volumes['/etc/group'] = {'bind': '/etc/group', 'mode': 'ro'} binds = [_['bind'] for _ in volumes.values()] for b1 in binds: for b2 in binds: if b1 == b2: continue if b1.startswith(b2): msg = 'Warning, it might be a problem to have binds with overlap' msg += '\n b1: %s' % b1 msg += '\n b2: %s' % b2 dtslogger.warn(msg) # command.extend(['-C', fake_dir]) env = {} extra_environment = dict(username=USERNAME, uid=UID, USER=USERNAME, HOME=dir_fake_home_guest) env.update(extra_environment) dtslogger.debug('Volumes:\n\n%s' % yaml.safe_dump(volumes, default_flow_style=False)) dtslogger.debug('Environment:\n\n%s' % yaml.safe_dump(env, default_flow_style=False)) from duckietown_challenges.rest import get_duckietown_server_url url = get_duckietown_server_url() dtslogger.info('The server URL is: %s' % url) if 'localhost' in url: h = socket.gethostname() replacement = h + '.local' dtslogger.warning( 'There is "localhost" inside, so I will try to change it to %r' % replacement) dtslogger.warning( 'This is because Docker cannot see the host as "localhost".') url = url.replace("localhost", replacement) dtslogger.warning('The new url is: %s' % url) dtslogger.warning( 'This will be passed to the evaluator in the Docker container.' ) env['DTSERVER'] = url container_name = 'local-evaluator' image = parsed.image name, tag = image.split(':') if not parsed.no_pull: dtslogger.info('Updating container %s' % image) dtslogger.info('This might take some time.') client.images.pull(name, tag) # try: container = client.containers.get(container_name) except: pass else: dtslogger.error('stopping previous %s' % container_name) container.stop() dtslogger.error('removing') container.remove() dtslogger.info('Starting container %s with %s' % (container_name, image)) detach = True env[DTShellConstants.DT1_TOKEN_CONFIG_KEY] = shell.get_dt1_token() dtslogger.info('Container command: %s' % " ".join(command)) # add all the groups on_mac = 'Darwin' in platform.system() if on_mac: group_add = [] else: group_add = [ g.gr_gid for g in grp.getgrall() if USERNAME in g.gr_mem ] interactive = False if parsed.shell: interactive = True detach = False command = ['/bin/bash', '-l'] params = dict(working_dir=os.getcwd(), user=UID, group_add=group_add, command=command, tty=interactive, volumes=volumes, environment=env, remove=True, network_mode='host', detach=detach, name=container_name) dtslogger.info('Parameters:\n%s' % json.dumps(params, indent=4)) client.containers.run(image, **params) if parsed.visualize: start_rqt_image_view() continuously_monitor(client, container_name)
def go(token, impersonate, parsed, challenge, base, client, no_cache): ri = get_registry_info(token=token, impersonate=impersonate) if parsed.steps: use_steps = parsed.steps.split(",") else: use_steps = list(challenge.steps) for step_name in use_steps: if step_name not in challenge.steps: msg = 'Could not find step "%s" in %s.' % (step_name, list(challenge.steps)) raise Exception(msg) step = challenge.steps[step_name] services = step.evaluation_parameters.services for service_name, service in services.items(): if service.build: dockerfile = service.build.dockerfile context = os.path.join(base, service.build.context) if not os.path.exists(context): msg = 'Context does not exist %s' % context raise Exception(msg) dockerfile_abs = os.path.join(context, dockerfile) if not os.path.exists(dockerfile_abs): msg = 'Cannot find Dockerfile %s' % dockerfile_abs raise Exception(msg) dtslogger.info('context: %s' % context) args = service.build.args if args: dtslogger.warning('arguments not supported yet: %s' % args) br = \ build_image(client, context, challenge.name, step_name, service_name, dockerfile_abs, no_cache, registry_info=ri) complete = get_complete_tag(br) service.image = complete # very important: get rid of it! service.build = None else: if service.image == ChallengesConstants.SUBMISSION_CONTAINER_TAG: pass else: msg = 'Finding digest for image %s' % service.image dtslogger.info(msg) image = client.images.get(service.image) service.image_digest = image.id dtslogger.info('Found: %s' % image.id) data2 = yaml.dump(challenge.as_dict()) res = dtserver_challenge_define(token, data2, parsed.force_invalidate_subs, impersonate=impersonate) challenge_id = res['challenge_id'] steps_updated = res['steps_updated'] if steps_updated: print('Updated challenge %s' % challenge_id) print('The following steps were updated and will be invalidated.') for step_name, reason in steps_updated.items(): print('\n\n' + indent(reason, ' ', step_name + ' ')) else: msg = 'No update needed - the container digests did not change.' print(msg)