def _run_cmd(cmd, get_output=False, print_output=False): dtslogger.debug('$ %s' % cmd) if get_output: proc = subprocess.Popen(cmd, stdout=subprocess.PIPE) p = re.compile(CATKIN_REGEX, re.IGNORECASE) lines = [] last_matched = False for line in io.TextIOWrapper(proc.stdout, encoding="utf-8"): line = line.rstrip() if print_output: matches = p.match(line.strip()) is not None if matches and last_matched: sys.stdout.write("\033[F") sys.stdout.write(line + "\033[K" + "\n") sys.stdout.flush() last_matched = matches if line: lines.append(line) proc.wait() if proc.returncode != 0: msg = 'The command {} returned exit code {}'.format( cmd, proc.returncode) dtslogger.error(msg) raise RuntimeError(msg) return lines else: subprocess.check_call(cmd)
def _run_cmd(cmd, get_output=False): dtslogger.debug('$ %s' % cmd) if get_output: return [ l for l in subprocess.check_output(cmd).decode('utf-8').split('\n') if l ] else: subprocess.check_call(cmd)
def build_image(client, path, challenge_name, step_name, service_name, filename, no_cache: bool, registry_info: RegistryInfo) -> BuildResult: d = datetime.datetime.now() username = get_dockerhub_username() from duckietown_challenges.utils import tag_from_date if username.lower() != username: msg = f'Are you sure that the DockerHub username is not lowercase? You gave "{username}".' dtslogger.warning(msg) username = username.lower() br = BuildResult( repository=('%s-%s-%s' % (challenge_name, step_name, service_name)).lower(), organization=username, registry=registry_info.registry, tag=tag_from_date(d), digest=None) complete = get_complete_tag(br) cmd = ['docker', 'build', '--pull', '-t', complete, '-f', filename] if no_cache: cmd.append('--no-cache') cmd.append(path) dtslogger.debug('$ %s' % " ".join(cmd)) subprocess.check_call(cmd) image = client.images.get(complete) repo_digests = image.attrs.get('RepoDigests', []) if repo_digests: msg = 'Already found repo digest: %s' % repo_digests dtslogger.info(msg) else: dtslogger.info('Image not present on registry. Need to push.') cmd = ['docker', 'push', complete] dtslogger.debug('$ %s' % " ".join(cmd)) subprocess.check_call(cmd) image = client.images.get(complete) dtslogger.info('image id: %s' % image.id) dtslogger.info('complete: %s' % get_complete_tag(br)) repo_digests = image.attrs.get('RepoDigests', []) if not repo_digests: msg = 'Could not find any repo digests (push not succeeded?)' raise Exception(msg) dtslogger.info('RepoDigests: %s' % repo_digests) _, digest = repo_digests[0].split('@') # br.digest = image.id br.digest = digest br = parse_complete_tag(get_complete_tag(br)) return br
def check_user_in_docker_group(): # first, let's see if there exists a group "docker" group_names = [g.gr_name for g in grp.getgrall()] G = 'docker' if G not in group_names: msg = 'No group %s defined.' % G dtslogger.warning(msg) else: group_id = grp.getgrnam(G).gr_gid my_groups = os.getgroups() if group_id not in my_groups: msg = 'My groups are %s and "%s" group is %s ' % (my_groups, G, group_id) msg += '\n\nNote that when you add a user to a group, you need to login in and out.' dtslogger.debug(msg)
def check_docker_environment(): username = getpass.getuser() from . import dtslogger dtslogger.debug('Checking docker environment for user %s' % username) check_executable_exists('docker') check_user_in_docker_group() # # if on_linux(): # # if username != 'root': # check_user_in_docker_group() # # print('checked groups') # else: # dtslogger.debug('skipping env check because not on Linux') try: import docker except Exception as e: msg = 'Could not import package docker:\n%s' % e msg += '\n\nYou need to install the package' raise InvalidEnvironment(msg) if 'DOCKER_HOST' in os.environ: msg = 'Note that the variable DOCKER_HOST is set to "%s"' % os.environ[ 'DOCKER_HOST'] dtslogger.warning(msg) try: client = docker.from_env() containers = client.containers.list(filters=dict(status='running')) # dtslogger.debug(json.dumps(client.info(), indent=4)) except Exception as e: msg = 'I cannot communicate with Docker:\n%s' % e msg += '\n\nMake sure the docker service is running.' raise InvalidEnvironment(msg) return client
def _run_cmd(cmd, get_output=False, print_output=False): dtslogger.debug('$ %s' % cmd) if get_output: proc = subprocess.Popen(cmd, stdout=subprocess.PIPE) lines = [] for line in io.TextIOWrapper(proc.stdout, encoding="utf-8"): line = line.rstrip() if print_output: print(line) if line: lines.append(line) proc.wait() if proc.returncode != 0: msg = 'The command {} returned exit code {}'.format( cmd, proc.returncode) dtslogger.error(msg) raise RuntimeError(msg) return lines else: subprocess.check_call(cmd)
def command(shell, args): check_docker_environment() home = os.path.expanduser('~') prog = 'dts challenges evaluator' parser = argparse.ArgumentParser(prog=prog, usage=usage) group = parser.add_argument_group('Basic') group.add_argument('--submission', type=int, default=None, help='Run a specific submission.') group.add_argument( '--reset', dest='reset', action='store_true', default=False, help='(needs --submission) Re-evaluate the specific submission.') group = parser.add_argument_group('Advanced') group.add_argument('--no-watchtower', dest='no_watchtower', action='store_true', default=False, help="Disable starting of watchtower") group.add_argument('--no-pull', dest='no_pull', action='store_true', default=False, help="Disable pulling of container") group.add_argument('--image', help="Evaluator image to run", default='duckietown/dt-challenges-evaluator:v3') group.add_argument('--name', default=None, help='Name for this evaluator') group.add_argument("--features", default=None, help="Pretend to be what you are not.") dtslogger.debug('args: %s' % args) parsed = parser.parse_args(args) machine_id = socket.gethostname() if parsed.name is None: container_name = '%s-%s' % (socket.gethostname(), os.getpid()) else: container_name = parsed.name import docker client = docker.from_env() command = ['dt-challenges-evaluator'] if parsed.submission: command += ['--submission', str(parsed.submission)] if parsed.reset: command += ['--reset'] else: command += ['--continuous'] command += ['--name', container_name] command += ['--machine-id', machine_id] if parsed.features: dtslogger.debug('Passing features %r' % parsed.features) command += ['--features', parsed.features] volumes = { '/var/run/docker.sock': { 'bind': '/var/run/docker.sock', 'mode': 'rw' }, os.path.join(home, '.dt-shell'): { 'bind': '/root/.dt-shell', 'mode': 'ro' }, '/tmp': { 'bind': '/tmp', 'mode': 'rw' } } env = {} UID = os.getuid() USERNAME = getpass.getuser() extra_environment = dict(username=USERNAME, uid=UID) env.update(extra_environment) if not parsed.no_watchtower: ensure_watchtower_active(client) url = get_duckietown_server_url() dtslogger.info('The server URL is: %s' % url) if 'localhost' in url: h = socket.gethostname() replacement = h + '.local' dtslogger.warning( 'There is "localhost" inside, so I will try to change it to %r' % replacement) dtslogger.warning( 'This is because Docker cannot see the host as "localhost".') url = url.replace("localhost", replacement) dtslogger.warning('The new url is: %s' % url) dtslogger.warning( 'This will be passed to the evaluator in the Docker container.' ) env['DTSERVER'] = url image = parsed.image name, tag = image.split(':') if not parsed.no_pull: dtslogger.info('Updating container %s' % image) client.images.pull(name, tag) try: container = client.containers.get(container_name) except: pass else: dtslogger.error('stopping previous %s' % container_name) container.stop() dtslogger.error('removing') container.remove() dtslogger.info('Starting container %s with %s' % (container_name, image)) dtslogger.info('Container command: %s' % " ".join(command)) client.containers.run(image, command=command, volumes=volumes, environment=env, network_mode='host', detach=True, name=container_name, tty=True) while True: try: container = client.containers.get(container_name) except Exception as e: msg = 'Cannot get container %s: %s' % (container_name, e) dtslogger.error(msg) dtslogger.info('Will wait.') time.sleep(5) continue dtslogger.info('status: %s' % container.status) if container.status == 'exited': msg = 'The container exited.' logs = '' for c in container.logs(stdout=True, stderr=True, stream=True): logs += c dtslogger.error(msg) tf = 'evaluator.log' with open(tf, 'w') as f: f.write(logs) msg = 'Logs saved at %s' % (tf) dtslogger.info(msg) break try: for c in container.logs(stdout=True, stderr=True, stream=True, follow=True): sys.stdout.write(c) time.sleep(3) except Exception as e: dtslogger.error(e) dtslogger.info('Will try to re-attach to container.') time.sleep(3) except KeyboardInterrupt: dtslogger.info('Received CTRL-C. Stopping container...') container.stop() dtslogger.info('Removing container') container.remove() dtslogger.info('Container removed.') break
def check_program_dependency(exe): p = which(exe) if p is None: msg = 'Could not find program %r' % exe raise Exception(msg) dtslogger.debug('Found %r at %s' % (exe, p))
def _run_cmd(cmd): dtslogger.debug('$ %s' % cmd) subprocess.check_call(cmd)
def configure_images(parsed, user_data, add_file_local, add_file): import psutil # read and validate duckiebot-compose stacks_to_load = parsed.stacks_to_load.split(',') stacks_to_run = parsed.stacks_to_run.split(',') dtslogger.info('Stacks to load: %s' % stacks_to_load) dtslogger.info('Stacks to run: %s' % stacks_to_run) for _ in stacks_to_run: if _ not in stacks_to_load: msg = 'If you want to run %r you need to load it as well.' % _ raise Exception(msg) configuration = parsed.configuration for cf in stacks_to_load: # local path lpath = get_resource(os.path.join('stacks', configuration, cf + '.yaml')) # path on PI rpath = '/var/local/%s.yaml' % cf if which('docker-compose') is None: msg = 'Could not find docker-compose. Cannot validate file.' dtslogger.error(msg) else: _run_cmd(['docker-compose', '-f', lpath, 'config', '--quiet']) add_file_local(path=rpath, local=lpath) stack2yaml = get_stack2yaml(stacks_to_load, get_resource(os.path.join('stacks', configuration))) if not stack2yaml: msg = 'Not even one stack specified' raise Exception(msg) stack2info = save_images(stack2yaml, compress=parsed.compress) buffer_bytes = 100 * 1024 * 1024 stacks_written = [] stack2archive_rpath = {} dtslogger.debug(stack2info) for stack, stack_info in stack2info.items(): tgz = stack_info.archive size = os.stat(tgz).st_size dtslogger.info('Considering copying %s of size %s' % (tgz, friendly_size_file(tgz))) rpath = os.path.join('var', 'local', os.path.basename(tgz)) destination = os.path.join(TMP_ROOT_MOUNTPOINT, rpath) available = psutil.disk_usage(TMP_ROOT_MOUNTPOINT).free dtslogger.info('available %s' % friendly_size(available)) if available < size + buffer_bytes: msg = 'You have %s available on %s but need %s for %s' % ( friendly_size(available), TMP_ROOT_MOUNTPOINT, friendly_size_file(tgz), tgz) dtslogger.info(msg) continue dtslogger.info('OK, copying, and loading it on first boot.') if os.path.exists(destination): msg = 'Skipping copying image that already exist at %s.' % destination dtslogger.info(msg) else: if which('rsync'): cmd = ['sudo', 'rsync', '-avP', tgz, destination] else: cmd = ['sudo', 'cp', tgz, destination] _run_cmd(cmd) sync_data() stack2archive_rpath[stack] = os.path.join('/', rpath) stacks_written.append(stack) client = check_docker_environment() stacks_not_to_run = [_ for _ in stacks_to_load if _ not in stacks_to_run] order = stacks_to_run + stacks_not_to_run for cf in order: if cf in stacks_written: log_current_phase(user_data, PHASE_LOADING, "Stack %s: Loading containers" % cf) cmd = 'docker load --input %s && rm %s' % (stack2archive_rpath[cf], stack2archive_rpath[cf]) add_run_cmd(user_data, cmd) add_file(stack2archive_rpath[cf] + '.labels.json', json.dumps(stack2info[cf].image_name2id, indent=4)) # cmd = ['docker', 'load', '--input', stack2archive_rpath[cf]] # add_run_cmd(user_data, cmd) # cmd = ['rm', stack2archive_rpath[cf]] # add_run_cmd(user_data, cmd) for image_name, image_id in stack2info[cf].image_name2id.items(): image = client.images.get(image_name) image_id = str(image.id) dtslogger.info('id for %s: %s' % (image_name, image_id)) cmd = ['docker', 'tag', image_id, image_name] print(cmd) add_run_cmd(user_data, cmd) if cf in stacks_to_run: msg = 'Adding the stack %r as default running' % cf dtslogger.info(msg) log_current_phase(user_data, PHASE_LOADING, "Stack %s: docker-compose up" % cf) cmd = ['docker-compose', '--file', '/var/local/%s.yaml' % cf, '-p', cf, 'up', '-d'] add_run_cmd(user_data, cmd) # XXX cmd = ['docker-compose', '-p', cf, '--file', '/var/local/%s.yaml' % cf, 'up', '-d'] user_data['bootcmd'].append(cmd) # every boot # The RPi blinking feedback expects that "All stacks up" will be written to the /data/boot-log.txt file. # If modifying, make sure to adjust the blinking feedback log_current_phase(user_data, PHASE_DONE, "All stacks up")
def step_expand(shell, parsed): deps = ['parted', 'resize2fs', 'e2fsck', 'lsblk', 'fdisk', 'umount'] for dep in deps: check_program_dependency(dep) global SD_CARD_DEVICE if not os.path.exists(SD_CARD_DEVICE): msg = 'This only works assuming device == %s' % SD_CARD_DEVICE raise Exception(msg) else: msg = 'Found device %s.' % SD_CARD_DEVICE dtslogger.info(msg) # Some devices get only a number added to the disk name, other get p + a number if os.path.exists(SD_CARD_DEVICE + '1'): DEVp1 = SD_CARD_DEVICE + '1' DEVp2 = SD_CARD_DEVICE + '2' elif os.path.exists(SD_CARD_DEVICE + 'p1'): DEVp1 = SD_CARD_DEVICE + 'p1' DEVp2 = SD_CARD_DEVICE + 'p2' else: msg = 'The two partitions of device %s could not be found.' % SD_CARD_DEVICE raise Exception(msg) # Unmount the devices and check if this worked, otherwise parted will fail p = subprocess.Popen(['lsblk'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) ret, err = p.communicate() if DEVp1 in ret.decode('utf-8'): cmd = ['sudo', 'umount', DEVp1] _run_cmd(cmd) if DEVp2 in ret.decode('utf-8'): cmd = ['sudo', 'umount', DEVp2] _run_cmd(cmd) p = subprocess.Popen(['lsblk'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) ret, err = p.communicate() if DEVp1 in ret.decode('utf-8') or DEVp2 in ret.decode('utf-8'): msg = 'Automatic unmounting of %s and %s was unsuccessful. Please do it manually and run again.' % ( DEVp1, DEVp2) raise Exception(msg) # Do the expansion dtslogger.info('Current status:') cmd = ['sudo', 'lsblk', SD_CARD_DEVICE] _run_cmd(cmd) # get the disk identifier of the SD card. # IMPORTANT: This must be executed before `parted` p = re.compile(".*Disk identifier: 0x([0-9a-z]*).*") cmd = ['sudo' ,'fdisk', '-l', SD_CARD_DEVICE] dtslogger.debug('$ %s' % cmd) pc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) ret, err = pc.communicate() m = p.search(ret.decode('utf-8')) uuid = m.group(1) cmd = ['sudo', 'parted', '-s', SD_CARD_DEVICE, 'resizepart', '2', '100%'] _run_cmd(cmd) cmd = ['sudo', 'e2fsck', '-f', DEVp2] _run_cmd(cmd) cmd = ['sudo', 'resize2fs', DEVp2] _run_cmd(cmd) # restore the original disk identifier cmd = ['sudo' ,'fdisk', SD_CARD_DEVICE] dtslogger.debug('$ %s' % cmd) p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE) ret, err = p.communicate(input=('x\ni\n0x%s\nr\nw' % uuid).encode('ascii')) print(ret.decode('utf-8')) dtslogger.info('Updated status:') cmd = ['sudo', 'lsblk', SD_CARD_DEVICE] _run_cmd(cmd)
def command(shell, args): check_docker_environment() home = os.path.expanduser('~') prog = 'dts challenges evaluator' parser = argparse.ArgumentParser(prog=prog, usage=usage) group = parser.add_argument_group('Basic') group.add_argument('--submission', type=int, default=None, help='Run a specific submission.') group.add_argument('--reset', dest='reset', action='store_true', default=False, help='(needs --submission) Re-evaluate the specific submission.') group = parser.add_argument_group('Advanced') group.add_argument('--no-watchtower', dest='no_watchtower', action='store_true', default=False, help="Disable starting of watchtower") group.add_argument('--no-pull', dest='no_pull', action='store_true', default=False, help="Disable pulling of containers") group.add_argument('--no-upload', dest='no_upload', action='store_true', default=False, help="Disable upload of artifacts") group.add_argument('--no-delete', dest='no_delete', action='store_true', default=False, help="Does not erase temporary files in /tmp/duckietown") group.add_argument('--image', help="Evaluator image to run", default='duckietown/dt-challenges-evaluator:v4') group.add_argument('--name', default=None, help='Name for this evaluator') group.add_argument("--features", default=None, help="Pretend to be what you are not.") group.add_argument("--ipfs", action='store_true', default=False, help="Run with IPFS available") group.add_argument("--one", action='store_true', default=False, help="Only run 1 submission") # dtslogger.debug('args: %s' % args) parsed = parser.parse_args(args) machine_id = socket.gethostname() if parsed.name is None: container_name = '%s-%s' % (socket.gethostname(), os.getpid()) else: container_name = parsed.name client = check_docker_environment() command = ['dt-challenges-evaluator'] if parsed.submission: command += ['--submission', str(parsed.submission)] if parsed.reset: command += ['--reset'] else: if not parsed.one: command += ['--continuous'] command += ['--name', container_name] command += ['--machine-id', machine_id] if parsed.no_upload: command += ['--no-upload'] if parsed.no_pull: command += ['--no-pull'] if parsed.no_delete: command += ['--no-delete'] if parsed.one: command += ['--one'] if parsed.features: dtslogger.debug('Passing features %r' % parsed.features) command += ['--features', parsed.features] mounts = [] volumes = { '/var/run/docker.sock': {'bind': '/var/run/docker.sock', 'mode': 'rw'}, os.path.join(home, '.dt-shell'): {'bind': '/root/.dt-shell', 'mode': 'ro'}, '/tmp': {'bind': '/tmp', 'mode': 'rw'} } if parsed.ipfs: if not ipfs_available(): msg = 'IPFS not available/mounted correctly.' raise UserError(msg) command += ['--ipfs'] # volumes['/ipfs'] = {'bind': '/ipfs', 'mode': 'ro'} from docker.types import Mount mount = Mount(type='bind', source='/ipfs', target='/ipfs', read_only=True) mounts.append(mount) env = {} UID = os.getuid() USERNAME = getpass.getuser() extra_environment = dict(username=USERNAME, uid=UID) env.update(extra_environment) if not parsed.no_watchtower: ensure_watchtower_active(client) from duckietown_challenges.rest import get_duckietown_server_url url = get_duckietown_server_url() dtslogger.info('The server URL is: %s' % url) if 'localhost' in url: h = socket.gethostname() replacement = h + '.local' dtslogger.warning('There is "localhost" inside, so I will try to change it to %r' % replacement) dtslogger.warning('This is because Docker cannot see the host as "localhost".') url = url.replace("localhost", replacement) dtslogger.warning('The new url is: %s' % url) dtslogger.warning('This will be passed to the evaluator in the Docker container.') env['DTSERVER'] = url image = parsed.image dtslogger.info('Using evaluator image %s' % image) name, tag = image.split(':') if not parsed.no_pull: dtslogger.info('Updating container %s' % image) client.images.pull(name, tag) # noinspection PyBroadException try: container = client.containers.get(container_name) except: pass else: dtslogger.error('stopping previous %s' % container_name) container.stop() dtslogger.error('removing') container.remove() dtslogger.info('Starting container %s with %s' % (container_name, image)) dtslogger.info('Container command: %s' % " ".join(command)) # add all the groups import grp group_add = [g.gr_gid for g in grp.getgrall() if USERNAME in g.gr_mem] client.containers.run(image, group_add=group_add, command=command, volumes=volumes, environment=env, mounts=mounts, network_mode='host', detach=True, name=container_name, tty=True) last_log_timestamp = None while True: try: container = client.containers.get(container_name) except Exception as e: msg = 'Cannot get container %s: %s' % (container_name, e) dtslogger.error(msg) dtslogger.info('Will wait.') time.sleep(5) continue dtslogger.info('status: %s' % container.status) if container.status == 'exited': logs = '' for c in container.logs(stdout=True, stderr=True, stream=True, since=last_log_timestamp): logs += c.decode('utf-8') last_log_timestamp = datetime.datetime.now() tf = 'evaluator.log' with open(tf, 'w') as f: f.write(logs) msg = 'The container exited.' msg += '\nLogs saved at %s' % tf dtslogger.info(msg) break try: if last_log_timestamp is not None: print('since: %s' % last_log_timestamp.isoformat()) for c0 in container.logs(stdout=True, stderr=True, stream=True, # follow=True, since=last_log_timestamp, tail=0): c: bytes = c0 try: s = c.decode('utf-8') except: s = c.decode('utf-8', errors='replace') sys.stdout.write(s) last_log_timestamp = datetime.datetime.now() time.sleep(3) except KeyboardInterrupt: dtslogger.info('Received CTRL-C. Stopping container...') container.stop() dtslogger.info('Removing container') container.remove() dtslogger.info('Container removed.') break except BaseException: s = traceback.format_exc() if 'Read timed out' in s: dtslogger.debug('(reattaching)') else: dtslogger.error(s) dtslogger.info('Will try to re-attach to container.') time.sleep(3)
def command(shell, args): parser = argparse.ArgumentParser() parser.add_argument('--image', default=IMAGE, help="Which image to use") parsed = parser.parse_args(args=args) image = parsed.image check_docker_environment() pwd = os.getcwd() bookdir = os.path.join(pwd, 'book') if not os.path.exists(bookdir): msg = 'Could not find "book" directory %r.' % bookdir raise UserError(msg) # check that the resources directory is present resources = os.path.join(pwd, 'resources') if not os.path.exists(os.path.join(resources, 'templates')): msg = 'It looks like that the "resources" repo is not checked out.' msg += '\nMaybe try:\n' msg += '\n git submodule init' msg += '\n git submodule update' raise Exception(msg) # XXX entries = list(os.listdir(bookdir)) entries = [_ for _ in entries if not _[0] == '.'] if len(entries) > 1: msg = 'Found more than one directory in "book": %s' % entries DTCommandAbs.fail(msg) bookname = entries[0] src = os.path.join(bookdir, bookname) git_version = system_cmd_result(pwd, ['git', '--version']).strip() dtslogger.debug('git version: %s' % git_version) cmd = ['git', 'rev-parse', '--show-superproject-working-tree'] gitdir_super = system_cmd_result(pwd, cmd).strip() dtslogger.debug('gitdir_super: %r' % gitdir_super) gitdir = system_cmd_result(pwd, ['git', 'rev-parse', '--show-toplevel']).strip() dtslogger.debug('gitdir: %r' % gitdir) if '--show' in gitdir_super: # or not gitdir_super: msg = "Your git version is too low, as it does not support --show-superproject-working-tree" msg += '\n\nDetected: %s' % git_version raise InvalidEnvironment(msg) if '--show' in gitdir or not gitdir: msg = "Your git version is too low, as it does not support --show-toplevel" msg += '\n\nDetected: %s' % git_version raise InvalidEnvironment(msg) pwd1 = os.path.realpath(pwd) user = getpass.getuser() tmpdir = '/tmp' fake_home = os.path.join(tmpdir, 'fake-%s-home' % user) if not os.path.exists(fake_home): os.makedirs(fake_home) resources = 'resources' uid1 = os.getuid() if sys.platform == 'darwin': flag = ':delegated' else: flag = '' cache = '/tmp/cache' if not os.path.exists(cache): os.makedirs(cache) cmd = ['docker', 'run', '-v', '%s:%s%s' % (gitdir, gitdir, flag), '-v', '%s:%s%s' % (pwd1, pwd1, flag), '-v', '%s:%s%s' % (cache, cache, flag), '-v', '%s:%s%s' % (fake_home, '/home/%s' % user, flag), '-e', 'USER=%s' % user, '-e', 'USERID=%s' % uid1, '-m', '4GB', '--user', '%s' % uid1] if gitdir_super: cmd += ['-v', '%s:%s%s' % (gitdir_super, gitdir_super, flag)] interactive = True if interactive: cmd.append('-it') cmd += [ image, '/project/run-book-native.sh', bookname, src, resources, pwd1 ] dtslogger.info('executing:\nls ' + " ".join(cmd)) # res = system_cmd_result(pwd, cmd, raise_on_error=True) try: p = subprocess.Popen(cmd, bufsize=0, executable=None, stdin=None, stdout=None, stderr=None, preexec_fn=None, shell=False, cwd=pwd, env=None) except OSError as e: if e.errno == 2: msg = 'Could not find "docker" executable.' DTCommandAbs.fail(msg) raise p.communicate() dtslogger.info('\n\nCompleted.')
def command(shell, args): prog = 'dts challenges evaluate' parser = argparse.ArgumentParser(prog=prog, usage=usage) group = parser.add_argument_group('Basic') group.add_argument('--no-cache', action='store_true', default=False, help="") group.add_argument('--no-build', action='store_true', default=False, help="") group.add_argument('--no-pull', action='store_true', default=False, help="") group.add_argument('--challenge', help="Specific challenge to evaluate") group.add_argument('--image', help="Evaluator image to run", default='duckietown/dt-challenges-evaluator:v4') group.add_argument('--shell', action='store_true', default=False, help="Runs a shell in the container") group.add_argument('--output', help="", default='output') group.add_argument('--visualize', help="Visualize the evaluation", action='store_true', default=False) parser.add_argument('--impersonate', type=str, default=None) group.add_argument('-C', dest='change', default=None) parsed = parser.parse_args(args) if parsed.change: os.chdir(parsed.change) client = check_docker_environment() command = ['dt-challenges-evaluate-local'] if parsed.no_cache: command.append('--no-cache') if parsed.no_build: command.append('--no-build') if parsed.challenge: command.extend(['--challenge', parsed.challenge]) if parsed.impersonate: command.extend(['--impersonate', parsed.impersonate]) output_rp = os.path.realpath(parsed.output) command.extend(['--output', parsed.output]) # # if parsed.features: # dtslogger.debug('Passing features %r' % parsed.features) # command += ['--features', parsed.features] # fake_dir = '/submission' tmpdir = '/tmp' UID = os.getuid() USERNAME = getpass.getuser() dir_home_guest = '/fake-home/%s' % USERNAME # os.path.expanduser('~') dir_fake_home_host = os.path.join(tmpdir, 'fake-%s-home' % USERNAME) if not os.path.exists(dir_fake_home_host): os.makedirs(dir_fake_home_host) dir_fake_home_guest = dir_home_guest dir_dtshell_host = os.path.join(os.path.expanduser('~'), '.dt-shell') dir_dtshell_guest = os.path.join(dir_fake_home_guest, '.dt-shell') dir_tmpdir_host = '/tmp' dir_tmpdir_guest = '/tmp' volumes = { '/var/run/docker.sock': { 'bind': '/var/run/docker.sock', 'mode': 'rw' } } d = os.path.join(os.getcwd(), parsed.output) if not os.path.exists(d): os.makedirs(d) volumes[output_rp] = {'bind': d, 'mode': 'rw'} volumes[os.getcwd()] = {'bind': os.getcwd(), 'mode': 'ro'} volumes[dir_tmpdir_host] = {'bind': dir_tmpdir_guest, 'mode': 'rw'} volumes[dir_dtshell_host] = {'bind': dir_dtshell_guest, 'mode': 'ro'} volumes[dir_fake_home_host] = { 'bind': dir_fake_home_guest, 'mode': 'rw' } volumes['/etc/group'] = {'bind': '/etc/group', 'mode': 'ro'} binds = [_['bind'] for _ in volumes.values()] for b1 in binds: for b2 in binds: if b1 == b2: continue if b1.startswith(b2): msg = 'Warning, it might be a problem to have binds with overlap' msg += '\n b1: %s' % b1 msg += '\n b2: %s' % b2 dtslogger.warn(msg) # command.extend(['-C', fake_dir]) env = {} extra_environment = dict(username=USERNAME, uid=UID, USER=USERNAME, HOME=dir_fake_home_guest) env.update(extra_environment) dtslogger.debug('Volumes:\n\n%s' % yaml.safe_dump(volumes, default_flow_style=False)) dtslogger.debug('Environment:\n\n%s' % yaml.safe_dump(env, default_flow_style=False)) from duckietown_challenges.rest import get_duckietown_server_url url = get_duckietown_server_url() dtslogger.info('The server URL is: %s' % url) if 'localhost' in url: h = socket.gethostname() replacement = h + '.local' dtslogger.warning( 'There is "localhost" inside, so I will try to change it to %r' % replacement) dtslogger.warning( 'This is because Docker cannot see the host as "localhost".') url = url.replace("localhost", replacement) dtslogger.warning('The new url is: %s' % url) dtslogger.warning( 'This will be passed to the evaluator in the Docker container.' ) env['DTSERVER'] = url container_name = 'local-evaluator' image = parsed.image name, tag = image.split(':') if not parsed.no_pull: dtslogger.info('Updating container %s' % image) dtslogger.info('This might take some time.') client.images.pull(name, tag) # try: container = client.containers.get(container_name) except: pass else: dtslogger.error('stopping previous %s' % container_name) container.stop() dtslogger.error('removing') container.remove() dtslogger.info('Starting container %s with %s' % (container_name, image)) detach = True env[DTShellConstants.DT1_TOKEN_CONFIG_KEY] = shell.get_dt1_token() dtslogger.info('Container command: %s' % " ".join(command)) # add all the groups on_mac = 'Darwin' in platform.system() if on_mac: group_add = [] else: group_add = [ g.gr_gid for g in grp.getgrall() if USERNAME in g.gr_mem ] interactive = False if parsed.shell: interactive = True detach = False command = ['/bin/bash', '-l'] params = dict(working_dir=os.getcwd(), user=UID, group_add=group_add, command=command, tty=interactive, volumes=volumes, environment=env, remove=True, network_mode='host', detach=detach, name=container_name) dtslogger.info('Parameters:\n%s' % json.dumps(params, indent=4)) client.containers.run(image, **params) if parsed.visualize: start_rqt_image_view() continuously_monitor(client, container_name)
def continuously_monitor(client, container_name): from docker.errors import NotFound, APIError dtslogger.debug('Monitoring container %s' % container_name) last_log_timestamp = None while True: try: container = client.containers.get(container_name) except Exception as e: # msg = 'Cannot get container %s: %s' % (container_name, e) # dtslogger.info(msg) break # dtslogger.info('Will wait.') # time.sleep(5) # continue dtslogger.info('status: %s' % container.status) if container.status == 'exited': msg = 'The container exited.' logs = '' for c in container.logs(stdout=True, stderr=True, stream=True, since=last_log_timestamp): last_log_timestamp = datetime.datetime.now() logs += c.decode() dtslogger.error(msg) tf = 'evaluator.log' with open(tf, 'w') as f: f.write(logs) msg = 'Logs saved at %s' % tf dtslogger.info(msg) # return container.exit_code return # XXX try: for c in container.logs(stdout=True, stderr=True, stream=True, follow=True, since=last_log_timestamp): if six.PY2: sys.stdout.write(c) else: sys.stdout.write(c.decode('utf-8')) last_log_timestamp = datetime.datetime.now() time.sleep(3) except KeyboardInterrupt: dtslogger.info('Received CTRL-C. Stopping container...') try: container.stop() dtslogger.info('Removing container') container.remove() dtslogger.info('Container removed.') except NotFound: pass except APIError as e: # if e.errno == 409: # pass break except BaseException: dtslogger.error(traceback.format_exc()) dtslogger.info('Will try to re-attach to container.') time.sleep(3)