def command(shell, args): token = shell.get_dt1_token() parser = argparse.ArgumentParser() parser.add_argument('--config', default='challenge.yaml', help="YAML configuration file") parser.add_argument('--no-cache', default=False, action='store_true') parser.add_argument('--steps', default=None, help='Which steps (comma separated)') parser.add_argument('--force-invalidate-subs', default=False, action='store_true') parser.add_argument('-C', dest='cwd', default=None, help='Base directory') parser.add_argument('--impersonate', type=str, default=None) parsed = parser.parse_args(args) impersonate = parsed.impersonate from dt_shell.env_checks import check_docker_environment client = check_docker_environment() if client is None: # To remove when done client = check_docker_environment() if parsed.cwd is not None: dtslogger.info('Changing to directory %s' % parsed.cwd) os.chdir(parsed.cwd) no_cache = parsed.no_cache fn = os.path.join(parsed.config) if not os.path.exists(fn): msg = 'File %s does not exist.' % fn raise UserError(msg) data = read_yaml_file(fn) if 'description' not in data or data['description'] is None: fnd = os.path.join(os.path.dirname(fn), 'challenge.description.md') if os.path.exists(fnd): desc = open(fnd).read() data['description'] = desc msg = 'Read description from %s' % fnd dtslogger.info(msg) base = os.path.dirname(fn) challenge = ChallengeDescription.from_yaml(data) with wrap_server_operations(): go(token, impersonate, parsed, challenge, base, client, no_cache)
def start_gui_tools(duckiebot_name): duckiebot_ip = get_duckiebot_ip(duckiebot_name) local_client = check_docker_environment() operating_system = platform.system() local_client.images.pull(RPI_GUI_TOOLS) env_vars = default_env(duckiebot_name, duckiebot_ip) env_vars['DISPLAY'] = True container_name = 'gui-tools-interactive' if operating_system == 'Linux': subprocess.call(["xhost", "+"]) local_client.containers.run(image=RPI_GUI_TOOLS, network_mode='host', privileged=True, tty=True, name=container_name, environment=env_vars) elif operating_system == 'Darwin': IP = subprocess.check_output([ '/bin/sh', '-c', 'ifconfig en0 | grep inet | awk \'$1=="inet" {print $2}\'' ]) env_vars['IP'] = IP subprocess.call(["xhost", "+IP"]) local_client.containers.run(image=RPI_GUI_TOOLS, network_mode='host', privileged=True, tty=True, name=container_name, environment=env_vars) attach_terminal(container_name)
def save_images(stack2yaml, compress): """ returns stack2info """ cache_dir = DOCKER_IMAGES_CACHE_DIR if not os.path.exists(cache_dir): os.makedirs(cache_dir) client = check_docker_environment() stack2info = {} for cf, config in stack2yaml.items(): image_name2id = {} for service, service_config in config['services'].items(): image_name = service_config['image'] dtslogger.info('Pulling %s' % image_name) cmd = ['docker', 'pull', image_name] _run_cmd(cmd) image = client.images.get(image_name) image_id = str(image.id) image_name2id[image_name] = image_id hname = get_md5("-".join(sorted(list(image_name2id.values()))))[:8] if compress: destination = os.path.join(cache_dir, cf + '-' + hname + '.tar.gz') else: destination = os.path.join(cache_dir, cf + '-' + hname + '.tar') destination0 = os.path.join(cache_dir, cf + '-' + hname + '.tar') stack2info[cf] = StackInfo(archive=destination, image_name2id=image_name2id, hname=hname) if os.path.exists(destination): msg = 'Using cached file %s' % destination dtslogger.info(msg) continue dtslogger.info('Saving to %s' % destination0) cmd = ['docker', 'save', '-o', destination0] + list(image_name2id.values()) _run_cmd(cmd) if compress: cmd = ['gzip', '-f', destination0] _run_cmd(cmd) assert not os.path.exists(destination0) assert os.path.exists(destination) else: assert destination == destination0 msg = 'Saved archive %s of size %s' % (destination, friendly_size_file(destination)) dtslogger.info(msg) assert len(stack2info) == len(stack2yaml) return stack2info
def run_gui_controller(hostname, image): client = check_docker_environment() container_name = "joystick_gui_%s" % hostname remove_if_running(client, container_name) duckiebot_ip = get_duckiebot_ip(hostname) env = { 'HOSTNAME': hostname, 'ROS_MASTER': hostname, 'DUCKIEBOT_NAME': hostname, 'ROS_MASTER_URI': 'http://%s:11311' % duckiebot_ip } env['QT_X11_NO_MITSHM'] = 1 volumes = {} subprocess.call(["xhost", "+"]) p = platform.system().lower() if 'darwin' in p: dtslogger.warn( "We can try but running the joystick gui on MacOSx is not expected to work..." ) env['DISPLAY'] = 'host.docker.internal:0' volumes = {'/tmp/.X11-unix': {'bind': '/tmp/.X11-unix', 'mode': 'rw'}} else: env['DISPLAY'] = os.environ['DISPLAY'] dtslogger.info("Running %s on localhost with environment vars: %s" % (container_name, env)) if 'master19' in image: image = "duckietown/rpi-duckiebot-base:master19-no-arm" cmd = "python misc/virtualJoy/virtualJoy.py %s" % hostname elif 'daffy' in image: image = "duckietown/dt-core:daffy-amd64" cmd = "roslaunch virtual_joystick virtual_joystick_gui.launch veh:=%s" % hostname params = { 'image': image, 'name': container_name, 'network_mode': 'host', 'environment': env, 'privileged': True, 'stdin_open': True, 'tty': True, 'command': cmd, 'detach': True, 'volumes': volumes } container = client.containers.run(**params) cmd = 'docker attach %s' % container_name start_command_in_subprocess(cmd)
def run_cli_controller(hostname, image, duckiebot_ip, network_mode, sim): if sim: duckiebot_client = check_docker_environment() else: duckiebot_client = docker.DockerClient('tcp://' + duckiebot_ip + ':2375') container_name = "joystick_cli_%s" % hostname remove_if_running(duckiebot_client, container_name) env = { 'HOSTNAME': hostname, 'ROS_MASTER': hostname, 'VEHICLE_NAME': hostname, 'ROS_MASTER_URI': 'http://%s:11311' % duckiebot_ip } dtslogger.info("Running %s on localhost with environment vars: %s" % (container_name, env)) if 'master19' in image: image = "duckietown/rpi-duckiebot-base:master19" # run on robot cmd = "python misc/virtualJoy/joy_cli.py %s" % hostname elif 'daffy' in image: if sim: image = "duckietown/dt-core:daffy-amd64" else: image = "duckietown/dt-core:daffy" cmd = "roslaunch virtual_joystick virtual_joystick_cli.launch veh:=%s" % hostname params = { 'image': image, 'name': container_name, 'network_mode': network_mode, 'environment': env, 'privileged': True, 'stdin_open': True, 'tty': True, 'command': cmd, 'detach': True } container = duckiebot_client.containers.run(**params) if sim: cmd = 'docker attach %s' % container_name else: cmd = 'docker -H %s.local attach %s' % (hostname, container_name) start_command_in_subprocess(cmd)
def record_bag(duckiebot_name, duration): duckiebot_ip = get_duckiebot_ip(duckiebot_name) local_client = check_docker_environment() dtslogger.info("Starting bag recording...") parameters = { 'image': RPI_DUCKIEBOT_BASE, 'remove': True, 'network_mode': 'host', 'privileged': True, 'detach': True, 'environment': default_env(duckiebot_name, duckiebot_ip), 'command': 'bash -c "cd /data && rosbag record --duration %s -a"' % duration, 'volumes': bind_local_data_dir() } # Mac Docker has ARM support directly in the Docker environment, so we don't need to run qemu... if platform.system() != 'Darwin': parameters['entrypoint'] = 'qemu3-arm-static' return local_client.containers.run(**parameters)
def run_image_on_localhost(image_name, duckiebot_name, container_name, env=None, volumes=None): duckiebot_ip = get_duckiebot_ip(duckiebot_name) local_client = check_docker_environment() env_vars = default_env(duckiebot_name, duckiebot_ip) if env is not None: env_vars.update(env) try: container = local_client.containers.get(container_name) dtslogger.info("an image already on localhost - stopping it first..") stop_container(container) remove_container(container) except Exception as e: dtslogger.warn("coulgn't remove existing container: %s" % e) dtslogger.info("Running %s on localhost with environment vars: %s" % (image_name, env_vars)) params = { 'image': image_name, 'remove': True, 'network_mode': 'host', 'privileged': True, 'detach': True, 'tty': True, 'name': container_name, 'environment': env_vars } if volumes is not None: params['volumes'] = volumes new_local_container = local_client.containers.run(**params) return new_local_container
def start_rqt_image_view(duckiebot_name=None): dtslogger.info( """{}\nOpening a camera feed by running xhost+ and running rqt_image_view...""" .format('*' * 20)) local_client = check_docker_environment() local_client.images.pull(RPI_GUI_TOOLS) env_vars = {'QT_X11_NO_MITSHM': 1} if duckiebot_name is not None: duckiebot_ip = get_duckiebot_ip(duckiebot_name) env_vars.update(default_env(duckiebot_name, duckiebot_ip)) operating_system = platform.system() if operating_system == 'Linux': subprocess.call(["xhost", "+"]) env_vars['DISPLAY'] = ':0' elif operating_system == 'Darwin': IP = subprocess.check_output([ '/bin/sh', '-c', 'ifconfig en0 | grep inet | awk \'$1=="inet" {print $2}\'' ]) env_vars['IP'] = IP subprocess.call(["xhost", "+IP"]) dtslogger.info("Running %s on localhost with environment vars: %s" % (RPI_GUI_TOOLS, env_vars)) return local_client.containers.run( image=RPI_GUI_TOOLS, remove=True, privileged=True, detach=True, network_mode='host', environment=env_vars, command= 'bash -c "source /home/software/docker/env.sh && rqt_image_view"')
def command(shell, args): check_docker_environment() home = os.path.expanduser('~') prog = 'dts challenges evaluator' parser = argparse.ArgumentParser(prog=prog, usage=usage) group = parser.add_argument_group('Basic') group.add_argument('--submission', type=int, default=None, help='Run a specific submission.') group.add_argument('--reset', dest='reset', action='store_true', default=False, help='(needs --submission) Re-evaluate the specific submission.') group = parser.add_argument_group('Advanced') group.add_argument('--no-watchtower', dest='no_watchtower', action='store_true', default=False, help="Disable starting of watchtower") group.add_argument('--no-pull', dest='no_pull', action='store_true', default=False, help="Disable pulling of containers") group.add_argument('--no-upload', dest='no_upload', action='store_true', default=False, help="Disable upload of artifacts") group.add_argument('--no-delete', dest='no_delete', action='store_true', default=False, help="Does not erase temporary files in /tmp/duckietown") group.add_argument('--image', help="Evaluator image to run", default='duckietown/dt-challenges-evaluator:v4') group.add_argument('--name', default=None, help='Name for this evaluator') group.add_argument("--features", default=None, help="Pretend to be what you are not.") group.add_argument("--ipfs", action='store_true', default=False, help="Run with IPFS available") group.add_argument("--one", action='store_true', default=False, help="Only run 1 submission") # dtslogger.debug('args: %s' % args) parsed = parser.parse_args(args) machine_id = socket.gethostname() if parsed.name is None: container_name = '%s-%s' % (socket.gethostname(), os.getpid()) else: container_name = parsed.name client = check_docker_environment() command = ['dt-challenges-evaluator'] if parsed.submission: command += ['--submission', str(parsed.submission)] if parsed.reset: command += ['--reset'] else: if not parsed.one: command += ['--continuous'] command += ['--name', container_name] command += ['--machine-id', machine_id] if parsed.no_upload: command += ['--no-upload'] if parsed.no_pull: command += ['--no-pull'] if parsed.no_delete: command += ['--no-delete'] if parsed.one: command += ['--one'] if parsed.features: dtslogger.debug('Passing features %r' % parsed.features) command += ['--features', parsed.features] mounts = [] volumes = { '/var/run/docker.sock': {'bind': '/var/run/docker.sock', 'mode': 'rw'}, os.path.join(home, '.dt-shell'): {'bind': '/root/.dt-shell', 'mode': 'ro'}, '/tmp': {'bind': '/tmp', 'mode': 'rw'} } if parsed.ipfs: if not ipfs_available(): msg = 'IPFS not available/mounted correctly.' raise UserError(msg) command += ['--ipfs'] # volumes['/ipfs'] = {'bind': '/ipfs', 'mode': 'ro'} from docker.types import Mount mount = Mount(type='bind', source='/ipfs', target='/ipfs', read_only=True) mounts.append(mount) env = {} UID = os.getuid() USERNAME = getpass.getuser() extra_environment = dict(username=USERNAME, uid=UID) env.update(extra_environment) if not parsed.no_watchtower: ensure_watchtower_active(client) from duckietown_challenges.rest import get_duckietown_server_url url = get_duckietown_server_url() dtslogger.info('The server URL is: %s' % url) if 'localhost' in url: h = socket.gethostname() replacement = h + '.local' dtslogger.warning('There is "localhost" inside, so I will try to change it to %r' % replacement) dtslogger.warning('This is because Docker cannot see the host as "localhost".') url = url.replace("localhost", replacement) dtslogger.warning('The new url is: %s' % url) dtslogger.warning('This will be passed to the evaluator in the Docker container.') env['DTSERVER'] = url image = parsed.image dtslogger.info('Using evaluator image %s' % image) name, tag = image.split(':') if not parsed.no_pull: dtslogger.info('Updating container %s' % image) client.images.pull(name, tag) # noinspection PyBroadException try: container = client.containers.get(container_name) except: pass else: dtslogger.error('stopping previous %s' % container_name) container.stop() dtslogger.error('removing') container.remove() dtslogger.info('Starting container %s with %s' % (container_name, image)) dtslogger.info('Container command: %s' % " ".join(command)) # add all the groups import grp group_add = [g.gr_gid for g in grp.getgrall() if USERNAME in g.gr_mem] client.containers.run(image, group_add=group_add, command=command, volumes=volumes, environment=env, mounts=mounts, network_mode='host', detach=True, name=container_name, tty=True) last_log_timestamp = None while True: try: container = client.containers.get(container_name) except Exception as e: msg = 'Cannot get container %s: %s' % (container_name, e) dtslogger.error(msg) dtslogger.info('Will wait.') time.sleep(5) continue dtslogger.info('status: %s' % container.status) if container.status == 'exited': logs = '' for c in container.logs(stdout=True, stderr=True, stream=True, since=last_log_timestamp): logs += c.decode('utf-8') last_log_timestamp = datetime.datetime.now() tf = 'evaluator.log' with open(tf, 'w') as f: f.write(logs) msg = 'The container exited.' msg += '\nLogs saved at %s' % tf dtslogger.info(msg) break try: if last_log_timestamp is not None: print('since: %s' % last_log_timestamp.isoformat()) for c0 in container.logs(stdout=True, stderr=True, stream=True, # follow=True, since=last_log_timestamp, tail=0): c: bytes = c0 try: s = c.decode('utf-8') except: s = c.decode('utf-8', errors='replace') sys.stdout.write(s) last_log_timestamp = datetime.datetime.now() time.sleep(3) except KeyboardInterrupt: dtslogger.info('Received CTRL-C. Stopping container...') container.stop() dtslogger.info('Removing container') container.remove() dtslogger.info('Container removed.') break except BaseException: s = traceback.format_exc() if 'Read timed out' in s: dtslogger.debug('(reattaching)') else: dtslogger.error(s) dtslogger.info('Will try to re-attach to container.') time.sleep(3)
def command(shell, args): prog = 'dts duckiebot demo' parser = argparse.ArgumentParser(prog=prog, usage=usage) parser.add_argument('--demo_name', dest="demo_name", default=None, help="Name of the demo to run") parser.add_argument( '--duckiebot_name', dest="duckiebot_name", default=None, help="Name of the Duckiebot on which to run the demo") parser.add_argument( '--package_name', dest="package_name", default="duckietown", help= "You can specify the package that you want to use to look for launch files" ) parser.add_argument( '--image', dest="image_to_run", default="duckietown/rpi-duckiebot-base:master19", help="Docker image to use, you probably don't need to change ", ) parser.add_argument('--debug', action='store_true', default=False, help="will enter you into the running container") parsed = parser.parse_args(args) check_docker_environment() demo_name = parsed.demo_name if demo_name is None: msg = 'You must specify a demo_name' raise InvalidUserInput(msg) duckiebot_name = parsed.duckiebot_name if duckiebot_name is None: msg = 'You must specify a duckiebot_name' raise InvalidUserInput(msg) package_name = parsed.package_name dtslogger.info("Using package %s" % package_name) duckiebot_ip = get_duckiebot_ip(duckiebot_name) duckiebot_client = docker.DockerClient('tcp://' + duckiebot_ip + ':2375') container_name = 'demo_%s' % demo_name remove_if_running(duckiebot_client, container_name) image_base = parsed.image_to_run env_vars = default_env(duckiebot_name, duckiebot_ip) env_vars.update({'VEHICLE_NAME': duckiebot_name}) if demo_name == 'base': cmd = '/bin/bash' else: cmd = 'roslaunch %s %s.launch veh:=%s' % (package_name, demo_name, duckiebot_name) dtslogger.info("Running command %s" % cmd) demo_container = duckiebot_client.containers.run( image=image_base, command=cmd, network_mode='host', volumes=bind_duckiebot_data_dir(), privileged=True, name=container_name, mem_limit='800m', memswap_limit='2800m', stdin_open=True, tty=True, detach=True, environment=env_vars) if demo_name == 'base' or parsed.debug: attach_cmd = 'docker -H %s.local attach %s' % (duckiebot_name, container_name) start_command_in_subprocess(attach_cmd)
def command(shell, args): prog = 'dts duckiebot calibrate_intrinsics DUCKIEBOT_NAME' usage = """ Calibrate: %(prog)s """ parser = argparse.ArgumentParser(prog=prog, usage=usage) parser.add_argument('hostname', default=None, help='Name of the Duckiebot to calibrate') parser.add_argument('--base_image', dest='image', default="duckietown/rpi-duckiebot-base:master19-no-arm") parser.add_argument('--debug', action='store_true', default=False, help="will enter you into the running container") parsed_args = parser.parse_args(args) hostname = parsed_args.hostname duckiebot_ip = get_duckiebot_ip(hostname) duckiebot_client = get_remote_client(duckiebot_ip) # is the interface running? try: duckiebot_containers = duckiebot_client.containers.list() interface_container_found = False for c in duckiebot_containers: if 'duckiebot-interface' in c.name: interface_container_found = True if not interface_container_found: dtslogger.error("The duckiebot-interface is not running on the duckiebot") exit() except Exception as e: dtslogger.warn( "Not sure if the duckiebot-interface is running because we got and exception when trying: %s" % e) # is the raw imagery being published? try: duckiebot_containers = duckiebot_client.containers.list() raw_imagery_found = False for c in duckiebot_containers: if 'demo_camera' in c.name: raw_imagery_found = True if not raw_imagery_found: dtslogger.error("The demo_camera is not running on the duckiebot - please run `dts duckiebot demo --demo_name camera --duckiebot_name %s" % hostname) exit() except Exception as e: dtslogger.warn("%s" % e) image = parsed_args.image client = check_docker_environment() container_name = "intrinsic_calibration_%s" % hostname remove_if_running(client,container_name) env = {'HOSTNAME': hostname, 'ROS_MASTER': hostname, 'DUCKIEBOT_NAME': hostname, 'ROS_MASTER_URI': 'http://%s:11311' % duckiebot_ip, 'QT_X11_NO_MITSHM': 1} volumes = {} subprocess.call(["xhost", "+"]) p = platform.system().lower() if 'darwin' in p: env['DISPLAY'] = 'host.docker.internal:0' volumes = { '/tmp/.X11-unix': {'bind': '/tmp/.X11-unix', 'mode': 'rw'} } else: env['DISPLAY'] = os.environ['DISPLAY'] dtslogger.info("Running %s on localhost with environment vars: %s" % (container_name, env)) dtslogger.info("When the window opens you will need to move the checkerboard around in front of the Duckiebot camera") cmd = "roslaunch pi_camera intrinsic_calibration.launch veh:=%s" % hostname params = {'image': image, 'name': container_name, 'network_mode': 'host', 'environment': env, 'privileged': True, 'stdin_open': True, 'tty': True, 'detach': True, 'command': cmd, 'volumes': volumes } container = client.containers.run(**params) if parsed_args.debug: attach_cmd = 'docker attach %s' % (container_name) start_command_in_subprocess(attach_cmd)
def command(shell, args): prog = 'dts duckiebot evaluate' parser = argparse.ArgumentParser(prog=prog, usage=usage) group = parser.add_argument_group('Basic') group.add_argument('--duckiebot_name', default=None, help="Name of the Duckiebot on which to perform evaluation") group.add_argument('--duckiebot_username', default="duckie", help="The duckiebot username") group.add_argument('--image', dest='image_name', help="Image to evaluate, if none specified then we will build your current context", default=None) group.add_argument('--glue_node_image', default="duckietown/challenge-aido_lf-duckiebot:aido2", help="The node that glues your submission with ROS on the duckiebot. Probably don't change") group.add_argument('--duration', help="Number of seconds to run evaluation", default=15) group.add_argument('--remotely', action='store_true', default=True, help="If true run the image over network without pushing to Duckiebot") group.add_argument('--no_cache', help="disable cache on docker build", action="store_true", default=False) group.add_argument('--record_bag', action='store_true', default=False, help="If true record a rosbag") group.add_argument("--debug", action='store_true', default=False, help="If true you will get a shell instead of executing") group.add_argument('--max_vel', help="the max velocity for the duckiebot", default=0.7) group.add_argument('--challenge', help="Specific challenge to evaluate") parsed = parser.parse_args(args) tmpdir = '/tmp' USERNAME = getpass.getuser() dir_home_guest = os.path.expanduser('~') dir_fake_home = os.path.join(tmpdir, 'fake-%s-home' % USERNAME) if not os.path.exists(dir_fake_home): os.makedirs(dir_fake_home) get_calibration_files(dir_fake_home, parsed.duckiebot_username, parsed.duckiebot_name) client = check_docker_environment() agent_container_name = "agent" glue_container_name = "aido_glue" # remove the containers if they are already running remove_if_running(client, agent_container_name) remove_if_running(client, glue_container_name) # setup the fifos2 volume (requires pruning it if it's still hanging around from last time) try: client.volumes.prune() fifo2_volume=client.volumes.create(name='fifos2') except Exception as e: dtslogger.warn("error creating volume: %s" % e) duckiebot_ip = get_duckiebot_ip(parsed.duckiebot_name) duckiebot_client = get_remote_client(duckiebot_ip) try: duckiebot_containers = duckiebot_client.containers.list() interface_container_found=False for c in duckiebot_containers: if 'duckiebot-interface' in c.name: interface_container_found=True if not interface_container_found: dtslogger.error("The duckiebot-interface is not running on the duckiebot") except Exception as e: dtslogger.warn("Not sure if the duckiebot-interface is running because we got and exception when trying: %s" % e) # let's start building stuff for the "glue" node glue_volumes = {fifo2_volume.name: {'bind': '/fifos', 'mode': 'rw'}} glue_env = {'HOSTNAME':parsed.duckiebot_name, 'DUCKIEBOT_NAME':parsed.duckiebot_name, 'ROS_MASTER_URI':'http://%s:11311' % duckiebot_ip} dtslogger.info("Running %s on localhost with environment vars: %s" % (parsed.glue_node_image, glue_env)) params = {'image': parsed.glue_node_image, 'name': glue_container_name, 'network_mode': 'host', 'privileged': True, 'environment': glue_env, 'detach': True, 'tty': True, 'volumes': glue_volumes} # run the glue container glue_container = client.containers.run(**params) if not parsed.debug: monitor_thread = threading.Thread(target=continuously_monitor, args=(client, glue_container.name)) monitor_thread.start() if parsed.image_name is None: # if we didn't get an `image_name` we try need to build the local container path = '.' dockerfile = os.path.join(path, 'Dockerfile') if not os.path.exists(dockerfile): msg = 'No Dockerfile' raise Exception(msg) tag='myimage' if parsed.no_cache: cmd = ['docker', 'build', '--no-cache', '-t', tag, '-f', dockerfile] else: cmd = ['docker', 'build', '-t', tag, '-f', dockerfile] dtslogger.info("Running command: %s" % cmd) cmd.append(path) subprocess.check_call(cmd) image_name = tag else: image_name = parsed.image_name # start to build the agent stuff agent_env = {'AIDONODE_DATA_IN':'/fifos/agent-in', 'AIDONODE_DATA_OUT':'fifo:/fifos/agent-out'} agent_volumes = {fifo2_volume.name: {'bind': '/fifos', 'mode': 'rw'}, dir_fake_home: {'bind': '/data/config', 'mode': 'rw'} } params = {'image': image_name, 'remove': True, 'name': agent_container_name, 'environment': agent_env, 'detach': True, 'tty': True, 'volumes': agent_volumes} if parsed.debug: params['command'] = '/bin/bash' params['stdin_open'] = True dtslogger.info("Running %s on localhost with environment vars: %s" % (image_name, agent_env)) agent_container = client.containers.run(**params) if parsed.debug: attach_cmd = 'docker attach %s' % agent_container_name start_command_in_subprocess(attach_cmd) else: monitor_thread = threading.Thread(target=continuously_monitor,args=(client, agent_container_name)) monitor_thread.start() duration = int(parsed.duration) # should we record a bag? if parsed.record_bag: bag_container = record_bag(parsed.hostname, duration) dtslogger.info("Running for %d s" % duration) time.sleep(duration) stop_container(glue_container) stop_container(agent_container) if parsed.record_bag: stop_container(bag_container)
def command(shell, args): parser = argparse.ArgumentParser() parser.add_argument( '--image', default= 'andreacensi/mcdp_books:duckuments@sha256:ae2fcdbb8ce409e4817ed74c67b04bb91cd14ca96bed887e75e5275fa2efc933', help="Which image to use") parsed = parser.parse_args(args=args) image = parsed.image check_docker_environment() # check_git_supports_superproject() from system_cmd import system_cmd_result pwd = os.getcwd() bookdir = os.path.join(pwd, 'book') if not os.path.exists(bookdir): msg = 'Could not find "book" directory %r.' % bookdir DTCommandAbs.fail(msg) # check that the resources directory is present resources = os.path.join(pwd, 'resources') if not os.path.exists(os.path.join(resources, 'templates')): msg = 'It looks like that the "resources" repo is not checked out.' msg += '\nMaybe try:\n' msg += '\n git submodule init' msg += '\n git submodule update' raise Exception(msg) # XXX entries = list(os.listdir(bookdir)) entries = [_ for _ in entries if not _[0] == '.'] if len(entries) > 1: msg = 'Found more than one directory in "book": %s' % entries DTCommandAbs.fail(msg) bookname = entries[0] src = os.path.join(bookdir, bookname) res = system_cmd_result(pwd, ['git', '--version'], raise_on_error=True) git_version = res.stdout print('git version: %s' % git_version) cmd = ['git', 'rev-parse', '--show-superproject-working-tree'] res = system_cmd_result(pwd, cmd, raise_on_error=True) gitdir_super = res.stdout.strip() if '--show' in gitdir_super or not gitdir_super: msg = "Your git version is too low, as it does not support --show-superproject-working-tree" msg += '\n\nDetected: %s' % git_version raise InvalidEnvironment(msg) print('gitdir_super: %r' % gitdir_super) res = system_cmd_result(pwd, ['git', 'rev-parse', '--show-toplevel'], raise_on_error=True) gitdir = res.stdout.strip() if '--show' in gitdir or not gitdir: msg = "Your git version is too low, as it does not support --show-toplevel" msg += '\n\nDetected: %s' % git_version raise InvalidEnvironment(msg) print('gitdir: %r' % gitdir) pwd1 = os.path.realpath(pwd) user = getpass.getuser() tmpdir = '/tmp' fake_home = os.path.join(tmpdir, 'fake-%s-home' % user) if not os.path.exists(fake_home): os.makedirs(fake_home) resources = 'resources' uid1 = os.getuid() if sys.platform == 'darwin': flag = ':delegated' else: flag = '' cmd = [ 'docker', 'run', '-v', '%s:%s%s' % (gitdir, gitdir, flag), '-v', '%s:%s%s' % (gitdir_super, gitdir_super, flag), '-v', '%s:%s%s' % (pwd1, pwd1, flag), '-v', '%s:%s%s' % (fake_home, '/home/%s' % user, flag), '-e', 'USER=%s' % user, '-e', 'USERID=%s' % uid1, '-m', '4GB', '--user', '%s' % uid1 ] interactive = True if interactive: cmd.append('-it') cmd += [ image, '/project/run-book-native.sh', bookname, src, resources, pwd1 ] print('executing:\nls ' + " ".join(cmd)) # res = system_cmd_result(pwd, cmd, raise_on_error=True) try: p = subprocess.Popen(cmd, bufsize=0, executable=None, stdin=None, stdout=None, stderr=None, preexec_fn=None, shell=False, cwd=pwd, env=None) except OSError as e: if e.errno == 2: msg = 'Could not find "docker" executable.' DTCommandAbs.fail(msg) raise p.communicate() print('\n\nCompleted.')
def configure_images(parsed, user_data, add_file_local, add_file): import psutil # read and validate duckiebot-compose stacks_to_load = parsed.stacks_to_load.split(',') stacks_to_run = parsed.stacks_to_run.split(',') dtslogger.info('Stacks to load: %s' % stacks_to_load) dtslogger.info('Stacks to run: %s' % stacks_to_run) for _ in stacks_to_run: if _ not in stacks_to_load: msg = 'If you want to run %r you need to load it as well.' % _ raise Exception(msg) configuration = parsed.configuration for cf in stacks_to_load: # local path lpath = get_resource(os.path.join('stacks', configuration, cf + '.yaml')) # path on PI rpath = '/var/local/%s.yaml' % cf if which('docker-compose') is None: msg = 'Could not find docker-compose. Cannot validate file.' dtslogger.error(msg) else: _run_cmd(['docker-compose', '-f', lpath, 'config', '--quiet']) add_file_local(path=rpath, local=lpath) stack2yaml = get_stack2yaml(stacks_to_load, get_resource(os.path.join('stacks', configuration))) if not stack2yaml: msg = 'Not even one stack specified' raise Exception(msg) stack2info = save_images(stack2yaml, compress=parsed.compress) buffer_bytes = 100 * 1024 * 1024 stacks_written = [] stack2archive_rpath = {} dtslogger.debug(stack2info) for stack, stack_info in stack2info.items(): tgz = stack_info.archive size = os.stat(tgz).st_size dtslogger.info('Considering copying %s of size %s' % (tgz, friendly_size_file(tgz))) rpath = os.path.join('var', 'local', os.path.basename(tgz)) destination = os.path.join(TMP_ROOT_MOUNTPOINT, rpath) available = psutil.disk_usage(TMP_ROOT_MOUNTPOINT).free dtslogger.info('available %s' % friendly_size(available)) if available < size + buffer_bytes: msg = 'You have %s available on %s but need %s for %s' % ( friendly_size(available), TMP_ROOT_MOUNTPOINT, friendly_size_file(tgz), tgz) dtslogger.info(msg) continue dtslogger.info('OK, copying, and loading it on first boot.') if os.path.exists(destination): msg = 'Skipping copying image that already exist at %s.' % destination dtslogger.info(msg) else: if which('rsync'): cmd = ['sudo', 'rsync', '-avP', tgz, destination] else: cmd = ['sudo', 'cp', tgz, destination] _run_cmd(cmd) sync_data() stack2archive_rpath[stack] = os.path.join('/', rpath) stacks_written.append(stack) client = check_docker_environment() stacks_not_to_run = [_ for _ in stacks_to_load if _ not in stacks_to_run] order = stacks_to_run + stacks_not_to_run for cf in order: if cf in stacks_written: log_current_phase(user_data, PHASE_LOADING, "Stack %s: Loading containers" % cf) cmd = 'docker load --input %s && rm %s' % (stack2archive_rpath[cf], stack2archive_rpath[cf]) add_run_cmd(user_data, cmd) add_file(stack2archive_rpath[cf] + '.labels.json', json.dumps(stack2info[cf].image_name2id, indent=4)) # cmd = ['docker', 'load', '--input', stack2archive_rpath[cf]] # add_run_cmd(user_data, cmd) # cmd = ['rm', stack2archive_rpath[cf]] # add_run_cmd(user_data, cmd) for image_name, image_id in stack2info[cf].image_name2id.items(): image = client.images.get(image_name) image_id = str(image.id) dtslogger.info('id for %s: %s' % (image_name, image_id)) cmd = ['docker', 'tag', image_id, image_name] print(cmd) add_run_cmd(user_data, cmd) if cf in stacks_to_run: msg = 'Adding the stack %r as default running' % cf dtslogger.info(msg) log_current_phase(user_data, PHASE_LOADING, "Stack %s: docker-compose up" % cf) cmd = ['docker-compose', '--file', '/var/local/%s.yaml' % cf, '-p', cf, 'up', '-d'] add_run_cmd(user_data, cmd) # XXX cmd = ['docker-compose', '-p', cf, '--file', '/var/local/%s.yaml' % cf, 'up', '-d'] user_data['bootcmd'].append(cmd) # every boot # The RPi blinking feedback expects that "All stacks up" will be written to the /data/boot-log.txt file. # If modifying, make sure to adjust the blinking feedback log_current_phase(user_data, PHASE_DONE, "All stacks up")
def dt_challenges_evaluator(): elogger.info("dt-challenges-evaluator (DTC %s)" % __version__) check_docker_environment() try: check_executable_exists('docker-compose') except InvalidEnvironment: msg = 'Could not find docker-compose. Please install it.' msg += '\n\nSee: https://docs.docker.com/compose/install/#install-compose' raise InvalidEnvironment(msg) parser = argparse.ArgumentParser() parser.add_argument("--continuous", action="store_true", default=False) parser.add_argument("--no-pull", dest='no_pull', action="store_true", default=False) parser.add_argument("extra", nargs=argparse.REMAINDER) parsed = parser.parse_args() do_pull = not parsed.no_pull # # try: # docker_username = get_dockerhub_username() # except Exception: # msg = 'Skipping push because docker_username is not set.' # elogger.debug(msg) # docker_username = None if parsed.continuous: timeout = 5.0 # seconds multiplier = 1.0 max_multiplier = 10 while True: multiplier = min(multiplier, max_multiplier) try: go_(None, do_pull) multiplier = 1.0 except NothingLeft: sys.stderr.write('.') # time.sleep(timeout * multiplier) # elogger.info('No submissions available to evaluate.') except ConnectionError as e: elogger.error(e) multiplier *= 1.5 except Exception as e: msg = 'Uncaught exception:\n%s' % traceback.format_exc(e) elogger.error(msg) multiplier *= 1.5 time.sleep(timeout * multiplier) else: submissions = parsed.extra if not submissions: submissions = [None] for submission_id in submissions: try: go_(submission_id, do_pull) except NothingLeft: elogger.info('No submissions available to evaluate.')
def command(shell, args): check_docker_environment() token = shell.get_dt1_token() prog = 'dts challenges submit' usage = """ Submission: %(prog)s --challenge NAME ## Building options Rebuilds ignoring Docker cache %(prog)s --no-cache ## Attaching user data Submission with an identifying label: %(prog)s --user-label "My submission" Submission with an arbitrary JSON payload: %(prog)s --user-meta '{"param1": 123}' """ parser = argparse.ArgumentParser(prog=prog, usage=usage) group = parser.add_argument_group("Submission identification") parser.add_argument('--challenge', help="Specify challenge name.", default=None) group.add_argument('--user-label', dest='message', default=None, type=str, help="Submission message") group.add_argument( '--user-meta', dest='metadata', default=None, type=str, help="Custom JSON structure to attach to the submission") group = parser.add_argument_group("Building settings.") group.add_argument('--no-cache', dest='no_cache', action='store_true', default=False) group.add_argument('--impersonate', type=int, default=None) group.add_argument('-C', dest='cwd', default=None, help='Base directory') parsed = parser.parse_args(args) impersonate = parsed.impersonate if parsed.cwd is not None: dtslogger.info('Changing to directory %s' % parsed.cwd) os.chdir(parsed.cwd) if not os.path.exists('submission.yaml'): msg = 'Expected a submission.yaml file in %s.' % (os.path.realpath( os.getcwd())) raise UserError(msg) sub_info = read_submission_info('.') with wrap_server_operations(): ri = get_registry_info(token=token, impersonate=impersonate) registry = ri.registry compat = dtserver_get_compatible_challenges( token=token, impersonate=impersonate, submission_protocols=sub_info.protocols) if not compat.compatible: msg = 'There are no compatible challenges with protocols %s,\n' \ ' or you might not have the necessary permissions.' % sub_info.protocols raise UserError(msg) if parsed.message: sub_info.user_label = parsed.message if parsed.metadata: sub_info.user_metadata = json.loads(parsed.metadata) if parsed.challenge: sub_info.challenge_names = parsed.challenge.split(',') if sub_info.challenge_names is None: sub_info.challenge_names = compat.compatible print('I will submit to the challenges %s' % sub_info.challenge_names) for c in sub_info.challenge_names: if not c in compat.available_submit: msg = 'The challenge "%s" does not exist among %s.' % ( c, list(compat.available_submit)) raise UserError(msg) if not c in compat.compatible: msg = 'The challenge "%s" is not compatible with protocols %s .' % ( c, sub_info.protocols) raise UserError(msg) username = get_dockerhub_username(shell) print('') print('') br = submission_build(username=username, registry=registry, no_cache=parsed.no_cache) data = { 'image': dataclasses.asdict(br), 'user_label': sub_info.user_label, 'user_payload': sub_info.user_metadata, 'protocols': sub_info.protocols } data = dtserver_submit2(token=token, challenges=sub_info.challenge_names, data=data, impersonate=impersonate) # print('obtained:\n%s' % json.dumps(data, indent=2)) component_id = data['component_id'] submissions = data['submissions'] # url_component = href(get_duckietown_server_url() + '/humans/components/%s' % component_id) msg = f''' Successfully created component. This component has been entered in {len(submissions)} challenge(s). ''' for challenge_name, sub_info in submissions.items(): submission_id = sub_info['submission_id'] url_submission = href(get_duckietown_server_url() + '/humans/submissions/%s' % submission_id) challenge_title = sub_info['challenge']['title'] submission_id_color = termcolor.colored(submission_id, 'cyan') P = dark('$') head = bright( f'## Challenge {challenge_name} - {challenge_title}') msg += '\n\n' + f''' {head} Track this submission at: {url_submission} You can follow its fate using: {P} dts challenges follow --submission {submission_id_color} You can speed up the evaluation using your own evaluator: {P} dts challenges evaluator --submission {submission_id_color} '''.strip() manual = href('http://docs.duckietown.org/DT19/AIDO/out/') msg += f''' For more information, see the manual at {manual} ''' shell.sprint(msg)
def command(shell, args): check_docker_environment() token = shell.get_dt1_token() prog = 'dts challenges submit' usage = """ Submission: %(prog)s --challenge NAME ## Building options Rebuilds ignoring Docker cache %(prog)s --no-cache ## Attaching user data Submission with an identifying label: %(prog)s --user-label "My submission" Submission with an arbitrary JSON payload: %(prog)s --user-meta '{"param1": 123}' """ parser = argparse.ArgumentParser(prog=prog, usage=usage) group = parser.add_argument_group("Submission identification") parser.add_argument('--challenge', help="Specify challenge name.", default=None) group.add_argument('--user-label', dest='message', default=None, type=str, help="Submission message") group.add_argument( '--user-meta', dest='metadata', default=None, type=str, help="Custom JSON structure to attach to the submission") group = parser.add_argument_group("Building settings.") group.add_argument('--no-push', dest='no_push', action='store_true', default=False, help="Disable pushing of container") group.add_argument('--no-submit', dest='no_submit', action='store_true', default=False, help="Disable submission (only build and push)") group.add_argument('--no-cache', dest='no_cache', action='store_true', default=False) group.add_argument('-C', dest='cwd', default=None, help='Base directory') parsed = parser.parse_args(args) do_push = not parsed.no_push if parsed.cwd is not None: dtslogger.info('Changing to directory %s' % parsed.cwd) os.chdir(parsed.cwd) if not os.path.exists('submission.yaml'): msg = 'Expected a submission.yaml file in %s.' % (os.path.realpath( os.getcwd())) raise Exception(msg) sub_info = read_submission_info('.') if parsed.message: sub_info.user_label = parsed.message if parsed.metadata: sub_info.user_payload = json.loads(parsed.metadata) if parsed.challenge: sub_info.challenge_name = parsed.challenge username = get_dockerhub_username(shell) hashname = build(username, sub_info.challenge_name, do_push, no_cache=parsed.no_cache) data = { 'hash': hashname, 'user_label': sub_info.user_label, 'user_payload': sub_info.user_payload, 'protocols': sub_info.protocols } if not parsed.no_submit: submission_id = dtserver_submit(token, sub_info.challenge_name, data) url = get_duckietown_server_url( ) + '/humans/submissions/%s' % submission_id url = href(url) manual = href('http://docs.duckietown.org/DT18/AIDO/out/') ID = termcolor.colored(submission_id, 'cyan') msg = ''' Successfully created submission {ID}. You can track the progress at: {url} You can also use the command `follow` to follow its fate: {P} dts challenges follow --submission {ID} You can speed up the evaluation using your own evaluator: {P} dts challenges evaluator --submission {ID} For more information, see the manual at {manual} '''.format(ID=ID, P=dark('$'), url=url, manual=manual) if hasattr(shell, 'sprint'): shell.sprint(msg) else: print(msg)
def command(shell, args): parser = argparse.ArgumentParser() parser.add_argument('--hostname', default='duckiebot') parser.add_argument('--linux-username', default='duckie') parser.add_argument('--linux-password', default='quackquack') parser.add_argument('--wifi-ssid', dest="wifissid", default='duckietown') parser.add_argument('--wifi-password', dest="wifipass", default='quackquack') parsed = parser.parse_args(args=args) check_docker_environment() if not is_valid_hostname(parsed.hostname): msg = 'This is not a valid hostname: %r.' % parsed.hostname raise Exception(msg) if '-' in parsed.hostname: msg = 'Cannot use the hostname %r. It cannot contain "-" because of a ROS limitation. ' % parsed.hostname raise Exception(msg) if len(parsed.hostname) < 3: msg = 'This hostname is too short. Choose something more descriptive.' raise Exception(msg) MIN_AVAILABLE_GB = 0.0 try: import psutil except ImportError: msg = 'Skipping disk check because psutil not installed.' dtslogger.info(msg) else: disk = psutil.disk_usage(os.getcwd()) disk_available_gb = disk.free / (1024 * 1024 * 1024.0) if disk_available_gb < MIN_AVAILABLE_GB: msg = 'This procedure requires that you have at least %f GB of memory.' % MIN_AVAILABLE_GB msg += '\nYou only have %f GB available.' % disk_available_gb raise Exception(msg) p = platform.system().lower() if 'darwin' in p: msg = 'This procedure cannot be run on Mac. You need an Ubuntu machine.' raise Exception(msg) this = dirname(realpath(__file__)) script_files = realpath(join(this, '..', 'init_sd_card.scripts')) script_file = join(script_files, 'init_sd_card.sh') if not os.path.exists(script_file): msg = 'Could not find script %s' % script_file raise Exception(msg) ssh_key_pri = join(script_files, 'DT18_key_00') ssh_key_pub = join(script_files, 'DT18_key_00.pub') for f in [ssh_key_pri, ssh_key_pub]: if not os.path.exists(f): msg = 'Could not find file %s' % f raise Exception(msg) script_cmd = '/bin/bash %s' % script_file token = shell.get_dt1_token() env = dict() env['DUCKIE_TOKEN'] = token env['IDENTITY_FILE'] = ssh_key_pub env['WIFISSID'] = parsed.wifissid env['WIFIPASS'] = parsed.wifipass env['HOST_NAME'] = parsed.hostname env['DTS_USERNAME'] = parsed.linux_username env['PASSWORD'] = parsed.linux_password # add other environment env.update(os.environ) if 'DOCKER_HOST' in env: r = env['DOCKER_HOST'] msg = 'I will IGNORE the DOCKER_HOST variable that is currently set to %r' % r dtslogger.warning(msg) env.pop('DOCKER_HOST') ssh_dir = os.path.expanduser('~/.ssh') if not os.path.exists(ssh_dir): os.makedirs(ssh_dir) ssh_key_pri_copied = os.path.join(ssh_dir, 'DT18_key_00') ssh_key_pub_copied = os.path.join(ssh_dir, 'DT18_key_00.pub') if not os.path.exists(ssh_key_pri_copied): shutil.copy(ssh_key_pri, ssh_key_pri_copied) if not os.path.exists(ssh_key_pub_copied): shutil.copy(ssh_key_pub, ssh_key_pub_copied) ssh_config = os.path.join(ssh_dir, 'config') if not os.path.exists(ssh_config): msg = ('Could not find ssh config file %s' % ssh_config) dtslogger.info(msg) current = "" else: current = open(ssh_config).read() bit0 = """ # --- init_sd_card generated --- # Use the key for all hosts IdentityFile $IDENTITY Host $HOSTNAME User $DTS_USERNAME Hostname $HOSTNAME.local IdentityFile $IDENTITY StrictHostKeyChecking no # ------------------------------ """ subs = dict(HOSTNAME=parsed.hostname, IDENTITY=ssh_key_pri_copied, DTS_USERNAME=parsed.linux_username) bit = Template(bit0).substitute(**subs) if not bit in current: dtslogger.info('Updating ~/.ssh/config with: ' + bit) with open(ssh_config, 'a') as f: f.write(bit) else: dtslogger.info('Configuration already found in ~/.ssh/config') ret = subprocess.call(script_cmd, shell=True, env=env, stdin=sys.stdin, stderr=sys.stderr, stdout=sys.stdout) if ret == 0: dtslogger.info('Done!') else: msg = ( 'An error occurred while initializing the SD card, please check and try again (%s).' % ret) raise Exception(msg)
def command(shell, args): parser = argparse.ArgumentParser() parser.add_argument('--steps', default="flash,expand,mount,setup,unmount", help="Steps to perform") parser.add_argument('--hostname', required=True) parser.add_argument('--linux-username', default='duckie') parser.add_argument('--linux-password', default='quackquack') parser.add_argument('--stacks-load', dest="stacks_to_load", default="DT18_00_basic,DT18_01_health_stats,DT18_02_others,DT18_03_roscore,DT18_05_duckiebot_base,DT18_06_dashboard", help="which stacks to load") parser.add_argument('--stacks-run', dest="stacks_to_run", default="DT18_00_basic,DT18_01_health_stats,DT18_03_roscore,DT18_06_dashboard", help="which stacks to RUN by default") parser.add_argument('--reset-cache', dest='reset_cache', default=False, action='store_true', help='Deletes the cached images') parser.add_argument('--compress', dest='compress', default=False, action='store_true', help='Compress the images - use if you have a 16GB SD card') parser.add_argument('--device', dest='device', default='', help='The device with the SD card') parser.add_argument('--aido', dest='aido', default=False, action='store_true', help='Only load what is necessary for an AI-DO submission') # parser.add_argument('--swap', default=False, action='store_true', # help='Create swap space') parser.add_argument('--country', default="US", help="2-letter country code (US, CA, CH, etc.)") parser.add_argument('--wifi', dest="wifi", default='duckietown:quackquack', help=""" Can specify one or more networks: "network:password,network:password,..." """) parser.add_argument('--ethz-username', default=None) parser.add_argument('--ethz-password', default=None) parser.add_argument('--experimental', dest='experimental', default=False, action='store_true', help='Use experimental settings') parser.add_argument('--configuration', dest='configuration', default=DEFAULT_CONFIGURATION, help='Which configuration of Docker stacks to flash') parser.add_argument('--type', dest='robot_type', default=None, choices=['duckiebot', 'watchtower'], help='Which type of robot we are setting up') parsed = parser.parse_args(args=args) global SD_CARD_DEVICE SD_CARD_DEVICE = parsed.device if parsed.reset_cache: dtslogger.info('Removing cache') if os.path.exists(DUCKIETOWN_TMP): shutil.rmtree(DUCKIETOWN_TMP) # if aido is set overwrite the stacks (don't load the base) if parsed.aido: parsed.stacks_to_load = 'DT18_00_basic,DT18_01_health_stats,DT18_03_roscore' parsed.stacks_to_run = parsed.stacks_to_load msg = """ ## Tips and tricks ### Multiple networks dts init_sd_card --wifi network1:password1,network2:password2 --country US ### Steps Without arguments the script performs the steps: flash expand mount setup unmount You can use --steps to run only some of those: dts init_sd_card --steps expand,mount """ print(msg) if 'DOCKER_HOST' in os.environ: msg = 'Removing DOCKER_HOST from os.environ.' dtslogger.info(msg) os.environ.pop('DOCKER_HOST') check_docker_environment() check_good_platform() check_dependencies() if parsed.experimental: dtslogger.info('Running experimental mode!') if parsed.robot_type is None: while True: r = input('You did not specify a robot type. Default is "{}". Do you confirm? [y]'.format(DEFAULT_ROBOT_TYPE)) if r.strip() in ['', 'y', 'Y', 'yes', 'YES', 'yup', 'YUP']: parsed.robot_type = DEFAULT_ROBOT_TYPE break; elif r.strip() in ['', 'n', 'N', 'no', 'NO', 'nope', 'NOPE']: dtslogger.info('Please retry while specifying a robot type. Bye bye!') exit(1) configuration = parsed.configuration try: get_resource(os.path.join('stacks', configuration)) except: msg = 'Cannot find configuration "%s"' % configuration raise InvalidUserInput(msg) dtslogger.info('Configuration: %s' % configuration) dtslogger.setLevel(logging.DEBUG) steps = parsed.steps.split(',') step2function = { 'flash': step_flash, 'expand': step_expand, 'mount': step_mount, 'setup': step_setup, 'unmount': step_unmount } for step_name in steps: if step_name not in step2function: msg = 'Cannot find step %r in %s' % (step_name, list(step2function)) raise InvalidUserInput(msg) step2function[step_name](shell, parsed)
def dt_challenges_evaluator(): from .col_logging import setup_logging setup_logging() elogger.info("dt-challenges-evaluator (DTC %s)" % __version__) elogger.info('called with:\n%s' % sys.argv) check_docker_environment() try: check_executable_exists('docker-compose') except InvalidEnvironment: msg = 'Could not find docker-compose. Please install it.' msg += '\n\nSee: https://docs.docker.com/compose/install/#install-compose' raise InvalidEnvironment(msg) usage = """ Usage: """ parser = argparse.ArgumentParser(usage=usage) parser.add_argument("--continuous", action="store_true", default=False) parser.add_argument("--no-pull", dest='no_pull', action="store_true", default=False) parser.add_argument("--no-upload", dest='no_upload', action="store_true", default=False) parser.add_argument("--no-delete", dest='no_delete', action="store_true", default=False) parser.add_argument("--machine-id", default=None, help='Machine name') parser.add_argument("--name", default=None, help='Evaluator name') parser.add_argument("--submission", default=None, help='evaluate this particular submission') parser.add_argument("--reset", dest='reset', action="store_true", default=False, help='Reset submission') parser.add_argument("--features", default='{}') parsed = parser.parse_args() tmpdir = '/tmp/duckietown/DT18/evaluator/executions' try: more_features = yaml.load(parsed.features) except BaseException as e: msg = 'Could not evaluate your YAML string %r:\n%s' % (parsed.features, e) raise Exception(msg) if not isinstance(more_features, dict): msg = 'I expected that the features are a dict; obtained %s: %r' % ( type(more_features).__name__, more_features) raise Exception(msg) do_pull = not parsed.no_pull do_upload = not parsed.no_upload delete = not parsed.no_delete reset = parsed.reset evaluator_name = parsed.name or 'p-%s' % os.getpid() machine_id = parsed.machine_id or socket.gethostname() args = dict(do_upload=do_upload, do_pull=do_pull, more_features=more_features, delete=delete, evaluator_name=evaluator_name, machine_id=machine_id, tmpdir=tmpdir) if parsed.continuous: timeout = 5.0 # seconds multiplier = 1.0 max_multiplier = 10 while True: multiplier = min(multiplier, max_multiplier) try: go_(None, reset=False, **args) multiplier = 1.0 except NothingLeft: sys.stderr.write('.') # time.sleep(timeout * multiplier) # elogger.info('No submissions available to evaluate.') except ConnectionError as e: elogger.error(e) multiplier *= 1.5 except BaseException as e: msg = 'Uncaught exception:\n%s' % traceback.format_exc(e) elogger.error(msg) multiplier *= 1.5 time.sleep(timeout * multiplier) else: if parsed.submission: submissions = [parsed.submission] else: submissions = [None] for submission_id in submissions: try: go_(submission_id, reset=reset, **args) except NothingLeft as e: if submission_id is None: msg = 'No submissions available to evaluate.' else: msg = 'Could not evaluate submission %s.' % submission_id msg += '\n' + str(e) elogger.error(msg)
def command(shell, args): prog = 'dts challenges evaluate' parser = argparse.ArgumentParser(prog=prog, usage=usage) group = parser.add_argument_group('Basic') group.add_argument('--no-cache', action='store_true', default=False, help="") group.add_argument('--no-build', action='store_true', default=False, help="") group.add_argument('--no-pull', action='store_true', default=False, help="") group.add_argument('--challenge', help="Specific challenge to evaluate") group.add_argument('--image', help="Evaluator image to run", default='duckietown/dt-challenges-evaluator:v4') group.add_argument('--shell', action='store_true', default=False, help="Runs a shell in the container") group.add_argument('--output', help="", default='output') group.add_argument('--visualize', help="Visualize the evaluation", action='store_true', default=False) parser.add_argument('--impersonate', type=str, default=None) group.add_argument('-C', dest='change', default=None) parsed = parser.parse_args(args) if parsed.change: os.chdir(parsed.change) client = check_docker_environment() command = ['dt-challenges-evaluate-local'] if parsed.no_cache: command.append('--no-cache') if parsed.no_build: command.append('--no-build') if parsed.challenge: command.extend(['--challenge', parsed.challenge]) if parsed.impersonate: command.extend(['--impersonate', parsed.impersonate]) output_rp = os.path.realpath(parsed.output) command.extend(['--output', parsed.output]) # # if parsed.features: # dtslogger.debug('Passing features %r' % parsed.features) # command += ['--features', parsed.features] # fake_dir = '/submission' tmpdir = '/tmp' UID = os.getuid() USERNAME = getpass.getuser() dir_home_guest = '/fake-home/%s' % USERNAME # os.path.expanduser('~') dir_fake_home_host = os.path.join(tmpdir, 'fake-%s-home' % USERNAME) if not os.path.exists(dir_fake_home_host): os.makedirs(dir_fake_home_host) dir_fake_home_guest = dir_home_guest dir_dtshell_host = os.path.join(os.path.expanduser('~'), '.dt-shell') dir_dtshell_guest = os.path.join(dir_fake_home_guest, '.dt-shell') dir_tmpdir_host = '/tmp' dir_tmpdir_guest = '/tmp' volumes = { '/var/run/docker.sock': { 'bind': '/var/run/docker.sock', 'mode': 'rw' } } d = os.path.join(os.getcwd(), parsed.output) if not os.path.exists(d): os.makedirs(d) volumes[output_rp] = {'bind': d, 'mode': 'rw'} volumes[os.getcwd()] = {'bind': os.getcwd(), 'mode': 'ro'} volumes[dir_tmpdir_host] = {'bind': dir_tmpdir_guest, 'mode': 'rw'} volumes[dir_dtshell_host] = {'bind': dir_dtshell_guest, 'mode': 'ro'} volumes[dir_fake_home_host] = { 'bind': dir_fake_home_guest, 'mode': 'rw' } volumes['/etc/group'] = {'bind': '/etc/group', 'mode': 'ro'} binds = [_['bind'] for _ in volumes.values()] for b1 in binds: for b2 in binds: if b1 == b2: continue if b1.startswith(b2): msg = 'Warning, it might be a problem to have binds with overlap' msg += '\n b1: %s' % b1 msg += '\n b2: %s' % b2 dtslogger.warn(msg) # command.extend(['-C', fake_dir]) env = {} extra_environment = dict(username=USERNAME, uid=UID, USER=USERNAME, HOME=dir_fake_home_guest) env.update(extra_environment) dtslogger.debug('Volumes:\n\n%s' % yaml.safe_dump(volumes, default_flow_style=False)) dtslogger.debug('Environment:\n\n%s' % yaml.safe_dump(env, default_flow_style=False)) from duckietown_challenges.rest import get_duckietown_server_url url = get_duckietown_server_url() dtslogger.info('The server URL is: %s' % url) if 'localhost' in url: h = socket.gethostname() replacement = h + '.local' dtslogger.warning( 'There is "localhost" inside, so I will try to change it to %r' % replacement) dtslogger.warning( 'This is because Docker cannot see the host as "localhost".') url = url.replace("localhost", replacement) dtslogger.warning('The new url is: %s' % url) dtslogger.warning( 'This will be passed to the evaluator in the Docker container.' ) env['DTSERVER'] = url container_name = 'local-evaluator' image = parsed.image name, tag = image.split(':') if not parsed.no_pull: dtslogger.info('Updating container %s' % image) dtslogger.info('This might take some time.') client.images.pull(name, tag) # try: container = client.containers.get(container_name) except: pass else: dtslogger.error('stopping previous %s' % container_name) container.stop() dtslogger.error('removing') container.remove() dtslogger.info('Starting container %s with %s' % (container_name, image)) detach = True env[DTShellConstants.DT1_TOKEN_CONFIG_KEY] = shell.get_dt1_token() dtslogger.info('Container command: %s' % " ".join(command)) # add all the groups on_mac = 'Darwin' in platform.system() if on_mac: group_add = [] else: group_add = [ g.gr_gid for g in grp.getgrall() if USERNAME in g.gr_mem ] interactive = False if parsed.shell: interactive = True detach = False command = ['/bin/bash', '-l'] params = dict(working_dir=os.getcwd(), user=UID, group_add=group_add, command=command, tty=interactive, volumes=volumes, environment=env, remove=True, network_mode='host', detach=detach, name=container_name) dtslogger.info('Parameters:\n%s' % json.dumps(params, indent=4)) client.containers.run(image, **params) if parsed.visualize: start_rqt_image_view() continuously_monitor(client, container_name)
def command(shell, args): prog = 'dts start_gui_tools DUCKIEBOT_NAME' usage = """ Keyboard control: %(prog)s """ parser = argparse.ArgumentParser(prog=prog, usage=usage) parser.add_argument('hostname', default=None, help='Name of the Duckiebot to calibrate') parser.add_argument('--network', default='host', help='Name of the network which to connect') parser.add_argument('--sim', action='store_true', default=False, help='are we running in simulator?') parser.add_argument( '--base_image', dest='image', default="duckietown/rpi-duckiebot-base:master19-no-arm", help="The base image, probably don't change the default") parsed_args = parser.parse_args(args) if parsed_args.sim: duckiebot_ip = "localhost" else: duckiebot_ip = get_duckiebot_ip( duckiebot_name=parsed_args.hostname) hostname = parsed_args.hostname image = parsed_args.image client = check_docker_environment() container_name = "interactive_gui_tools_%s" % hostname remove_if_running(client, container_name) if parsed_args.sim: env = { 'HOSTNAME': 'default', 'ROS_MASTER': hostname, 'DUCKIEBOT_NAME': hostname, 'ROS_MASTER_URI': 'http://%s:11311' % hostname } else: env = { 'HOSTNAME': hostname, 'ROS_MASTER': hostname, 'DUCKIEBOT_NAME': hostname, 'ROS_MASTER_URI': 'http://%s:11311' % duckiebot_ip } env['QT_X11_NO_MITSHM'] = 1 p = platform.system().lower() volumes = {'/tmp/.X11-unix': {'bind': '/tmp/.X11-unix', 'mode': 'rw'}} if 'darwin' in p: subprocess.call(["xhost", "+", '127.0.0.1']) env['DISPLAY'] = 'host.docker.internal:0' else: subprocess.call(["xhost", "+"]) env['DISPLAY'] = os.environ['DISPLAY'] dtslogger.info("Running %s on localhost with environment vars: %s" % (container_name, env)) cmd = "/bin/bash" params = { 'image': image, 'name': container_name, 'network_mode': parsed_args.network, 'environment': env, 'privileged': True, 'stdin_open': True, 'tty': True, 'detach': True, 'command': cmd, 'volumes': volumes } container = client.containers.run(**params) attach_cmd = 'docker attach %s' % container_name start_command_in_subprocess(attach_cmd)
def command(shell, args): check_docker_environment() home = os.path.expanduser('~') prog = 'dts challenges evaluator' parser = argparse.ArgumentParser(prog=prog, usage=usage) group = parser.add_argument_group('Basic') group.add_argument('--submission', type=int, default=None, help='Run a specific submission.') group.add_argument( '--reset', dest='reset', action='store_true', default=False, help='(needs --submission) Re-evaluate the specific submission.') group = parser.add_argument_group('Advanced') group.add_argument('--no-watchtower', dest='no_watchtower', action='store_true', default=False, help="Disable starting of watchtower") group.add_argument('--no-pull', dest='no_pull', action='store_true', default=False, help="Disable pulling of container") group.add_argument('--image', help="Evaluator image to run", default='duckietown/dt-challenges-evaluator:v3') group.add_argument('--name', default=None, help='Name for this evaluator') group.add_argument("--features", default=None, help="Pretend to be what you are not.") dtslogger.debug('args: %s' % args) parsed = parser.parse_args(args) machine_id = socket.gethostname() if parsed.name is None: container_name = '%s-%s' % (socket.gethostname(), os.getpid()) else: container_name = parsed.name import docker client = docker.from_env() command = ['dt-challenges-evaluator'] if parsed.submission: command += ['--submission', str(parsed.submission)] if parsed.reset: command += ['--reset'] else: command += ['--continuous'] command += ['--name', container_name] command += ['--machine-id', machine_id] if parsed.features: dtslogger.debug('Passing features %r' % parsed.features) command += ['--features', parsed.features] volumes = { '/var/run/docker.sock': { 'bind': '/var/run/docker.sock', 'mode': 'rw' }, os.path.join(home, '.dt-shell'): { 'bind': '/root/.dt-shell', 'mode': 'ro' }, '/tmp': { 'bind': '/tmp', 'mode': 'rw' } } env = {} UID = os.getuid() USERNAME = getpass.getuser() extra_environment = dict(username=USERNAME, uid=UID) env.update(extra_environment) if not parsed.no_watchtower: ensure_watchtower_active(client) url = get_duckietown_server_url() dtslogger.info('The server URL is: %s' % url) if 'localhost' in url: h = socket.gethostname() replacement = h + '.local' dtslogger.warning( 'There is "localhost" inside, so I will try to change it to %r' % replacement) dtslogger.warning( 'This is because Docker cannot see the host as "localhost".') url = url.replace("localhost", replacement) dtslogger.warning('The new url is: %s' % url) dtslogger.warning( 'This will be passed to the evaluator in the Docker container.' ) env['DTSERVER'] = url image = parsed.image name, tag = image.split(':') if not parsed.no_pull: dtslogger.info('Updating container %s' % image) client.images.pull(name, tag) try: container = client.containers.get(container_name) except: pass else: dtslogger.error('stopping previous %s' % container_name) container.stop() dtslogger.error('removing') container.remove() dtslogger.info('Starting container %s with %s' % (container_name, image)) dtslogger.info('Container command: %s' % " ".join(command)) client.containers.run(image, command=command, volumes=volumes, environment=env, network_mode='host', detach=True, name=container_name, tty=True) while True: try: container = client.containers.get(container_name) except Exception as e: msg = 'Cannot get container %s: %s' % (container_name, e) dtslogger.error(msg) dtslogger.info('Will wait.') time.sleep(5) continue dtslogger.info('status: %s' % container.status) if container.status == 'exited': msg = 'The container exited.' logs = '' for c in container.logs(stdout=True, stderr=True, stream=True): logs += c dtslogger.error(msg) tf = 'evaluator.log' with open(tf, 'w') as f: f.write(logs) msg = 'Logs saved at %s' % (tf) dtslogger.info(msg) break try: for c in container.logs(stdout=True, stderr=True, stream=True, follow=True): sys.stdout.write(c) time.sleep(3) except Exception as e: dtslogger.error(e) dtslogger.info('Will try to re-attach to container.') time.sleep(3) except KeyboardInterrupt: dtslogger.info('Received CTRL-C. Stopping container...') container.stop() dtslogger.info('Removing container') container.remove() dtslogger.info('Container removed.') break
def command(shell, args): parser = argparse.ArgumentParser() parser.add_argument('--steps', default="flash,expand,mount,setup,unmount", help="Steps to perform") parser.add_argument('--hostname', default='duckiebot') parser.add_argument('--linux-username', default='duckie') parser.add_argument('--linux-password', default='quackquack') parser.add_argument( '--stacks-load', dest="stacks_to_load", default= "DT18_00_basic,DT18_01_health_stats,DT18_02_others,DT18_05_duckiebot_base", help="which stacks to load") parser.add_argument('--stacks-run', dest="stacks_to_run", default="DT18_00_basic,DT18_01_health_stats", help="which stacks to RUN by default") parser.add_argument('--reset-cache', dest='reset_cache', default=False, action='store_true', help='Deletes the cached images') parser.add_argument( '--compress', dest='compress', default=False, action='store_true', help='Compress the images - use if you have a 16GB SD card') parser.add_argument('--device', dest='device', default='', help='The device with the SD card') # parser.add_argument('--swap', default=False, action='store_true', # help='Create swap space') parser.add_argument('--country', default="US", help="2-letter country code (US, CA, CH, etc.)") parser.add_argument('--wifi', dest="wifi", default='duckietown:quackquack', help=""" Can specify one or more networks: "network:password,network:password,..." """) parser.add_argument('--ethz-username', default=None) parser.add_argument('--ethz-password', default=None) parsed = parser.parse_args(args=args) global SD_CARD_DEVICE SD_CARD_DEVICE = parsed.device if parsed.reset_cache: dtslogger.info('Removing cache') if os.path.exists(DUCKIETOWN_TMP): shutil.rmtree(DUCKIETOWN_TMP) msg = """ ## Tips and tricks ### Multiple networks dts init_sd_card2 --wifi network1:password1,network2:password2 --country US ### Steps Without arguments the script performs the steps: flash expand mount setup unmount You can use --steps to run only some of those: dts init_sd_card2 --steps expand,mount """ print(msg) if 'DOCKER_HOST' in os.environ: msg = 'Removing DOCKER_HOST from os.environ.' dtslogger.info(msg) os.environ.pop('DOCKER_HOST') check_docker_environment() check_good_platform() check_dependencies() dtslogger.setLevel(logging.DEBUG) steps = parsed.steps.split(',') step2function = { 'flash': step_flash, 'expand': step_expand, 'mount': step_mount, 'setup': step_setup, 'unmount': step_unmount } for step_name in steps: if step_name not in step2function: msg = 'Cannot find step %r in %s' % (step_name, list(step2function)) raise InvalidUserInput(msg) step2function[step_name](shell, parsed)