def command(shell, args):
token = shell.get_dt1_token()
parser = argparse.ArgumentParser()
parser.add_argument('--config',
default='challenge.yaml',
help="YAML configuration file")
parser.add_argument('--no-cache', default=False, action='store_true')
parser.add_argument('--steps',
default=None,
help='Which steps (comma separated)')
parser.add_argument('--force-invalidate-subs',
default=False,
action='store_true')
parser.add_argument('-C',
dest='cwd',
default=None,
help='Base directory')
parser.add_argument('--impersonate', type=str, default=None)
parsed = parser.parse_args(args)
impersonate = parsed.impersonate
from dt_shell.env_checks import check_docker_environment
client = check_docker_environment()
if client is None: # To remove when done
client = check_docker_environment()
if parsed.cwd is not None:
dtslogger.info('Changing to directory %s' % parsed.cwd)
os.chdir(parsed.cwd)
no_cache = parsed.no_cache
fn = os.path.join(parsed.config)
if not os.path.exists(fn):
msg = 'File %s does not exist.' % fn
raise UserError(msg)
data = read_yaml_file(fn)
if 'description' not in data or data['description'] is None:
fnd = os.path.join(os.path.dirname(fn), 'challenge.description.md')
if os.path.exists(fnd):
desc = open(fnd).read()
data['description'] = desc
msg = 'Read description from %s' % fnd
dtslogger.info(msg)
base = os.path.dirname(fn)
challenge = ChallengeDescription.from_yaml(data)
with wrap_server_operations():
go(token, impersonate, parsed, challenge, base, client, no_cache)
def start_gui_tools(duckiebot_name):
duckiebot_ip = get_duckiebot_ip(duckiebot_name)
local_client = check_docker_environment()
operating_system = platform.system()
local_client.images.pull(RPI_GUI_TOOLS)
env_vars = default_env(duckiebot_name, duckiebot_ip)
env_vars['DISPLAY'] = True
container_name = 'gui-tools-interactive'
if operating_system == 'Linux':
subprocess.call(["xhost", "+"])
local_client.containers.run(image=RPI_GUI_TOOLS,
network_mode='host',
privileged=True,
tty=True,
name=container_name,
environment=env_vars)
elif operating_system == 'Darwin':
IP = subprocess.check_output([
'/bin/sh', '-c',
'ifconfig en0 | grep inet | awk \'$1=="inet" {print $2}\''
])
env_vars['IP'] = IP
subprocess.call(["xhost", "+IP"])
local_client.containers.run(image=RPI_GUI_TOOLS,
network_mode='host',
privileged=True,
tty=True,
name=container_name,
environment=env_vars)
attach_terminal(container_name)
def save_images(stack2yaml, compress):
"""
returns stack2info
"""
cache_dir = DOCKER_IMAGES_CACHE_DIR
if not os.path.exists(cache_dir):
os.makedirs(cache_dir)
client = check_docker_environment()
stack2info = {}
for cf, config in stack2yaml.items():
image_name2id = {}
for service, service_config in config['services'].items():
image_name = service_config['image']
dtslogger.info('Pulling %s' % image_name)
cmd = ['docker', 'pull', image_name]
_run_cmd(cmd)
image = client.images.get(image_name)
image_id = str(image.id)
image_name2id[image_name] = image_id
hname = get_md5("-".join(sorted(list(image_name2id.values()))))[:8]
if compress:
destination = os.path.join(cache_dir, cf + '-' + hname + '.tar.gz')
else:
destination = os.path.join(cache_dir, cf + '-' + hname + '.tar')
destination0 = os.path.join(cache_dir, cf + '-' + hname + '.tar')
stack2info[cf] = StackInfo(archive=destination, image_name2id=image_name2id,
hname=hname)
if os.path.exists(destination):
msg = 'Using cached file %s' % destination
dtslogger.info(msg)
continue
dtslogger.info('Saving to %s' % destination0)
cmd = ['docker', 'save', '-o', destination0] + list(image_name2id.values())
_run_cmd(cmd)
if compress:
cmd = ['gzip', '-f', destination0]
_run_cmd(cmd)
assert not os.path.exists(destination0)
assert os.path.exists(destination)
else:
assert destination == destination0
msg = 'Saved archive %s of size %s' % (destination, friendly_size_file(destination))
dtslogger.info(msg)
assert len(stack2info) == len(stack2yaml)
return stack2info
def run_gui_controller(hostname, image):
client = check_docker_environment()
container_name = "joystick_gui_%s" % hostname
remove_if_running(client, container_name)
duckiebot_ip = get_duckiebot_ip(hostname)
env = {
'HOSTNAME': hostname,
'ROS_MASTER': hostname,
'DUCKIEBOT_NAME': hostname,
'ROS_MASTER_URI': 'http://%s:11311' % duckiebot_ip
}
env['QT_X11_NO_MITSHM'] = 1
volumes = {}
subprocess.call(["xhost", "+"])
p = platform.system().lower()
if 'darwin' in p:
dtslogger.warn(
"We can try but running the joystick gui on MacOSx is not expected to work..."
)
env['DISPLAY'] = 'host.docker.internal:0'
volumes = {'/tmp/.X11-unix': {'bind': '/tmp/.X11-unix', 'mode': 'rw'}}
else:
env['DISPLAY'] = os.environ['DISPLAY']
dtslogger.info("Running %s on localhost with environment vars: %s" %
(container_name, env))
if 'master19' in image:
image = "duckietown/rpi-duckiebot-base:master19-no-arm"
cmd = "python misc/virtualJoy/virtualJoy.py %s" % hostname
elif 'daffy' in image:
image = "duckietown/dt-core:daffy-amd64"
cmd = "roslaunch virtual_joystick virtual_joystick_gui.launch veh:=%s" % hostname
params = {
'image': image,
'name': container_name,
'network_mode': 'host',
'environment': env,
'privileged': True,
'stdin_open': True,
'tty': True,
'command': cmd,
'detach': True,
'volumes': volumes
}
container = client.containers.run(**params)
cmd = 'docker attach %s' % container_name
start_command_in_subprocess(cmd)
def run_cli_controller(hostname, image, duckiebot_ip, network_mode, sim):
if sim:
duckiebot_client = check_docker_environment()
else:
duckiebot_client = docker.DockerClient('tcp://' + duckiebot_ip +
':2375')
container_name = "joystick_cli_%s" % hostname
remove_if_running(duckiebot_client, container_name)
env = {
'HOSTNAME': hostname,
'ROS_MASTER': hostname,
'VEHICLE_NAME': hostname,
'ROS_MASTER_URI': 'http://%s:11311' % duckiebot_ip
}
dtslogger.info("Running %s on localhost with environment vars: %s" %
(container_name, env))
if 'master19' in image:
image = "duckietown/rpi-duckiebot-base:master19" # run on robot
cmd = "python misc/virtualJoy/joy_cli.py %s" % hostname
elif 'daffy' in image:
if sim:
image = "duckietown/dt-core:daffy-amd64"
else:
image = "duckietown/dt-core:daffy"
cmd = "roslaunch virtual_joystick virtual_joystick_cli.launch veh:=%s" % hostname
params = {
'image': image,
'name': container_name,
'network_mode': network_mode,
'environment': env,
'privileged': True,
'stdin_open': True,
'tty': True,
'command': cmd,
'detach': True
}
container = duckiebot_client.containers.run(**params)
if sim:
cmd = 'docker attach %s' % container_name
else:
cmd = 'docker -H %s.local attach %s' % (hostname, container_name)
start_command_in_subprocess(cmd)
def record_bag(duckiebot_name, duration):
duckiebot_ip = get_duckiebot_ip(duckiebot_name)
local_client = check_docker_environment()
dtslogger.info("Starting bag recording...")
parameters = {
'image': RPI_DUCKIEBOT_BASE,
'remove': True,
'network_mode': 'host',
'privileged': True,
'detach': True,
'environment': default_env(duckiebot_name, duckiebot_ip),
'command':
'bash -c "cd /data && rosbag record --duration %s -a"' % duration,
'volumes': bind_local_data_dir()
}
# Mac Docker has ARM support directly in the Docker environment, so we don't need to run qemu...
if platform.system() != 'Darwin':
parameters['entrypoint'] = 'qemu3-arm-static'
return local_client.containers.run(**parameters)
def run_image_on_localhost(image_name,
duckiebot_name,
container_name,
env=None,
volumes=None):
duckiebot_ip = get_duckiebot_ip(duckiebot_name)
local_client = check_docker_environment()
env_vars = default_env(duckiebot_name, duckiebot_ip)
if env is not None:
env_vars.update(env)
try:
container = local_client.containers.get(container_name)
dtslogger.info("an image already on localhost - stopping it first..")
stop_container(container)
remove_container(container)
except Exception as e:
dtslogger.warn("coulgn't remove existing container: %s" % e)
dtslogger.info("Running %s on localhost with environment vars: %s" %
(image_name, env_vars))
params = {
'image': image_name,
'remove': True,
'network_mode': 'host',
'privileged': True,
'detach': True,
'tty': True,
'name': container_name,
'environment': env_vars
}
if volumes is not None:
params['volumes'] = volumes
new_local_container = local_client.containers.run(**params)
return new_local_container
def start_rqt_image_view(duckiebot_name=None):
dtslogger.info(
"""{}\nOpening a camera feed by running xhost+ and running rqt_image_view..."""
.format('*' * 20))
local_client = check_docker_environment()
local_client.images.pull(RPI_GUI_TOOLS)
env_vars = {'QT_X11_NO_MITSHM': 1}
if duckiebot_name is not None:
duckiebot_ip = get_duckiebot_ip(duckiebot_name)
env_vars.update(default_env(duckiebot_name, duckiebot_ip))
operating_system = platform.system()
if operating_system == 'Linux':
subprocess.call(["xhost", "+"])
env_vars['DISPLAY'] = ':0'
elif operating_system == 'Darwin':
IP = subprocess.check_output([
'/bin/sh', '-c',
'ifconfig en0 | grep inet | awk \'$1=="inet" {print $2}\''
])
env_vars['IP'] = IP
subprocess.call(["xhost", "+IP"])
dtslogger.info("Running %s on localhost with environment vars: %s" %
(RPI_GUI_TOOLS, env_vars))
return local_client.containers.run(
image=RPI_GUI_TOOLS,
remove=True,
privileged=True,
detach=True,
network_mode='host',
environment=env_vars,
command=
'bash -c "source /home/software/docker/env.sh && rqt_image_view"')
def command(shell, args):
check_docker_environment()
home = os.path.expanduser('~')
prog = 'dts challenges evaluator'
parser = argparse.ArgumentParser(prog=prog, usage=usage)
group = parser.add_argument_group('Basic')
group.add_argument('--submission', type=int, default=None,
help='Run a specific submission.')
group.add_argument('--reset', dest='reset', action='store_true', default=False,
help='(needs --submission) Re-evaluate the specific submission.')
group = parser.add_argument_group('Advanced')
group.add_argument('--no-watchtower', dest='no_watchtower', action='store_true', default=False,
help="Disable starting of watchtower")
group.add_argument('--no-pull', dest='no_pull', action='store_true', default=False,
help="Disable pulling of containers")
group.add_argument('--no-upload', dest='no_upload', action='store_true', default=False,
help="Disable upload of artifacts")
group.add_argument('--no-delete', dest='no_delete', action='store_true', default=False,
help="Does not erase temporary files in /tmp/duckietown")
group.add_argument('--image', help="Evaluator image to run", default='duckietown/dt-challenges-evaluator:v4')
group.add_argument('--name', default=None, help='Name for this evaluator')
group.add_argument("--features", default=None, help="Pretend to be what you are not.")
group.add_argument("--ipfs", action='store_true', default=False, help="Run with IPFS available")
group.add_argument("--one", action='store_true', default=False, help="Only run 1 submission")
# dtslogger.debug('args: %s' % args)
parsed = parser.parse_args(args)
machine_id = socket.gethostname()
if parsed.name is None:
container_name = '%s-%s' % (socket.gethostname(), os.getpid())
else:
container_name = parsed.name
client = check_docker_environment()
command = ['dt-challenges-evaluator']
if parsed.submission:
command += ['--submission', str(parsed.submission)]
if parsed.reset:
command += ['--reset']
else:
if not parsed.one:
command += ['--continuous']
command += ['--name', container_name]
command += ['--machine-id', machine_id]
if parsed.no_upload:
command += ['--no-upload']
if parsed.no_pull:
command += ['--no-pull']
if parsed.no_delete:
command += ['--no-delete']
if parsed.one:
command += ['--one']
if parsed.features:
dtslogger.debug('Passing features %r' % parsed.features)
command += ['--features', parsed.features]
mounts = []
volumes = {
'/var/run/docker.sock': {'bind': '/var/run/docker.sock', 'mode': 'rw'},
os.path.join(home, '.dt-shell'): {'bind': '/root/.dt-shell', 'mode': 'ro'},
'/tmp': {'bind': '/tmp', 'mode': 'rw'}
}
if parsed.ipfs:
if not ipfs_available():
msg = 'IPFS not available/mounted correctly.'
raise UserError(msg)
command += ['--ipfs']
# volumes['/ipfs'] = {'bind': '/ipfs', 'mode': 'ro'}
from docker.types import Mount
mount = Mount(type='bind', source='/ipfs', target='/ipfs', read_only=True)
mounts.append(mount)
env = {}
UID = os.getuid()
USERNAME = getpass.getuser()
extra_environment = dict(username=USERNAME, uid=UID)
env.update(extra_environment)
if not parsed.no_watchtower:
ensure_watchtower_active(client)
from duckietown_challenges.rest import get_duckietown_server_url
url = get_duckietown_server_url()
dtslogger.info('The server URL is: %s' % url)
if 'localhost' in url:
h = socket.gethostname()
replacement = h + '.local'
dtslogger.warning('There is "localhost" inside, so I will try to change it to %r' % replacement)
dtslogger.warning('This is because Docker cannot see the host as "localhost".')
url = url.replace("localhost", replacement)
dtslogger.warning('The new url is: %s' % url)
dtslogger.warning('This will be passed to the evaluator in the Docker container.')
env['DTSERVER'] = url
image = parsed.image
dtslogger.info('Using evaluator image %s' % image)
name, tag = image.split(':')
if not parsed.no_pull:
dtslogger.info('Updating container %s' % image)
client.images.pull(name, tag)
# noinspection PyBroadException
try:
container = client.containers.get(container_name)
except:
pass
else:
dtslogger.error('stopping previous %s' % container_name)
container.stop()
dtslogger.error('removing')
container.remove()
dtslogger.info('Starting container %s with %s' % (container_name, image))
dtslogger.info('Container command: %s' % " ".join(command))
# add all the groups
import grp
group_add = [g.gr_gid for g in grp.getgrall() if USERNAME in g.gr_mem]
client.containers.run(image,
group_add=group_add,
command=command,
volumes=volumes,
environment=env,
mounts=mounts,
network_mode='host',
detach=True,
name=container_name,
tty=True)
last_log_timestamp = None
while True:
try:
container = client.containers.get(container_name)
except Exception as e:
msg = 'Cannot get container %s: %s' % (container_name, e)
dtslogger.error(msg)
dtslogger.info('Will wait.')
time.sleep(5)
continue
dtslogger.info('status: %s' % container.status)
if container.status == 'exited':
logs = ''
for c in container.logs(stdout=True, stderr=True, stream=True, since=last_log_timestamp):
logs += c.decode('utf-8')
last_log_timestamp = datetime.datetime.now()
tf = 'evaluator.log'
with open(tf, 'w') as f:
f.write(logs)
msg = 'The container exited.'
msg += '\nLogs saved at %s' % tf
dtslogger.info(msg)
break
try:
if last_log_timestamp is not None:
print('since: %s' % last_log_timestamp.isoformat())
for c0 in container.logs(stdout=True, stderr=True, stream=True, # follow=True,
since=last_log_timestamp, tail=0):
c: bytes = c0
try:
s = c.decode('utf-8')
except:
s = c.decode('utf-8', errors='replace')
sys.stdout.write(s)
last_log_timestamp = datetime.datetime.now()
time.sleep(3)
except KeyboardInterrupt:
dtslogger.info('Received CTRL-C. Stopping container...')
container.stop()
dtslogger.info('Removing container')
container.remove()
dtslogger.info('Container removed.')
break
except BaseException:
s = traceback.format_exc()
if 'Read timed out' in s:
dtslogger.debug('(reattaching)')
else:
dtslogger.error(s)
dtslogger.info('Will try to re-attach to container.')
time.sleep(3)
def command(shell, args):
prog = 'dts duckiebot demo'
parser = argparse.ArgumentParser(prog=prog, usage=usage)
parser.add_argument('--demo_name',
dest="demo_name",
default=None,
help="Name of the demo to run")
parser.add_argument(
'--duckiebot_name',
dest="duckiebot_name",
default=None,
help="Name of the Duckiebot on which to run the demo")
parser.add_argument(
'--package_name',
dest="package_name",
default="duckietown",
help=
"You can specify the package that you want to use to look for launch files"
)
parser.add_argument(
'--image',
dest="image_to_run",
default="duckietown/rpi-duckiebot-base:master19",
help="Docker image to use, you probably don't need to change ",
)
parser.add_argument('--debug',
action='store_true',
default=False,
help="will enter you into the running container")
parsed = parser.parse_args(args)
check_docker_environment()
demo_name = parsed.demo_name
if demo_name is None:
msg = 'You must specify a demo_name'
raise InvalidUserInput(msg)
duckiebot_name = parsed.duckiebot_name
if duckiebot_name is None:
msg = 'You must specify a duckiebot_name'
raise InvalidUserInput(msg)
package_name = parsed.package_name
dtslogger.info("Using package %s" % package_name)
duckiebot_ip = get_duckiebot_ip(duckiebot_name)
duckiebot_client = docker.DockerClient('tcp://' + duckiebot_ip +
':2375')
container_name = 'demo_%s' % demo_name
remove_if_running(duckiebot_client, container_name)
image_base = parsed.image_to_run
env_vars = default_env(duckiebot_name, duckiebot_ip)
env_vars.update({'VEHICLE_NAME': duckiebot_name})
if demo_name == 'base':
cmd = '/bin/bash'
else:
cmd = 'roslaunch %s %s.launch veh:=%s' % (package_name, demo_name,
duckiebot_name)
dtslogger.info("Running command %s" % cmd)
demo_container = duckiebot_client.containers.run(
image=image_base,
command=cmd,
network_mode='host',
volumes=bind_duckiebot_data_dir(),
privileged=True,
name=container_name,
mem_limit='800m',
memswap_limit='2800m',
stdin_open=True,
tty=True,
detach=True,
environment=env_vars)
if demo_name == 'base' or parsed.debug:
attach_cmd = 'docker -H %s.local attach %s' % (duckiebot_name,
container_name)
start_command_in_subprocess(attach_cmd)
def command(shell, args):
prog = 'dts duckiebot calibrate_intrinsics DUCKIEBOT_NAME'
usage = """
Calibrate:
%(prog)s
"""
parser = argparse.ArgumentParser(prog=prog, usage=usage)
parser.add_argument('hostname', default=None, help='Name of the Duckiebot to calibrate')
parser.add_argument('--base_image', dest='image',
default="duckietown/rpi-duckiebot-base:master19-no-arm")
parser.add_argument('--debug', action='store_true', default=False,
help="will enter you into the running container")
parsed_args = parser.parse_args(args)
hostname = parsed_args.hostname
duckiebot_ip = get_duckiebot_ip(hostname)
duckiebot_client = get_remote_client(duckiebot_ip)
# is the interface running?
try:
duckiebot_containers = duckiebot_client.containers.list()
interface_container_found = False
for c in duckiebot_containers:
if 'duckiebot-interface' in c.name:
interface_container_found = True
if not interface_container_found:
dtslogger.error("The duckiebot-interface is not running on the duckiebot")
exit()
except Exception as e:
dtslogger.warn(
"Not sure if the duckiebot-interface is running because we got and exception when trying: %s" % e)
# is the raw imagery being published?
try:
duckiebot_containers = duckiebot_client.containers.list()
raw_imagery_found = False
for c in duckiebot_containers:
if 'demo_camera' in c.name:
raw_imagery_found = True
if not raw_imagery_found:
dtslogger.error("The demo_camera is not running on the duckiebot - please run `dts duckiebot demo --demo_name camera --duckiebot_name %s" % hostname)
exit()
except Exception as e:
dtslogger.warn("%s" % e)
image = parsed_args.image
client = check_docker_environment()
container_name = "intrinsic_calibration_%s" % hostname
remove_if_running(client,container_name)
env = {'HOSTNAME': hostname,
'ROS_MASTER': hostname,
'DUCKIEBOT_NAME': hostname,
'ROS_MASTER_URI': 'http://%s:11311' % duckiebot_ip,
'QT_X11_NO_MITSHM': 1}
volumes = {}
subprocess.call(["xhost", "+"])
p = platform.system().lower()
if 'darwin' in p:
env['DISPLAY'] = 'host.docker.internal:0'
volumes = {
'/tmp/.X11-unix': {'bind': '/tmp/.X11-unix', 'mode': 'rw'}
}
else:
env['DISPLAY'] = os.environ['DISPLAY']
dtslogger.info("Running %s on localhost with environment vars: %s" %
(container_name, env))
dtslogger.info("When the window opens you will need to move the checkerboard around in front of the Duckiebot camera")
cmd = "roslaunch pi_camera intrinsic_calibration.launch veh:=%s" % hostname
params = {'image': image,
'name': container_name,
'network_mode': 'host',
'environment': env,
'privileged': True,
'stdin_open': True,
'tty': True,
'detach': True,
'command': cmd,
'volumes': volumes
}
container = client.containers.run(**params)
if parsed_args.debug:
attach_cmd = 'docker attach %s' % (container_name)
start_command_in_subprocess(attach_cmd)
def command(shell, args):
prog = 'dts duckiebot evaluate'
parser = argparse.ArgumentParser(prog=prog, usage=usage)
group = parser.add_argument_group('Basic')
group.add_argument('--duckiebot_name', default=None,
help="Name of the Duckiebot on which to perform evaluation")
group.add_argument('--duckiebot_username', default="duckie",
help="The duckiebot username")
group.add_argument('--image', dest='image_name',
help="Image to evaluate, if none specified then we will build your current context",
default=None)
group.add_argument('--glue_node_image', default="duckietown/challenge-aido_lf-duckiebot:aido2",
help="The node that glues your submission with ROS on the duckiebot. Probably don't change")
group.add_argument('--duration', help="Number of seconds to run evaluation", default=15)
group.add_argument('--remotely', action='store_true', default=True,
help="If true run the image over network without pushing to Duckiebot")
group.add_argument('--no_cache', help="disable cache on docker build",
action="store_true", default=False)
group.add_argument('--record_bag', action='store_true', default=False,
help="If true record a rosbag")
group.add_argument("--debug", action='store_true', default=False,
help="If true you will get a shell instead of executing")
group.add_argument('--max_vel', help="the max velocity for the duckiebot", default=0.7)
group.add_argument('--challenge', help="Specific challenge to evaluate")
parsed = parser.parse_args(args)
tmpdir = '/tmp'
USERNAME = getpass.getuser()
dir_home_guest = os.path.expanduser('~')
dir_fake_home = os.path.join(tmpdir, 'fake-%s-home' % USERNAME)
if not os.path.exists(dir_fake_home):
os.makedirs(dir_fake_home)
get_calibration_files(dir_fake_home, parsed.duckiebot_username, parsed.duckiebot_name)
client = check_docker_environment()
agent_container_name = "agent"
glue_container_name = "aido_glue"
# remove the containers if they are already running
remove_if_running(client, agent_container_name)
remove_if_running(client, glue_container_name)
# setup the fifos2 volume (requires pruning it if it's still hanging around from last time)
try:
client.volumes.prune()
fifo2_volume=client.volumes.create(name='fifos2')
except Exception as e:
dtslogger.warn("error creating volume: %s" % e)
duckiebot_ip = get_duckiebot_ip(parsed.duckiebot_name)
duckiebot_client = get_remote_client(duckiebot_ip)
try:
duckiebot_containers = duckiebot_client.containers.list()
interface_container_found=False
for c in duckiebot_containers:
if 'duckiebot-interface' in c.name:
interface_container_found=True
if not interface_container_found:
dtslogger.error("The duckiebot-interface is not running on the duckiebot")
except Exception as e:
dtslogger.warn("Not sure if the duckiebot-interface is running because we got and exception when trying: %s" % e)
# let's start building stuff for the "glue" node
glue_volumes = {fifo2_volume.name: {'bind': '/fifos', 'mode': 'rw'}}
glue_env = {'HOSTNAME':parsed.duckiebot_name,
'DUCKIEBOT_NAME':parsed.duckiebot_name,
'ROS_MASTER_URI':'http://%s:11311' % duckiebot_ip}
dtslogger.info("Running %s on localhost with environment vars: %s" %
(parsed.glue_node_image, glue_env))
params = {'image': parsed.glue_node_image,
'name': glue_container_name,
'network_mode': 'host',
'privileged': True,
'environment': glue_env,
'detach': True,
'tty': True,
'volumes': glue_volumes}
# run the glue container
glue_container = client.containers.run(**params)
if not parsed.debug:
monitor_thread = threading.Thread(target=continuously_monitor, args=(client, glue_container.name))
monitor_thread.start()
if parsed.image_name is None:
# if we didn't get an `image_name` we try need to build the local container
path = '.'
dockerfile = os.path.join(path, 'Dockerfile')
if not os.path.exists(dockerfile):
msg = 'No Dockerfile'
raise Exception(msg)
tag='myimage'
if parsed.no_cache:
cmd = ['docker', 'build', '--no-cache', '-t', tag, '-f', dockerfile]
else:
cmd = ['docker', 'build', '-t', tag, '-f', dockerfile]
dtslogger.info("Running command: %s" % cmd)
cmd.append(path)
subprocess.check_call(cmd)
image_name = tag
else:
image_name = parsed.image_name
# start to build the agent stuff
agent_env = {'AIDONODE_DATA_IN':'/fifos/agent-in',
'AIDONODE_DATA_OUT':'fifo:/fifos/agent-out'}
agent_volumes = {fifo2_volume.name: {'bind': '/fifos', 'mode': 'rw'},
dir_fake_home: {'bind': '/data/config', 'mode': 'rw'}
}
params = {'image': image_name,
'remove': True,
'name': agent_container_name,
'environment': agent_env,
'detach': True,
'tty': True,
'volumes': agent_volumes}
if parsed.debug:
params['command'] = '/bin/bash'
params['stdin_open'] = True
dtslogger.info("Running %s on localhost with environment vars: %s" % (image_name, agent_env))
agent_container = client.containers.run(**params)
if parsed.debug:
attach_cmd = 'docker attach %s' % agent_container_name
start_command_in_subprocess(attach_cmd)
else:
monitor_thread = threading.Thread(target=continuously_monitor,args=(client, agent_container_name))
monitor_thread.start()
duration = int(parsed.duration)
# should we record a bag?
if parsed.record_bag:
bag_container = record_bag(parsed.hostname, duration)
dtslogger.info("Running for %d s" % duration)
time.sleep(duration)
stop_container(glue_container)
stop_container(agent_container)
if parsed.record_bag:
stop_container(bag_container)
def command(shell, args):
parser = argparse.ArgumentParser()
parser.add_argument(
'--image',
default=
'andreacensi/mcdp_books:duckuments@sha256:ae2fcdbb8ce409e4817ed74c67b04bb91cd14ca96bed887e75e5275fa2efc933',
help="Which image to use")
parsed = parser.parse_args(args=args)
image = parsed.image
check_docker_environment()
# check_git_supports_superproject()
from system_cmd import system_cmd_result
pwd = os.getcwd()
bookdir = os.path.join(pwd, 'book')
if not os.path.exists(bookdir):
msg = 'Could not find "book" directory %r.' % bookdir
DTCommandAbs.fail(msg)
# check that the resources directory is present
resources = os.path.join(pwd, 'resources')
if not os.path.exists(os.path.join(resources, 'templates')):
msg = 'It looks like that the "resources" repo is not checked out.'
msg += '\nMaybe try:\n'
msg += '\n git submodule init'
msg += '\n git submodule update'
raise Exception(msg) # XXX
entries = list(os.listdir(bookdir))
entries = [_ for _ in entries if not _[0] == '.']
if len(entries) > 1:
msg = 'Found more than one directory in "book": %s' % entries
DTCommandAbs.fail(msg)
bookname = entries[0]
src = os.path.join(bookdir, bookname)
res = system_cmd_result(pwd, ['git', '--version'], raise_on_error=True)
git_version = res.stdout
print('git version: %s' % git_version)
cmd = ['git', 'rev-parse', '--show-superproject-working-tree']
res = system_cmd_result(pwd, cmd, raise_on_error=True)
gitdir_super = res.stdout.strip()
if '--show' in gitdir_super or not gitdir_super:
msg = "Your git version is too low, as it does not support --show-superproject-working-tree"
msg += '\n\nDetected: %s' % git_version
raise InvalidEnvironment(msg)
print('gitdir_super: %r' % gitdir_super)
res = system_cmd_result(pwd, ['git', 'rev-parse', '--show-toplevel'],
raise_on_error=True)
gitdir = res.stdout.strip()
if '--show' in gitdir or not gitdir:
msg = "Your git version is too low, as it does not support --show-toplevel"
msg += '\n\nDetected: %s' % git_version
raise InvalidEnvironment(msg)
print('gitdir: %r' % gitdir)
pwd1 = os.path.realpath(pwd)
user = getpass.getuser()
tmpdir = '/tmp'
fake_home = os.path.join(tmpdir, 'fake-%s-home' % user)
if not os.path.exists(fake_home):
os.makedirs(fake_home)
resources = 'resources'
uid1 = os.getuid()
if sys.platform == 'darwin':
flag = ':delegated'
else:
flag = ''
cmd = [
'docker', 'run', '-v',
'%s:%s%s' % (gitdir, gitdir, flag), '-v',
'%s:%s%s' % (gitdir_super, gitdir_super, flag), '-v',
'%s:%s%s' % (pwd1, pwd1, flag), '-v',
'%s:%s%s' % (fake_home, '/home/%s' % user, flag), '-e',
'USER=%s' % user, '-e',
'USERID=%s' % uid1, '-m', '4GB', '--user',
'%s' % uid1
]
interactive = True
if interactive:
cmd.append('-it')
cmd += [
image, '/project/run-book-native.sh', bookname, src, resources,
pwd1
]
print('executing:\nls ' + " ".join(cmd))
# res = system_cmd_result(pwd, cmd, raise_on_error=True)
try:
p = subprocess.Popen(cmd,
bufsize=0,
executable=None,
stdin=None,
stdout=None,
stderr=None,
preexec_fn=None,
shell=False,
cwd=pwd,
env=None)
except OSError as e:
if e.errno == 2:
msg = 'Could not find "docker" executable.'
DTCommandAbs.fail(msg)
raise
p.communicate()
print('\n\nCompleted.')
def configure_images(parsed, user_data, add_file_local, add_file):
import psutil
# read and validate duckiebot-compose
stacks_to_load = parsed.stacks_to_load.split(',')
stacks_to_run = parsed.stacks_to_run.split(',')
dtslogger.info('Stacks to load: %s' % stacks_to_load)
dtslogger.info('Stacks to run: %s' % stacks_to_run)
for _ in stacks_to_run:
if _ not in stacks_to_load:
msg = 'If you want to run %r you need to load it as well.' % _
raise Exception(msg)
configuration = parsed.configuration
for cf in stacks_to_load:
# local path
lpath = get_resource(os.path.join('stacks', configuration, cf + '.yaml'))
# path on PI
rpath = '/var/local/%s.yaml' % cf
if which('docker-compose') is None:
msg = 'Could not find docker-compose. Cannot validate file.'
dtslogger.error(msg)
else:
_run_cmd(['docker-compose', '-f', lpath, 'config', '--quiet'])
add_file_local(path=rpath, local=lpath)
stack2yaml = get_stack2yaml(stacks_to_load, get_resource(os.path.join('stacks', configuration)))
if not stack2yaml:
msg = 'Not even one stack specified'
raise Exception(msg)
stack2info = save_images(stack2yaml, compress=parsed.compress)
buffer_bytes = 100 * 1024 * 1024
stacks_written = []
stack2archive_rpath = {}
dtslogger.debug(stack2info)
for stack, stack_info in stack2info.items():
tgz = stack_info.archive
size = os.stat(tgz).st_size
dtslogger.info('Considering copying %s of size %s' % (tgz, friendly_size_file(tgz)))
rpath = os.path.join('var', 'local', os.path.basename(tgz))
destination = os.path.join(TMP_ROOT_MOUNTPOINT, rpath)
available = psutil.disk_usage(TMP_ROOT_MOUNTPOINT).free
dtslogger.info('available %s' % friendly_size(available))
if available < size + buffer_bytes:
msg = 'You have %s available on %s but need %s for %s' % (
friendly_size(available), TMP_ROOT_MOUNTPOINT, friendly_size_file(tgz), tgz)
dtslogger.info(msg)
continue
dtslogger.info('OK, copying, and loading it on first boot.')
if os.path.exists(destination):
msg = 'Skipping copying image that already exist at %s.' % destination
dtslogger.info(msg)
else:
if which('rsync'):
cmd = ['sudo', 'rsync', '-avP', tgz, destination]
else:
cmd = ['sudo', 'cp', tgz, destination]
_run_cmd(cmd)
sync_data()
stack2archive_rpath[stack] = os.path.join('/', rpath)
stacks_written.append(stack)
client = check_docker_environment()
stacks_not_to_run = [_ for _ in stacks_to_load if _ not in stacks_to_run]
order = stacks_to_run + stacks_not_to_run
for cf in order:
if cf in stacks_written:
log_current_phase(user_data, PHASE_LOADING,
"Stack %s: Loading containers" % cf)
cmd = 'docker load --input %s && rm %s' % (stack2archive_rpath[cf], stack2archive_rpath[cf])
add_run_cmd(user_data, cmd)
add_file(stack2archive_rpath[cf] + '.labels.json',
json.dumps(stack2info[cf].image_name2id, indent=4))
# cmd = ['docker', 'load', '--input', stack2archive_rpath[cf]]
# add_run_cmd(user_data, cmd)
# cmd = ['rm', stack2archive_rpath[cf]]
# add_run_cmd(user_data, cmd)
for image_name, image_id in stack2info[cf].image_name2id.items():
image = client.images.get(image_name)
image_id = str(image.id)
dtslogger.info('id for %s: %s' % (image_name, image_id))
cmd = ['docker', 'tag', image_id, image_name]
print(cmd)
add_run_cmd(user_data, cmd)
if cf in stacks_to_run:
msg = 'Adding the stack %r as default running' % cf
dtslogger.info(msg)
log_current_phase(user_data, PHASE_LOADING, "Stack %s: docker-compose up" % cf)
cmd = ['docker-compose', '--file', '/var/local/%s.yaml' % cf, '-p', cf, 'up', '-d']
add_run_cmd(user_data, cmd)
# XXX
cmd = ['docker-compose', '-p', cf, '--file', '/var/local/%s.yaml' % cf, 'up', '-d']
user_data['bootcmd'].append(cmd) # every boot
# The RPi blinking feedback expects that "All stacks up" will be written to the /data/boot-log.txt file.
# If modifying, make sure to adjust the blinking feedback
log_current_phase(user_data, PHASE_DONE, "All stacks up")
def dt_challenges_evaluator():
elogger.info("dt-challenges-evaluator (DTC %s)" % __version__)
check_docker_environment()
try:
check_executable_exists('docker-compose')
except InvalidEnvironment:
msg = 'Could not find docker-compose. Please install it.'
msg += '\n\nSee: https://docs.docker.com/compose/install/#install-compose'
raise InvalidEnvironment(msg)
parser = argparse.ArgumentParser()
parser.add_argument("--continuous", action="store_true", default=False)
parser.add_argument("--no-pull",
dest='no_pull',
action="store_true",
default=False)
parser.add_argument("extra", nargs=argparse.REMAINDER)
parsed = parser.parse_args()
do_pull = not parsed.no_pull
#
# try:
# docker_username = get_dockerhub_username()
# except Exception:
# msg = 'Skipping push because docker_username is not set.'
# elogger.debug(msg)
# docker_username = None
if parsed.continuous:
timeout = 5.0 # seconds
multiplier = 1.0
max_multiplier = 10
while True:
multiplier = min(multiplier, max_multiplier)
try:
go_(None, do_pull)
multiplier = 1.0
except NothingLeft:
sys.stderr.write('.')
# time.sleep(timeout * multiplier)
# elogger.info('No submissions available to evaluate.')
except ConnectionError as e:
elogger.error(e)
multiplier *= 1.5
except Exception as e:
msg = 'Uncaught exception:\n%s' % traceback.format_exc(e)
elogger.error(msg)
multiplier *= 1.5
time.sleep(timeout * multiplier)
else:
submissions = parsed.extra
if not submissions:
submissions = [None]
for submission_id in submissions:
try:
go_(submission_id, do_pull)
except NothingLeft:
elogger.info('No submissions available to evaluate.')
def command(shell, args):
check_docker_environment()
token = shell.get_dt1_token()
prog = 'dts challenges submit'
usage = """
Submission:
%(prog)s --challenge NAME
## Building options
Rebuilds ignoring Docker cache
%(prog)s --no-cache
## Attaching user data
Submission with an identifying label:
%(prog)s --user-label "My submission"
Submission with an arbitrary JSON payload:
%(prog)s --user-meta '{"param1": 123}'
"""
parser = argparse.ArgumentParser(prog=prog, usage=usage)
group = parser.add_argument_group("Submission identification")
parser.add_argument('--challenge',
help="Specify challenge name.",
default=None)
group.add_argument('--user-label',
dest='message',
default=None,
type=str,
help="Submission message")
group.add_argument(
'--user-meta',
dest='metadata',
default=None,
type=str,
help="Custom JSON structure to attach to the submission")
group = parser.add_argument_group("Building settings.")
group.add_argument('--no-cache',
dest='no_cache',
action='store_true',
default=False)
group.add_argument('--impersonate', type=int, default=None)
group.add_argument('-C',
dest='cwd',
default=None,
help='Base directory')
parsed = parser.parse_args(args)
impersonate = parsed.impersonate
if parsed.cwd is not None:
dtslogger.info('Changing to directory %s' % parsed.cwd)
os.chdir(parsed.cwd)
if not os.path.exists('submission.yaml'):
msg = 'Expected a submission.yaml file in %s.' % (os.path.realpath(
os.getcwd()))
raise UserError(msg)
sub_info = read_submission_info('.')
with wrap_server_operations():
ri = get_registry_info(token=token, impersonate=impersonate)
registry = ri.registry
compat = dtserver_get_compatible_challenges(
token=token,
impersonate=impersonate,
submission_protocols=sub_info.protocols)
if not compat.compatible:
msg = 'There are no compatible challenges with protocols %s,\n' \
' or you might not have the necessary permissions.' % sub_info.protocols
raise UserError(msg)
if parsed.message:
sub_info.user_label = parsed.message
if parsed.metadata:
sub_info.user_metadata = json.loads(parsed.metadata)
if parsed.challenge:
sub_info.challenge_names = parsed.challenge.split(',')
if sub_info.challenge_names is None:
sub_info.challenge_names = compat.compatible
print('I will submit to the challenges %s' %
sub_info.challenge_names)
for c in sub_info.challenge_names:
if not c in compat.available_submit:
msg = 'The challenge "%s" does not exist among %s.' % (
c, list(compat.available_submit))
raise UserError(msg)
if not c in compat.compatible:
msg = 'The challenge "%s" is not compatible with protocols %s .' % (
c, sub_info.protocols)
raise UserError(msg)
username = get_dockerhub_username(shell)
print('')
print('')
br = submission_build(username=username,
registry=registry,
no_cache=parsed.no_cache)
data = {
'image': dataclasses.asdict(br),
'user_label': sub_info.user_label,
'user_payload': sub_info.user_metadata,
'protocols': sub_info.protocols
}
data = dtserver_submit2(token=token,
challenges=sub_info.challenge_names,
data=data,
impersonate=impersonate)
# print('obtained:\n%s' % json.dumps(data, indent=2))
component_id = data['component_id']
submissions = data['submissions']
# url_component = href(get_duckietown_server_url() + '/humans/components/%s' % component_id)
msg = f'''
Successfully created component.
This component has been entered in {len(submissions)} challenge(s).
'''
for challenge_name, sub_info in submissions.items():
submission_id = sub_info['submission_id']
url_submission = href(get_duckietown_server_url() +
'/humans/submissions/%s' % submission_id)
challenge_title = sub_info['challenge']['title']
submission_id_color = termcolor.colored(submission_id, 'cyan')
P = dark('$')
head = bright(
f'## Challenge {challenge_name} - {challenge_title}')
msg += '\n\n' + f'''
{head}
Track this submission at:
{url_submission}
You can follow its fate using:
{P} dts challenges follow --submission {submission_id_color}
You can speed up the evaluation using your own evaluator:
{P} dts challenges evaluator --submission {submission_id_color}
'''.strip()
manual = href('http://docs.duckietown.org/DT19/AIDO/out/')
msg += f'''
For more information, see the manual at {manual}
'''
shell.sprint(msg)
def command(shell, args):
check_docker_environment()
token = shell.get_dt1_token()
prog = 'dts challenges submit'
usage = """
Submission:
%(prog)s --challenge NAME
## Building options
Rebuilds ignoring Docker cache
%(prog)s --no-cache
## Attaching user data
Submission with an identifying label:
%(prog)s --user-label "My submission"
Submission with an arbitrary JSON payload:
%(prog)s --user-meta '{"param1": 123}'
"""
parser = argparse.ArgumentParser(prog=prog, usage=usage)
group = parser.add_argument_group("Submission identification")
parser.add_argument('--challenge',
help="Specify challenge name.",
default=None)
group.add_argument('--user-label',
dest='message',
default=None,
type=str,
help="Submission message")
group.add_argument(
'--user-meta',
dest='metadata',
default=None,
type=str,
help="Custom JSON structure to attach to the submission")
group = parser.add_argument_group("Building settings.")
group.add_argument('--no-push',
dest='no_push',
action='store_true',
default=False,
help="Disable pushing of container")
group.add_argument('--no-submit',
dest='no_submit',
action='store_true',
default=False,
help="Disable submission (only build and push)")
group.add_argument('--no-cache',
dest='no_cache',
action='store_true',
default=False)
group.add_argument('-C',
dest='cwd',
default=None,
help='Base directory')
parsed = parser.parse_args(args)
do_push = not parsed.no_push
if parsed.cwd is not None:
dtslogger.info('Changing to directory %s' % parsed.cwd)
os.chdir(parsed.cwd)
if not os.path.exists('submission.yaml'):
msg = 'Expected a submission.yaml file in %s.' % (os.path.realpath(
os.getcwd()))
raise Exception(msg)
sub_info = read_submission_info('.')
if parsed.message:
sub_info.user_label = parsed.message
if parsed.metadata:
sub_info.user_payload = json.loads(parsed.metadata)
if parsed.challenge:
sub_info.challenge_name = parsed.challenge
username = get_dockerhub_username(shell)
hashname = build(username,
sub_info.challenge_name,
do_push,
no_cache=parsed.no_cache)
data = {
'hash': hashname,
'user_label': sub_info.user_label,
'user_payload': sub_info.user_payload,
'protocols': sub_info.protocols
}
if not parsed.no_submit:
submission_id = dtserver_submit(token, sub_info.challenge_name,
data)
url = get_duckietown_server_url(
) + '/humans/submissions/%s' % submission_id
url = href(url)
manual = href('http://docs.duckietown.org/DT18/AIDO/out/')
ID = termcolor.colored(submission_id, 'cyan')
msg = '''
Successfully created submission {ID}.
You can track the progress at:
{url}
You can also use the command `follow` to follow its fate:
{P} dts challenges follow --submission {ID}
You can speed up the evaluation using your own evaluator:
{P} dts challenges evaluator --submission {ID}
For more information, see the manual at {manual}
'''.format(ID=ID, P=dark('$'), url=url, manual=manual)
if hasattr(shell, 'sprint'):
shell.sprint(msg)
else:
print(msg)
def command(shell, args):
parser = argparse.ArgumentParser()
parser.add_argument('--hostname', default='duckiebot')
parser.add_argument('--linux-username', default='duckie')
parser.add_argument('--linux-password', default='quackquack')
parser.add_argument('--wifi-ssid',
dest="wifissid",
default='duckietown')
parser.add_argument('--wifi-password',
dest="wifipass",
default='quackquack')
parsed = parser.parse_args(args=args)
check_docker_environment()
if not is_valid_hostname(parsed.hostname):
msg = 'This is not a valid hostname: %r.' % parsed.hostname
raise Exception(msg)
if '-' in parsed.hostname:
msg = 'Cannot use the hostname %r. It cannot contain "-" because of a ROS limitation. ' % parsed.hostname
raise Exception(msg)
if len(parsed.hostname) < 3:
msg = 'This hostname is too short. Choose something more descriptive.'
raise Exception(msg)
MIN_AVAILABLE_GB = 0.0
try:
import psutil
except ImportError:
msg = 'Skipping disk check because psutil not installed.'
dtslogger.info(msg)
else:
disk = psutil.disk_usage(os.getcwd())
disk_available_gb = disk.free / (1024 * 1024 * 1024.0)
if disk_available_gb < MIN_AVAILABLE_GB:
msg = 'This procedure requires that you have at least %f GB of memory.' % MIN_AVAILABLE_GB
msg += '\nYou only have %f GB available.' % disk_available_gb
raise Exception(msg)
p = platform.system().lower()
if 'darwin' in p:
msg = 'This procedure cannot be run on Mac. You need an Ubuntu machine.'
raise Exception(msg)
this = dirname(realpath(__file__))
script_files = realpath(join(this, '..', 'init_sd_card.scripts'))
script_file = join(script_files, 'init_sd_card.sh')
if not os.path.exists(script_file):
msg = 'Could not find script %s' % script_file
raise Exception(msg)
ssh_key_pri = join(script_files, 'DT18_key_00')
ssh_key_pub = join(script_files, 'DT18_key_00.pub')
for f in [ssh_key_pri, ssh_key_pub]:
if not os.path.exists(f):
msg = 'Could not find file %s' % f
raise Exception(msg)
script_cmd = '/bin/bash %s' % script_file
token = shell.get_dt1_token()
env = dict()
env['DUCKIE_TOKEN'] = token
env['IDENTITY_FILE'] = ssh_key_pub
env['WIFISSID'] = parsed.wifissid
env['WIFIPASS'] = parsed.wifipass
env['HOST_NAME'] = parsed.hostname
env['DTS_USERNAME'] = parsed.linux_username
env['PASSWORD'] = parsed.linux_password
# add other environment
env.update(os.environ)
if 'DOCKER_HOST' in env:
r = env['DOCKER_HOST']
msg = 'I will IGNORE the DOCKER_HOST variable that is currently set to %r' % r
dtslogger.warning(msg)
env.pop('DOCKER_HOST')
ssh_dir = os.path.expanduser('~/.ssh')
if not os.path.exists(ssh_dir):
os.makedirs(ssh_dir)
ssh_key_pri_copied = os.path.join(ssh_dir, 'DT18_key_00')
ssh_key_pub_copied = os.path.join(ssh_dir, 'DT18_key_00.pub')
if not os.path.exists(ssh_key_pri_copied):
shutil.copy(ssh_key_pri, ssh_key_pri_copied)
if not os.path.exists(ssh_key_pub_copied):
shutil.copy(ssh_key_pub, ssh_key_pub_copied)
ssh_config = os.path.join(ssh_dir, 'config')
if not os.path.exists(ssh_config):
msg = ('Could not find ssh config file %s' % ssh_config)
dtslogger.info(msg)
current = ""
else:
current = open(ssh_config).read()
bit0 = """
# --- init_sd_card generated ---
# Use the key for all hosts
IdentityFile $IDENTITY
Host $HOSTNAME
User $DTS_USERNAME
Hostname $HOSTNAME.local
IdentityFile $IDENTITY
StrictHostKeyChecking no
# ------------------------------
"""
subs = dict(HOSTNAME=parsed.hostname,
IDENTITY=ssh_key_pri_copied,
DTS_USERNAME=parsed.linux_username)
bit = Template(bit0).substitute(**subs)
if not bit in current:
dtslogger.info('Updating ~/.ssh/config with: ' + bit)
with open(ssh_config, 'a') as f:
f.write(bit)
else:
dtslogger.info('Configuration already found in ~/.ssh/config')
ret = subprocess.call(script_cmd,
shell=True,
env=env,
stdin=sys.stdin,
stderr=sys.stderr,
stdout=sys.stdout)
if ret == 0:
dtslogger.info('Done!')
else:
msg = (
'An error occurred while initializing the SD card, please check and try again (%s).'
% ret)
raise Exception(msg)
def command(shell, args):
parser = argparse.ArgumentParser()
parser.add_argument('--steps', default="flash,expand,mount,setup,unmount",
help="Steps to perform")
parser.add_argument('--hostname', required=True)
parser.add_argument('--linux-username', default='duckie')
parser.add_argument('--linux-password', default='quackquack')
parser.add_argument('--stacks-load', dest="stacks_to_load",
default="DT18_00_basic,DT18_01_health_stats,DT18_02_others,DT18_03_roscore,DT18_05_duckiebot_base,DT18_06_dashboard",
help="which stacks to load")
parser.add_argument('--stacks-run', dest="stacks_to_run",
default="DT18_00_basic,DT18_01_health_stats,DT18_03_roscore,DT18_06_dashboard",
help="which stacks to RUN by default")
parser.add_argument('--reset-cache', dest='reset_cache', default=False, action='store_true',
help='Deletes the cached images')
parser.add_argument('--compress', dest='compress', default=False, action='store_true',
help='Compress the images - use if you have a 16GB SD card')
parser.add_argument('--device', dest='device', default='',
help='The device with the SD card')
parser.add_argument('--aido', dest='aido', default=False, action='store_true',
help='Only load what is necessary for an AI-DO submission')
# parser.add_argument('--swap', default=False, action='store_true',
# help='Create swap space')
parser.add_argument('--country', default="US",
help="2-letter country code (US, CA, CH, etc.)")
parser.add_argument('--wifi', dest="wifi", default='duckietown:quackquack',
help="""
Can specify one or more networks: "network:password,network:password,..."
""")
parser.add_argument('--ethz-username', default=None)
parser.add_argument('--ethz-password', default=None)
parser.add_argument('--experimental', dest='experimental', default=False, action='store_true',
help='Use experimental settings')
parser.add_argument('--configuration', dest='configuration', default=DEFAULT_CONFIGURATION,
help='Which configuration of Docker stacks to flash')
parser.add_argument('--type', dest='robot_type', default=None,
choices=['duckiebot', 'watchtower'],
help='Which type of robot we are setting up')
parsed = parser.parse_args(args=args)
global SD_CARD_DEVICE
SD_CARD_DEVICE = parsed.device
if parsed.reset_cache:
dtslogger.info('Removing cache')
if os.path.exists(DUCKIETOWN_TMP):
shutil.rmtree(DUCKIETOWN_TMP)
# if aido is set overwrite the stacks (don't load the base)
if parsed.aido:
parsed.stacks_to_load = 'DT18_00_basic,DT18_01_health_stats,DT18_03_roscore'
parsed.stacks_to_run = parsed.stacks_to_load
msg = """
## Tips and tricks
### Multiple networks
dts init_sd_card --wifi network1:password1,network2:password2 --country US
### Steps
Without arguments the script performs the steps:
flash
expand
mount
setup
unmount
You can use --steps to run only some of those:
dts init_sd_card --steps expand,mount
"""
print(msg)
if 'DOCKER_HOST' in os.environ:
msg = 'Removing DOCKER_HOST from os.environ.'
dtslogger.info(msg)
os.environ.pop('DOCKER_HOST')
check_docker_environment()
check_good_platform()
check_dependencies()
if parsed.experimental:
dtslogger.info('Running experimental mode!')
if parsed.robot_type is None:
while True:
r = input('You did not specify a robot type. Default is "{}". Do you confirm? [y]'.format(DEFAULT_ROBOT_TYPE))
if r.strip() in ['', 'y', 'Y', 'yes', 'YES', 'yup', 'YUP']:
parsed.robot_type = DEFAULT_ROBOT_TYPE
break;
elif r.strip() in ['', 'n', 'N', 'no', 'NO', 'nope', 'NOPE']:
dtslogger.info('Please retry while specifying a robot type. Bye bye!')
exit(1)
configuration = parsed.configuration
try:
get_resource(os.path.join('stacks', configuration))
except:
msg = 'Cannot find configuration "%s"' % configuration
raise InvalidUserInput(msg)
dtslogger.info('Configuration: %s' % configuration)
dtslogger.setLevel(logging.DEBUG)
steps = parsed.steps.split(',')
step2function = {
'flash': step_flash,
'expand': step_expand,
'mount': step_mount,
'setup': step_setup,
'unmount': step_unmount
}
for step_name in steps:
if step_name not in step2function:
msg = 'Cannot find step %r in %s' % (step_name, list(step2function))
raise InvalidUserInput(msg)
step2function[step_name](shell, parsed)
def dt_challenges_evaluator():
from .col_logging import setup_logging
setup_logging()
elogger.info("dt-challenges-evaluator (DTC %s)" % __version__)
elogger.info('called with:\n%s' % sys.argv)
check_docker_environment()
try:
check_executable_exists('docker-compose')
except InvalidEnvironment:
msg = 'Could not find docker-compose. Please install it.'
msg += '\n\nSee: https://docs.docker.com/compose/install/#install-compose'
raise InvalidEnvironment(msg)
usage = """
Usage:
"""
parser = argparse.ArgumentParser(usage=usage)
parser.add_argument("--continuous", action="store_true", default=False)
parser.add_argument("--no-pull",
dest='no_pull',
action="store_true",
default=False)
parser.add_argument("--no-upload",
dest='no_upload',
action="store_true",
default=False)
parser.add_argument("--no-delete",
dest='no_delete',
action="store_true",
default=False)
parser.add_argument("--machine-id", default=None, help='Machine name')
parser.add_argument("--name", default=None, help='Evaluator name')
parser.add_argument("--submission",
default=None,
help='evaluate this particular submission')
parser.add_argument("--reset",
dest='reset',
action="store_true",
default=False,
help='Reset submission')
parser.add_argument("--features", default='{}')
parsed = parser.parse_args()
tmpdir = '/tmp/duckietown/DT18/evaluator/executions'
try:
more_features = yaml.load(parsed.features)
except BaseException as e:
msg = 'Could not evaluate your YAML string %r:\n%s' % (parsed.features,
e)
raise Exception(msg)
if not isinstance(more_features, dict):
msg = 'I expected that the features are a dict; obtained %s: %r' % (
type(more_features).__name__, more_features)
raise Exception(msg)
do_pull = not parsed.no_pull
do_upload = not parsed.no_upload
delete = not parsed.no_delete
reset = parsed.reset
evaluator_name = parsed.name or 'p-%s' % os.getpid()
machine_id = parsed.machine_id or socket.gethostname()
args = dict(do_upload=do_upload,
do_pull=do_pull,
more_features=more_features,
delete=delete,
evaluator_name=evaluator_name,
machine_id=machine_id,
tmpdir=tmpdir)
if parsed.continuous:
timeout = 5.0 # seconds
multiplier = 1.0
max_multiplier = 10
while True:
multiplier = min(multiplier, max_multiplier)
try:
go_(None, reset=False, **args)
multiplier = 1.0
except NothingLeft:
sys.stderr.write('.')
# time.sleep(timeout * multiplier)
# elogger.info('No submissions available to evaluate.')
except ConnectionError as e:
elogger.error(e)
multiplier *= 1.5
except BaseException as e:
msg = 'Uncaught exception:\n%s' % traceback.format_exc(e)
elogger.error(msg)
multiplier *= 1.5
time.sleep(timeout * multiplier)
else:
if parsed.submission:
submissions = [parsed.submission]
else:
submissions = [None]
for submission_id in submissions:
try:
go_(submission_id, reset=reset, **args)
except NothingLeft as e:
if submission_id is None:
msg = 'No submissions available to evaluate.'
else:
msg = 'Could not evaluate submission %s.' % submission_id
msg += '\n' + str(e)
elogger.error(msg)
def command(shell, args):
prog = 'dts challenges evaluate'
parser = argparse.ArgumentParser(prog=prog, usage=usage)
group = parser.add_argument_group('Basic')
group.add_argument('--no-cache',
action='store_true',
default=False,
help="")
group.add_argument('--no-build',
action='store_true',
default=False,
help="")
group.add_argument('--no-pull',
action='store_true',
default=False,
help="")
group.add_argument('--challenge',
help="Specific challenge to evaluate")
group.add_argument('--image',
help="Evaluator image to run",
default='duckietown/dt-challenges-evaluator:v4')
group.add_argument('--shell',
action='store_true',
default=False,
help="Runs a shell in the container")
group.add_argument('--output', help="", default='output')
group.add_argument('--visualize',
help="Visualize the evaluation",
action='store_true',
default=False)
parser.add_argument('--impersonate', type=str, default=None)
group.add_argument('-C', dest='change', default=None)
parsed = parser.parse_args(args)
if parsed.change:
os.chdir(parsed.change)
client = check_docker_environment()
command = ['dt-challenges-evaluate-local']
if parsed.no_cache:
command.append('--no-cache')
if parsed.no_build:
command.append('--no-build')
if parsed.challenge:
command.extend(['--challenge', parsed.challenge])
if parsed.impersonate:
command.extend(['--impersonate', parsed.impersonate])
output_rp = os.path.realpath(parsed.output)
command.extend(['--output', parsed.output])
#
# if parsed.features:
# dtslogger.debug('Passing features %r' % parsed.features)
# command += ['--features', parsed.features]
# fake_dir = '/submission'
tmpdir = '/tmp'
UID = os.getuid()
USERNAME = getpass.getuser()
dir_home_guest = '/fake-home/%s' % USERNAME # os.path.expanduser('~')
dir_fake_home_host = os.path.join(tmpdir, 'fake-%s-home' % USERNAME)
if not os.path.exists(dir_fake_home_host):
os.makedirs(dir_fake_home_host)
dir_fake_home_guest = dir_home_guest
dir_dtshell_host = os.path.join(os.path.expanduser('~'), '.dt-shell')
dir_dtshell_guest = os.path.join(dir_fake_home_guest, '.dt-shell')
dir_tmpdir_host = '/tmp'
dir_tmpdir_guest = '/tmp'
volumes = {
'/var/run/docker.sock': {
'bind': '/var/run/docker.sock',
'mode': 'rw'
}
}
d = os.path.join(os.getcwd(), parsed.output)
if not os.path.exists(d):
os.makedirs(d)
volumes[output_rp] = {'bind': d, 'mode': 'rw'}
volumes[os.getcwd()] = {'bind': os.getcwd(), 'mode': 'ro'}
volumes[dir_tmpdir_host] = {'bind': dir_tmpdir_guest, 'mode': 'rw'}
volumes[dir_dtshell_host] = {'bind': dir_dtshell_guest, 'mode': 'ro'}
volumes[dir_fake_home_host] = {
'bind': dir_fake_home_guest,
'mode': 'rw'
}
volumes['/etc/group'] = {'bind': '/etc/group', 'mode': 'ro'}
binds = [_['bind'] for _ in volumes.values()]
for b1 in binds:
for b2 in binds:
if b1 == b2:
continue
if b1.startswith(b2):
msg = 'Warning, it might be a problem to have binds with overlap'
msg += '\n b1: %s' % b1
msg += '\n b2: %s' % b2
dtslogger.warn(msg)
# command.extend(['-C', fake_dir])
env = {}
extra_environment = dict(username=USERNAME,
uid=UID,
USER=USERNAME,
HOME=dir_fake_home_guest)
env.update(extra_environment)
dtslogger.debug('Volumes:\n\n%s' %
yaml.safe_dump(volumes, default_flow_style=False))
dtslogger.debug('Environment:\n\n%s' %
yaml.safe_dump(env, default_flow_style=False))
from duckietown_challenges.rest import get_duckietown_server_url
url = get_duckietown_server_url()
dtslogger.info('The server URL is: %s' % url)
if 'localhost' in url:
h = socket.gethostname()
replacement = h + '.local'
dtslogger.warning(
'There is "localhost" inside, so I will try to change it to %r'
% replacement)
dtslogger.warning(
'This is because Docker cannot see the host as "localhost".')
url = url.replace("localhost", replacement)
dtslogger.warning('The new url is: %s' % url)
dtslogger.warning(
'This will be passed to the evaluator in the Docker container.'
)
env['DTSERVER'] = url
container_name = 'local-evaluator'
image = parsed.image
name, tag = image.split(':')
if not parsed.no_pull:
dtslogger.info('Updating container %s' % image)
dtslogger.info('This might take some time.')
client.images.pull(name, tag)
#
try:
container = client.containers.get(container_name)
except:
pass
else:
dtslogger.error('stopping previous %s' % container_name)
container.stop()
dtslogger.error('removing')
container.remove()
dtslogger.info('Starting container %s with %s' %
(container_name, image))
detach = True
env[DTShellConstants.DT1_TOKEN_CONFIG_KEY] = shell.get_dt1_token()
dtslogger.info('Container command: %s' % " ".join(command))
# add all the groups
on_mac = 'Darwin' in platform.system()
if on_mac:
group_add = []
else:
group_add = [
g.gr_gid for g in grp.getgrall() if USERNAME in g.gr_mem
]
interactive = False
if parsed.shell:
interactive = True
detach = False
command = ['/bin/bash', '-l']
params = dict(working_dir=os.getcwd(),
user=UID,
group_add=group_add,
command=command,
tty=interactive,
volumes=volumes,
environment=env,
remove=True,
network_mode='host',
detach=detach,
name=container_name)
dtslogger.info('Parameters:\n%s' % json.dumps(params, indent=4))
client.containers.run(image, **params)
if parsed.visualize:
start_rqt_image_view()
continuously_monitor(client, container_name)
def command(shell, args):
prog = 'dts start_gui_tools DUCKIEBOT_NAME'
usage = """
Keyboard control:
%(prog)s
"""
parser = argparse.ArgumentParser(prog=prog, usage=usage)
parser.add_argument('hostname',
default=None,
help='Name of the Duckiebot to calibrate')
parser.add_argument('--network',
default='host',
help='Name of the network which to connect')
parser.add_argument('--sim',
action='store_true',
default=False,
help='are we running in simulator?')
parser.add_argument(
'--base_image',
dest='image',
default="duckietown/rpi-duckiebot-base:master19-no-arm",
help="The base image, probably don't change the default")
parsed_args = parser.parse_args(args)
if parsed_args.sim:
duckiebot_ip = "localhost"
else:
duckiebot_ip = get_duckiebot_ip(
duckiebot_name=parsed_args.hostname)
hostname = parsed_args.hostname
image = parsed_args.image
client = check_docker_environment()
container_name = "interactive_gui_tools_%s" % hostname
remove_if_running(client, container_name)
if parsed_args.sim:
env = {
'HOSTNAME': 'default',
'ROS_MASTER': hostname,
'DUCKIEBOT_NAME': hostname,
'ROS_MASTER_URI': 'http://%s:11311' % hostname
}
else:
env = {
'HOSTNAME': hostname,
'ROS_MASTER': hostname,
'DUCKIEBOT_NAME': hostname,
'ROS_MASTER_URI': 'http://%s:11311' % duckiebot_ip
}
env['QT_X11_NO_MITSHM'] = 1
p = platform.system().lower()
volumes = {'/tmp/.X11-unix': {'bind': '/tmp/.X11-unix', 'mode': 'rw'}}
if 'darwin' in p:
subprocess.call(["xhost", "+", '127.0.0.1'])
env['DISPLAY'] = 'host.docker.internal:0'
else:
subprocess.call(["xhost", "+"])
env['DISPLAY'] = os.environ['DISPLAY']
dtslogger.info("Running %s on localhost with environment vars: %s" %
(container_name, env))
cmd = "/bin/bash"
params = {
'image': image,
'name': container_name,
'network_mode': parsed_args.network,
'environment': env,
'privileged': True,
'stdin_open': True,
'tty': True,
'detach': True,
'command': cmd,
'volumes': volumes
}
container = client.containers.run(**params)
attach_cmd = 'docker attach %s' % container_name
start_command_in_subprocess(attach_cmd)
def command(shell, args):
check_docker_environment()
home = os.path.expanduser('~')
prog = 'dts challenges evaluator'
parser = argparse.ArgumentParser(prog=prog, usage=usage)
group = parser.add_argument_group('Basic')
group.add_argument('--submission',
type=int,
default=None,
help='Run a specific submission.')
group.add_argument(
'--reset',
dest='reset',
action='store_true',
default=False,
help='(needs --submission) Re-evaluate the specific submission.')
group = parser.add_argument_group('Advanced')
group.add_argument('--no-watchtower',
dest='no_watchtower',
action='store_true',
default=False,
help="Disable starting of watchtower")
group.add_argument('--no-pull',
dest='no_pull',
action='store_true',
default=False,
help="Disable pulling of container")
group.add_argument('--image',
help="Evaluator image to run",
default='duckietown/dt-challenges-evaluator:v3')
group.add_argument('--name',
default=None,
help='Name for this evaluator')
group.add_argument("--features",
default=None,
help="Pretend to be what you are not.")
dtslogger.debug('args: %s' % args)
parsed = parser.parse_args(args)
machine_id = socket.gethostname()
if parsed.name is None:
container_name = '%s-%s' % (socket.gethostname(), os.getpid())
else:
container_name = parsed.name
import docker
client = docker.from_env()
command = ['dt-challenges-evaluator']
if parsed.submission:
command += ['--submission', str(parsed.submission)]
if parsed.reset:
command += ['--reset']
else:
command += ['--continuous']
command += ['--name', container_name]
command += ['--machine-id', machine_id]
if parsed.features:
dtslogger.debug('Passing features %r' % parsed.features)
command += ['--features', parsed.features]
volumes = {
'/var/run/docker.sock': {
'bind': '/var/run/docker.sock',
'mode': 'rw'
},
os.path.join(home, '.dt-shell'): {
'bind': '/root/.dt-shell',
'mode': 'ro'
},
'/tmp': {
'bind': '/tmp',
'mode': 'rw'
}
}
env = {}
UID = os.getuid()
USERNAME = getpass.getuser()
extra_environment = dict(username=USERNAME, uid=UID)
env.update(extra_environment)
if not parsed.no_watchtower:
ensure_watchtower_active(client)
url = get_duckietown_server_url()
dtslogger.info('The server URL is: %s' % url)
if 'localhost' in url:
h = socket.gethostname()
replacement = h + '.local'
dtslogger.warning(
'There is "localhost" inside, so I will try to change it to %r'
% replacement)
dtslogger.warning(
'This is because Docker cannot see the host as "localhost".')
url = url.replace("localhost", replacement)
dtslogger.warning('The new url is: %s' % url)
dtslogger.warning(
'This will be passed to the evaluator in the Docker container.'
)
env['DTSERVER'] = url
image = parsed.image
name, tag = image.split(':')
if not parsed.no_pull:
dtslogger.info('Updating container %s' % image)
client.images.pull(name, tag)
try:
container = client.containers.get(container_name)
except:
pass
else:
dtslogger.error('stopping previous %s' % container_name)
container.stop()
dtslogger.error('removing')
container.remove()
dtslogger.info('Starting container %s with %s' %
(container_name, image))
dtslogger.info('Container command: %s' % " ".join(command))
client.containers.run(image,
command=command,
volumes=volumes,
environment=env,
network_mode='host',
detach=True,
name=container_name,
tty=True)
while True:
try:
container = client.containers.get(container_name)
except Exception as e:
msg = 'Cannot get container %s: %s' % (container_name, e)
dtslogger.error(msg)
dtslogger.info('Will wait.')
time.sleep(5)
continue
dtslogger.info('status: %s' % container.status)
if container.status == 'exited':
msg = 'The container exited.'
logs = ''
for c in container.logs(stdout=True, stderr=True, stream=True):
logs += c
dtslogger.error(msg)
tf = 'evaluator.log'
with open(tf, 'w') as f:
f.write(logs)
msg = 'Logs saved at %s' % (tf)
dtslogger.info(msg)
break
try:
for c in container.logs(stdout=True,
stderr=True,
stream=True,
follow=True):
sys.stdout.write(c)
time.sleep(3)
except Exception as e:
dtslogger.error(e)
dtslogger.info('Will try to re-attach to container.')
time.sleep(3)
except KeyboardInterrupt:
dtslogger.info('Received CTRL-C. Stopping container...')
container.stop()
dtslogger.info('Removing container')
container.remove()
dtslogger.info('Container removed.')
break
def command(shell, args):
parser = argparse.ArgumentParser()
parser.add_argument('--steps',
default="flash,expand,mount,setup,unmount",
help="Steps to perform")
parser.add_argument('--hostname', default='duckiebot')
parser.add_argument('--linux-username', default='duckie')
parser.add_argument('--linux-password', default='quackquack')
parser.add_argument(
'--stacks-load',
dest="stacks_to_load",
default=
"DT18_00_basic,DT18_01_health_stats,DT18_02_others,DT18_05_duckiebot_base",
help="which stacks to load")
parser.add_argument('--stacks-run',
dest="stacks_to_run",
default="DT18_00_basic,DT18_01_health_stats",
help="which stacks to RUN by default")
parser.add_argument('--reset-cache',
dest='reset_cache',
default=False,
action='store_true',
help='Deletes the cached images')
parser.add_argument(
'--compress',
dest='compress',
default=False,
action='store_true',
help='Compress the images - use if you have a 16GB SD card')
parser.add_argument('--device',
dest='device',
default='',
help='The device with the SD card')
# parser.add_argument('--swap', default=False, action='store_true',
# help='Create swap space')
parser.add_argument('--country',
default="US",
help="2-letter country code (US, CA, CH, etc.)")
parser.add_argument('--wifi',
dest="wifi",
default='duckietown:quackquack',
help="""
Can specify one or more networks: "network:password,network:password,..."
""")
parser.add_argument('--ethz-username', default=None)
parser.add_argument('--ethz-password', default=None)
parsed = parser.parse_args(args=args)
global SD_CARD_DEVICE
SD_CARD_DEVICE = parsed.device
if parsed.reset_cache:
dtslogger.info('Removing cache')
if os.path.exists(DUCKIETOWN_TMP):
shutil.rmtree(DUCKIETOWN_TMP)
msg = """
## Tips and tricks
### Multiple networks
dts init_sd_card2 --wifi network1:password1,network2:password2 --country US
### Steps
Without arguments the script performs the steps:
flash
expand
mount
setup
unmount
You can use --steps to run only some of those:
dts init_sd_card2 --steps expand,mount
"""
print(msg)
if 'DOCKER_HOST' in os.environ:
msg = 'Removing DOCKER_HOST from os.environ.'
dtslogger.info(msg)
os.environ.pop('DOCKER_HOST')
check_docker_environment()
check_good_platform()
check_dependencies()
dtslogger.setLevel(logging.DEBUG)
steps = parsed.steps.split(',')
step2function = {
'flash': step_flash,
'expand': step_expand,
'mount': step_mount,
'setup': step_setup,
'unmount': step_unmount
}
for step_name in steps:
if step_name not in step2function:
msg = 'Cannot find step %r in %s' % (step_name,
list(step2function))
raise InvalidUserInput(msg)
step2function[step_name](shell, parsed)
