def _run_cmd(cmd, get_output=False, print_output=False):
dtslogger.debug('$ %s' % cmd)
if get_output:
proc = subprocess.Popen(cmd, stdout=subprocess.PIPE)
p = re.compile(CATKIN_REGEX, re.IGNORECASE)
lines = []
last_matched = False
for line in io.TextIOWrapper(proc.stdout, encoding="utf-8"):
line = line.rstrip()
if print_output:
matches = p.match(line.strip()) is not None
if matches and last_matched:
sys.stdout.write("\033[F")
sys.stdout.write(line + "\033[K" + "\n")
sys.stdout.flush()
last_matched = matches
if line:
lines.append(line)
proc.wait()
if proc.returncode != 0:
msg = 'The command {} returned exit code {}'.format(
cmd, proc.returncode)
dtslogger.error(msg)
raise RuntimeError(msg)
return lines
else:
subprocess.check_call(cmd)
def command(shell, args):
if shell.commands_path_leave_alone:
dtslogger.warn(
'Will not update the commands because the path was set explicitly.'
)
else:
if shell.update_commands():
dtslogger.info('Duckietown Shell commands updated.')
else:
dtslogger.error('Update was not successful.')
def check_has_space(where, min_available_gb):
try:
import psutil
except ImportError:
msg = 'Skipping disk check because psutil not installed.'
dtslogger.info(msg)
else:
disk = psutil.disk_usage(where)
disk_available_gb = disk.free / (1024 * 1024 * 1024.0)
if disk_available_gb < min_available_gb:
msg = 'This procedure requires that you have at least %f GB of memory.' % min_available_gb
msg += '\nYou only have %.2f GB available on %s.' % (disk_available_gb, where)
dtslogger.error(msg)
raise NotEnoughSpace(msg)
else:
msg = 'You have %.2f GB available on %s. ' % (disk_available_gb, where)
dtslogger.info(msg)
def validate_user_data(user_data_yaml):
if 'VARIABLE' in user_data_yaml:
msg = 'Invalid user_data_yaml:\n' + user_data_yaml
msg += '\n\nThe above contains VARIABLE'
raise Exception(msg)
try:
import requests
except ImportError:
msg = 'Skipping validation because "requests" not installed.'
dtslogger.warning(msg)
else:
url = 'https://validate.core-os.net/validate'
r = requests.put(url, data=user_data_yaml)
info = json.loads(r.content)
result = info['result']
nerrors = 0
for x in result:
kind = x['kind']
line = x['line']
message = x['message']
m = 'Invalid at line %s: %s' % (line, message)
m += '| %s' % user_data_yaml.split('\n')[line - 1]
if kind == 'error':
dtslogger.error(m)
nerrors += 1
else:
ignore = [
'bootcmd', 'package_upgrade', 'runcmd', 'ssh_pwauth',
'sudo', 'chpasswd', 'lock_passwd', 'plain_text_passwd'
]
show = False
for i in ignore:
if 'unrecognized key "%s"' % i in m:
break
else:
show = True
if show:
dtslogger.warning(m)
if nerrors:
msg = 'There are %d errors: exiting' % nerrors
raise Exception(msg)
def _run_cmd(cmd, get_output=False, print_output=False):
dtslogger.debug('$ %s' % cmd)
if get_output:
proc = subprocess.Popen(cmd, stdout=subprocess.PIPE)
lines = []
for line in io.TextIOWrapper(proc.stdout, encoding="utf-8"):
line = line.rstrip()
if print_output:
print(line)
if line:
lines.append(line)
proc.wait()
if proc.returncode != 0:
msg = 'The command {} returned exit code {}'.format(
cmd, proc.returncode)
dtslogger.error(msg)
raise RuntimeError(msg)
return lines
else:
subprocess.check_call(cmd)
def command(shell, args):
check_docker_environment()
home = os.path.expanduser('~')
prog = 'dts challenges evaluator'
parser = argparse.ArgumentParser(prog=prog, usage=usage)
group = parser.add_argument_group('Basic')
group.add_argument('--submission',
type=int,
default=None,
help='Run a specific submission.')
group.add_argument(
'--reset',
dest='reset',
action='store_true',
default=False,
help='(needs --submission) Re-evaluate the specific submission.')
group = parser.add_argument_group('Advanced')
group.add_argument('--no-watchtower',
dest='no_watchtower',
action='store_true',
default=False,
help="Disable starting of watchtower")
group.add_argument('--no-pull',
dest='no_pull',
action='store_true',
default=False,
help="Disable pulling of container")
group.add_argument('--image',
help="Evaluator image to run",
default='duckietown/dt-challenges-evaluator:v3')
group.add_argument('--name',
default=None,
help='Name for this evaluator')
group.add_argument("--features",
default=None,
help="Pretend to be what you are not.")
dtslogger.debug('args: %s' % args)
parsed = parser.parse_args(args)
machine_id = socket.gethostname()
if parsed.name is None:
container_name = '%s-%s' % (socket.gethostname(), os.getpid())
else:
container_name = parsed.name
import docker
client = docker.from_env()
command = ['dt-challenges-evaluator']
if parsed.submission:
command += ['--submission', str(parsed.submission)]
if parsed.reset:
command += ['--reset']
else:
command += ['--continuous']
command += ['--name', container_name]
command += ['--machine-id', machine_id]
if parsed.features:
dtslogger.debug('Passing features %r' % parsed.features)
command += ['--features', parsed.features]
volumes = {
'/var/run/docker.sock': {
'bind': '/var/run/docker.sock',
'mode': 'rw'
},
os.path.join(home, '.dt-shell'): {
'bind': '/root/.dt-shell',
'mode': 'ro'
},
'/tmp': {
'bind': '/tmp',
'mode': 'rw'
}
}
env = {}
UID = os.getuid()
USERNAME = getpass.getuser()
extra_environment = dict(username=USERNAME, uid=UID)
env.update(extra_environment)
if not parsed.no_watchtower:
ensure_watchtower_active(client)
url = get_duckietown_server_url()
dtslogger.info('The server URL is: %s' % url)
if 'localhost' in url:
h = socket.gethostname()
replacement = h + '.local'
dtslogger.warning(
'There is "localhost" inside, so I will try to change it to %r'
% replacement)
dtslogger.warning(
'This is because Docker cannot see the host as "localhost".')
url = url.replace("localhost", replacement)
dtslogger.warning('The new url is: %s' % url)
dtslogger.warning(
'This will be passed to the evaluator in the Docker container.'
)
env['DTSERVER'] = url
image = parsed.image
name, tag = image.split(':')
if not parsed.no_pull:
dtslogger.info('Updating container %s' % image)
client.images.pull(name, tag)
try:
container = client.containers.get(container_name)
except:
pass
else:
dtslogger.error('stopping previous %s' % container_name)
container.stop()
dtslogger.error('removing')
container.remove()
dtslogger.info('Starting container %s with %s' %
(container_name, image))
dtslogger.info('Container command: %s' % " ".join(command))
client.containers.run(image,
command=command,
volumes=volumes,
environment=env,
network_mode='host',
detach=True,
name=container_name,
tty=True)
while True:
try:
container = client.containers.get(container_name)
except Exception as e:
msg = 'Cannot get container %s: %s' % (container_name, e)
dtslogger.error(msg)
dtslogger.info('Will wait.')
time.sleep(5)
continue
dtslogger.info('status: %s' % container.status)
if container.status == 'exited':
msg = 'The container exited.'
logs = ''
for c in container.logs(stdout=True, stderr=True, stream=True):
logs += c
dtslogger.error(msg)
tf = 'evaluator.log'
with open(tf, 'w') as f:
f.write(logs)
msg = 'Logs saved at %s' % (tf)
dtslogger.info(msg)
break
try:
for c in container.logs(stdout=True,
stderr=True,
stream=True,
follow=True):
sys.stdout.write(c)
time.sleep(3)
except Exception as e:
dtslogger.error(e)
dtslogger.info('Will try to re-attach to container.')
time.sleep(3)
except KeyboardInterrupt:
dtslogger.info('Received CTRL-C. Stopping container...')
container.stop()
dtslogger.info('Removing container')
container.remove()
dtslogger.info('Container removed.')
break
def configure_images(parsed, user_data, add_file_local, add_file):
import psutil
# read and validate duckiebot-compose
stacks_to_load = parsed.stacks_to_load.split(',')
stacks_to_run = parsed.stacks_to_run.split(',')
dtslogger.info('Stacks to load: %s' % stacks_to_load)
dtslogger.info('Stacks to run: %s' % stacks_to_run)
for _ in stacks_to_run:
if _ not in stacks_to_load:
msg = 'If you want to run %r you need to load it as well.' % _
raise Exception(msg)
configuration = parsed.configuration
for cf in stacks_to_load:
# local path
lpath = get_resource(os.path.join('stacks', configuration, cf + '.yaml'))
# path on PI
rpath = '/var/local/%s.yaml' % cf
if which('docker-compose') is None:
msg = 'Could not find docker-compose. Cannot validate file.'
dtslogger.error(msg)
else:
_run_cmd(['docker-compose', '-f', lpath, 'config', '--quiet'])
add_file_local(path=rpath, local=lpath)
stack2yaml = get_stack2yaml(stacks_to_load, get_resource(os.path.join('stacks', configuration)))
if not stack2yaml:
msg = 'Not even one stack specified'
raise Exception(msg)
stack2info = save_images(stack2yaml, compress=parsed.compress)
buffer_bytes = 100 * 1024 * 1024
stacks_written = []
stack2archive_rpath = {}
dtslogger.debug(stack2info)
for stack, stack_info in stack2info.items():
tgz = stack_info.archive
size = os.stat(tgz).st_size
dtslogger.info('Considering copying %s of size %s' % (tgz, friendly_size_file(tgz)))
rpath = os.path.join('var', 'local', os.path.basename(tgz))
destination = os.path.join(TMP_ROOT_MOUNTPOINT, rpath)
available = psutil.disk_usage(TMP_ROOT_MOUNTPOINT).free
dtslogger.info('available %s' % friendly_size(available))
if available < size + buffer_bytes:
msg = 'You have %s available on %s but need %s for %s' % (
friendly_size(available), TMP_ROOT_MOUNTPOINT, friendly_size_file(tgz), tgz)
dtslogger.info(msg)
continue
dtslogger.info('OK, copying, and loading it on first boot.')
if os.path.exists(destination):
msg = 'Skipping copying image that already exist at %s.' % destination
dtslogger.info(msg)
else:
if which('rsync'):
cmd = ['sudo', 'rsync', '-avP', tgz, destination]
else:
cmd = ['sudo', 'cp', tgz, destination]
_run_cmd(cmd)
sync_data()
stack2archive_rpath[stack] = os.path.join('/', rpath)
stacks_written.append(stack)
client = check_docker_environment()
stacks_not_to_run = [_ for _ in stacks_to_load if _ not in stacks_to_run]
order = stacks_to_run + stacks_not_to_run
for cf in order:
if cf in stacks_written:
log_current_phase(user_data, PHASE_LOADING,
"Stack %s: Loading containers" % cf)
cmd = 'docker load --input %s && rm %s' % (stack2archive_rpath[cf], stack2archive_rpath[cf])
add_run_cmd(user_data, cmd)
add_file(stack2archive_rpath[cf] + '.labels.json',
json.dumps(stack2info[cf].image_name2id, indent=4))
# cmd = ['docker', 'load', '--input', stack2archive_rpath[cf]]
# add_run_cmd(user_data, cmd)
# cmd = ['rm', stack2archive_rpath[cf]]
# add_run_cmd(user_data, cmd)
for image_name, image_id in stack2info[cf].image_name2id.items():
image = client.images.get(image_name)
image_id = str(image.id)
dtslogger.info('id for %s: %s' % (image_name, image_id))
cmd = ['docker', 'tag', image_id, image_name]
print(cmd)
add_run_cmd(user_data, cmd)
if cf in stacks_to_run:
msg = 'Adding the stack %r as default running' % cf
dtslogger.info(msg)
log_current_phase(user_data, PHASE_LOADING, "Stack %s: docker-compose up" % cf)
cmd = ['docker-compose', '--file', '/var/local/%s.yaml' % cf, '-p', cf, 'up', '-d']
add_run_cmd(user_data, cmd)
# XXX
cmd = ['docker-compose', '-p', cf, '--file', '/var/local/%s.yaml' % cf, 'up', '-d']
user_data['bootcmd'].append(cmd) # every boot
# The RPi blinking feedback expects that "All stacks up" will be written to the /data/boot-log.txt file.
# If modifying, make sure to adjust the blinking feedback
log_current_phase(user_data, PHASE_DONE, "All stacks up")
def command(shell, args):
check_docker_environment()
home = os.path.expanduser('~')
prog = 'dts challenges evaluator'
parser = argparse.ArgumentParser(prog=prog, usage=usage)
group = parser.add_argument_group('Basic')
group.add_argument('--submission', type=int, default=None,
help='Run a specific submission.')
group.add_argument('--reset', dest='reset', action='store_true', default=False,
help='(needs --submission) Re-evaluate the specific submission.')
group = parser.add_argument_group('Advanced')
group.add_argument('--no-watchtower', dest='no_watchtower', action='store_true', default=False,
help="Disable starting of watchtower")
group.add_argument('--no-pull', dest='no_pull', action='store_true', default=False,
help="Disable pulling of containers")
group.add_argument('--no-upload', dest='no_upload', action='store_true', default=False,
help="Disable upload of artifacts")
group.add_argument('--no-delete', dest='no_delete', action='store_true', default=False,
help="Does not erase temporary files in /tmp/duckietown")
group.add_argument('--image', help="Evaluator image to run", default='duckietown/dt-challenges-evaluator:v4')
group.add_argument('--name', default=None, help='Name for this evaluator')
group.add_argument("--features", default=None, help="Pretend to be what you are not.")
group.add_argument("--ipfs", action='store_true', default=False, help="Run with IPFS available")
group.add_argument("--one", action='store_true', default=False, help="Only run 1 submission")
# dtslogger.debug('args: %s' % args)
parsed = parser.parse_args(args)
machine_id = socket.gethostname()
if parsed.name is None:
container_name = '%s-%s' % (socket.gethostname(), os.getpid())
else:
container_name = parsed.name
client = check_docker_environment()
command = ['dt-challenges-evaluator']
if parsed.submission:
command += ['--submission', str(parsed.submission)]
if parsed.reset:
command += ['--reset']
else:
if not parsed.one:
command += ['--continuous']
command += ['--name', container_name]
command += ['--machine-id', machine_id]
if parsed.no_upload:
command += ['--no-upload']
if parsed.no_pull:
command += ['--no-pull']
if parsed.no_delete:
command += ['--no-delete']
if parsed.one:
command += ['--one']
if parsed.features:
dtslogger.debug('Passing features %r' % parsed.features)
command += ['--features', parsed.features]
mounts = []
volumes = {
'/var/run/docker.sock': {'bind': '/var/run/docker.sock', 'mode': 'rw'},
os.path.join(home, '.dt-shell'): {'bind': '/root/.dt-shell', 'mode': 'ro'},
'/tmp': {'bind': '/tmp', 'mode': 'rw'}
}
if parsed.ipfs:
if not ipfs_available():
msg = 'IPFS not available/mounted correctly.'
raise UserError(msg)
command += ['--ipfs']
# volumes['/ipfs'] = {'bind': '/ipfs', 'mode': 'ro'}
from docker.types import Mount
mount = Mount(type='bind', source='/ipfs', target='/ipfs', read_only=True)
mounts.append(mount)
env = {}
UID = os.getuid()
USERNAME = getpass.getuser()
extra_environment = dict(username=USERNAME, uid=UID)
env.update(extra_environment)
if not parsed.no_watchtower:
ensure_watchtower_active(client)
from duckietown_challenges.rest import get_duckietown_server_url
url = get_duckietown_server_url()
dtslogger.info('The server URL is: %s' % url)
if 'localhost' in url:
h = socket.gethostname()
replacement = h + '.local'
dtslogger.warning('There is "localhost" inside, so I will try to change it to %r' % replacement)
dtslogger.warning('This is because Docker cannot see the host as "localhost".')
url = url.replace("localhost", replacement)
dtslogger.warning('The new url is: %s' % url)
dtslogger.warning('This will be passed to the evaluator in the Docker container.')
env['DTSERVER'] = url
image = parsed.image
dtslogger.info('Using evaluator image %s' % image)
name, tag = image.split(':')
if not parsed.no_pull:
dtslogger.info('Updating container %s' % image)
client.images.pull(name, tag)
# noinspection PyBroadException
try:
container = client.containers.get(container_name)
except:
pass
else:
dtslogger.error('stopping previous %s' % container_name)
container.stop()
dtslogger.error('removing')
container.remove()
dtslogger.info('Starting container %s with %s' % (container_name, image))
dtslogger.info('Container command: %s' % " ".join(command))
# add all the groups
import grp
group_add = [g.gr_gid for g in grp.getgrall() if USERNAME in g.gr_mem]
client.containers.run(image,
group_add=group_add,
command=command,
volumes=volumes,
environment=env,
mounts=mounts,
network_mode='host',
detach=True,
name=container_name,
tty=True)
last_log_timestamp = None
while True:
try:
container = client.containers.get(container_name)
except Exception as e:
msg = 'Cannot get container %s: %s' % (container_name, e)
dtslogger.error(msg)
dtslogger.info('Will wait.')
time.sleep(5)
continue
dtslogger.info('status: %s' % container.status)
if container.status == 'exited':
logs = ''
for c in container.logs(stdout=True, stderr=True, stream=True, since=last_log_timestamp):
logs += c.decode('utf-8')
last_log_timestamp = datetime.datetime.now()
tf = 'evaluator.log'
with open(tf, 'w') as f:
f.write(logs)
msg = 'The container exited.'
msg += '\nLogs saved at %s' % tf
dtslogger.info(msg)
break
try:
if last_log_timestamp is not None:
print('since: %s' % last_log_timestamp.isoformat())
for c0 in container.logs(stdout=True, stderr=True, stream=True, # follow=True,
since=last_log_timestamp, tail=0):
c: bytes = c0
try:
s = c.decode('utf-8')
except:
s = c.decode('utf-8', errors='replace')
sys.stdout.write(s)
last_log_timestamp = datetime.datetime.now()
time.sleep(3)
except KeyboardInterrupt:
dtslogger.info('Received CTRL-C. Stopping container...')
container.stop()
dtslogger.info('Removing container')
container.remove()
dtslogger.info('Container removed.')
break
except BaseException:
s = traceback.format_exc()
if 'Read timed out' in s:
dtslogger.debug('(reattaching)')
else:
dtslogger.error(s)
dtslogger.info('Will try to re-attach to container.')
time.sleep(3)
def command(shell, args):
prog = 'dts duckiebot calibrate_intrinsics DUCKIEBOT_NAME'
usage = """
Calibrate:
%(prog)s
"""
parser = argparse.ArgumentParser(prog=prog, usage=usage)
parser.add_argument('hostname', default=None, help='Name of the Duckiebot to calibrate')
parser.add_argument('--base_image', dest='image',
default="duckietown/rpi-duckiebot-base:master19-no-arm")
parser.add_argument('--debug', action='store_true', default=False,
help="will enter you into the running container")
parsed_args = parser.parse_args(args)
hostname = parsed_args.hostname
duckiebot_ip = get_duckiebot_ip(hostname)
duckiebot_client = get_remote_client(duckiebot_ip)
# is the interface running?
try:
duckiebot_containers = duckiebot_client.containers.list()
interface_container_found = False
for c in duckiebot_containers:
if 'duckiebot-interface' in c.name:
interface_container_found = True
if not interface_container_found:
dtslogger.error("The duckiebot-interface is not running on the duckiebot")
exit()
except Exception as e:
dtslogger.warn(
"Not sure if the duckiebot-interface is running because we got and exception when trying: %s" % e)
# is the raw imagery being published?
try:
duckiebot_containers = duckiebot_client.containers.list()
raw_imagery_found = False
for c in duckiebot_containers:
if 'demo_camera' in c.name:
raw_imagery_found = True
if not raw_imagery_found:
dtslogger.error("The demo_camera is not running on the duckiebot - please run `dts duckiebot demo --demo_name camera --duckiebot_name %s" % hostname)
exit()
except Exception as e:
dtslogger.warn("%s" % e)
image = parsed_args.image
client = check_docker_environment()
container_name = "intrinsic_calibration_%s" % hostname
remove_if_running(client,container_name)
env = {'HOSTNAME': hostname,
'ROS_MASTER': hostname,
'DUCKIEBOT_NAME': hostname,
'ROS_MASTER_URI': 'http://%s:11311' % duckiebot_ip,
'QT_X11_NO_MITSHM': 1}
volumes = {}
subprocess.call(["xhost", "+"])
p = platform.system().lower()
if 'darwin' in p:
env['DISPLAY'] = 'host.docker.internal:0'
volumes = {
'/tmp/.X11-unix': {'bind': '/tmp/.X11-unix', 'mode': 'rw'}
}
else:
env['DISPLAY'] = os.environ['DISPLAY']
dtslogger.info("Running %s on localhost with environment vars: %s" %
(container_name, env))
dtslogger.info("When the window opens you will need to move the checkerboard around in front of the Duckiebot camera")
cmd = "roslaunch pi_camera intrinsic_calibration.launch veh:=%s" % hostname
params = {'image': image,
'name': container_name,
'network_mode': 'host',
'environment': env,
'privileged': True,
'stdin_open': True,
'tty': True,
'detach': True,
'command': cmd,
'volumes': volumes
}
container = client.containers.run(**params)
if parsed_args.debug:
attach_cmd = 'docker attach %s' % (container_name)
start_command_in_subprocess(attach_cmd)
def command(shell, args):
prog = 'dts duckiebot evaluate'
parser = argparse.ArgumentParser(prog=prog, usage=usage)
group = parser.add_argument_group('Basic')
group.add_argument('--duckiebot_name', default=None,
help="Name of the Duckiebot on which to perform evaluation")
group.add_argument('--duckiebot_username', default="duckie",
help="The duckiebot username")
group.add_argument('--image', dest='image_name',
help="Image to evaluate, if none specified then we will build your current context",
default=None)
group.add_argument('--glue_node_image', default="duckietown/challenge-aido_lf-duckiebot:aido2",
help="The node that glues your submission with ROS on the duckiebot. Probably don't change")
group.add_argument('--duration', help="Number of seconds to run evaluation", default=15)
group.add_argument('--remotely', action='store_true', default=True,
help="If true run the image over network without pushing to Duckiebot")
group.add_argument('--no_cache', help="disable cache on docker build",
action="store_true", default=False)
group.add_argument('--record_bag', action='store_true', default=False,
help="If true record a rosbag")
group.add_argument("--debug", action='store_true', default=False,
help="If true you will get a shell instead of executing")
group.add_argument('--max_vel', help="the max velocity for the duckiebot", default=0.7)
group.add_argument('--challenge', help="Specific challenge to evaluate")
parsed = parser.parse_args(args)
tmpdir = '/tmp'
USERNAME = getpass.getuser()
dir_home_guest = os.path.expanduser('~')
dir_fake_home = os.path.join(tmpdir, 'fake-%s-home' % USERNAME)
if not os.path.exists(dir_fake_home):
os.makedirs(dir_fake_home)
get_calibration_files(dir_fake_home, parsed.duckiebot_username, parsed.duckiebot_name)
client = check_docker_environment()
agent_container_name = "agent"
glue_container_name = "aido_glue"
# remove the containers if they are already running
remove_if_running(client, agent_container_name)
remove_if_running(client, glue_container_name)
# setup the fifos2 volume (requires pruning it if it's still hanging around from last time)
try:
client.volumes.prune()
fifo2_volume=client.volumes.create(name='fifos2')
except Exception as e:
dtslogger.warn("error creating volume: %s" % e)
duckiebot_ip = get_duckiebot_ip(parsed.duckiebot_name)
duckiebot_client = get_remote_client(duckiebot_ip)
try:
duckiebot_containers = duckiebot_client.containers.list()
interface_container_found=False
for c in duckiebot_containers:
if 'duckiebot-interface' in c.name:
interface_container_found=True
if not interface_container_found:
dtslogger.error("The duckiebot-interface is not running on the duckiebot")
except Exception as e:
dtslogger.warn("Not sure if the duckiebot-interface is running because we got and exception when trying: %s" % e)
# let's start building stuff for the "glue" node
glue_volumes = {fifo2_volume.name: {'bind': '/fifos', 'mode': 'rw'}}
glue_env = {'HOSTNAME':parsed.duckiebot_name,
'DUCKIEBOT_NAME':parsed.duckiebot_name,
'ROS_MASTER_URI':'http://%s:11311' % duckiebot_ip}
dtslogger.info("Running %s on localhost with environment vars: %s" %
(parsed.glue_node_image, glue_env))
params = {'image': parsed.glue_node_image,
'name': glue_container_name,
'network_mode': 'host',
'privileged': True,
'environment': glue_env,
'detach': True,
'tty': True,
'volumes': glue_volumes}
# run the glue container
glue_container = client.containers.run(**params)
if not parsed.debug:
monitor_thread = threading.Thread(target=continuously_monitor, args=(client, glue_container.name))
monitor_thread.start()
if parsed.image_name is None:
# if we didn't get an `image_name` we try need to build the local container
path = '.'
dockerfile = os.path.join(path, 'Dockerfile')
if not os.path.exists(dockerfile):
msg = 'No Dockerfile'
raise Exception(msg)
tag='myimage'
if parsed.no_cache:
cmd = ['docker', 'build', '--no-cache', '-t', tag, '-f', dockerfile]
else:
cmd = ['docker', 'build', '-t', tag, '-f', dockerfile]
dtslogger.info("Running command: %s" % cmd)
cmd.append(path)
subprocess.check_call(cmd)
image_name = tag
else:
image_name = parsed.image_name
# start to build the agent stuff
agent_env = {'AIDONODE_DATA_IN':'/fifos/agent-in',
'AIDONODE_DATA_OUT':'fifo:/fifos/agent-out'}
agent_volumes = {fifo2_volume.name: {'bind': '/fifos', 'mode': 'rw'},
dir_fake_home: {'bind': '/data/config', 'mode': 'rw'}
}
params = {'image': image_name,
'remove': True,
'name': agent_container_name,
'environment': agent_env,
'detach': True,
'tty': True,
'volumes': agent_volumes}
if parsed.debug:
params['command'] = '/bin/bash'
params['stdin_open'] = True
dtslogger.info("Running %s on localhost with environment vars: %s" % (image_name, agent_env))
agent_container = client.containers.run(**params)
if parsed.debug:
attach_cmd = 'docker attach %s' % agent_container_name
start_command_in_subprocess(attach_cmd)
else:
monitor_thread = threading.Thread(target=continuously_monitor,args=(client, agent_container_name))
monitor_thread.start()
duration = int(parsed.duration)
# should we record a bag?
if parsed.record_bag:
bag_container = record_bag(parsed.hostname, duration)
dtslogger.info("Running for %d s" % duration)
time.sleep(duration)
stop_container(glue_container)
stop_container(agent_container)
if parsed.record_bag:
stop_container(bag_container)
def command(shell, args):
# configure arguments
parser = argparse.ArgumentParser()
parser.add_argument('-C',
'--workdir',
default=None,
help="Directory containing the project to build")
parser.add_argument('-a',
'--arch',
default=DEFAULT_ARCH,
choices=set(CANONICAL_ARCH.values()),
help="Target architecture for the image to build")
parser.add_argument(
'-H',
'--machine',
default=DEFAULT_MACHINE,
help="Docker socket or hostname where to build the image")
parser.add_argument(
'--pull',
default=False,
action='store_true',
help="Whether to pull the latest base image used by the Dockerfile"
)
parser.add_argument('--no-cache',
default=False,
action='store_true',
help="Whether to use the Docker cache")
parser.add_argument(
'--no-multiarch',
default=False,
action='store_true',
help="Whether to disable multiarch support (based on bin_fmt)")
parser.add_argument(
'-f',
'--force',
default=False,
action='store_true',
help="Whether to force the build when the git index is not clean")
parser.add_argument('--push',
default=False,
action='store_true',
help="Whether to push the resulting image")
parser.add_argument(
'--rm',
default=False,
action='store_true',
help=
"Whether to remove the images once the build succeded (after pushing)"
)
parser.add_argument(
'--loop',
default=False,
action='store_true',
help=
"(Experimental) Whether to reuse the same base image to speed up the build process"
)
parser.add_argument(
'--ignore-watchtower',
default=False,
action='store_true',
help="Whether to ignore a running Docker watchtower")
parsed, _ = parser.parse_known_args(args=args)
# ---
code_dir = parsed.workdir if parsed.workdir else os.getcwd()
dtslogger.info('Project workspace: {}'.format(code_dir))
# show info about project
shell.include.devel.info.command(shell, args)
# get info about current repo
repo_info = shell.include.devel.info.get_repo_info(code_dir)
repo = repo_info['REPOSITORY']
branch = repo_info['BRANCH']
nmodified = repo_info['INDEX_NUM_MODIFIED']
nadded = repo_info['INDEX_NUM_ADDED']
# check if the index is clean
if nmodified + nadded > 0:
dtslogger.warning(
'Your index is not clean (some files are not committed).')
dtslogger.warning(
'If you know what you are doing, use --force (-f) to force the execution of the command.'
)
if not parsed.force:
exit(1)
dtslogger.warning('Forced!')
# create defaults
default_tag = "duckietown/%s:%s" % (repo, branch)
tag = "%s-%s" % (default_tag, parsed.arch)
# get info about docker endpoint
dtslogger.info('Retrieving info about Docker endpoint...')
epoint = _run_cmd([
'docker',
'-H=%s' % parsed.machine, 'info', '--format', '{{json .}}'
],
get_output=True,
print_output=False)
epoint = json.loads(epoint[0])
if 'ServerErrors' in epoint:
dtslogger.error('\n'.join(epoint['ServerErrors']))
return
epoint['MemTotal'] = _sizeof_fmt(epoint['MemTotal'])
print(DOCKER_INFO.format(**epoint))
# check if there is a watchtower instance running on the endpoint
if shell.include.devel.watchtower.is_running(parsed.machine):
dtslogger.warning(
'An instance of a Docker watchtower was found running on the Docker endpoint.'
)
dtslogger.warning(
'Building new images next to an active watchtower might (sure it will) create race conditions.'
)
dtslogger.warning('Solutions:')
dtslogger.warning(
' - Recommended: Use the command `dts devel watchtower stop [options]` to stop the watchtower.'
)
dtslogger.warning(
' - NOT Recommended: Use the flag `--ignore-watchtower` to ignore this warning and continue.'
)
if not parsed.ignore_watchtower:
exit(2)
dtslogger.warning('Ignored!')
# print info about multiarch
msg = 'Building an image for {} on {}.'.format(parsed.arch,
epoint['Architecture'])
dtslogger.info(msg)
# register bin_fmt in the target machine (if needed)
if not parsed.no_multiarch:
if epoint['Architecture'] not in ARCH_MAP[CANONICAL_ARCH[
parsed.arch]]:
dtslogger.info('Configuring machine for multiarch builds...')
try:
_run_cmd([
'docker',
'-H=%s' % parsed.machine, 'run', '--rm',
'--privileged', 'multiarch/qemu-user-static:register',
'--reset'
], True)
dtslogger.info('Multiarch Enabled!')
except:
msg = 'Multiarch cannot be enabled on the target machine. This might create issues.'
dtslogger.warning(msg)
else:
msg = 'Building an image for {} on {}. Multiarch not needed!'.format(
parsed.arch, epoint['Architecture'])
dtslogger.info(msg)
# define labels
buildlabels = []
# define build args
buildargs = ['--build-arg', 'ARCH={}'.format(parsed.arch)]
# loop mode (Experimental)
if parsed.loop:
buildargs += ['--build-arg', 'BASE_IMAGE={}'.format(repo)]
buildargs += [
'--build-arg', 'BASE_TAG={}-{}'.format(branch, parsed.arch)
]
buildlabels += ['--label', 'LOOP=1']
# ---
msg = "WARNING: Experimental mode 'loop' is enabled!. Use with caution"
dtslogger.warn(msg)
# build
buildlog = _run_cmd([
'docker',
'-H=%s' % parsed.machine,
'build',
'--pull=%d' % int(parsed.pull),
'--no-cache=%d' % int(parsed.no_cache),
'-t', tag] + \
buildlabels + \
buildargs + [
code_dir
], True, True)
# get image history
historylog = _run_cmd([
'docker',
'-H=%s' % parsed.machine, 'history', '-H=false', '--format',
'{{.ID}}:{{.Size}}', tag
], True)
historylog = [l.split(':') for l in historylog if len(l.strip()) > 0]
# run docker image analysis
ImageAnalyzer.process(buildlog, historylog, codens=100)
# image tagging
if parsed.arch == DEFAULT_ARCH:
dtslogger.info("Tagging image {} as {}.".format(tag, default_tag))
_run_cmd(
['docker',
'-H=%s' % parsed.machine, 'tag', tag, default_tag])
# perform push (if needed)
if parsed.push:
if not parsed.loop:
shell.include.devel.push.command(shell, args)
else:
msg = "Forbidden: You cannot push an image when using the experimental mode `--loop`."
dtslogger.warn(msg)
# perform remove (if needed)
if parsed.rm:
shell.include.devel.clean.command(shell, args)
def command(shell, args):
prog = 'dts challenges evaluate'
parser = argparse.ArgumentParser(prog=prog, usage=usage)
group = parser.add_argument_group('Basic')
group.add_argument('--no-cache',
action='store_true',
default=False,
help="")
group.add_argument('--no-build',
action='store_true',
default=False,
help="")
group.add_argument('--no-pull',
action='store_true',
default=False,
help="")
group.add_argument('--challenge',
help="Specific challenge to evaluate")
group.add_argument('--image',
help="Evaluator image to run",
default='duckietown/dt-challenges-evaluator:v4')
group.add_argument('--shell',
action='store_true',
default=False,
help="Runs a shell in the container")
group.add_argument('--output', help="", default='output')
group.add_argument('--visualize',
help="Visualize the evaluation",
action='store_true',
default=False)
parser.add_argument('--impersonate', type=str, default=None)
group.add_argument('-C', dest='change', default=None)
parsed = parser.parse_args(args)
if parsed.change:
os.chdir(parsed.change)
client = check_docker_environment()
command = ['dt-challenges-evaluate-local']
if parsed.no_cache:
command.append('--no-cache')
if parsed.no_build:
command.append('--no-build')
if parsed.challenge:
command.extend(['--challenge', parsed.challenge])
if parsed.impersonate:
command.extend(['--impersonate', parsed.impersonate])
output_rp = os.path.realpath(parsed.output)
command.extend(['--output', parsed.output])
#
# if parsed.features:
# dtslogger.debug('Passing features %r' % parsed.features)
# command += ['--features', parsed.features]
# fake_dir = '/submission'
tmpdir = '/tmp'
UID = os.getuid()
USERNAME = getpass.getuser()
dir_home_guest = '/fake-home/%s' % USERNAME # os.path.expanduser('~')
dir_fake_home_host = os.path.join(tmpdir, 'fake-%s-home' % USERNAME)
if not os.path.exists(dir_fake_home_host):
os.makedirs(dir_fake_home_host)
dir_fake_home_guest = dir_home_guest
dir_dtshell_host = os.path.join(os.path.expanduser('~'), '.dt-shell')
dir_dtshell_guest = os.path.join(dir_fake_home_guest, '.dt-shell')
dir_tmpdir_host = '/tmp'
dir_tmpdir_guest = '/tmp'
volumes = {
'/var/run/docker.sock': {
'bind': '/var/run/docker.sock',
'mode': 'rw'
}
}
d = os.path.join(os.getcwd(), parsed.output)
if not os.path.exists(d):
os.makedirs(d)
volumes[output_rp] = {'bind': d, 'mode': 'rw'}
volumes[os.getcwd()] = {'bind': os.getcwd(), 'mode': 'ro'}
volumes[dir_tmpdir_host] = {'bind': dir_tmpdir_guest, 'mode': 'rw'}
volumes[dir_dtshell_host] = {'bind': dir_dtshell_guest, 'mode': 'ro'}
volumes[dir_fake_home_host] = {
'bind': dir_fake_home_guest,
'mode': 'rw'
}
volumes['/etc/group'] = {'bind': '/etc/group', 'mode': 'ro'}
binds = [_['bind'] for _ in volumes.values()]
for b1 in binds:
for b2 in binds:
if b1 == b2:
continue
if b1.startswith(b2):
msg = 'Warning, it might be a problem to have binds with overlap'
msg += '\n b1: %s' % b1
msg += '\n b2: %s' % b2
dtslogger.warn(msg)
# command.extend(['-C', fake_dir])
env = {}
extra_environment = dict(username=USERNAME,
uid=UID,
USER=USERNAME,
HOME=dir_fake_home_guest)
env.update(extra_environment)
dtslogger.debug('Volumes:\n\n%s' %
yaml.safe_dump(volumes, default_flow_style=False))
dtslogger.debug('Environment:\n\n%s' %
yaml.safe_dump(env, default_flow_style=False))
from duckietown_challenges.rest import get_duckietown_server_url
url = get_duckietown_server_url()
dtslogger.info('The server URL is: %s' % url)
if 'localhost' in url:
h = socket.gethostname()
replacement = h + '.local'
dtslogger.warning(
'There is "localhost" inside, so I will try to change it to %r'
% replacement)
dtslogger.warning(
'This is because Docker cannot see the host as "localhost".')
url = url.replace("localhost", replacement)
dtslogger.warning('The new url is: %s' % url)
dtslogger.warning(
'This will be passed to the evaluator in the Docker container.'
)
env['DTSERVER'] = url
container_name = 'local-evaluator'
image = parsed.image
name, tag = image.split(':')
if not parsed.no_pull:
dtslogger.info('Updating container %s' % image)
dtslogger.info('This might take some time.')
client.images.pull(name, tag)
#
try:
container = client.containers.get(container_name)
except:
pass
else:
dtslogger.error('stopping previous %s' % container_name)
container.stop()
dtslogger.error('removing')
container.remove()
dtslogger.info('Starting container %s with %s' %
(container_name, image))
detach = True
env[DTShellConstants.DT1_TOKEN_CONFIG_KEY] = shell.get_dt1_token()
dtslogger.info('Container command: %s' % " ".join(command))
# add all the groups
on_mac = 'Darwin' in platform.system()
if on_mac:
group_add = []
else:
group_add = [
g.gr_gid for g in grp.getgrall() if USERNAME in g.gr_mem
]
interactive = False
if parsed.shell:
interactive = True
detach = False
command = ['/bin/bash', '-l']
params = dict(working_dir=os.getcwd(),
user=UID,
group_add=group_add,
command=command,
tty=interactive,
volumes=volumes,
environment=env,
remove=True,
network_mode='host',
detach=detach,
name=container_name)
dtslogger.info('Parameters:\n%s' % json.dumps(params, indent=4))
client.containers.run(image, **params)
if parsed.visualize:
start_rqt_image_view()
continuously_monitor(client, container_name)
def continuously_monitor(client, container_name):
from docker.errors import NotFound, APIError
dtslogger.debug('Monitoring container %s' % container_name)
last_log_timestamp = None
while True:
try:
container = client.containers.get(container_name)
except Exception as e:
# msg = 'Cannot get container %s: %s' % (container_name, e)
# dtslogger.info(msg)
break
# dtslogger.info('Will wait.')
# time.sleep(5)
# continue
dtslogger.info('status: %s' % container.status)
if container.status == 'exited':
msg = 'The container exited.'
logs = ''
for c in container.logs(stdout=True,
stderr=True,
stream=True,
since=last_log_timestamp):
last_log_timestamp = datetime.datetime.now()
logs += c.decode()
dtslogger.error(msg)
tf = 'evaluator.log'
with open(tf, 'w') as f:
f.write(logs)
msg = 'Logs saved at %s' % tf
dtslogger.info(msg)
# return container.exit_code
return # XXX
try:
for c in container.logs(stdout=True,
stderr=True,
stream=True,
follow=True,
since=last_log_timestamp):
if six.PY2:
sys.stdout.write(c)
else:
sys.stdout.write(c.decode('utf-8'))
last_log_timestamp = datetime.datetime.now()
time.sleep(3)
except KeyboardInterrupt:
dtslogger.info('Received CTRL-C. Stopping container...')
try:
container.stop()
dtslogger.info('Removing container')
container.remove()
dtslogger.info('Container removed.')
except NotFound:
pass
except APIError as e:
# if e.errno == 409:
#
pass
break
except BaseException:
dtslogger.error(traceback.format_exc())
dtslogger.info('Will try to re-attach to container.')
time.sleep(3)
