def _run_cmd(cmd, get_output=False, print_output=False):
dtslogger.debug('$ %s' % cmd)
if get_output:
proc = subprocess.Popen(cmd, stdout=subprocess.PIPE)
p = re.compile(CATKIN_REGEX, re.IGNORECASE)
lines = []
last_matched = False
for line in io.TextIOWrapper(proc.stdout, encoding="utf-8"):
line = line.rstrip()
if print_output:
matches = p.match(line.strip()) is not None
if matches and last_matched:
sys.stdout.write("\033[F")
sys.stdout.write(line + "\033[K" + "\n")
sys.stdout.flush()
last_matched = matches
if line:
lines.append(line)
proc.wait()
if proc.returncode != 0:
msg = 'The command {} returned exit code {}'.format(
cmd, proc.returncode)
dtslogger.error(msg)
raise RuntimeError(msg)
return lines
else:
subprocess.check_call(cmd)
def _run_cmd(cmd, get_output=False):
dtslogger.debug('$ %s' % cmd)
if get_output:
return [
l for l in subprocess.check_output(cmd).decode('utf-8').split('\n')
if l
]
else:
subprocess.check_call(cmd)
def build_image(client, path, challenge_name, step_name, service_name,
filename, no_cache: bool,
registry_info: RegistryInfo) -> BuildResult:
d = datetime.datetime.now()
username = get_dockerhub_username()
from duckietown_challenges.utils import tag_from_date
if username.lower() != username:
msg = f'Are you sure that the DockerHub username is not lowercase? You gave "{username}".'
dtslogger.warning(msg)
username = username.lower()
br = BuildResult(
repository=('%s-%s-%s' %
(challenge_name, step_name, service_name)).lower(),
organization=username,
registry=registry_info.registry,
tag=tag_from_date(d),
digest=None)
complete = get_complete_tag(br)
cmd = ['docker', 'build', '--pull', '-t', complete, '-f', filename]
if no_cache:
cmd.append('--no-cache')
cmd.append(path)
dtslogger.debug('$ %s' % " ".join(cmd))
subprocess.check_call(cmd)
image = client.images.get(complete)
repo_digests = image.attrs.get('RepoDigests', [])
if repo_digests:
msg = 'Already found repo digest: %s' % repo_digests
dtslogger.info(msg)
else:
dtslogger.info('Image not present on registry. Need to push.')
cmd = ['docker', 'push', complete]
dtslogger.debug('$ %s' % " ".join(cmd))
subprocess.check_call(cmd)
image = client.images.get(complete)
dtslogger.info('image id: %s' % image.id)
dtslogger.info('complete: %s' % get_complete_tag(br))
repo_digests = image.attrs.get('RepoDigests', [])
if not repo_digests:
msg = 'Could not find any repo digests (push not succeeded?)'
raise Exception(msg)
dtslogger.info('RepoDigests: %s' % repo_digests)
_, digest = repo_digests[0].split('@')
# br.digest = image.id
br.digest = digest
br = parse_complete_tag(get_complete_tag(br))
return br
def check_user_in_docker_group():
# first, let's see if there exists a group "docker"
group_names = [g.gr_name for g in grp.getgrall()]
G = 'docker'
if G not in group_names:
msg = 'No group %s defined.' % G
dtslogger.warning(msg)
else:
group_id = grp.getgrnam(G).gr_gid
my_groups = os.getgroups()
if group_id not in my_groups:
msg = 'My groups are %s and "%s" group is %s ' % (my_groups, G, group_id)
msg += '\n\nNote that when you add a user to a group, you need to login in and out.'
dtslogger.debug(msg)
def check_docker_environment():
username = getpass.getuser()
from . import dtslogger
dtslogger.debug('Checking docker environment for user %s' % username)
check_executable_exists('docker')
check_user_in_docker_group()
#
# if on_linux():
#
# if username != 'root':
# check_user_in_docker_group()
# # print('checked groups')
# else:
# dtslogger.debug('skipping env check because not on Linux')
try:
import docker
except Exception as e:
msg = 'Could not import package docker:\n%s' % e
msg += '\n\nYou need to install the package'
raise InvalidEnvironment(msg)
if 'DOCKER_HOST' in os.environ:
msg = 'Note that the variable DOCKER_HOST is set to "%s"' % os.environ[
'DOCKER_HOST']
dtslogger.warning(msg)
try:
client = docker.from_env()
containers = client.containers.list(filters=dict(status='running'))
# dtslogger.debug(json.dumps(client.info(), indent=4))
except Exception as e:
msg = 'I cannot communicate with Docker:\n%s' % e
msg += '\n\nMake sure the docker service is running.'
raise InvalidEnvironment(msg)
return client
def _run_cmd(cmd, get_output=False, print_output=False):
dtslogger.debug('$ %s' % cmd)
if get_output:
proc = subprocess.Popen(cmd, stdout=subprocess.PIPE)
lines = []
for line in io.TextIOWrapper(proc.stdout, encoding="utf-8"):
line = line.rstrip()
if print_output:
print(line)
if line:
lines.append(line)
proc.wait()
if proc.returncode != 0:
msg = 'The command {} returned exit code {}'.format(
cmd, proc.returncode)
dtslogger.error(msg)
raise RuntimeError(msg)
return lines
else:
subprocess.check_call(cmd)
def command(shell, args):
check_docker_environment()
home = os.path.expanduser('~')
prog = 'dts challenges evaluator'
parser = argparse.ArgumentParser(prog=prog, usage=usage)
group = parser.add_argument_group('Basic')
group.add_argument('--submission',
type=int,
default=None,
help='Run a specific submission.')
group.add_argument(
'--reset',
dest='reset',
action='store_true',
default=False,
help='(needs --submission) Re-evaluate the specific submission.')
group = parser.add_argument_group('Advanced')
group.add_argument('--no-watchtower',
dest='no_watchtower',
action='store_true',
default=False,
help="Disable starting of watchtower")
group.add_argument('--no-pull',
dest='no_pull',
action='store_true',
default=False,
help="Disable pulling of container")
group.add_argument('--image',
help="Evaluator image to run",
default='duckietown/dt-challenges-evaluator:v3')
group.add_argument('--name',
default=None,
help='Name for this evaluator')
group.add_argument("--features",
default=None,
help="Pretend to be what you are not.")
dtslogger.debug('args: %s' % args)
parsed = parser.parse_args(args)
machine_id = socket.gethostname()
if parsed.name is None:
container_name = '%s-%s' % (socket.gethostname(), os.getpid())
else:
container_name = parsed.name
import docker
client = docker.from_env()
command = ['dt-challenges-evaluator']
if parsed.submission:
command += ['--submission', str(parsed.submission)]
if parsed.reset:
command += ['--reset']
else:
command += ['--continuous']
command += ['--name', container_name]
command += ['--machine-id', machine_id]
if parsed.features:
dtslogger.debug('Passing features %r' % parsed.features)
command += ['--features', parsed.features]
volumes = {
'/var/run/docker.sock': {
'bind': '/var/run/docker.sock',
'mode': 'rw'
},
os.path.join(home, '.dt-shell'): {
'bind': '/root/.dt-shell',
'mode': 'ro'
},
'/tmp': {
'bind': '/tmp',
'mode': 'rw'
}
}
env = {}
UID = os.getuid()
USERNAME = getpass.getuser()
extra_environment = dict(username=USERNAME, uid=UID)
env.update(extra_environment)
if not parsed.no_watchtower:
ensure_watchtower_active(client)
url = get_duckietown_server_url()
dtslogger.info('The server URL is: %s' % url)
if 'localhost' in url:
h = socket.gethostname()
replacement = h + '.local'
dtslogger.warning(
'There is "localhost" inside, so I will try to change it to %r'
% replacement)
dtslogger.warning(
'This is because Docker cannot see the host as "localhost".')
url = url.replace("localhost", replacement)
dtslogger.warning('The new url is: %s' % url)
dtslogger.warning(
'This will be passed to the evaluator in the Docker container.'
)
env['DTSERVER'] = url
image = parsed.image
name, tag = image.split(':')
if not parsed.no_pull:
dtslogger.info('Updating container %s' % image)
client.images.pull(name, tag)
try:
container = client.containers.get(container_name)
except:
pass
else:
dtslogger.error('stopping previous %s' % container_name)
container.stop()
dtslogger.error('removing')
container.remove()
dtslogger.info('Starting container %s with %s' %
(container_name, image))
dtslogger.info('Container command: %s' % " ".join(command))
client.containers.run(image,
command=command,
volumes=volumes,
environment=env,
network_mode='host',
detach=True,
name=container_name,
tty=True)
while True:
try:
container = client.containers.get(container_name)
except Exception as e:
msg = 'Cannot get container %s: %s' % (container_name, e)
dtslogger.error(msg)
dtslogger.info('Will wait.')
time.sleep(5)
continue
dtslogger.info('status: %s' % container.status)
if container.status == 'exited':
msg = 'The container exited.'
logs = ''
for c in container.logs(stdout=True, stderr=True, stream=True):
logs += c
dtslogger.error(msg)
tf = 'evaluator.log'
with open(tf, 'w') as f:
f.write(logs)
msg = 'Logs saved at %s' % (tf)
dtslogger.info(msg)
break
try:
for c in container.logs(stdout=True,
stderr=True,
stream=True,
follow=True):
sys.stdout.write(c)
time.sleep(3)
except Exception as e:
dtslogger.error(e)
dtslogger.info('Will try to re-attach to container.')
time.sleep(3)
except KeyboardInterrupt:
dtslogger.info('Received CTRL-C. Stopping container...')
container.stop()
dtslogger.info('Removing container')
container.remove()
dtslogger.info('Container removed.')
break
def check_program_dependency(exe):
p = which(exe)
if p is None:
msg = 'Could not find program %r' % exe
raise Exception(msg)
dtslogger.debug('Found %r at %s' % (exe, p))
def _run_cmd(cmd):
dtslogger.debug('$ %s' % cmd)
subprocess.check_call(cmd)
def configure_images(parsed, user_data, add_file_local, add_file):
import psutil
# read and validate duckiebot-compose
stacks_to_load = parsed.stacks_to_load.split(',')
stacks_to_run = parsed.stacks_to_run.split(',')
dtslogger.info('Stacks to load: %s' % stacks_to_load)
dtslogger.info('Stacks to run: %s' % stacks_to_run)
for _ in stacks_to_run:
if _ not in stacks_to_load:
msg = 'If you want to run %r you need to load it as well.' % _
raise Exception(msg)
configuration = parsed.configuration
for cf in stacks_to_load:
# local path
lpath = get_resource(os.path.join('stacks', configuration, cf + '.yaml'))
# path on PI
rpath = '/var/local/%s.yaml' % cf
if which('docker-compose') is None:
msg = 'Could not find docker-compose. Cannot validate file.'
dtslogger.error(msg)
else:
_run_cmd(['docker-compose', '-f', lpath, 'config', '--quiet'])
add_file_local(path=rpath, local=lpath)
stack2yaml = get_stack2yaml(stacks_to_load, get_resource(os.path.join('stacks', configuration)))
if not stack2yaml:
msg = 'Not even one stack specified'
raise Exception(msg)
stack2info = save_images(stack2yaml, compress=parsed.compress)
buffer_bytes = 100 * 1024 * 1024
stacks_written = []
stack2archive_rpath = {}
dtslogger.debug(stack2info)
for stack, stack_info in stack2info.items():
tgz = stack_info.archive
size = os.stat(tgz).st_size
dtslogger.info('Considering copying %s of size %s' % (tgz, friendly_size_file(tgz)))
rpath = os.path.join('var', 'local', os.path.basename(tgz))
destination = os.path.join(TMP_ROOT_MOUNTPOINT, rpath)
available = psutil.disk_usage(TMP_ROOT_MOUNTPOINT).free
dtslogger.info('available %s' % friendly_size(available))
if available < size + buffer_bytes:
msg = 'You have %s available on %s but need %s for %s' % (
friendly_size(available), TMP_ROOT_MOUNTPOINT, friendly_size_file(tgz), tgz)
dtslogger.info(msg)
continue
dtslogger.info('OK, copying, and loading it on first boot.')
if os.path.exists(destination):
msg = 'Skipping copying image that already exist at %s.' % destination
dtslogger.info(msg)
else:
if which('rsync'):
cmd = ['sudo', 'rsync', '-avP', tgz, destination]
else:
cmd = ['sudo', 'cp', tgz, destination]
_run_cmd(cmd)
sync_data()
stack2archive_rpath[stack] = os.path.join('/', rpath)
stacks_written.append(stack)
client = check_docker_environment()
stacks_not_to_run = [_ for _ in stacks_to_load if _ not in stacks_to_run]
order = stacks_to_run + stacks_not_to_run
for cf in order:
if cf in stacks_written:
log_current_phase(user_data, PHASE_LOADING,
"Stack %s: Loading containers" % cf)
cmd = 'docker load --input %s && rm %s' % (stack2archive_rpath[cf], stack2archive_rpath[cf])
add_run_cmd(user_data, cmd)
add_file(stack2archive_rpath[cf] + '.labels.json',
json.dumps(stack2info[cf].image_name2id, indent=4))
# cmd = ['docker', 'load', '--input', stack2archive_rpath[cf]]
# add_run_cmd(user_data, cmd)
# cmd = ['rm', stack2archive_rpath[cf]]
# add_run_cmd(user_data, cmd)
for image_name, image_id in stack2info[cf].image_name2id.items():
image = client.images.get(image_name)
image_id = str(image.id)
dtslogger.info('id for %s: %s' % (image_name, image_id))
cmd = ['docker', 'tag', image_id, image_name]
print(cmd)
add_run_cmd(user_data, cmd)
if cf in stacks_to_run:
msg = 'Adding the stack %r as default running' % cf
dtslogger.info(msg)
log_current_phase(user_data, PHASE_LOADING, "Stack %s: docker-compose up" % cf)
cmd = ['docker-compose', '--file', '/var/local/%s.yaml' % cf, '-p', cf, 'up', '-d']
add_run_cmd(user_data, cmd)
# XXX
cmd = ['docker-compose', '-p', cf, '--file', '/var/local/%s.yaml' % cf, 'up', '-d']
user_data['bootcmd'].append(cmd) # every boot
# The RPi blinking feedback expects that "All stacks up" will be written to the /data/boot-log.txt file.
# If modifying, make sure to adjust the blinking feedback
log_current_phase(user_data, PHASE_DONE, "All stacks up")
def step_expand(shell, parsed):
deps = ['parted', 'resize2fs', 'e2fsck', 'lsblk', 'fdisk', 'umount']
for dep in deps:
check_program_dependency(dep)
global SD_CARD_DEVICE
if not os.path.exists(SD_CARD_DEVICE):
msg = 'This only works assuming device == %s' % SD_CARD_DEVICE
raise Exception(msg)
else:
msg = 'Found device %s.' % SD_CARD_DEVICE
dtslogger.info(msg)
# Some devices get only a number added to the disk name, other get p + a number
if os.path.exists(SD_CARD_DEVICE + '1'):
DEVp1 = SD_CARD_DEVICE + '1'
DEVp2 = SD_CARD_DEVICE + '2'
elif os.path.exists(SD_CARD_DEVICE + 'p1'):
DEVp1 = SD_CARD_DEVICE + 'p1'
DEVp2 = SD_CARD_DEVICE + 'p2'
else:
msg = 'The two partitions of device %s could not be found.' % SD_CARD_DEVICE
raise Exception(msg)
# Unmount the devices and check if this worked, otherwise parted will fail
p = subprocess.Popen(['lsblk'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
ret, err = p.communicate()
if DEVp1 in ret.decode('utf-8'):
cmd = ['sudo', 'umount', DEVp1]
_run_cmd(cmd)
if DEVp2 in ret.decode('utf-8'):
cmd = ['sudo', 'umount', DEVp2]
_run_cmd(cmd)
p = subprocess.Popen(['lsblk'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
ret, err = p.communicate()
if DEVp1 in ret.decode('utf-8') or DEVp2 in ret.decode('utf-8'):
msg = 'Automatic unmounting of %s and %s was unsuccessful. Please do it manually and run again.' % (
DEVp1, DEVp2)
raise Exception(msg)
# Do the expansion
dtslogger.info('Current status:')
cmd = ['sudo', 'lsblk', SD_CARD_DEVICE]
_run_cmd(cmd)
# get the disk identifier of the SD card.
# IMPORTANT: This must be executed before `parted`
p = re.compile(".*Disk identifier: 0x([0-9a-z]*).*")
cmd = ['sudo' ,'fdisk', '-l', SD_CARD_DEVICE]
dtslogger.debug('$ %s' % cmd)
pc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
ret, err = pc.communicate()
m = p.search(ret.decode('utf-8'))
uuid = m.group(1)
cmd = ['sudo', 'parted', '-s', SD_CARD_DEVICE, 'resizepart', '2', '100%']
_run_cmd(cmd)
cmd = ['sudo', 'e2fsck', '-f', DEVp2]
_run_cmd(cmd)
cmd = ['sudo', 'resize2fs', DEVp2]
_run_cmd(cmd)
# restore the original disk identifier
cmd = ['sudo' ,'fdisk', SD_CARD_DEVICE]
dtslogger.debug('$ %s' % cmd)
p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE)
ret, err = p.communicate(input=('x\ni\n0x%s\nr\nw' % uuid).encode('ascii'))
print(ret.decode('utf-8'))
dtslogger.info('Updated status:')
cmd = ['sudo', 'lsblk', SD_CARD_DEVICE]
_run_cmd(cmd)
def command(shell, args):
check_docker_environment()
home = os.path.expanduser('~')
prog = 'dts challenges evaluator'
parser = argparse.ArgumentParser(prog=prog, usage=usage)
group = parser.add_argument_group('Basic')
group.add_argument('--submission', type=int, default=None,
help='Run a specific submission.')
group.add_argument('--reset', dest='reset', action='store_true', default=False,
help='(needs --submission) Re-evaluate the specific submission.')
group = parser.add_argument_group('Advanced')
group.add_argument('--no-watchtower', dest='no_watchtower', action='store_true', default=False,
help="Disable starting of watchtower")
group.add_argument('--no-pull', dest='no_pull', action='store_true', default=False,
help="Disable pulling of containers")
group.add_argument('--no-upload', dest='no_upload', action='store_true', default=False,
help="Disable upload of artifacts")
group.add_argument('--no-delete', dest='no_delete', action='store_true', default=False,
help="Does not erase temporary files in /tmp/duckietown")
group.add_argument('--image', help="Evaluator image to run", default='duckietown/dt-challenges-evaluator:v4')
group.add_argument('--name', default=None, help='Name for this evaluator')
group.add_argument("--features", default=None, help="Pretend to be what you are not.")
group.add_argument("--ipfs", action='store_true', default=False, help="Run with IPFS available")
group.add_argument("--one", action='store_true', default=False, help="Only run 1 submission")
# dtslogger.debug('args: %s' % args)
parsed = parser.parse_args(args)
machine_id = socket.gethostname()
if parsed.name is None:
container_name = '%s-%s' % (socket.gethostname(), os.getpid())
else:
container_name = parsed.name
client = check_docker_environment()
command = ['dt-challenges-evaluator']
if parsed.submission:
command += ['--submission', str(parsed.submission)]
if parsed.reset:
command += ['--reset']
else:
if not parsed.one:
command += ['--continuous']
command += ['--name', container_name]
command += ['--machine-id', machine_id]
if parsed.no_upload:
command += ['--no-upload']
if parsed.no_pull:
command += ['--no-pull']
if parsed.no_delete:
command += ['--no-delete']
if parsed.one:
command += ['--one']
if parsed.features:
dtslogger.debug('Passing features %r' % parsed.features)
command += ['--features', parsed.features]
mounts = []
volumes = {
'/var/run/docker.sock': {'bind': '/var/run/docker.sock', 'mode': 'rw'},
os.path.join(home, '.dt-shell'): {'bind': '/root/.dt-shell', 'mode': 'ro'},
'/tmp': {'bind': '/tmp', 'mode': 'rw'}
}
if parsed.ipfs:
if not ipfs_available():
msg = 'IPFS not available/mounted correctly.'
raise UserError(msg)
command += ['--ipfs']
# volumes['/ipfs'] = {'bind': '/ipfs', 'mode': 'ro'}
from docker.types import Mount
mount = Mount(type='bind', source='/ipfs', target='/ipfs', read_only=True)
mounts.append(mount)
env = {}
UID = os.getuid()
USERNAME = getpass.getuser()
extra_environment = dict(username=USERNAME, uid=UID)
env.update(extra_environment)
if not parsed.no_watchtower:
ensure_watchtower_active(client)
from duckietown_challenges.rest import get_duckietown_server_url
url = get_duckietown_server_url()
dtslogger.info('The server URL is: %s' % url)
if 'localhost' in url:
h = socket.gethostname()
replacement = h + '.local'
dtslogger.warning('There is "localhost" inside, so I will try to change it to %r' % replacement)
dtslogger.warning('This is because Docker cannot see the host as "localhost".')
url = url.replace("localhost", replacement)
dtslogger.warning('The new url is: %s' % url)
dtslogger.warning('This will be passed to the evaluator in the Docker container.')
env['DTSERVER'] = url
image = parsed.image
dtslogger.info('Using evaluator image %s' % image)
name, tag = image.split(':')
if not parsed.no_pull:
dtslogger.info('Updating container %s' % image)
client.images.pull(name, tag)
# noinspection PyBroadException
try:
container = client.containers.get(container_name)
except:
pass
else:
dtslogger.error('stopping previous %s' % container_name)
container.stop()
dtslogger.error('removing')
container.remove()
dtslogger.info('Starting container %s with %s' % (container_name, image))
dtslogger.info('Container command: %s' % " ".join(command))
# add all the groups
import grp
group_add = [g.gr_gid for g in grp.getgrall() if USERNAME in g.gr_mem]
client.containers.run(image,
group_add=group_add,
command=command,
volumes=volumes,
environment=env,
mounts=mounts,
network_mode='host',
detach=True,
name=container_name,
tty=True)
last_log_timestamp = None
while True:
try:
container = client.containers.get(container_name)
except Exception as e:
msg = 'Cannot get container %s: %s' % (container_name, e)
dtslogger.error(msg)
dtslogger.info('Will wait.')
time.sleep(5)
continue
dtslogger.info('status: %s' % container.status)
if container.status == 'exited':
logs = ''
for c in container.logs(stdout=True, stderr=True, stream=True, since=last_log_timestamp):
logs += c.decode('utf-8')
last_log_timestamp = datetime.datetime.now()
tf = 'evaluator.log'
with open(tf, 'w') as f:
f.write(logs)
msg = 'The container exited.'
msg += '\nLogs saved at %s' % tf
dtslogger.info(msg)
break
try:
if last_log_timestamp is not None:
print('since: %s' % last_log_timestamp.isoformat())
for c0 in container.logs(stdout=True, stderr=True, stream=True, # follow=True,
since=last_log_timestamp, tail=0):
c: bytes = c0
try:
s = c.decode('utf-8')
except:
s = c.decode('utf-8', errors='replace')
sys.stdout.write(s)
last_log_timestamp = datetime.datetime.now()
time.sleep(3)
except KeyboardInterrupt:
dtslogger.info('Received CTRL-C. Stopping container...')
container.stop()
dtslogger.info('Removing container')
container.remove()
dtslogger.info('Container removed.')
break
except BaseException:
s = traceback.format_exc()
if 'Read timed out' in s:
dtslogger.debug('(reattaching)')
else:
dtslogger.error(s)
dtslogger.info('Will try to re-attach to container.')
time.sleep(3)
def command(shell, args):
parser = argparse.ArgumentParser()
parser.add_argument('--image',
default=IMAGE,
help="Which image to use")
parsed = parser.parse_args(args=args)
image = parsed.image
check_docker_environment()
pwd = os.getcwd()
bookdir = os.path.join(pwd, 'book')
if not os.path.exists(bookdir):
msg = 'Could not find "book" directory %r.' % bookdir
raise UserError(msg)
# check that the resources directory is present
resources = os.path.join(pwd, 'resources')
if not os.path.exists(os.path.join(resources, 'templates')):
msg = 'It looks like that the "resources" repo is not checked out.'
msg += '\nMaybe try:\n'
msg += '\n git submodule init'
msg += '\n git submodule update'
raise Exception(msg) # XXX
entries = list(os.listdir(bookdir))
entries = [_ for _ in entries if not _[0] == '.']
if len(entries) > 1:
msg = 'Found more than one directory in "book": %s' % entries
DTCommandAbs.fail(msg)
bookname = entries[0]
src = os.path.join(bookdir, bookname)
git_version = system_cmd_result(pwd, ['git', '--version']).strip()
dtslogger.debug('git version: %s' % git_version)
cmd = ['git', 'rev-parse', '--show-superproject-working-tree']
gitdir_super = system_cmd_result(pwd, cmd).strip()
dtslogger.debug('gitdir_super: %r' % gitdir_super)
gitdir = system_cmd_result(pwd, ['git', 'rev-parse', '--show-toplevel']).strip()
dtslogger.debug('gitdir: %r' % gitdir)
if '--show' in gitdir_super: # or not gitdir_super:
msg = "Your git version is too low, as it does not support --show-superproject-working-tree"
msg += '\n\nDetected: %s' % git_version
raise InvalidEnvironment(msg)
if '--show' in gitdir or not gitdir:
msg = "Your git version is too low, as it does not support --show-toplevel"
msg += '\n\nDetected: %s' % git_version
raise InvalidEnvironment(msg)
pwd1 = os.path.realpath(pwd)
user = getpass.getuser()
tmpdir = '/tmp'
fake_home = os.path.join(tmpdir, 'fake-%s-home' % user)
if not os.path.exists(fake_home):
os.makedirs(fake_home)
resources = 'resources'
uid1 = os.getuid()
if sys.platform == 'darwin':
flag = ':delegated'
else:
flag = ''
cache = '/tmp/cache'
if not os.path.exists(cache):
os.makedirs(cache)
cmd = ['docker', 'run',
'-v', '%s:%s%s' % (gitdir, gitdir, flag),
'-v', '%s:%s%s' % (pwd1, pwd1, flag),
'-v', '%s:%s%s' % (cache, cache, flag),
'-v', '%s:%s%s' % (fake_home, '/home/%s' % user, flag),
'-e', 'USER=%s' % user,
'-e', 'USERID=%s' % uid1,
'-m', '4GB',
'--user', '%s' % uid1]
if gitdir_super:
cmd += ['-v', '%s:%s%s' % (gitdir_super, gitdir_super, flag)]
interactive = True
if interactive:
cmd.append('-it')
cmd += [
image,
'/project/run-book-native.sh',
bookname,
src,
resources,
pwd1
]
dtslogger.info('executing:\nls ' + " ".join(cmd))
# res = system_cmd_result(pwd, cmd, raise_on_error=True)
try:
p = subprocess.Popen(cmd, bufsize=0, executable=None, stdin=None, stdout=None, stderr=None, preexec_fn=None,
shell=False, cwd=pwd, env=None)
except OSError as e:
if e.errno == 2:
msg = 'Could not find "docker" executable.'
DTCommandAbs.fail(msg)
raise
p.communicate()
dtslogger.info('\n\nCompleted.')
def command(shell, args):
prog = 'dts challenges evaluate'
parser = argparse.ArgumentParser(prog=prog, usage=usage)
group = parser.add_argument_group('Basic')
group.add_argument('--no-cache',
action='store_true',
default=False,
help="")
group.add_argument('--no-build',
action='store_true',
default=False,
help="")
group.add_argument('--no-pull',
action='store_true',
default=False,
help="")
group.add_argument('--challenge',
help="Specific challenge to evaluate")
group.add_argument('--image',
help="Evaluator image to run",
default='duckietown/dt-challenges-evaluator:v4')
group.add_argument('--shell',
action='store_true',
default=False,
help="Runs a shell in the container")
group.add_argument('--output', help="", default='output')
group.add_argument('--visualize',
help="Visualize the evaluation",
action='store_true',
default=False)
parser.add_argument('--impersonate', type=str, default=None)
group.add_argument('-C', dest='change', default=None)
parsed = parser.parse_args(args)
if parsed.change:
os.chdir(parsed.change)
client = check_docker_environment()
command = ['dt-challenges-evaluate-local']
if parsed.no_cache:
command.append('--no-cache')
if parsed.no_build:
command.append('--no-build')
if parsed.challenge:
command.extend(['--challenge', parsed.challenge])
if parsed.impersonate:
command.extend(['--impersonate', parsed.impersonate])
output_rp = os.path.realpath(parsed.output)
command.extend(['--output', parsed.output])
#
# if parsed.features:
# dtslogger.debug('Passing features %r' % parsed.features)
# command += ['--features', parsed.features]
# fake_dir = '/submission'
tmpdir = '/tmp'
UID = os.getuid()
USERNAME = getpass.getuser()
dir_home_guest = '/fake-home/%s' % USERNAME # os.path.expanduser('~')
dir_fake_home_host = os.path.join(tmpdir, 'fake-%s-home' % USERNAME)
if not os.path.exists(dir_fake_home_host):
os.makedirs(dir_fake_home_host)
dir_fake_home_guest = dir_home_guest
dir_dtshell_host = os.path.join(os.path.expanduser('~'), '.dt-shell')
dir_dtshell_guest = os.path.join(dir_fake_home_guest, '.dt-shell')
dir_tmpdir_host = '/tmp'
dir_tmpdir_guest = '/tmp'
volumes = {
'/var/run/docker.sock': {
'bind': '/var/run/docker.sock',
'mode': 'rw'
}
}
d = os.path.join(os.getcwd(), parsed.output)
if not os.path.exists(d):
os.makedirs(d)
volumes[output_rp] = {'bind': d, 'mode': 'rw'}
volumes[os.getcwd()] = {'bind': os.getcwd(), 'mode': 'ro'}
volumes[dir_tmpdir_host] = {'bind': dir_tmpdir_guest, 'mode': 'rw'}
volumes[dir_dtshell_host] = {'bind': dir_dtshell_guest, 'mode': 'ro'}
volumes[dir_fake_home_host] = {
'bind': dir_fake_home_guest,
'mode': 'rw'
}
volumes['/etc/group'] = {'bind': '/etc/group', 'mode': 'ro'}
binds = [_['bind'] for _ in volumes.values()]
for b1 in binds:
for b2 in binds:
if b1 == b2:
continue
if b1.startswith(b2):
msg = 'Warning, it might be a problem to have binds with overlap'
msg += '\n b1: %s' % b1
msg += '\n b2: %s' % b2
dtslogger.warn(msg)
# command.extend(['-C', fake_dir])
env = {}
extra_environment = dict(username=USERNAME,
uid=UID,
USER=USERNAME,
HOME=dir_fake_home_guest)
env.update(extra_environment)
dtslogger.debug('Volumes:\n\n%s' %
yaml.safe_dump(volumes, default_flow_style=False))
dtslogger.debug('Environment:\n\n%s' %
yaml.safe_dump(env, default_flow_style=False))
from duckietown_challenges.rest import get_duckietown_server_url
url = get_duckietown_server_url()
dtslogger.info('The server URL is: %s' % url)
if 'localhost' in url:
h = socket.gethostname()
replacement = h + '.local'
dtslogger.warning(
'There is "localhost" inside, so I will try to change it to %r'
% replacement)
dtslogger.warning(
'This is because Docker cannot see the host as "localhost".')
url = url.replace("localhost", replacement)
dtslogger.warning('The new url is: %s' % url)
dtslogger.warning(
'This will be passed to the evaluator in the Docker container.'
)
env['DTSERVER'] = url
container_name = 'local-evaluator'
image = parsed.image
name, tag = image.split(':')
if not parsed.no_pull:
dtslogger.info('Updating container %s' % image)
dtslogger.info('This might take some time.')
client.images.pull(name, tag)
#
try:
container = client.containers.get(container_name)
except:
pass
else:
dtslogger.error('stopping previous %s' % container_name)
container.stop()
dtslogger.error('removing')
container.remove()
dtslogger.info('Starting container %s with %s' %
(container_name, image))
detach = True
env[DTShellConstants.DT1_TOKEN_CONFIG_KEY] = shell.get_dt1_token()
dtslogger.info('Container command: %s' % " ".join(command))
# add all the groups
on_mac = 'Darwin' in platform.system()
if on_mac:
group_add = []
else:
group_add = [
g.gr_gid for g in grp.getgrall() if USERNAME in g.gr_mem
]
interactive = False
if parsed.shell:
interactive = True
detach = False
command = ['/bin/bash', '-l']
params = dict(working_dir=os.getcwd(),
user=UID,
group_add=group_add,
command=command,
tty=interactive,
volumes=volumes,
environment=env,
remove=True,
network_mode='host',
detach=detach,
name=container_name)
dtslogger.info('Parameters:\n%s' % json.dumps(params, indent=4))
client.containers.run(image, **params)
if parsed.visualize:
start_rqt_image_view()
continuously_monitor(client, container_name)
def continuously_monitor(client, container_name):
from docker.errors import NotFound, APIError
dtslogger.debug('Monitoring container %s' % container_name)
last_log_timestamp = None
while True:
try:
container = client.containers.get(container_name)
except Exception as e:
# msg = 'Cannot get container %s: %s' % (container_name, e)
# dtslogger.info(msg)
break
# dtslogger.info('Will wait.')
# time.sleep(5)
# continue
dtslogger.info('status: %s' % container.status)
if container.status == 'exited':
msg = 'The container exited.'
logs = ''
for c in container.logs(stdout=True,
stderr=True,
stream=True,
since=last_log_timestamp):
last_log_timestamp = datetime.datetime.now()
logs += c.decode()
dtslogger.error(msg)
tf = 'evaluator.log'
with open(tf, 'w') as f:
f.write(logs)
msg = 'Logs saved at %s' % tf
dtslogger.info(msg)
# return container.exit_code
return # XXX
try:
for c in container.logs(stdout=True,
stderr=True,
stream=True,
follow=True,
since=last_log_timestamp):
if six.PY2:
sys.stdout.write(c)
else:
sys.stdout.write(c.decode('utf-8'))
last_log_timestamp = datetime.datetime.now()
time.sleep(3)
except KeyboardInterrupt:
dtslogger.info('Received CTRL-C. Stopping container...')
try:
container.stop()
dtslogger.info('Removing container')
container.remove()
dtslogger.info('Container removed.')
except NotFound:
pass
except APIError as e:
# if e.errno == 409:
#
pass
break
except BaseException:
dtslogger.error(traceback.format_exc())
dtslogger.info('Will try to re-attach to container.')
time.sleep(3)
