def test_run(*sigs, **kwargs):
"""Prepare the test for running or reporting."""
build_dn = kwargs.pop('build', 'docker_builds')
username = kwargs.pop('username', container_user)
config_fn = kwargs.pop('config', None)
visit = kwargs.pop('visit', False)
if config_fn == None:
config_fn = read_config().get('docks_config', 'docker_config.py')
mods_fn = kwargs.pop('mods', None)
backrun = kwargs.pop('back', False)
dump_raw_test = kwargs.pop('dump_raw_test', None)
if kwargs: raise Exception('unprocessed kwargs %s' % kwargs)
# get the interpreted docker configuration
instruct = interpret_docker_instructions(config=config_fn, mods=mods_fn)
# use the sigs list to select a test set
tests = instruct.get('tests', {})
keys = [key for key in tests if set(sigs) == set(key.split())]
if len(keys) != 1:
raise Exception(
'cannot find a unique key in the testset for sigs %s: %s' %
(sigs, tests.keys()))
else:
name = keys[0]
if dump_raw_test:
import yaml
with open(dump_raw_test, 'w') as fp:
yaml.dump(tests[name],
fp,
default_flow_style=False,
default_style='|')
return dict(config_fn=config_fn,
background=backrun,
visit=visit,
**tests[name])
def avail(config=None, mods=None, **kwargs):
"""List available tests."""
build_dn = kwargs.pop('build', 'builds')
toc_fn = kwargs.pop('toc_fn', 'docker.json')
username = kwargs.pop('username', container_user)
config_dict = read_config()
if config == None:
config = config_dict.get('docks_config', 'docker_config.py')
if kwargs: raise Exception('unprocessed kwargs %s' % kwargs)
# get the interpreted docker configuration
instruct = interpret_docker_instructions(config=config, mods=mods)
from datapack import asciitree
tests_these = dict(tests=instruct.get('tests', {}).keys())
asciitree(tests_these)
return tests_these
def docker_recap(longest=True, log=False):
"""Summarize docker compile times."""
config = read_config()
from datapack import asciitree
docker_history = config.get('docker_history', {})
keys = list(set([i[0] for i in docker_history]))
timings = dict([(key, {}) for key in keys])
for key in keys:
stamps = [i[1] for i in docker_history if i[0] == key]
timings[key]['timings'] = dict([
(s, '%.1f min' % (docker_history[(key, s)]['total_time'] / 60.))
for s in stamps
])
timings[key]['sub-timings'] = [
'%s, %.1f min' % (s['name'], s['elapsed'] / 60.)
for s in docker_history[(key, s)]['series']
]
timings[key]['longest'] = max(timings[key]['timings'].values())
if not log: asciitree(timings)
else: print(json.dumps(timings))
def docker(name,
config=None,
report=None,
sequential=False,
mods=None,
**kwargs):
"""
Manage the DOCKER.
"""
build_dn = kwargs.pop('build', 'builds')
toc_fn = kwargs.pop('toc_fn', 'docker.json')
username = kwargs.pop('username', container_user)
config_dict = read_config()
if config == None:
config = config_dict.get('docks_config', 'docker_config.py')
if kwargs: raise Exception('unprocessed kwargs %s' % kwargs)
# get the interpreted docker configuration
instruct = interpret_docker_instructions(config=config, mods=mods)
config = config_dict
# the name is a sequence
if name not in instruct.get('sequences', {}):
raise Exception('docker configuration lacks a sequence called %s' %
name)
seqspec = instruct['sequences'][name]
# prepare a build directory
if os.path.isdir(build_dn): shutil.rmtree(build_dn)
os.mkdir(build_dn)
# get the docker history
docker_history = config.get('docker_history', {})
# process all requirements before making the texts
reqs = instruct.get('requirements', {})
for key, val in reqs.items():
# only check the requirement if the key points to an item in the sequence
if key not in seqspec.get('seq', '').split(): continue
# perform a simple copy command in docker with a regex substitution for the filename
if set(val.keys()) >= set(['config_keys', 'filename_sub']):
key_path = val['config_keys']
key_path = tuple([key_path
]) if type(key_path) in str_types else key_path
#! previously used delve from datapack to do a nested lookup in config, however top-level
#! ...keys should be fine and the testset_processor uses a raw lookup of config and
#! ...I would prefer to have the testsets be self-contained.
if len(key_path) != 1:
raise Exception(
'not ready to do nested lookups. need to add delve back.')
if key_path[0] not in config:
raise Exception(
'failed to get item %s from the config dictionary.' %
str(key_path))
spot = config[key_path[0]]
# substitute in the dockerfile
if key not in instruct['dockerfiles']:
raise Exception('cannot find %s in dockerfiles' % key)
instruct['dockerfiles'][key] = re.sub(val['filename_sub'],
os.path.basename(spot),
instruct['dockerfiles'][key])
for sub_from, sub_to in val.get('subs', {}).items():
instruct['dockerfiles'][key] = re.sub(
sub_from, sub_to, instruct['dockerfiles'][key])
# we always have to copy the file to the docker build directory
shutil.copyfile(spot, os.path.join(build_dn,
os.path.basename(spot)))
else:
raise Exception('cannot get requirement for %s: %s' % (key, val))
# we allow the sequence to be a dictionary (extra features) or a string (default)
if type(seqspec) in str_types: seqspec = {'seq': seqspec}
# defaults and extra settings passed through a sequence dictionary
seq = seqspec['seq']
user_coda = seqspec.get('user', False)
coda = seqspec.get('coda', None)
if coda != None and user_coda == False:
raise Exception('cannot allow a coda if not user')
# prepare the texts of the dockerfiles
steps = seq.split()
texts = [(step, instruct['dockerfiles'][step]) for step in steps]
# final stage adds the user
this_user = pwd.getpwnam(os.environ['USER'])
this_user_details = {
'gid': this_user.pw_gid,
'uid': this_user.pw_uid,
'user': os.environ['USER']
}
this_user_details.update(
gname=grp.getgrgid(this_user_details['gid']).gr_name)
# handle special keys
special_keys = {'@USER'}
for ii, (step, text) in enumerate(texts):
# handle config calls with some hardcoded items
special_keys_this = set(re.findall(
'@[\w_]+', texts[ii][1])).intersection(special_keys)
for key in special_keys_this:
if key == '@USER':
texts[ii] = (step,
re.sub('@USER', this_user_details['user'],
texts[ii][1]))
else:
raise Exception
# substitutions of @read_config('<key>') from the config
for ii, (step, text) in enumerate(texts):
comp = re.compile(r'@read_config\((.*?)\)', flags=re.M)
for config_key in comp.findall(texts[ii][1]):
val = config.get(eval(config_key))
texts[ii] = (step,
re.sub('@read_config\(%s\)' % config_key, val,
texts[ii][1]))
# at the end of each run we set the user so that permissions work properly and dockers are run as user
if user_coda:
this_user_details['user_passwd'] = config.get('user_creds')
if not this_user_details['user_passwd']:
raise Exception(
'set docker user password with: `make set user_creds="password"`'
)
texts += [
('su', "RUN groupmod -g %(gid)d %(gname)s\n" % this_user_details +
("RUN useradd -m -u %(uid)d -g %(gid)d %(user)s\n" +
"RUN echo '%(user)s:%(user_passwd)s' | chpasswd\n" +
"USER %(user)s\nWORKDIR /home/%(user)s\n") % this_user_details)
]
# report on what happened
docker_details = dict(user_coda=user_coda)
# commands to run after setting the user
if coda != None: texts += [('coda', coda)]
# if we are reporting then write the file and exit
if report != None:
with open(report, 'w') as fp:
fp.write('\n'.join(list(zip(*texts))[1]))
return docker_details
# never rebuild if unnecessary (docker builds are extremely quick but why waste the time)
# we use the texts of the docker instead of timestamps, since users might be updating other parts
# ... of the config file pretty frequently
if docker_history.keys():
keys = sorted([i for i in docker_history.keys() if i[0] == name])
if keys:
if docker_history[keys[-1]]['texts'] == texts:
print(
('[STATUS] the docker called "%s" has already been built '
+ 'and the instructions have not changed') % name)
return docker_details
# record the history for docker_history
updates, total_time = [], 0.0
# in the sequential method we generate larger dockerfiles from small ones so that adding
# ... to the end of a long dockerfile resumes at the previous image. since the sequential dockerfiles
# ... are always built in the same way, this saves time
#! the sequential feature and the text checking feature is highly redundant with docker
#! note clean cluttered images with `docker rmi $(docker images | awk '$1 ~ /-s/ {print $3}')`
if sequential:
# loop over stages, each of which gets a separate image
for stage, (stage_name, text) in enumerate(texts):
start_time = time.time()
# prepare names
print('[STATUS] processing %s, stage %d: %s' %
(name, stage, stage_name))
# subsequent stages depend on the previous one so we prepend it
if stage > 0: text = 'FROM %s/%s\n' % (username, image_name) + text
image_name = '%s-s%d' % (name, stage)
if stage == len(texts) - 1: image_name = name
print('[STATUS] image name: %s' % image_name)
print('\n'.join(['[CONFIG] | %s' % i for i in text.splitlines()]))
docker_fn = os.path.join(build_dn, 'Dockerfile-%s' % image_name)
# write the docker file
with open(docker_fn, 'w') as fp:
fp.write(text)
# generate the image
cmd = 'docker build -t %s/%s -f %s %s' % (
username, image_name, docker_fn, os.path.join(build_dn, ''))
print('[STATUS] running "%s"' % cmd)
subprocess.check_call(cmd, shell=True)
elapsed_sec = time.time() - start_time
total_time += elapsed_sec
elapsed = '%.1f min' % (elapsed_sec / 60.)
print('[TIME] elapsed: %s' % elapsed)
updates.append(
dict(name=stage_name, image=image_name, elapsed=elapsed_sec))
# standard, non-sequential method
else:
start_time = time.time()
image_name = name
text = '\n'.join(list(zip(*texts))[1])
docker_fn = os.path.join(build_dn, 'Dockerfile-%s' % name)
# write the docker file
with open(docker_fn, 'w') as fp:
fp.write(text)
cmd = 'docker build -t %s/%s -f %s %s' % (
username, image_name, docker_fn, os.path.join(build_dn, ''))
print('[STATUS] running "%s"' % cmd)
subprocess.check_call(cmd, shell=True)
elapsed_sec = time.time() - start_time
total_time += elapsed_sec
elapsed = '%.1f min' % (elapsed_sec / 60.)
print('[TIME] elapsed: %s' % elapsed)
updates.append(
dict(name='everything', image=image_name, elapsed=elapsed_sec))
# save to the history with a docker style in contrast to a test style
# since we only save at the end, a failure means no times get written
ts = datetime.datetime.fromtimestamp(time.time()).strftime('%Y.%m.%d.%H%M')
docker_history[(name, ts)] = dict(series=updates,
texts=texts,
total_time=total_time)
config.update(docker_history=docker_history)
write_config(config)
return docker_details
def test_report(*sigs, **kwargs):
"""Write a report for a specific testset to a file."""
import textwrap
liner = lambda x, indent=0, indent_sub=2: '\n'.join(
textwrap.wrap(
x, width=110 - indent, subsequent_indent=' ' * indent_sub))
formatter = lambda title, *x: '%s\n' % title + '\n'.join(
['%s%s' % (' ' * 2, liner(i, indent=2, indent_sub=4)) for i in x])
# get the testset instructions
prepped = test_run(*sigs, **kwargs)
# generate timestamp
ts = datetime.datetime.fromtimestamp(time.time()).strftime('%Y.%m.%d.%H%M')
text = ['# FACTORY TESTSET REPORT: "%s"' % '_'.join(sigs)]
text += ['This report was generated on: %s.' % ts]
# start with notes
notes = prepped.get('notes', None)
if notes:
text += [
'> NOTES:\n%s' %
'\n'.join([' %s' % i for i in notes.strip().splitlines()])
]
# tell the user how to run it
text += ['RUN COMMAND: `make test %s`' % ' '.join(sigs)]
# handle once flag
if prepped.get('once', False):
text += [
formatter(
'SINGLE USE:',
'This testset runs once and is then recorded in the `factory/config.py` variable ',
'called "testset_history" so that it is not repeated.')
]
# report the docker
text += [
formatter(
'DOCKER IMAGE:', 'This testset uses the docker image "%s/%s".' %
(container_user, prepped['docker']),
'See available images with `docker images`.')
]
# location on disk
where = prepped.get('where', False)
if where:
text += [
formatter(
'LOCATION:',
'This docker is mounted to the host disk at `host/%s`.' %
where)
]
# you can embed some files directly in the YAML (this is useful for files that change ports)
write_files = prepped.get('write files', {})
if write_files:
for fn, text in write_files.items():
with open(
os.path.join(os.path.dirname(config['docks_config']), fn),
'w') as fp:
fp.write(text)
# files that get copied
collect_files = prepped.get('collect files', {})
if collect_files:
text += [
formatter(
'COPY FILES:',
'The following files are copied from the host to the user directory in the container.',
*[
' ' * 2 + '%s > %s' % (k, v)
for k, v in collect_files.items()
])
]
# write preliminary script
prelim = prepped.get('preliminary', False)
if prelim:
text += [
formatter('PRELIMINARY SCRIPT (runs in the host):\n',
'#!/bin/bash', 'set -e', *prelim.splitlines())
]
# write the main script
script = prepped.get('script', False)
if script:
text += [
formatter('MAIN SCRIPT (runs in the container):\n', '#!/bin/bash',
'set -e', *script.splitlines())
]
# reproduce other files
report_files = prepped.get('report files', None)
if report_files:
# get location of the testset sources via the docks_config
config = read_config()
if 'docks_config' not in config:
raise Exception('cannot find docks_config')
for fn in report_files:
with open(os.path.join(os.path.dirname(config['docks_config']),
fn)) as fp:
text += [
'## file contents: "%s"\n\n~~~\n%s\n~~~' %
(fn, fp.read().strip())
]
# write the report
fn = 'report-%s.md' % ('_'.join(sigs))
with open(fn, 'w') as fp:
fp.write('\n\n'.join(text))
print('[STATUS] write report to %s' % fn)
def docker_local(**kwargs):
"""
Use a prepared docker to run some code.
"""
config_fn = kwargs.pop('config_fn', 'docker_config.py')
mods_fn = kwargs.pop('mods_fn', None)
# get the docker history
config = read_config()
docker_history = config.get('docker_history', {})
testset_history = config.get('testset_history', {})
# check if the docker is ready
docker_name = kwargs.get('docker')
# call docker which only builds if the docker is not stored in the history
docker_details = docker(name=docker_name, config=config_fn, mods=mods_fn)
# check for once
do_once = kwargs.get('once', False)
# check for an identical event in the testset history
if do_once:
"""
we only continue if the kwargs are not stored in the testset history
which means that any change to the instructions triggers a remake
but many "do once" routines will fail if you try to rerun them with slightly different settings
so it is best to be mindful of making changes to one-time-only executions
"""
kwargs_no_notes = copy.deepcopy(kwargs)
kwargs_no_notes.pop('notes', None)
events = testset_history.get('events', [])
if any([i == kwargs_no_notes for i in events]):
print(
'[STATUS] found an exact match for this test so we are exiting'
)
return
# check that the location is ready
spot = os.path.abspath(os.path.expanduser(kwargs.get('where')))
if not os.path.isdir(spot):
try:
os.mkdir(spot)
except Exception as e:
raise Exception(
'exception is: %s. you might need to mkdir. we failed to make %s'
% (e, spot))
else:
print('[STATUS] found %s' % spot)
# run the custom requirements script
if 'preliminary' in kwargs:
import tempfile
script = tempfile.NamedTemporaryFile(delete=True)
script_header = '#!/bin/bash\nset -e\n\n'
with open(script.name, 'w') as fp:
fp.write(script_header + kwargs['preliminary'])
subprocess.check_call('bash %s' % fp.name, shell=True)
# you can embed some files directly in the YAML (this is useful for files that change ports)
write_files = kwargs.get('write files', {})
if write_files:
for fn, text in write_files.items():
with open(
os.path.join(os.path.dirname(config['docks_config']), fn),
'w') as fp:
fp.write(text)
# collect local files
for key, val in kwargs.get('collect files', {}).items():
shutil.copyfile(os.path.join(os.path.dirname(config_fn), key),
os.path.join(spot, val))
# write the testset to the top directory. this is a transient file which only lives in the host?
if 'script' in kwargs:
ts = datetime.datetime.fromtimestamp(
time.time()).strftime('%Y.%m.%d.%H%M')
testset_fn = 'script-run-%s.sh' % ts
script_header = ('#!/bin/bash\nset -e\n' + 'log_file=%s\n' %
('log-run-%s' % ts) +
'exec &> >(tee -a "$log_file")' + '\n\n')
with open(os.path.join(spot, testset_fn), 'w') as fp:
fp.write(script_header + kwargs['script'])
elif kwargs.get('visit', True):
testset_fn = None
else:
raise Exception('need either script or visit')
#! by default we work in the home directory of the user. this needs documented
user = os.environ['USER']
container_site = os.path.join('/', 'home', user, 'host')
# default to root if there is no user_coda in the docker specification
if not docker_details.get('user_coda', False): user = '******'
# prepare the run settings
run_settings = dict(
user=user,
host_site=spot,
container_site=container_site,
container_user=container_user,
image=docker_name,
testset_file=testset_fn if not kwargs.get('visit', True) else None,
mounts_extra='')
# extra mounts
for mount_from, mount_to in kwargs.get('mounts', {}).items():
run_settings['mounts_extra'] += ' -v %s:%s' % (
mount_from, os.path.join('/home/%s' % user, mount_to))
run_settings['ports'] = ' '.join([
'--publish=%d:%d' %
((p, p) if type(p) == int else tuple([int(j) for j in p]))
for p in kwargs.get('ports', [])
])
if run_settings['ports'] != '':
run_settings['ports'] = ' %s ' % run_settings['ports']
# run the docker
if kwargs.get('background', False) == False or kwargs.get('visit', True):
run_settings['attach_mode'] = '--rm -it '
else:
run_settings['attach_mode'] = '-d '
if run_settings['testset_file'] != None:
run_settings[
'tail'] = " bash %(container_site)s/%(testset_file)s" % run_settings
else:
run_settings['tail'] = ""
run_settings['namer'] = "--name=%s " % kwargs[
'container_name'] if 'container_name' in kwargs else ""
cmd = (
"docker run %(namer)s %(attach_mode)s"
"-u %(user)s -v %(host_site)s:%(container_site)s%(mounts_extra)s%(ports)s "
"%(container_user)s/%(image)s%(tail)s") % run_settings
#! delete the original below after testing
cmd_alt = ((
"docker run %s" %
('--rm -it ' if
(kwargs.get('background', False) == False
or kwargs.get('visit', True)) else '-d ') + "--name %s " %
kwargs['container_name'] if 'container_name' in kwargs else "" +
"-u %(user)s -v %(host_site)s:%(container_site)s%(mounts_extra)s%(ports)s "
+ "%(container_user)s/%(image)s ") % run_settings +
("bash %(container_site)s/%(testset_file)s" %
run_settings if run_settings['testset_file'] != None else ""))
print('[STATUS] calling docker via: %s' % cmd)
# we wait if do_once
do_wait = do_once or kwargs.get('wait', False)
# check_call raises exception on failure
subprocess.check_call(cmd, shell=True)
# wait until the container is removed. best with the back flag for detached mode
if do_wait:
#! useful place to fork a process and tail the log? obv difficult because GIL
subprocess.check_call('docker wait %s' % kwargs['container_name'],
shell=True)
# clean up the testset script
#! currently skipping the script cleanup. RESOLVE LATER!
if False and testset_fn != None:
try:
os.remove(os.path.join(spot, testset_fn))
except:
pass
# it is no longer necessary to clean up external mounts if they are mounted in ~/host/
# register this in the config if it runs only once
if do_once:
testset_history['events'] = testset_history.get('events', [])
event = copy.deepcopy(kwargs)
#---never save the notes
event.pop('notes', None)
testset_history['events'].append(event)
config.update(testset_history=testset_history)
write_config(config)
respond = {'spot': spot}
if testset_fn: respond['script'] = testset_fn
return respond