def main(argv):
Common.init("/mnt/config/config.yml", docker_container=argv[1])
Common.message(_('Loading pipeline...'))
pipeline = Pipeline(path="/mnt/pipeline/pipeline.yml", host_path=argv[0])
if len(argv) > 2 and argv[2] == "test":
pipeline.run_tests()
elif len(argv) > 2 and argv[2]:
Common.message(_('Unknown argument')+": "+argv[2])
else:
Web(pipeline, host='0.0.0.0')
def run_tests(self):
focus = None
for test_name in self.tests:
for test_id in self.tests[test_name]:
if 'focus' in self.tests[test_name][test_id]:
focus = test_name
Common.message(_('Focusing on test:')+' '+test_name)
break
tests_run = 0
tests_failed = 0
tests_skipped = 0
for test_name in self.tests:
if test_name == '*':
continue
if focus and test_name != focus:
continue
Common.message("\n---\n"+_('Running test:')+' '+test_name+"\n")
test = self.tests[test_name]
temp_path = tempfile.mkdtemp(prefix='test-')
tests_path = os.path.dirname(self.path)+"/tests"
config_path = None # tests_path+"/"+test_name+"/config"
input_path = tests_path+"/"+test_name
output_path = temp_path+"/"+test_name+"/output"
status_path = temp_path+"/"+test_name+"/status"
os.makedirs(output_path)
os.makedirs(status_path)
result = self.run(config_path, input_path, output_path, status_path, test=test_name)
tests_run += result['tests']['run']
tests_failed += result['tests']['failed']
tests_skipped += result['tests']['skipped']
Common.message("")
Common.message(_('Host system path to pipeline output is:')+' '+Common.host_path(os.path.dirname(output_path)))
Common.message("")
if focus:
Common.message(_('Test in focus:')+' '+test_name)
Common.message("TESTS RUN: "+str(tests_run))
Common.message("TESTS FAILED: "+str(tests_failed))
def run(self, config_path, input_path, output_path, status_path, status="", test=None, step_id='', step_depth=0):
temp_path = tempfile.mkdtemp(prefix='pipeline-')
tests_run = 0
tests_failed = 0
tests_skipped = 0
pipeline_context_name = os.path.basename(input_path)
padding = (self.location if len(self.steps) == 0 else self.steps[0][list(self.steps[0].keys())[0]]['location']).ljust(20+step_depth*2)
Common.message(padding+'-- Context is: '+pipeline_context_name)
current_input_path = None
current_output_path = None
current_status_path = None
current_status = ''
exit = False
position = 0
step_count = len([ step_object for step_object in self.steps if (list(step_object.keys())[0] != 'test' and list(step_object.keys())[0] != 'assert') ])
if step_count == 0:
current_status = status
current_input_path = input_path
current_output_path = output_path
current_status_path = status_path
if status != None:
Common.write_file(current_status_path+'/status.txt', status)
if os.path.isdir(current_input_path):
for item in os.listdir(current_input_path):
item_fullpath = os.path.join(current_input_path, item)
if os.path.isdir(item_fullpath):
shutil.copytree(item_fullpath, current_output_path+'/'+os.path.basename(item_fullpath))
else:
shutil.copy2(item_fullpath, current_output_path+'/'+os.path.basename(item_fullpath))
else:
shutil.copy2(current_input_path, current_output_path+'/'+os.path.basename(current_input_path))
for step_object in self.steps:
step_type = list(step_object.keys())[0]
step = step_object[step_type]
padding = step['location'].ljust(20+step_depth*2)
if exit or step_type == 'exit':
if step_type == 'exit':
Common.message(padding+'-- Exiting')
exit = True
break
current_status = status if not current_status_path else Common.first_line(current_status_path)
if step_type != 'test' and step_type != 'assert':
current_step_id = step_id+'/'+str(position)
Common.message(padding+'-- Running '+step_type+': '+(step_type if not 'name' in step else step['name']))
if position > 0:
current_input_path = current_output_path
else:
current_input_path = input_path
if position < step_count - 1:
current_output_path = temp_path+"/"+str(position)+"/output"
current_status_path = temp_path+"/"+str(position)+"/status"
os.makedirs(current_output_path)
os.makedirs(current_status_path)
else:
current_output_path = output_path
current_status_path = status_path
if step_type == 'image':
volumes = {}
if config_path:
volumes[config_path] = { 'bind': '/mnt/config', 'mode': 'ro' }
volumes[current_input_path] = { 'bind': '/mnt/input'+('/'+pipeline_context_name if not os.path.isdir(current_input_path) else ''), 'mode': 'ro' }
volumes[current_output_path] = { 'bind': '/mnt/output', 'mode': 'rw' }
volumes[current_status_path] = { 'bind': '/mnt/status', 'mode': 'rw' }
command = None if not 'command' in step else step['command']
Common.docker_run(step['id'], volumes, command=command)
elif step_type == 'choose':
for when in step['choices']:
when_key = list(when.keys())[0]
if (not 'test' in when[when_key]) or when[when_key]['test'] == current_status:
result = when[when_key]['pipeline'].run(config_path, file_or_dir, current_output_path, current_status_path, status=current_status, test=test, step_id=current_step_id, step_depth=step_depth+1)
exit = exit or result['exit']
tests_skipped += result['tests']['skipped']
tests_run += result['tests']['run']
tests_failed += result['tests']['failed']
break
elif step_type == 'foreach':
for file_or_dir in Common.list_files(current_input_path):
result = step['pipeline'].run(config_path, file_or_dir, current_output_path+'/'+os.path.basename(file_or_dir), current_status_path, status=current_status, test=test, step_id=current_step_id, step_depth=step_depth+1)
exit = exit or result['exit']
tests_skipped += result['tests']['skipped']
tests_run += result['tests']['run']
tests_failed += result['tests']['failed']
elif step_type == 'unfold':
prev_paths = []
paths = [ current_input_path ]
depth = step['depth']
while depth >= 0:
depth -= 1
prev_paths = list(paths)
paths = []
for current_path in prev_paths:
if not os.path.isdir(current_path):
continue
for item in os.listdir(current_path):
paths.append(os.path.join(current_path, item))
for path in paths:
if os.path.exists(current_output_path+'/'+os.path.basename(path)):
Common.message(padding+" WARNING: file naming collision when unfolding: "+os.path.basename(path))
continue
if os.path.isdir(path):
shutil.copytree(path, current_output_path+'/'+os.path.basename(path))
else:
shutil.copy2(path, current_output_path+'/'+os.path.basename(path))
else:
Common.message(padding+" ERROR: could not process step type: "+step_type)
current_output_path = current_input_path
position += 1
else:
if step['name'] == test or test == '*' or step['name'] == '*':
if 'context' in step and not step['context'] == pipeline_context_name:
continue
success = True
Common.message(padding+'-- Evaluating test: '+step['name'])
if 'expect' in step:
tests_run += 1
expected_files = {}
actual_files = {}
actual_path = current_output_path if current_output_path else input_path
expected_path = os.path.dirname(self.path)+"/tests/"+step['expect']
if os.path.isdir(actual_path):
for root, subdirs, files in os.walk(actual_path):
for file in files:
actual_files[os.path.relpath(root, actual_path)+'/'+file] = {'hash': Common.hashfile(root+'/'+file),
'fullpath': root+'/'+file }
else:
actual_files['./'+os.path.basename(actual_path)] = {'hash': Common.hashfile(actual_path),
'fullpath': actual_path }
if os.path.isdir(expected_path):
for root, subdirs, files in os.walk(expected_path):
for file in files:
expected_files[os.path.relpath(root, expected_path)+'/'+file] = {'hash': Common.hashfile(root+'/'+file),
'fullpath': root+'/'+file }
else:
expected_files['./'+os.path.basename(expected_path)] = {'hash': Common.hashfile(expected_path),
'fullpath': expected_path }
for file in actual_files:
if not file in expected_files:
Common.message(padding+" FAILED: file should not be present in output: "+file)
success = False
elif expected_files[file]['hash'] != actual_files[file]['hash']:
Common.message(padding+" FAILED: content in actual and expected files differ: "+file)
success = False
mime = mimetypes.guess_type(file)
if mime[0].startswith('text/') and os.stat(actual_files[file]['fullpath']).st_size < 1000000 and os.stat(expected_files[file]['fullpath']).st_size < 1000000:
with open(expected_files[file]['fullpath'], 'r') as expected_file:
with open(actual_files[file]['fullpath'], 'r') as actual_file:
expected_lines = expected_file.readlines()
actual_lines = actual_file.readlines()
max_lines = 30
for diffline in difflib.unified_diff(expected_lines, actual_lines, fromfile=file, tofile=file):
Common.message(padding+diffline.strip())
max_lines -= 1
if max_lines <= 0:
break
for file in expected_files:
if not file in actual_files:
Common.message(padding+" FAILED: file should be present in output: "+file)
success = False
if not success:
actual_compact = list(actual_files.keys())
actual_compact.sort()
actual_compact = ', '.join(actual_compact)
if len(actual_compact) > 80:
actual_compact = actual_compact[:79]+'…'
expected_compact = list(expected_files.keys())
expected_compact.sort()
expected_compact = ', '.join(expected_compact)
if len(expected_compact) > 80:
expected_compact = expected_compact[:79]+'…'
Common.message(padding+" ACTUAL: "+actual_compact)
Common.message(padding+" EXPECTED: "+expected_compact)
if 'status' in step:
expected_status = '' if step['status'] == None else str(step['status'])
if current_status != expected_status:
Common.message(padding+" FAILED: actual status and expected status differ:")
Common.message(padding+" expected: "+expected_status)
Common.message(padding+" actual: "+current_status)
success = False
if not success:
tests_failed += 1
tests_run += 1
return {
'exit': exit,
'tests': {
'skipped': tests_skipped,
'run': tests_run,
'failed': tests_failed
}
}
def __init__(self, path, host_path, pipeline=None, start_location=None):
self.path = path
self.host_path = host_path
if pipeline:
self.pipeline = pipeline
else:
Common.message("host_path: "+host_path)
self.pipeline = Common.load_yaml(path)
if start_location:
self.location = start_location
elif '__location__' in self.pipeline:
self.location = self.pipeline['__location__']
else:
os.path.basename(self.path)+":1"
self.steps = []
self.tests = {}
for step in self.pipeline['pipeline']:
if isinstance(step, str):
if step == "exit":
self.steps.append({ 'exit': { 'location': self.location.split(':',1)[0] }})
else:
Common.message("WARNING: unknown instruction found: '"+step+"' (skipping!)")
continue
step_line = [key for key in list(step.keys()) if not key.startswith('_')]
step_line = None if len(step_line) == 0 else step_line[0]
if 'image' in step:
image = { 'location': str(step['__location__']), 'name': step['image'], 'id': step['image'] }
if 'command' in step:
image['command'] = step['command']
self.steps.append({ 'image': image })
elif 'dockerfile' in step:
image_id = Common.docker_build(os.path.dirname(path)+"/"+step['dockerfile']+"/")
image = { 'location': str(step['__location__']), 'name': step['dockerfile'], 'id': image_id }
if 'command' in step:
image['command'] = step['command']
self.steps.append({ 'image': image })
elif 'unfold' in step:
self.steps.append({'unfold':{'location': str(step['__location__']), 'name':'unfold', 'depth':step['unfold']}})
elif 'test' in step:
test_input = '*' if not 'input' in step['test'] else step['test']['input']
test = { 'location': str(step['__location__']), 'name': test_input }
for key in ['input', 'expect', 'status', 'focus', 'context']:
if key in step['test']:
test[key] = step['test'][key]
if not test_input in self.tests:
self.tests[test_input] = {}
self.tests[test_input][str(len(self.steps))] = test
self.steps.append({ 'test': test })
elif 'assert' in step:
assertion = { 'test': { 'location': str(step['__location__']), 'name': '*', 'input': '*', 'status': step['assert'] if not step['assert'] == None else "" }}
if not '*' in self.tests:
self.tests['*'] = {}
self.tests['*'][str(len(self.steps))] = assertion['test']
self.steps.append(assertion)
elif step_line.startswith('if'):
test = step_line[2:].split(":",1)[0].strip()
subpipeline_name = 'pipeline' if not 'name' in self.pipeline['pipeline'] else self.pipeline['pipeline']['name']
subpipeline_if = Pipeline(
self.path,
self.host_path,
pipeline={
'name': subpipeline_name,
'pipeline': step[step_line]
},
start_location=step['__location__']
)
subpipeline_else = Pipeline(
self.path,
self.host_path,
pipeline={
'name': subpipeline_name,
'pipeline': []
},
start_location=step['__location__']
)
for test_input in subpipeline_if.tests:
if not test_input in self.tests:
self.tests[test_input] = {}
for subtest_id in subpipeline_if.tests[test_input]:
self.tests[test_input][str(len(self.steps))+'.'+subtest_id] = subpipeline_if.tests[test_input][subtest_id]
self.steps.append({ 'choose':
{ 'location': str(step['__location__']),
'choices': [
{ 'when': {
'location': str(step['__location__']),
'test': test,
'pipeline': subpipeline_if
}},
{ 'otherwise': {
'location': str(step['__location__']),
'pipeline': subpipeline_else
}}
]
}
})
elif step_line.startswith('elif'):
choose = self.steps[len(self.steps)-1]
test = step_line[4:].split(":",1)[0].strip()
subpipeline_name = 'pipeline' if not 'name' in self.pipeline['pipeline'] else self.pipeline['pipeline']['name']
subpipeline_elif = Pipeline(
self.path,
self.host_path,
pipeline={
'name': subpipeline_name,
'pipeline': step[step_line]
},
start_location=step['__location__']
)
for test_input in subpipeline_elif.tests:
if not test_input in self.tests:
self.tests[test_input] = {}
for subtest_id in subpipeline_elif.tests[test_input]:
self.tests[test_input][str(len(self.steps))+'.'+subtest_id] = subpipeline_elif.tests[test_input][subtest_id]
when = { 'when': { 'location': str(step['__location__']), 'test': test, 'pipeline': subpipeline_elif } }
choose['choose']['choices'].insert(len(choose['choose']['choices'])-1, when)
elif 'else' in step:
choose = self.steps[len(self.steps)-1]
subpipeline_name = 'pipeline' if not 'name' in self.pipeline['pipeline'] else self.pipeline['pipeline']['name']
subpipeline_else = Pipeline(
self.path,
self.host_path,
pipeline={
'name': subpipeline_name,
'pipeline': step['else']
},
start_location=step['__location__']
)
for test_input in subpipeline_else.tests:
if not test_input in self.tests:
self.tests[test_input] = {}
for subtest_id in subpipeline_else.tests[test_input]:
self.tests[test_input][str(len(self.steps))+'.'+subtest_id] = subpipeline_else.tests[test_input][subtest_id]
otherwise = { 'otherwise': { 'location': str(step['__location__']), 'pipeline': subpipeline_else } }
choose['choose']['choices'][len(choose['choose']['choices'])-1] = otherwise
elif 'foreach' in step:
subpipeline_name = 'pipeline' if not 'name' in self.pipeline['pipeline'] else self.pipeline['pipeline']['name']
subpipeline = Pipeline(
self.path,
self.host_path,
pipeline={
'name': subpipeline_name,
'pipeline': step['foreach']
},
start_location=step['__location__']
)
for test_input in subpipeline.tests:
if not test_input in self.tests:
self.tests[test_input] = {}
for subtest_id in subpipeline.tests[test_input]:
self.tests[test_input][str(len(self.steps))+'.'+subtest_id] = subpipeline.tests[test_input][subtest_id]
self.steps.append({ 'foreach': { 'location': str(step['__location__']), 'name': subpipeline_name, 'pipeline': subpipeline }})
else:
Common.message("WARNING: unknown step type: dictionary with key '"+step_line+"' (skipping!)")