def run(task, inputs, outputs, task_inputs, task_outputs, **kwargs):
script = task['script']
script_fname = tempfile.mktemp()
with open(script_fname, 'w') as script_file:
script_file.write(script)
tmpDir = kwargs.get('_tempdir')
args = _expand_args(task['swift_args'], inputs, task_inputs, tmpDir)
pipes = {}
for id in ('_stdout', '_stderr'):
if id in task_outputs and id in outputs:
pipes[id] = utils.AccumulateDictAdapter(outputs[id], 'script_data')
command = ['swift', script_fname] + args
print('Running swift: "%s"' % ' '.join(command))
p = utils.run_process(command, output_pipes=pipes)
if p.returncode != 0:
raise Exception('Error: swift run returned code {}.'.format(
p.returncode))
for name, task_output in task_outputs.iteritems():
with open(name) as output_file:
outputs[name]['script_data'] = output_file.read()
def _run(spark, task, inputs, outputs, task_inputs, task_outputs, **kwargs):
tmp_dir = kwargs.get('_tempdir')
script_fname = _write_scala_script(
task['script'], inputs, task_outputs, tmp_dir)
pipes = {}
for id in ('_stdout', '_stderr'):
if id in task_outputs and id in outputs:
pipes[id] = utils.AccumulateDictAdapter(outputs[id], 'script_data')
if spark:
command = ['spark-shell', '-i', script_fname]
else:
command = ['scala', script_fname]
print('Running scala: "%s"' % ' '.join(command))
p = utils.run_process(command, output_pipes=pipes)
if p.returncode != 0:
raise Exception('Error: scala run returned code {}.'.format(
p.returncode))
for name, task_output in task_outputs.iteritems():
if name != '_stderr' and name != '_stdout':
fname = os.path.join(tmp_dir, name)
with open(fname) as output_file:
outputs[name]['script_data'] = output_file.read()
# Deal with converting from string - assume JSON
if task_output['type'] in ('number', 'boolean'):
outputs[name]['script_data'] = json.loads(
outputs[name]['script_data'])
def testOutputStreams(self):
output_spec = {"mode": "http", "method": "PUT", "url": "http://localhost:%d" % _socket_port}
fd = os.open(_pipepath, os.O_RDONLY | os.O_NONBLOCK)
adapters = {fd: make_stream_push_adapter(output_spec)}
cmd = [sys.executable, _oscript, _pipepath]
try:
with captureOutput() as stdpipes:
run_process(cmd, adapters)
except Exception:
print("Stdout/stderr from exception: ")
print(stdpipes)
raise
self.assertEqual(stdpipes, ["start\ndone\n", ""])
self.assertEqual(len(_req_chunks), 1)
self.assertEqual(_req_chunks[0], (9, "a message"))
def testInputStreams(self):
input_spec = {"mode": "http", "method": "GET", "url": "http://mockedhost"}
@httmock.urlmatch(netloc="^mockedhost$", method="GET")
def mock_fetch(url, request):
return "hello\nworld"
adapters = {_pipepath: make_stream_fetch_adapter(input_spec)}
cmd = [sys.executable, _iscript, _pipepath]
try:
with captureOutput() as stdpipes, httmock.HTTMock(mock_fetch):
run_process(cmd, input_pipes=adapters)
except Exception:
print("Stdout/stderr from exception: ")
print(stdpipes)
raise
self.assertEqual(stdpipes, ["olleh\ndlrow\n", ""])
def testOutputStreams(self):
output_spec = {
'mode': 'http',
'method': 'PUT',
'url': 'http://localhost:%d' % _socket_port
}
fd = os.open(_pipepath, os.O_RDONLY | os.O_NONBLOCK)
adapters = {fd: make_stream_push_adapter(output_spec)}
cmd = [sys.executable, _oscript, _pipepath]
try:
with captureOutput() as stdpipes:
run_process(cmd, adapters)
except Exception:
print('Stdout/stderr from exception: ')
print(stdpipes)
raise
self.assertEqual(stdpipes, ['start\ndone\n', ''])
self.assertEqual(len(_req_chunks), 1)
self.assertEqual(_req_chunks[0], (9, 'a message'))
def testInputStreams(self):
input_spec = {
'mode': 'http',
'method': 'GET',
'url': 'http://mockedhost'
}
@httmock.urlmatch(netloc='^mockedhost$', method='GET')
def mock_fetch(url, request):
return 'hello\nworld'
adapters = {_pipepath: make_stream_fetch_adapter(input_spec)}
cmd = [sys.executable, _iscript, _pipepath]
try:
with captureOutput() as stdpipes, httmock.HTTMock(mock_fetch):
run_process(cmd, input_pipes=adapters)
except Exception:
print('Stdout/stderr from exception: ')
print(stdpipes)
raise
self.assertEqual(stdpipes, ['olleh\ndlrow\n', ''])
def run(task, inputs, outputs, task_inputs, task_outputs, **kwargs):
image = task['docker_image']
if task.get('pull_image', True):
print('Pulling Docker image: ' + image)
_pull_image(image)
tempdir = kwargs.get('_tempdir')
args = _expand_args(task.get('container_args', []), inputs, task_inputs,
tempdir)
ipipes, opipes = _setup_pipes(
task_inputs, inputs, task_outputs, outputs, tempdir)
pre_args = _get_pre_args(task, os.getuid(), os.getgid())
command = [
'docker', 'run',
'-v', '%s:%s' % (tempdir, DATA_VOLUME),
'-v', '%s:%s:ro' % (SCRIPTS_DIR, SCRIPTS_VOLUME),
'--entrypoint', os.path.join(SCRIPTS_VOLUME, 'entrypoint.sh')
] + task.get('docker_run_args', []) + [image] + pre_args + args
print('Running container: %s' % repr(command))
p = utils.run_process(command, output_pipes=opipes, input_pipes=ipipes)
if p.returncode != 0:
raise Exception('Error: docker run returned code %d.' % p.returncode)
print('Garbage collecting old containers and images.')
gc_dir = os.path.join(tempdir, 'docker_gc_scratch')
os.mkdir(gc_dir)
p = _docker_gc(gc_dir)
for name, spec in task_outputs.iteritems():
if spec.get('target') == 'filepath' and not spec.get('stream'):
path = spec.get('path', name)
if not path.startswith('/'):
# Assume relative paths are relative to the data volume
path = os.path.join(DATA_VOLUME, path)
# Convert data volume refs to the temp dir on the host
path = path.replace(DATA_VOLUME, tempdir, 1)
if not os.path.exists(path):
raise Exception('Output filepath %s does not exist.' % path)
outputs[name]['script_data'] = path
p.wait() # Wait for garbage collection subprocess to finish
if p.returncode != 0:
raise Exception('Docker GC returned code %d.' % p.returncode)
