def read_thunk(thunk_hash, s3_bucket=None):
tpath = GGPaths.blob_path(thunk_hash)
if not os.path.exists(tpath) and not s3_bucket:
raise Exception("thunk is not locally available")
elif not os.path.exists(tpath):
try:
s3_client.download_file(Bucket=s3_bucket,
Key=thunk_hash,
Filename=tpath)
tags = s3_client.get_object_tagging(Bucket=s3_bucket,
Key=thunk_hash)
tags = tags.get('TagSet', [])
for tag in tags:
if tag['Key'] == 'gg:executable' and tag['Value'] == 'true':
make_executable(tpath)
break
except:
Exception("thunk is not available: {}".format(thunk_hash))
with open(tpath, "rb") as fin:
magic = fin.read(len(MAGIC_CODE))
if magic != MAGIC_CODE:
raise NotAThunkException("not a thunk: {}".format(thunk_hash))
thunk = gg_pb2.Thunk()
thunk.ParseFromString(fin.read())
return thunk
def test_js_engine_path(self):
# Test that running JS commands works for node, d8, and jsc and is not path dependent
restore_and_set_up()
sample_script = test_file('print_args.js')
# Fake some JS engines
# Note that the path contains 'd8'.
test_path = self.in_dir('fake', 'abcd8765')
ensure_dir(test_path)
jsengines = [('d8', config.V8_ENGINE),
('d8_g', config.V8_ENGINE),
('js', config.SPIDERMONKEY_ENGINE),
('node', config.NODE_JS),
('nodejs', config.NODE_JS)]
for filename, engine in jsengines:
try_delete(SANITY_FILE)
if type(engine) is list:
engine = engine[0]
if not engine:
print('WARNING: Not testing engine %s, not configured.' % (filename))
continue
print(filename, engine)
test_engine_path = os.path.join(test_path, filename)
with open(test_engine_path, 'w') as f:
f.write('#!/bin/sh\n')
f.write('exec %s $@\n' % (engine))
make_executable(test_engine_path)
out = self.run_js(sample_script, engine=test_engine_path, args=['--foo'])
self.assertEqual('0: --foo', out.strip())
def make_fake_llc(filename, targets):
"""Create a fake llc that only handles --version and writes target
list to stdout.
"""
print('make_fake_llc: %s' % filename)
ensure_dir(os.path.dirname(filename))
with open(filename, 'w') as f:
f.write('#!/bin/sh\n')
f.write('echo "llc fake output\nRegistered Targets:\n%s"' % targets)
make_executable(filename)
def make_fake_tool(filename, version, report_name=None):
if not report_name:
report_name = os.path.basename(filename)
print('make_fake_tool: %s' % filename)
ensure_dir(os.path.dirname(filename))
with open(filename, 'w') as f:
f.write('#!/bin/sh\n')
f.write('echo "%s version %s"\n' % (report_name, version))
f.write('echo "..."\n')
f.write('exit 0\n')
make_executable(filename)
def test_node(self):
NODE_WARNING = 'node version appears too old'
NODE_WARNING_2 = 'cannot check node version'
restore_and_set_up()
# Clang should report the version number we expect, and emcc should not warn
assert shared.check_node_version()
output = self.check_working(EMCC)
self.assertNotContained(NODE_WARNING, output)
# Fake a different node version
restore_and_set_up()
with open(EM_CONFIG, 'a') as f:
f.write('NODE_JS = "' + self.in_dir('fake', 'nodejs') + '"')
ensure_dir('fake')
for version, succeed in [('v0.8.0', False),
('v4.1.0', False),
('v4.1.1', True),
('v4.2.3-pre', True),
('cheez', False)]:
print(version, succeed)
try_delete(SANITY_FILE)
f = open(self.in_dir('fake', 'nodejs'), 'w')
f.write('#!/bin/sh\n')
f.write('''if [ $1 = "--version" ]; then
echo "%s"
else
%s $@
fi
''' % (version, ' '.join(config.NODE_JS)))
f.close()
make_executable(self.in_dir('fake', 'nodejs'))
if not succeed:
if version[0] == 'v':
self.check_working(EMCC, NODE_WARNING)
else:
self.check_working(EMCC, NODE_WARNING_2)
else:
output = self.check_working(EMCC)
self.assertNotContained(NODE_WARNING, output)
def main(event):
startTime = datetime.now()
print('startTime: %s' % startTime.strftime('[%Y-%m-%dT%H:%M:%S.%fZ]'))
#return params
os.environ['GG_STORAGE_URI'] = event['storageBackend']
thunks = event['thunks']
timelog = event.get('timelog')
# Remove old thunk-execute directories
os.system("rm -rf /tmp/thunk-execute.*")
# Write thunks to disk
tried_once = False
while True:
try:
for thunk_item in thunks:
thunk_data = b64decode(thunk_item['data'])
if os.path.exists(GGPaths.blob_path(thunk_item['hash'])):
os.remove(GGPaths.blob_path(thunk_item['hash']))
with open(GGPaths.blob_path(thunk_item['hash']), "wb") as fout:
fout.write(thunk_data)
# Move executables from Lambda package to .gg directory
executables_dir = os.path.join(curdir, 'executables')
if os.path.exists(executables_dir):
for exe in os.listdir(executables_dir):
blob_path = GGPaths.blob_path(exe)
exe_path = os.path.join(executables_dir, exe)
if not os.path.exists(blob_path):
shutil.copy(exe_path, blob_path)
make_executable(blob_path)
break
except OSError as ex:
if not tried_once and ex.errno == errno.ENOSPC:
# there's no space left; let's get rid of GG_DIR and try again
tried_once = True
os.system("rm -rf '{}'".format(GGPaths.blobs))
os.system("rm -rf '{}'".format(GGPaths.reductions))
make_gg_dirs()
continue
else:
raise
# Execute the thunk, and upload the result
# command = ["gg-execute-static",
# "--get-dependencies",
# "--put-output",
# "--cleanup"]
command = "./gg-execute-static --get-dependencies --put-output --cleanup"
if timelog:
#command += ["--timelog"]
command += " --timelog"
for x in thunks:
command += " " + x['hash']
print('command: ' + command)
return_code, stdout = run_command(command)
# return_code, stdout = run_command(command +
# [x['hash'] for x in thunks])
print('command output: ' + stdout)
executed_thunks = []
for thunk in thunks:
outputs = []
for output_tag in thunk['outputs']:
output_hash = GGCache.check(thunk['hash'], output_tag)
if not output_hash:
return {'returnCode': return_code, 'stdout': stdout}
data = None
if is_hash_for_thunk(output_hash):
with open(GGPaths.blob_path(output_hash), 'rb') as tin:
data = b64encode(tin.read()).decode('ascii')
outputs += [{
'tag':
output_tag,
'hash':
output_hash,
'size':
os.path.getsize(GGPaths.blob_path(output_hash)),
'executable':
is_executable(GGPaths.blob_path(output_hash)),
'data':
data
}]
executed_thunks += [{'thunkHash': thunk['hash'], 'outputs': outputs}]
return {'returnCode': 0, 'stdout': '', 'executedThunks': executed_thunks}
def handler(event, context):
gginfo = GGInfo()
gginfo.thunk_hash = event['thunk_hash']
gginfo.s3_bucket = event['s3_bucket']
gginfo.s3_region = event['s3_region']
gginfo.infiles = event['infiles']
enable_timelog = event.get('timelog', True)
timelogger = TimeLog(enabled=enable_timelog)
thunk_data = b64decode(event['thunk_data'])
with open(GGPaths.blob_path(gginfo.thunk_hash), "wb") as fout:
fout.write(thunk_data)
timelogger.add_point("write thunk to disk")
executables_dir = os.path.join(curdir, 'executables')
if os.path.exists(executables_dir):
for exe in os.listdir(executables_dir):
blob_path = GGPaths.blob_path(exe)
exe_path = os.path.join(executables_dir, exe)
if not os.path.exists(blob_path):
shutil.copy(exe_path, blob_path)
make_executable(blob_path)
timelogger.add_point("copy executables to ggdir")
# only clean up the gg directory if running on Lambda.
if not fetch_dependencies(gginfo, gginfo.infiles, not GG_RUNNER):
return {'errorType': 'GG-FetchDependenciesFailed'}
for infile in gginfo.infiles:
if infile['executable']:
print("exe %s" % infile['hash'])
make_executable(GGPaths.blob_path(infile['hash']))
timelogger.add_point("fetching the dependencies")
return_code, output = run_command(["gg-execute-static", gginfo.thunk_hash])
if return_code:
return {
'errorType': 'GG-ExecutionFailed',
}
timelogger.add_point("gg-execute")
result = GGCache.check(gginfo.thunk_hash)
if not result:
return {'errorType': 'GG-ExecutionFailed'}
executable = is_executable(GGPaths.blob_path(result))
timelogger.add_point("check the outfile")
s3_client = boto3.client('s3')
print("writing %s" % GGPaths.blob_path(result))
with open(GGPaths.blob_path(result), "rb") as bfile:
blob = bfile.read()
print("file read")
r.set(result, blob)
print("redis key set")
"""
s3_client.upload_file(GGPaths.blob_path(result), gginfo.s3_bucket, result)
s3_client.put_object_acl(
ACL='public-read',
Bucket=gginfo.s3_bucket,
Key=result
)
s3_client.put_object_tagging(
Bucket=gginfo.s3_bucket,
Key=result,
Tagging={
'TagSet': [
{ 'Key': 'gg:reduced_from', 'Value': gginfo.thunk_hash },
{ 'Key': 'gg:executable', 'Value': 'true' if executable else 'false' }
]
}
)
"""
timelogger.add_point("upload outfile to s3")
if enable_timelog:
print("s3 timelog uploading..")
s3_client.put_object(ACL='public-read',
Bucket=gginfo.s3_bucket,
Key="runlogs/{}".format(gginfo.thunk_hash),
Body=str({
'output_hash': result,
'started': timelogger.start,
'timelog': timelogger.points
}).encode('utf-8'))
print("s3 timelog uploaded!")
return {
'thunk_hash': gginfo.thunk_hash,
'output_hash': result,
'output_size': os.path.getsize(GGPaths.blob_path(result)),
'executable_output': executable
}
def handler(event, context):
os.environ['GG_STORAGE_URI'] = event['storageBackend']
thunks = event['thunks']
timelog = event.get('timelog')
# Remove old thunk-execute directories
os.system("rm -rf /tmp/thunk-execute.*")
# Write thunks to disk
for thunk_item in thunks:
thunk_data = b64decode(thunk_item['data'])
if os.path.exists(GGPaths.blob_path(thunk_item['hash'])):
os.remove(GGPaths.blob_path(thunk_item['hash']))
with open(GGPaths.blob_path(thunk_item['hash']), "wb") as fout:
fout.write(thunk_data)
# Move executables from Lambda package to .gg directory
executables_dir = os.path.join(curdir, 'executables')
if os.path.exists(executables_dir):
for exe in os.listdir(executables_dir):
blob_path = GGPaths.blob_path(exe)
exe_path = os.path.join(executables_dir, exe)
if not os.path.exists(blob_path):
shutil.copy(exe_path, blob_path)
make_executable(blob_path)
# Execute the thunk, and upload the result
command = [
"gg-execute-static", "--get-dependencies", "--put-output", "--cleanup"
]
if timelog:
command += ["--timelog"]
return_code, stdout = run_command(command + [x['hash'] for x in thunks])
executed_thunks = []
for thunk in thunks:
outputs = []
for output_tag in thunk['outputs']:
output_hash = GGCache.check(thunk['hash'], output_tag)
if not output_hash:
return {'returnCode': return_code, 'stdout': stdout}
data = None
if is_hash_for_thunk(output_hash):
with open(GGPaths.blob_path(output_hash), 'rb') as tin:
data = b64encode(tin.read()).decode('ascii')
outputs += [{
'tag':
output_tag,
'hash':
output_hash,
'size':
os.path.getsize(GGPaths.blob_path(output_hash)),
'executable':
is_executable(GGPaths.blob_path(output_hash)),
'data':
data
}]
executed_thunks += [{'thunkHash': thunk['hash'], 'outputs': outputs}]
return {'returnCode': 0, 'stdout': '', 'executedThunks': executed_thunks}
def make_new_executable(name): open(os.path.join(temp_bin, name), 'w').close() make_executable(os.path.join(temp_bin, name))
