def read_thunk(thunk_hash, s3_bucket=None):
tpath = GGPaths.blob_path(thunk_hash)
if not os.path.exists(tpath) and not s3_bucket:
raise Exception("thunk is not locally available")
elif not os.path.exists(tpath):
try:
s3_client.download_file(Bucket=s3_bucket,
Key=thunk_hash,
Filename=tpath)
tags = s3_client.get_object_tagging(Bucket=s3_bucket,
Key=thunk_hash)
tags = tags.get('TagSet', [])
for tag in tags:
if tag['Key'] == 'gg:executable' and tag['Value'] == 'true':
make_executable(tpath)
break
except:
Exception("thunk is not available: {}".format(thunk_hash))
with open(tpath, "rb") as fin:
magic = fin.read(len(MAGIC_CODE))
if magic != MAGIC_CODE:
raise NotAThunkException("not a thunk: {}".format(thunk_hash))
thunk = gg_pb2.Thunk()
thunk.ParseFromString(fin.read())
return thunk
def fetch_dependencies(gginfo, infiles, cleanup_first=True):
download_list = []
blob_path = GGPaths.blobs
infile_hashes = {x['hash'] for x in infiles}
infile_hashes.add(gginfo.thunk_hash)
if cleanup_first:
# remove temp execution directories
os.system("rm -rf /tmp/thunk-execute.*")
for x in os.listdir(blob_path):
if x not in infile_hashes:
os.remove(os.path.join(blob_path, x))
for infile in infiles:
bpath = GGPaths.blob_path(infile['hash'])
if os.path.exists(bpath) and os.path.getsize(bpath) == infile['size']:
continue
print("About to download: %s" % infile['hash'])
blob = r.get(infile['hash'])
if blob is None:
return False
print("Downloaded: %s" % infile['hash'])
with open(bpath, "wb") as bfile:
bfile.write(blob)
print("Finished writing %s" % infile['hash'])
#download_list += [infile['hash']]
print("Finished downloads!")
#s3_ip = socket.gethostbyname('{}.s3.amazonaws.com'.format(gginfo.s3_bucket))
#p = sub.Popen(['gg-s3-download', gginfo.s3_region, gginfo.s3_bucket, s3_ip], stdout=sub.PIPE, stdin=sub.PIPE, stderr=sub.PIPE)
#out, err = p.communicate(input="\n".join(download_list).encode('ascii'))
#if p.returncode != 0:
# return False
return True
def main(event):
startTime = datetime.now()
print('startTime: %s' % startTime.strftime('[%Y-%m-%dT%H:%M:%S.%fZ]'))
#return params
os.environ['GG_STORAGE_URI'] = event['storageBackend']
thunks = event['thunks']
timelog = event.get('timelog')
# Remove old thunk-execute directories
os.system("rm -rf /tmp/thunk-execute.*")
# Write thunks to disk
tried_once = False
while True:
try:
for thunk_item in thunks:
thunk_data = b64decode(thunk_item['data'])
if os.path.exists(GGPaths.blob_path(thunk_item['hash'])):
os.remove(GGPaths.blob_path(thunk_item['hash']))
with open(GGPaths.blob_path(thunk_item['hash']), "wb") as fout:
fout.write(thunk_data)
# Move executables from Lambda package to .gg directory
executables_dir = os.path.join(curdir, 'executables')
if os.path.exists(executables_dir):
for exe in os.listdir(executables_dir):
blob_path = GGPaths.blob_path(exe)
exe_path = os.path.join(executables_dir, exe)
if not os.path.exists(blob_path):
shutil.copy(exe_path, blob_path)
make_executable(blob_path)
break
except OSError as ex:
if not tried_once and ex.errno == errno.ENOSPC:
# there's no space left; let's get rid of GG_DIR and try again
tried_once = True
os.system("rm -rf '{}'".format(GGPaths.blobs))
os.system("rm -rf '{}'".format(GGPaths.reductions))
make_gg_dirs()
continue
else:
raise
# Execute the thunk, and upload the result
# command = ["gg-execute-static",
# "--get-dependencies",
# "--put-output",
# "--cleanup"]
command = "./gg-execute-static --get-dependencies --put-output --cleanup"
if timelog:
#command += ["--timelog"]
command += " --timelog"
for x in thunks:
command += " " + x['hash']
print('command: ' + command)
return_code, stdout = run_command(command)
# return_code, stdout = run_command(command +
# [x['hash'] for x in thunks])
print('command output: ' + stdout)
executed_thunks = []
for thunk in thunks:
outputs = []
for output_tag in thunk['outputs']:
output_hash = GGCache.check(thunk['hash'], output_tag)
if not output_hash:
return {'returnCode': return_code, 'stdout': stdout}
data = None
if is_hash_for_thunk(output_hash):
with open(GGPaths.blob_path(output_hash), 'rb') as tin:
data = b64encode(tin.read()).decode('ascii')
outputs += [{
'tag':
output_tag,
'hash':
output_hash,
'size':
os.path.getsize(GGPaths.blob_path(output_hash)),
'executable':
is_executable(GGPaths.blob_path(output_hash)),
'data':
data
}]
executed_thunks += [{'thunkHash': thunk['hash'], 'outputs': outputs}]
return {'returnCode': 0, 'stdout': '', 'executedThunks': executed_thunks}
def handler(event, context):
gginfo = GGInfo()
gginfo.thunk_hash = event['thunk_hash']
gginfo.s3_bucket = event['s3_bucket']
gginfo.s3_region = event['s3_region']
gginfo.infiles = event['infiles']
enable_timelog = event.get('timelog', True)
timelogger = TimeLog(enabled=enable_timelog)
thunk_data = b64decode(event['thunk_data'])
with open(GGPaths.blob_path(gginfo.thunk_hash), "wb") as fout:
fout.write(thunk_data)
timelogger.add_point("write thunk to disk")
executables_dir = os.path.join(curdir, 'executables')
if os.path.exists(executables_dir):
for exe in os.listdir(executables_dir):
blob_path = GGPaths.blob_path(exe)
exe_path = os.path.join(executables_dir, exe)
if not os.path.exists(blob_path):
shutil.copy(exe_path, blob_path)
make_executable(blob_path)
timelogger.add_point("copy executables to ggdir")
# only clean up the gg directory if running on Lambda.
if not fetch_dependencies(gginfo, gginfo.infiles, not GG_RUNNER):
return {'errorType': 'GG-FetchDependenciesFailed'}
for infile in gginfo.infiles:
if infile['executable']:
print("exe %s" % infile['hash'])
make_executable(GGPaths.blob_path(infile['hash']))
timelogger.add_point("fetching the dependencies")
return_code, output = run_command(["gg-execute-static", gginfo.thunk_hash])
if return_code:
return {
'errorType': 'GG-ExecutionFailed',
}
timelogger.add_point("gg-execute")
result = GGCache.check(gginfo.thunk_hash)
if not result:
return {'errorType': 'GG-ExecutionFailed'}
executable = is_executable(GGPaths.blob_path(result))
timelogger.add_point("check the outfile")
s3_client = boto3.client('s3')
print("writing %s" % GGPaths.blob_path(result))
with open(GGPaths.blob_path(result), "rb") as bfile:
blob = bfile.read()
print("file read")
r.set(result, blob)
print("redis key set")
"""
s3_client.upload_file(GGPaths.blob_path(result), gginfo.s3_bucket, result)
s3_client.put_object_acl(
ACL='public-read',
Bucket=gginfo.s3_bucket,
Key=result
)
s3_client.put_object_tagging(
Bucket=gginfo.s3_bucket,
Key=result,
Tagging={
'TagSet': [
{ 'Key': 'gg:reduced_from', 'Value': gginfo.thunk_hash },
{ 'Key': 'gg:executable', 'Value': 'true' if executable else 'false' }
]
}
)
"""
timelogger.add_point("upload outfile to s3")
if enable_timelog:
print("s3 timelog uploading..")
s3_client.put_object(ACL='public-read',
Bucket=gginfo.s3_bucket,
Key="runlogs/{}".format(gginfo.thunk_hash),
Body=str({
'output_hash': result,
'started': timelogger.start,
'timelog': timelogger.points
}).encode('utf-8'))
print("s3 timelog uploaded!")
return {
'thunk_hash': gginfo.thunk_hash,
'output_hash': result,
'output_size': os.path.getsize(GGPaths.blob_path(result)),
'executable_output': executable
}
def handler(event, context):
os.environ['GG_STORAGE_URI'] = event['storageBackend']
thunks = event['thunks']
timelog = event.get('timelog')
# Remove old thunk-execute directories
os.system("rm -rf /tmp/thunk-execute.*")
# Write thunks to disk
for thunk_item in thunks:
thunk_data = b64decode(thunk_item['data'])
if os.path.exists(GGPaths.blob_path(thunk_item['hash'])):
os.remove(GGPaths.blob_path(thunk_item['hash']))
with open(GGPaths.blob_path(thunk_item['hash']), "wb") as fout:
fout.write(thunk_data)
# Move executables from Lambda package to .gg directory
executables_dir = os.path.join(curdir, 'executables')
if os.path.exists(executables_dir):
for exe in os.listdir(executables_dir):
blob_path = GGPaths.blob_path(exe)
exe_path = os.path.join(executables_dir, exe)
if not os.path.exists(blob_path):
shutil.copy(exe_path, blob_path)
make_executable(blob_path)
# Execute the thunk, and upload the result
command = [
"gg-execute-static", "--get-dependencies", "--put-output", "--cleanup"
]
if timelog:
command += ["--timelog"]
return_code, stdout = run_command(command + [x['hash'] for x in thunks])
executed_thunks = []
for thunk in thunks:
outputs = []
for output_tag in thunk['outputs']:
output_hash = GGCache.check(thunk['hash'], output_tag)
if not output_hash:
return {'returnCode': return_code, 'stdout': stdout}
data = None
if is_hash_for_thunk(output_hash):
with open(GGPaths.blob_path(output_hash), 'rb') as tin:
data = b64encode(tin.read()).decode('ascii')
outputs += [{
'tag':
output_tag,
'hash':
output_hash,
'size':
os.path.getsize(GGPaths.blob_path(output_hash)),
'executable':
is_executable(GGPaths.blob_path(output_hash)),
'data':
data
}]
executed_thunks += [{'thunkHash': thunk['hash'], 'outputs': outputs}]
return {'returnCode': 0, 'stdout': '', 'executedThunks': executed_thunks}