def __init__(self, exec_file, extra_args=None, stdout_file = None, stderr_file = None): self.exec_file = exec_file self.cwd = None self.stdout_file = stdout_file self.stderr_file = stderr_file self.sp = None if extra_args is None: self.extra_args = None else: self.extra_args = [str(a) for a in list(extra_args)] # make a duplicate if not is_executable(self.exec_file): raise Exception("Not executable ? : %s" % self.exec_file)
def handle_dat(request): allowed_directory = "global/" file = allowed_directory + request.fields[0] args = request.fields[2:] if not in_directory(file, allowed_directory): raise SecurityViolation("Forbidden", request.fields[0]) if is_executable(file): p = subprocess.run([file] + args, input=request.fields[1].encode("utf-8") if len(request.fields) >= 1 else b'', capture_output=True) if len(p.stderr) > 0: print_err("stderr: " + p.stderr.decode("utf-8")) return unet_request.Request(p.stdout) elif is_readable(file): with open(file, "rb") as f: return unet_request.Request(f.read()) else: raise InvalidRequest("Not Found", file[len(allowed_directory) - 1:])
def run_exp(name, schedule, scheduler, kernel, duration, work_dir, out_dir): proc_entries = [] executables = [] # Parse values for proc entries for entry_conf in schedule['proc']: path = entry_conf[0] data = entry_conf[1] if not os.path.exists(path): raise IOError("Invalid proc path %s: %s" % (path, name)) proc_entries += [ProcEntry(path, data)] # Parse spinners for spin_conf in schedule['spin']: if isinstance(spin_conf, str): # Just a string defaults to default spin (spin, args) = (conf.DEFAULTS['spin'], spin_conf) else: # Otherwise its a pair, the type and the args if len(spin_conf) != 2: raise IOError("Invalid spin conf %s: %s" % (spin_conf, name)) (spin, args) = (spin_conf[0], spin_conf[1]) # if not conf.BINS[spin]: # raise IndexError("No knowledge of program %s: %s" % (spin, name)) real_spin = com.get_executable(spin, "") real_args = args.split() if re.match(".*spin", real_spin): real_args = ['-w'] + real_args + [duration] if not com.is_executable(real_spin): raise OSError("Cannot run spin %s: %s" % (real_spin, name)) executables += [Executable(real_spin, real_args)] exp = Experiment(name, scheduler, work_dir, out_dir, proc_entries, executables) exp.run_exp()
def main(event): startTime = datetime.now() print('startTime: %s' % startTime.strftime('[%Y-%m-%dT%H:%M:%S.%fZ]')) #return params os.environ['GG_STORAGE_URI'] = event['storageBackend'] thunks = event['thunks'] timelog = event.get('timelog') # Remove old thunk-execute directories os.system("rm -rf /tmp/thunk-execute.*") # Write thunks to disk tried_once = False while True: try: for thunk_item in thunks: thunk_data = b64decode(thunk_item['data']) if os.path.exists(GGPaths.blob_path(thunk_item['hash'])): os.remove(GGPaths.blob_path(thunk_item['hash'])) with open(GGPaths.blob_path(thunk_item['hash']), "wb") as fout: fout.write(thunk_data) # Move executables from Lambda package to .gg directory executables_dir = os.path.join(curdir, 'executables') if os.path.exists(executables_dir): for exe in os.listdir(executables_dir): blob_path = GGPaths.blob_path(exe) exe_path = os.path.join(executables_dir, exe) if not os.path.exists(blob_path): shutil.copy(exe_path, blob_path) make_executable(blob_path) break except OSError as ex: if not tried_once and ex.errno == errno.ENOSPC: # there's no space left; let's get rid of GG_DIR and try again tried_once = True os.system("rm -rf '{}'".format(GGPaths.blobs)) os.system("rm -rf '{}'".format(GGPaths.reductions)) make_gg_dirs() continue else: raise # Execute the thunk, and upload the result # command = ["gg-execute-static", # "--get-dependencies", # "--put-output", # "--cleanup"] command = "./gg-execute-static --get-dependencies --put-output --cleanup" if timelog: #command += ["--timelog"] command += " --timelog" for x in thunks: command += " " + x['hash'] print('command: ' + command) return_code, stdout = run_command(command) # return_code, stdout = run_command(command + # [x['hash'] for x in thunks]) print('command output: ' + stdout) executed_thunks = [] for thunk in thunks: outputs = [] for output_tag in thunk['outputs']: output_hash = GGCache.check(thunk['hash'], output_tag) if not output_hash: return {'returnCode': return_code, 'stdout': stdout} data = None if is_hash_for_thunk(output_hash): with open(GGPaths.blob_path(output_hash), 'rb') as tin: data = b64encode(tin.read()).decode('ascii') outputs += [{ 'tag': output_tag, 'hash': output_hash, 'size': os.path.getsize(GGPaths.blob_path(output_hash)), 'executable': is_executable(GGPaths.blob_path(output_hash)), 'data': data }] executed_thunks += [{'thunkHash': thunk['hash'], 'outputs': outputs}] return {'returnCode': 0, 'stdout': '', 'executedThunks': executed_thunks}
def handler(event, context): gginfo = GGInfo() gginfo.thunk_hash = event['thunk_hash'] gginfo.s3_bucket = event['s3_bucket'] gginfo.s3_region = event['s3_region'] gginfo.infiles = event['infiles'] enable_timelog = event.get('timelog', True) timelogger = TimeLog(enabled=enable_timelog) thunk_data = b64decode(event['thunk_data']) with open(GGPaths.blob_path(gginfo.thunk_hash), "wb") as fout: fout.write(thunk_data) timelogger.add_point("write thunk to disk") executables_dir = os.path.join(curdir, 'executables') if os.path.exists(executables_dir): for exe in os.listdir(executables_dir): blob_path = GGPaths.blob_path(exe) exe_path = os.path.join(executables_dir, exe) if not os.path.exists(blob_path): shutil.copy(exe_path, blob_path) make_executable(blob_path) timelogger.add_point("copy executables to ggdir") # only clean up the gg directory if running on Lambda. if not fetch_dependencies(gginfo, gginfo.infiles, not GG_RUNNER): return {'errorType': 'GG-FetchDependenciesFailed'} for infile in gginfo.infiles: if infile['executable']: print("exe %s" % infile['hash']) make_executable(GGPaths.blob_path(infile['hash'])) timelogger.add_point("fetching the dependencies") return_code, output = run_command(["gg-execute-static", gginfo.thunk_hash]) if return_code: return { 'errorType': 'GG-ExecutionFailed', } timelogger.add_point("gg-execute") result = GGCache.check(gginfo.thunk_hash) if not result: return {'errorType': 'GG-ExecutionFailed'} executable = is_executable(GGPaths.blob_path(result)) timelogger.add_point("check the outfile") s3_client = boto3.client('s3') print("writing %s" % GGPaths.blob_path(result)) with open(GGPaths.blob_path(result), "rb") as bfile: blob = bfile.read() print("file read") r.set(result, blob) print("redis key set") """ s3_client.upload_file(GGPaths.blob_path(result), gginfo.s3_bucket, result) s3_client.put_object_acl( ACL='public-read', Bucket=gginfo.s3_bucket, Key=result ) s3_client.put_object_tagging( Bucket=gginfo.s3_bucket, Key=result, Tagging={ 'TagSet': [ { 'Key': 'gg:reduced_from', 'Value': gginfo.thunk_hash }, { 'Key': 'gg:executable', 'Value': 'true' if executable else 'false' } ] } ) """ timelogger.add_point("upload outfile to s3") if enable_timelog: print("s3 timelog uploading..") s3_client.put_object(ACL='public-read', Bucket=gginfo.s3_bucket, Key="runlogs/{}".format(gginfo.thunk_hash), Body=str({ 'output_hash': result, 'started': timelogger.start, 'timelog': timelogger.points }).encode('utf-8')) print("s3 timelog uploaded!") return { 'thunk_hash': gginfo.thunk_hash, 'output_hash': result, 'output_size': os.path.getsize(GGPaths.blob_path(result)), 'executable_output': executable }
def handler(event, context): os.environ['GG_STORAGE_URI'] = event['storageBackend'] thunks = event['thunks'] timelog = event.get('timelog') # Remove old thunk-execute directories os.system("rm -rf /tmp/thunk-execute.*") # Write thunks to disk for thunk_item in thunks: thunk_data = b64decode(thunk_item['data']) if os.path.exists(GGPaths.blob_path(thunk_item['hash'])): os.remove(GGPaths.blob_path(thunk_item['hash'])) with open(GGPaths.blob_path(thunk_item['hash']), "wb") as fout: fout.write(thunk_data) # Move executables from Lambda package to .gg directory executables_dir = os.path.join(curdir, 'executables') if os.path.exists(executables_dir): for exe in os.listdir(executables_dir): blob_path = GGPaths.blob_path(exe) exe_path = os.path.join(executables_dir, exe) if not os.path.exists(blob_path): shutil.copy(exe_path, blob_path) make_executable(blob_path) # Execute the thunk, and upload the result command = [ "gg-execute-static", "--get-dependencies", "--put-output", "--cleanup" ] if timelog: command += ["--timelog"] return_code, stdout = run_command(command + [x['hash'] for x in thunks]) executed_thunks = [] for thunk in thunks: outputs = [] for output_tag in thunk['outputs']: output_hash = GGCache.check(thunk['hash'], output_tag) if not output_hash: return {'returnCode': return_code, 'stdout': stdout} data = None if is_hash_for_thunk(output_hash): with open(GGPaths.blob_path(output_hash), 'rb') as tin: data = b64encode(tin.read()).decode('ascii') outputs += [{ 'tag': output_tag, 'hash': output_hash, 'size': os.path.getsize(GGPaths.blob_path(output_hash)), 'executable': is_executable(GGPaths.blob_path(output_hash)), 'data': data }] executed_thunks += [{'thunkHash': thunk['hash'], 'outputs': outputs}] return {'returnCode': 0, 'stdout': '', 'executedThunks': executed_thunks}