def local_main(): current_dir = os.path.dirname(os.path.realpath(__file__)) data_path = os.path.join(current_dir, "data") for pid in {"1", "2"}: # define name for the workflow workflow_name = "aspirin-local-test-" + pid # configure conclave conclave_config = CodeGenConfig(workflow_name, int(pid)) conclave_config.all_pids = [int(pid)] sharemind_conf = SharemindCodeGenConfig("/mnt/shared", use_docker=False, use_hdfs=False) conclave_config.with_sharemind_config(sharemind_conf) # point conclave to the directory where the generated code should be stored/ read from conclave_config.code_path = os.path.join("/mnt/shared", workflow_name) # point conclave to directory where data is to be read from... conclave_config.input_path = data_path # and written to conclave_config.output_path = data_path suffix = "left" if pid == "1" else "right" # define this party's unique ID (in this demo there is only one party) job_queue = generate_code(lambda: protocol_local(suffix, int(pid)), conclave_config, ["sharemind"], ["python"], apply_optimizations=False) dispatch_jobs(job_queue, conclave_config) res_mpc = read_rel(data_path + "/" + "actual_mpc_open.csv") res_left = read_rel(data_path + "/" + "actual_left.csv") res_right = read_rel(data_path + "/" + "actual_right.csv") assert len(res_mpc) == 1 assert len(res_left) == 1 assert len(res_right) == 1 res = [[res_mpc[0][0] + res_left[0][0] + res_right[0][0]]] write_rel(data_path, "actual_open.csv", res, "1")
def run_mpc(pid: str, data_root: str, mpc_backend: str): workflow_name = "aspirin-mpc-join-" + pid + "-" + data_root conclave_config = CodeGenConfig(workflow_name, int(pid)) conclave_config.use_leaky_ops = False conclave_config.code_path = os.path.join("/mnt/shared", workflow_name) conclave_config.input_path = os.path.join("/mnt/shared", data_root) conclave_config.output_path = os.path.join("/mnt/shared", data_root) job_queue = generate_code(lambda: protocol_mpc(conclave_config.all_pids), conclave_config, [mpc_backend], ["python"], apply_optimizations=True) dispatch_jobs(job_queue, conclave_config)
def main_mpc(pid: str, mpc_backend: str): # define name for the workflow workflow_name = "real-aspirin-partitioned-" + pid # configure conclave mpc_backend = sys.argv[2] conclave_config = CodeGenConfig(workflow_name, int(pid)) \ .with_default_mpc_config(mpc_backend) current_dir = os.path.dirname(os.path.realpath(__file__)) # point conclave to the directory where the generated code should be stored/ read from conclave_config.code_path = os.path.join("/mnt/shared", workflow_name) # point conclave to directory where data is to be read from... conclave_config.input_path = os.path.join(current_dir, "data") # and written to conclave_config.output_path = os.path.join(current_dir, "data") job_queue = generate_code(lambda: protocol_mpc(conclave_config.all_pids), conclave_config, [mpc_backend], ["python"], apply_optimizations=True) dispatch_jobs(job_queue, conclave_config)
def run_local(pid: str, data_root: str): workflow_name = "aspirin-local-join-" + pid + "-" + data_root conclave_config = CodeGenConfig(workflow_name, int(pid)) conclave_config.all_pids = [int(pid)] sharemind_conf = SharemindCodeGenConfig("/mnt/shared", use_docker=False, use_hdfs=False) conclave_config.with_sharemind_config(sharemind_conf) conclave_config.code_path = os.path.join("/mnt/shared", workflow_name) conclave_config.input_path = os.path.join("/mnt/shared", data_root) conclave_config.output_path = os.path.join("/mnt/shared", data_root) suffix = "left" if pid == "1" else "right" job_queue = generate_code(lambda: protocol_local(suffix, int(pid)), conclave_config, ["sharemind"], ["python"], apply_optimizations=False) dispatch_jobs(job_queue, conclave_config)
def main(): pid = sys.argv[1] data_root = sys.argv[2] mpc_backend = sys.argv[3] # define name for the workflow workflow_name = "aspirin-large-join-" + pid + "-" + data_root # configure conclave conclave_config = CodeGenConfig(workflow_name, int(pid)) if mpc_backend == "sharemind": sharemind_conf = SharemindCodeGenConfig("/mnt/shared", use_docker=True, use_hdfs=False) conclave_config.with_sharemind_config(sharemind_conf) elif mpc_backend == "obliv-c": conclave_config.all_pids = [1, 2] net_conf = [{ "host": "ca-spark-node-0", "port": 8001 }, { "host": "cb-spark-node-0", "port": 8002 }] net = NetworkConfig(net_conf, int(pid)) conclave_config.with_network_config(net) oc_conf = OblivcConfig("/obliv-c/bin/oblivcc", "ca-spark-node-0:9000") conclave_config.with_oc_config(oc_conf) else: raise Exception("Unknown MPC backend {}".format(mpc_backend)) conclave_config.code_path = os.path.join("/mnt/shared", workflow_name) conclave_config.input_path = os.path.join("/mnt/shared", data_root) conclave_config.output_path = os.path.join("/mnt/shared", data_root) job_queue = generate_code(protocol, conclave_config, [mpc_backend], ["python"], apply_optimizations=True) dispatch_jobs(job_queue, conclave_config)
joined = cc.join(left, right, "joined", ["a"], ["c"]) cc.aggregate(joined, "expected", ["b"], "d", "sum", "total") return {left, right} if __name__ == "__main__": pid = sys.argv[1] # define name for the workflow workflow_name = "simple-oblivious-test-" + pid # configure conclave conclave_config = CodeGenConfig(workflow_name, int(pid)) conclave_config.all_pids = [1] sharemind_conf = SharemindCodeGenConfig("/mnt/shared", use_docker=False, use_hdfs=False) conclave_config.with_sharemind_config(sharemind_conf) current_dir = os.path.dirname(os.path.realpath(__file__)) # point conclave to the directory where the generated code should be stored/ read from conclave_config.code_path = os.path.join("/mnt/shared", workflow_name) # point conclave to directory where data is to be read from... conclave_config.input_path = os.path.join(current_dir, "data") # and written to conclave_config.output_path = os.path.join(current_dir, "data") # define this party's unique ID (in this demo there is only one party) job_queue = generate_code(protocol, conclave_config, ["sharemind"], ["python"], apply_optimizations=False) dispatch_jobs(job_queue, conclave_config)
heart_patients = cc.cc_filter(aspirin, "heart_patients", diag_col_diags, "==", scalar=1) cc.collect(cc.distinct_count(heart_patients, "actual", pid_col_meds), 1) return {left_medication, left_diagnosis, right_medication, right_diagnosis} if __name__ == "__main__": pid = sys.argv[1] # define name for the workflow workflow_name = "real-aspirin-test-" + pid # configure conclave mpc_backend = sys.argv[2] conclave_config = CodeGenConfig(workflow_name, int(pid)) \ .with_default_mpc_config(mpc_backend) current_dir = os.path.dirname(os.path.realpath(__file__)) # point conclave to the directory where the generated code should be stored/ read from conclave_config.code_path = os.path.join("/mnt/shared", workflow_name) # point conclave to directory where data is to be read from... conclave_config.input_path = os.path.join(current_dir, "data") # and written to conclave_config.output_path = os.path.join(current_dir, "data") job_queue = generate_code(lambda: protocol(conclave_config.all_pids), conclave_config, [mpc_backend], ["python"], apply_optimizations=True) dispatch_jobs(job_queue, conclave_config)
def testHybridJoinWorkflow(): def protocol(): # define inputs colsIn1 = [ defCol("a", "INTEGER", [1]), defCol("b", "INTEGER", [1]), ] in1 = sal.create("in1", colsIn1, set([1])) proj1 = sal.project(in1, "proj1", ["a", "b"]) colsIn2 = [ defCol("c", "INTEGER", [1], [2]), defCol("d", "INTEGER", [2]) ] in2 = sal.create("in2", colsIn2, set([2])) proj2 = sal.project(in2, "proj2", ["c", "d"]) res = sal.join(proj1, proj2, "res", ["a"], ["c"]) # open result to party 1 sal.collect(res, 1) # return roots of dag return set([in1, in2]) pid = int(sys.argv[1]) size = sys.argv[2] workflow_name = "hybrid-join-" + str(pid) sm_cg_config = SharemindCodeGenConfig(workflow_name, "/mnt/shared", use_hdfs=False, use_docker=True) codegen_config = CodeGenConfig(workflow_name).with_sharemind_config( sm_cg_config) codegen_config.pid = pid codegen_config.code_path = "/mnt/shared/" + workflow_name codegen_config.input_path = "/mnt/shared/hybridjoin/" + size codegen_config.output_path = "/mnt/shared/hybridjoin/" + size jobqueue = generate_code(protocol, codegen_config, ["sharemind"], ["python"]) sharemind_config = { "pid": pid, "parties": { 1: { "host": "localhost", "port": 9001 }, 2: { "host": "localhost", "port": 9002 }, 3: { "host": "localhost", "port": 9003 } } } sm_peer = setup_peer(sharemind_config) conclave.dispatch.dispatch_all(None, sm_peer, jobqueue)
defCol("column_a", "INTEGER", [2]), defCol("column_b", "INTEGER", [2]) ] right = cc.create("right", input_columns_right, {2}) rel = cc.concat([left, right], "rel") filtered = cc.cc_filter(rel, "filtered", "column_b", "==", scalar=1) in_order = cc.sort_by(filtered, "in_order", "column_a") actual = cc.distinct_count(in_order, "actual", "column_a") cc.collect(actual, 1) return {left, right} if __name__ == "__main__": pid = sys.argv[1] # define name for the workflow workflow_name = "real-distinct-count-test-" + pid # configure conclave mpc_backend = sys.argv[2] conclave_config = CodeGenConfig(workflow_name, int(pid)) \ .with_default_mpc_config(mpc_backend) current_dir = os.path.dirname(os.path.realpath(__file__)) # point conclave to the directory where the generated code should be stored/ read from conclave_config.code_path = os.path.join("/mnt/shared", workflow_name) # point conclave to directory where data is to be read from... conclave_config.input_path = os.path.join(current_dir, "data") # and written to conclave_config.output_path = os.path.join(current_dir, "data") # define this party's unique ID (in this demo there is only one party) job_queue = generate_code(protocol, conclave_config, [mpc_backend], ["python"], apply_optimizations=True) dispatch_jobs(job_queue, conclave_config)