def no_hdfs():
pid = int(sys.argv[1])
num_tuples = sys.argv[2]
workflow_name = "sharemind_join_{}_{}".format(num_tuples, pid)
sm_cg_config = SharemindCodeGenConfig(
workflow_name, "/mnt/shared", use_hdfs=False, use_docker=True)
codegen_config = CodeGenConfig(
workflow_name).with_sharemind_config(sm_cg_config)
codegen_config.code_path = "/mnt/shared/" + workflow_name
codegen_config.input_path = "/mnt/shared/join/" + num_tuples
codegen_config.output_path = "/mnt/shared/join/" + num_tuples
sharemind_config = {
"pid": pid,
"parties": {
1: {"host": "ca-spark-node-0", "port": 9001},
2: {"host": "cb-spark-node-0", "port": 9002},
3: {"host": "cc-spark-node-0", "port": 9003}
}
}
sm_peer = conclave.net.setup_peer(sharemind_config)
join(pid, codegen_config, sm_peer, num_tuples)
def scalar_div(namenode, root, f_size, master_url):
@dag_only
def protocol():
colsInA = [
defCol('a', 'INTEGER', [1]),
defCol('b', 'INTEGER', [1]),
]
in1 = sal.create("in1", colsInA, set([1]))
div1 = sal.divide(in1, 'div1', 'a', ['a', 5])
return set([in1])
dag = protocol()
config = CodeGenConfig('scalar_div_spark_{}'.format(f_size))
config.code_path = "/mnt/shared/" + config.name
config.input_path = "hdfs://{}/{}/{}" \
.format(namenode, root, f_size)
config.output_path = "hdfs://{}/{}/scalar_div_sp{}" \
.format(namenode, root, f_size)
cg = spark.SparkCodeGen(config, dag)
job = cg.generate(config.name, config.output_path)
job_queue = [job]
dis.dispatch_all(master_url, None, job_queue)
def project(namenode, root, f_size, master_url):
@dag_only
def protocol():
colsInA = [
defCol('a', 'INTEGER', [1]),
defCol('b', 'INTEGER', [1]),
defCol('c', 'INTEGER', [1]),
defCol('d', 'INTEGER', [1])
]
in1 = sal.create("in1", colsInA, set([1]))
cols = ([column.name for column in in1.out_rel.columns])
shuffle(cols)
proja = sal.project(in1, "proja", cols)
return set([in1])
dag = protocol()
config = CodeGenConfig('project_spark_{}'.format(f_size))
config.code_path = "/mnt/shared/" + config.name
config.input_path = "hdfs://{}/{}/{}" \
.format(namenode, root, f_size)
config.output_path = "hdfs://{}/{}/project_sp{}" \
.format(namenode, root, f_size)
cg = spark.SparkCodeGen(config, dag)
job = cg.generate(config.name, config.output_path)
job_queue = [job]
dis.dispatch_all(master_url, None, job_queue)
def setup(conf: Dict):
pid = conf["pid"]
hdfs_node_name = conf["spark"]["hdfs"]["node_name"]
hdfs_root = conf["spark"]["hdfs"]["root"]
spark_master_url = conf["spark"]["master_url"]
workflow_name = conf["workflow_name"]
sm_config = SharemindCodeGenConfig(conf["code_path"])
spark_config = SparkConfig(spark_master_url)
conclave_config = CodeGenConfig(workflow_name) \
.with_sharemind_config(sm_config) \
.with_spark_config(spark_config)
conclave_config.code_path = conf["code_path"] + workflow_name
conclave_config.input_path = "hdfs://{}/{}/{}".format(
hdfs_node_name, hdfs_root, conf["name"])
conclave_config.output_path = "hdfs://{}/{}/{}".format(
hdfs_node_name, hdfs_root, conf["name"])
conclave_config.pid = pid
conclave_config.name = workflow_name
conclave_config.all_pids = [int(p) for p in conf["all_pids"]]
network_config = NetworkConfig(conf["sharemind"]["parties"], pid)
conclave_config.with_network_config(network_config)
return conclave_config
def join(namenode, root, f_size, master_url):
@dag_only
def protocol():
colsInA = [
defCol('a', 'INTEGER', [1]),
defCol('b', 'INTEGER', [1]),
]
colsInB = [
defCol('a', 'INTEGER', [1]),
defCol('c', 'INTEGER', [1]),
]
in1 = sal.create("in1", colsInA, set([1]))
in2 = sal.create("in2", colsInB, set([1]))
join1 = sal.join(in1, in2, 'join1', ['a'], ['a'])
return set([in1, in2])
dag = protocol()
config = CodeGenConfig('join_spark_{}'.format(f_size))
config.code_path = "/mnt/shared/" + config.name
config.input_path = "hdfs://{}/{}/{}" \
.format(namenode, root, f_size)
config.output_path = "hdfs://{}/{}/join_sp{}" \
.format(namenode, root, f_size)
cg = spark.SparkCodeGen(config, dag)
job = cg.generate(config.name, config.output_path)
job_queue = [job]
dis.dispatch_all(master_url, None, job_queue)
def setup(conf: dict):
# GENERAL
pid = conf["user_config"]["pid"]
workflow_name = conf["user_config"]["workflow_name"]
all_pids = conf["user_config"]['all_pids']
use_leaky = conf["user_config"]["leaky_ops"]
conclave_config = CodeGenConfig(workflow_name)
# SPARK
try:
spark_avail = conf["backends"]["spark"]["available"]
if spark_avail:
spark_master_url = conf["backends"]["spark"]["master_url"]
spark_config = SparkConfig(spark_master_url)
conclave_config.with_spark_config(spark_config)
except KeyError:
pass
# OBLIV-C
try:
oc_avail = conf["backends"]["oblivc"]["available"]
if oc_avail:
oc_path = conf["backends"]["oblivc"]["oc_path"]
ip_port = conf["backends"]["oblivc"]["ip_port"]
oc_config = OblivcConfig(oc_path, ip_port)
conclave_config.with_oc_config(oc_config)
except KeyError:
pass
# JIFF
try:
jiff_avail = conf["backends"]["jiff"]["available"]
if jiff_avail:
jiff_path = conf["backends"]["jiff"]["jiff_path"]
party_count = conf["backends"]["jiff"]["party_count"]
server_ip = conf["backends"]["jiff"]["server_ip"]
server_port = conf["backends"]["jiff"]["server_port"]
jiff_config = JiffConfig(jiff_path, party_count, server_ip,
server_port)
conclave_config.with_jiff_config(jiff_config)
except KeyError:
pass
# NET
hosts = conf["net"]["parties"]
net_config = NetworkConfig(hosts, pid)
conclave_config.with_network_config(net_config)
conclave_config.pid = pid
conclave_config.all_pids = all_pids
conclave_config.name = workflow_name
conclave_config.use_leaky_ops = use_leaky
conclave_config.code_path = conf["user_config"]["paths"]["code_path"]
conclave_config.output_path = conf["user_config"]["paths"]["output_path"]
conclave_config.input_path = conf["user_config"]["paths"]["input_path"]
return conclave_config
def main():
pid = sys.argv[1]
data_root = sys.argv[2]
backend = sys.argv[3]
workflow_name = "hhi-benchmark-" + pid
if backend == "python":
sharemind_conf = SharemindCodeGenConfig("/mnt/shared",
use_docker=True,
use_hdfs=False)
conclave_config = CodeGenConfig(workflow_name, int(pid))
conclave_config.with_sharemind_config(sharemind_conf)
conclave_config.code_path = os.path.join("/mnt/shared", workflow_name)
conclave_config.input_path = os.path.join("/mnt/shared", data_root)
conclave_config.output_path = os.path.join("/mnt/shared", data_root)
generate_and_dispatch(protocol,
conclave_config, ["sharemind"], ["python"],
apply_optimizations=True)
elif backend == "spark":
sharemind_conf = SharemindCodeGenConfig("/mnt/shared",
use_docker=True,
use_hdfs=True)
conclave_config = CodeGenConfig(workflow_name, int(pid))
host = conclave_config.network_config["parties"][int(pid)]["host"]
# Update this if your spark master and HDFS namenode are mapped to a different host than your Conclave node
spark_master_url = "spark://{}:7077".format(host)
hdfs_namenode = "{}:9000".format(host)
spark_config = SparkConfig(spark_master_url)
conclave_config \
.with_sharemind_config(sharemind_conf) \
.with_spark_config(spark_config)
conclave_config.code_path = os.path.join("/mnt/shared", workflow_name)
conclave_config.input_path = "hdfs://{}/{}".format(
hdfs_namenode, data_root)
conclave_config.output_path = "hdfs://{}/{}".format(
hdfs_namenode, data_root)
generate_and_dispatch(protocol,
conclave_config, ["sharemind"], ["spark"],
apply_optimizations=True)
else:
raise Exception("Unknown backend {}".format(backend))
)
sys.exit(1)
pid = int(sys.argv[1])
hdfs_namenode = sys.argv[2]
hdfs_root = sys.argv[3]
spark_master_url = sys.argv[4]
workflow_name = "job-" + str(pid)
sm_config = SharemindCodeGenConfig("/mnt/shared")
spark_config = SparkConfig(spark_master_url)
conclave_config = CodeGenConfig(workflow_name) \
.with_sharemind_config(sm_config) \
.with_spark_config(spark_config)
conclave_config.code_path = "/mnt/shared/" + workflow_name
conclave_config.input_path = "hdfs://{}/{}/taxi".format(
hdfs_namenode, hdfs_root)
conclave_config.output_path = "hdfs://{}/{}/taxi".format(
hdfs_namenode, hdfs_root)
conclave_config.pid = pid
conclave_config.name = workflow_name
network_config = {
"pid": pid,
"parties": {
1: {
"host": "ca-spark-node-0",
"port": 9001
},
2: {
"host": "cb-spark-node-0",
"port": 9002
},
def testHybridJoinWorkflow():
def protocol():
# define inputs
colsIn1 = [
defCol("a", "INTEGER", [1]),
defCol("b", "INTEGER", [1]),
]
in1 = sal.create("in1", colsIn1, set([1]))
proj1 = sal.project(in1, "proj1", ["a", "b"])
colsIn2 = [
defCol("c", "INTEGER", [1], [2]),
defCol("d", "INTEGER", [2])
]
in2 = sal.create("in2", colsIn2, set([2]))
proj2 = sal.project(in2, "proj2", ["c", "d"])
res = sal.join(proj1, proj2, "res", ["a"], ["c"])
# open result to party 1
sal.collect(res, 1)
# return roots of dag
return set([in1, in2])
pid = int(sys.argv[1])
size = sys.argv[2]
workflow_name = "hybrid-join-" + str(pid)
sm_cg_config = SharemindCodeGenConfig(workflow_name,
"/mnt/shared",
use_hdfs=False,
use_docker=True)
codegen_config = CodeGenConfig(workflow_name).with_sharemind_config(
sm_cg_config)
codegen_config.pid = pid
codegen_config.code_path = "/mnt/shared/" + workflow_name
codegen_config.input_path = "/mnt/shared/hybridjoin/" + size
codegen_config.output_path = "/mnt/shared/hybridjoin/" + size
jobqueue = generate_code(protocol, codegen_config, ["sharemind"],
["python"])
sharemind_config = {
"pid": pid,
"parties": {
1: {
"host": "localhost",
"port": 9001
},
2: {
"host": "localhost",
"port": 9002
},
3: {
"host": "localhost",
"port": 9003
}
}
}
sm_peer = setup_peer(sharemind_config)
conclave.dispatch.dispatch_all(None, sm_peer, jobqueue)
return set([in1, in2, in3])
if __name__ == "__main__":
pid = int(sys.argv[1])
workflow_name = "sharemind-ssn-" + str(pid)
sm_cg_config = SharemindCodeGenConfig(workflow_name,
"/mnt/shared",
use_hdfs=False,
use_docker=True)
codegen_config = CodeGenConfig(workflow_name).with_sharemind_config(
sm_cg_config)
codegen_config.code_path = "/mnt/shared/" + workflow_name
codegen_config.input_path = "/mnt/shared/ssn-data"
codegen_config.output_path = "/mnt/shared/ssn-data"
job = SharemindCodeGen(codegen_config, protocol(),
pid).generate("sharemind-0", "")
job_queue = [job]
sharemind_config = {
"pid": pid,
"parties": {
1: {
"host": "ca-spark-node-0",
"port": 9001
},
2: {
"host": "cb-spark-node-0",
def testHybridAggWorkflow():
@dag_only
def protocol():
# define inputs
colsInA = [
defCol("a", "INTEGER", [1]),
defCol("b", "INTEGER", [1]),
]
in1 = sal.create("in1", colsInA, set([1]))
in1.isMPC = False
proja = sal.project(in1, "proja", ["a", "b"])
proja.isMPC = False
proja.out_rel.storedWith = set([1])
# define inputs
colsInB = [
defCol("a", "INTEGER", [2]),
defCol("b", "INTEGER", [2]),
]
in2 = sal.create("in2", colsInB, set([2]))
in2.isMPC = False
projb = sal.project(in2, "projb", ["a", "b"])
projb.isMPC = False
projb.out_rel.storedWith = set([2])
# define inputs
colsInC = [
defCol("a", "INTEGER", [3]),
defCol("b", "INTEGER", [3]),
]
in3 = sal.create("in3", colsInC, set([3]))
in3.isMPC = False
projc = sal.project(in3, "projc", ["a", "b"])
projc.isMPC = False
projc.out_rel.storedWith = set([3])
clA = sal._close(proja, "clA", set([1, 2, 3]))
clA.isMPC = True
clB = sal._close(projb, "clB", set([1, 2, 3]))
clB.isMPC = True
clC = sal._close(projc, "clC", set([1, 2, 3]))
clC.isMPC = True
comb = sal.concat([clA, clB, clC], "comb")
comb.out_rel.storedWith = set([1, 2, 3])
comb.isMPC = True
shuffled = sal.shuffle(comb, "shuffled")
shuffled.out_rel.storedWith = set([1, 2, 3])
shuffled.isMPC = True
persisted = sal._persist(shuffled, "persisted")
persisted.out_rel.storedWith = set([1, 2, 3])
persisted.isMPC = True
keysclosed = sal.project(shuffled, "keysclosed", ["a"])
keysclosed.out_rel.storedWith = set([1, 2, 3])
keysclosed.isMPC = True
keys = sal._open(keysclosed, "keys", 1)
keys.isMPC = True
indexed = sal.index(keys, "indexed", "rowIndex")
indexed.isMPC = False
indexed.out_rel.storedWith = set([1])
sortedByKey = sal.sort_by(indexed, "sortedByKey", "a")
sortedByKey.isMPC = False
sortedByKey.out_rel.storedWith = set([1])
eqFlags = sal._comp_neighs(sortedByKey, "eqFlags", "a")
eqFlags.isMPC = False
eqFlags.out_rel.storedWith = set([1])
# TODO: hack to get keys stored
# need to fix later!
sortedByKey = sal.project(sortedByKey, "sortedByKey",
["rowIndex", "a"])
sortedByKey.isMPC = False
sortedByKey.out_rel.storedWith = set([1])
closedEqFlags = sal._close(eqFlags, "closedEqFlags", set([1, 2, 3]))
closedEqFlags.isMPC = True
closedSortedByKey = sal._close(sortedByKey, "closedSortedByKey",
set([1, 2, 3]))
closedSortedByKey.isMPC = True
agg = sal.index_aggregate(persisted, "agg", ["a"], "b", "+", "b",
closedEqFlags, closedSortedByKey)
agg.out_rel.storedWith = set([1, 2, 3])
agg.isMPC = True
sal._open(agg, "opened", 1)
# create condag
return set([in1, in2, in3])
pid = int(sys.argv[1])
size = sys.argv[2]
workflow_name = "hybrid-agg-" + str(pid)
sm_cg_config = SharemindCodeGenConfig(workflow_name,
"/mnt/shared",
use_hdfs=False,
use_docker=True)
codegen_config = CodeGenConfig(workflow_name).with_sharemind_config(
sm_cg_config)
codegen_config.code_path = "/mnt/shared/" + workflow_name
codegen_config.input_path = "/mnt/shared/" + size
codegen_config.output_path = "/mnt/shared/" + size
dag = protocol()
mapping = part.heupart(dag, ["sharemind"], ["python"])
job_queue = []
for idx, (fmwk, subdag, storedWith) in enumerate(mapping):
if fmwk == "sharemind":
job = SharemindCodeGen(codegen_config, subdag,
pid).generate("sharemind-" + str(idx), None)
else:
job = PythonCodeGen(codegen_config,
subdag).generate("python-" + str(idx), None)
# TODO: this probably doesn't belong here
if not pid in storedWith:
job.skip = True
job_queue.append(job)
sharemind_config = {
"pid": pid,
"parties": {
1: {
"host": "ca-spark-node-0",
"port": 9001
},
2: {
"host": "cb-spark-node-0",
"port": 9002
},
3: {
"host": "cc-spark-node-0",
"port": 9003
}
}
}
sm_peer = setup_peer(sharemind_config)
dispatch_all(None, sm_peer, job_queue)
def testPublicJoinWorkflow():
@dag_only
def protocol():
# define inputs
colsInA = [
defCol("a", "INTEGER", [1]),
defCol("b", "INTEGER", [1]),
]
in1 = sal.create("in1", colsInA, set([1]))
in1.isMPC = False
proja = sal.project(in1, "proja", ["a", "b"])
proja.isMPC = False
proja.out_rel.storedWith = set([1])
colsInB = [
defCol("c", "INTEGER", [1], [2]),
defCol("d", "INTEGER", [2])
]
in2 = sal.create("in2", colsInB, set([2]))
in2.isMPC = False
projb = sal.project(in2, "projb", ["c", "d"])
projb.isMPC = False
projb.out_rel.storedWith = set([2])
clA = sal._close(proja, "clA", set([1, 2, 3]))
clA.isMPC = True
clB = sal._close(projb, "clB", set([1, 2, 3]))
clB.isMPC = True
persistedA = sal._persist(clA, "persistedA")
persistedB = sal._persist(clB, "persistedB")
keysaclosed = sal.project(clA, "keysaclosed", ["a"])
keysaclosed.out_rel.storedWith = set([1, 2, 3])
keysaclosed.isMPC = True
keysbclosed = sal.project(clB, "keysbclosed", ["c"])
keysbclosed.isMPC = True
keysbclosed.out_rel.storedWith = set([1, 2, 3])
keysa = sal._open(keysaclosed, "keysa", 1)
keysa.isMPC = True
keysb = sal._open(keysbclosed, "keysb", 1)
keysb.isMPC = True
indexedA = sal.index(keysa, "indexedA", "indexA")
indexedA.isMPC = False
indexedA.out_rel.storedWith = set([1])
indexedB = sal.index(keysb, "indexedB", "indexB")
indexedB.isMPC = False
indexedB.out_rel.storedWith = set([1])
joinedindeces = sal.join(
indexedA, indexedB, "joinedindeces", ["a"], ["c"])
joinedindeces.isMPC = False
joinedindeces.out_rel.storedWith = set([1])
indecesonly = sal.project(
joinedindeces, "indecesonly", ["indexA", "indexB"])
indecesonly.isMPC = False
indecesonly.out_rel.storedWith = set([1])
indecesclosed = sal._close(
indecesonly, "indecesclosed", set([1, 2, 3]))
indecesclosed.isMPC = True
joined = sal._index_join(persistedA, persistedB, "joined",
["a"], ["c"], indecesclosed)
joined.isMPC = True
sal._open(joined, "opened", 1)
# create condag
return set([in1, in2])
pid = int(sys.argv[1])
workflow_name = "hybrid-join-" + str(pid)
sm_cg_config = SharemindCodeGenConfig(
workflow_name, "/mnt/shared", use_hdfs=False)
codegen_config = CodeGenConfig(
workflow_name).with_sharemind_config(sm_cg_config)
codegen_config.code_path = "/mnt/shared/" + workflow_name
codegen_config.input_path = "/mnt/shared"
codegen_config.output_path = "/mnt/shared"
exampleutils.generate_data(pid, codegen_config.output_path)
dag = protocol()
mapping = part.heupart(dag, ["sharemind"], ["python"])
job_queue = []
for idx, (fmwk, subdag, storedWith) in enumerate(mapping):
if fmwk == "sharemind":
job = SharemindCodeGen(codegen_config, subdag, pid).generate(
"sharemind-" + str(idx), None)
else:
job = PythonCodeGen(codegen_config, subdag).generate(
"python-" + str(idx), None)
# TODO: this probably doesn't belong here
if not pid in storedWith:
job.skip = True
job_queue.append(job)
sharemind_config = exampleutils.get_sharemind_config(pid, True)
sm_peer = setup_peer(sharemind_config)
dispatch_all(None, sm_peer, job_queue)
if pid == 1:
expected = ['', '2,200,2001', '3,300,3001', '4,400,4001', '42,42,1001', '5,500,5001',
'6,600,6001', '7,700,7001', '7,800,7001', '7,900,7001', '8,1000,8001', '9,1100,9001']
exampleutils.check_res(expected, "/mnt/shared/opened.csv")
print("Success")
agg = sal.aggregate(rel, "agg", ["a"], "b", "+", "total")
opened = sal._open(agg, "opened", 1)
# return root nodes
return set([in1, in2, in3])
if __name__ == "__main__":
pid = int(sys.argv[1])
sharemind_config = exampleutils.get_sharemind_config(pid)
sm_peer = conclave.net.setup_peer(sharemind_config)
workflow_name = "job-" + str(pid)
sm_cg_config = SharemindCodeGenConfig(workflow_name,
"/mnt/shared",
use_hdfs=False)
codegen_config = CodeGenConfig(workflow_name).with_sharemind_config(
sm_cg_config)
codegen_config.code_path = "/mnt/shared/" + workflow_name
codegen_config.input_path = "/mnt/shared"
codegen_config.output_path = "/mnt/shared"
exampleutils.generate_data(pid, codegen_config.output_path)
job = SharemindCodeGen(codegen_config, protocol(),
pid).generate("sharemind-0", "")
job_queue = [job]
conclave.dispatch.dispatch_all(None, sm_peer, job_queue)
def testPublicJoinWorkflow():
@dag_only
def protocol():
# define inputs
colsInA = [
defCol("a", "INTEGER", [1]),
defCol("b", "INTEGER", [1]),
]
in1 = sal.create("in1", colsInA, set([1]))
in1.isMPC = False
proja = sal.project(in1, "proja", ["a", "b"])
proja.isMPC = False
proja.out_rel.storedWith = set([1])
colsInB = [
defCol("c", "INTEGER", [1], [2]),
defCol("d", "INTEGER", [2])
]
in2 = sal.create("in2", colsInB, set([2]))
in2.isMPC = False
projb = sal.project(in2, "projb", ["c", "d"])
projb.isMPC = False
projb.out_rel.storedWith = set([2])
clA = sal._close(proja, "clA", set([1, 2, 3]))
clA.isMPC = True
clB = sal._close(projb, "clB", set([1, 2, 3]))
clB.isMPC = True
persistedA = sal._persist(clA, "persistedA")
persistedA.isMPC = True
persistedB = sal._persist(clB, "persistedB")
persistedB.isMPC = True
keysaclosed = sal.project(clA, "keysaclosed", ["a"])
keysaclosed.out_rel.storedWith = set([1, 2, 3])
keysaclosed.isMPC = True
keysbclosed = sal.project(clB, "keysbclosed", ["c"])
keysbclosed.isMPC = True
keysbclosed.out_rel.storedWith = set([1, 2, 3])
keysa = sal._open(keysaclosed, "keysa", 1)
keysa.isMPC = True
keysb = sal._open(keysbclosed, "keysb", 1)
keysb.isMPC = True
indexedA = sal.index(keysa, "indexedA", "indexA")
indexedA.isMPC = False
indexedA.out_rel.storedWith = set([1])
indexedB = sal.index(keysb, "indexedB", "indexB")
indexedB.isMPC = False
indexedB.out_rel.storedWith = set([1])
joinedindeces = sal.join(
indexedA, indexedB, "joinedindeces", ["a"], ["c"])
joinedindeces.isMPC = False
joinedindeces.out_rel.storedWith = set([1])
indecesonly = sal.project(
joinedindeces, "indecesonly", ["indexA", "indexB"])
indecesonly.isMPC = False
indecesonly.out_rel.storedWith = set([1])
indecesclosed = sal._close(
indecesonly, "indecesclosed", set([1, 2, 3]))
indecesclosed.isMPC = True
joined = sal._index_join(persistedA, persistedB, "joined", [
"a"], ["c"], indecesclosed)
joined.out_rel.storedWith = set([1, 2, 3])
joined.isMPC = True
sal._open(joined, "opened", 1)
# create condag
return set([in1, in2])
pid = int(sys.argv[1])
size = sys.argv[2]
workflow_name = "public-join-" + str(pid)
sm_cg_config = SharemindCodeGenConfig(
workflow_name, "/mnt/shared", use_hdfs=False, use_docker=True)
codegen_config = CodeGenConfig(
workflow_name).with_sharemind_config(sm_cg_config)
codegen_config.code_path = "/mnt/shared/" + workflow_name
codegen_config.input_path = "/mnt/shared/hybridjoin/" + size
codegen_config.output_path = "/mnt/shared/hybridjoin/" + size
dag = protocol()
mapping = part.heupart(dag, ["sharemind"], ["python"])
job_queue = []
for idx, (fmwk, subdag, storedWith) in enumerate(mapping):
if fmwk == "sharemind":
job = SharemindCodeGen(codegen_config, subdag, pid).generate(
"sharemind-" + str(idx), None)
else:
job = PythonCodeGen(codegen_config, subdag).generate(
"python-" + str(idx), None)
# TODO: this probably doesn't belong here
if not pid in storedWith:
job.skip = True
job_queue.append(job)
sharemind_config = {
"pid": pid,
"parties": {
1: {"host": "ca-spark-node-0", "port": 9001},
2: {"host": "cb-spark-node-0", "port": 9002},
3: {"host": "cc-spark-node-0", "port": 9003}
}
}
sm_peer = setup_peer(sharemind_config)
dispatch_all(None, sm_peer, job_queue)
def testHybridJoinWorkflow():
def hybrid_join():
# define inputs
colsInA = [
defCol("a", "INTEGER", [1]),
defCol("b", "INTEGER", [1]),
]
in1 = sal.create("in1", colsInA, set([1]))
in1.isMPC = False
proja = sal.project(in1, "proja", ["a", "b"])
proja.isMPC = False
proja.out_rel.storedWith = set([1])
colsInB = [
defCol("c", "INTEGER", [1], [2]),
defCol("d", "INTEGER", [2])
]
in2 = sal.create("in2", colsInB, set([2]))
in2.isMPC = False
projb = sal.project(in2, "projb", ["c", "d"])
projb.isMPC = False
projb.out_rel.storedWith = set([2])
clA = sal._close(proja, "clA", set([1, 2, 3]))
clA.isMPC = True
clB = sal._close(projb, "clB", set([1, 2, 3]))
clB.isMPC = True
shuffledA = sal.shuffle(clA, "shuffledA")
shuffledA.isMPC = True
persistedA = sal._persist(shuffledA, "persistedA")
persistedA.isMPC = True
shuffledB = sal.shuffle(clB, "shuffledB")
shuffledB.isMPC = True
persistedB = sal._persist(shuffledB, "persistedB")
persistedB.isMPC = True
keysaclosed = sal.project(shuffledA, "keysaclosed", ["a"])
keysaclosed.out_rel.storedWith = set([1, 2, 3])
keysaclosed.isMPC = True
keysbclosed = sal.project(shuffledB, "keysbclosed", ["c"])
keysbclosed.isMPC = True
keysbclosed.out_rel.storedWith = set([1, 2, 3])
keysa = sal._open(keysaclosed, "keysa", 1)
keysa.isMPC = True
keysb = sal._open(keysbclosed, "keysb", 1)
keysb.isMPC = True
indexedA = sal.index(keysa, "indexedA", "indexA")
indexedA.isMPC = False
indexedA.out_rel.storedWith = set([1])
indexedB = sal.index(keysb, "indexedB", "indexB")
indexedB.isMPC = False
indexedB.out_rel.storedWith = set([1])
joinedindeces = sal.join(
indexedA, indexedB, "joinedindeces", ["a"], ["c"])
joinedindeces.isMPC = False
joinedindeces.out_rel.storedWith = set([1])
indecesonly = sal.project(
joinedindeces, "indecesonly", ["indexA", "indexB"])
indecesonly.isMPC = False
indecesonly.out_rel.storedWith = set([1])
indecesclosed = sal._close(
indecesonly, "indecesclosed", set([1, 2, 3]))
indecesclosed.isMPC = True
joined = sal._index_join(persistedA, persistedB, "joined", [
"a"], ["c"], indecesclosed)
joined.isMPC = True
return joined, set([in1, in2])
def hybrid_agg(in1):
shuffled = sal.shuffle(in1, "shuffled")
shuffled.out_rel.storedWith = set([1, 2, 3])
shuffled.isMPC = True
persisted = sal._persist(shuffled, "persisted")
persisted.out_rel.storedWith = set([1, 2, 3])
persisted.isMPC = True
keysclosed = sal.project(shuffled, "keysclosed", ["b"])
keysclosed.out_rel.storedWith = set([1, 2, 3])
keysclosed.isMPC = True
keys = sal._open(keysclosed, "keys", 1)
keys.isMPC = True
indexed = sal.index(keys, "indexed", "rowIndex")
indexed.isMPC = False
indexed.out_rel.storedWith = set([1])
distinctKeys = sal.distinct(keys, "distinctKeys", ["b"])
distinctKeys.isMPC = False
distinctKeys.out_rel.storedWith = set([1])
# TODO: hack to get keys stored
# need to fix later!
fakeDistinctKeys = sal.distinct(keys, "distinctKeys", ["b"])
fakeDistinctKeys.isMPC = False
fakeDistinctKeys.out_rel.storedWith = set([1])
indexedDistinct = sal.index(distinctKeys, "indexedDistinct", "keyIndex")
indexedDistinct.isMPC = False
indexedDistinct.out_rel.storedWith = set([1])
joinedindeces = sal.join(
indexed, indexedDistinct, "joinedindeces", ["b"], ["b"])
joinedindeces.isMPC = False
joinedindeces.out_rel.storedWith = set([1])
# TODO: could project row indeces away too
indecesonly = sal.project(
joinedindeces, "indecesonly", ["rowIndex", "keyIndex"])
indecesonly.isMPC = False
indecesonly.out_rel.storedWith = set([1])
closedDistinct = sal._close(distinctKeys, "closedDistinct", set([1, 2, 3]))
closedDistinct.isMPC = True
closedLookup = sal._close(indecesonly, "closedLookup", set([1, 2, 3]))
closedLookup.isMPC = True
agg = sal.index_aggregate(persisted, "agg", ["b"], "d", "+", "d", closedLookup, closedDistinct)
agg.isMPC = True
sal._open(agg, "aggopened", 1)
def protocol():
joinedres, inputs = hybrid_join()
hybrid_agg(joinedres)
return saldag.OpDag(inputs)
pid = int(sys.argv[1])
workflow_name = "ssn-" + str(pid)
sm_cg_config = SharemindCodeGenConfig(
workflow_name, "/mnt/shared", use_hdfs=False, use_docker=False)
codegen_config = CodeGenConfig(
workflow_name).with_sharemind_config(sm_cg_config)
codegen_config.code_path = "/mnt/shared/" + workflow_name
codegen_config.input_path = "/mnt/shared"
codegen_config.output_path = "/mnt/shared"
exampleutils.generate_ssn_data(pid, codegen_config.output_path)
dag = protocol()
mapping = part.heupart(dag, ["sharemind"], ["python"])
job_queue = []
for idx, (fmwk, subdag, storedWith) in enumerate(mapping):
if fmwk == "sharemind":
job = SharemindCodeGen(codegen_config, subdag, pid).generate(
"sharemind-" + str(idx), None)
else:
job = PythonCodeGen(codegen_config, subdag).generate(
"python-" + str(idx), None)
# TODO: this probably doesn't belong here
if not pid in storedWith:
job.skip = True
job_queue.append(job)
sharemind_config = exampleutils.get_sharemind_config(pid, True)
sm_peer = setup_peer(sharemind_config)
dispatch_all(None, sm_peer, job_queue)
if pid == 1:
expected = ['', '1,30', '2,50', '3,30']
exampleutils.check_res(expected, "/mnt/shared/aggopened.csv")
print("Success")