# from eggroll.api import eggroll from eggroll.api import clustercomm if __name__ == '__main__': eggroll.init("atest") clustercomm.init( "atest", { "local": { "role": "host", "party_id": 10001 }, "role": { "host": [10001, 10002], "arbiter": [99999], "guest": [10001] } }) for _tag in range(0, 1000, 2): c = eggroll.parallelize(range(_tag), partition=3, persistent=True).map(lambda k, v: (v, k + 1)) print(c) a = _tag clustercomm.remote(a, "RsaIntersectTransferVariable.rsa_pubkey", tag="{}".format(_tag)) clustercomm.remote(c, "RsaIntersectTransferVariable.rsa_pubkey", tag="{}".format(_tag + 1))
import numpy as np def f(iterator): sum = 0 for k, v in iterator: sum += v return sum if __name__ == "__main__": init() _matrix = np.ones([400, 50]) _table = parallelize(_matrix, partition=40) c = _table.mapValues(lambda _x: _x) dict(c.collect()) print(list(c.collect())) _table = parallelize(["b", "a", "c"], partition=5) a = _table.mapValues(lambda _x: _x + "1") print(list(a.collect())) print(dict(a.collect())) print(list(_table.collect())) x = _table.map(lambda k, v: (v, v + "1")) print(list(x.collect())) _table = parallelize([("a", ["apple", "banana", "lemon"]), ("b", ["grapes"])],