def run_command(key):
host = '172.31.12.102'
logger.info("Connecting to the jiffy server")
em = JiffyClient(host)
logger.info("Jiffy connected")
data_path = "/test"
data_size = 30 * 1024 * 1024
sample_data = 'a' * data_size
test_queue = em.open_or_create_queue(data_path,"local://tmp", 10,1)
logger.info("queue created")
test_queue.put(sample_data)
logger.info("Data put")
obj = test_queue.get()
logger.info("Data read: " + str(len(obj)))
em.close(data_path)
return 0
def run_command(key):
pywren.wrenlogging.default_config('INFO')
logging.basicConfig(level=logging.DEBUG)
logger = logging.getLogger(__name__)
logger.info("before everything")
logger.info(key)
t_s = time.time()
partition_num = key['partition_num']
rounds = key['rounds']
#em = key['em']
taskId = key['taskId']
appName = key['appName']
# partition_num = 1
# rounds = 8
# #em = key['em']
# taskId = 1
# appName = 'test-1'
em = JiffyClient(host=key['em'])
logger.info("berfore queue")
data_ques, msg_que = open_or_create_jiffy_queues(
em, appName, partition_num, 1, 'sender')
logger.info("queue opend")
for i in range(rounds):
### dd = read_s3_table(key)
msg = create_msg(rounds, taskId, i, partition_num)
######### create a table here to replace the input
right_table = 100000
indices = tm.makeStringIndex(right_table).values
key = np.tile(indices[:right_table], 1)
right = DataFrame({
"key": key,
"value": np.random.randn(right_table)
})
logger.info("Finish generating data")
x = 0
if i == rounds - 1:
x = 1
encoded = right.to_csv(sep="|", header=False,
index=False).encode('utf-8')
a = np.random.randint(1, 10, 500000)
encoded = np.asarray(a).astype('S100').tobytes()
# print(sys.getsizeof(encoded))
data_path = "/" + appName + "/" + '01'
test_que = em.open_or_create_queue(data_path, "local://tmp", 10, 1)
logger.info("get encoded size" + str(sys.getsizeof(encoded)))
ta = time.time()
test_que.put(encoded)
tb = time.time()
logger.info("wirte takes" + str(tb - ta))
logger.info("before get")
obj = test_que.get()
logger.info("get obj of size" + str(sys.getsizeof(obj)))
tc = time.time()
logger.info("get takes " + str(tc - tb))
# data_ques[0].put(encoded)
# logger.info("wirte finished")
# logger.info("before get")
# obj = data_ques[0].get()
#res = write_jiffy_partitions(right, ['key'], 'uniform', partition_num, data_ques, msg_que = msg_que, msg = msg, fin = x)
t_f = time.time()
# share.append([t_s,t_f])
return ([t_s, t_f])