def handle_register_job(reader, writer):
print(
"-------------------------- REGISTER JOB --------------------------------",
time.time())
jobname_len = yield from reader.read(INT)
jobname_len, = struct.Struct("!i").unpack(jobname_len)
jobname = yield from reader.read(jobname_len + 3 * INT + SHORT)
jobname, num_lambdas, jobGB, peakMbps, latency_sensitive = struct.Struct(
"!" + str(jobname_len) + "siiih").unpack(jobname)
jobname = jobname.decode('utf-8')
# generate jobid
if 'gg' in jobname:
jobid = jobname + '-1234'
jobid_int = 1234
else:
jobid_int = randint(0, 1000000)
jobid = jobname + "-" + str(jobid_int)
print("received hints ", jobid, num_lambdas, jobGB, peakMbps,
latency_sensitive)
# create dir named jobid
createdirsock = pocket.connect(NAMENODE_IP, NAMENODE_PORT)
if createdirsock is None:
return
pocket.create_dir(createdirsock, None, jobid)
#pocket.close(createdirsock)
if jobGB == 0 or peakMbps == 0:
jobGB, peakMbps = compute_GB_Mbps_with_hints(num_lambdas, jobGB,
peakMbps,
latency_sensitive)
# generate weightmask
wmask, wmask_str = yield from generate_weightmask(jobid, jobGB, peakMbps,
latency_sensitive)
# wmask = [(ioctlcmd.calculate_datanode_hash("10.1.88.82", 50030), 1)]
# register job in table
err = add_job(jobid, jobGB, peakMbps, wmask, wmask_str)
# send wmask to metadata server
ioctlsock = yield from ioctlcmd.connect(NAMENODE_IP, NAMENODE_PORT)
if ioctlsock is None:
return
yield from ioctlcmd.send_weightmask(ioctlsock, jobid, wmask)
# reply to client with jobid int
resp_packer = struct.Struct(RESP_STRUCT_FORMAT + "i")
resp = (RESP_LEN_BYTES + INT, TICKET, JOB_CMD, err, REGISTER_OPCODE,
jobid_int)
pkt = resp_packer.pack(*resp)
writer.write(pkt)
print(
"-------------------------- REGISTERED JOB --------------------------------"
)
return
def handle_register_job(reader, writer):
print(
"-------------------------- REGISTER JOB --------------------------------",
time.time())
jobname_len = yield from reader.read(INT)
jobname_len, = struct.Struct("!i").unpack(jobname_len)
jobname = yield from reader.read(jobname_len + 3 * INT + SHORT)
jobname, num_lambdas, jobGB, peakMbps, latency_sensitive = struct.Struct(
"!" + str(jobname_len) + "siiih").unpack(jobname)
jobname = jobname.decode('utf-8')
# generate jobid
if 'gg' in jobname:
jobid = jobname + '-1234'
jobid_int = 1234
else:
jobid_int = randint(0, 1000000)
jobid = jobname + "-" + str(jobid_int)
print("received hints ", jobid, num_lambdas, jobGB, peakMbps,
latency_sensitive)
# create dir named jobid
# NOTE: this is blocking but we are not yielding
createdirsock = pocket.connect(NAMENODE_IP, NAMENODE_PORT)
if createdirsock is None:
return
ret = pocket.create_dir(createdirsock, None, jobid)
if jobGB == 0 or peakMbps == 0:
jobGB, peakMbps = compute_GB_Mbps_with_hints(num_lambdas, jobGB,
peakMbps,
latency_sensitive)
nvm_ip = []
for each_ip in nvm_ip:
print("Adding this nvme server" + each_ip)
add_nvme_datanodes(each_ip)
# generate weightmask
print("Generating weightmask")
wmask, wmask_str = yield from generate_weightmask(jobid, jobGB, peakMbps,
latency_sensitive)
# wmask = [(ioctlcmd.calculate_datanode_hash("10.1.88.82", 50030), 1)]
# register job in table
# nvm_ip = ['10.1.80.147', '10.1.71.111']
# for each_ip in nvm_ip:
# print("Adding this nvme server" + each_ip)
# add_nvme_datanodes(each_ip)
#wmask[1] = (ioctlcmd.calculate_datanode_hash(nvm_ip, 1234), 0.25)
#wmask_str[1] = (nvm_ip + ':1234', 0.24999999999999994)
err = add_job(jobid, jobGB, peakMbps, wmask, wmask_str)
# send wmask to metadata server
ioctlsock = yield from ioctlcmd.connect(NAMENODE_IP, NAMENODE_PORT)
if ioctlsock is None:
return
yield from ioctlcmd.send_weightmask(ioctlsock, jobid, wmask)
# reply to client with jobid int
resp_packer = struct.Struct(RESP_STRUCT_FORMAT + "i")
resp = (RESP_LEN_BYTES + INT, TICKET, JOB_CMD, err, REGISTER_OPCODE,
jobid_int)
pkt = resp_packer.pack(*resp)
writer.write(pkt)
print(
"-------------------------- REGISTERED JOB --------------------------------"
)
return