def __init__(self):
self.node = Server(node_id)
self.node.connect_neighbours()
class RequestHandler(server_pb2_grpc.CommunicationServiceServicer):
'''
'''
def __init__(self):
self.node = Server(node_id)
self.node.connect_neighbours()
def pingInternal(self, request, context):
return server_pb2.BoolResponse(result=True)
def getClientStatus(self, request, context):
return server_pb2.ClientResponse(id=self.node.id,
is_leader=self.node.is_leader(),
leader_id=self.node.getLeaderId())
def setLeader(self, request, context):
logger.info("Setting leader %d", request.id)
self.node.leader_id = request.id
self.node.voted = False
return server_pb2.BoolResponse(result=True)
def requestVote(self, request, context):
logger.info("Requesting vote %d", request.id)
res = self.node.giveVote(request.id)
return server_pb2.BoolResponse(result=res)
def getLeaderNode(self, request, context):
leader_node = self.node.getLeaderNode()
logger.info("leader node for node-id {} is {}".format(
str(self.node.id), str(leader_node)))
return server_pb2.ReplicationRequest(id=leader_node)
def getHandler(self, request, context):
print(request.getRequest.queryParams)
print(request)
#checking bloomfilter
d = int(
time.mktime(
time.strptime(request.getRequest.queryParams.from_utc,
'%Y-%m-%d %H:%M:%S')))
toDate = int(
time.mktime(
time.strptime(request.getRequest.queryParams.to_utc,
'%Y-%m-%d %H:%M:%S')))
process_internal = False
while d <= toDate:
date_to_check = (
datetime.datetime.fromtimestamp(d).strftime('%Y%m%d'))
c = self.node.bloomfilter
print("checking bloom filter ", date_to_check)
if c.testdate(date_to_check):
print("blloom filter said yes")
process_internal = True
break
d += (24 * 60 * 60)
if process_internal:
serverlist = self.node.get_active_node_ids()
return_queue = Queue(maxsize=0)
for node_id in serverlist:
print("Connecting to node", node_id)
assign_to_node = (Thread(target=self.connect_to_node,
args=(
node_id,
request,
return_queue,
)))
assign_to_node.setDaemon(True)
assign_to_node.start()
null_count = 0
while (True):
if (return_queue.qsize()):
print("queue data ", return_queue.qsize())
d = return_queue.get()
if not d:
print("incrementing null count")
null_count += 1
continue
yield (d)
if null_count == len(serverlist):
print("Breaking null check")
print("null_count", null_count)
print("server list", serverlist)
break
else:
if request.fromSender == "prof":
external_hosts = requests.get(
"http://cmpe275-spring-18.mybluemix.net/get").text
external_hosts = external_hosts.split(",")
request.fromSender = ""
leader_details = config.get_node_details(self.node.leader_id)
return_queue_external = Queue(maxsize=0)
for host_details in external_hosts:
if host_details == leader_details[0]:
continue
print("Connecting to host", host_details)
assign_to_node = (Thread(
target=self.connect_to_external_node,
args=(
host_details,
request,
return_queue_external,
)))
assign_to_node.setDaemon(True)
assign_to_node.start()
null_count = 0
while (True):
if (return_queue_external.qsize()):
print("queue data ", return_queue_external.qsize())
d = return_queue_external.get()
if not d:
print("incrementing null count")
null_count += 1
continue
yield (d)
if null_count == len(external_hosts) - 1:
print("Breaking null check")
print("null_count", null_count)
print("server list", external_hosts)
break
def connect_to_external_node(self, host_details, request, return_queue):
try:
client = Client(host=host_details, port=8080)
print("at..." + str(host_details))
print("Inside connect_to_node", request.getRequest.queryParams)
#fromTimestamp = getEpochTime(request.getRequest.queryParams.from_utc)
#toTimestamp = getEpochTime(request.getRequest.queryParams.to_utc)
stream = client.getHandler(request.getRequest.queryParams.from_utc,
request.getRequest.queryParams.to_utc)
for res in stream:
print("inserting into queue")
if res.datFragment.data:
print(res.datFragment.data)
print("yes data")
return_queue.put(res)
else:
print(res.datFragment.data)
print("no data")
finally:
return_queue.put(None)
def connect_to_node(self, node_id, request, return_queue):
# channel = grpc.insecure_channel(hostdetails)
# stub = request_pb2_grpc.CommunicationServiceStub(channel)
client = self.node.get_client(node_id)
print("at..." + str(node_id))
print("Inside connect_to_node", request.getRequest.queryParams)
#fromTimestamp = getEpochTime(request.getRequest.queryParams.from_utc)
#toTimestamp = getEpochTime(request.getRequest.queryParams.to_utc)
stream = client.GetFromLocalCluster(
request.getRequest.queryParams.from_utc,
request.getRequest.queryParams.to_utc)
for res in stream:
print("inserting into queue")
if res.datFragment.data:
print(res.datFragment.data)
print("yes data")
return_queue.put(res)
else:
print(res.datFragment.data)
print("no data")
return_queue.put(None)
#print ("data count from "+str(node_id)+" is "+str(return_queue.qsize()))
def GetFromLocalCluster(self, request, context):
print("Inside GetFromLocalCluster")
print((request.getRequest.queryParams))
fromTimestamp = getEpochTime(request.getRequest.queryParams.from_utc)
toTimestamp = getEpochTime(request.getRequest.queryParams.to_utc)
#fromTimestamp, toTimestamp = 1328114400000, 1328155200000
data_count = mongoTestNew.get_count_of_data(fromTimestamp, toTimestamp)
print("Data count is", data_count)
#TODO Move to config
offset = 0
limit = 2000
yield_count = 1
while (offset <= data_count):
query_data = mongoTestNew.get_data(fromTimestamp, toTimestamp,
offset, limit)
response = server_pb2.Response(
code=1,
msg="froms-1",
metaData=server_pb2.MetaData(uuid="",
numOfFragment=int(data_count)),
datFragment=server_pb2.DatFragment(
timestamp_utc="",
data=str(query_data).encode(encoding='utf_8')))
print("yield count", yield_count)
yield_count += 1
yield (response)
offset = offset + limit
def pushDataToNode(self, req, node_id):
print("Pusing data to ", node_id)
client = self.node.get_client(node_id)
res = client.PutToLocalCluster(
(req.putRequest.datFragment.data).decode('utf-8'))
if res.code != 1:
return False
return True
def putHandler(self, request_iterator, context):
serverlist = self.node.get_active_node_ids_for_push()
print("serverlist ", serverlist)
print("Inside put handler")
st_idx = 0
for req in request_iterator:
if not serverlist:
return server_pb2.Response(code=2)
node_id = serverlist[st_idx]
while (True):
if self.pushDataToNode(req, node_id):
st_idx = st_idx + 1
if st_idx > len(serverlist) - 1:
st_idx = 0
break
else:
print("Marking node as full ", node_id)
self.node.markNodeAsFull(node_id)
serverlist.pop(st_idx)
if not serverlist:
return server_pb2.Response(code=2)
node_id = serverlist[st_idx]
return server_pb2.Response(code=1)
def PutToLocalCluster(self, request_iterator, context):
for req in request_iterator:
if (mongoTestNew.get_mongo_connection().mesowest.command("dbstats")
["dataSize"] > space):
print("Inside PutToLocalCluster returening node full")
return server_pb2.Response(code=2)
mongoTestNew.put_data(
(req.putRequest.datFragment.data).decode('utf-8'))
return server_pb2.Response(code=1)
def ping(self, req, context):
print("Inside server ping")
return server_pb2.Response(code=1)
def getUniqueDateIds(self, request, context):
print(request)
data = mongoTestNew.get_dates()
dates = server_pb2.DateResponse()
for d in data:
d_t = dates.dates.add()
d_t.date = d
#extend
return dates
def updateBloomFilter(self, request, context):
nodeids = self.node.get_active_node_ids()
dates_set = set()
for node in nodeids:
client = self.node.get_client(node)
if client is None:
continue
data = client.getUniqueDateIds()
for d in data.dates:
dates_set.add(d.date)
print("bloom filter dates", dates_set)
c = CreateBloomFilter(len(dates_set), dates_set)
self.node.bloomfilter = c
return server_pb2.BoolResponse(result=True)
class RequestHandler(server_pb2_grpc.CommunicationServiceServicer):
def __init__(self):
self.node = Server(node_id)
self.node.connect_neighbours()
def pingInternal(self, request, context):
return server_pb2.BoolResponse(result=True)
def getClientStatus(self, request, context):
return server_pb2.ClientResponse(id=self.node.id,
is_leader=self.node.is_leader(),
leader_id=self.node.getLeaderId())
def setLeader(self, request, context):
logger.info("Setting leader %d", request.id)
self.node.leader_id = request.id
self.node.voted = False
return server_pb2.BoolResponse(result=True)
def requestVote(self, request, context):
logger.info("Requesting vote %d", request.id)
res = self.node.giveVote(request.id)
return server_pb2.BoolResponse(result=res)
def getLeaderNode(self, request, context):
leader_node = self.node.getLeaderNode()
logger.info("leader node for node-id {} is {}".format(
str(self.node.id), str(leader_node)))
return server_pb2.ReplicationRequest(id=leader_node)
def getHandler(self, request, context):
#checking bloomfilter
d = int(
time.mktime(
time.strptime(request.getRequest.queryParams.from_utc,
'%Y-%m-%d %H:%M:%S')))
toDate = int(
time.mktime(
time.strptime(request.getRequest.queryParams.to_utc,
'%Y-%m-%d %H:%M:%S')))
process_internal = False
while d <= toDate:
date_to_check = (
datetime.datetime.fromtimestamp(d).strftime('%Y%m%d'))
c = self.node.bloomfilter
if c.testdate(date_to_check):
process_internal = True
break
d += (24 * 60 * 60)
if process_internal:
serverlist = self.node.get_active_node_ids()
return_queue = Queue(maxsize=0)
for node_id in serverlist:
assign_to_node = (Thread(target=self.connect_to_node,
args=(
node_id,
request,
return_queue,
)))
assign_to_node.setDaemon(True)
assign_to_node.start()
null_count = 0
while (True):
if (return_queue.qsize()):
d = return_queue.get()
if not d:
null_count += 1
continue
if request.fromSender == "external-client":
yield (provideJson(d))
else:
yield (d)
if null_count == len(serverlist):
break
else:
if request.fromSender == "external-client":
#external_hosts = requests.get("http://cmpe275-spring-18.mybluemix.net/get").text
external_hosts = "169.254.149.215"
external_hosts = external_hosts.split(",")
request.fromSender = ""
leader_details = config.get_node_details(self.node.leader_id)
return_queue_external = Queue(maxsize=0)
for host_details in external_hosts:
if host_details == leader_details[0]:
continue
assign_to_node = (Thread(
target=self.connect_to_external_node,
args=(
host_details,
request,
return_queue_external,
)))
assign_to_node.setDaemon(True)
assign_to_node.start()
null_count = 0
while (True):
if (return_queue_external.qsize()):
d = return_queue_external.get()
if not d:
null_count += 1
continue
yield (provideJson(d))
if null_count == len(external_hosts) - 1:
break
def connect_to_external_node(self, host_details, request, return_queue):
try:
client = Client(host=host_details, port=8080)
stream = client.getHandler(request.getRequest.queryParams.from_utc,
request.getRequest.queryParams.to_utc)
for res in stream:
if res.datFragment.data:
return_queue.put(res)
finally:
return_queue.put(None)
def connect_to_node(self, node_id, request, return_queue):
client = self.node.get_client(node_id)
if client is not None:
stream = client.GetFromLocalCluster(
request.getRequest.queryParams.from_utc,
request.getRequest.queryParams.to_utc)
for res in stream:
if res.datFragment.data:
return_queue.put(res)
return_queue.put(None)
def GetFromLocalCluster(self, request, context):
fromTimestamp = getEpochTime(request.getRequest.queryParams.from_utc)
toTimestamp = getEpochTime(request.getRequest.queryParams.to_utc)
data_count = mongoTestNew.get_count_of_data(
fromTimestamp, toTimestamp,
request.getRequest.queryParams.params_json)
offset = 0
limit = 2000
yield_count = 1
while (offset <= data_count):
query_data = mongoTestNew.get_data(
fromTimestamp, toTimestamp, offset, limit,
request.getRequest.queryParams.params_json)
response = server_pb2.Response(
code=1,
metaData=server_pb2.MetaData(uuid="",
numOfFragment=int(data_count)),
datFragment=server_pb2.DatFragment(
timestamp_utc="",
data=str(query_data).encode(encoding='utf_8')))
yield_count += 1
yield (response)
offset = offset + limit
def pushDataToNode(self, req, node_id):
client = self.node.get_client(node_id)
res = client.PutToLocalCluster(
(req.putRequest.datFragment.data).decode('utf-8'))
if res.code != 1:
return False
return True
def pushDataToExternalCluster(self, req):
#external_hosts = requests.get("http://cmpe275-spring-18.mybluemix.net/get").text
external_hosts = "169.254.230.239"
external_hosts = external_hosts.split(",")
req.fromSender = ""
leader_details = config.get_node_details(self.node.leader_id)
for host_details in external_hosts:
if host_details == leader_details[0]:
continue
try:
client = Client(host=host_details, port=8080)
res = client.putHandler(
(req.putRequest.datFragment.data).decode('utf-8'))
if res.code == 1:
break
except Exception as e:
print(e)
def putHandler(self, request_iterator, context):
serverlist = self.node.get_active_node_ids_for_push()
st_idx = 0
counter = 0
for req in request_iterator:
if counter % 3 == 0 or not serverlist:
self.pushDataToExternalCluster(req)
counter = counter + 1
continue
if not serverlist:
return server_pb2.Response(code=2)
node_id = serverlist[st_idx]
while (True):
if self.pushDataToNode(req, node_id):
st_idx = st_idx + 1
if st_idx > len(serverlist) - 1:
st_idx = 0
break
else:
self.node.markNodeAsFull(node_id)
serverlist.pop(st_idx)
if not serverlist:
return server_pb2.Response(code=2)
if st_idx > len(serverlist) - 1:
st_idx = 0
node_id = serverlist[st_idx]
counter += 1
return server_pb2.Response(code=1)
def PutToLocalCluster(self, request_iterator, context):
for req in request_iterator:
if (mongoTestNew.get_mongo_connection().mesowest.command("dbstats")
["dataSize"] > space):
return server_pb2.Response(code=2)
mongoTestNew.put_data(req.putRequest.datFragment)
return server_pb2.Response(code=1)
def ping(self, req, context):
return server_pb2.Response(code=1)
def getUniqueDateIds(self, request, context):
data = mongoTestNew.get_dates()
dates = server_pb2.DateResponse()
for d in data:
d_t = dates.dates.add()
d_t.date = d
return dates
def updateBloomFilter(self, request, context):
nodeids = self.node.get_active_node_ids()
dates_set = set()
for node in nodeids:
client = self.node.get_client(node)
if client is None:
continue
data = client.getUniqueDateIds()
for d in data.dates:
dates_set.add(d.date)
c = CreateBloomFilter(len(dates_set), dates_set)
self.node.bloomfilter = c
return server_pb2.BoolResponse(result=True)
class RequestHandler(server_pb2_grpc.CommunicationServiceServicer):
'''
'''
def __init__(self):
self.node = Server(node_id)
self.node.connect_neighbours()
def pingInternal(self, request, context):
return server_pb2.BoolResponse(result=True)
def getClientStatus(self, request, context):
return server_pb2.ClientResponse(id=self.node.id,
is_leader=self.node.is_leader(),
leader_id=self.node.getLeaderId())
def setLeader(self, request, context):
logger.info("Setting leader %d", request.id)
self.node.leader_id = request.id
self.node.voted = False
return server_pb2.BoolResponse(result=True)
def requestVote(self, request, context):
logger.info("Requesting vote %d", request.id)
res = self.node.giveVote(request.id)
return server_pb2.BoolResponse(result=res)
def getLeaderNode(self, request, context):
leader_node = self.node.getLeaderNode()
logger.info("leader node for node-id {} is {}".format(
str(self.node.id), str(leader_node)))
return server_pb2.ReplicationRequest(id=leader_node)
def getHandler(self, request, context):
print("Inside gethandler")
print(request.getRequest.queryParams)
serverlist = self.node.get_active_node_ids()
return_queue = Queue(maxsize=0)
for node_id in serverlist:
print("Connecting to node", node_id)
assign_to_node = (Thread(target=self.connect_to_node,
args=(
node_id,
request,
return_queue,
)))
assign_to_node.setDaemon(True)
assign_to_node.start()
null_count = 0
while (True):
if (return_queue.qsize()):
print("queue data ", return_queue.qsize())
d = return_queue.get()
if not d:
print("incrementing null count")
null_count += 1
continue
yield (d)
if null_count == len(serverlist):
print("Breaking null check")
print("null_count", null_count)
print("server list", serverlist)
break
def connect_to_node(self, node_id, request, return_queue):
# channel = grpc.insecure_channel(hostdetails)
# stub = request_pb2_grpc.CommunicationServiceStub(channel)
client = self.node.get_client(node_id)
print("at..." + str(node_id))
print("Inside connect_to_node", request.getRequest.queryParams)
#fromTimestamp = getEpochTime(request.getRequest.queryParams.from_utc)
#toTimestamp = getEpochTime(request.getRequest.queryParams.to_utc)
stream = client.GetFromLocalCluster(
request.getRequest.queryParams.from_utc,
request.getRequest.queryParams.to_utc)
for res in stream:
print("inserting into queue")
return_queue.put(res)
return_queue.put(None)
#print ("data count from "+str(node_id)+" is "+str(return_queue.qsize()))
def GetFromLocalCluster(self, request, context):
print("Inside GetFromLocalCluster")
print((request.getRequest.queryParams))
fromTimestamp = getEpochTime(request.getRequest.queryParams.from_utc)
toTimestamp = getEpochTime(request.getRequest.queryParams.to_utc)
#fromTimestamp, toTimestamp = 1328114400000, 1328155200000
data_count = mongoTestNew.get_count_of_data(fromTimestamp, toTimestamp)
print("Data count is", data_count)
#TODO Move to config
offset = 0
limit = 2000
yield_count = 1
while (offset <= data_count):
query_data = mongoTestNew.get_data(fromTimestamp, toTimestamp,
offset, limit)
response = server_pb2.Response(
code=1,
msg="froms-1",
metaData=server_pb2.MetaData(uuid="",
numOfFragment=int(data_count)),
datFragment=server_pb2.DatFragment(
timestamp_utc="",
data=str(query_data).encode(encoding='utf_8')))
print("yield count", yield_count)
yield_count += 1
yield (response)
offset = offset + limit
def pushDataToNode(self, req, node_id):
print("Pusing data to ", node_id)
client = self.node.get_client(node_id)
res = client.PutToLocalCluster(
(req.putRequest.datFragment.data).decode('utf-8'))
if res != 1:
return False
return True
def putHandler(self, request_iterator, context):
serverlist = self.node.get_active_node_ids_for_push()
print("Inside put handler")
st_idx = 0
for req in request_iterator:
if not serverlist:
return server_pb2.Response(code=2)
node_id = serverlist[st_idx]
while (True):
if self.pushDataToNode(req, node_id):
st_idx = st_idx + 1
if st_idx > len(serverlist) - 1:
st_idx = 0
break
else:
print("Marking node as full ", node_id)
self.node.markNodeAsFull(node_id)
serverlist.pop(st_inx)
return server_pb2.Response(code=1)
def PutToLocalCluster(self, request_iterator, context):
print("server inside PutToLocalCluster")
for req in request_iterator:
print((req.putRequest.datFragment.data).decode('utf-8'))
mongoTestNew.put_data(
(req.putRequest.datFragment.data).decode('utf-8'))
return server_pb2.Response(code=1)
def ping(self, req, context):
print("Inside server ping")
return server_pb2.Response(code=1)