def run():
global host
config.populate()
node_id = config.get_node_id()
node_details = config.get_node_details(node_id)
logger.info("Connecting to host {} on port {}".format(
node_details[0], node_details[1]))
c = Client(node_details[0], node_details[1])
c.updateBloomFilter()
leader_node = 0
heart_beat_count = 0
requests.get("http://cmpe275-spring-18.mybluemix.net/delete/" +
(config.get_node_details(node_id))[0])
while (True):
time.sleep(heartbeat_interval)
heart_beat_count += 1
leader = c.getLeaderNode(node_id)
if leader_node != leader.id and leader.id == node_id:
if leader_node != 0:
requests.get("http://cmpe275-spring-18.mybluemix.net/delete/" +
config.get_node_details(leader_node)[0])
requests.get("http://cmpe275-spring-18.mybluemix.net/put/" +
config.get_node_details(leader.id)[0])
leader_node = leader.id
logger.info(
"Publish node_id {} to external cluster".format(node_id))
if heart_beat_count * heartbeat_interval == bloom_filter_interval:
c.updateBloomFilter()
def pushDataToExternalCluster(self, req):
#external_hosts = requests.get("http://cmpe275-spring-18.mybluemix.net/get").text
external_hosts = "169.254.230.239"
external_hosts = external_hosts.split(",")
req.fromSender = ""
leader_details = config.get_node_details(self.node.leader_id)
for host_details in external_hosts:
if host_details == leader_details[0]:
continue
try:
client = Client(host=host_details, port=8080)
res = client.putHandler(
(req.putRequest.datFragment.data).decode('utf-8'))
if res.code == 1:
break
except Exception as e:
print(e)
import glob
import gzip
import shutil
import re
from chunktest import process
from config import get_client_map, get_node_details, populate
from client import Client
import server_pb2
import server_pb2_grpc
from server_pb2 import Request, PutRequest, QueryParams, DatFragment
populate()
node_details = get_node_details(1)
print(node_details)
c = Client(node_details[0], node_details[1])
list_of_files = glob.glob('./data/*.gz')
for file_name in list_of_files:
with gzip.open(file_name, 'rb') as f_in:
with open(file_name[:-3], 'wb') as f_out:
shutil.copyfileobj(f_in, f_out)
# Mesonet CDFNet file format for now just imagine that the parser did its job and we have CSV format
list_of_files = glob.glob('./data/*')
for file_name in list_of_files:
print("file_name", file_name)
if file_name.endswith('.out'):
# mesowest format ending with .out
c.streamFile(file_name)
elif '_' in file_name and file_name.endswith('.csv'):
def getHandler(self, request, context):
#checking bloomfilter
d = int(
time.mktime(
time.strptime(request.getRequest.queryParams.from_utc,
'%Y-%m-%d %H:%M:%S')))
toDate = int(
time.mktime(
time.strptime(request.getRequest.queryParams.to_utc,
'%Y-%m-%d %H:%M:%S')))
process_internal = False
while d <= toDate:
date_to_check = (
datetime.datetime.fromtimestamp(d).strftime('%Y%m%d'))
c = self.node.bloomfilter
if c.testdate(date_to_check):
process_internal = True
break
d += (24 * 60 * 60)
if process_internal:
serverlist = self.node.get_active_node_ids()
return_queue = Queue(maxsize=0)
for node_id in serverlist:
assign_to_node = (Thread(target=self.connect_to_node,
args=(
node_id,
request,
return_queue,
)))
assign_to_node.setDaemon(True)
assign_to_node.start()
null_count = 0
while (True):
if (return_queue.qsize()):
d = return_queue.get()
if not d:
null_count += 1
continue
if request.fromSender == "external-client":
yield (provideJson(d))
else:
yield (d)
if null_count == len(serverlist):
break
else:
if request.fromSender == "external-client":
#external_hosts = requests.get("http://cmpe275-spring-18.mybluemix.net/get").text
external_hosts = "169.254.149.215"
external_hosts = external_hosts.split(",")
request.fromSender = ""
leader_details = config.get_node_details(self.node.leader_id)
return_queue_external = Queue(maxsize=0)
for host_details in external_hosts:
if host_details == leader_details[0]:
continue
assign_to_node = (Thread(
target=self.connect_to_external_node,
args=(
host_details,
request,
return_queue_external,
)))
assign_to_node.setDaemon(True)
assign_to_node.start()
null_count = 0
while (True):
if (return_queue_external.qsize()):
d = return_queue_external.get()
if not d:
null_count += 1
continue
yield (provideJson(d))
if null_count == len(external_hosts) - 1:
break
'SLON': dlineValues[4],
'SELV': dlineValues[5],
'TMPF': dlineValues[6],
'SKNT': dlineValues[7],
'DRCT': dlineValues[8],
'GUST': dlineValues[9],
'PMSL': dlineValues[10],
'ALTI': dlineValues[11],
'DWPF': dlineValues[12],
'RELH': dlineValues[13],
'WTHR': dlineValues[14],
'P24I': dlineValues[15]
}
jsonChunk.append(json.dumps(dlineJsonValues))
responseChunk = server_pb2.Response(
code=1,
metaData=server_pb2.MetaData(uuid="",
numOfFragment=int(
chunk.metaData.numOfFragment)),
datFragment=server_pb2.DatFragment(
timestamp_utc="",
data=str((',').join(jsonChunk)).encode(encoding='utf_8')))
return responseChunk
if __name__ == '__main__':
config.populate()
node_id = config.get_node_id()
node_details = config.get_node_details(node_id)
space = config.get_space()
run(node_details[0], node_details[1], node_id)
def getHandler(self, request, context):
print(request.getRequest.queryParams)
print(request)
#checking bloomfilter
d = int(
time.mktime(
time.strptime(request.getRequest.queryParams.from_utc,
'%Y-%m-%d %H:%M:%S')))
toDate = int(
time.mktime(
time.strptime(request.getRequest.queryParams.to_utc,
'%Y-%m-%d %H:%M:%S')))
process_internal = False
while d <= toDate:
date_to_check = (
datetime.datetime.fromtimestamp(d).strftime('%Y%m%d'))
c = self.node.bloomfilter
print("checking bloom filter ", date_to_check)
if c.testdate(date_to_check):
print("blloom filter said yes")
process_internal = True
break
d += (24 * 60 * 60)
if process_internal:
serverlist = self.node.get_active_node_ids()
return_queue = Queue(maxsize=0)
for node_id in serverlist:
print("Connecting to node", node_id)
assign_to_node = (Thread(target=self.connect_to_node,
args=(
node_id,
request,
return_queue,
)))
assign_to_node.setDaemon(True)
assign_to_node.start()
null_count = 0
while (True):
if (return_queue.qsize()):
print("queue data ", return_queue.qsize())
d = return_queue.get()
if not d:
print("incrementing null count")
null_count += 1
continue
yield (d)
if null_count == len(serverlist):
print("Breaking null check")
print("null_count", null_count)
print("server list", serverlist)
break
else:
if request.fromSender == "prof":
external_hosts = requests.get(
"http://cmpe275-spring-18.mybluemix.net/get").text
external_hosts = external_hosts.split(",")
request.fromSender = ""
leader_details = config.get_node_details(self.node.leader_id)
return_queue_external = Queue(maxsize=0)
for host_details in external_hosts:
if host_details == leader_details[0]:
continue
print("Connecting to host", host_details)
assign_to_node = (Thread(
target=self.connect_to_external_node,
args=(
host_details,
request,
return_queue_external,
)))
assign_to_node.setDaemon(True)
assign_to_node.start()
null_count = 0
while (True):
if (return_queue_external.qsize()):
print("queue data ", return_queue_external.qsize())
d = return_queue_external.get()
if not d:
print("incrementing null count")
null_count += 1
continue
yield (d)
if null_count == len(external_hosts) - 1:
print("Breaking null check")
print("null_count", null_count)
print("server list", external_hosts)
break