def run(): global host config.populate() node_id = config.get_node_id() node_details = config.get_node_details(node_id) logger.info("Connecting to host {} on port {}".format( node_details[0], node_details[1])) c = Client(node_details[0], node_details[1]) c.updateBloomFilter() leader_node = 0 heart_beat_count = 0 requests.get("http://cmpe275-spring-18.mybluemix.net/delete/" + (config.get_node_details(node_id))[0]) while (True): time.sleep(heartbeat_interval) heart_beat_count += 1 leader = c.getLeaderNode(node_id) if leader_node != leader.id and leader.id == node_id: if leader_node != 0: requests.get("http://cmpe275-spring-18.mybluemix.net/delete/" + config.get_node_details(leader_node)[0]) requests.get("http://cmpe275-spring-18.mybluemix.net/put/" + config.get_node_details(leader.id)[0]) leader_node = leader.id logger.info( "Publish node_id {} to external cluster".format(node_id)) if heart_beat_count * heartbeat_interval == bloom_filter_interval: c.updateBloomFilter()
def pushDataToExternalCluster(self, req): #external_hosts = requests.get("http://cmpe275-spring-18.mybluemix.net/get").text external_hosts = "169.254.230.239" external_hosts = external_hosts.split(",") req.fromSender = "" leader_details = config.get_node_details(self.node.leader_id) for host_details in external_hosts: if host_details == leader_details[0]: continue try: client = Client(host=host_details, port=8080) res = client.putHandler( (req.putRequest.datFragment.data).decode('utf-8')) if res.code == 1: break except Exception as e: print(e)
import glob import gzip import shutil import re from chunktest import process from config import get_client_map, get_node_details, populate from client import Client import server_pb2 import server_pb2_grpc from server_pb2 import Request, PutRequest, QueryParams, DatFragment populate() node_details = get_node_details(1) print(node_details) c = Client(node_details[0], node_details[1]) list_of_files = glob.glob('./data/*.gz') for file_name in list_of_files: with gzip.open(file_name, 'rb') as f_in: with open(file_name[:-3], 'wb') as f_out: shutil.copyfileobj(f_in, f_out) # Mesonet CDFNet file format for now just imagine that the parser did its job and we have CSV format list_of_files = glob.glob('./data/*') for file_name in list_of_files: print("file_name", file_name) if file_name.endswith('.out'): # mesowest format ending with .out c.streamFile(file_name) elif '_' in file_name and file_name.endswith('.csv'):
def getHandler(self, request, context): #checking bloomfilter d = int( time.mktime( time.strptime(request.getRequest.queryParams.from_utc, '%Y-%m-%d %H:%M:%S'))) toDate = int( time.mktime( time.strptime(request.getRequest.queryParams.to_utc, '%Y-%m-%d %H:%M:%S'))) process_internal = False while d <= toDate: date_to_check = ( datetime.datetime.fromtimestamp(d).strftime('%Y%m%d')) c = self.node.bloomfilter if c.testdate(date_to_check): process_internal = True break d += (24 * 60 * 60) if process_internal: serverlist = self.node.get_active_node_ids() return_queue = Queue(maxsize=0) for node_id in serverlist: assign_to_node = (Thread(target=self.connect_to_node, args=( node_id, request, return_queue, ))) assign_to_node.setDaemon(True) assign_to_node.start() null_count = 0 while (True): if (return_queue.qsize()): d = return_queue.get() if not d: null_count += 1 continue if request.fromSender == "external-client": yield (provideJson(d)) else: yield (d) if null_count == len(serverlist): break else: if request.fromSender == "external-client": #external_hosts = requests.get("http://cmpe275-spring-18.mybluemix.net/get").text external_hosts = "169.254.149.215" external_hosts = external_hosts.split(",") request.fromSender = "" leader_details = config.get_node_details(self.node.leader_id) return_queue_external = Queue(maxsize=0) for host_details in external_hosts: if host_details == leader_details[0]: continue assign_to_node = (Thread( target=self.connect_to_external_node, args=( host_details, request, return_queue_external, ))) assign_to_node.setDaemon(True) assign_to_node.start() null_count = 0 while (True): if (return_queue_external.qsize()): d = return_queue_external.get() if not d: null_count += 1 continue yield (provideJson(d)) if null_count == len(external_hosts) - 1: break
'SLON': dlineValues[4], 'SELV': dlineValues[5], 'TMPF': dlineValues[6], 'SKNT': dlineValues[7], 'DRCT': dlineValues[8], 'GUST': dlineValues[9], 'PMSL': dlineValues[10], 'ALTI': dlineValues[11], 'DWPF': dlineValues[12], 'RELH': dlineValues[13], 'WTHR': dlineValues[14], 'P24I': dlineValues[15] } jsonChunk.append(json.dumps(dlineJsonValues)) responseChunk = server_pb2.Response( code=1, metaData=server_pb2.MetaData(uuid="", numOfFragment=int( chunk.metaData.numOfFragment)), datFragment=server_pb2.DatFragment( timestamp_utc="", data=str((',').join(jsonChunk)).encode(encoding='utf_8'))) return responseChunk if __name__ == '__main__': config.populate() node_id = config.get_node_id() node_details = config.get_node_details(node_id) space = config.get_space() run(node_details[0], node_details[1], node_id)
def getHandler(self, request, context): print(request.getRequest.queryParams) print(request) #checking bloomfilter d = int( time.mktime( time.strptime(request.getRequest.queryParams.from_utc, '%Y-%m-%d %H:%M:%S'))) toDate = int( time.mktime( time.strptime(request.getRequest.queryParams.to_utc, '%Y-%m-%d %H:%M:%S'))) process_internal = False while d <= toDate: date_to_check = ( datetime.datetime.fromtimestamp(d).strftime('%Y%m%d')) c = self.node.bloomfilter print("checking bloom filter ", date_to_check) if c.testdate(date_to_check): print("blloom filter said yes") process_internal = True break d += (24 * 60 * 60) if process_internal: serverlist = self.node.get_active_node_ids() return_queue = Queue(maxsize=0) for node_id in serverlist: print("Connecting to node", node_id) assign_to_node = (Thread(target=self.connect_to_node, args=( node_id, request, return_queue, ))) assign_to_node.setDaemon(True) assign_to_node.start() null_count = 0 while (True): if (return_queue.qsize()): print("queue data ", return_queue.qsize()) d = return_queue.get() if not d: print("incrementing null count") null_count += 1 continue yield (d) if null_count == len(serverlist): print("Breaking null check") print("null_count", null_count) print("server list", serverlist) break else: if request.fromSender == "prof": external_hosts = requests.get( "http://cmpe275-spring-18.mybluemix.net/get").text external_hosts = external_hosts.split(",") request.fromSender = "" leader_details = config.get_node_details(self.node.leader_id) return_queue_external = Queue(maxsize=0) for host_details in external_hosts: if host_details == leader_details[0]: continue print("Connecting to host", host_details) assign_to_node = (Thread( target=self.connect_to_external_node, args=( host_details, request, return_queue_external, ))) assign_to_node.setDaemon(True) assign_to_node.start() null_count = 0 while (True): if (return_queue_external.qsize()): print("queue data ", return_queue_external.qsize()) d = return_queue_external.get() if not d: print("incrementing null count") null_count += 1 continue yield (d) if null_count == len(external_hosts) - 1: print("Breaking null check") print("null_count", null_count) print("server list", external_hosts) break