def run(self): for message in self._sub.recv(1): if message is None: continue if (message.type == "have"): sat = message.data["satellite"] utctime = strp_isoformat(message.data["timecode"]) # This should take care of address translation. sender = (message.sender.split("@")[1] + ":" + message.data["origin"].split(":")[1]) elevation = message.data["elevation"] self.scanlines.setdefault(sat, {}) if utctime not in self.scanlines[sat]: self.scanlines[sat][utctime] = [(sender, elevation)] # TODO: This implies that we always wait BUFFER_TIME before # sending to queue. In the case were the "have" messages of # all servers were sent in less time, we should not be # waiting... if len(self._requesters) == 1: self.send_to_queues(sat, utctime) else: timer = Timer(BUFFER_TIME, self.send_to_queues, args=[sat, utctime]) timer.start() self._timers[(sat, utctime)] = timer else: # Since append is atomic in CPython, this should work. # However, if it is not, then this is not thread safe. self.scanlines[sat][utctime].append((sender, elevation)) if (len(self.scanlines[sat][utctime]) == len( self._requesters)): self.send_to_queues(sat, utctime)
def run(self): for message in self._sub.recv(1): if message is None: continue if(message.type == "have"): sat = message.data["satellite"] utctime = strp_isoformat(message.data["timecode"]) # This should take care of address translation. sender = (message.sender.split("@")[1] + ":" + message.data["origin"].split(":")[1]) elevation = message.data["elevation"] self.scanlines.setdefault(sat, {}) if utctime not in self.scanlines[sat]: self.scanlines[sat][utctime] = [(sender, elevation)] # TODO: This implies that we always wait BUFFER_TIME before # sending to queue. In the case were the "have" messages of # all servers were sent in less time, we should not be # waiting... if len(self._requesters) == 1: self.send_to_queues(sat, utctime) else: timer = Timer(BUFFER_TIME, self.send_to_queues, args=[sat, utctime]) timer.start() self._timers[(sat, utctime)] = timer else: # Since append is atomic in CPython, this should work. # However, if it is not, then this is not thread safe. self.scanlines[sat][utctime].append((sender, elevation)) if (len(self.scanlines[sat][utctime]) == len(self._requesters)): self.send_to_queues(sat, utctime)
def run(self): for message in self._sub.recv(1): if message is None: continue if message.type == "have": sat = message.data["satellite"] utctime = strp_isoformat(message.data["timecode"]) # This should take care of address translation. senderhost = message.sender.split("@")[1] sender = senderhost + ":" + message.data["origin"].split(":")[1] elevation = message.data["elevation"] try: quality = message.data["quality"] except KeyError: quality = 100 self.scanlines.setdefault(sat, {}) if utctime not in self.scanlines[sat]: self.scanlines[sat][utctime] = [ (sender, elevation, quality, self._ping_times.get(senderhost, 3600000)) ] # TODO: This implies that we always wait BUFFER_TIME before # sending to queue. In the case were the "have" messages of # all servers were sent in less time, we should not be # waiting... if len(self._requesters) == 1: self.send_to_queues(sat, utctime) else: timer = Timer(BUFFER_TIME, self.send_to_queues, args=[sat, utctime]) timer.start() self._timers[(sat, utctime)] = timer else: # Since append is atomic in CPython, this should work. # However, if it is not, then this is not thread safe. self.scanlines[sat][utctime].append( (sender, elevation, quality, self._ping_times.get(senderhost, 3600000)) ) if len(self.scanlines[sat][utctime]) == len(self._requesters): self.send_to_queues(sat, utctime) elif message.type == "heartbeat": senderhost = message.sender.split("@")[1] sender = "tcp://" + senderhost + ":" + message.data["addr"].split(":")[1] logger.debug("receive heartbeat from " + str(sender) + ": " + str(message)) self._hb[str(sender)].reset() for addr, req in self._requesters.items(): # can we get the ip adress from the socket somehow ? # because right now the pubaddr and sender are not the same # (name vs ip) if addr == senderhost: rstation, rtime = req.ping() if rstation is None: logger.warning("Can't ping " + str(sender)) else: rtime = rtime.seconds * 1000 + rtime.microseconds / 1000.0 self._ping_times[senderhost] = rtime logger.debug("ping roundtrip to " + rstation + ": " + str(rtime) + "ms") break
def order(self, time_slice, satellite, filename): """Get all the scanlines for a *satellite* within a *time_slice* and save them in *filename*. The scanlines will be saved in a contiguous manner. """ start_time = time_slice.start end_time = time_slice.stop saved = [] # Create a file of the right length, filled with zeros. The alternative # would be to store all the scanlines in memory. tsize = (end_time - start_time).seconds * LINES_PER_SECOND * LINE_SIZE with open(filename, "wb") as fp_: fp_.write("\x00" * (tsize)) # Do the retrieval. with open(filename, "r+b") as fp_: queue = Queue() self.add_queue(queue) linepos = None lines_to_get = {} # first, get the existing scanlines from self (client) logger.info("Getting list of existing scanlines from client.") for utctime, hosts in self.scanlines.get(satellite, {}).iteritems(): if(utctime >= start_time and utctime < end_time and utctime not in saved): lines_to_get[utctime] = hosts # then, get scanlines from the server logger.info("Getting list of existing scanlines from server.") for host, req in self._requesters.iteritems(): try: response = req.get_slice(satellite, start_time, end_time) for utcstr, elevation in response: utctime = strp_isoformat(utcstr) lines_to_get.setdefault(utctime, []).append((host, elevation)) except IOError, e__: logger.warning(e__) # get lines with highest elevation and add them to current scene logger.info("Getting old scanlines.") for utctime, data, elevation in self.get_lines(satellite, lines_to_get): if linepos is None: linepos = compute_line_times(utctime, start_time, end_time) time_diff = utctime - start_time time_diff = (time_diff.seconds + time_diff.microseconds / 1000000.0) pos = LINE_SIZE * int(np.floor(time_diff * LINES_PER_SECOND)) fp_.seek(pos, 0) fp_.write(data) self.send_lineinfo_to_server(satellite, utctime, elevation, filename, pos) saved.append(utctime) linepos -= set([utctime]) # then, get the newly arrived scanlines logger.info("Getting new scanlines") #timethres = datetime.utcnow() + CLIENT_TIMEOUT delay = timedelta(days=1000) timethres = datetime.utcnow() + delay while ((start_time > datetime.utcnow() or timethres > datetime.utcnow()) and ((linepos is None) or (len(linepos) > 0))): try: sat, utctime, senders = queue.get(True, CLIENT_TIMEOUT.seconds) logger.debug("Picking line " + " ".join([str(utctime), str(senders)])) # choose the highest elevation sender, elevation = max(senders, key=(lambda x: x[1])) except Empty: continue if linepos is None: linepos = compute_line_times(utctime, start_time, end_time) if(sat == satellite and utctime >= start_time and utctime < end_time and utctime not in saved): saved.append(utctime) # getting line logger.debug("requesting " + " ".join([str(satellite), str(utctime), str(sender), str(elevation)])) host = sender.split(":")[0] # TODO: this should be parallelized, and timed. I case of # failure, another source should be used. Choking ? line = self._requesters[host].get_line(satellite, utctime) # compute line position in file time_diff = utctime - start_time time_diff = (time_diff.seconds + time_diff.microseconds / 1000000.0) pos = LINE_SIZE * int(np.floor(time_diff * LINES_PER_SECOND)) fp_.seek(pos, 0) fp_.write(line) self.send_lineinfo_to_server(satellite, utctime, elevation, filename, pos) # removing from line check list linepos -= set([utctime]) delay = min(delay, datetime.utcnow() - utctime) if len(linepos) > 0: timethres = max(linepos) + CLIENT_TIMEOUT + delay else: timethres = datetime.utcnow() # shut down self.del_queue(queue)
def run(self): for message in self._sub.recv(1): if message is None: continue if(message.type == "have"): sat = message.data["satellite"] utctime = strp_isoformat(message.data["timecode"]) # This should take care of address translation. senderhost = message.sender.split("@")[1] sender = (senderhost + ":" + message.data["origin"].split(":")[1]) elevation = message.data["elevation"] try: quality = message.data["quality"] except KeyError: quality = 100 self.scanlines.setdefault(sat, {}) if utctime not in self.scanlines[sat]: self.scanlines[sat][utctime] = [ (sender, elevation, quality, self._ping_times.get(senderhost, 3600000))] # TODO: This implies that we always wait BUFFER_TIME before # sending to queue. In the case were the "have" messages of # all servers were sent in less time, we should not be # waiting... if len(self._requesters) == 1: self.send_to_queues(sat, utctime) else: timer = Timer(BUFFER_TIME, self.send_to_queues, args=[sat, utctime]) timer.start() self._timers[(sat, utctime)] = timer else: # Since append is atomic in CPython, this should work. # However, if it is not, then this is not thread safe. self.scanlines[sat][utctime].append((sender, elevation, quality, self._ping_times.get(senderhost, 3600000))) if (len(self.scanlines[sat][utctime]) == len(self._requesters)): self.send_to_queues(sat, utctime) elif(message.type == "heartbeat"): senderhost = message.sender.split("@")[1] sender = ("tcp://" + senderhost + ":" + message.data["addr"].split(":")[1]) logger.debug("receive heartbeat from " + str(sender) + ": " + str(message)) self._hb[str(sender)].reset() for addr, req in self._requesters.items(): # can we get the ip adress from the socket somehow ? # because right now the pubaddr and sender are not the same # (name vs ip) if addr == senderhost: rstation, rtime = req.ping() if rstation is None: logger.warning("Can't ping " + str(sender)) else: rtime = (rtime.seconds * 1000 + rtime.microseconds / 1000.0) self._ping_times[senderhost] = rtime logger.debug("ping roundtrip to " + rstation + ": " + str(rtime) + "ms") break