def run(self):
for message in self._sub.recv(1):
if message is None:
continue
if (message.type == "have"):
sat = message.data["satellite"]
utctime = strp_isoformat(message.data["timecode"])
# This should take care of address translation.
sender = (message.sender.split("@")[1] + ":" +
message.data["origin"].split(":")[1])
elevation = message.data["elevation"]
self.scanlines.setdefault(sat, {})
if utctime not in self.scanlines[sat]:
self.scanlines[sat][utctime] = [(sender, elevation)]
# TODO: This implies that we always wait BUFFER_TIME before
# sending to queue. In the case were the "have" messages of
# all servers were sent in less time, we should not be
# waiting...
if len(self._requesters) == 1:
self.send_to_queues(sat, utctime)
else:
timer = Timer(BUFFER_TIME,
self.send_to_queues,
args=[sat, utctime])
timer.start()
self._timers[(sat, utctime)] = timer
else:
# Since append is atomic in CPython, this should work.
# However, if it is not, then this is not thread safe.
self.scanlines[sat][utctime].append((sender, elevation))
if (len(self.scanlines[sat][utctime]) == len(
self._requesters)):
self.send_to_queues(sat, utctime)
def run(self):
for message in self._sub.recv(1):
if message is None:
continue
if(message.type == "have"):
sat = message.data["satellite"]
utctime = strp_isoformat(message.data["timecode"])
# This should take care of address translation.
sender = (message.sender.split("@")[1] + ":" +
message.data["origin"].split(":")[1])
elevation = message.data["elevation"]
self.scanlines.setdefault(sat, {})
if utctime not in self.scanlines[sat]:
self.scanlines[sat][utctime] = [(sender, elevation)]
# TODO: This implies that we always wait BUFFER_TIME before
# sending to queue. In the case were the "have" messages of
# all servers were sent in less time, we should not be
# waiting...
if len(self._requesters) == 1:
self.send_to_queues(sat, utctime)
else:
timer = Timer(BUFFER_TIME,
self.send_to_queues,
args=[sat, utctime])
timer.start()
self._timers[(sat, utctime)] = timer
else:
# Since append is atomic in CPython, this should work.
# However, if it is not, then this is not thread safe.
self.scanlines[sat][utctime].append((sender, elevation))
if (len(self.scanlines[sat][utctime]) ==
len(self._requesters)):
self.send_to_queues(sat, utctime)
def run(self):
for message in self._sub.recv(1):
if message is None:
continue
if message.type == "have":
sat = message.data["satellite"]
utctime = strp_isoformat(message.data["timecode"])
# This should take care of address translation.
senderhost = message.sender.split("@")[1]
sender = senderhost + ":" + message.data["origin"].split(":")[1]
elevation = message.data["elevation"]
try:
quality = message.data["quality"]
except KeyError:
quality = 100
self.scanlines.setdefault(sat, {})
if utctime not in self.scanlines[sat]:
self.scanlines[sat][utctime] = [
(sender, elevation, quality, self._ping_times.get(senderhost, 3600000))
]
# TODO: This implies that we always wait BUFFER_TIME before
# sending to queue. In the case were the "have" messages of
# all servers were sent in less time, we should not be
# waiting...
if len(self._requesters) == 1:
self.send_to_queues(sat, utctime)
else:
timer = Timer(BUFFER_TIME, self.send_to_queues, args=[sat, utctime])
timer.start()
self._timers[(sat, utctime)] = timer
else:
# Since append is atomic in CPython, this should work.
# However, if it is not, then this is not thread safe.
self.scanlines[sat][utctime].append(
(sender, elevation, quality, self._ping_times.get(senderhost, 3600000))
)
if len(self.scanlines[sat][utctime]) == len(self._requesters):
self.send_to_queues(sat, utctime)
elif message.type == "heartbeat":
senderhost = message.sender.split("@")[1]
sender = "tcp://" + senderhost + ":" + message.data["addr"].split(":")[1]
logger.debug("receive heartbeat from " + str(sender) + ": " + str(message))
self._hb[str(sender)].reset()
for addr, req in self._requesters.items():
# can we get the ip adress from the socket somehow ?
# because right now the pubaddr and sender are not the same
# (name vs ip)
if addr == senderhost:
rstation, rtime = req.ping()
if rstation is None:
logger.warning("Can't ping " + str(sender))
else:
rtime = rtime.seconds * 1000 + rtime.microseconds / 1000.0
self._ping_times[senderhost] = rtime
logger.debug("ping roundtrip to " + rstation + ": " + str(rtime) + "ms")
break
def order(self, time_slice, satellite, filename):
"""Get all the scanlines for a *satellite* within a *time_slice* and
save them in *filename*. The scanlines will be saved in a contiguous
manner.
"""
start_time = time_slice.start
end_time = time_slice.stop
saved = []
# Create a file of the right length, filled with zeros. The alternative
# would be to store all the scanlines in memory.
tsize = (end_time - start_time).seconds * LINES_PER_SECOND * LINE_SIZE
with open(filename, "wb") as fp_:
fp_.write("\x00" * (tsize))
# Do the retrieval.
with open(filename, "r+b") as fp_:
queue = Queue()
self.add_queue(queue)
linepos = None
lines_to_get = {}
# first, get the existing scanlines from self (client)
logger.info("Getting list of existing scanlines from client.")
for utctime, hosts in self.scanlines.get(satellite, {}).iteritems():
if(utctime >= start_time and
utctime < end_time and
utctime not in saved):
lines_to_get[utctime] = hosts
# then, get scanlines from the server
logger.info("Getting list of existing scanlines from server.")
for host, req in self._requesters.iteritems():
try:
response = req.get_slice(satellite, start_time, end_time)
for utcstr, elevation in response:
utctime = strp_isoformat(utcstr)
lines_to_get.setdefault(utctime, []).append((host,
elevation))
except IOError, e__:
logger.warning(e__)
# get lines with highest elevation and add them to current scene
logger.info("Getting old scanlines.")
for utctime, data, elevation in self.get_lines(satellite,
lines_to_get):
if linepos is None:
linepos = compute_line_times(utctime, start_time, end_time)
time_diff = utctime - start_time
time_diff = (time_diff.seconds
+ time_diff.microseconds / 1000000.0)
pos = LINE_SIZE * int(np.floor(time_diff * LINES_PER_SECOND))
fp_.seek(pos, 0)
fp_.write(data)
self.send_lineinfo_to_server(satellite, utctime, elevation,
filename, pos)
saved.append(utctime)
linepos -= set([utctime])
# then, get the newly arrived scanlines
logger.info("Getting new scanlines")
#timethres = datetime.utcnow() + CLIENT_TIMEOUT
delay = timedelta(days=1000)
timethres = datetime.utcnow() + delay
while ((start_time > datetime.utcnow()
or timethres > datetime.utcnow())
and ((linepos is None) or (len(linepos) > 0))):
try:
sat, utctime, senders = queue.get(True,
CLIENT_TIMEOUT.seconds)
logger.debug("Picking line " + " ".join([str(utctime),
str(senders)]))
# choose the highest elevation
sender, elevation = max(senders, key=(lambda x: x[1]))
except Empty:
continue
if linepos is None:
linepos = compute_line_times(utctime, start_time, end_time)
if(sat == satellite and
utctime >= start_time and
utctime < end_time and
utctime not in saved):
saved.append(utctime)
# getting line
logger.debug("requesting " +
" ".join([str(satellite), str(utctime),
str(sender), str(elevation)]))
host = sender.split(":")[0]
# TODO: this should be parallelized, and timed. I case of
# failure, another source should be used. Choking ?
line = self._requesters[host].get_line(satellite, utctime)
# compute line position in file
time_diff = utctime - start_time
time_diff = (time_diff.seconds
+ time_diff.microseconds / 1000000.0)
pos = LINE_SIZE * int(np.floor(time_diff *
LINES_PER_SECOND))
fp_.seek(pos, 0)
fp_.write(line)
self.send_lineinfo_to_server(satellite, utctime, elevation,
filename, pos)
# removing from line check list
linepos -= set([utctime])
delay = min(delay, datetime.utcnow() - utctime)
if len(linepos) > 0:
timethres = max(linepos) + CLIENT_TIMEOUT + delay
else:
timethres = datetime.utcnow()
# shut down
self.del_queue(queue)
def run(self):
for message in self._sub.recv(1):
if message is None:
continue
if(message.type == "have"):
sat = message.data["satellite"]
utctime = strp_isoformat(message.data["timecode"])
# This should take care of address translation.
senderhost = message.sender.split("@")[1]
sender = (senderhost + ":" +
message.data["origin"].split(":")[1])
elevation = message.data["elevation"]
try:
quality = message.data["quality"]
except KeyError:
quality = 100
self.scanlines.setdefault(sat, {})
if utctime not in self.scanlines[sat]:
self.scanlines[sat][utctime] = [
(sender, elevation, quality, self._ping_times.get(senderhost, 3600000))]
# TODO: This implies that we always wait BUFFER_TIME before
# sending to queue. In the case were the "have" messages of
# all servers were sent in less time, we should not be
# waiting...
if len(self._requesters) == 1:
self.send_to_queues(sat, utctime)
else:
timer = Timer(BUFFER_TIME,
self.send_to_queues,
args=[sat, utctime])
timer.start()
self._timers[(sat, utctime)] = timer
else:
# Since append is atomic in CPython, this should work.
# However, if it is not, then this is not thread safe.
self.scanlines[sat][utctime].append((sender, elevation,
quality, self._ping_times.get(senderhost, 3600000)))
if (len(self.scanlines[sat][utctime]) ==
len(self._requesters)):
self.send_to_queues(sat, utctime)
elif(message.type == "heartbeat"):
senderhost = message.sender.split("@")[1]
sender = ("tcp://" + senderhost +
":" + message.data["addr"].split(":")[1])
logger.debug("receive heartbeat from " + str(sender) +
": " + str(message))
self._hb[str(sender)].reset()
for addr, req in self._requesters.items():
# can we get the ip adress from the socket somehow ?
# because right now the pubaddr and sender are not the same
# (name vs ip)
if addr == senderhost:
rstation, rtime = req.ping()
if rstation is None:
logger.warning("Can't ping " + str(sender))
else:
rtime = (rtime.seconds * 1000 +
rtime.microseconds / 1000.0)
self._ping_times[senderhost] = rtime
logger.debug("ping roundtrip to " + rstation +
": " + str(rtime) +
"ms")
break
def order(self, time_slice, satellite, filename):
"""Get all the scanlines for a *satellite* within a *time_slice* and
save them in *filename*. The scanlines will be saved in a contiguous
manner.
"""
start_time = time_slice.start
end_time = time_slice.stop
saved = []
# Create a file of the right length, filled with zeros. The alternative
# would be to store all the scanlines in memory.
tsize = (end_time - start_time).seconds * LINES_PER_SECOND * LINE_SIZE
with open(filename, "wb") as fp_:
fp_.write("\x00" * (tsize))
# Do the retrieval.
with open(filename, "r+b") as fp_:
queue = Queue()
self.add_queue(queue)
linepos = None
lines_to_get = {}
# first, get the existing scanlines from self (client)
logger.info("Getting list of existing scanlines from client.")
for utctime, hosts in self.scanlines.get(satellite, {}).iteritems():
if(utctime >= start_time and
utctime < end_time and
utctime not in saved):
lines_to_get[utctime] = hosts
# then, get scanlines from the server
logger.info("Getting list of existing scanlines from server.")
for host, req in self._requesters.iteritems():
try:
response = req.get_slice(satellite, start_time, end_time)
for utcstr, elevation in response:
utctime = strp_isoformat(utcstr)
lines_to_get.setdefault(utctime, []).append((host,
elevation))
except IOError, e__:
logger.warning(e__)
# get lines with highest elevation and add them to current scene
logger.info("Getting old scanlines.")
for utctime, data, elevation in self.get_lines(satellite,
lines_to_get):
if linepos is None:
linepos = compute_line_times(utctime, start_time, end_time)
time_diff = utctime - start_time
time_diff = (time_diff.seconds
+ time_diff.microseconds / 1000000.0)
pos = LINE_SIZE * int(np.floor(time_diff * LINES_PER_SECOND))
fp_.seek(pos, 0)
fp_.write(data)
self.send_lineinfo_to_server(satellite, utctime, elevation,
filename, pos)
saved.append(utctime)
linepos -= set([utctime])
# then, get the newly arrived scanlines
logger.info("Getting new scanlines")
#timethres = datetime.utcnow() + CLIENT_TIMEOUT
delay = timedelta(days=1000)
timethres = datetime.utcnow() + delay
while ((start_time > datetime.utcnow()
or timethres > datetime.utcnow())
and ((linepos is None) or (len(linepos) > 0))):
try:
sat, utctime, senders = queue.get(True,
CLIENT_TIMEOUT.seconds)
logger.debug("Picking line " + " ".join([str(utctime),
str(senders)]))
# choose the highest elevation
sender, elevation = max(senders, key=(lambda x: x[1]))
except Empty:
continue
if linepos is None:
linepos = compute_line_times(utctime, start_time, end_time)
if(sat == satellite and
utctime >= start_time and
utctime < end_time and
utctime not in saved):
saved.append(utctime)
# getting line
logger.debug("requesting " +
" ".join([str(satellite), str(utctime),
str(sender), str(elevation)]))
host = sender.split(":")[0]
# TODO: this should be parallelized, and timed. I case of
# failure, another source should be used. Choking ?
line = self._requesters[host].get_line(satellite, utctime)
# compute line position in file
time_diff = utctime - start_time
time_diff = (time_diff.seconds
+ time_diff.microseconds / 1000000.0)
pos = LINE_SIZE * int(np.floor(time_diff *
LINES_PER_SECOND))
fp_.seek(pos, 0)
fp_.write(line)
self.send_lineinfo_to_server(satellite, utctime, elevation,
filename, pos)
# removing from line check list
linepos -= set([utctime])
delay = min(delay, datetime.utcnow() - utctime)
if len(linepos) > 0:
timethres = max(linepos) + CLIENT_TIMEOUT + delay
else:
timethres = datetime.utcnow()
# shut down
self.del_queue(queue)
