def __init__(self, args, manifest, monitored_devices, check_period, notifier):
super(Surveyor, self).__init__()
self._args = args
self._manifest = manifest
self._monitored_devices = monitored_devices
self._notifer = notifier
self._roll_call = SharedVarCollection({})
self._devices_poll = Periodic(check_period, self.poll_devices, "poll_devices")
return
async def create_program_and_run_it():
init_logging()
prog = MainRuntime(logger)
prog.init_services()
start_http_server(8000)
host = prog.cfg.get_fw_api_server_hostname()
poll_interval = prog.cfg.get_polling_delay_seconds()
logger.info(
f"Extra Metrics - connecting to {host}, using poll interval of {poll_interval} sec"
)
# fetches and stores the current server version, this is important as the REST API's depend on this
# information. Prior to 14.2.0 some of the API calls were different and 14.2.0 put them all under ../api/v1/...
prog.fw_query.fetch_server_version()
log_config_summary(prog.cfg, prog.fw_query.major_version,
prog.fw_query.minor_version,
prog.fw_query.patch_version)
if prog.fw_query.major_version is None or prog.fw_query.major_version == 0:
logger.error("Unable to reach FileWave server, aborting...")
return
p = Periodic(poll_interval, prog.validate_and_collect_data)
await p.start()
while (True):
if prog.rerun_data_collection:
await prog.validate_and_collect_data()
await asyncio.sleep(1)
def __init__(self, args, config, notifier):
super(Tracker, self).__init__()
self._args = args
self._config = config
self._monitored_devices = config.devices_details()["monitored_devices"]
self._notifer = notifier
positive_poll_period = config.general_details()["positive_poll_period"]
negative_poll_period = config.general_details()["negative_poll_period"]
self._poll = Periodic(negative_poll_period,
self.poll_devices,
"poll_devices",
notifier=notifier)
redis_info = config.redis_details()
self._rdb = redis.StrictRedis(host=redis_info["host"],
port=redis_info["port"],
db=redis_info["db_no"])
self._roll_call = {}
return
async def _schedule_obsolete_game_check() -> None:
"""Schedule the Obsolete Check task to run periodically, with a delay before starting."""
period = config().obsolete_game_check_period_sec
delay = config().obsolete_game_check_delay_sec
p = Periodic(period, _execute_obsolete_game_check)
await p.start(delay=delay)
log.debug(
"Completed scheduling obsolete game check with period %d and delay %d",
period, delay)
async def _schedule_idle_player_check() -> None:
"""Schedule the Idle Player Check task to run periodically, with a delay before starting."""
period = config().idle_player_check_period_sec
delay = config().idle_player_check_delay_sec
p = Periodic(period, _execute_idle_player_check)
await p.start(delay=delay)
log.debug(
"Completed scheduling idle player check with period %d and delay %d",
period, delay)
class Surveyor(threading.Thread):
def __init__(self, args, manifest, monitored_devices, check_period, notifier):
super(Surveyor, self).__init__()
self._args = args
self._manifest = manifest
self._monitored_devices = monitored_devices
self._notifer = notifier
self._roll_call = SharedVarCollection({})
self._devices_poll = Periodic(check_period, self.poll_devices, "poll_devices")
return
def poll_devices(self):
for name in self._monitored_devices:
self._notifer.diagnostic("pinging %s" % name)
found = self.ping(self._manifest.address(name))
if found:
self._roll_call.set(name, True)
self._notifer.note("%s found" % name)
else:
self._roll_call.set(name, False)
self._notifer.note("%s missing" % name)
return
def ping(self, ip_address):
# NOTE: ping requires root access. Fake it during development with a random#
if hasattr(args, 'test') and args.test:
found = (random.randint(0, 3) == 3)
else:
response = pyping.ping(ip_address)
found = (response.ret_code == 0)
return found
def roll_call(self):
return self._roll_call.get()
def check(self):
self._devices_poll.check()
return
def run(self):
while gRunningFlag:
self._devices_poll.check()
return
async def main():
cm = mpl.cm.get_cmap('jet')
context = ModelContext('./data/emission.pkl', 30, 0.6, 500)
p = Periodic(lambda: payload(context, cm), 1)
try:
print('Start')
await p.start()
await asyncio.sleep(100.1)
finally:
await p.stop() # we should stop task finally
class Job(object):
def __init__(self, proxy, num, ping_interval, info):
self.proxy = proxy
self.num = num
self.ping_thread = Periodic(self.ping_refresh, ping_interval, delay=ping_interval)
self.info = info
def ping_refresh(self):
print "pinging for job %d" % self.num
self.proxy.refresh_job(self.num)
def start(self, work, msg):
print "job %d starting" % self.num
self.ping_thread.start()
try:
work(self.num, msg)
print "job %d done" % self.num
self.finish(success=True)
except Exception, e:
print "job %d failed" % self.num
self.finish(success=False, info={'error': repr(e)})
finally:
def __init__(self, n, chunksize, ping_timeout, cleanup_interval):
self.n = n
self.chunksize = chunksize
self.ping_timeout = ping_timeout
self.cleanup_interval = cleanup_interval
self.packnum = self.n / self.chunksize + (1 if self.n % 1 > 0 else 0)
self.undone = set([ x for x in xrange(self.packnum) ])
self.working = set()
self.done = set()
self.pingtimes = {}
self.cleanup_thread = Periodic(self.cleanup, self.cleanup_interval)
def __init__(self, listenPort):
#Make a UDP socket
self.sock = socket.socket( socket.AF_INET, socket.SOCK_DGRAM )
self.sock.bind( ("0.0.0.0", listenPort) )
self.sock.settimeout(0.01)
self.packetSize = 0
self.t = 0
self.buf = ""
self.packetTimestamps = []
self.packetsPerSecond = 0
self.simulatedLatency = 0
self.simulatedRandomLatencyVal = 0
self.simulatedPacketloss = 0
self.simulatedRandomLatency = 0
self.simulatedPackets = []
self.packet_outbound_last_id = defaultdict(lambda:0)
self.packet_inbound_last_id = defaultdict(lambda:0)
self.packetloss = defaultdict(lambda:0)
self.ensured_send_packet_ids = defaultdict(lambda:0)
self.ensured_sent_packets = defaultdict(dict)
self.ensured_recv_packet_ids = defaultdict(lambda:-1)
self.ensured_packets_received_early = defaultdict(list)
self.resend_unconfirmed_timer = 0
self.averagedData = AveragedData()
self.netinfotimer = 1.0
self.debug_lines = []
self.periodic = Periodic()
self.periodic.add(self.resendUnconfirmed, 0.5)
def refresh_period_or_duty_cycle(self):
self.periodic = Periodic(
period_sec=self.state.get_period(),
semi_period_percent=self.state.get_duty_cycle()
)
class NetCommon:
netEntities = { "player": Player, "projectile":Projectile }
def __init__(self, listenPort):
#Make a UDP socket
self.sock = socket.socket( socket.AF_INET, socket.SOCK_DGRAM )
self.sock.bind( ("0.0.0.0", listenPort) )
self.sock.settimeout(0.01)
self.packetSize = 0
self.t = 0
self.buf = ""
self.packetTimestamps = []
self.packetsPerSecond = 0
self.simulatedLatency = 0
self.simulatedRandomLatencyVal = 0
self.simulatedPacketloss = 0
self.simulatedRandomLatency = 0
self.simulatedPackets = []
self.packet_outbound_last_id = defaultdict(lambda:0)
self.packet_inbound_last_id = defaultdict(lambda:0)
self.packetloss = defaultdict(lambda:0)
self.ensured_send_packet_ids = defaultdict(lambda:0)
self.ensured_sent_packets = defaultdict(dict)
self.ensured_recv_packet_ids = defaultdict(lambda:-1)
self.ensured_packets_received_early = defaultdict(list)
self.resend_unconfirmed_timer = 0
self.averagedData = AveragedData()
self.netinfotimer = 1.0
self.debug_lines = []
self.periodic = Periodic()
self.periodic.add(self.resendUnconfirmed, 0.5)
def readPacket(self, info, data):
self.averagedData.add(self.t, "packets")
self.averagedData.add(self.t, "packetsize", len(data))
unpacked = pickle.loads(zlib.decompress(data))
addr, port = info
addrportstr = addr + ":" + str(port)
if "ensured_id" in unpacked:
if unpacked["ensured_id"] == self.ensured_recv_packet_ids[addrportstr]+1:
print "recv " + str(unpacked["ensured_id"])
self.ensured_recv_packet_ids[addrportstr] += 1
self.sendReceipt(addr, port, unpacked["ensured_id"])
elif unpacked["ensured_id"] < self.ensured_recv_packet_ids[addrportstr]+1:
print unpacked
print "got ensured packet twice; resending receipt for " + str(unpacked["ensured_id"])
self.sendReceipt(addr, port, unpacked["ensured_id"])
return []
else:
print "got packet " + str(unpacked["ensured_id"]) + " before " + str(self.ensured_recv_packet_ids[addrportstr]+1)
self.ensured_packets_received_early[addrportstr].append(unpacked)
return []
allPackets = []
to_remove = []
self.ensured_packets_received_early[addrportstr].sort(lambda a,b:cmp(a["ensured_id"], b["ensured_id"]))
for p in self.ensured_packets_received_early[addrportstr]:
print "resolving old " + str(p["ensured_id"])
if p["ensured_id"] <= self.ensured_recv_packet_ids[addrportstr]+1:
self.ensured_recv_packet_ids[addrportstr] += 1
self.sendReceipt(addr, port, p["ensured_id"])
allPackets.extend(self.readUnpackedPacket(p, addrportstr))
to_remove.append(p)
for p in to_remove:
self.ensured_packets_received_early[addrportstr].remove(p)
allPackets.extend(self.readUnpackedPacket(unpacked, addrportstr))
return allPackets
def sendReceipt(self, addr, port, q):
self.sendPacket({"type":"confirmReceipt","other_ensured_id":q}, addr, port)
def readUnpackedPacket(self, unpacked, addrportstr):
pid = unpacked["packet_id"]
lid = self.packet_inbound_last_id[addrportstr]
if pid > lid + 1:
self.packetloss[addrportstr] += 1
self.packet_inbound_last_id[addrportstr] = pid
if self.packet_inbound_last_id[addrportstr] > 0:
packetloss = self.packetloss[addrportstr] / float(self.packet_inbound_last_id[addrportstr])
self.averagedData.add(self.t, "packetloss_" + addrportstr, packetloss)
return [unpacked]
def sendPacket(self, data, addr, port):
print "packet: " + data["type"]
addrportstr = addr + ":" + str(port)
data["packet_id"] = self.packet_outbound_last_id[addrportstr]
self.packet_outbound_last_id[addrportstr] += 1
self.sock.sendto(zlib.compress(pickle.dumps(data, 2)), (addr, port))
def sendEnsuredPacket(self, data, addr, port):
addrportstr = addr + ":" + str(port)
ensured_id = self.ensured_send_packet_ids[addrportstr]
print "packet: " + data["type"] + " (ensured id: " + str(ensured_id) + ")"
data["packet_id"] = self.packet_outbound_last_id[addrportstr]
self.packet_outbound_last_id[addrportstr] += 1
data["ensured_id"] = ensured_id
cdata = zlib.compress(pickle.dumps(data, 2))
sent = {
"id":ensured_id,
"data":cdata,
"time":self.t,
"info":(addr,port)
}
self.ensured_sent_packets[addrportstr][ensured_id] = sent
self.sock.sendto(cdata, (addr, port))
self.ensured_send_packet_ids[addrportstr] = ensured_id + 1
def process_confirmReceipt(self, data, game, info):
(addr, port) = info
addrportstr = addr + ":" + str(port)
pending_packets = self.ensured_sent_packets[addrportstr]
pid = data["other_ensured_id"]
print "got receipt for " + str(pid)
if pid in pending_packets:
del pending_packets[pid]
else:
if pid > self.ensured_send_packet_ids:
print "got receipt for packet i haven't sent yet!!"
def update(self, game, dt):
self.game = game
self.t = pygame.time.get_ticks() / 1000.0
self.periodic.update()
self.packetsPerSecond = self.averagedData.get_ct(self.t, "packets", 1.0)
self.packetSize = self.averagedData.get_sum(self.t, "packetsize", 1.0)
allPackets = []
try:
(data, info) = self.sock.recvfrom(4096)
#self.packetSize = len(data)
if self.simulatedPacketloss > 0 and random.random() < self.simulatedPacketloss:
pass
else:
allPackets = self.readPacket(info, data)
except socket.timeout:
pass
except socket.error as err:
#print err
pass
#print self.simulatedPackets
if self.simulatedLatency == 0:
for d in allPackets:
self.process(d, game, info)
else:
off = self.simulatedLatency + self.simulatedRandomLatency * random.random()
self.simulatedPackets.extend( [(d, off, info) for d in allPackets] )
thisFramePackets = [ s for s in self.simulatedPackets if s[1] <= 0]
self.simulatedPackets = [ s for s in self.simulatedPackets if s[1] > 0 ]
for (p, t, info) in thisFramePackets:
self.process(p, game, info)
self.simulatedPackets = [ (s[0], s[1] - dt, s[2]) for s in self.simulatedPackets ]
def resendUnconfirmed(self):
for k,packets in self.ensured_sent_packets.items():
for i,packet in packets.items():
if self.t > packet["time"] + 1.5:
print "resending unreceipted packet: " + str(packet["id"])
self.sock.sendto(packet["data"], packet["info"])
def process(self, data, game, info):
if(hasattr(self, "process_" + data["type"])):
f = getattr(self, "process_" + data["type"])
f(data, game, info)
else:
print("Got packet of type '" + data["type"] + "' but there is no process_" + data["type"] + " method to handle it." )
def __init__(self, proxy, num, ping_interval, info): self.proxy = proxy self.num = num self.ping_thread = Periodic(self.ping_refresh, ping_interval, delay=ping_interval) self.info = info
class Balancer(object):
def __init__(self, n, chunksize, ping_timeout, cleanup_interval):
self.n = n
self.chunksize = chunksize
self.ping_timeout = ping_timeout
self.cleanup_interval = cleanup_interval
self.packnum = self.n / self.chunksize + (1 if self.n % 1 > 0 else 0)
self.undone = set([ x for x in xrange(self.packnum) ])
self.working = set()
self.done = set()
self.pingtimes = {}
self.cleanup_thread = Periodic(self.cleanup, self.cleanup_interval)
def get_status(self):
return "status: %d/%d/%d (undone/working/done)" % (len(self.undone), len(self.working), len(self.done))
def get_start_job(self):
if self.undone:
job = self.undone.pop()
self.working.add(job)
self.pingtimes[job] = time()
print "job %d started" % job
print self.get_status()
msg = {'work': True, 'n': self.n, 'job': job, 'chunksize': self.chunksize}
return msg
else:
return {'work': False, 'shutdown': len(self.working)==0}
def refresh_job(self, job, info={}):
if job in self.working:
print "received refresh ping for job %d" % job
self.pingtimes[job] = time()
return True
else:
print "ignoring unexpected refresh ping for job %d" % job
return False
def finish_job(self, job, success, info={}):
try:
self.working.remove(job)
if success:
self.done.add(job)
print "job %d completed successfully" % job
else:
self.undone.add(job)
print "job %d failed with error: %s" % (job, info.get('error', "unspecified"))
print self.get_status()
del self.pingtimes[job]
return True
except KeyError:
print "got unexpected finishing notification for job %d" % job
return False
def clean_dead_jobs(self):
now = time()
dead = set()
for job in self.pingtimes:
if now - self.pingtimes[job] > self.ping_timeout:
dead.add(job)
num = len(self.working)
self.undone.union(dead)
self.working -= dead
return num - len(self.working)
# TODO: delete done jobs from pingtimes to save some memory
def cleanup(self):
numcleaned = self.clean_dead_jobs()
if numcleaned:
print "cleaned %d dead jobs" % numcleaned
print self.get_status()
def start(self):
self.cleanup_thread.start()
def stop(self):
self.cleanup_thread.stop()
class Tracker(threading.Thread):
def __init__(self, args, config, notifier):
super(Tracker, self).__init__()
self._args = args
self._config = config
self._monitored_devices = config.devices_details()["monitored_devices"]
self._notifer = notifier
positive_poll_period = config.general_details()["positive_poll_period"]
negative_poll_period = config.general_details()["negative_poll_period"]
self._poll = Periodic(negative_poll_period,
self.poll_devices,
"poll_devices",
notifier=notifier)
redis_info = config.redis_details()
self._rdb = redis.StrictRedis(host=redis_info["host"],
port=redis_info["port"],
db=redis_info["db_no"])
self._roll_call = {}
return
def any_detected(self):
for thisDevice, presence in self._roll_call.items():
if presence:
return True
return False
def poll_devices(self):
roll_call = {}
redis_info = config.redis_details()
previously_detected = self.any_detected()
any_detected = False
for name, address in self._monitored_devices.items():
self._notifer.diagnostic("pinging %s at %s" % (name, address))
found = self.ping(address)
if found:
any_detected = True
roll_call[name] = True
self._notifer.note("%s found" % name)
else:
roll_call[name] = False
self._notifer.note("%s missing" % name)
if previously_detected:
self._poll.reset()
# set up next poll
if any_detected and not previously_detected:
self._poll.set_period(
config.general_details()["positive_poll_period"])
elif not any_detected and previously_detected:
self._poll.set_period(
config.general_details()["negative_poll_period"])
self._roll_call = roll_call
self._rdb.set(redis_info["key_detail"], json.dumps(roll_call))
self._rdb.set(redis_info["key_summary"], json.dumps(any_detected))
return
def ping(self, ip_address):
# NOTE: ping requires root access. Fake it during development with a random#
if hasattr(args, 'test') and args.test:
found = (random.randint(0, 3) == 3)
elif USE_PYPING:
response = pyping.ping(ip_address)
found = (response.ret_code == 0)
else:
response = os.system("ping -c i %s" % ip_address)
found = (response == 0)
return found
def roll_call(self):
return self._roll_call
def check(self):
self._poll.check()
return
def run(self):
while gRunningFlag:
self.check()
return
