def __init__(self, args, manifest, monitored_devices, check_period, notifier): super(Surveyor, self).__init__() self._args = args self._manifest = manifest self._monitored_devices = monitored_devices self._notifer = notifier self._roll_call = SharedVarCollection({}) self._devices_poll = Periodic(check_period, self.poll_devices, "poll_devices") return
async def create_program_and_run_it(): init_logging() prog = MainRuntime(logger) prog.init_services() start_http_server(8000) host = prog.cfg.get_fw_api_server_hostname() poll_interval = prog.cfg.get_polling_delay_seconds() logger.info( f"Extra Metrics - connecting to {host}, using poll interval of {poll_interval} sec" ) # fetches and stores the current server version, this is important as the REST API's depend on this # information. Prior to 14.2.0 some of the API calls were different and 14.2.0 put them all under ../api/v1/... prog.fw_query.fetch_server_version() log_config_summary(prog.cfg, prog.fw_query.major_version, prog.fw_query.minor_version, prog.fw_query.patch_version) if prog.fw_query.major_version is None or prog.fw_query.major_version == 0: logger.error("Unable to reach FileWave server, aborting...") return p = Periodic(poll_interval, prog.validate_and_collect_data) await p.start() while (True): if prog.rerun_data_collection: await prog.validate_and_collect_data() await asyncio.sleep(1)
def __init__(self, args, config, notifier): super(Tracker, self).__init__() self._args = args self._config = config self._monitored_devices = config.devices_details()["monitored_devices"] self._notifer = notifier positive_poll_period = config.general_details()["positive_poll_period"] negative_poll_period = config.general_details()["negative_poll_period"] self._poll = Periodic(negative_poll_period, self.poll_devices, "poll_devices", notifier=notifier) redis_info = config.redis_details() self._rdb = redis.StrictRedis(host=redis_info["host"], port=redis_info["port"], db=redis_info["db_no"]) self._roll_call = {} return
async def _schedule_obsolete_game_check() -> None: """Schedule the Obsolete Check task to run periodically, with a delay before starting.""" period = config().obsolete_game_check_period_sec delay = config().obsolete_game_check_delay_sec p = Periodic(period, _execute_obsolete_game_check) await p.start(delay=delay) log.debug( "Completed scheduling obsolete game check with period %d and delay %d", period, delay)
async def _schedule_idle_player_check() -> None: """Schedule the Idle Player Check task to run periodically, with a delay before starting.""" period = config().idle_player_check_period_sec delay = config().idle_player_check_delay_sec p = Periodic(period, _execute_idle_player_check) await p.start(delay=delay) log.debug( "Completed scheduling idle player check with period %d and delay %d", period, delay)
class Surveyor(threading.Thread): def __init__(self, args, manifest, monitored_devices, check_period, notifier): super(Surveyor, self).__init__() self._args = args self._manifest = manifest self._monitored_devices = monitored_devices self._notifer = notifier self._roll_call = SharedVarCollection({}) self._devices_poll = Periodic(check_period, self.poll_devices, "poll_devices") return def poll_devices(self): for name in self._monitored_devices: self._notifer.diagnostic("pinging %s" % name) found = self.ping(self._manifest.address(name)) if found: self._roll_call.set(name, True) self._notifer.note("%s found" % name) else: self._roll_call.set(name, False) self._notifer.note("%s missing" % name) return def ping(self, ip_address): # NOTE: ping requires root access. Fake it during development with a random# if hasattr(args, 'test') and args.test: found = (random.randint(0, 3) == 3) else: response = pyping.ping(ip_address) found = (response.ret_code == 0) return found def roll_call(self): return self._roll_call.get() def check(self): self._devices_poll.check() return def run(self): while gRunningFlag: self._devices_poll.check() return
async def main(): cm = mpl.cm.get_cmap('jet') context = ModelContext('./data/emission.pkl', 30, 0.6, 500) p = Periodic(lambda: payload(context, cm), 1) try: print('Start') await p.start() await asyncio.sleep(100.1) finally: await p.stop() # we should stop task finally
class Job(object): def __init__(self, proxy, num, ping_interval, info): self.proxy = proxy self.num = num self.ping_thread = Periodic(self.ping_refresh, ping_interval, delay=ping_interval) self.info = info def ping_refresh(self): print "pinging for job %d" % self.num self.proxy.refresh_job(self.num) def start(self, work, msg): print "job %d starting" % self.num self.ping_thread.start() try: work(self.num, msg) print "job %d done" % self.num self.finish(success=True) except Exception, e: print "job %d failed" % self.num self.finish(success=False, info={'error': repr(e)}) finally:
def __init__(self, n, chunksize, ping_timeout, cleanup_interval): self.n = n self.chunksize = chunksize self.ping_timeout = ping_timeout self.cleanup_interval = cleanup_interval self.packnum = self.n / self.chunksize + (1 if self.n % 1 > 0 else 0) self.undone = set([ x for x in xrange(self.packnum) ]) self.working = set() self.done = set() self.pingtimes = {} self.cleanup_thread = Periodic(self.cleanup, self.cleanup_interval)
def __init__(self, listenPort): #Make a UDP socket self.sock = socket.socket( socket.AF_INET, socket.SOCK_DGRAM ) self.sock.bind( ("0.0.0.0", listenPort) ) self.sock.settimeout(0.01) self.packetSize = 0 self.t = 0 self.buf = "" self.packetTimestamps = [] self.packetsPerSecond = 0 self.simulatedLatency = 0 self.simulatedRandomLatencyVal = 0 self.simulatedPacketloss = 0 self.simulatedRandomLatency = 0 self.simulatedPackets = [] self.packet_outbound_last_id = defaultdict(lambda:0) self.packet_inbound_last_id = defaultdict(lambda:0) self.packetloss = defaultdict(lambda:0) self.ensured_send_packet_ids = defaultdict(lambda:0) self.ensured_sent_packets = defaultdict(dict) self.ensured_recv_packet_ids = defaultdict(lambda:-1) self.ensured_packets_received_early = defaultdict(list) self.resend_unconfirmed_timer = 0 self.averagedData = AveragedData() self.netinfotimer = 1.0 self.debug_lines = [] self.periodic = Periodic() self.periodic.add(self.resendUnconfirmed, 0.5)
def refresh_period_or_duty_cycle(self): self.periodic = Periodic( period_sec=self.state.get_period(), semi_period_percent=self.state.get_duty_cycle() )
class NetCommon: netEntities = { "player": Player, "projectile":Projectile } def __init__(self, listenPort): #Make a UDP socket self.sock = socket.socket( socket.AF_INET, socket.SOCK_DGRAM ) self.sock.bind( ("0.0.0.0", listenPort) ) self.sock.settimeout(0.01) self.packetSize = 0 self.t = 0 self.buf = "" self.packetTimestamps = [] self.packetsPerSecond = 0 self.simulatedLatency = 0 self.simulatedRandomLatencyVal = 0 self.simulatedPacketloss = 0 self.simulatedRandomLatency = 0 self.simulatedPackets = [] self.packet_outbound_last_id = defaultdict(lambda:0) self.packet_inbound_last_id = defaultdict(lambda:0) self.packetloss = defaultdict(lambda:0) self.ensured_send_packet_ids = defaultdict(lambda:0) self.ensured_sent_packets = defaultdict(dict) self.ensured_recv_packet_ids = defaultdict(lambda:-1) self.ensured_packets_received_early = defaultdict(list) self.resend_unconfirmed_timer = 0 self.averagedData = AveragedData() self.netinfotimer = 1.0 self.debug_lines = [] self.periodic = Periodic() self.periodic.add(self.resendUnconfirmed, 0.5) def readPacket(self, info, data): self.averagedData.add(self.t, "packets") self.averagedData.add(self.t, "packetsize", len(data)) unpacked = pickle.loads(zlib.decompress(data)) addr, port = info addrportstr = addr + ":" + str(port) if "ensured_id" in unpacked: if unpacked["ensured_id"] == self.ensured_recv_packet_ids[addrportstr]+1: print "recv " + str(unpacked["ensured_id"]) self.ensured_recv_packet_ids[addrportstr] += 1 self.sendReceipt(addr, port, unpacked["ensured_id"]) elif unpacked["ensured_id"] < self.ensured_recv_packet_ids[addrportstr]+1: print unpacked print "got ensured packet twice; resending receipt for " + str(unpacked["ensured_id"]) self.sendReceipt(addr, port, unpacked["ensured_id"]) return [] else: print "got packet " + str(unpacked["ensured_id"]) + " before " + str(self.ensured_recv_packet_ids[addrportstr]+1) self.ensured_packets_received_early[addrportstr].append(unpacked) return [] allPackets = [] to_remove = [] self.ensured_packets_received_early[addrportstr].sort(lambda a,b:cmp(a["ensured_id"], b["ensured_id"])) for p in self.ensured_packets_received_early[addrportstr]: print "resolving old " + str(p["ensured_id"]) if p["ensured_id"] <= self.ensured_recv_packet_ids[addrportstr]+1: self.ensured_recv_packet_ids[addrportstr] += 1 self.sendReceipt(addr, port, p["ensured_id"]) allPackets.extend(self.readUnpackedPacket(p, addrportstr)) to_remove.append(p) for p in to_remove: self.ensured_packets_received_early[addrportstr].remove(p) allPackets.extend(self.readUnpackedPacket(unpacked, addrportstr)) return allPackets def sendReceipt(self, addr, port, q): self.sendPacket({"type":"confirmReceipt","other_ensured_id":q}, addr, port) def readUnpackedPacket(self, unpacked, addrportstr): pid = unpacked["packet_id"] lid = self.packet_inbound_last_id[addrportstr] if pid > lid + 1: self.packetloss[addrportstr] += 1 self.packet_inbound_last_id[addrportstr] = pid if self.packet_inbound_last_id[addrportstr] > 0: packetloss = self.packetloss[addrportstr] / float(self.packet_inbound_last_id[addrportstr]) self.averagedData.add(self.t, "packetloss_" + addrportstr, packetloss) return [unpacked] def sendPacket(self, data, addr, port): print "packet: " + data["type"] addrportstr = addr + ":" + str(port) data["packet_id"] = self.packet_outbound_last_id[addrportstr] self.packet_outbound_last_id[addrportstr] += 1 self.sock.sendto(zlib.compress(pickle.dumps(data, 2)), (addr, port)) def sendEnsuredPacket(self, data, addr, port): addrportstr = addr + ":" + str(port) ensured_id = self.ensured_send_packet_ids[addrportstr] print "packet: " + data["type"] + " (ensured id: " + str(ensured_id) + ")" data["packet_id"] = self.packet_outbound_last_id[addrportstr] self.packet_outbound_last_id[addrportstr] += 1 data["ensured_id"] = ensured_id cdata = zlib.compress(pickle.dumps(data, 2)) sent = { "id":ensured_id, "data":cdata, "time":self.t, "info":(addr,port) } self.ensured_sent_packets[addrportstr][ensured_id] = sent self.sock.sendto(cdata, (addr, port)) self.ensured_send_packet_ids[addrportstr] = ensured_id + 1 def process_confirmReceipt(self, data, game, info): (addr, port) = info addrportstr = addr + ":" + str(port) pending_packets = self.ensured_sent_packets[addrportstr] pid = data["other_ensured_id"] print "got receipt for " + str(pid) if pid in pending_packets: del pending_packets[pid] else: if pid > self.ensured_send_packet_ids: print "got receipt for packet i haven't sent yet!!" def update(self, game, dt): self.game = game self.t = pygame.time.get_ticks() / 1000.0 self.periodic.update() self.packetsPerSecond = self.averagedData.get_ct(self.t, "packets", 1.0) self.packetSize = self.averagedData.get_sum(self.t, "packetsize", 1.0) allPackets = [] try: (data, info) = self.sock.recvfrom(4096) #self.packetSize = len(data) if self.simulatedPacketloss > 0 and random.random() < self.simulatedPacketloss: pass else: allPackets = self.readPacket(info, data) except socket.timeout: pass except socket.error as err: #print err pass #print self.simulatedPackets if self.simulatedLatency == 0: for d in allPackets: self.process(d, game, info) else: off = self.simulatedLatency + self.simulatedRandomLatency * random.random() self.simulatedPackets.extend( [(d, off, info) for d in allPackets] ) thisFramePackets = [ s for s in self.simulatedPackets if s[1] <= 0] self.simulatedPackets = [ s for s in self.simulatedPackets if s[1] > 0 ] for (p, t, info) in thisFramePackets: self.process(p, game, info) self.simulatedPackets = [ (s[0], s[1] - dt, s[2]) for s in self.simulatedPackets ] def resendUnconfirmed(self): for k,packets in self.ensured_sent_packets.items(): for i,packet in packets.items(): if self.t > packet["time"] + 1.5: print "resending unreceipted packet: " + str(packet["id"]) self.sock.sendto(packet["data"], packet["info"]) def process(self, data, game, info): if(hasattr(self, "process_" + data["type"])): f = getattr(self, "process_" + data["type"]) f(data, game, info) else: print("Got packet of type '" + data["type"] + "' but there is no process_" + data["type"] + " method to handle it." )
def __init__(self, proxy, num, ping_interval, info): self.proxy = proxy self.num = num self.ping_thread = Periodic(self.ping_refresh, ping_interval, delay=ping_interval) self.info = info
class Balancer(object): def __init__(self, n, chunksize, ping_timeout, cleanup_interval): self.n = n self.chunksize = chunksize self.ping_timeout = ping_timeout self.cleanup_interval = cleanup_interval self.packnum = self.n / self.chunksize + (1 if self.n % 1 > 0 else 0) self.undone = set([ x for x in xrange(self.packnum) ]) self.working = set() self.done = set() self.pingtimes = {} self.cleanup_thread = Periodic(self.cleanup, self.cleanup_interval) def get_status(self): return "status: %d/%d/%d (undone/working/done)" % (len(self.undone), len(self.working), len(self.done)) def get_start_job(self): if self.undone: job = self.undone.pop() self.working.add(job) self.pingtimes[job] = time() print "job %d started" % job print self.get_status() msg = {'work': True, 'n': self.n, 'job': job, 'chunksize': self.chunksize} return msg else: return {'work': False, 'shutdown': len(self.working)==0} def refresh_job(self, job, info={}): if job in self.working: print "received refresh ping for job %d" % job self.pingtimes[job] = time() return True else: print "ignoring unexpected refresh ping for job %d" % job return False def finish_job(self, job, success, info={}): try: self.working.remove(job) if success: self.done.add(job) print "job %d completed successfully" % job else: self.undone.add(job) print "job %d failed with error: %s" % (job, info.get('error', "unspecified")) print self.get_status() del self.pingtimes[job] return True except KeyError: print "got unexpected finishing notification for job %d" % job return False def clean_dead_jobs(self): now = time() dead = set() for job in self.pingtimes: if now - self.pingtimes[job] > self.ping_timeout: dead.add(job) num = len(self.working) self.undone.union(dead) self.working -= dead return num - len(self.working) # TODO: delete done jobs from pingtimes to save some memory def cleanup(self): numcleaned = self.clean_dead_jobs() if numcleaned: print "cleaned %d dead jobs" % numcleaned print self.get_status() def start(self): self.cleanup_thread.start() def stop(self): self.cleanup_thread.stop()
class Tracker(threading.Thread): def __init__(self, args, config, notifier): super(Tracker, self).__init__() self._args = args self._config = config self._monitored_devices = config.devices_details()["monitored_devices"] self._notifer = notifier positive_poll_period = config.general_details()["positive_poll_period"] negative_poll_period = config.general_details()["negative_poll_period"] self._poll = Periodic(negative_poll_period, self.poll_devices, "poll_devices", notifier=notifier) redis_info = config.redis_details() self._rdb = redis.StrictRedis(host=redis_info["host"], port=redis_info["port"], db=redis_info["db_no"]) self._roll_call = {} return def any_detected(self): for thisDevice, presence in self._roll_call.items(): if presence: return True return False def poll_devices(self): roll_call = {} redis_info = config.redis_details() previously_detected = self.any_detected() any_detected = False for name, address in self._monitored_devices.items(): self._notifer.diagnostic("pinging %s at %s" % (name, address)) found = self.ping(address) if found: any_detected = True roll_call[name] = True self._notifer.note("%s found" % name) else: roll_call[name] = False self._notifer.note("%s missing" % name) if previously_detected: self._poll.reset() # set up next poll if any_detected and not previously_detected: self._poll.set_period( config.general_details()["positive_poll_period"]) elif not any_detected and previously_detected: self._poll.set_period( config.general_details()["negative_poll_period"]) self._roll_call = roll_call self._rdb.set(redis_info["key_detail"], json.dumps(roll_call)) self._rdb.set(redis_info["key_summary"], json.dumps(any_detected)) return def ping(self, ip_address): # NOTE: ping requires root access. Fake it during development with a random# if hasattr(args, 'test') and args.test: found = (random.randint(0, 3) == 3) elif USE_PYPING: response = pyping.ping(ip_address) found = (response.ret_code == 0) else: response = os.system("ping -c i %s" % ip_address) found = (response == 0) return found def roll_call(self): return self._roll_call def check(self): self._poll.check() return def run(self): while gRunningFlag: self.check() return