class NetworkSpeedDaemon(VMCDaemon): """ I enque fake SIG_SPEED UnsolicitedNotifications """ def __init__(self, frequency, device, notification_manager): super(NetworkSpeedDaemon, self).__init__(frequency, device, notification_manager) self.netspeed = NetworkSpeed() def start(self): log.msg("DAEMONS: DAEMON %s started..." % self.__class__.__name__) self.netspeed.start() self.loop = LoopingCall(self.function) self.loop.start(self.frequency) def stop(self): log.msg("DAEMONS: DAEMON %s stopped..." % self.__class__.__name__) self.loop.stop() self.netspeed.stop() def function(self): up, down = self.netspeed["up"], self.netspeed["down"] n = N.UnsolicitedNotification(N.SIG_SPEED, bps_to_human(up, down)) self.manager.on_notification_received(n)
class LeaseService(Service): """ Manage leases. In particular, clear out expired leases once a second. :ivar _reactor: A ``twisted.internet.reactor`` implementation. :ivar _persistence_service: The persistence service to act with. :ivar _lc: A ``twisted.internet.task.LoopingCall`` run every second to update the configured leases by releasing leases that have expired. """ def __init__(self, reactor, persistence_service): if reactor is None: reactor = default_reactor self._reactor = reactor self._persistence_service = persistence_service def startService(self): self._lc = LoopingCall(self._expire) self._lc.clock = self._reactor self._lc.start(1) def stopService(self): self._lc.stop() def _expire(self): now = datetime.now(tz=UTC) return update_leases(lambda leases: leases.expire(now), self._persistence_service)
class USBClient(Protocol): def __init__(self, network): self.network = network self.usb_list = [] self.lc = LoopingCall(self.timerEvent) self.serialBuffer = bytes("") def timerEvent(self): self.lc.stop() for cli in client_list: cli.transport.write(self.serialBuffer) self.serialBuffer = bytes("") def connectionFailed(self): log.msg ("Connection Failed:", self) reactor.stop() def connectionMade(self): usb_list.append(self) log.msg("Connected to %s device" % device) def dataReceived(self, data): #log.msg("data resrvr") self.serialBuffer += data if self.lc.running: self.lc.stop() self.lc.start(0.2, False) def sendLine(self, cmd): print cmd self.transport.write(cmd + "\r\n") def outReceived(self, data): self.data = self.data + data
class IconAnimator: lc = None def __init__(self, box, icons): self.box = box self.icons = icons for icon in icons: icon.show() self.stop() def tick(self): self.position += 1 self.position %= len(self.icons) c = self.box.get_children() if c: self.box.remove(c[0]) self.box.add(self.icons[self.position]) def start(self): self.animating = True self.tick() self.lc = LoopingCall(self.tick) self.lc.start(1.0) def stop(self, index=0): self.animating = False self.position = index - 1 self.tick() if self.lc is not None: self.lc.stop() self.lc = None
class Joiner: def __init__(self, screen, address, port): self.screen = screen self.address = address self.port = port def start(self): # Set up the connection between the state and the network; # using queues gives a degree of seperation between the # communication and game logic, and making them be deferred # keeps it all asynchronous. inqueue = DeferredQueue() outqueue = DeferredQueue() #Initialize GameState gs = GameState(self,self.screen,2,inqueue,outqueue) #Create Looping Call self.lc = LoopingCall(gs.gameLoop) self.lc.start(.0166666666) #Begin Connecting connfactory = CommandConnFactory(inqueue,outqueue) reactor.connectTCP(self.address,self.port,connfactory) def stop(self, nextscreen=None): # Stop the GameState logic self.lc.stop() # Start up the next screen, if there is one if nextscreen: nextscreen.start() else: reactor.stop()
class EnvisalinkClientFactory(ReconnectingClientFactory): # we will only ever have one connection to envisalink at a time envisalinkClient = None _currentLoopingCall = None def __init__(self, config): self._config = config def buildProtocol(self, addr): logging.debug("%s connection estblished to %s:%s", addr.type, addr.host, addr.port) logging.debug("resetting connection delay") self.resetDelay() self.envisalinkClient = EnvisalinkClient(self._config) # check on the state of the envisalink connection repeatedly self._currentLoopingCall = LoopingCall(self.envisalinkClient.check_alive) self._currentLoopingCall.start(1) return self.envisalinkClient def startedConnecting(self, connector): logging.debug("Started to connect to Envisalink...") def clientConnectionLost(self, connector, reason): logging.debug('Lost connection to Envisalink. Reason: %s', str(reason)) self._currentLoopingCall.stop() ReconnectingClientFactory.clientConnectionLost(self, connector, reason) def clientConnectionFailed(self, connector, reason): logging.debug('Connection failed to Envisalink. Reason: %s', str(reason)) self._currentLoopingCall.stop() ReconnectingClientFactory.clientConnectionFailed(self, connector, reason)
class VOEventBroadcasterFactory(ServerFactory): IAMALIVE_INTERVAL = 60 # Sent iamalive every IAMALIVE_INTERVAL seconds protocol = VOEventBroadcaster def __init__(self, local_ivo, test_interval): # test_interval is the time in seconds between sending test events to # the network. 0 to disable. self.local_ivo = local_ivo self.test_interval = test_interval self.broadcasters = [] self.alive_loop = LoopingCall(self.sendIAmAlive) self.test_loop = LoopingCall(self.sendTestEvent) def startFactory(self): self.alive_loop.start(self.IAMALIVE_INTERVAL) if self.test_interval: self.test_loop.start(self.test_interval) return ServerFactory.startFactory(self) def stopFactory(self): self.alive_loop.stop() if self.test_loop.running: self.test_loop.stop() return ServerFactory.stopFactory(self) def sendIAmAlive(self): log.debug("Broadcasting iamalive") for broadcaster in self.broadcasters: broadcaster.sendIAmAlive() def sendTestEvent(self): log.debug("Broadcasting test event") test_event = broker_test_message(self.local_ivo) for broadcaster in self.broadcasters: broadcaster.send_event(test_event)
class Autofetcher(object): """ Download name claims as they occur """ def __init__(self, api): self._api = api self._checker = LoopingCall(self._check_for_new_claims) self.best_block = None def start(self): reactor.addSystemEventTrigger('before', 'shutdown', self.stop) self._checker.start(5) def stop(self): log.info("Stopping autofetcher") self._checker.stop() def _check_for_new_claims(self): block = self._api.get_best_blockhash() if block != self.best_block: log.info("Checking new block for name claims, block hash: %s" % block) self.best_block = block transactions = self._api.get_block({'blockhash': block})['tx'] for t in transactions: c = self._api.get_claims_for_tx({'txid': t}) if len(c): for i in c: log.info("Downloading stream for claim txid: %s" % t) self._api.get({'name': t, 'stream_info': json.loads(i['value'])})
class FileConsumer(object): _looper = None _producer = None def __init__(self, reactor=None, interval=0.01): self._reactor = reactor self._interval = interval self._io = StringIO() def registerProducer(self, producer, isPush): if not isPush and not self._reactor: raise ValueError("can't read from a pull producer with no reactor") self._producer = producer if not isPush: self._looper = LoopingCall(self._pull) self._looper.clock = self._reactor self._looper.start(self._interval, now=False) def unregisterProducer(self): self._producer = None if self._looper: self._looper.stop() def _pull(self): self._producer.resumeProducing() def write(self, data): self._io.write(data) def value(self): return self._io.getvalue()
class TimerService(_VolatileDataService): """Service to periodically call a function Every C{step} seconds call the given function with the given arguments. The service starts the calls when it starts, and cancels them when it stops. """ volatile = ["_loop"] def __init__(self, step, callable, *args, **kwargs): self.step = step self.call = (callable, args, kwargs) def startService(self): service.Service.startService(self) from twisted.internet.task import LoopingCall callable, args, kwargs = self.call self._loop = LoopingCall(callable, *args, **kwargs) self._loop.start(self.step, now=True).addErrback(self._failed) def _failed(self, why): self._loop.running = False log.err(why) def stopService(self): if self._loop.running: self._loop.stop() return service.Service.stopService(self)
class TimeOnlineLabel(gtk.Label): """A display of how long the current session has been online for.""" def __init__(self): gtk.Label.__init__(self) self.looping_call = LoopingCall(self.update_time) self.start_time = None def start_counting(self): """Start ticking.""" self.start_time = datetime.now() #don't leave our display blank self.update_time() self.looping_call.start(0.5) #update it twice per second def stop_counting(self): """We only count time online; stop counting.""" self.looping_call.stop() def update_time(self): """Should tick once a second. Displays the current running count of how long's been spent online. """ delta = datetime.now() - self.start_time #chop off microseconds, for neatness delta = timedelta(delta.days, delta.seconds) self.set_text('Time online: %s' % delta)
class RelayProtocol(RTMProtocol): clock = reactor def __init__(self, session_data): RTMProtocol.__init__(self) self.session_data = session_data self.bot_user_id = session_data['self']['id'] self.lc = None self.relay = None def onOpen(self): self.relay = self.factory.relay self.lc = LoopingCall(self.send_ping) self.lc.clock = self.clock self.lc.start(3, now=True) def onClose(self, wasClean, code, reason): if self.relay is not None: self.relay.remove_protocol(self.bot_user_id) if self.lc is not None: self.lc.stop() def onMessage(self, payload, isBinary): data = json.loads(payload) self.relay.relay(self.bot_user_id, data) def send_ping(self): return self.send_message({ "type": "ping", }) def send_message(self, data): return self.sendMessage(json.dumps(data))
class UdpSender(DatagramProtocol): """ Class that will send UDP packets to UDP Listener """ def __init__(self, host, count, size, duration): # LoopingCall does not know whether UDP socket is actually writable self.looper = None self.host = host self.count = count self.duration = duration self.start = time.time() self.sent = 0 self.data = array.array('c', 'X' * size) def startProtocol(self): self.looper = LoopingCall(self.sendData) period = self.duration / float(self.count) self.looper.start(period, now=False) def stopProtocol(self): if (self.looper is not None): self.looper.stop() self.looper = None def datagramReceived(self, data, (host, port)): pass
class Rain(object): """ Make it rain. Rain only occurs during spring. """ implements(IAutomaton) blocks = tuple() def __init__(self): self.season_loop = LoopingCall(self.check_season) def scan(self, chunk): pass def feed(self, coords): pass def start(self): self.season_loop.start(5 * 60) def stop(self): self.season_loop.stop() def check_season(self): if factory.world.season.name == "spring": factory.vane.weather = "rainy" reactor.callLater(1 * 60, setattr, factory.vane, "weather", "sunny") name = "rain"
class CopyEncoder(object): """ This encoder just reads from a file. It ignores the requirement to limit the playback speed so clients will drift around depending on when they start and how large of a buffer they have. The UI updates will also drift from the audio playback. But it has no external requirments, and it simple. """ zope.interface.classProvides(IEncoder) def __init__(self, song, data_callback): self.song = song self.data_callback = data_callback self.done = defer.Deferred() def encode(self): self.file = urllib2.urlopen(self.song.uri) self.lc = LoopingCall(self.process_file) self.lc.start(.25) # TODO: bit rate? return self.done def process_file(self): data = self.file.read(1024 * 8) if not data: self.lc.stop() self.file.close() self.done.callback(None) return self.data_callback(data)
class PeriodicTasksService(Service): def __init__(self, clock, taskFunction, interval=60): self.clock = clock self.taskFunction = taskFunction self.interval = interval self._delayedCall = None self._looper = None self._loopFinished = None def startService(self): Service.startService(self) now = time.time() startDelta = now // self.interval * self.interval + self.interval - now self._delayedCall = self.clock.callLater(startDelta, self._startLooping) def _startLooping(self): self._delayedCall = None self._looper = LoopingCall(self._call) self._looper.clock = self.clock self._loopFinished = self._looper.start(self.interval) def _call(self): d = deferToThread(self.taskFunction) d.addErrback(log.err, "error calling periodic tasks") return d def stopService(self): if self._delayedCall: self._delayedCall.cancel() self._delayedCall = None return self._looper.stop() self._loopFinished.addCallback(lambda _: Service.stopService(self)) return self._loopFinished
class Pinger(object): """ An periodic AMP ping helper. """ def __init__(self, reactor): """ :param IReactorTime reactor: The reactor to use to schedule the pings. """ self.reactor = reactor def start(self, protocol, interval): """ Start sending some pings. :param AMP protocol: The protocol over which to send the pings. :param timedelta interval: The interval at which to send the pings. """ def ping(): protocol.callRemote(NoOp) self._pinging = LoopingCall(ping) self._pinging.clock = self.reactor self._pinging.start(interval.total_seconds(), now=False) def stop(self): """ Stop sending the pings. """ self._pinging.stop()
class BandwidthEstimator: bufsize = 20 totalBytes = 0 def __init__(self, message, length): self.length = length self.message = message self.estim = [] self.bytes = 0 self.call = LoopingCall(self.estimateBandwidth) self.call.start(1) def estimateBandwidth(self): bytes = self.bytes self.totalBytes += bytes self.estim.append(bytes) self.message("%0.2f k/s (%0.2d%%)" % ((sum(self.estim) / len(self.estim)) / 1024., (float(self.totalBytes) / self.length) * 100)) if len(self.estim) > self.bufsize: self.estim.pop(0) self.bytes = 0 def stop(self): self.call.stop() self.estimateBandwidth() self.message("Finished receiving: %d bytes (%d%%)" % ( self.totalBytes, (float(self.totalBytes) / self.length) * 100))
class SendsManyFileDescriptors(ConnectableProtocol): paused = False def connectionMade(self): self.socket = socket() self.transport.registerProducer(self, True) def sender(): self.transport.sendFileDescriptor(self.socket.fileno()) self.transport.write(b"x") self.task = LoopingCall(sender) self.task.clock = self.transport.reactor self.task.start(0).addErrback(err, "Send loop failure") def stopProducing(self): self._disconnect() def resumeProducing(self): self._disconnect() def pauseProducing(self): self.paused = True self.transport.unregisterProducer() self._disconnect() def _disconnect(self): self.task.stop() self.transport.abortConnection() self.other.transport.abortConnection()
class PollingDataStream(DataStream): """ A self-polling data stream. This class represents a data stream that wakes up at a given frequency, and calls the :meth:`poll` method. """ frequency = None # Either a timedelta object, or the number of seconds now = False def __init__(self): super(PollingDataStream, self).__init__() self.timer = LoopingCall(self.poll) if isinstance(self.frequency, timedelta): seconds = ( self.frequency.seconds + (self.frequency.days * 24 * 60 * 60) + (self.frequency.microseconds / 1000000.0) ) else: seconds = self.frequency log.debug("Setting a %s second timer" % seconds) self.timer.start(seconds, now=self.now) def poll(self): raise NotImplementedError def stop(self): super(PollingDataStream, self).stop() try: if hasattr(self, "timer"): self.timer.stop() except Exception, e: self.log.warn(e)
class ICUMonitorFactory(protocol.ClientFactory): def __init__(self, gui_msg_callback, gui_ip, gui_port): self.ip = gui_ip self.port = gui_port self.msg_callback = gui_msg_callback self.objmonit = None def buildProtocol(self, addr): return ICUMonitor(self.msg_callback) def clientConnectionLost(self, connector, reason): pass def clientConnectionFailed(self, connector, reason): pass def sendmsg(self, gui_reactor): gui_reactor.connectTCP(self.ip, self.port, self) def startsend(self, gui_reactor, timerep): self.loop = LoopingCall(self.sendmsg, gui_reactor) self.loop.start(timerep) def stopsend(self): self.loop.stop()
class CommandExecutor(Runnable): """Executes a method every x seconds""" __metaclass__ = ABCMeta def __init__(self, pauseBetweenDuties=emu_config.botcommand_timeout, **kwargs): """:param pauseBetweenDuties: How long to wait between invocations of performDuty() in seconds""" super(CommandExecutor, self).__init__(**kwargs) logging.debug("pause: %d, kwargs: %s"%(pauseBetweenDuties, kwargs)) self.pauseBetweenDuties = float(pauseBetweenDuties) self.lc = LoopingCall(self.performDuty) def start(self): """Starts the runnable, so that it regularly calls performDuty()""" logging.debug("Started performing duties every %d seconds" % self.pauseBetweenDuties) observer = log.PythonLoggingObserver() observer.start() lcDeferred = self.lc.start(self.pauseBetweenDuties) lcDeferred.addErrback(self.errback) @abstractmethod def performDuty(self, *args, **kwargs): """Does the actual work. Is called regularly and should be implemented by subclasses.""" def stop(self): """Stops performDuty() from being called""" logging.debug("Stopping %s" % self.name) if self.lc.running: self.lc.stop() def errback(self, failure): """Given to defereds to report errors""" logging.warning( "%s in %s: %s %s" % (failure.type, self.name, failure.getErrorMessage(), failure.getBriefTraceback()))
class LeaseService(Service): """ Manage leases. In particular, clear out expired leases once a second. :ivar _reactor: A ``IReactorTime`` provider. :ivar _persistence_service: The persistence service to act with. :ivar _lc: A ``twisted.internet.task.LoopingCall`` run every second to update the configured leases by releasing leases that have expired. """ def __init__(self, reactor, persistence_service): self._reactor = reactor self._persistence_service = persistence_service def startService(self): self._lc = LoopingCall(self._expire) self._lc.clock = self._reactor self._lc.start(1) def stopService(self): self._lc.stop() def _expire(self): now = datetime.fromtimestamp(self._reactor.seconds(), tz=UTC) def expire(leases): updated_leases = leases.expire(now) for dataset_id in set(leases) - set(updated_leases): _LOG_EXPIRE(dataset_id=dataset_id, node_id=leases[dataset_id].node_id).write() return updated_leases return update_leases(expire, self._persistence_service)
class ProvisionerQueryService(ServiceProcess): """Provisioner querying service """ declare = ServiceProcess.service_declare(name='provisioner_query', version='0.1.0', dependencies=[]) def slc_init(self): interval = float(self.spawn_args.get("interval_seconds", DEFAULT_QUERY_INTERVAL)) self.client = ProvisionerClient(self) log.debug('Starting provisioner query loop - %s second interval', interval) self.loop = LoopingCall(self.query) self.loop.start(interval) def slc_terminate(self): if self.loop: self.loop.stop() @defer.inlineCallbacks def query(self): try: yield self._do_query() except Exception,e: log.error("Error sending provisioner query request: %s", e, exc_info=True)
class CacheService(Service): def __init__(self, *args, **kwargs): self.call = None self.node_updater = NodeCacheUpdater() self.cluster_updater = ClusterCacheUpdater() self.vm_updater = VirtualMachineCacheUpdater() self.job_updater = JobCacheUpdater() def update_cache(self): """ a single run of all update classes """ return DeferredList( [ self.vm_updater.update(), self.job_updater.update(), self.node_updater.update(), self.cluster_updater.update(), ] ) def startService(self): self.call = LoopingCall(self.update_cache) self.call.start(settings.PERIODIC_CACHE_REFRESH) def stopService(self): if self.call is not None: self.call.stop()
class myApp(App): '''a simple app''' def __init__(self, reactor): self.menu = [('&start', self.start), ('sto&p', self.stop), ('&quit', self.quit)] App.__init__(self, reactor, 'Simple App', self.menu) def start_clock(self): self.__i__ = 0 self.__lc = LoopingCall(self.clock) self.__lc.start(1.0) def clock(self): log.msg("clock", self.__i__) self.widget('table').set_cells([(0, 0, self.__i__)]) self.widget('table').draw() self.__i__ += 1 def start(self, key): log.msg("menu: start", key) def stop(self, key): self.__lc.stop() log.msg("menu: stop", key) def list_box_active_item_changed(self, arg): '''test callback, here, we just add the callback to other listbox ''' self.widget('table').set_cells([(1, 2, arg['active'])]) self.widget('table').draw()
class Blinker(object): def __init__(self, port, interval=0.5): self.port = port self.interval = interval self.state = None self.transition = None def set(self, *args, **kwargs): self.blink() def unset(self): self.transition_stop() self.port.unset() def blink(self): self.transition_stop() self.transition = LoopingCall(self.blink_task) self.transition.start(self.interval) def transition_stop(self): if self.transition and self.transition.running: self.transition.stop() def blink_task(self): if self.state: self.state = False self.port.set() else: self.state = True self.port.unset()
class Hoster: def __init__(self, screen, port): self.screen = screen self.port = port def start(self): # Set up the connection between the state and the network; # using queues gives a degree of seperation between the # communication and game logic, and making them be deferred # keeps it all asynchronous. inqueue = DeferredQueue() outqueue = DeferredQueue() #Initialize GameState gs = GameState(self,self.screen,1,inqueue,outqueue) #Create Looping Call self.lc = LoopingCall(gs.gameLoop) self.lc.start(.0166666666) #Begin Listening connfactory = CommandConnFactory(inqueue,outqueue) self.listening = reactor.listenTCP(self.port,connfactory) def stop(self, nextscreen=None): # Stop the GameState logic, and let go of the port on which we're listening self.lc.stop() self.listening.stopListening() # Start up the next screen, if there is one if nextscreen: nextscreen.start() else: reactor.stop()
class WebSocketConnection(WebSocketProtocol): def __init__(self, stateObject): self.state = stateObject self.opcode = TEXT WebSocketProtocol.__init__(self, WebSocketReceiver(self)) self.finished = Deferred() self.pingLoop = LoopingCall(self.doPing) def doPing(self): self.receiver.transport.sendFrame(PING, '') def write(self, data): self.receiver.transport.sendFrame(self.opcode, data) def sendFrame(self, opcode, data): self.receiver.transport.sendFrame(opcode, data) def writeSequence(self, data): for chunk in data: self.write(chunk) def connectionLost(self, reason): if self.pingLoop.running: self.pingLoop.stop() self.finished.callback(self)
class TrueAfricanUssdTransport(Transport): CONFIG_CLASS = TrueAfricanUssdTransportConfig TRANSPORT_TYPE = 'ussd' SESSION_STATE_MAP = { TransportUserMessage.SESSION_NONE: 'cont', TransportUserMessage.SESSION_RESUME: 'cont', TransportUserMessage.SESSION_CLOSE: 'end', } TIMEOUT_TASK_INTERVAL = 10 @inlineCallbacks def setup_transport(self): super(TrueAfricanUssdTransport, self).setup_transport() config = self.get_static_config() # Session handling key_prefix = "trueafrican:%s" % self.transport_name self.session_manager = yield SessionManager.from_redis_config( config.redis_manager, key_prefix, config.session_timeout) # XMLRPC Resource self.web_resource = reactor.listenTCP(config.port, server.Site( XmlRpcResource(self)), interface=config.interface) # request tracking self.clock = self.get_clock() self._requests = {} self.request_timeout = config.request_timeout self.timeout_task = LoopingCall(self.request_timeout_cb) self.timeout_task.clock = self.clock self.timeout_task_d = self.timeout_task.start( self.TIMEOUT_TASK_INTERVAL, now=False) self.timeout_task_d.addErrback(log.err, "Request timeout handler failed") @inlineCallbacks def teardown_transport(self): yield self.web_resource.loseConnection() if self.timeout_task.running: self.timeout_task.stop() yield self.timeout_task_d yield self.session_manager.stop() yield super(TrueAfricanUssdTransport, self).teardown_transport() def get_clock(self): """ For easier stubbing in tests """ return reactor def request_timeout_cb(self): for request_id, request in self._requests.items(): timestamp = request.timestamp if timestamp < self.clock.seconds() - self.request_timeout: self.finish_expired_request(request_id, request) def track_request(self, request_id, http_request, session): d = Deferred() self._requests[request_id] = Request(d, http_request, session, self.clock.seconds()) return d def _send_inbound(self, session_id, session, session_event, content): transport_metadata = {'session_id': session_id} request_id = self.generate_message_id() self.publish_message( message_id=request_id, content=content, to_addr=session['to_addr'], from_addr=session['from_addr'], session_event=session_event, transport_name=self.transport_name, transport_type=self.TRANSPORT_TYPE, transport_metadata=transport_metadata, ) return request_id @inlineCallbacks def handle_session_new(self, request, session_id, msisdn, to_addr): session = yield self.session_manager.create_session(session_id, from_addr=msisdn, to_addr=to_addr) session_event = TransportUserMessage.SESSION_NEW request_id = self._send_inbound(session_id, session, session_event, None) r = yield self.track_request(request_id, request, session) returnValue(r) @inlineCallbacks def handle_session_resume(self, request, session_id, content): # This is an existing session. session = yield self.session_manager.load_session(session_id) if not session: returnValue(self.response_for_error()) session_event = TransportUserMessage.SESSION_RESUME request_id = self._send_inbound(session_id, session, session_event, content) r = yield self.track_request(request_id, request, session) returnValue(r) @inlineCallbacks def handle_session_end(self, request, session_id): session = yield self.session_manager.load_session(session_id) if not session: returnValue(self.response_for_error()) session_event = TransportUserMessage.SESSION_CLOSE # send a response immediately, and don't (n)ack # since this is not application-initiated self._send_inbound(session_id, session, session_event, None) response = {} returnValue(response) def handle_outbound_message(self, message): in_reply_to = message['in_reply_to'] session_id = message['transport_metadata'].get('session_id') content = message['content'] if not (in_reply_to and session_id and content): return self.publish_nack( user_message_id=message['message_id'], sent_message_id=message['message_id'], reason="Missing in_reply_to, content or session_id fields") response = { 'session': session_id, 'type': self.SESSION_STATE_MAP[message['session_event']], 'message': content } log.msg("Sending outbound message %s: %s" % (message['message_id'], response)) self.finish_request(in_reply_to, message['message_id'], response) def response_for_error(self): """ Generic response for abnormal server side errors. """ response = { 'message': 'We encountered an error while processing your message', 'type': 'end' } return response def finish_request(self, request_id, message_id, response): request = self._requests.get(request_id) if request is None: # send a nack back, indicating that the original request had # timed out before the outbound message reached us. self.publish_nack(user_message_id=message_id, sent_message_id=message_id, reason='Exceeded request timeout') else: del self._requests[request_id] # (n)ack publishing. # # Add a callback and errback, either of which will be invoked # depending on whether the response was written to the client # successfully or not if request.http_request.content.closed: request_done = fail( Failure(TrueAfricanError("HTTP client closed connection"))) else: request_done = request.http_request.notifyFinish() request_done.addCallbacks( lambda _: self._finish_success_cb(message_id), lambda f: self._finish_failure_cb(f, message_id)) request.deferred.callback(response) def finish_expired_request(self, request_id, request): """ Called on requests that timed out. """ del self._requests[request_id] log.msg('Timing out on response for %s' % request.session['from_addr']) request.deferred.callback(self.response_for_error()) def _finish_success_cb(self, message_id): self.publish_ack(message_id, message_id) def _finish_failure_cb(self, failure, message_id): self.publish_nack(user_message_id=message_id, sent_message_id=message_id, reason=str(failure))
class MetricBuffer: __slots__ = ('metric_path', 'interval_buffers', 'compute_task', 'configured', 'aggregation_frequency', 'aggregation_func') def __init__(self, metric_path): self.metric_path = metric_path self.interval_buffers = {} self.compute_task = None self.configured = False self.aggregation_frequency = None self.aggregation_func = None def input(self, datapoint): (timestamp, value) = datapoint interval = timestamp - (timestamp % self.aggregation_frequency) if interval in self.interval_buffers: buffer = self.interval_buffers[interval] else: buffer = self.interval_buffers[interval] = IntervalBuffer(interval) buffer.input(datapoint) def configure_aggregation(self, frequency, func): self.aggregation_frequency = int(frequency) self.aggregation_func = func self.compute_task = LoopingCall(self.compute_value) if settings['WRITE_BACK_FREQUENCY'] is not None: compute_frequency = min(settings['WRITE_BACK_FREQUENCY'], frequency) else: compute_frequency = frequency self.compute_task.start(compute_frequency, now=False) self.configured = True def compute_value(self): now = int(time.time()) current_interval = now - (now % self.aggregation_frequency) age_threshold = current_interval - ( settings['MAX_AGGREGATION_INTERVALS'] * self.aggregation_frequency) for buffer in list(self.interval_buffers.values()): if buffer.active: value = self.aggregation_func(buffer.values) datapoint = (buffer.interval, value) state.events.metricGenerated(self.metric_path, datapoint) state.instrumentation.increment('aggregateDatapointsSent') buffer.mark_inactive() if buffer.interval < age_threshold: del self.interval_buffers[buffer.interval] if not self.interval_buffers: self.close() self.configured = False del BufferManager.buffers[self.metric_path] def close(self): if self.compute_task and self.compute_task.running: self.compute_task.stop() @property def size(self): return sum([len(buf.values) for buf in self.interval_buffers.values()])
class InfiltrationProtocol(protocol): game_mode = CTF_MODE balanced_teams = None defender = None defender_score_loop = None defender_return_call = None attacker = None attacker_dummy = None attacker_dummy_calls = None def on_map_change(self, map): self.attacker = self.teams[ATTACKER_TEAM] self.defender = self.attacker.other self.defender_score_loop = LoopingCall(self.defender_score_cycle) self.start_defender_score_loop() protocol.on_map_change(self, map) def on_map_leave(self): if self.defender_score_loop and self.defender_score_loop.running: self.defender_score_loop.stop() self.defender_score_loop = None self.end_attacker_dummy_calls() protocol.on_map_leave(self) def on_game_end(self): self.end_attacker_dummy_calls() self.start_defender_score_loop() protocol.on_game_end(self) def on_flag_spawn(self, x, y, z, flag, entity_id): if flag.team is self.attacker: return 0, 0, 63 return protocol.on_flag_spawn(self, x, y, z, flag, entity_id) def start_defender_score_loop(self): if self.defender_score_loop.running: self.defender_score_loop.stop() self.defender_score_loop.start(DEFENDER_SCORE_INTERVAL, now=False) def defender_score_cycle(self): dummy = DummyPlayer(self, self.defender) dummy.score() def attacker_dummy_score(self): self.attacker_dummy.score() if self.attacker_dummy is None: # this could happen if the dummy capture caused the game to end return self.attacker_dummy_calls.pop(0) if not self.attacker_dummy_calls: self.end_attacker_dummy_calls() def end_attacker_dummy_calls(self): if self.attacker_dummy_calls: for dummy_call in self.attacker_dummy_calls: if dummy_call and dummy_call.active(): dummy_call.cancel() self.attacker_dummy_calls = None self.attacker_dummy = None def cancel_defender_return_call(self): if self.defender_return_call and self.defender_return_call.active( ): self.defender_return_call.cancel() self.defender_return_call = None def fog_flash(self, color): old_color = self.get_fog_color() self.set_fog_color(color) callLater(0.2, self.set_fog_color, old_color)
class Tunnel(object): __single = None def __init__(self, settings, crawl_keypair_filename=None, dispersy_port=-1): if Tunnel.__single: raise RuntimeError("Tunnel is singleton") Tunnel.__single = self self.settings = settings self.should_run = True self.crawl_keypair_filename = crawl_keypair_filename self.dispersy_port = dispersy_port self.crawl_data = defaultdict(lambda: []) self.crawl_message = {} self.current_stats = [0, 0, 0] self.history_stats = deque(maxlen=180) self.start_tribler() self.dispersy = self.session.lm.dispersy self.community = None self.clean_messages_lc = LoopingCall(self.clean_messages) self.clean_messages_lc.start(1800) self.build_history_lc = LoopingCall(self.build_history) self.build_history_lc.start(60, now=True) def get_instance(*args, **kw): if Tunnel.__single is None: Tunnel(*args, **kw) return Tunnel.__single get_instance = staticmethod(get_instance) def clean_messages(self): now = int(time.time()) for k in self.crawl_message.keys(): if now - 3600 > self.crawl_message[k]['time']: self.crawl_message.pop(k) clean_twisted_observers() def build_history(self): self.history_stats.append(self.get_stats()) def stats_handler(self, candidate, stats, message): now = int(time.time()) logger.debug('@%d %r %r', now, message.candidate.get_member().mid.encode('hex'), json.dumps(stats)) candidate_mid = candidate.get_member().mid stats = self.preprocess_stats(stats) stats['time'] = now stats_old = self.crawl_message.get(candidate_mid, None) self.crawl_message[candidate_mid] = stats if stats_old is None: return time_dif = float(stats['uptime'] - stats_old['uptime']) if time_dif > 0: for index, key in enumerate( ['bytes_orig', 'bytes_exit', 'bytes_relay']): self.current_stats[index] = self.current_stats[index] * 0.875 + \ (((stats[key] - stats_old[key]) / time_dif) / 1024) * 0.125 def start_tribler(self): config = SessionStartupConfig() config.set_state_dir( os.path.join(config.get_state_dir(), "tunnel-%d") % self.settings.socks_listen_ports[0]) config.set_torrent_checking(False) config.set_multicast_local_peer_discovery(False) config.set_megacache(False) config.set_dispersy(True) config.set_mainline_dht(True) config.set_torrent_collecting(False) config.set_libtorrent(True) config.set_dht_torrent_collecting(False) config.set_enable_torrent_search(False) config.set_videoserver_enabled(False) config.set_dispersy_port(self.dispersy_port) config.set_enable_torrent_search(False) config.set_enable_channel_search(False) config.set_enable_multichain(self.settings.enable_multichain) # We do not want to load the TunnelCommunity in the session but instead our own community config.set_tunnel_community_enabled(False) self.session = Session(config) self.session.start() logger.info("Using Dispersy port %d" % self.session.get_dispersy_port()) def start(self, introduce_port): def start_community(): if self.crawl_keypair_filename: keypair = read_keypair(self.crawl_keypair_filename) member = self.dispersy.get_member( private_key=self.dispersy.crypto.key_to_bin(keypair)) cls = TunnelCommunityCrawler else: if self.settings.enable_multichain: from Tribler.community.multichain.community import MultiChainCommunity member = self.dispersy.get_member( private_key=self.session.multichain_keypair.key_to_bin( )) self.dispersy.define_auto_load(MultiChainCommunity, member, load=True) else: member = self.dispersy.get_new_member(u"curve25519") cls = HiddenTunnelCommunity self.community = self.dispersy.define_auto_load( cls, member, (self.session, self.settings), load=True)[0] self.session.set_anon_proxy_settings( 2, ("127.0.0.1", self.session.get_tunnel_community_socks5_listen_ports())) if introduce_port: self.community.add_discovered_candidate( Candidate(('127.0.0.1', introduce_port), tunnel=False)) blockingCallFromThread(reactor, start_community) self.session.set_download_states_callback( self.download_states_callback, interval=4.0) def download_states_callback(self, dslist): try: self.community.monitor_downloads(dslist) except: logger.error("Monitoring downloads failed") return [] def stop(self): if self.clean_messages_lc: self.clean_messages_lc.stop() self.clean_messages_lc = None if self.build_history_lc: self.build_history_lc.stop() self.build_history_lc = None if self.session: logger.info("Going to shutdown session") return self.session.shutdown() def preprocess_stats(self, stats): result = defaultdict(int) result['uptime'] = stats['uptime'] keys_to_from = { 'bytes_orig': ('bytes_up', 'bytes_down'), 'bytes_exit': ('bytes_enter', 'bytes_exit'), 'bytes_relay': ('bytes_relay_up', 'bytes_relay_down') } for key_to, key_from in keys_to_from.iteritems(): result[key_to] = sum([stats.get(k, 0) for k in key_from]) return result def get_stats(self): return [round(f, 2) for f in self.current_stats]
class AdapterPmMetrics(object): """ Base class for Device Adapter PM Metrics Manager Device specific (OLT, ONU, OpenOMCI, ...) will derive groups of PM information and this base class is primarily used to provide a consistent interface to configure, start, and stop statistics collection. """ DEFAULT_FREQUENCY_KEY = 'default-collection-frequency' DEFAULT_COLLECTION_FREQUENCY = 15 * 10 # 1/10ths of a second # If the collection object has a property of the following name, it will be used # to retrieve the UTC Collection Timestamp (UTC seconds since epoch). If the collection # object does not support this attribute, the current time will be used. If the attribute # is supported, but returns None, this signals that no metrics are currently available # for collection. TIMESTAMP_ATTRIBUTE = 'timestamp' def __init__(self, adapter_agent, device_id, logical_device_id, grouped=False, freq_override=False, **kwargs): """ Initializer for shared Device Adapter PM metrics manager :param adapter_agent: (AdapterAgent) Adapter agent for the device :param device_id: (str) Device ID :param logical_device_id: (str) VOLTHA Logical Device ID :param grouped: (bool) Flag indicating if statistics are managed as a group :param freq_override: (bool) Flag indicating if frequency collection can be specified on a per group basis :param kwargs: (dict) Device Adapter specific values """ self.log = structlog.get_logger(device_id=device_id) self.device_id = device_id self.adapter_agent = adapter_agent self.name = adapter_agent.adapter_name # Sanitize the vcore ID in the logical device ID self.logical_device_id = '0000' + logical_device_id[4:] device = self.adapter_agent.get_device(self.device_id) self.serial_number = device.serial_number self.default_freq = kwargs.get(AdapterPmMetrics.DEFAULT_FREQUENCY_KEY, AdapterPmMetrics.DEFAULT_COLLECTION_FREQUENCY) self.grouped = grouped self.freq_override = grouped and freq_override self.lc = None self.pm_group_metrics = dict() # name -> PmGroupConfig def update(self, pm_config): # TODO: Move any common steps into base class raise NotImplementedError('Your derived class should override this method') def make_proto(self, pm_config=None): raise NotImplementedError('Your derived class should override this method') def start_collector(self, callback=None): """ Start the collection loop for an adapter if the frequency > 0 :param callback: (callable) Function to call to collect PM data """ self.log.info("starting-pm-collection", device_name=self.name) if callback is None: callback = self.collect_and_publish_metrics if self.lc is None: self.lc = LoopingCall(callback) if self.default_freq > 0: self.lc.start(interval=self.default_freq / 10) def stop_collector(self): """ Stop the collection loop""" if self.lc is not None and self.default_freq > 0: self.lc.stop() def collect_group_metrics(self, group_name, group, names, config): """ Collect the metrics for a specific PM group. This common collection method expects that the 'group object' provide as the second parameter supports an attribute or property with the name of the value to retrieve. :param group_name: (str) The unique collection name. The name should not contain spaces. :param group: (object) The object to query for the value of various attributes (PM names) :param names: (set) A collection of PM names that, if implemented as a property in the object, will return a value to store in the returned PM dictionary :param config: (PMConfig) PM Configuration settings. The enabled flag is examined to determine if the data associated with a PM Name will be collected. :return: (MetricInformation) collected metrics """ assert ' ' not in group_name, 'Spaces are not allowed in metric titles, use an underscore' if group is None: return None metrics = dict() context = dict() now = getattr(group, AdapterPmMetrics.TIMESTAMP_ATTRIBUTE) \ if hasattr(group, AdapterPmMetrics.TIMESTAMP_ATTRIBUTE) \ else arrow.utcnow().float_timestamp if now is None: return None # No metrics available at this time for collection for (metric, t) in names: if config[metric].type == PmConfig.CONTEXT and hasattr(group, metric): context[metric] = str(getattr(group, metric)) elif config[metric].type in (PmConfig.COUNTER, PmConfig.GAUGE, PmConfig.STATE): if config[metric].enabled and hasattr(group, metric): metrics[metric] = getattr(group, metric) # Check length of metric data. Will be zero if if/when individual group # metrics can be disabled and all are (or or not supported by the # underlying adapter) if len(metrics) == 0: return None return MetricInformation(metadata=MetricMetaData(title=group_name, ts=now, logical_device_id=self.logical_device_id, serial_no=self.serial_number, device_id=self.device_id, context=context), metrics=metrics) def collect_metrics(self, data=None): """ Collect metrics for this adapter. The adapter type (OLT, ONU, ..) should provide a derived class where this method iterates through all metrics and collects them up in a dictionary with the group/metric name as the key, and the metric values as the contents. The data collected (or passed in) is a list of pairs/tuples. Each pair is composed of a MetricMetaData metadata-portion and list of MetricValuePairs that contains a single individual metric or list of metrics if this is a group metric. This method is called for each adapter at a fixed frequency. TODO: Currently all group metrics are collected on a single timer tick. This needs to be fixed as independent group or instance collection is desirable. :param data: (list) Existing list of collected metrics (MetricInformation). This is provided to allow derived classes to call into further encapsulated classes. :return: (list) metadata and metrics pairs - see description above """ raise NotImplementedError('Your derived class should override this method') def collect_and_publish_metrics(self): """ Request collection of all enabled metrics and publish them """ try: data = self.collect_metrics() self.publish_metrics(data) except Exception as e: self.log.exception('failed-to-collect-kpis', e=e) def publish_metrics(self, data): """ Publish the metrics during a collection. The data collected (or passed in) is a list of dictionary pairs/tuple. Each pair is composed of a metadata-portion and a metrics-portion that contains information for a specific instance of an individual metric or metric group. :param data: (list) Existing list of collected metrics (MetricInformation) to convert to a KPIEvent and publish """ self.log.debug('publish-metrics', data=data) if len(data): try: # TODO: Existing adapters use the KpiEvent, if/when all existing # adapters use the shared KPI library, we may want to # deprecate the KPIEvent kpi_event = KpiEvent2( type=KpiEventType.slice, ts=arrow.utcnow().float_timestamp, slice_data=data ) self.adapter_agent.submit_kpis(kpi_event) except Exception as e: self.log.exception('failed-to-submit-kpis', e=e)
class _ContinuousPolling(_PollLikeMixin, _DisconnectSelectableMixin): """ Schedule reads and writes based on the passage of time, rather than notification. This is useful for supporting polling filesystem files, which C{epoll(7)} does not support. The implementation uses L{_PollLikeMixin}, which is a bit hacky, but re-implementing and testing the relevant code yet again is unappealing. @ivar _reactor: The L{EPollReactor} that is using this instance. @ivar _loop: A C{LoopingCall} that drives the polling, or L{None}. @ivar _readers: A C{set} of C{FileDescriptor} objects that should be read from. @ivar _writers: A C{set} of C{FileDescriptor} objects that should be written to. """ # Attributes for _PollLikeMixin _POLL_DISCONNECTED = 1 _POLL_IN = 2 _POLL_OUT = 4 def __init__(self, reactor): self._reactor = reactor self._loop = None self._readers = set() self._writers = set() def _checkLoop(self): """ Start or stop a C{LoopingCall} based on whether there are readers and writers. """ if self._readers or self._writers: if self._loop is None: from twisted.internet.task import LoopingCall, _EPSILON self._loop = LoopingCall(self.iterate) self._loop.clock = self._reactor # LoopingCall seems unhappy with timeout of 0, so use very # small number: self._loop.start(_EPSILON, now=False) elif self._loop: self._loop.stop() self._loop = None def iterate(self): """ Call C{doRead} and C{doWrite} on all readers and writers respectively. """ for reader in list(self._readers): self._doReadOrWrite(reader, reader, self._POLL_IN) for writer in list(self._writers): self._doReadOrWrite(writer, writer, self._POLL_OUT) def addReader(self, reader): """ Add a C{FileDescriptor} for notification of data available to read. """ self._readers.add(reader) self._checkLoop() def addWriter(self, writer): """ Add a C{FileDescriptor} for notification of data available to write. """ self._writers.add(writer) self._checkLoop() def removeReader(self, reader): """ Remove a C{FileDescriptor} from notification of data available to read. """ try: self._readers.remove(reader) except KeyError: return self._checkLoop() def removeWriter(self, writer): """ Remove a C{FileDescriptor} from notification of data available to write. """ try: self._writers.remove(writer) except KeyError: return self._checkLoop() def removeAll(self): """ Remove all readers and writers. """ result = list(self._readers | self._writers) # Don't reset to new value, since self.isWriting and .isReading refer # to the existing instance: self._readers.clear() self._writers.clear() return result def getReaders(self): """ Return a list of the readers. """ return list(self._readers) def getWriters(self): """ Return a list of the writers. """ return list(self._writers) def isReading(self, fd): """ Checks if the file descriptor is currently being observed for read readiness. @param fd: The file descriptor being checked. @type fd: L{twisted.internet.abstract.FileDescriptor} @return: C{True} if the file descriptor is being observed for read readiness, C{False} otherwise. @rtype: C{bool} """ return fd in self._readers def isWriting(self, fd): """ Checks if the file descriptor is currently being observed for write readiness. @param fd: The file descriptor being checked. @type fd: L{twisted.internet.abstract.FileDescriptor} @return: C{True} if the file descriptor is being observed for write readiness, C{False} otherwise. @rtype: C{bool} """ return fd in self._writers
class PreferencesManager(component.Component): def __init__(self): component.Component.__init__(self, 'PreferencesManager') self.config = deluge.configmanager.ConfigManager( 'core.conf', DEFAULT_PREFS) if 'proxies' in self.config: log.warning( 'Updating config file for proxy, using "peer" values to fill new "proxy" setting' ) self.config['proxy'].update(self.config['proxies']['peer']) log.warning('New proxy config is: %s', self.config['proxy']) del self.config['proxies'] if 'i2p_proxy' in self.config and self.config['i2p_proxy']['hostname']: self.config['proxy'].update(self.config['i2p_proxy']) self.config['proxy']['type'] = 6 del self.config['i2p_proxy'] if 'anonymous_mode' in self.config: self.config['proxy']['anonymous_mode'] = self.config[ 'anonymous_mode'] del self.config['anonymous_mode'] if 'proxy' in self.config: for key in DEFAULT_PREFS['proxy']: if key not in self.config['proxy']: self.config['proxy'][key] = DEFAULT_PREFS['proxy'][key] self.core = component.get('Core') self.new_release_timer = None def start(self): # Set the initial preferences on start-up for key in DEFAULT_PREFS: self.do_config_set_func(key, self.config[key]) self.config.register_change_callback(self._on_config_value_change) def stop(self): if self.new_release_timer and self.new_release_timer.running: self.new_release_timer.stop() # Config set functions def do_config_set_func(self, key, value): on_set_func = getattr(self, '_on_set_' + key, None) if on_set_func: if log.isEnabledFor(logging.DEBUG): log.debug('Config key: %s set to %s..', key, value) on_set_func(key, value) def _on_config_value_change(self, key, value): if self.get_state() == 'Started': self.do_config_set_func(key, value) component.get('EventManager').emit( ConfigValueChangedEvent(key, value)) def _on_set_torrentfiles_location(self, key, value): if self.config['copy_torrent_file']: try: os.makedirs(value) except OSError as ex: log.debug('Unable to make directory: %s', ex) def _on_set_listen_ports(self, key, value): self.__set_listen_on() def _on_set_listen_interface(self, key, value): self.__set_listen_on() def _on_set_outgoing_interface(self, key, value): """Set interface name or IP address for outgoing BitTorrent connections.""" value = value.strip() if value else '' self.core.apply_session_settings({'outgoing_interfaces': value}) def _on_set_random_port(self, key, value): self.__set_listen_on() def __set_listen_on(self): """ Set the ports and interface address to listen for incoming connections on.""" if self.config['random_port']: if not self.config['listen_random_port']: self.config['listen_random_port'] = random.randrange( 49152, 65525) listen_ports = [self.config['listen_random_port'] ] * 2 # use single port range else: self.config['listen_random_port'] = None listen_ports = self.config['listen_ports'] if self.config['listen_interface']: interface = self.config['listen_interface'].strip() else: interface = '0.0.0.0' log.debug( 'Listen Interface: %s, Ports: %s with use_sys_port: %s', interface, listen_ports, self.config['listen_use_sys_port'], ) interfaces = [ '%s:%s' % (interface, port) for port in range(listen_ports[0], listen_ports[1] + 1) ] self.core.apply_session_settings({ 'listen_system_port_fallback': self.config['listen_use_sys_port'], 'listen_interfaces': ','.join(interfaces), }) def _on_set_outgoing_ports(self, key, value): self.__set_outgoing_ports() def _on_set_random_outgoing_ports(self, key, value): self.__set_outgoing_ports() def __set_outgoing_ports(self): port = (0 if self.config['random_outgoing_ports'] else self.config['outgoing_ports'][0]) if port: num_ports = (self.config['outgoing_ports'][1] - self.config['outgoing_ports'][0]) num_ports = num_ports if num_ports > 1 else 5 else: num_ports = 0 log.debug('Outgoing port set to %s with range: %s', port, num_ports) self.core.apply_session_settings({ 'outgoing_port': port, 'num_outgoing_ports': num_ports }) def _on_set_peer_tos(self, key, value): try: self.core.apply_session_setting('peer_tos', int(value, 16)) except ValueError as ex: log.error('Invalid tos byte: %s', ex) def _on_set_dht(self, key, value): lt_bootstraps = self.core.session.get_settings()['dht_bootstrap_nodes'] # Update list of lt bootstraps, using set to remove duplicates. dht_bootstraps = set( lt_bootstraps.split(',') + [ 'router.bittorrent.com:6881', 'router.utorrent.com:6881', 'router.bitcomet.com:6881', 'dht.transmissionbt.com:6881', 'dht.aelitis.com:6881', ]) self.core.apply_session_settings({ 'dht_bootstrap_nodes': ','.join(dht_bootstraps), 'enable_dht': value }) def _on_set_upnp(self, key, value): self.core.apply_session_setting('enable_upnp', value) def _on_set_natpmp(self, key, value): self.core.apply_session_setting('enable_natpmp', value) def _on_set_lsd(self, key, value): self.core.apply_session_setting('enable_lsd', value) def _on_set_utpex(self, key, value): if value: self.core.session.add_extension('ut_pex') def _on_set_enc_in_policy(self, key, value): self._on_set_encryption(key, value) def _on_set_enc_out_policy(self, key, value): self._on_set_encryption(key, value) def _on_set_enc_level(self, key, value): self._on_set_encryption(key, value) def _on_set_encryption(self, key, value): # Convert Deluge enc_level values to libtorrent enc_level values. pe_enc_level = { 0: lt.enc_level.plaintext, 1: lt.enc_level.rc4, 2: lt.enc_level.both, } self.core.apply_session_settings({ 'out_enc_policy': lt.enc_policy(self.config['enc_out_policy']), 'in_enc_policy': lt.enc_policy(self.config['enc_in_policy']), 'allowed_enc_level': lt.enc_level(pe_enc_level[self.config['enc_level']]), 'prefer_rc4': True, }) def _on_set_max_connections_global(self, key, value): self.core.apply_session_setting('connections_limit', value) def _on_set_max_upload_speed(self, key, value): # We need to convert Kb/s to B/s value = -1 if value < 0 else int(value * 1024) self.core.apply_session_setting('upload_rate_limit', value) def _on_set_max_download_speed(self, key, value): # We need to convert Kb/s to B/s value = -1 if value < 0 else int(value * 1024) self.core.apply_session_setting('download_rate_limit', value) def _on_set_max_upload_slots_global(self, key, value): self.core.apply_session_setting('unchoke_slots_limit', value) def _on_set_max_half_open_connections(self, key, value): self.core.apply_session_setting('half_open_limit', value) def _on_set_max_connections_per_second(self, key, value): self.core.apply_session_setting('connection_speed', value) def _on_set_ignore_limits_on_local_network(self, key, value): self.core.apply_session_setting('ignore_limits_on_local_network', value) def _on_set_share_ratio_limit(self, key, value): # This value is a float percentage in deluge, but libtorrent needs int percentage. self.core.apply_session_setting('share_ratio_limit', int(value * 100)) def _on_set_seed_time_ratio_limit(self, key, value): # This value is a float percentage in deluge, but libtorrent needs int percentage. self.core.apply_session_setting('seed_time_ratio_limit', int(value * 100)) def _on_set_seed_time_limit(self, key, value): # This value is stored in minutes in deluge, but libtorrent wants seconds self.core.apply_session_setting('seed_time_limit', int(value * 60)) def _on_set_max_active_downloading(self, key, value): self.core.apply_session_setting('active_downloads', value) def _on_set_max_active_seeding(self, key, value): self.core.apply_session_setting('active_seeds', value) def _on_set_max_active_limit(self, key, value): self.core.apply_session_setting('active_limit', value) def _on_set_dont_count_slow_torrents(self, key, value): self.core.apply_session_setting('dont_count_slow_torrents', value) def _on_set_send_info(self, key, value): """sends anonymous stats home""" log.debug('Sending anonymous stats..') class SendInfoThread(threading.Thread): def __init__(self, config): self.config = config threading.Thread.__init__(self) def run(self): import time now = time.time() # check if we've done this within the last week or never if (now - self.config['info_sent']) >= (60 * 60 * 24 * 7): try: url = ( 'http://deluge-torrent.org/stats_get.php?processor=' + platform.machine() + '&python=' + platform.python_version() + '&deluge=' + deluge.common.get_version() + '&os=' + platform.system() + '&plugins=' + quote_plus( ':'.join(self.config['enabled_plugins']))) urlopen(url) except IOError as ex: log.debug( 'Network error while trying to send info: %s', ex) else: self.config['info_sent'] = now if value: SendInfoThread(self.config).start() def _on_set_new_release_check(self, key, value): if value: log.debug('Checking for new release..') threading.Thread(target=self.core.get_new_release).start() if self.new_release_timer and self.new_release_timer.running: self.new_release_timer.stop() # Set a timer to check for a new release every 3 days self.new_release_timer = LoopingCall( self._on_set_new_release_check, 'new_release_check', True) self.new_release_timer.start(72 * 60 * 60, False) else: if self.new_release_timer and self.new_release_timer.running: self.new_release_timer.stop() def _on_set_proxy(self, key, value): # Initialise with type none and blank hostnames. proxy_settings = { 'proxy_type': lt.proxy_type_t.none, 'i2p_hostname': '', 'proxy_hostname': '', 'proxy_hostnames': value['proxy_hostnames'], 'proxy_peer_connections': value['proxy_peer_connections'], 'proxy_tracker_connections': value['proxy_tracker_connections'], 'force_proxy': value['force_proxy'], 'anonymous_mode': value['anonymous_mode'], } if value['type'] == lt.proxy_type_t.i2p_proxy: proxy_settings.update({ 'proxy_type': lt.proxy_type_t.i2p_proxy, 'i2p_hostname': value['hostname'], 'i2p_port': value['port'], }) elif value['type'] != lt.proxy_type_t.none: proxy_settings.update({ 'proxy_type': value['type'], 'proxy_hostname': value['hostname'], 'proxy_port': value['port'], 'proxy_username': value['username'], 'proxy_password': value['password'], }) self.core.apply_session_settings(proxy_settings) def _on_set_rate_limit_ip_overhead(self, key, value): self.core.apply_session_setting('rate_limit_ip_overhead', value) def _on_set_geoip_db_location(self, key, geoipdb_path): # Load the GeoIP DB for country look-ups if available if os.path.exists(geoipdb_path): try: self.core.geoip_instance = GeoIP.open(geoipdb_path, GeoIP.GEOIP_STANDARD) except AttributeError: log.warning('GeoIP Unavailable') else: log.warning('Unable to find GeoIP database file: %s', geoipdb_path) def _on_set_cache_size(self, key, value): self.core.apply_session_setting('cache_size', value) def _on_set_cache_expiry(self, key, value): self.core.apply_session_setting('cache_expiry', value) def _on_auto_manage_prefer_seeds(self, key, value): self.core.apply_session_setting('auto_manage_prefer_seeds', value)
class MetricLogObserver(config.ReconfigurableServiceMixin, service.MultiService): _reactor = reactor def __init__(self): service.MultiService.__init__(self) self.setName('metrics') self.enabled = False self.periodic_task = None self.periodic_interval = None self.log_task = None self.log_interval = None # Mapping of metric type to handlers for that type self.handlers = {} # Register our default handlers self.registerHandler(MetricCountEvent, MetricCountHandler(self)) self.registerHandler(MetricTimeEvent, MetricTimeHandler(self)) self.registerHandler(MetricAlarmEvent, MetricAlarmHandler(self)) self.getHandler(MetricCountEvent).addWatcher( AttachedSlavesWatcher(self)) def reconfigService(self, new_config): # first, enable or disable if new_config.metrics is None: self.disable() else: self.enable() metrics_config = new_config.metrics # Start up periodic logging log_interval = metrics_config.get('log_interval', 60) if log_interval != self.log_interval: if self.log_task: self.log_task.stop() self.log_task = None if log_interval: self.log_task = LoopingCall(self.report) self.log_task.clock = self._reactor self.log_task.start(log_interval) # same for the periodic task periodic_interval = metrics_config.get('periodic_interval', 10) if periodic_interval != self.periodic_interval: if self.periodic_task: self.periodic_task.stop() self.periodic_task = None if periodic_interval: self.periodic_task = LoopingCall(periodicCheck, self._reactor) self.periodic_task.clock = self._reactor self.periodic_task.start(periodic_interval) # upcall return config.ReconfigurableServiceMixin.reconfigService( self, new_config) def stopService(self): self.disable() service.MultiService.stopService(self) def enable(self): if self.enabled: return log.addObserver(self.emit) self.enabled = True def disable(self): if not self.enabled: return if self.periodic_task: self.periodic_task.stop() self.periodic_task = None if self.log_task: self.log_task.stop() self.log_task = None log.removeObserver(self.emit) self.enabled = False def registerHandler(self, interface, handler): old = self.getHandler(interface) self.handlers[interface] = handler return old def getHandler(self, interface): return self.handlers.get(interface) def emit(self, eventDict): # Ignore non-statistic events metric = eventDict.get('metric') if not metric or not isinstance(metric, MetricEvent): return if metric.__class__ not in self.handlers: return h = self.handlers[metric.__class__] h.handle(eventDict, metric) for w in h.watchers: w.run() def asDict(self): retval = {} for interface, handler in self.handlers.iteritems(): retval.update(handler.asDict()) return retval def report(self): try: for interface, handler in self.handlers.iteritems(): report = handler.report() if not report: continue for line in report.split("\n"): log.msg(line) except Exception: log.err(None, "generating metric report")
class IntervalTask(BaseTask): """ IntervalTask executes a callback at an interval. Example: IntervalTask(myFunc, 60) -- calls myFunc() every 60 seconds. """ _transient_vars = ['_looper'] _callback = None _interval = None _immediate = False _iterations = None _args = [] _kwargs = {} #: @type: LoopingCall _looper = None _last_run_time = None _scheduled_time = None run_count = 0 paused = False _elapsed_at_pause = None _paused_at_shutdown = False def __init__(self, callback, interval, immediate=False, iterations=None, args=None, kwargs=None): """ @param callback: The callable to call at intervals. @param interval: The interval at which to execute the callback. @type interval: int or float or None @param immediate: If True, then the first iteration fires immediately. @type immediate: bool @param iterations: The maximum number of iterations this task should run. If None, then there is no limit. @type iterations: int """ super(IntervalTask, self).__init__() self._callback = StoredCallback(callback) self._interval = interval self._immediate = immediate self._iterations = iterations if args is not None: self._args = args if kwargs is not None: self._kwargs = kwargs def __str__(self): return "%s: %s" % (self.__class__.__name__, self._callback) @property def last_run_time(self): return self._last_run_time if self.run_count else None @property def next_run_time(self): if self.paused: return None last = max(self.last_run_time, self._scheduled_time) return last + self._interval @property def active(self): return (not self.paused) and (self._looper is not None) def _schedule(self, interval=None): interval = max(0, interval if interval is not None else self._interval) self._scheduled_time = time.time() now = self._immediate and not self.run_count self._looper = LoopingCall(self._run) d = self._looper.start(interval, now=now) d.addErrback(self._errback) def kill(self): self._kill_looper() super(IntervalTask, self).kill() def _kill_looper(self): if self._looper is not None and self._looper.running: self._looper.stop() def pause(self): if not self.paused: last = self._last_run_time or self._scheduled_time or time.time() self._elapsed_at_pause = time.time() - last self.paused = True self._kill_looper() return True else: return False def unpause(self): if self.paused: del self.paused elapsed = self._elapsed_at_pause or 0 self._schedule(self._interval - elapsed) def _run(self): if not self.alive: self._kill_looper() return self.run_count += 1 self._last_run_time = time.time() #noinspection PyBroadException try: self._callback(*self._args, **self._kwargs) except: logging.error("Error in %s:\n%s" % (self, traceback.format_exc())) if self._iterations is not None and self.run_count >= self._iterations: self.kill() if self._elapsed_at_pause: del self._elapsed_at_pause self._kill_looper() self._schedule() def _errback(self, error): """ @type error: twisted.python.failure.Failure """ error.trap(Exception) if error.type == CancelledError: return tb = ''.join(traceback.format_exception(error.type, error.value, error.getTracebackObject())) logging.error("Error in %s:\n%s" % (self, tb)) def server_startup(self): if not self.alive: return if not self._paused_at_shutdown: self.unpause() try: del self._paused_at_shutdown except AttributeError: pass super(IntervalTask, self).server_startup() def server_shutdown(self): self._paused_at_shutdown = self.paused self.pause() super(IntervalTask, self).server_shutdown()
class BlobAvailabilityTracker(object): """ Class to track peer counts for known blobs, and to discover new popular blobs Attributes: availability (dict): dictionary of peers for known blobs """ def __init__(self, blob_manager, peer_finder, dht_node): self.availability = {} self._last_mean_availability = Decimal(0.0) self._blob_manager = blob_manager self._peer_finder = peer_finder self._dht_node = dht_node self._check_popular = LoopingCall(self._update_most_popular) self._check_mine = LoopingCall(self._update_mine) def start(self): log.info("Starting %s", self) self._check_popular.start(30) self._check_mine.start(600) def stop(self): log.info("Stopping %s", self) if self._check_popular.running: self._check_popular.stop() if self._check_mine.running: self._check_mine.stop() def get_blob_availability(self, blob): def _get_peer_count(peers): have_blob = sum(1 for peer in peers if peer.is_available()) return {blob: have_blob} d = self._peer_finder.find_peers_for_blob(blob) d.addCallback(_get_peer_count) return d def get_availability_for_blobs(self, blobs): dl = [self.get_blob_availability(blob) for blob in blobs if blob] d = defer.DeferredList(dl) d.addCallback( lambda results: [val for success, val in results if success]) return d @property def last_mean_availability(self): return max(Decimal(0.01), self._last_mean_availability) def _update_peers_for_blob(self, blob): def _save_peer_info(blob_hash, peers): v = {blob_hash: peers} self.availability.update(v) return v d = self._peer_finder.find_peers_for_blob(blob) d.addCallback( lambda r: [[c.host, c.port, c.is_available()] for c in r]) d.addCallback(lambda peers: _save_peer_info(blob, peers)) return d def _get_most_popular(self): dl = [] for (hash, _) in self._dht_node.get_most_popular_hashes(100): encoded = hash.encode('hex') dl.append(self._update_peers_for_blob(encoded)) return defer.DeferredList(dl) def _update_most_popular(self): d = self._get_most_popular() d.addCallback(lambda _: self._set_mean_peers()) def _update_mine(self): def _get_peers(blobs): dl = [] for hash in blobs: dl.append(self._update_peers_for_blob(hash)) return defer.DeferredList(dl) def sample(blobs): return random.sample(blobs, min(len(blobs), 100)) start = time.time() log.debug('==> Updating the peers for my blobs') d = self._blob_manager.get_all_verified_blobs() # as far as I can tell, this only is used to set _last_mean_availability # which... seems like a very expensive operation for such little payoff. # so taking a sample should get about the same effect as querying the entire # list of blobs d.addCallback(sample) d.addCallback(_get_peers) d.addCallback(lambda _: self._set_mean_peers()) d.addCallback(lambda _: log.debug( '<== Done updating peers for my blobs. Took %s seconds', time.time() - start)) # although unused, need to return or else the looping call # could overrun on a previous call return d def _set_mean_peers(self): num_peers = [ len(self.availability[blob]) for blob in self.availability ] mean = Decimal(sum(num_peers)) / Decimal(max(1, len(num_peers))) self._last_mean_availability = mean
class MonotoneSource(service.Service, util.ComparableMixin): """This source will poll a monotone server for changes and submit them to the change master. @param server_addr: monotone server specification (host:portno) @param branch: monotone branch to watch @param trusted_keys: list of keys whose code you trust @param db_path: path to monotone database to pull into @param pollinterval: interval in seconds between polls, defaults to 10 minutes @param monotone_exec: path to monotone executable, defaults to "monotone" """ __implements__ = IChangeSource, service.Service.__implements__ compare_attrs = [ "server_addr", "trusted_keys", "db_path", "pollinterval", "branch", "monotone_exec" ] parent = None # filled in when we're added done_revisions = [] last_revision = None loop = None d = None tmpfile = None monotone = None volatile = ["loop", "d", "tmpfile", "monotone"] def __init__(self, server_addr, branch, trusted_keys, db_path, pollinterval=60 * 10, monotone_exec="monotone"): self.server_addr = server_addr self.branch = branch self.trusted_keys = trusted_keys self.db_path = db_path self.pollinterval = pollinterval self.monotone_exec = monotone_exec self.monotone = Monotone(self.monotone_exec, self.db_path) def startService(self): self.loop = LoopingCall(self.start_poll) self.loop.start(self.pollinterval) service.Service.startService(self) def stopService(self): self.loop.stop() return service.Service.stopService(self) def describe(self): return "monotone_source %s %s" % (self.server_addr, self.branch) def start_poll(self): if self.d is not None: log.msg("last poll still in progress, skipping next poll") return log.msg("starting poll") self.d = self._maybe_init_db() self.d.addCallback(self._do_netsync) self.d.addCallback(self._get_changes) self.d.addErrback(self._handle_error) def _handle_error(self, failure): log.err(failure) self.d = None def _maybe_init_db(self): if not os.path.exists(self.db_path): log.msg("init'ing db") return self.monotone.db_init() else: log.msg("db already exists, migrating") return self.monotone.db_migrate() def _do_netsync(self, output): return self.monotone.pull(self.server_addr, self.branch) def _get_changes(self, output): d = self._get_new_head() d.addCallback(self._process_new_head) return d def _get_new_head(self): # This function returns a deferred that resolves to a good pick of new # head (or None if there is no good new head.) # First need to get all new heads... rcfile = """function get_revision_cert_trust(signers, id, name, val) local trusted_signers = { %s } local ts_table = {} for k, v in pairs(trusted_signers) do ts_table[v] = 1 end for k, v in pairs(signers) do if ts_table[v] then return true end end return false end """ trusted_list = ", ".join( ['"' + key + '"' for key in self.trusted_keys]) # mktemp is unsafe, but mkstemp is not 2.2 compatible. tmpfile_name = tempfile.mktemp() f = open(tmpfile_name, "w") f.write(rcfile % trusted_list) f.close() d = self.monotone.get_heads(self.branch, tmpfile_name) d.addCallback(self._find_new_head, tmpfile_name) return d def _find_new_head(self, new_heads, tmpfile_name): os.unlink(tmpfile_name) # Now get the old head's descendents... if self.last_revision is not None: d = self.monotone.descendents(self.last_revision) else: d = defer.succeed(new_heads) d.addCallback(self._pick_new_head, new_heads) return d def _pick_new_head(self, old_head_descendents, new_heads): for r in new_heads: if r in old_head_descendents: return r return None def _process_new_head(self, new_head): if new_head is None: log.msg("No new head") self.d = None return None # Okay, we have a new head; we need to get all the revisions since # then and create change objects for them. # Step 1: simplify set of processed revisions. d = self._simplify_revisions() # Step 2: get the list of new revisions d.addCallback(self._get_new_revisions, new_head) # Step 3: add a change for each d.addCallback(self._add_changes_for_revisions) # Step 4: all done d.addCallback(self._finish_changes, new_head) return d def _simplify_revisions(self): d = self.monotone.erase_ancestors(self.done_revisions) d.addCallback(self._reset_done_revisions) return d def _reset_done_revisions(self, new_done_revisions): self.done_revisions = new_done_revisions return None def _get_new_revisions(self, blah, new_head): if self.done_revisions: return self.monotone.ancestry_difference(new_head, self.done_revisions) else: # Don't force feed the builder with every change since the # beginning of time when it's first started up. return defer.succeed([new_head]) def _add_changes_for_revisions(self, revs): d = defer.succeed(None) for rid in revs: d.addCallback(self._add_change_for_revision, rid) return d def _add_change_for_revision(self, blah, rid): d = self.monotone.log(rid, 1) d.addCallback(self._add_change_from_log, rid) return d def _add_change_from_log(self, log, rid): d = self.monotone.get_revision(rid) d.addCallback(self._add_change_from_log_and_revision, log, rid) return d def _add_change_from_log_and_revision(self, revision, log, rid): # Stupid way to pull out everything inside quotes (which currently # uniquely identifies filenames inside a changeset). pieces = revision.split('"') files = [] for i in range(len(pieces)): if (i % 2) == 1: files.append(pieces[i]) # Also pull out author key and date author = "unknown author" pieces = log.split('\n') for p in pieces: if p.startswith("Author:"): author = p.split()[1] self.parent.addChange(Change(author, files, log, revision=rid)) def _finish_changes(self, blah, new_head): self.done_revisions.append(new_head) self.last_revision = new_head self.d = None
class AdapterPmMetrics: def __init__(self, device): self.pm_names = { 'tx_64_pkts', 'tx_65_127_pkts', 'tx_128_255_pkts', 'tx_256_511_pkts', 'tx_512_1023_pkts', 'tx_1024_1518_pkts', 'tx_1519_9k_pkts', 'rx_64_pkts', 'rx_65_127_pkts', 'rx_128_255_pkts', 'rx_256_511_pkts', 'rx_512_1023_pkts', 'rx_1024_1518_pkts', 'rx_1519_9k_pkts' } self.device = device self.id = device.id self.name = 'ponsim_onu' self.default_freq = 150 self.grouped = False self.freq_override = False self.pm_metrics = None self.pon_metrics_config = dict() self.uni_metrics_config = dict() self.lc = None for m in self.pm_names: self.pon_metrics_config[m] = PmConfig(name=m, type=PmConfig.COUNTER, enabled=True) self.uni_metrics_config[m] = PmConfig(name=m, type=PmConfig.COUNTER, enabled=True) def update(self, pm_config): if self.default_freq != pm_config.default_freq: # Update the callback to the new frequency. self.default_freq = pm_config.default_freq self.lc.stop() self.lc.start(interval=self.default_freq / 10) for m in pm_config.metrics: self.pon_metrics_config[m.name].enabled = m.enabled self.uni_metrics_config[m.name].enabled = m.enabled def make_proto(self): pm_config = PmConfigs(id=self.id, default_freq=self.default_freq, grouped=False, freq_override=False) for m in sorted(self.pon_metrics_config): pm = self.pon_metrics_config[m] # Either will do they're the same pm_config.metrics.extend( [PmConfig(name=pm.name, type=pm.type, enabled=pm.enabled)]) return pm_config def extract_metrics(self, stats): rtrn_port_metrics = dict() rtrn_port_metrics['pon'] = self.extract_pon_metrics(stats) rtrn_port_metrics['uni'] = self.extract_uni_metrics(stats) return rtrn_port_metrics def extract_pon_metrics(self, stats): rtrn_pon_metrics = dict() for m in stats.metrics: if m.port_name == "pon": for p in m.packets: if self.pon_metrics_config[p.name].enabled: rtrn_pon_metrics[p.name] = p.value return rtrn_pon_metrics def extract_uni_metrics(self, stats): rtrn_pon_metrics = dict() for m in stats.metrics: if m.port_name == "uni": for p in m.packets: if self.pon_metrics_config[p.name].enabled: rtrn_pon_metrics[p.name] = p.value return rtrn_pon_metrics def start_collector(self, callback): log.info("starting-pm-collection", device_name=self.name, device_id=self.device.id) prefix = 'voltha.{}.{}'.format(self.name, self.device.id) self.lc = LoopingCall(callback, self.device.id, prefix) self.lc.start(interval=self.default_freq / 10) def stop_collector(self): log.info("stopping-pm-collection", device_name=self.name, device_id=self.device.id) self.lc.stop()
class PahoMqttAdapter(BaseMqttAdapter, Service): # If connection fails at first, retry connecting each X seconds retry_interval = 5 def connect(self): """ Connect to MQTT broker. """ # TODO: Check if we can do asynchronous connection establishment. # Currently, this is done synchronously which could harm # other subsystems in timeout or otherwise blocking situations. # Make MQTT client identifier even more unique by adding process id pid = os.getpid() client_id = '{}:{}'.format(self.name, str(pid)) # Connection establishment self.client = mqtt.Client(client_id=client_id, clean_session=True) # Optionally authenticate connection if self.broker_username: self.client.username_pw_set(self.broker_username, self.broker_password) # Set event handlers self.client.on_connect = lambda *args: reactor.callFromThread( self.on_connect, *args) self.client.on_message = lambda *args: reactor.callFromThread( self.on_message, *args) self.client.on_log = lambda *args: reactor.callFromThread( self.on_log, *args) # Connect with retry self.connect_loop = LoopingCall(self.connect_with_retry) self.connect_loop.start(self.retry_interval, now=True) def connect_with_retry(self): try: self.client.connect(self.broker_host, port=self.broker_port, keepalive=60) self.connect_loop.stop() except: log.failure( u'Error connecting to MQTT broker but retrying each {retry_interval} seconds', retry_interval=self.retry_interval) return """ This is part of the threaded client interface. Call this once to start a new thread to process network traffic. This provides an alternative to repeatedly calling loop() yourself. """ self.client.loop_start() reactor.addSystemEventTrigger('before', 'shutdown', self.client.loop_stop, True) # The callback for when the client receives a CONNACK response from the server. def on_connect(self, client, userdata, flags, rc): """ on_connect(client, userdata, flags, rc): called when the broker responds to our connection request. flags is a dict that contains response flags from the broker: flags['session present'] - this flag is useful for clients that are using clean session set to 0 only. If a client with clean session=0, that reconnects to a broker that it has previously connected to, this flag indicates whether the broker still has the session information for the client. If 1, the session still exists. The value of rc determines success or not: 0: Connection successful 1: Connection refused - incorrect protocol version 2: Connection refused - invalid client identifier 3: Connection refused - server unavailable 4: Connection refused - bad username or password 5: Connection refused - not authorised 6-255: Currently unused. """ log.debug( "Connected to MQTT. userdata={userdata}, flags={flags}, rc={rc}", userdata=userdata, flags=flags, rc=rc) # Subscribing in on_connect() means that if we lose the connection and # reconnect then subscriptions will be renewed. #client.subscribe("$SYS/#") self.subscribe() # The callback for when a PUBLISH message is received from the server. def on_message(self, client, userdata, message): """ on_message(client, userdata, message): called when a message has been received on a topic that the client subscribes to. The message variable is a MQTTMessage that describes all of the message parameters. """ # TODO: Do something with "client" object (paho.mqtt.client.Client) topic = message.topic payload = message.payload # Make metadata dictionary to be passed as kwargs later # Mungle topic and payload out of metadata metadata = message.__dict__.copy() del metadata['topic'] del metadata['payload'] # Mungle userdata into message metadata['userdata'] = userdata if not topic.endswith('error.json'): log.debug( 'on_message: name={name}, topic={topic}, payload={payload}, kwargs={kwargs}', name=self.name, topic=topic, payload=payload, kwargs=metadata) return self.callback(topic=topic, payload=payload, **metadata) def publish(self, topic, payload): log.debug(u'Publishing to topic={topic}, payload={payload}', topic=topic, payload=payload) return self.client.publish(topic, payload) def subscribe(self, *args): #d = self.protocol.subscribe("foo/bar/baz", 0) log.info(u"Subscribing to topics {subscriptions}. client={client}", subscriptions=self.subscriptions, client=self.client) for topic in self.subscriptions: log.info(u"Subscribing to topic '{topic}'", topic=topic) # Topic name **must not** be unicode, so casting to string e = self.client.subscribe(str(topic), qos=0) def on_log(self, client, userdata, level, buf): """ on_log(client, userdata, level, buf): called when the client has log information. Define to allow debugging. The level variable gives the severity of the message and will be one of MQTT_LOG_INFO, MQTT_LOG_NOTICE, MQTT_LOG_WARNING, MQTT_LOG_ERR, and MQTT_LOG_DEBUG. The message itself is in buf. """ log.debug(u'{message}. level={level_mqtt}, userdata={userdata}', message=buf, level_mqtt=level, userdata=userdata)
class TinderboxPoller(base.ChangeSource): """This source will poll a tinderbox server for changes and submit them to the change master.""" compare_attrs = ["tinderboxURL", "pollInterval", "tree", "branch"] parent = None # filled in when we're added loop = None volatile = ['loop'] working = False def __init__(self, tinderboxURL, branch, tree="Firefox", machines=[], pollInterval=30): """ @type tinderboxURL: string @param tinderboxURL: The base URL of the Tinderbox server (ie. http://tinderbox.mozilla.org) @type tree: string @param tree: The tree to look for changes in. For example, Firefox trunk is 'Firefox' @type branch: string @param branch: The branch to look for changes in. This must match the 'branch' option for the Scheduler. @type machines: list @param machines: A list of machine names to search for. Changes will only register for machines that match individual "machine" substrings @type pollInterval: int @param pollInterval: The time (in seconds) between queries for changes """ self.tinderboxURL = tinderboxURL self.tree = tree self.branch = branch self.machines = machines self.pollInterval = pollInterval self.previousChange = '' self.lastPoll = time.time() self.lastChange = time.time() def startService(self): self.loop = LoopingCall(self.poll) base.ChangeSource.startService(self) reactor.callLater(0, self.loop.start, self.pollInterval) def stopService(self): self.loop.stop() return base.ChangeSource.stopService(self) def describe(self): str = "" str += "Getting changes from the Tinderbox service running at %s " \ % self.tinderboxURL str += "<br>Using tree: %s, branch %s, hostname %s" % (self.tree, self.branch, str(self.machines)) return str def poll(self): if self.working: log.msg("Not polling Tinderbox because last poll is still working") else: self.working = True d = self._get_changes() d.addCallback(self._process_changes) d.addBoth(self._finished) return def _finished(self, res): assert self.working self.working = False # check for failure if isinstance(res, failure.Failure): log.msg("Tinderbox poll failed: %s" % res) return res def _make_url(self): # build the tinderbox URL url = self.tinderboxURL url += "/" + self.tree url += "/" + "quickparse.txt" return url def _get_changes(self): url = self._make_url() log.msg("Polling Tinderbox tree at %s" % url) self.lastPoll = time.time() # get the page, in pipe-delimited format return defer.maybeDeferred(urlopen, url) def _process_changes(self, query): try: tp = TinderboxParser(query) buildList = tp.getData() except InvalidResultError, e: log.msg("Could not process Tinderbox query: " + e.value) return except EmptyResult: return
if __name__ == '__main__': if len(sys.argv) != 3: print 'GAMESERVER: Usage: python game_server <number of players> <PORT>' exit() # grab the number of players in this game player_count = int(sys.argv[1]) GAME_PORT = int(sys.argv[2]) player_range = range(player_count) # initialize gamespace and generate the player objects within game gs = GameSpace(player_range) # Listen for players to connect reactor.listenTCP(GAME_PORT, GameServerConnectionFactory(gs.players)) # initialize game loop lc = LoopingCall(gs.game_loop_iterate) lc.start(1.0 / 20) # begin reactor event loop reactor.run() # after reactor stops, end game loop lc.stop() # clean exit os._exit(0)
class TriggerBouncerCheck(Triggerable): compare_attrs = Triggerable.compare_attrs + \ ('minUptake', 'configRepo', 'checkMARs', 'username', 'password', 'pollInterval', 'pollTimeout') working = False loop = None release_config = None script_repo_revision = None configRepo = None def __init__(self, minUptake, configRepo, checkMARs=True, username=None, password=None, pollInterval=5*60, pollTimeout=12*60*60, **kwargs): self.minUptake = minUptake self.configRepo = configRepo self.checkMARs = checkMARs self.username = username self.password = password self.pollInterval = pollInterval self.pollTimeout = pollTimeout self.ss = None self.set_props = None Triggerable.__init__(self, **kwargs) def trigger(self, ss, set_props=None): self.ss = ss self.set_props = set_props props = Properties() props.updateFromProperties(self.properties) if set_props: props.updateFromProperties(set_props) self.script_repo_revision = props.getProperty('script_repo_revision') assert self.script_repo_revision, 'script_repo_revision should be set' self.release_config = props.getProperty('release_config') assert self.release_config, 'release_config should be set' def _run_loop(_): self.loop = LoopingCall(self.poll) reactor.callLater(0, self.loop.start, self.pollInterval) reactor.callLater(self.pollTimeout, self.stopLoop, 'Timeout after %s' % self.pollTimeout) d = self.getReleaseConfig() d.addCallback(_run_loop) def stopLoop(self, reason=None): if reason: log.msg('%s: Stopping uptake monitoring: %s' % (self.__class__.__name__, reason)) if self.loop.running: self.loop.stop() else: log.msg('%s: Loop has been alredy stopped' % self.__class__.__name__) def getReleaseConfig(self): url = str('%s/raw-file/%s/%s' % (self.configRepo, self.script_repo_revision, self.release_config)) d = getPage(url) def setReleaseConfig(res): c = {} exec res in c self.release_config = c.get('releaseConfig') log.msg('%s: release_config loaded' % self.__class__.__name__) d.addCallback(setReleaseConfig) return d def poll(self): if self.working: log.msg('%s: Not polling because last poll is still working' % self.__class__.__name__) return defer.succeed(None) self.working = True log.msg('%s: polling' % self.__class__.__name__) bouncerProductName = self.release_config.get('bouncerProductName') or \ self.release_config.get('productName').capitalize() d = get_release_uptake( tuxedoServerUrl=self.release_config.get('tuxedoServerUrl'), bouncerProductName=bouncerProductName, version=self.release_config.get('version'), platforms=self.release_config.get('enUSPlatforms'), oldVersion=self.release_config.get('oldVersion'), checkMARs=self.checkMARs, username=self.username, password=self.password) d.addCallback(self.checkUptake) d.addCallbacks(self.finished_ok, self.finished_failure) return d def checkUptake(self, uptake): log.msg('%s: uptake is %s' % (self.__class__.__name__, uptake)) if uptake >= self.minUptake: self.stopLoop('Reached required uptake: %s' % uptake) Triggerable.trigger(self, self.ss, self.set_props) def finished_ok(self, res): log.msg('%s: polling finished' % (self.__class__.__name__)) assert self.working self.working = False return res def finished_failure(self, f): log.msg('%s failed:\n%s' % (self.__class__.__name__, f.getTraceback())) assert self.working self.working = False return None # eat the failure
class triviabot(irc.IRCClient): ''' This is the irc bot portion of the trivia bot. It implements the whole program so is kinda big. The algorithm is implemented by a series of callbacks, initiated by an admin on the server. ''' def __init__(self): self._answer = Answer() self._question = '' self._scores = {} self._clue_number = 0 self._admins = list(config.ADMINS) self._game_channel = config.GAME_CHANNEL self._current_points = 5 self._questions_dir = config.Q_DIR self._lc = LoopingCall(self._play_game) self._load_game() self._votes = 0 self._voters = [] def _get_nickname(self): return self.factory.nickname nickname = property(_get_nickname) def _get_lineRate(self): return self.factory.lineRate lineRate = property(_get_lineRate) def _cmsg(self, dest, msg): """ Write a colorized message. """ self.msg(dest, "%s%s" % (config.COLOR_CODE, msg)) def _gmsg(self, msg): """ Write a message to the channel playing the trivia game. """ self._cmsg(self._game_channel, msg) def _play_game(self): ''' Implements the main loop of the game. ''' points = {0: 5, 1: 3, 2: 2, 3: 1} if self._clue_number == 0: self._votes = 0 self._voters = [] self._get_new_question() self._current_points = points[self._clue_number] # Blank line. self._gmsg("") self._gmsg("Next question:") self._gmsg(self._question) self._gmsg("Clue: %s" % self._answer.current_clue()) self._clue_number += 1 # we must be somewhere in between elif self._clue_number < 4: self._current_points = points[self._clue_number] self._gmsg("Question:") self._gmsg(self._question) self._gmsg('Clue: %s' % self._answer.give_clue()) self._clue_number += 1 # no one must have gotten it. else: self._gmsg('No one got it. The answer was: %s' % self._answer.answer) self._clue_number = 0 self._get_new_question() #self._lc.reset() def signedOn(self): ''' Actions to perform on signon to the server. ''' self.join(self._game_channel) self.msg('NickServ', 'identify %s' % config.IDENT_STRING) print("Signed on as %s." % (self.nickname, )) if self.factory.running: self._start(None, None, None) else: self._gmsg('Welcome to %s!' % self._game_channel) self._gmsg("Have an admin start the game when you are ready.") self._gmsg("For how to use this bot, just say ?help or") self._gmsg("%s help." % self.nickname) def joined(self, channel): ''' Callback runs when the bot joins a channel ''' print("Joined %s." % (channel, )) def privmsg(self, user, channel, msg): ''' Parses out each message and initiates doing the right thing with it. ''' user, temp = user.split('!') print(user + " : " + channel + " : " + msg) # need to strip off colors if present. try: while not msg[0].isalnum() and not msg[0] == '?': msg = msg[1:] except IndexError: return # parses each incoming line, and sees if it's a command for the bot. try: if (msg[0] == "?"): command = msg.replace('?', '').split()[0] args = msg.replace('?', '').split()[1:] self.select_command(command, args, user, channel) return elif (msg.split()[0].find(self.nickname) == 0): command = msg.split()[1] args = msg.replace(self.nickname, '').split()[2:] self.select_command(command, args, user, channel) return # if not, try to match the message to the answer. else: if msg.lower().strip() == self._answer.answer.lower(): self._winner(user, channel) self._save_game() except: return def _winner(self, user, channel): ''' Congratulates the winner for guessing correctly and assigns points appropriately, then signals that it was guessed. ''' if channel != self._game_channel: self.msg(channel, "I'm sorry, answers must be given in the game channel.") return self._gmsg("%s GOT IT!" % user.upper()) try: self._scores[user] += self._current_points except: self._scores[user] = self._current_points if self._current_points == 1: self._gmsg("%s point has been added to your score!" % str(self._current_points)) else: self._gmsg("%s points has been added to your score!" % str(self._current_points)) self._clue_number = 0 self._get_new_question() def ctcpQuery(self, user, channel, msg): ''' Responds to ctcp requests. Currently just reports them. ''' print("CTCP recieved: " + user + ":" + channel + ": " + msg[0][0] + " " + msg[0][1]) def _help(self, args, user, channel): ''' Tells people how to use the bot. Replies differently if you are an admin or a regular user. Only responds to the user since there could be a game in progress. ''' try: self._admins.index(user) except: self._cmsg(user, "I'm nameless's trivia bot.") self._cmsg( user, "Commands: score, standings, giveclue, help, " "next, source") return self._cmsg(user, "I'm nameless's trivia bot.") self._cmsg( user, "Commands: score, standings, giveclue, help, next, " "skip, source") self._cmsg("Admin commands: die, set <user> <score>, start, stop, " "save") def _show_source(self, args, user, channel): ''' Tells people how to use the bot. Only responds to the user since there could be a game in progress. ''' self._cmsg( user, 'My source can be found at: ' 'https://github.com/rawsonj/triviabot') def select_command(self, command, args, user, channel): ''' Callback that responds to commands given to the bot. Need to differentiate between priviledged users and regular users. ''' # set up command dicts. unpriviledged_commands = { 'score': self._score, 'help': self._help, 'source': self._show_source, 'standings': self._standings, 'giveclue': self._give_clue, 'next': self._next_vote, 'skip': self._next_question } priviledged_commands = { 'die': self._die, 'set': self._set_user_score, 'start': self._start, 'stop': self._stop, 'save': self._save_game, } print command, args, user, channel try: self._admins.index(user) is_admin = True except: is_admin = False # the following takes care of sorting out functions and # priviledges. if not is_admin and priviledged_commands.has_key(command): self.msg(channel, "%s: You don't tell me what to do." % user) return elif is_admin and priviledged_commands.has_key(command): priviledged_commands[command](args, user, channel) elif unpriviledged_commands.has_key(command): unpriviledged_commands[command](args, user, channel) else: self.describe(channel, '%slooks at %s oddly.' % (config.COLOR_CODE, user)) def _next_vote(self, args, user, channel): '''Implements user voting for the next question. Need to keep track of who voted, and how many votes. ''' if not self._lc.running: self._gmsg("We aren't playing right now.") return try: self._voters.index(user) self._gmsg("You already voted, %s, give someone else a chance to " "hate this question" % user) return except: if self._votes < 2: self._votes += 1 self._voters.append(user) print self._voters self._gmsg("%s, you have voted. %s more votes needed to " "skip." % (user, str(3 - self._votes))) else: self._votes = 0 self._voters = [] self._next_question(None, None, None) def _start(self, args, user, channel): ''' Starts the trivia game. TODO: Load scores from last game, if any. ''' if self._lc.running: return else: self._lc.start(config.WAIT_INTERVAL) self.factory.running = True def _stop(self, *args): ''' Stops the game and thanks people for playing, then saves the scores. ''' if not self._lc.running: return else: self._lc.stop() self._gmsg('Thanks for playing trivia!') self._gmsg('Current rankings were:') self._standings(None, self._game_channel, None) self._gmsg('''Scores have been saved, and see you next game!''') self._save_game() self.factory.running = False def _save_game(self, *args): ''' Saves the game to the data directory. ''' if not path.exists(config.SAVE_DIR): makedirs(config.SAVE_DIR) with open(config.SAVE_DIR + 'scores.json', 'w') as savefile: json.dump(self._scores, savefile) print "Scores have been saved." def _load_game(self): ''' Loads the running data from previous games. ''' # ensure initialization self._scores = {} if not path.exists(config.SAVE_DIR): print "Save directory doesn't exist." return try: with open(config.SAVE_DIR + 'scores.json', 'r') as savefile: temp_dict = json.load(savefile) except: print "Save file doesn't exist." return for name in temp_dict.keys(): self._scores[str(name)] = int(temp_dict[name]) print self._scores print "Scores loaded." def _set_user_score(self, args, user, channel): ''' Administrative action taken to adjust scores, if needed. ''' try: self._scores[args[0]] = int(args[1]) except: self._cmsg(user, args[0] + " not in scores database.") return self._cmsg(user, args[0] + " score set to " + args[1]) def _die(self, *args): ''' Terminates execution of the bot. Need to dig into twisted to figure out how this happens. ''' global reactor self.quit(message='This is triviabot, signing off.') reactor.stop() # figure out how to kill the bot def _score(self, args, user, channel): ''' Tells the user their score. ''' try: self._cmsg(user, "Your current score is: %s" % str(self._scores[user])) except: self._cmsg(user, "You aren't in my database.") def _next_question(self, args, user, channel): ''' Administratively skips the current question. ''' if not self._lc.running: self._gmsg("We are not playing right now.") return self._gmsg("Question has been skipped. The answer was: %s" % self._answer.answer) self._clue_number = 0 self._lc.stop() self._lc.start(config.WAIT_INTERVAL) def _standings(self, args, user, channel): ''' Tells the user the complete standings in the game. TODO: order them. ''' self._cmsg(user, "The current trivia standings are: ") sorted_scores = sorted(self._scores.iteritems(), key=lambda (k, v): (v, k), reverse=True) for rank, (player, score) in enumerate(sorted_scores, start=1): formatted_score = "%s: %s: %s" % (rank, player, score) self._cmsg(user, str(formatted_score)) def _give_clue(self, args, user, channel): if not self._lc.running: self._gmsg("we are not playing right now.") return self._cmsg(channel, "Question: ") self._cmsg(channel, self._question) self._cmsg(channel, "Clue: " + self._answer.current_clue()) def _get_new_question(self): ''' Selects a new question from the questions directory and sets it. ''' damaged_question = True while damaged_question: #randomly select file filename = choice(listdir(self._questions_dir)) fd = open(config.Q_DIR + filename) lines = fd.read().splitlines() myline = choice(lines) fd.close() try: self._question, temp_answer = myline.split('`') except ValueError: print "Broken question:" print myline continue self._answer.set_answer(temp_answer.strip()) damaged_question = False
class IRCUser(IRCBase): def __init__(self, ircd, ip, uuid=None, host=None): self.ircd = ircd self.uuid = ircd.createUUID() if uuid is None else uuid registrationTimeout = self.ircd.config.get("user_registration_timeout", 10) self.nick = None self.ident = None if ip[0] == ":": # Normalize IPv6 address for IRC ip = "0{}".format(ip) if host is None: self.realHost = ip else: self.realHost = host self.ip = ip self._hostStack = [] self._hostsByType = {} self.gecos = None self._metadata = CaseInsensitiveDictionary() self.cache = {} self.channels = [] self.modes = {} self.connectedSince = now() self.nickSince = now() self.idleSince = now() self._registerHolds = set(("connection", "dns", "NICK", "USER")) self.disconnectedDeferred = Deferred() self._messageBatches = {} self._errorBatchName = None self._errorBatch = [] self.ircd.users[self.uuid] = self self.localOnly = False self.secureConnection = False self._pinger = LoopingCall(self._ping) self._registrationTimeoutTimer = reactor.callLater( registrationTimeout, self._timeoutRegistration) self._connectHandlerTimer = None self._startDNSResolving(registrationTimeout) def _startDNSResolving(self, timeout): ip = self.ip if ipIsV4(ip): addr = "{}.in-addr.arpa".format(".".join(reversed(ip.split(".")))) else: addr = reversed(expandIPv6Address(ip).replace(":", "")) addr = "{}.ip6.arpa".format(".".join(addr)) resolveDeferred = dnsClient.lookupPointer(addr, ((timeout / 2), )) resolveDeferred.addCallbacks(callback=self._verifyDNSResolution, callbackArgs=(timeout, ), errback=self._cancelDNSResolution) def _verifyDNSResolution(self, result, timeout): name = result[0][0].payload.name.name if len(name) > self.ircd.config.get("hostname_length", 64): self._cancelDNSResolution() return if not isValidHost(name): self._cancelDNSResolution() return resolveDeferred = dnsClient.getHostByName(name, ((timeout / 2), )) resolveDeferred.addCallbacks(callback=self._completeDNSResolution, errback=self._cancelDNSResolution, callbackArgs=(name, )) def _completeDNSResolution(self, result, name): if result == self.ip: self.realHost = name self.register("dns") def _cancelDNSResolution(self, error=None): self.register("dns") def connectionMade(self): # We need to callLater the connect action call because the connection isn't fully set up yet, # nor is it fully set up even with a delay of zero, which causes the message buffer not to be sent # when the connection is closed. # The "connection" register hold is used basically solely for the purposes of this to prevent potential # race conditions with registration. self._connectHandlerTimer = reactor.callLater(0.1, self._callConnectAction) if ISSLTransport.providedBy(self.transport): self.secureConnection = True def _callConnectAction(self): self._connectHandlerTimer = None if self.ircd.runActionUntilFalse("userconnect", self, users=[self]): self.transport.loseConnection() else: self.register("connection") def dataReceived(self, data): self.ircd.runActionStandard("userrecvdata", self, data, users=[self]) try: IRCBase.dataReceived(self, data) except Exception: self.ircd.log.failure( "An error occurred while processing incoming data.") if self.uuid in self.ircd.users: self.disconnect("Error occurred") def sendLine(self, line): self.ircd.runActionStandard("usersenddata", self, line, users=[self]) IRCBase.sendLine(self, line) def sendMessage(self, command, *args, **kw): """ Sends the given message to this user. Accepts the following keyword arguments: - prefix: The message prefix or None to suppress the default prefix If not given, defaults to the server name. - to: The destination of the message or None if the message has no destination. The implicit destination is this user if this argument isn't specified. - tags: Dict of message tags to send. - alwaysPrefixLastParam: For compatibility with some broken clients, you might want some messages to always have the last parameter prefixed with a colon. To do that, pass this as True. """ if "prefix" not in kw: kw["prefix"] = self.ircd.name if kw["prefix"] is None: del kw["prefix"] to = self.nick if self.nick else "*" if "to" in kw: to = kw["to"] del kw["to"] if to: args = [to] + list(args) self.ircd.runActionStandard("modifyoutgoingmessage", self, command, args, kw) IRCBase.sendMessage(self, command, *args, **kw) def handleCommand(self, command, params, prefix, tags): if self.uuid not in self.ircd.users: return # we have been disconnected - ignore all further commands if command in self.ircd.userCommands: handlers = self.ircd.userCommands[command] if not handlers: return data = None spewRegWarning = True affectedUsers = [] affectedChannels = [] for handler in handlers: if handler[0].forRegistered is not None: if (handler[0].forRegistered is True and not self.isRegistered()) or ( handler[0].forRegistered is False and self.isRegistered()): continue spewRegWarning = False data = handler[0].parseParams(self, params, prefix, tags) if data is not None: affectedUsers = handler[0].affectedUsers(self, data) affectedChannels = handler[0].affectedChannels(self, data) if self not in affectedUsers: affectedUsers.append(self) break if data is None: if spewRegWarning: if self.isRegistered(): self.sendMessage(irc.ERR_ALREADYREGISTERED, "You may not reregister") else: self.sendMessage(irc.ERR_NOTREGISTERED, command, "You have not registered") elif self._hasBatchedErrors(): self._dispatchErrorBatch() return self._clearErrorBatch() if self.ircd.runComboActionUntilValue( (("commandpermission-{}".format(command), self, data), ("commandpermission", self, command, data)), users=affectedUsers, channels=affectedChannels) is False: return self.ircd.runComboActionStandard( (("commandmodify-{}".format(command), self, data), ("commandmodify", self, command, data)), users=affectedUsers, channels=affectedChannels ) # This allows us to do processing without the "stop on empty" feature of runActionProcessing for handler in handlers: if handler[0].execute(self, data): if handler[0].resetsIdleTime: self.idleSince = now() break # If the command executor returns True, it was handled else: return # Don't process commandextra if it wasn't handled self.ircd.runComboActionStandard( (("commandextra-{}".format(command), self, data), ("commandextra", self, command, data)), users=affectedUsers, channels=affectedChannels) else: if not self.ircd.runActionFlagTrue("commandunknown", self, command, params, {}): self.sendMessage(irc.ERR_UNKNOWNCOMMAND, command, "Unknown command") def createMessageBatch(self, batchName, batchType, batchParameters=None): """ Start a new message batch with the given batch name, type, and list of parameters. If a batch with the given name already exists, that batch will be overwritten. """ self._messageBatches[batchName] = { "type": batchType, "parameters": batchParameters, "messages": [] } def sendMessageInBatch(self, batchName, command, *args, **kw): """ Adds a message to the batch with the given name. """ if batchName not in self._messageBatches: return self._messageBatches[batchName]["messages"].append((command, args, kw)) def sendBatch(self, batchName): """ Sends the messages in the given batch to the user. """ if batchName not in self._messageBatches: return batchType = self._messageBatches[batchName]["type"] batchParameters = self._messageBatches[batchName]["parameters"] self.ircd.runActionStandard("startbatchsend", self, batchName, batchType, batchParameters) for messageData in self._messageBatches[batchName]["messages"]: self.sendMessage(messageData[0], *messageData[1], **messageData[2]) self.ircd.runActionStandard("endbatchsend", self, batchName, batchType, batchParameters) def startErrorBatch(self, batchName): """ Used to start an error batch when sending multiple error messages to a user from a command's parseParams or from the commandpermission action. """ if not self._errorBatchName or not self._errorBatch: # Only the first batch should apply self._errorBatchName = batchName def sendBatchedError(self, batchName, command, *args, **kw): """ Adds an error to the current error batch if the specified error batch is the current error batch. """ if batchName and self._errorBatchName == batchName: self._errorBatch.append((command, args, kw)) def sendSingleError(self, batchName, command, *args, **kw): """ Creates a batch containing a single error and adds the specified error to it. """ if not self._errorBatchName: self._errorBatchName = batchName self._errorBatch.append((command, args, kw)) def _hasBatchedErrors(self): if self._errorBatch: return True return False def _clearErrorBatch(self): self._errorBatchName = None self._errorBatch = [] def _dispatchErrorBatch(self): for error in self._errorBatch: self.sendMessage(error[0], *error[1], **error[2]) self._clearErrorBatch() def filterConditionalTags(self, conditionalTags): applyTags = {} for tag, data in conditionalTags.iteritems(): value, check = data if check(self): applyTags[tag] = value return applyTags def connectionLost(self, reason): if self.uuid in self.ircd.users: self.disconnect("Connection reset") self.disconnectedDeferred.callback(None) def disconnect(self, reason): """ Disconnects the user from the server. """ self.ircd.log.debug( "Disconnecting user {user.uuid} ({user.hostmask()}): {reason}", user=self, reason=reason) # Sometimes, actions deferred from initial connection may cause registration to occur after disconnection if # disconnection happens before registration completes. If the user is unregistered on disconnection, this prevents # the user from completing registration. self.addRegisterHold("QUIT") if self._pinger: if self._pinger.running: self._pinger.stop() self._pinger = None if self._registrationTimeoutTimer: if self._registrationTimeoutTimer.active(): self._registrationTimeoutTimer.cancel() self._registrationTimeoutTimer = None if self._connectHandlerTimer and self._connectHandlerTimer.active(): self._connectHandlerTimer.cancel() self._connectHandlerTimer = None self.ircd.recentlyQuitUsers[self.uuid] = now() del self.ircd.users[self.uuid] if self.isRegistered(): del self.ircd.userNicks[self.nick] userSendList = [self] while self.channels: channel = self.channels[0] userSendList.extend(channel.users.keys()) self._leaveChannel(channel) userSendList = [ u for u in set(userSendList) if u.uuid[:3] == self.ircd.serverID ] userSendList.remove(self) self.ircd.runActionProcessing("quitmessage", userSendList, self, reason, users=[self] + userSendList) self.ircd.runActionStandard("quit", self, reason, users=self) self.transport.loseConnection() def _timeoutRegistration(self): if self.isRegistered(): self._pinger.start(self.ircd.config.get("user_ping_frequency", 60), False) return self.disconnect("Registration timeout") def _ping(self): self.ircd.runActionStandard("pinguser", self) def isRegistered(self): """ Returns True if this user session is fully registered. """ return not self._registerHolds def register(self, holdName): """ Removes the specified hold on a user's registration. If this is the last hold on a user, completes registration on the user. """ if holdName not in self._registerHolds: return self._registerHolds.remove(holdName) if not self._registerHolds: if not self.nick or self.nick in self.ircd.userNicks: self._registerHolds.add("NICK") if not self.ident or not self.gecos: self._registerHolds.add("USER") if self._registerHolds: return self._registerHolds.add( "registercheck" ) # The user shouldn't be considered registered until we complete these final checks if self.ircd.runActionUntilFalse("register", self, users=[self]): self.transport.loseConnection() return self._registerHolds.remove("registercheck") self.ircd.userNicks[self.nick] = self.uuid self.ircd.log.debug( "Registering user {user.uuid} ({user.hostmask()})", user=self) versionWithName = "txircd-{}".format(version) self.sendMessage( irc.RPL_WELCOME, "Welcome to the {} Internet Relay Chat Network {}".format( self.ircd.config["network_name"], self.hostmask())) self.sendMessage( irc.RPL_YOURHOST, "Your host is {}, running version {}".format( self.ircd.name, versionWithName)) self.sendMessage( irc.RPL_CREATED, "This server was created {}".format( self.ircd.startupTime.replace(microsecond=0))) chanModes = "".join( ["".join(modes.keys()) for modes in self.ircd.channelModes]) chanModes += "".join(self.ircd.channelStatuses.keys()) self.sendMessage( irc.RPL_MYINFO, self.ircd.name, versionWithName, "".join( ["".join(modes.keys()) for modes in self.ircd.userModes]), chanModes) self.sendISupport() self.ircd.runActionStandard("welcome", self, users=[self]) def addRegisterHold(self, holdName): """ Adds a register hold to this user if the user is not yet registered. """ if not self._registerHolds: return self._registerHolds.add(holdName) def sendISupport(self): """ Sends ISUPPORT to this user.""" isupportList = self.ircd.generateISupportList() isupportMsgList = splitMessage(" ".join(isupportList), 350) for line in isupportMsgList: lineArgs = line.split(" ") lineArgs.append("are supported by this server") self.sendMessage(irc.RPL_ISUPPORT, *lineArgs) def hostmask(self): """ Returns the user's hostmask. """ return "{}!{}@{}".format(self.nick, self.ident, self.host()) def hostmaskWithRealHost(self): """ Returns the user's hostmask using the user's real host rather than any vhost that may have been applied. """ return "{}!{}@{}".format(self.nick, self.ident, self.realHost) def hostmaskWithIP(self): """ Returns the user's hostmask using the user's IP address instead of the host. """ return "{}!{}@{}".format(self.nick, self.ident, self.ip) def changeNick(self, newNick, fromServer=None): """ Changes this user's nickname. If initiated by a remote server, that server should be specified in the fromServer parameter. """ if newNick == self.nick: return if newNick in self.ircd.userNicks and self.ircd.userNicks[ newNick] != self.uuid: return oldNick = self.nick if oldNick and oldNick in self.ircd.userNicks: del self.ircd.userNicks[self.nick] self.nick = newNick self.nickSince = now() if self.isRegistered(): self.ircd.userNicks[self.nick] = self.uuid userSendList = [self] for channel in self.channels: userSendList.extend(channel.users.keys()) userSendList = [ u for u in set(userSendList) if u.uuid[:3] == self.ircd.serverID ] self.ircd.runActionProcessing("changenickmessage", userSendList, self, oldNick, users=userSendList) self.ircd.runActionStandard("changenick", self, oldNick, fromServer, users=[self]) def changeIdent(self, newIdent, fromServer=None): """ Changes this user's ident. If initiated by a remote server, that server should be specified in the fromServer parameter. """ if newIdent == self.ident: return if len(newIdent) > self.ircd.config.get("ident_length", 12): return oldIdent = self.ident self.ident = newIdent if self.isRegistered(): self.ircd.runActionStandard("changeident", self, oldIdent, fromServer, users=[self]) def host(self): if not self._hostStack: return self.realHost return self._hostsByType[self._hostStack[-1]] def changeHost(self, hostType, newHost, fromServer=None): """ Changes a user's host. If initiated by a remote server, that server should be specified in the fromServer parameter. """ if hostType == "*": return if len(newHost) > self.ircd.config.get("hostname_length", 64): return if hostType in self._hostsByType and self._hostsByType[ hostType] == newHost: return oldHost = self.host() self._hostsByType[hostType] = newHost if hostType in self._hostStack: self._hostStack.remove(hostType) self._hostStack.append(hostType) if self.isRegistered(): self.ircd.runComboActionStandard( (("changehost", self, hostType, oldHost, fromServer), ("updatehost", self, hostType, oldHost, newHost, fromServer)), users=[self]) def updateHost(self, hostType, newHost, fromServer=None): """ Updates the host of a given host type for the user. If initiated by a remote server, that server should be specified in the fromServer parameter. """ if hostType not in self._hostStack: self.changeHost(hostType, newHost, fromServer) return if hostType == "*": return if len(newHost) > self.ircd.config.get("hostname_length", 64): return if hostType in self._hostsByType and self._hostsByType[ hostType] == newHost: return oldHost = self.host() oldHostOfType = None if hostType in self._hostsByType: oldHostOfType = self._hostsByType[hostType] self._hostsByType[hostType] = newHost changedUserHost = (oldHost != self.host()) changedHostOfType = (oldHostOfType != newHost) if self.isRegistered(): if changedUserHost and changedHostOfType: self.ircd.runComboActionStandard( (("changehost", self, hostType, oldHost, fromServer), ("updatehost", self, hostType, oldHost, newHost, fromServer)), users=[self]) elif changedHostOfType: self.ircd.runActionStandard("updatehost", self, hostType, oldHost, newHost, fromServer, users=[self]) def resetHost(self, hostType, fromServer=None): """ Resets the user's host to the real host. """ if hostType not in self._hostsByType: return oldHost = self.host() if hostType in self._hostStack: self._hostStack.remove(hostType) del self._hostsByType[hostType] currentHost = self.host() if currentHost != oldHost: self.ircd.runComboActionStandard( (("changehost", self, hostType, oldHost, fromServer), ("updatehost", self, hostType, oldHost, None, fromServer)), users=[self]) else: self.ircd.runActionStandard("updatehost", self, hostType, oldHost, None, fromServer, users=[self]) def currentHostType(self): if self._hostStack: return self._hostStack[-1] return "*" def changeGecos(self, newGecos, fromServer=None): """ Changes a user's real name. If initiated by a remote server, that server should be specified in the fromServer parameter. """ if len(newGecos) > self.ircd.config.get("gecos_length", 128): return if newGecos == self.gecos: return oldGecos = self.gecos self.gecos = newGecos if self.isRegistered(): self.ircd.runActionStandard("changegecos", self, oldGecos, fromServer, users=[self]) def metadataKeyExists(self, key): """ Checks whether the specified key exists in the user's metadata. """ return key in self._metadata def metadataKeyCase(self, key): """ Returns the specified key in the user's metadata in its original case. Returns None if the given key is not in the user's metadata. """ if key not in self._metadata: return None return self._metadata[key][0] def metadataValue(self, key): """ Returns the value of the given key in the user's metadata or None if the given key is not in the user's metadata. """ if key not in self._metadata: return None return self._metadata[key][1] def metadataVisibility(self, key): """ Returns the visibility value of the given key in the user's metadata or None if the given key is not in the user's metadata. """ if key not in self._metadata: return None return self._metadata[key][2] def metadataSetByUser(self, key): """ Returns whether the given key in the user's metadata was set by a user or None if the given key is not in the user's metadata. """ if key not in self._metadata: return None return self._metadata[key][3] def metadataList(self): """ Returns the list of metadata keys/values for the user as a list of tuples in the format [ (key, value, visibility, setByUser) ] """ return self._metadata.values() def setMetadata(self, key, value, visibility, setByUser, fromServer=None): """ Sets metadata for the user. If initiated by a remote server, that server should be specified in the fromServer parameter. If the value is None, deletes the metadata at the provided key. """ if not isValidMetadataKey(key): return False oldData = None if key in self._metadata: oldData = self._metadata[key] if setByUser and oldData and not oldData[3]: return False if setByUser and self.ircd.runActionUntilValue( "usercansetmetadata", key, users=[self]) is False: return False if value is None: if key in self._metadata: del self._metadata[key] elif not visibility: return False else: self._metadata[key] = (key, value, visibility, setByUser) oldValue = oldData[1] if oldData else None self.ircd.runActionStandard("usermetadataupdate", self, key, oldValue, value, visibility, setByUser, fromServer, users=[self]) return True def canSeeMetadataVisibility(self, visibility): if visibility == "*": return True return self.ircd.runActionUntilValue("usercanseemetadata", self, visibility) is not False def joinChannel(self, channel, override=False): """ Joins the user to a channel. Specify the override parameter only if all permission checks should be bypassed. """ if channel in self.channels: return if not override: if self.ircd.runActionUntilValue("joinpermission", channel, self, users=[self], channels=[channel]) is False: return channel.users[self] = {"status": ""} self.channels.append(channel) newChannel = False if channel.name not in self.ircd.channels: newChannel = True self.ircd.channels[channel.name] = channel self.ircd.recentlyDestroyedChannels[channel.name] = False # We need to send the JOIN message before doing other processing, as chancreate will do things like # mode defaulting, which will send messages about the channel before the JOIN message, which is bad. messageUsers = [ u for u in channel.users.iterkeys() if u.uuid[:3] == self.ircd.serverID ] self.ircd.runActionProcessing("joinmessage", messageUsers, channel, self, users=messageUsers, channels=[channel]) if newChannel: self.ircd.runActionStandard("channelcreate", channel, self, channels=[channel]) self.ircd.runActionStandard("join", channel, self, users=[self], channels=[channel]) def leaveChannel(self, channel, partType="PART", typeData={}, fromServer=None): """ Removes the user from a channel. The partType and typeData are used for the leavemessage action to send the parting message. If the channel leaving is initiated by a remote server, that server should be specified in the fromServer parameter. """ if channel not in self.channels: return messageUsers = [ u for u in channel.users.iterkeys() if u.uuid[:3] == self.ircd.serverID ] self.ircd.runActionProcessing("leavemessage", messageUsers, channel, self, partType, typeData, fromServer, users=[self], channels=[channel]) self._leaveChannel(channel) def _leaveChannel(self, channel): self.ircd.runActionStandard("leave", channel, self, users=[self], channels=[channel]) self.channels.remove(channel) del channel.users[self] def setModes(self, modes, defaultSource): """ Sets modes on the user. Accepts modes as a list of tuples in the format: [ (adding, mode, param, setBy, setTime) ] - adding: True if we're setting the mode; False if unsetting - mode: The mode letter - param: The mode's parameter; None if no parameter is needed for that mode - setBy: Optional, only used for list modes; a human-readable string (typically server name or nick!user@host) for who/what set this mode) - setTime: Optional, only used for list modes; a datetime object containing when the mode was set The defaultSource is a valid user ID or server ID of someone who set the modes. It is used as the source for announcements about the mode change and as the default setter for any list modes who do not have the setBy parameter specified. The default time for list modes with no setTime specified is now(). """ modeChanges = [] defaultSourceName = self._sourceName(defaultSource) if defaultSourceName is None: raise ValueError("Source must be a valid user or server ID.") nowTime = now() for modeData in modes: mode = modeData[1] if mode not in self.ircd.userModeTypes: continue setBy = defaultSourceName setTime = nowTime modeType = self.ircd.userModeTypes[mode] adding = modeData[0] if modeType in (ModeType.List, ModeType.ParamOnUnset, ModeType.Param): param = modeData[2] else: param = None if modeType == ModeType.List: dataCount = len(modeData) if dataCount >= 4: setBy = modeData[3] if dataCount >= 5: setTime = modeData[4] if adding: paramList = self.ircd.userModes[modeType][mode].checkSet( self, param) else: paramList = self.ircd.userModes[modeType][mode].checkUnset( self, param) if paramList is None: continue for parameter in paramList: if self._applyMode(adding, modeType, mode, parameter, setBy, setTime): modeChanges.append( (adding, mode, parameter, setBy, setTime)) self._notifyModeChanges(modeChanges, defaultSource, defaultSourceName) return modeChanges def setModesByUser(self, user, modes, params, override=False): """ Parses a mode string specified by a user and sets those modes on the user. The user parameter should be the user who set the modes (usually, but not always, this user). The modes parameter is the actual modes string; parameters specified by the user should be as a list of strings in params. The override parameter should be used only when all permission checks should be overridden. """ adding = True changes = [] setBy = self._sourceName(user.uuid) setTime = now() for mode in modes: if len(changes) >= self.ircd.config.get("modes_per_line", 20): break if mode == "+": adding = True continue if mode == "-": adding = False continue if mode not in self.ircd.userModeTypes: user.sendMessage(irc.ERR_UNKNOWNMODE, mode, "is unknown mode char to me") continue modeType = self.ircd.userModeTypes[mode] param = None if modeType in (ModeType.List, ModeType.ParamOnUnset) or ( adding and modeType == ModeType.Param): try: param = params.pop(0) except IndexError: if modeType == ModeType.List: self.ircd.userModes[modeType][mode].showListParams( user, self) continue if adding: paramList = self.ircd.userModes[modeType][mode].checkSet( self, param) else: paramList = self.ircd.userModes[modeType][mode].checkUnset( self, param) if paramList is None: continue for parameter in paramList: if len(changes) >= self.ircd.config.get("modes_per_line", 20): break if not override and self.ircd.runActionUntilValue( "modepermission-user-{}".format(mode), self, user, adding, parameter, users=[self, user]) is False: continue if adding: if modeType == ModeType.List: if mode in self.modes and len( self.modes[mode]) > self.ircd.config.get( "user_listmode_limit", 128): user.sendMessage( irc.ERR_BANLISTFULL, self.name, parameter, "Channel +{} list is full".format(mode)) continue if self._applyMode(adding, modeType, mode, parameter, setBy, setTime): changes.append((adding, mode, parameter, setBy, setTime)) self._notifyModeChanges(changes, user.uuid, setBy) return changes def _applyMode(self, adding, modeType, mode, parameter, setBy, setTime): if parameter: if len(parameter) > 255: return False if " " in parameter: return False if adding: if modeType == ModeType.List: if mode in self.modes: if len(self.modes[mode]) > self.ircd.config.get( "user_listmode_limit", 128): return False for paramData in self.modes[mode]: if parameter == paramData[0]: return False else: self.modes[mode] = [] self.modes[mode].append((parameter, setBy, setTime)) return True if mode in self.modes and self.modes[mode] == parameter: return False self.modes[mode] = parameter return True if modeType == ModeType.List: if mode not in self.modes: return False for index, paramData in enumerate(self.modes[mode]): if paramData[0] == parameter: del self.modes[mode][index] break else: return False if not self.modes[mode]: del self.modes[mode] return True if mode not in self.modes: return False if modeType == ModeType.ParamOnUnset and parameter != self.modes[mode]: return False del self.modes[mode] return True def _notifyModeChanges(self, modeChanges, source, sourceName): if not modeChanges: return for change in modeChanges: self.ircd.runActionStandard("modechange-user-{}".format(change[1]), self, change[3], change[0], change[2], users=[self]) users = [] if source in self.ircd.users and source[:3] == self.ircd.serverID: users.append(self.ircd.users[source]) if self.uuid[:3] == self.ircd.serverID: users.append(self) if users: self.ircd.runActionProcessing("modemessage-user", users, self, source, sourceName, modeChanges, users=users) self.ircd.runActionStandard("modechanges-user", self, source, sourceName, modeChanges, users=[self]) def _sourceName(self, source): if source in self.ircd.users: return self.ircd.users[source].hostmask() if source == self.ircd.serverID: return self.ircd.name if source in self.ircd.servers: return self.ircd.servers[source].name return None def modeString(self, toUser): """ Get a user-reportable mode string for the modes set on the user. """ modeStr = ["+"] params = [] for mode in self.modes: modeType = self.ircd.userModeTypes[mode] if modeType not in (ModeType.ParamOnUnset, ModeType.Param, ModeType.NoParam): continue if modeType != ModeType.NoParam: param = None if toUser: param = self.ircd.userModes[modeType][mode].showParam( toUser, self) if not param: param = self.modes[mode] else: param = None modeStr.append(mode) if param: params.append(param) if params: return "{} {}".format("".join(modeStr), " ".join(params)) return "".join(modeStr)
class MBDBChangeSource(base.ChangeSource): debug = True def __init__(self, pollInterval=30, branch='default'): #base.ChangeSource.__init__(self) self.pollInterval = pollInterval self.latest = None self.branch, created = \ Branch.objects.get_or_create(name=branch) def startService(self): self.loop = LoopingCall(self.poll) base.ChangeSource.startService(self) reactor.callLater(0, self.loop.start, self.pollInterval) def stopService(self): self.loop.stop() return base.ChangeSource.stopService(self) @transaction.commit_on_success def poll(self): '''Check for new pushes. Hack around transactions on innodb, make this transaction aware and transaction.commit() to get a new transaction for our queries. ''' transaction.commit() if self.latest is None: try: self.latest = Push.objects.order_by('-pk')[0].id except IndexError: self.latest = 0 return new_pushes = Push.objects.filter(pk__gt=self.latest).order_by('pk') if self.debug: log.msg('mbdb changesource found %d pushes after %d' % (new_pushes.count(), self.latest)) push = None for push in new_pushes: self.submitChangesForPush(push) if push is not None: self.latest = push.id def submitChangesForPush(self, push): if self.debug: log.msg('submitChangesForPush called') repo = push.repository if repo.forest is not None: branch = repo.forest.name.encode('utf-8') locale = repo.name[len(branch) + 1:].encode('utf-8') else: branch = repo.name.encode('utf-8') for cs in push.changesets.filter( branch=self.branch).order_by('pk'): when = timegm(push.push_date.utctimetuple()) +\ push.push_date.microsecond/1000.0/1000 c = changes.Change(who=push.user.encode('utf-8'), files=map( lambda u: u.encode('utf-8'), cs.files.values_list('path', flat=True)), revision=cs.revision.encode('utf-8'), comments=cs.description.encode('utf-8'), when=when, branch=branch) if repo.forest is not None: # locale change c.locale = locale self.parent.addChange(c) def replay(self, builder, startPush=None, startTime=None, endTime=None): bm = self.parent.parent.botmaster qd = {} if startTime is not None: qd['push_date__gte'] = startTime if endTime is not None: qd['push_date__lte'] = endTime if startPush is not None: qd['id__gte'] = startPush q = Push.objects.filter(**qd).order_by('push_date') i = q.iterator() if self.debug: log.msg('replay called for %d pushes' % q.count()) def next(_cb): try: p = i.next() except StopIteration: log.msg("done iterating") return self.submitChangesForPush(p) def stumble(): bm.waitUntilBuilderIdle(builder).addCallback(_cb, _cb) reactor.callLater(.5, stumble) def cb(res, _cb): reactor.callLater(.5, next, _cb) next(cb) def describe(self): return str(self) def __str__(self): return "MBDBChangeSource"
def run_server(opt): global gp10block global gp40block store = {} if opt['gp10']['enable'] is True: gp10block = Gp10ModbusSlave(opt['gp10']['deviceid'], **opt['gp10']) store[opt['gp10']['deviceid']] = gp10block.store if opt['gp40']['enable'] is True: gp40block = Gp40ModbusSlave(opt['gp40']['deviceid'], **opt['gp40']) store[opt['gp40']['deviceid']] = gp40block.store if 0 == len(store): print("ERR: enable GP10 or GP40.") return context = ModbusServerContext(slaves=store, single=False) # ----------------------------------------------------------------------- # # initialize the server information # ----------------------------------------------------------------------- # # If you don't set this or any fields, they are defaulted to empty strings. # ----------------------------------------------------------------------- # identity = ModbusDeviceIdentification() identity.VendorName = 'RatocSystems, Inc.' identity.ProductCode = 'RPi-Modbus' identity.VendorUrl = 'https://github.com/ratocsystems' identity.ProductName = 'RasPi Modbus Server' identity.ModelName = 'RasPi Modbus Server' identity.MajorMinorRevision = '1.0' # ----------------------------------------------------------------------- # # updater # ----------------------------------------------------------------------- # updater_500ms = LoopingCall(f=write_context_500ms, a=(context,)) updater_500ms.start(.5) updater_1000ms = LoopingCall(f=write_context_1000ms, a=(context,)) updater_1000ms.start(1) writer = LoopingCall(f=read_context, a=(context,)) writer.start(.1) # ----------------------------------------------------------------------- # # run the server you want # ----------------------------------------------------------------------- # if 'tcp' == opt['common']['protocol']: # Tcp: StartTcpServer(context, identity=identity, address=(opt['tcp']['host'], opt['tcp']['port'])) elif 'ascii' == opt['common']['protocol']: # Ascii: StartSerialServer(context, identity=identity, port=opt['serial']['device'], timeout=1, baudrate=opt['serial']['baudrate']) elif 'rtu' == opt['common']['protocol']: # RTU: StartSerialServer(context, framer=ModbusRtuFramer, identity=identity, port=opt['serial']['device'], timeout=.005, baudrate=opt['serial']['baudrate']) else: print("ERR: select protocol tcp, rtu or ascii") # ----------------------------------------------------------------------- # # end proc # ----------------------------------------------------------------------- # updater_500ms.stop() updater_1000ms.stop() writer.stop() if gp10block is not None: del gp10block if gp40block is not None: del gp40block
class Core(CorePluginBase): def enable(self): self.config = deluge.configmanager.ConfigManager( "centeragent.conf", DEFAULT_PREFS) self.session = component.get("Core").session self.alertmanager = component.get("AlertManager") # connect to Center via HTTP self.connect(self.config["IP"], self.config["Port"]) # polling timer every 5 min self.polling_timer = LoopingCall(self.polling) self.polling_timer.start(300) def disable(self): self.polling_timer.stop() pass def update(self): pass @export def set_config(self, config): """Sets the config dictionary""" for key in config.keys(): self.config[key] = config[key] self.config.save() @export def get_config(self): """Returns the config dictionary""" return self.config.config def connect(self, ip, port): log.info("Connecting...") self.url = "http://" + ip + ":" + str(port) response = urllib2.urlopen(self.url + "/register/") r = json.loads(response.read()) if r['result'] == 'OK': log.info("Connect to Center SUCCESS") return True return False def fn(self, handle): if not handle.is_valid(): return None status = handle.status() if status.is_finished: return None if not status.has_metadata: return None return handle def report(self): log.info("Report") jobs = len(filter(self.fn, self.session.get_torrents())) response = urllib2.urlopen(self.url + "/report/?" + urllib.urlencode({'jobs': jobs})) pass def on_save_resume_data_alert_factory(self, _save_path): save_path = _save_path def on_save_resume_data_alert(alert): # send to Center with save_path # TODO pass return on_save_resume_data_alert def add_torrent(self, torrent_resume_data, save_path): pass def migrate(self): log.info("Send job") # check if need to send a job to another while True: response = urllib2.urlopen(self.url + "/check/") r = json.loads(response) if r['result'] == "NO": break jobs = filter(self.fn, self.session.get_torrents()) if len(jobs) <= 0: break torrent = jobs[0] torrent.pause() torrent.flush_cache() torrent.save_resume_data(3) save_path = torrent.status().save_path self.alertmanager.register_handler( "save_resume_data_alert", self.on_save_resume_data_alert_factory(save_path)) log.info("Recv job") # check if need to recv job response = urllib2.urlopen(self.url + "/recv/") r = json.loads(response) if r['result'] == "NO": return for a, b in r['jobs']: self.add_torrent(a, b) pass def polling(self): log.info("Polling") if self.url: self.report() self.migrate()
class Tunnel(object): def __init__(self, options, ipv8_port=-1, ipv8_address="0.0.0.0"): self.options = options self.should_run = True self.ipv8_port = ipv8_port self.ipv8_address = ipv8_address self.session = None self.community = None self.clean_messages_lc = LoopingCall(self.clean_messages) self.clean_messages_lc.start(1800) self.clean_messages_lc = LoopingCall(self.periodic_bootstrap) self.clean_messages_lc.start(30, now=False) def clean_messages(self): clean_twisted_observers() def periodic_bootstrap(self): self.session.lm.tunnel_community.bootstrap() def tribler_started(self): new_strategies = [] with self.session.lm.ipv8.overlay_lock: for strategy, target_peers in self.session.lm.ipv8.strategies: if strategy.overlay == self.session.lm.tunnel_community: new_strategies.append((strategy, -1)) else: new_strategies.append((strategy, target_peers)) self.session.lm.ipv8.strategies = new_strategies def circuit_removed(self, _, __, ___, address): self.session.lm.ipv8.network.remove_by_address(address) self.session.lm.tunnel_community.bootstrap() def start(self): # Determine socks5 ports socks5_port = self.options['socks5'] if "HELPER_INDEX" in os.environ and "HELPER_BASE" in os.environ: base_port = int(os.environ["HELPER_BASE"]) socks5_port = base_port + int(os.environ["HELPER_INDEX"]) * 5 if socks5_port is not None: socks_listen_ports = range(socks5_port, socks5_port + 5) else: socks_listen_ports = [ random.randint(1000, 65535) for _ in range(5) ] config = TriblerConfig() config.set_state_dir( os.path.join(config.get_state_dir(), "tunnel-%d") % socks_listen_ports[0]) config.set_tunnel_community_socks5_listen_ports(socks_listen_ports) config.set_torrent_checking_enabled(False) config.set_megacache_enabled(False) config.set_dispersy_enabled(False) config.set_ipv8_enabled(True) config.set_mainline_dht_enabled(True) config.set_torrent_collecting_enabled(False) config.set_libtorrent_enabled(False) config.set_video_server_enabled(False) config.set_dispersy_port(self.ipv8_port) config.set_ipv8_address(self.ipv8_address) config.set_torrent_search_enabled(False) config.set_channel_search_enabled(False) config.set_trustchain_enabled(True) config.set_credit_mining_enabled(False) config.set_market_community_enabled(False) config.set_mainline_dht_enabled(False) config.set_tunnel_community_exitnode_enabled(bool( self.options["exit"])) config.set_trustchain_testnet(bool(self.options["testnet"])) if self.options["restapi"] is not None: config.set_http_api_enabled(True) config.set_http_api_port(self.options["restapi"]) if "ipv8_bootstrap_override" in self.options: config.set_ipv8_bootstrap_override( self.options["ipv8_bootstrap_override"]) self.session = Session(config) logger.info("Using IPv8 port %d" % self.session.config.get_dispersy_port()) self.session.notifier.add_observer(self.circuit_removed, NTFY_TUNNEL, [NTFY_REMOVE]) return self.session.start().addCallback(self.tribler_started) def stop(self): if self.clean_messages_lc: self.clean_messages_lc.stop() self.clean_messages_lc = None if self.session: logger.info("Going to shutdown session") return self.session.shutdown()
class datehandler(AbstractHandler): midnight = time(00, 00, 00) def updatedate(self): now = datetime.now().replace(microsecond=0) self._tags['date'] = now.strftime("%d.%m.%Y %H:%M:%S") self.checktags(now) if now.time() == self.midnight: self._tags['sunset'] = self.getsunset().strftime("%H:%M:%S") self._tags['sunrise'] = self.getsunrise().strftime("%H:%M:%S") if now.time().strftime("%H:%M:%S") == self._tags['sunset']: self._settag('issunset', '1') else: self._settag('issunset', '0') if now.time().strftime("%H:%M:%S") == self._tags['sunrise']: self._settag('issunrise', '1') else: self._settag('issunrise', '0') def loadtags(self): self._tags['sunset'] = self.getsunset().strftime("%H:%M:%S") self._tags['sunrise'] = self.getsunrise().strftime("%H:%M:%S") self._tags['issunset'] = '0' self._tags['issunrise'] = '0' self.rrules = {} for tag, params in self.config.items(): startdate = parse(params["start"]) enddate = parse(params["end"]) untildate = parse(params["until"]) freq = eval(params['freq']) startrr = rrule(freq, dtstart=startdate, until=untildate, cache=True) endrr = rrule(freq, dtstart=enddate, until=untildate, cache=True) self._tags[tag] = '0' tagrrule = self.rrules.setdefault(tag, {}) tagrrule["startrr"] = startrr tagrrule["endrr"] = endrr def earthtool(self, param): now = datetime.now() xml = urllib2.urlopen("http://www.earthtools.org/sun/54.32/36.16/" + str(now.day) + "/" + str(now.month) + "/+4/0").read() xmldoc = minidom.parseString(xml) itemlist = xmldoc.getElementsByTagName(param) return parse(itemlist[0].childNodes[0].nodeValue).time() def getsunset(self): sunset = time(hour=20, minute=00, second=0, microsecond=0) try: sunset = self.earthtool('sunset') except: self.logger.error("Cant get sunset", exc_info=1) return sunset def getsunrise(self): sunrise = time(hour=8, minute=00, second=0, microsecond=0) try: sunrise = self.earthtool('sunrise') except: self.logger.error("Cant get sunrise", exc_info=1) return sunrise def checktag(self, tag, dt): dt = dt.replace(second=0, microsecond=0) if dt in self.rrules[tag]["startrr"]: self._settag(tag, '1') elif dt in self.rrules[tag]["endrr"]: self._settag(tag, '0') def checktags(self, dt): for tag in self.rrules: self.checktag(tag, dt) def start(self): AbstractHandler.start(self) self.lc = LoopingCall(self.updatedate) self.lc.start(1) def stop(self): AbstractHandler.stop(self) if self.lc: self.lc.stop()
class Fluid(object): """ Fluid simulator. """ implements(IAutomaton, IDigHook) sponge = None """ Block that will soak up fluids and springs that are near it. Defaults to None, which effectively disables this feature. """ def __init__(self, factory): self.factory = factory self.sponges = Block3DSpatialDict() self.springs = Block2DSpatialDict() self.tracked = set() self.new = set() self.loop = LoopingCall(self.process) def start(self): if not self.loop.running: self.loop.start(self.step) def stop(self): if self.loop.running: self.loop.stop() def schedule(self): if self.tracked: self.start() else: self.stop() @property def blocks(self): retval = [self.spring, self.fluid] if self.sponge: retval.append(self.sponge) return retval def feed(self, coordinates): """ Accept the coordinates and stash them for later processing. """ self.tracked.add(coordinates) self.schedule() scan = naive_scan def update_fluid(self, w, coords, falling, level=0): if not 0 <= coords[1] < 256: return False block = w.sync_get_block(coords) if (block in self.whitespace and not any(self.sponges.iteritemsnear(coords, 2))): w.sync_set_block(coords, self.fluid) if falling: level |= FALLING w.sync_set_metadata(coords, level) self.new.add(coords) return True return False def add_sponge(self, w, x, y, z): # Track this sponge. self.sponges[x, y, z] = True # Destroy the water! Destroy! for coords in itercube(x, y, z, 2): try: target = w.sync_get_block(coords) if target == self.spring: if (coords[0], coords[2]) in self.springs: del self.springs[coords[0], coords[2]] w.sync_destroy(coords) elif target == self.fluid: w.sync_destroy(coords) except ChunkNotLoaded: pass # And now mark our surroundings so that they can be # updated appropriately. for coords in itercube(x, y, z, 3): if coords != (x, y, z): self.new.add(coords) def add_spring(self, w, x, y, z): # Double-check that we weren't placed inside a sponge. That's just # not going to work out. if any(self.sponges.iteritemsnear((x, y, z), 2)): w.sync_destroy((x, y, z)) return # Track this spring. self.springs[x, z] = y # Neighbors on the xz-level. neighbors = ((x - 1, y, z), (x + 1, y, z), (x, y, z - 1), (x, y, z + 1)) # Spawn water from springs. for coords in neighbors: try: self.update_fluid(w, coords, False) except ChunkNotLoaded: pass # Is this water falling down to the next y-level? We don't really # care, but we'll run the update nonetheless. if y > 0: # Our downstairs pal. below = x, y - 1, z self.update_fluid(w, below, True) def add_fluid(self, w, x, y, z): # Neighbors on the xz-level. neighbors = ((x - 1, y, z), (x + 1, y, z), (x, y, z - 1), (x, y, z + 1)) # Our downstairs pal. below = (x, y - 1, z) # Double-check that we weren't placed inside a sponge. if any(self.sponges.iteritemsnear((x, y, z), 2)): w.sync_destroy((x, y, z)) return # First, figure out whether or not we should be spreading. Let's see # if there are any springs nearby which are above us and thus able to # fuel us. if not any(springy >= y for springy in self.springs.itervaluesnear((x, z), self.levels + 1)): # Oh noes, we're drying up! We should mark our neighbors and dry # ourselves up. self.new.update(neighbors) if y: self.new.add(below) w.sync_destroy((x, y, z)) return newmd = self.levels + 1 for coords in neighbors: try: jones = w.sync_get_block(coords) if jones == self.spring: newmd = 0 self.new.update(neighbors) break elif jones == self.fluid: jonesmd = w.sync_get_metadata(coords) & ~FALLING if jonesmd + 1 < newmd: newmd = jonesmd + 1 except ChunkNotLoaded: pass current_md = w.sync_get_metadata((x,y,z)) if newmd > self.levels and current_md < FALLING: # We should dry up. self.new.update(neighbors) if y: self.new.add(below) w.sync_destroy((x, y, z)) return # Mark any neighbors which should adjust themselves. This will only # mark lower water levels than ourselves, and only if they are # definitely too low. for coords in neighbors: try: neighbor = w.sync_get_metadata(coords) if neighbor & ~FALLING > newmd + 1: self.new.add(coords) except ChunkNotLoaded: pass # Now, it's time to extend water. Remember, either the water flows # downward to the next y-level, or it flows out across the xz-level, # but *not* both. # Fall down to the next y-level, if possible. if y and self.update_fluid(w, below, True, newmd): return # Clamp our newmd and assign. Also, set ourselves again; we changed # this time and we might change again. if current_md < FALLING: w.sync_set_metadata((x, y, z), newmd) # If pending block is already above fluid, don't keep spreading. if neighbor == self.fluid: return # Otherwise, just fill our neighbors with water, where applicable, and # mark them. if newmd < self.levels: newmd += 1 for coords in neighbors: try: self.update_fluid(w, coords, False, newmd) except ChunkNotLoaded: pass def remove_sponge(self, x, y, z): # The evil sponge tyrant is gone. Flow, minions, flow! for coords in itercube(x, y, z, 3): if coords != (x, y, z): self.new.add(coords) def remove_spring(self, x, y, z): # Neighbors on the xz-level. neighbors = ((x - 1, y, z), (x + 1, y, z), (x, y, z - 1), (x, y, z + 1)) # Destroyed spring. Add neighbors and below to blocks to update. del self.springs[x, z] self.new.update(neighbors) if y: # Our downstairs pal. below = x, y - 1, z self.new.add(below) def process(self): w = self.factory.world for x, y, z in self.tracked: # Try each block separately. If it can't be done, it'll be # discarded from the set simply by not being added to the new set # for the next iteration. try: block = w.sync_get_block((x, y, z)) if block == self.sponge: self.add_sponge(w, x, y, z) elif block == self.spring: self.add_spring(w, x, y, z) elif block == self.fluid: self.add_fluid(w, x, y, z) else: # Hm, why would a pending block not be any of the things # we care about? Maybe it used to be a spring or # something? if (x, z) in self.springs: self.remove_spring(x, y, z) elif (x, y, z) in self.sponges: self.remove_sponge(x, y, z) except ChunkNotLoaded: pass # Flush affected chunks. to_flush = set() for x, y, z in chain(self.tracked, self.new): to_flush.add((x // 16, z // 16)) for x, z in to_flush: d = self.factory.world.request_chunk(x, z) d.addCallback(self.factory.flush_chunk) self.tracked = self.new self.new = set() # Prune, and reschedule. self.schedule() @inlineCallbacks def dig_hook(self, chunk, x, y, z, block): """ Check for neighboring water that might want to spread. Also check to see whether we are, for example, dug ice that has turned back into water. """ x += chunk.x * 16 z += chunk.z * 16 # Check for sponges first, since they will mark the entirety of the # area. if block == self.sponge: for coords in itercube(x, y, z, 3): self.tracked.add(coords) else: for coords in iterneighbors(x, y, z): test_block = yield self.factory.world.get_block(coords) if test_block in (self.spring, self.fluid): self.tracked.add(coords) self.schedule() before = ("build",) after = tuple()
class FamilyMember(automat.Automat): """ This class implements all the functionality of ``family_member()`` state machine. """ timers = { 'timer-10sec': (10.0, ['SUPPLIERS']), } def __init__(self, customer_idurl, debug_level=_DebugLevel, log_events=_Debug, log_transitions=_Debug, publish_events=False, **kwargs): """ Builds `family_member()` state machine. """ self.customer_idurl = customer_idurl self.supplier_idurl = my_id.getLocalIDURL() super(FamilyMember, self).__init__(name="family_member_%s_%s" % ( nameurl.GetName(self.customer_idurl), nameurl.GetName(my_id.getLocalIDURL()), ), state="AT_STARTUP", debug_level=debug_level, log_events=log_events, log_transitions=log_transitions, publish_events=publish_events, **kwargs) def state_changed(self, oldstate, newstate, event, *args, **kwargs): """ Method to catch the moment when `family_member()` state were changed. """ if event != 'instant' and newstate in [ 'CONNECTED', 'DISCONNECTED', ]: self.automat('instant') def state_not_changed(self, curstate, event, *args, **kwargs): """ This method intended to catch the moment when some event was fired in the `family_member()` but automat state was not changed. """ if event != 'instant' and curstate in [ 'CONNECTED', 'DISCONNECTED', ]: self.automat('instant') def A(self, event, *args, **kwargs): """ The state machine code, generated using `visio2python <http://bitdust.io/visio2python/>`_ tool. """ #---AT_STARTUP--- if self.state == 'AT_STARTUP': if event == 'init': self.state = 'DISCONNECTED' self.doInit(*args, **kwargs) #---DISCONNECTED--- elif self.state == 'DISCONNECTED': if event == 'shutdown': self.state = 'CLOSED' self.doDestroyMe(*args, **kwargs) elif event == 'instant' and self.isAnyRequests(*args, **kwargs): self.state = 'DHT_READ' self.Attempts = 0 self.doPull(*args, **kwargs) self.doDHTRead(*args, **kwargs) elif event == 'family-refresh' or event == 'family-join' or event == 'family-leave': self.doPush(event, *args, **kwargs) elif event == 'contacts-received': self.doCheckReply(*args, **kwargs) #---DHT_READ--- elif self.state == 'DHT_READ': if event == 'family-refresh' or event == 'family-join' or event == 'family-leave': self.doPush(event, *args, **kwargs) elif event == 'contacts-received': self.doCheckReply(*args, **kwargs) elif event == 'dht-read-fail': self.state = 'DISCONNECTED' self.Attempts = 0 self.doNotifyDisconnected(*args, **kwargs) elif event == 'dht-value-exist' and self.isMyPositionOK( *args, **kwargs) and not self.isLeaving(*args, **kwargs): self.state = 'CONNECTED' self.Attempts = 0 self.doNotifyConnected(*args, **kwargs) elif (event == 'dht-value-exist' and self.isMyPositionOK(*args, **kwargs) and self.isLeaving(*args, **kwargs)) or event == 'shutdown': self.state = 'CLOSED' self.doDestroyMe(*args, **kwargs) elif event == 'dht-value-not-exist' or ( event == 'dht-value-exist' and not self.isMyPositionOK(*args, **kwargs)): self.state = 'SUPPLIERS' self.Attempts += 1 self.doRebuildFamily(*args, **kwargs) self.doRequestSuppliersReview(*args, **kwargs) #---SUPPLIERS--- elif self.state == 'SUPPLIERS': if event == 'shutdown': self.state = 'CLOSED' self.doDestroyMe(*args, **kwargs) elif event == 'family-refresh' or event == 'family-join' or event == 'family-leave': self.doPush(event, *args, **kwargs) elif event == 'contacts-received': self.doCheckReply(*args, **kwargs) elif (event == 'all-suppliers-agree' or event == 'timer-10sec') and not self.isFamilyModified( *args, **kwargs): self.state = 'CONNECTED' self.Attempts = 0 self.doNotifyConnected(*args, **kwargs) elif (event == 'timer-10sec' or event == 'all-suppliers-agree') and self.isFamilyModified( *args, **kwargs): self.state = 'DHT_WRITE' self.doDHTWrite(*args, **kwargs) elif event == 'one-supplier-not-agree': self.doSolveConflict(*args, **kwargs) self.doRequestSuppliersReview(*args, **kwargs) #---DHT_WRITE--- elif self.state == 'DHT_WRITE': if event == 'shutdown': self.state = 'CLOSED' self.doDestroyMe(*args, **kwargs) elif event == 'family-refresh' or event == 'family-join' or event == 'family-leave': self.doPush(event, *args, **kwargs) elif event == 'contacts-received': self.doCheckReply(*args, **kwargs) elif event == 'dht-write-fail' and self.Attempts > 3: self.state = 'DISCONNECTED' self.Attempts = 0 self.doNotifyDisconnected(*args, **kwargs) elif event == 'dht-write-fail' and self.Attempts <= 3: self.state = 'DHT_READ' self.doDHTRead(*args, **kwargs) elif event == 'dht-write-ok': self.state = 'CONNECTED' self.Attempts = 0 self.doNotifyConnected(*args, **kwargs) #---CLOSED--- elif self.state == 'CLOSED': pass #---CONNECTED--- elif self.state == 'CONNECTED': if event == 'shutdown': self.state = 'CLOSED' self.doDestroyMe(*args, **kwargs) elif event == 'disconnect': self.state = 'DISCONNECTED' self.doNotifyDisconnected(*args, **kwargs) elif event == 'instant' and self.isAnyRequests(*args, **kwargs): self.state = 'DHT_READ' self.Attempts = 0 self.doPull(*args, **kwargs) self.doDHTRead(*args, **kwargs) elif event == 'family-refresh' or event == 'family-join' or event == 'family-leave': self.doPush(event, *args, **kwargs) elif event == 'contacts-received': self.doCheckReply(*args, **kwargs) return None def isAnyRequests(self, *args, **kwargs): """ Condition method. """ return len(self.requests) > 0 def isLeaving(self, *args, **kwargs): """ Condition method. """ return self.current_request and self.current_request[ 'command'] == 'family-leave' def isMyPositionOK(self, *args, **kwargs): """ Condition method. """ dht_info_valid = self._do_validate_dht_info(args[0]) if not dht_info_valid: return False if self.current_request and self.current_request[ 'command'] == 'family-leave': if my_id.getLocalIDURL() not in dht_info_valid['suppliers']: return True my_info_valid = self._do_validate_my_info(self.my_info) if not my_info_valid: return False latest_revision = self._do_detect_latest_revision( dht_info_valid, my_info_valid) if latest_revision == 0: return False try: my_position = my_info_valid['suppliers'].index( my_id.getLocalIDURL()) except: my_position = -1 if my_position < 0: return False try: existing_position = dht_info_valid['suppliers'].index( my_id.getLocalIDURL()) except: existing_position = -1 return existing_position > 0 and my_position > 0 and existing_position == my_position def isFamilyModified(self, *args, **kwargs): """ Condition method. """ return self.transaction is not None def doInit(self, *args, **kwargs): """ Action method. """ self.requests = [] self.current_request = None self.dht_info = None self.my_info = None self.transaction = None self.refresh_task = LoopingCall(self._on_family_refresh_task) def doPush(self, event, *args, **kwargs): """ Action method. """ if event not in _ValidRequests: raise Exception('Invalid request: %r' % args) request = (args[0] if args else {}) or {} request['command'] = event self.requests.append(request) def doPull(self, *args, **kwargs): """ Action method. """ self.current_request = self.requests.pop(0) def doRebuildFamily(self, *args, **kwargs): """ Action method. """ dht_info_valid = self._do_validate_dht_info(args[0]) my_info_valid = self._do_validate_my_info(self.my_info) latest_revision = self._do_detect_latest_revision( dht_info_valid, my_info_valid) merged_info = None if latest_revision > 0: merged_info = self._do_merge_revisions(dht_info_valid, my_info_valid, latest_revision) if not merged_info: merged_info = self._do_create_first_revision(self.current_request) # if not merged_info: # lg.err('failed to merge customer family info after reading from DHT, skip transaction') # self.transaction = None # return possible_transaction = self._do_process_request( merged_info, self.current_request) if not possible_transaction: lg.err( 'failed to process customer family change request, skip transaction' ) return self.transaction = self._do_increment_revision(possible_transaction) if _Debug: lg.out( _DebugLevel, 'family_member._do_build_transaction : %r' % self.transaction) self.refresh_period = DHT_RECORD_REFRESH_INTERVAL * settings.DefaultDesiredSuppliers( ) if self.transaction: known_ecc_map = self.transaction.get('ecc_map') if known_ecc_map: expected_suppliers_count = eccmap.GetEccMapSuppliersNumber( known_ecc_map) self.refresh_period = DHT_RECORD_REFRESH_INTERVAL * expected_suppliers_count def doRequestSuppliersReview(self, *args, **kwargs): """ Action method. """ if not self.transaction: self.automat('all-suppliers-agree') return self.suppliers_requests = [] for supplier_idurl in self.transaction['suppliers']: if not supplier_idurl: continue if supplier_idurl == my_id.getLocalIDURL(): continue outpacket = p2p_service.SendContacts( remote_idurl=supplier_idurl, json_payload={ 'space': 'family_member', 'type': 'suppliers_list', 'customer_idurl': self.customer_idurl, 'customer_ecc_map': self.transaction['ecc_map'], 'transaction_revision': self.transaction['revision'], 'suppliers_list': self.transaction['suppliers'], }, callbacks={ commands.Ack(): self._on_supplier_ack, commands.Fail(): self._on_supplier_fail, }, ) self.suppliers_requests.append(outpacket.PacketID) if not self.suppliers_requests: self.automat('all-suppliers-agree') else: if _Debug: lg.out( _DebugLevel, 'family_member.doRequestSuppliersReview sent to transaction for review to %d suppliers' % len(self.suppliers_requests)) def doSolveConflict(self, *args, **kwargs): """ Action method. """ # TODO: take in account ecc_map while solving the conflict # ecc_map = kwargs.get('ecc_map') suppliers_list = kwargs.get('suppliers_list') another_supplier_idurl = kwargs.get('supplier_idurl') try: another_supplier_position = suppliers_list.index( another_supplier_idurl) except: another_supplier_position = -1 if another_supplier_position < 0: # this must never happen actually... only if another supplier is really uncooperative # this is dangerous because can lead to infinite loop between me and another supplier lg.info( 'found uncooperative supplier %s who raised the conflict but replied with invalid response' % another_supplier_idurl) # TODO: solve later self.transaction = None else: if len(self.transaction['suppliers']) <= another_supplier_position: lg.warn( 'another supplier position larger than family size, failed to solve family conflict with supplier %s' % another_supplier_idurl) self.transaction = None else: if self.transaction['suppliers'][another_supplier_position]: lg.warn( 'given position is not empty, failed to solve family conflict with supplier %s' % another_supplier_idurl) self.transaction = None else: self.transaction['suppliers'][ another_supplier_position] = another_supplier_idurl lg.info( 'found desired position %d in the family and solved conflict with supplier %s' % ( another_supplier_position, another_supplier_idurl, )) def doDHTRead(self, *args, **kwargs): """ Action method. """ d = dht_relations.read_customer_suppliers(self.customer_idurl) d.addCallback(self._on_dht_read_success) d.addErrback(self._on_dht_read_failed) def doDHTWrite(self, *args, **kwargs): """ Action method. """ d = dht_relations.write_customer_suppliers( customer_idurl=self.customer_idurl, suppliers_list=self.transaction['suppliers'], ecc_map=self.transaction['ecc_map'], revision=self.transaction['revision'], publisher_idurl=self.transaction['publisher_idurl'], ) d.addCallback(self._on_dht_write_success) d.addErrback(self._on_dht_write_failed) def doNotifyConnected(self, *args, **kwargs): """ Action method. """ if _Debug: lg.out( _DebugLevel, 'family_memeber.doNotifyConnected\n my_info=%r\n dht_info=%r\n requests=%r' % ( self.my_info, self.dht_info, self.requests, )) to_be_closed = False if self.current_request['command'] == 'family-leave': to_be_closed = True self.current_request = None if self.refresh_task.running: self.refresh_task.stop() self.refresh_task.start(self.refresh_period, now=False) if to_be_closed: self.requests = [] self.automat('shutdown') def doNotifyDisconnected(self, *args, **kwargs): """ Action method. """ if self.refresh_task.running: self.refresh_task.stop() self.current_request = None def doCheckReply(self, *args, **kwargs): """ Action method. """ self._on_incoming_contacts_packet(args[0]) def doDestroyMe(self, *args, **kwargs): """ Remove all references to the state machine object to destroy it. """ self.requests = [] self.current_request = None self.my_info = None self.dht_info = None self.transaction = None self.refresh_task = None delete_family(self.customer_idurl) self.destroy() #------------------------------------------------------------------------------ def _do_validate_dht_info(self, inp): if not inp or not isinstance(inp, dict): return None out = inp.copy() try: dht_revision = int(out['revision']) suppliers = out['suppliers'] ecc_map = out['ecc_map'] if dht_revision < 1: raise Exception('invalid revision') if not isinstance(suppliers, list) or len(suppliers) < 1: raise Exception('must include some suppliers') if ecc_map and ecc_map not in eccmap.EccMapNames(): raise Exception('invalid ecc_map name') out['publisher_idurl'] # TODO: add publisher_signature and Validate method to check publisher signature out['customer_idurl'] # TODO: add customer_signature and Validate method to check customer signature except: lg.exc() lg.warn('skip invalid DHT info and assume DHT record is not exist') return None return out def _do_validate_my_info(self, inp): if not inp: return None if not inp or not isinstance(inp, dict): return None # self._do_prepare_my_default_info() out = inp.copy() try: my_revision = int(out['revision']) if my_revision < 1: raise Exception('invalid revision') except: lg.exc() # out['revision'] = 0 return None try: suppliers = out['suppliers'] if not isinstance(suppliers, list) or len(suppliers) < 1: raise Exception('must include some suppliers') except: lg.exc() return None try: ecc_map = out['ecc_map'] if ecc_map and ecc_map not in eccmap.EccMapNames(): raise Exception('invalid ecc_map name') except: lg.exc() return None try: out['publisher_idurl'] # TODO: if I am a publisher - revision number must be the same as my info except: return None try: customer_idurl = out['customer_idurl'] if customer_idurl != self.customer_idurl: raise Exception('invalid customer_idurl') except: return None return out def _do_create_first_revision(self, request): return { 'revision': 0, 'publisher_idurl': my_id.getLocalIDURL( ), # I will be a publisher of the first revision 'suppliers': request.get('family_snapshot'), 'ecc_map': request['ecc_map'], 'customer_idurl': self.customer_idurl, } def _do_create_possible_revision(self, latest_revision): local_customer_meta_info = contactsdb.get_customer_meta_info( self.customer_idurl) possible_position = local_customer_meta_info.get('position', -1) possible_suppliers = local_customer_meta_info.get('family_snapshot') if possible_position > 0 and my_id.getLocalIDURL( ) not in possible_suppliers: if len(possible_suppliers) > possible_position: possible_suppliers[possible_position] = my_id.getLocalIDURL() return { 'revision': latest_revision, 'publisher_idurl': my_id.getLocalIDURL(), # I will be a publisher of that revision 'suppliers': possible_suppliers, 'ecc_map': local_customer_meta_info.get('ecc_map'), 'customer_idurl': self.customer_idurl, } def _do_create_revision_from_another_supplier(self, another_revision, another_suppliers, another_ecc_map): local_customer_meta_info = contactsdb.get_customer_meta_info( self.customer_idurl) possible_position = local_customer_meta_info.get('position', -1) if possible_position >= 0: try: another_suppliers[possible_position] = my_id.getLocalIDURL() except: lg.exc() contactsdb.add_customer_meta_info( self.customer_idurl, { 'ecc_map': another_ecc_map, 'position': possible_position, 'family_snapshot': another_suppliers, }) return { 'revision': int(another_revision), 'publisher_idurl': my_id.getLocalIDURL(), # I will be a publisher of that revision 'suppliers': another_suppliers, 'ecc_map': another_ecc_map, 'customer_idurl': self.customer_idurl, } def _do_detect_latest_revision(self, dht_info, my_info): try: my_revision = int(my_info['revision']) except: lg.warn( 'my own info is unknown or invalid, assume my revision is 0') my_revision = 0 try: dht_revision = int(dht_info['revision']) except: lg.warn('DHT info is unknown or invalid, assume DHT revision is 0') dht_revision = 0 if my_revision == dht_revision: return dht_revision if my_revision > dht_revision: # TODO: SECURITY need to find a solution to prevent cheating here # another supplier could publish a record where he is only alone present and with a correct revision # that means he actually brutally dropped all other suppliers from the family lg.info( 'known DHT info for customer %s is more fresh, will rewrite DHT record' % self.customer_idurl) if my_revision > dht_revision + 1: lg.warn( 'switching revision too far, normally always increase by one on every change' ) return my_revision return dht_revision def _do_merge_revisions(self, dht_info, my_info, latest_revision): if dht_info is None or not isinstance(dht_info, dict): merged_info = my_info else: if latest_revision == int(dht_info['revision']): if my_info is not None: if latest_revision == int(my_info['revision']): # I have same revision as info from DHT merged_info = dht_info else: if int(my_info['revision']) > int( dht_info['revision']): # here my revision is higher, so I have some changes that needs to be published already merged_info = my_info else: # here my revision is lower, I need to take info from DHT merged_info = dht_info else: merged_info = dht_info else: # here my revision is higher, so I have some changes that needs to be published already merged_info = my_info if not merged_info: return None # make sure list of suppliers have correct length according to ecc_map if not merged_info['ecc_map']: known_ecc_map = contactsdb.get_customer_meta_info( self.customer_idurl).get('ecc_map', None) lg.warn('unknown ecc_map, will populate known value: %s' % known_ecc_map) merged_info['ecc_map'] = known_ecc_map if merged_info['ecc_map']: expected_suppliers_count = eccmap.GetEccMapSuppliersNumber( merged_info['ecc_map']) if len(merged_info['suppliers']) < expected_suppliers_count: merged_info['suppliers'] += [ b'', ] * (expected_suppliers_count - len(merged_info['suppliers'])) elif len(merged_info['suppliers']) > expected_suppliers_count: merged_info['suppliers'] = merged_info[ 'suppliers'][:expected_suppliers_count] if merged_info['revision'] != latest_revision: lg.info('will switch known revision %d to the latest: %d' % ( merged_info['revision'], latest_revision, )) merged_info['revision'] = latest_revision return merged_info def _do_increment_revision(self, possible_transaction): if self.dht_info: if self.dht_info['suppliers'] == possible_transaction['suppliers']: if self.dht_info['ecc_map'] == possible_transaction['ecc_map']: if _Debug: lg.out( _DebugLevel, 'family_member._do_increment_revision did not found any changes, skip transaction' ) return None possible_transaction['revision'] += 1 possible_transaction['publisher_idurl'] = my_id.getLocalIDURL() return possible_transaction def _do_process_family_join_request(self, merged_info, current_request): current_request_expected_suppliers_count = None if current_request['ecc_map']: current_request_expected_suppliers_count = eccmap.GetEccMapSuppliersNumber( current_request['ecc_map']) if current_request_expected_suppliers_count and current_request[ 'position'] > current_request_expected_suppliers_count: lg.warn( '"family-join" request is not valid, supplier position %d greater than expected suppliers count %d for %s' % (current_request['position'], current_request_expected_suppliers_count, current_request['ecc_map'])) return None if merged_info['ecc_map'] and current_request[ 'ecc_map'] and current_request['ecc_map'] != merged_info[ 'ecc_map']: lg.info( 'from "family-join" request, detected ecc_map change %s -> %s for customer %s' % (merged_info['ecc_map'], current_request['ecc_map'], self.customer_idurl)) merged_info['ecc_map'] = current_request['ecc_map'] if not merged_info['ecc_map'] and current_request['ecc_map']: lg.info( 'from "family-join" request, detected ecc_map was set to %s for the first time for customer %s' % (current_request['ecc_map'], self.customer_idurl)) merged_info['ecc_map'] = current_request['ecc_map'] if not merged_info['ecc_map']: lg.warn( 'still did not found actual ecc_map from DHT or from the request' ) return None expected_suppliers_count = eccmap.GetEccMapSuppliersNumber( merged_info['ecc_map']) if not merged_info['suppliers']: merged_info['suppliers'] = [ b'', ] * expected_suppliers_count if len(merged_info['suppliers']) < expected_suppliers_count: merged_info['suppliers'] += [ b'', ] * (expected_suppliers_count - len(merged_info['suppliers'])) else: merged_info['suppliers'] = merged_info[ 'suppliers'][:expected_suppliers_count] try: existing_position = merged_info['suppliers'].index( current_request['supplier_idurl']) except: existing_position = -1 if current_request[ 'position'] is not None and current_request['position'] >= 0: if current_request['position'] >= expected_suppliers_count: lg.warn( '"family-join" request is not valid, supplier position greater than expected suppliers count' ) return None if existing_position >= 0 and existing_position != current_request[ 'position']: merged_info['suppliers'][existing_position] = b'' merged_info['suppliers'][current_request[ 'position']] = current_request['supplier_idurl'] if _Debug: lg.out( _DebugLevel, ' found my IDURL on %d position and will move it on %d position in the family of customer %s' % (existing_position, current_request['position'], self.customer_idurl)) if merged_info['suppliers'][current_request[ 'position']] != current_request['supplier_idurl']: if merged_info['suppliers'][ current_request['position']] not in [b'', '', None]: # TODO: SECURITY need to implement a signature verification and # also build solution to validate that change was approved by customer lg.warn( 'overwriting another supplier %s with my IDURL at position %d in family of customer %s' % ( merged_info['suppliers'][ current_request['position']], current_request['position'], self.customer_idurl, )) merged_info['suppliers'][current_request[ 'position']] = current_request['supplier_idurl'] if _Debug: lg.out( _DebugLevel, ' placed supplier %s at known position %d in the family of customer %s' % (current_request['supplier_idurl'], current_request['position'], self.customer_idurl)) if current_request['supplier_idurl'] not in merged_info['suppliers']: if b'' in merged_info['suppliers']: first_empty_position = merged_info['suppliers'].index(b'') merged_info['suppliers'][ first_empty_position] = current_request['supplier_idurl'] if _Debug: lg.out( _DebugLevel, ' placed supplier %s at first empty position %d in family of customer %s' % (current_request['supplier_idurl'], first_empty_position, self.customer_idurl)) else: merged_info['suppliers'].append( current_request['supplier_idurl']) if _Debug: lg.out( _DebugLevel, ' added supplier %s to family of customer %s' % (current_request['supplier_idurl'], self.customer_idurl)) return merged_info def _do_process_family_leave_request(self, merged_info, current_request): try: existing_position = merged_info['suppliers'].index( current_request['supplier_idurl']) except ValueError: existing_position = -1 if existing_position < 0: lg.warn( 'skip "family-leave" request, did not found supplier %r in customer family %r' % ( current_request['supplier_idurl'], self.customer_idurl, )) return None merged_info['suppliers'][existing_position] = b'' return merged_info def _do_process_family_refresh_request(self, merged_info): if not self.my_info: self.my_info = self._do_create_possible_revision( int(merged_info['revision'])) lg.warn( '"family-refresh" request will use "possible" customer meta info: %r' % self.my_info) if int(self.my_info['revision']) > int(merged_info['revision']): lg.info( '"family-refresh" request will overwrite DHT record with my info because my revision is higher than record in DHT' ) return self.my_info.copy() try: my_position = self.my_info['suppliers'].index( my_id.getLocalIDURL()) except: my_position = -1 if my_position < 0: lg.warn( '"family-refresh" request failed because my info not exist or not valid, my own position in the family is unknown' ) return None my_expected_suppliers_count = None if self.my_info['ecc_map']: my_expected_suppliers_count = eccmap.GetEccMapSuppliersNumber( self.my_info['ecc_map']) if my_expected_suppliers_count and my_position >= my_expected_suppliers_count: lg.warn( '"family-refresh" request failed because my info is not valid, supplier position greater than expected suppliers count' ) return None if len(merged_info['suppliers']) != my_expected_suppliers_count: lg.warn( 'number of suppliers not expected during processing of "family-refresh" request' ) if len(merged_info['suppliers']) < my_expected_suppliers_count: merged_info['suppliers'] += [ b'', ] * (my_expected_suppliers_count - len(merged_info['suppliers'])) else: merged_info['suppliers'] = merged_info[ 'suppliers'][:my_expected_suppliers_count] try: existing_position = merged_info['suppliers'].index( my_id.getLocalIDURL()) except ValueError: existing_position = -1 if existing_position < 0: if merged_info['suppliers'][my_position] not in [b'', '', None]: # TODO: SECURITY need to implement a signature verification and # also build solution to validate that change was approved by customer lg.warn( 'overwriting another supplier %s with my IDURL at position %d in family of customer %s' % ( merged_info['suppliers'][my_position], my_position, self.customer_idurl, )) merged_info['suppliers'][my_position] = my_id.getLocalIDURL() if _Debug: lg.out( _DebugLevel, ' placed supplier %s at known position %d in the family of customer %s' % (my_id.getLocalIDURL(), my_position, self.customer_idurl)) existing_position = my_position if existing_position != my_position: merged_info['suppliers'][existing_position] = b'' merged_info['suppliers'][my_position] = my_id.getLocalIDURL() if _Debug: lg.out( _DebugLevel, ' found my IDURL on %d position and will move it on %d position in the family of customer %s' % (existing_position, my_position, self.customer_idurl)) return merged_info def _do_process_request(self, merged_info, current_request): if current_request['command'] == 'family-join': return self._do_process_family_join_request( merged_info, current_request) if current_request['command'] == 'family-leave': return self._do_process_family_leave_request( merged_info, current_request) if current_request['command'] == 'family-refresh': return self._do_process_family_refresh_request(merged_info) lg.err('invalid request command') return None def _on_family_refresh_task(self): self.automat('family-refresh') def _on_dht_read_success(self, dht_result): if _Debug: lg.out( _DebugLevel, 'family_member._on_dht_read_success result: %r' % dht_result) if dht_result: self.dht_info = dht_result self.automat('dht-value-exist', dht_result) else: self.dht_info = None self.automat('dht-value-not-exist', None) def _on_dht_read_failed(self, err): self.dht_info = None if _Debug: lg.out(_DebugLevel, 'family_member._on_dht_read_failed : %r' % err) self.automat('dht-read-fail') def _on_dht_write_success(self, dht_result): if _Debug: lg.out( _DebugLevel, 'family_member._on_dht_write_success result: %r' % dht_result) self.my_info = self.transaction.copy() self.dht_info = None self.transaction = None self.automat('dht-write-ok', dht_result) def _on_dht_write_failed(self, err): if _Debug: lg.out(_DebugLevel, 'family_member._on_dht_write_failed') self.transaction = None self.dht_info = None self.automat('dht-write-fail') def _on_incoming_suppliers_list(self, inp): # this packet came from another supplier who belongs to that family also incoming_packet = inp['packet'] if _Debug: lg.out( _DebugLevel, 'family_member._on_incoming_suppliers_list with %s' % incoming_packet) if not self.my_info: if _Debug: lg.out(_DebugLevel, ' current DHT info is not yet known, skip') return p2p_service.SendAck(incoming_packet) try: another_ecc_map = inp['customer_ecc_map'] another_suppliers_list = inp['suppliers_list'] another_revision = int(inp['transaction_revision']) except: lg.exc() return p2p_service.SendFail(incoming_packet, response=serialization.DictToBytes( self.my_info)) if _Debug: lg.out( _DebugLevel, ' another_revision=%d another_ecc_map=%s another_suppliers_list=%r' % (another_revision, another_ecc_map, another_suppliers_list)) if another_revision >= int(self.my_info['revision']): self.my_info = self._do_create_revision_from_another_supplier( another_revision, another_suppliers_list, another_ecc_map) lg.info( 'another supplier have more fresh revision, update my info and raise "family-refresh" event' ) self.automat('family-refresh') return p2p_service.SendAck(incoming_packet) if my_id.getLocalIDURL() not in another_suppliers_list: lg.warn( 'another supplier is trying to remove my IDURL from the family of customer %s' % self.customer_idurl) return p2p_service.SendFail(incoming_packet, response=serialization.DictToBytes( self.my_info)) my_position_in_transaction = another_suppliers_list.index( my_id.getLocalIDURL()) my_known_position = self.my_info['suppliers'].index( my_id.getLocalIDURL()) if my_position_in_transaction != my_known_position: lg.warn( 'another supplier is trying to put my IDURL on another position in the family of customer %s' % self.customer_idurl) return p2p_service.SendFail(incoming_packet, response=serialization.DictToBytes( self.my_info)) return p2p_service.SendAck(incoming_packet) def _on_incoming_supplier_position(self, inp): # this packet came from the customer, a godfather of the family ;))) incoming_packet = inp['packet'] try: ecc_map = inp['customer_ecc_map'] supplier_idurl = inp['supplier_idurl'] supplier_position = inp['supplier_position'] family_snapshot = inp.get('family_snapshot') except: lg.exc() return None if supplier_idurl != my_id.getLocalIDURL(): return p2p_service.SendFail( incoming_packet, 'contacts packet with supplier position not addressed to me') try: _existing_position = self.my_info['suppliers'].index( supplier_idurl) except: _existing_position = -1 contactsdb.add_customer_meta_info( self.customer_idurl, { 'ecc_map': ecc_map, 'position': supplier_position, 'family_snapshot': family_snapshot, }) if _Debug: lg.out( _DebugLevel, 'family_member._on_incoming_supplier_position stored new meta info for customer %s:\n' % self.customer_idurl) lg.out( _DebugLevel, ' ecc_map=%s position=%s family_snapshot=%s' % ( ecc_map, supplier_position, family_snapshot, )) return p2p_service.SendAck(incoming_packet) def _on_incoming_contacts_packet(self, inp): try: contacts_type = inp['type'] incoming_packet = inp['packet'] except: lg.exc() return None if contacts_type == 'suppliers_list': return self._on_incoming_suppliers_list(inp) elif contacts_type == 'supplier_position': return self._on_incoming_supplier_position(inp) return p2p_service.SendFail(incoming_packet, 'invalid contacts type') def _on_supplier_ack(self, response, info): if _Debug: lg.out(_DebugLevel, 'family_member._on_supplier_ack with %r' % response) if response.PacketID in self.suppliers_requests: self.suppliers_requests.remove(response.PacketID) if not self.suppliers_requests: self.automat('all-suppliers-agree') def _on_supplier_fail(self, response, info): if _Debug: lg.out(_DebugLevel, 'family_member._on_supplier_fail with %r' % response) if response.PacketID in self.suppliers_requests: self.suppliers_requests.remove(response.PacketID) try: json_payload = serialization.BytesToDict(response.Payload) ecc_map = strng.to_text(json_payload['ecc_map']) suppliers_list = list(map(strng.to_bin, json_payload['suppliers'])) except: lg.exc() if not self.suppliers_requests: self.automat('all-suppliers-agree') return None self.automat('one-supplier-not-agree', ecc_map=ecc_map, suppliers_list=suppliers_list, supplier_idurl=response.OwnerID)
class Character(): def __init__(self, name, address, udp, main): self.name = name self.address = address self.udp = udp self.heartBeatCount = 3 self.main = main self.world = None self.street = None self.room = None self.handlers = { "disconnect": self.connectionLost, "pos_upd": self.pos_upd_handler, "mov": self.mov_handler, "heartbeat": self.onHeartBeat } self.heartbeatloop = LoopingCall(self.sendHeartbeat) self.heartbeatloop.start(5) def stop(self): self.heartbeatloop.stop() self = None def sendHeartbeat(self): """ Checks if this character instance is still connected. Does three heartv beat retry """ if self.heartBeatCount == 0: self.connectionLost({"id": self.name}) return hb = json.dumps({'heartbeat': 1}) self.udp.reply(hb, self.address) self.heartBeatCount -= 1 def onHeartBeat(self, val): print "Hearbeat OK" self.heartBeatCount = 3 def onRelay(self, data): for handler, data in data.items(): if handler in self.handlers: self.handlers[handler](data) def reply(self, data): self.udp.reply(data, self.address) def connectionLost(self, data): name = data['id'] print name, "has disconnected" dc = json.dumps({'dc': {"id": name}}) self.main.send_to_others(name, dc) self.main.removePlayer(name) def pos_upd_handler(self, val): #update receiver with the my current position if val.has_key("pos") and val.has_key("id") and val.has_key("receiver")\ and len(val.keys())==2: mes = {"pos_upd":{"id":val['receiver'],"pos":[val["pos"][0],\ val["pos"][1]]}} self.send(val['receiver'], json.dumps(mes)) def mov_handler(self, val): if val.has_key("dest") and val.has_key("id") and len(val.keys()) == 2: mes = {"mov":{"id":val['id'],"dest":[val['dest'][0],\ val['dest'][1]]}} self.main.send_to_others(self.name, json.dumps(mes)) def send(self, player, mes): try: self.main.players[player].reply(mes) except Exception: print self.main.players, player print "Failed: Not in here anymore"
class _HamlibProxy(ExportedState): # pylint: disable=no-member """ Abstract class for objects which export state proxied to a hamlib daemon. """ implements(IComponent, IProxy) def __init__(self, protocol): # info from hamlib self.__cache = {} self.__caps = {} self.__levels = [] # invert command table # TODO: we only need to do this once per class, really self._how_to_command = { key: command for command, keys in self._commands.iteritems() for key in keys } # keys are same as __cache, values are functions to call with new values from rig self._cell_updaters = {} self.__communication_error = False self.__last_error = (-1e9, '', 0) self.__protocol = protocol self.__disconnect_deferred = defer.Deferred() protocol._set_proxy(self) # TODO: If hamlib backend supports "transceive mode", use it in lieu of polling self.__poller_slow = LoopingCall(self.__poll_slow) self.__poller_fast = LoopingCall(self.__poll_fast) self.__poller_slow.start(2.0) self.__poller_fast.start(0.2) self._ready_deferred = protocol.rc_send('dump_caps') def sync(self): # TODO: Replace 'sync' with more specifically meaningful operations d = self.__protocol.rc_send(self._dummy_command) d.addCallback(lambda _: None) # ignore result return d def close(self): """implements IComponent""" self.__protocol.transport.loseConnection() return self.when_closed() # used for tests, not part of IComponent def when_closed(self): return fork_deferred(self.__disconnect_deferred) def _ehs_get(self, name_in_cmd): if name_in_cmd in self.__cache: return self.__cache[name_in_cmd] else: return 0.0 def _clientReceived(self, command, key, value): self.__communication_error = False if command == 'dump_caps': def write(key): self.__caps[key] = value if key == 'Get level': # add to polling info for info in value.strip().split(' '): match = re.match(r'^(\w+)\([^()]+\)$', info) # part in parens is probably min/max/step info, but we don't have any working examples to test against (they are all 0) if match: self.__levels.append(match.group(1)) else: log.err( 'Unrecognized level description from %s: %r' % (self._server_name, info)) # remove irregularity keymatch = re.match(r'(Can [gs]et )([\w\s,/-]+)', key) if keymatch and keymatch.group(2) in _cap_remap: for mapped in _cap_remap[keymatch.group(2)]: write(keymatch.group(1) + mapped) else: write(key) else: self.__update_cache_and_cells(key, value) def _clientReceivedLevel(self, level_name, value_str): self.__update_cache_and_cells(level_name + ' level', value_str) def _clientError(self, cmd, error_number): if cmd.startswith('get_'): # these getter failures are boring, probably us polling something not implemented if error_number == RIG_ENIMPL or error_number == RIG_ENTARGET or error_number == RIG_BUSERROR: return elif error_number == RIG_ETIMEOUT: self.__communication_error = True return self.__last_error = (time.time(), cmd, error_number) def __update_cache_and_cells(self, key, value): self.__cache[key] = value if key in self._cell_updaters: self._cell_updaters[key](value) def _clientConnectionLost(self, reason): self.__poller_slow.stop() self.__poller_fast.stop() self.__disconnect_deferred.callback(None) def _ehs_set(self, name_full, value): if not isinstance(value, str): raise TypeError() name_in_cmd = self._how_to_command[name_full] # raises if cannot set if value != self.__cache[name_full]: self.__cache[name_full] = value self.__protocol.rc_send( 'set_' + name_in_cmd, ' '.join(self.__cache[arg_name] for arg_name in self._commands[name_in_cmd])) def state_def(self, callback): super(_HamlibProxy, self).state_def(callback) for name in self._info: can_get = self.__caps.get('Can get ' + name) if can_get is None: log.msg('No can-get information for ' + name) if can_get != 'Y': # TODO: Handle 'E' condition continue writable = name in self._how_to_command and self.__caps.get( 'Can set ' + name) == 'Y' _install_cell(self, name, False, writable, callback, self.__caps) for level_name in self.__levels: # TODO support writable levels _install_cell(self, level_name + ' level', True, False, callback, self.__caps) def __poll_fast(self): # TODO: Stop if we're getting behind p = self.__protocol self.poll_fast(p.rc_send) for level_name in self.__levels: p.rc_send('get_level', level_name) def __poll_slow(self): # TODO: Stop if we're getting behind p = self.__protocol self.poll_slow(p.rc_send) @exported_value(type=Notice(always_visible=False)) def get_errors(self): if self.__communication_error: return 'Rig not responding.' else: (error_time, cmd, error_number) = self.__last_error if error_time > time.time() - 10: return u'%s: %s' % (cmd, error_number) else: return u'' def poll_fast(self, send): raise NotImplementedError() def poll_slow(self, send): raise NotImplementedError()
class ResourceMonitor(object): """ This class is responsible for monitoring resources in Tribler. Specifically, it fetches information from the Tribler core and writes it to a file. """ def __init__(self, interval): self.interval = interval self.request_manager = HTTPRequestManager() self.monitor_memory_lc = LoopingCall(self.monitor_memory) self.monitor_cpu_lc = LoopingCall(self.monitor_cpu) self.start_time = time.time() self.latest_memory_time = 0 self.latest_cpu_time = 0 # Create the output directory if it does not exist yet output_dir = os.path.join(os.getcwd(), "output") if not os.path.exists(output_dir): os.makedirs(output_dir) self.memory_stats_file_path = os.path.join(output_dir, 'memory_stats.csv') with open(self.memory_stats_file_path, "w") as output_file: output_file.write("time,memory_usage\n") self.cpu_stats_file_path = os.path.join(output_dir, 'cpu_stats.csv') with open(self.cpu_stats_file_path, "w") as output_file: output_file.write("time,cpu_usage\n") def start(self): """ Start the monitoring loop for the resources. """ self.monitor_memory_lc.start(self.interval) self.monitor_cpu_lc.start(self.interval) def stop(self): """ Stop the monitoring loop for the resources. """ if self.monitor_memory_lc and self.monitor_memory_lc.running: self.monitor_memory_lc.stop() self.monitor_memory_lc = None if self.monitor_cpu_lc and self.monitor_cpu_lc.running: self.monitor_cpu_lc.stop() self.monitor_cpu_lc = None def on_memory_history(self, response): history = json.loads(response) for history_item in history["memory_history"]: if history_item["time"] > self.latest_memory_time: self.latest_memory_time = history_item["time"] time_diff = history_item["time"] - self.start_time with open(self.memory_stats_file_path, "a") as output_file: output_file.write("%s,%s\n" % (time_diff, history_item["mem"])) def on_cpu_history(self, response): history = json.loads(response) for history_item in history["cpu_history"]: if history_item["time"] > self.latest_cpu_time: self.latest_cpu_time = history_item["time"] time_diff = history_item["time"] - self.start_time with open(self.cpu_stats_file_path, "a") as output_file: output_file.write("%s,%s\n" % (time_diff, history_item["cpu"])) def monitor_memory(self): """ Monitor the memory usage in Tribler. """ return self.request_manager.get_memory_history_core().addCallback(self.on_memory_history) def monitor_cpu(self): """ Monitor the CPU usage in Tribler. """ return self.request_manager.get_cpu_history_core().addCallback(self.on_cpu_history)
class P2PConnector(automat.Automat): """ """ fast = False timers = { 'timer-20sec': (20.0, ['INCOMMING?']), } def init(self): self.health_check_task = None self.log_transitions = _Debug def state_changed(self, oldstate, newstate, event, *args, **kwargs): if newstate == 'INCOMMING?' and event != 'instant': self.automat('instant') if newstate == 'CONNECTED': self.health_check_task = LoopingCall( self._do_id_server_health_check) self.health_check_task.start(config.conf().getInt( 'services/identity-propagate/health-check-interval-seconds'), now=False) else: if self.health_check_task: self.health_check_task.stop() self.health_check_task = None def A(self, event, *args, **kwargs): #---AT_STARTUP--- if self.state == 'AT_STARTUP': if event == 'init': self.state = 'NETWORK?' self.NeedPropagate = True self.doInit(*args, **kwargs) network_connector.A('reconnect') #---NETWORK?--- elif self.state == 'NETWORK?': if (event == 'network_connector.state' and args[0] == 'CONNECTED'): self.state = 'MY_IDENTITY' self.doUpdateMyIdentity(*args, **kwargs) elif (event == 'network_connector.state' and args[0] == 'DISCONNECTED'): self.state = 'DISCONNECTED' #---INCOMMING?--- elif self.state == 'INCOMMING?': if event == 'inbox-packet' and not self.isUsingBestProto( *args, **kwargs): self.state = 'MY_IDENTITY' self.doUpdateMyIdentity(*args, **kwargs) self.doPopBestProto(*args, **kwargs) elif event == 'timer-20sec' or (event == 'network_connector.state' and args[0] == 'DISCONNECTED'): self.state = 'DISCONNECTED' self.doInitRatings(*args, **kwargs) elif event == 'check-synchronize' or (event == 'network_connector.state' and args[0] == 'CONNECTED'): self.state = 'MY_IDENTITY' self.doUpdateMyIdentity(*args, **kwargs) elif (event == 'instant' and not self.isAnyPeersKnown(*args, **kwargs)) or ( event == 'inbox-packet' and self.isUsingBestProto(*args, **kwargs)): self.state = 'CONNECTED' self.doInitRatings(*args, **kwargs) self.doRestartCustomersRejector(*args, **kwargs) #---CONNECTED--- elif self.state == 'CONNECTED': if event == 'ping-contact': self.doSendMyIdentity(*args, **kwargs) elif (event == 'network_connector.state' and args[0] == 'DISCONNECTED'): self.state = 'DISCONNECTED' elif event == 'check-synchronize' or (event == 'network_connector.state' and args[0] == 'CONNECTED'): self.state = 'MY_IDENTITY' self.doUpdateMyIdentity(*args, **kwargs) elif (event == 'network_connector.state' and args[0] not in ['CONNECTED', 'DISCONNECTED']): self.state = 'NETWORK?' #---DISCONNECTED--- elif self.state == 'DISCONNECTED': if event == 'ping-contact': self.doSendMyIdentity(*args, **kwargs) elif event == 'inbox-packet' or event == 'check-synchronize' or ( (event == 'network_connector.state' and args[0] == 'CONNECTED')): self.state = 'MY_IDENTITY' self.doUpdateMyIdentity(*args, **kwargs) elif (event == 'network_connector.state' and args[0] not in [ 'CONNECTED', 'DISCONNECTED', ]): self.state = 'NETWORK?' #---MY_IDENTITY--- elif self.state == 'MY_IDENTITY': if event == 'my-id-updated' and self.isMyContactsChanged( *args, **kwargs): self.state = 'NETWORK?' self.NeedPropagate = True network_connector.A('check-reconnect') elif event == 'my-id-updated' and not self.isMyContactsChanged( *args, **kwargs) and (self.NeedPropagate or self.isMyIdentityChanged( *args, **kwargs)): self.state = 'PROPAGATE' self.doCheckRotatePropagateMyIdentity(*args, **kwargs) elif event == 'my-id-updated' and not ( self.NeedPropagate or self.isMyIdentityChanged( *args, **kwargs)) and (network_connector.A().state is not 'CONNECTED'): self.state = 'DISCONNECTED' elif event == 'my-id-updated' and not ( self.NeedPropagate or self.isMyIdentityChanged(*args, **kwargs)) and ( network_connector.A().state is 'CONNECTED'): self.state = 'CONNECTED' #---PROPAGATE--- elif self.state == 'PROPAGATE': if event == 'my-id-propagated': self.state = 'INCOMMING?' self.NeedPropagate = False self.doRestartFireHire(*args, **kwargs) elif ((event == 'network_connector.state' and args[0] == 'CONNECTED')) or event == 'check-synchronize': self.state = 'MY_IDENTITY' self.doUpdateMyIdentity(*args, **kwargs) return None def isUsingBestProto(self, *args, **kwargs): """ Condition method. """ return self._check_to_use_best_proto() def isMyIdentityChanged(self, *args, **kwargs): """ Condition method. """ return args[0][1] def isMyContactsChanged(self, *args, **kwargs): """ Condition method. """ return args[0][0] def isAnyPeersKnown(self, *args, **kwargs): """ Condition method. """ return len(contactsdb.contacts_remote()) > 0 def doInit(self, *args, **kwargs): version_number = bpio.ReadTextFile( settings.VersionNumberFile()).strip() if _Debug: lg.out( _DebugLevel - 6, 'p2p_connector.doInit RevisionNumber=%s' % str(version_number)) callback.append_inbox_callback(inbox) def doSendMyIdentity(self, *args, **kwargs): """ Action method. """ propagate.single(args[0], wide=True) def doUpdateMyIdentity(self, *args, **kwargs): if _Debug: lg.out(_DebugLevel - 6, 'p2p_connector.doUpdateMyIdentity') self._update_my_identity() def doCheckRotatePropagateMyIdentity(self, *args, **kwargs): if _Debug: lg.out(_DebugLevel - 6, 'p2p_connector.doCheckRotatePropagateMyIdentity') self._check_rotate_propagate_my_identity() def doPopBestProto(self, *args, **kwargs): self._pop_active_proto() def doInitRatings(self, *args, **kwargs): ratings.init() def doRestartCustomersRejector(self, *args, **kwargs): """ Action method. """ if driver.is_on('service_customer_patrol'): # TODO: move this into a callback inside service_customer_patrol from supplier import customers_rejector customers_rejector.A('restart') def doRestartFireHire(self, *args, **kwargs): """ Action method. """ if driver.is_on('service_employer'): # TODO: use events instead from customer import fire_hire fire_hire.A('restart') def _check_to_use_best_proto(self): # if no incoming traffic - do nothing if len(active_protos()) == 0: return True lid = my_id.getLocalIdentity() order = lid.getProtoOrder() # if no protocols in local identity - do nothing if len(order) == 0: return True # when transport proxy is working we do not need to check our contacts at all if settings.transportIsEnabled('proxy'): if driver.is_on('service_proxy_transport'): if settings.transportReceivingIsEnabled('proxy'): try: # TODO: change here to receive the value directly from service_proxy_transport object router_idurl = driver.services( )['service_proxy_transport'].transport.options[ 'router_idurl'] except: router_idurl = None if router_idurl: router_identity = identitycache.FromCache(router_idurl) contacts_is_ok = True router_protos = router_identity.getContactsByProto() if lid.getContactsNumber() != len(router_protos): contacts_is_ok = False if contacts_is_ok: for proto, contact in router_protos.items(): if lid.getProtoContact(proto) != contact: contacts_is_ok = False if contacts_is_ok: if _Debug: lg.out( _DebugLevel - 6, 'p2p_connector._check_to_use_best_proto returning True : proxy_transport is OK' ) return True first = order[0] # if first contact in local identity is not working yet # but there is another working methods - switch first method if first not in active_protos(): if _Debug: lg.out( _DebugLevel - 6, 'p2p_connector._check_to_use_best_proto first contact (%s) is not working! active_protos()=%s' % (first, str(active_protos()))) return False # #small hack to make udp as first method if all is fine # if first != 'udp' and ('udp' in active_protos() and 'tcp' in active_protos()): # lg.out(2, 'p2p_connector._check_to_use_best_proto first contact (%s) but UDP also works! active_protos()=%s' % (first, str(active_protos()))) # return False # if tcp contact is on first place and it is working - we are VERY HAPPY! - no need to change anything - return False if first == 'tcp' and 'tcp' in active_protos(): return True # but if tcp method is not the first and it works - we want to TURN IT ON! - return True if first != 'tcp' and 'tcp' in active_protos(): if _Debug: lg.out( _DebugLevel - 6, 'p2p_connector._check_to_use_best_proto tcp is not first but it works active_protos()=%s' % str(active_protos())) return False # if we are using udp and it is working - this is fantastic! if first == 'udp' and 'udp' in active_protos(): # but let's check if TCP is also working # in that case we want to switch to TCP if 'tcp' in active_protos(): return False return True # udp seems to be working and first contact is not working - so switch to udp if first != 'udp' and 'udp' in active_protos(): if _Debug: lg.out( _DebugLevel - 6, 'p2p_connector._check_to_use_best_proto udp is not first but it works active_protos()=%s' % str(active_protos())) return False # http seems to work and it is first - cool! if first == 'http' and 'http' in active_protos(): return True # but if http method is not the first and it works - we want to TURN IT ON! - return True if first != 'http' and 'http' in active_protos(): if _Debug: lg.out( _DebugLevel - 6, 'p2p_connector._check_to_use_best_proto http is not first but it works active_protos()=%s' % str(active_protos())) return False # if we are using proxy and it is working - that is fine - it must work always! if first == 'proxy' and 'proxy' in active_protos(): return True # proxy seems to be working and first contact is not working - so switch to proxy if first != 'proxy' and 'proxy' in active_protos(): if _Debug: lg.out( _DebugLevel - 6, 'p2p_connector._check_to_use_best_proto proxy is not first but it works active_protos()=%s' % str(active_protos())) return False # in other cases - do nothing return True def _pop_active_proto(self): if len(active_protos()) == 0: return lid = my_id.getLocalIdentity() order = lid.getProtoOrder() first = order[0] wantedproto = '' # if first contact in local identity is not working yet # but there is another working methods - switch first method if first not in active_protos(): # take (but not remove) any item from the set wantedproto = active_protos().pop() active_protos().add(wantedproto) # if proxy method is not the first but it works - switch to proxy if first != 'proxy' and 'proxy' in active_protos(): wantedproto = 'proxy' # if http method is not the first but it works - switch to http if first != 'http' and 'http' in active_protos(): wantedproto = 'http' # if udp method is not the first but it works - switch to udp if first != 'udp' and 'udp' in active_protos(): wantedproto = 'udp' # if tcp method is not the first but it works - switch to tcp if first != 'tcp' and 'tcp' in active_protos(): wantedproto = 'tcp' if _Debug: lg.out( _DebugLevel - 6, 'p2p_connector.PopWorkingProto will pop %s contact order=%s active_protos()=%s' % (wantedproto, str(order), str(active_protos()))) # now move best proto on the top # other users will use this method to send to us lid.popProtoContact(wantedproto) # save local id # also need to propagate our identity # other users must know our new contacts my_id.setLocalIdentity(lid) my_id.saveLocalIdentity() def _update_my_identity(self): contacts_changed = False old_contacts = list(my_id.getLocalIdentity().getContacts()) identity_changed = my_id.rebuildLocalIdentity() if not identity_changed and len(active_protos()) > 0: self.automat('my-id-updated', (False, False)) return new_contacts = my_id.getLocalIdentity().getContacts() if len(old_contacts) != len(new_contacts): contacts_changed = True if not contacts_changed: for pos in range(len(old_contacts)): if old_contacts[pos] != new_contacts[pos]: contacts_changed = True break if contacts_changed: # erase all stats about received packets # if some contacts in my identity has been changed if _Debug: lg.out( 4, ' my contacts were changed, erase active_protos() flags' ) active_protos().clear() if _Debug: lg.out( 4, ' identity HAS %sBEEN CHANGED' % ('' if identity_changed else 'NOT ')) self.automat('my-id-updated', (contacts_changed, identity_changed)) def _check_rotate_propagate_my_identity(self): # TODO: rebuild that method into another state machine from p2p import id_rotator def _do_propagate(*args): if _Debug: lg.out(_DebugLevel, 'p2p_connector._do_propagate') if driver.is_on('service_entangled_dht'): from dht import dht_service dht_service.set_node_data('idurl', my_id.getLocalID().to_text()) d = propagate.start(wide=True, refresh_cache=True) d.addCallback(lambda contacts_list: self.automat( 'my-id-propagated', contacts_list)) if _Debug: d.addErrback( lg.errback, debug=_Debug, debug_level=_DebugLevel, method='_check_rotate_propagate_my_identity._do_propagate') def _on_propagate_failed(err): lg.err('failed propagate my identity: %r' % err) def _do_update(check_rotate_result): if _Debug: lg.out( _DebugLevel, 'p2p_connector._do_update check_rotate_result=%r' % check_rotate_result) if not check_rotate_result: lg.err( 'failed to rotate identity sources, skip propagating my identity' ) self.automat('my-id-propagated', []) return None d = propagate.update() d.addCallback(_do_propagate) d.addErrback(_on_propagate_failed) d.addErrback(lambda *args: self.automat('my-id-propagated', [])) def _do_rotate(check_result): if _Debug: lg.out( _DebugLevel, 'p2p_connector._do_rotate check_result=%r' % check_result) if check_result: lg.info('identity sources are healthy, send my identity now') _do_update(True) return None lg.err( 'identity sources are not healthy, will execute identity rotate flow now' ) d = id_rotator.run() d.addCallback(_do_update) d.addErrback(lambda _: _do_update(False)) def _do_check(x): if _Debug: lg.out(_DebugLevel, 'p2p_connector._do_check') d = id_rotator.check() d.addCallback(_do_rotate) d.addErrback(lambda _: _do_rotate(False)) _do_check(None) def _do_id_server_health_check(self): my_idurl = my_id.getLocalIdentity().getIDURL(as_original=True) if _Debug: lg.args(_DebugLevel, my_idurl=my_idurl) def _verify(xmlsrc=None): if not xmlsrc: lg.err('my current identity server not healthy') self.NeedPropagate = True self.automat('check-synchronize') return remote_ident = identity.identity(xmlsrc=xmlsrc) if not remote_ident.isCorrect() or not remote_ident.Valid(): lg.warn( 'my current identity server responded with bad identity file' ) self.NeedPropagate = True self.automat('check-synchronize') return if remote_ident.getIDURL(as_original=True) != my_idurl: lg.warn( 'my current identity server responded with unknown identity' ) self.NeedPropagate = True self.automat('check-synchronize') return if _Debug: lg.dbg(_DebugLevel, 'my current identity server is healthy') last_time = identitycache.last_time_cached(my_idurl) if last_time and time.time() - last_time < config.conf().getInt( 'services/identity-propagate/health-check-interval-seconds'): if _Debug: lg.dbg( _DebugLevel, 'skip health check of my current identity server, last time cached %d seconds ago' % (time.time() - last_time)) return d = identitycache.immediatelyCaching(my_idurl, try_other_sources=False) d.addCallback(_verify) d.addErrback(lambda _: _verify())
class ALSAAudioDevice(baseaudio.AudioDevice): writedev = None readdev = None def __repr__(self): return "ALSAAudioDevice %s" % (self.isOpen() and "open" or "closed") def openDev(self): print "alsa OPEN", self._closed try: from __main__ import app except: app = None if app is None: device = DEFAULT_ALSA_DEVICE else: device = app.getPref('audio_device', DEFAULT_ALSA_DEVICE) log.msg("alsaaudiodev opening device %s" % (device)) writedev = alsaaudio.PCM(alsaaudio.PCM_PLAYBACK, alsaaudio.PCM_NONBLOCK, device) self.writechannels = writedev.setchannels(1) writedev.setrate(8000) writedev.setformat(alsaaudio.PCM_FORMAT_S16_LE) writedev.setperiodsize(160) self.writedev = writedev readdev = alsaaudio.PCM(alsaaudio.PCM_CAPTURE, alsaaudio.PCM_NONBLOCK, device) self.readchannels = readdev.setchannels(1) readdev.setrate(8000) readdev.setformat(alsaaudio.PCM_FORMAT_S16_LE) readdev.setperiodsize(160) self.readdev = readdev self.LC = LoopingCall(self._push_up_some_data) self.LC.start(0.010) print "alsa OPENED", self._closed def _push_up_some_data(self): if self.readdev is None: return (l, data,) = self.readdev.read() if self.readchannels == 2: data = audioop.tomono(data, 2, 1, 1) if self.encoder and data: self.encoder.handle_audio(data) def write(self, data): if not hasattr(self, 'LC'): return assert self.isOpen(), "calling write() on closed %s"%(self,) if self.writechannels == 2: data = audioop.tostereo(data, 2, 1, 1) wrote = self.writedev.write(data) if not wrote: log.msg("ALSA overrun") def _close(self): print "alsa CLOSE", self._closed if self.isOpen(): log.msg("alsaaudiodev closing") try: self.LC.stop() except AttributeError: # ? bug in Twisted? Not sure. This catch-and-ignore is a temporary workaround. --Zooko pass del self.LC self.writedev = None self.readdev = None print "alsa CLOSED", self._closed
class ServerController(object): def __init__(self, realm, view): print('server cont!') self.realm = realm self.view = view def _handleInput(self): """ Handle currently available pygame input events. """ for event in pygame.event.get(): if (event.type == pygame.QUIT) or ((event.type == pygame.KEYDOWN) and (event.key == QUIT)): reactor.stop() sys.exit() if (event.type == pygame.KEYDOWN): if (event.key == START_GAME): self.realm.environment.startGame() elif (event.key == RESET_GAME): self.realm.environment.setPreGame() def go(self): self.previousTime = pygame.time.get_ticks() self._inputCall = LoopingCall(self._handleInput) d = self._inputCall.start(0.03) return d def stop(self): self._inputCall.stop()