def start(lc): global state, update, client, cleanup state = 0 client.addDispatcher(d) lc = LoopingCall(update) dd = lc.start(1.0 / 30) dd.addCallbacks(cleanup)
class Rain(object): """ Make it rain. Rain only occurs during spring. """ implements(IAutomaton) blocks = tuple() def __init__(self): self.season_loop = LoopingCall(self.check_season) def scan(self, chunk): pass def feed(self, coords): pass def start(self): self.season_loop.start(5 * 60) def stop(self): self.season_loop.stop() def check_season(self): if factory.world.season.name == "spring": factory.vane.weather = "rainy" reactor.callLater(1 * 60, setattr, factory.vane, "weather", "sunny") name = "rain"
def __init__( self, reactor, request, request_rate=10, sample_size=DEFAULT_SAMPLE_SIZE, timeout=45, tolerance_percentage=0.2 ): """ ``RequestLoadScenario`` constructor. :param tolerance_percentage: error percentage in the rate that is considered valid. For example, if we request a ``request_rate`` of 20, and we give a tolerance_percentage of 0.2 (20%), anything in [16,20] will be a valid rate. """ self.reactor = reactor self.request = request self.request_rate = request_rate self.timeout = timeout self._maintained = Deferred() self.rate_measurer = RateMeasurer(sample_size) self.max_outstanding = 10 * request_rate self.tolerated_errors = 5 * request_rate # Send requests per second self.loop = LoopingCall.withCount(self._request_and_measure) self.loop.clock = self.reactor # Monitor the status of the scenario self.monitor_loop = LoopingCall(self.check_rate) self.monitor_loop.clock = self.reactor self.is_started = False self.rate_tolerated = ( float(request_rate) - (request_rate*tolerance_percentage) )
def add(self, request, save_time = 0): self.data[request] = time.time() + save_time * 60 d = request.notifyFinish() d.addCallback(self.notifyFinished, request) d.addErrback(self.notifyFinished, request) call = LoopingCall(self.prune) call.start(60)
def modbus_master(module, properties): log.debug('Modbus master module : ' + str(module)) # Modbus Master #--------------------------------------------------------------------------# # initialize your data store #--------------------------------------------------------------------------# store = ModbusSlaveContext( co = ModbusSequentialDataBlock(0, [0]*100), hr = ModbusSequentialDataBlock(0, [0]*100)) context = ModbusServerContext(slaves=store, single=True) #--------------------------------------------------------------------------# # initialize the server information #--------------------------------------------------------------------------# identity = ModbusDeviceIdentification() identity.VendorName = 'ASO+AKO' identity.ProductCode = 'DYODE' identity.VendorUrl = 'yoloswag' identity.ProductName = 'DYODE' identity.ModelName = 'BSides LV release' identity.MajorMinorRevision = '0.9' #--------------------------------------------------------------------------# # run the server you want #--------------------------------------------------------------------------# time = 1 # 5 seconds delay loop = LoopingCall(f=modbus_master_update, a=(module, properties, context)) loop.start(time, now=False) # initially delay by time StartTcpServer(context, identity=identity, address=("0.0.0.0", \ properties['port_out']))
class Pinger(object): """ An periodic AMP ping helper. """ def __init__(self, reactor): """ :param IReactorTime reactor: The reactor to use to schedule the pings. """ self.reactor = reactor def start(self, protocol, interval): """ Start sending some pings. :param AMP protocol: The protocol over which to send the pings. :param timedelta interval: The interval at which to send the pings. """ def ping(): protocol.callRemote(NoOp) self._pinging = LoopingCall(ping) self._pinging.clock = self.reactor self._pinging.start(interval.total_seconds(), now=False) def stop(self): """ Stop sending the pings. """ self._pinging.stop()
class LeaseService(Service): """ Manage leases. In particular, clear out expired leases once a second. :ivar _reactor: A ``IReactorTime`` provider. :ivar _persistence_service: The persistence service to act with. :ivar _lc: A ``twisted.internet.task.LoopingCall`` run every second to update the configured leases by releasing leases that have expired. """ def __init__(self, reactor, persistence_service): self._reactor = reactor self._persistence_service = persistence_service def startService(self): self._lc = LoopingCall(self._expire) self._lc.clock = self._reactor self._lc.start(1) def stopService(self): self._lc.stop() def _expire(self): now = datetime.fromtimestamp(self._reactor.seconds(), tz=UTC) def expire(leases): updated_leases = leases.expire(now) for dataset_id in set(leases) - set(updated_leases): _LOG_EXPIRE(dataset_id=dataset_id, node_id=leases[dataset_id].node_id).write() return updated_leases return update_leases(expire, self._persistence_service)
class BandwidthEstimator: bufsize = 20 totalBytes = 0 def __init__(self, message, length): self.length = length self.message = message self.estim = [] self.bytes = 0 self.call = LoopingCall(self.estimateBandwidth) self.call.start(1) def estimateBandwidth(self): bytes = self.bytes self.totalBytes += bytes self.estim.append(bytes) self.message("%0.2f k/s (%0.2d%%)" % ((sum(self.estim) / len(self.estim)) / 1024., (float(self.totalBytes) / self.length) * 100)) if len(self.estim) > self.bufsize: self.estim.pop(0) self.bytes = 0 def stop(self): self.call.stop() self.estimateBandwidth() self.message("Finished receiving: %d bytes (%d%%)" % ( self.totalBytes, (float(self.totalBytes) / self.length) * 100))
def openShell(self, transport): """ Write 60 lines of data to the transport, then exit. """ proto = protocol.Protocol() proto.makeConnection(transport) transport.makeConnection(wrapProtocol(proto)) # Send enough bytes to the connection so that a rekey is triggered in # the client. def write(counter): i = counter() if i == 60: call.stop() transport.session.conn.sendRequest( transport.session, 'exit-status', '\x00\x00\x00\x00') transport.loseConnection() else: transport.write("line #%02d\n" % (i,)) # The timing for this loop is an educated guess (and/or the result of # experimentation) to exercise the case where a packet is generated # mid-rekey. Since the other side of the connection is (so far) the # OpenSSH command line client, there's no easy way to determine when the # rekey has been initiated. If there were, then generating a packet # immediately at that time would be a better way to test the # functionality being tested here. call = LoopingCall(write, count().next) call.start(0.01)
def __init__(self, settings, strategy_class): partition_id = settings.get('SCORING_PARTITION_ID') if partition_id is None or type(partition_id) != int: raise AttributeError("Scoring worker partition id isn't set.") messagebus = load_object(settings.get('MESSAGE_BUS')) mb = messagebus(settings) spider_log = mb.spider_log() scoring_log = mb.scoring_log() self.consumer = spider_log.consumer(partition_id=partition_id, type=b'sw') self.scoring_log_producer = scoring_log.producer() self._manager = FrontierManager.from_settings(settings, strategy_worker=True) codec_path = settings.get('MESSAGE_BUS_CODEC') encoder_cls = load_object(codec_path+".Encoder") decoder_cls = load_object(codec_path+".Decoder") self._decoder = decoder_cls(self._manager.request_model, self._manager.response_model) self._encoder = encoder_cls(self._manager.request_model) self.update_score = UpdateScoreStream(self._encoder, self.scoring_log_producer, 1024) self.states_context = StatesContext(self._manager.backend.states) self.consumer_batch_size = settings.get('SPIDER_LOG_CONSUMER_BATCH_SIZE') self.strategy = strategy_class.from_worker(self._manager, self.update_score, self.states_context) self.states = self._manager.backend.states self.stats = { 'consumed_since_start': 0 } self.job_id = 0 self.task = LoopingCall(self.work) self._logging_task = LoopingCall(self.log_status) self._flush_states_task = LoopingCall(self.flush_states) logger.info("Strategy worker is initialized and consuming partition %d", partition_id)
class PeriodicTasksService(Service): def __init__(self, clock, taskFunction, interval=60): self.clock = clock self.taskFunction = taskFunction self.interval = interval self._delayedCall = None self._looper = None self._loopFinished = None def startService(self): Service.startService(self) now = time.time() startDelta = now // self.interval * self.interval + self.interval - now self._delayedCall = self.clock.callLater(startDelta, self._startLooping) def _startLooping(self): self._delayedCall = None self._looper = LoopingCall(self._call) self._looper.clock = self.clock self._loopFinished = self._looper.start(self.interval) def _call(self): d = deferToThread(self.taskFunction) d.addErrback(log.err, "error calling periodic tasks") return d def stopService(self): if self._delayedCall: self._delayedCall.cancel() self._delayedCall = None return self._looper.stop() self._loopFinished.addCallback(lambda _: Service.stopService(self)) return self._loopFinished
def __init__(self): self.output("TWS init") self.username = environ["TXTRADER_USERNAME"] self.password = environ["TXTRADER_PASSWORD"] self.xmlrpc_port = int(environ["TXTRADER_XMLRPC_PORT"]) self.tcp_port = int(environ["TXTRADER_TCP_PORT"]) self.callback_timeout = int(environ["TXTRADER_TWS_CALLBACK_TIMEOUT"]) if not self.callback_timeout: self.callback_timeout = DEFAULT_TWS_CALLBACK_TIMEOUT self.output("callback_timeout=%d" % self.callback_timeout) self.enable_ticker = bool(int(environ["TXTRADER_ENABLE_TICKER"])) self.label = "TWS Gateway" self.channel = "tws" self.current_account = "" self.clients = set([]) self.orders = {} self.pending_orders = {} self.openorder_callbacks = [] self.accounts = [] self.account_data = {} self.positions = {} self.position_callbacks = [] self.executions = {} self.execution_callbacks = [] self.bardata_callbacks = [] self.cancel_callbacks = [] self.order_callbacks = [] self.addsymbol_callbacks = [] self.accountdata_callbacks = [] self.last_connection_status = "" self.connection_status = "Initializing" self.LastError = -1 self.next_order_id = -1 self.last_minute = -1 self.handlers = { "error": self.error_handler, "tickSize": self.handle_tick_size, "tickPrice": self.handle_tick_price, "tickString": self.handle_tick_string, "nextValidId": self.handle_next_valid_id, "currentTime": self.handle_time, "managedAccounts": self.handle_accounts, "orderStatus": self.handle_order_status, "openOrder": self.handle_open_order, "openOrderEnd": self.handle_open_order_end, "execDetails": self.handle_exec_details, "execDetailsEnd": self.handle_exec_details_end, "position": self.handle_position, "positionEnd": self.handle_position_end, "historicalData": self.handle_historical_data, "updateAccountValue": self.handle_account_value, "accountDownloadEnd": self.handle_account_download_end, } self.ticker_ids = {} self.symbols = {} self.symbols_by_id = {} self.primary_exchange_map = {} self.tws_conn = None repeater = LoopingCall(self.EverySecond) repeater.start(1)
class _ExchangeRate(object): """Download an exchange rate from Yahoo Finance using Twisted.""" def __init__(self, name): self._value = None self._name = name # External API: def latest_value(self): """Return the latest exchange rate value. May be None if no value is available. """ return self._value def start(self): """Start the background process.""" self._lc = LoopingCall(self._download) # Run immediately, and then every 30 seconds: self._lc.start(30, now=True) def _download(self): """Download the page.""" print("Downloading!") def parse(result): print("Got %r back from Yahoo." % (result,)) values = result.strip().split(",") self._value = float(values[1]) d = getPage( "http://download.finance.yahoo.com/d/quotes.csv?e=.csv&f=c4l1&s=%s=X" % (self._name,)) d.addCallback(parse) d.addErrback(log.err) return d
def make_lc(self, reactor, func): if DEBUG: self.stdout_length = 0 self.stderr_length = 0 def _(lc, reactor): if DEBUG: stdout = self.stdout.getvalue() stderr = self.stderr.getvalue() if self.stdout.getvalue()[self.stdout_length:]: print(self.stdout.getvalue()[self.stdout_length:], file=sys.__stdout__) if self.stderr.getvalue()[self.stderr_length:]: print(self.stderr.getvalue()[self.stderr_length:], file=sys.__stderr__) self.stdout_length = len(stdout) self.stderr_length = len(stderr) return func(lc, reactor) lc = LoopingCall(_) lc.a = (lc, reactor) lc.clock = reactor lc.start(0.1) return lc
class PollingDataStream(DataStream): """ A self-polling data stream. This class represents a data stream that wakes up at a given frequency, and calls the :meth:`poll` method. """ frequency = None # Either a timedelta object, or the number of seconds now = False def __init__(self): super(PollingDataStream, self).__init__() self.timer = LoopingCall(self.poll) if isinstance(self.frequency, timedelta): seconds = ( self.frequency.seconds + (self.frequency.days * 24 * 60 * 60) + (self.frequency.microseconds / 1000000.0) ) else: seconds = self.frequency log.debug("Setting a %s second timer" % seconds) self.timer.start(seconds, now=self.now) def poll(self): raise NotImplementedError def stop(self): super(PollingDataStream, self).stop() try: if hasattr(self, "timer"): self.timer.stop() except Exception, e: self.log.warn(e)
class SendsManyFileDescriptors(ConnectableProtocol): paused = False def connectionMade(self): self.socket = socket() self.transport.registerProducer(self, True) def sender(): self.transport.sendFileDescriptor(self.socket.fileno()) self.transport.write(b"x") self.task = LoopingCall(sender) self.task.clock = self.transport.reactor self.task.start(0).addErrback(err, "Send loop failure") def stopProducing(self): self._disconnect() def resumeProducing(self): self._disconnect() def pauseProducing(self): self.paused = True self.transport.unregisterProducer() self._disconnect() def _disconnect(self): self.task.stop() self.transport.abortConnection() self.other.transport.abortConnection()
def __init__(self, protocol): # info from hamlib self.__cache = {} self.__caps = {} self.__levels = [] # invert command table # TODO: we only need to do this once per class, really self._how_to_command = {key: command for command, keys in self._commands.iteritems() for key in keys} # keys are same as __cache, values are functions to call with new values from rig self._cell_updaters = {} self.__communication_error = False self.__last_error = (-1e9, '', 0) self.__protocol = protocol self.__disconnect_deferred = defer.Deferred() protocol._set_proxy(self) # TODO: If hamlib backend supports "transceive mode", use it in lieu of polling self.__poller_slow = LoopingCall(self.__poll_slow) self.__poller_fast = LoopingCall(self.__poll_fast) self.__poller_slow.start(2.0) self.__poller_fast.start(0.2) self._ready_deferred = protocol.rc_send('dump_caps')
class ICUMonitorFactory(protocol.ClientFactory): def __init__(self, gui_msg_callback, gui_ip, gui_port): self.ip = gui_ip self.port = gui_port self.msg_callback = gui_msg_callback self.objmonit = None def buildProtocol(self, addr): return ICUMonitor(self.msg_callback) def clientConnectionLost(self, connector, reason): pass def clientConnectionFailed(self, connector, reason): pass def sendmsg(self, gui_reactor): gui_reactor.connectTCP(self.ip, self.port, self) def startsend(self, gui_reactor, timerep): self.loop = LoopingCall(self.sendmsg, gui_reactor) self.loop.start(timerep) def stopsend(self): self.loop.stop()
class ProvisionerQueryService(ServiceProcess): """Provisioner querying service """ declare = ServiceProcess.service_declare(name='provisioner_query', version='0.1.0', dependencies=[]) def slc_init(self): interval = float(self.spawn_args.get("interval_seconds", DEFAULT_QUERY_INTERVAL)) self.client = ProvisionerClient(self) log.debug('Starting provisioner query loop - %s second interval', interval) self.loop = LoopingCall(self.query) self.loop.start(interval) def slc_terminate(self): if self.loop: self.loop.stop() @defer.inlineCallbacks def query(self): try: yield self._do_query() except Exception,e: log.error("Error sending provisioner query request: %s", e, exc_info=True)
def startService(self): self.log.info("Starting up...") self.startupTime = now() self.log.info("Loading configuration...") self.config.reload() self.name = self.config["server_name"] self.serverID = self.config["server_id"] self.log.info("Loading storage...") self.storage = shelve.open(self.config["datastore_path"], writeback=True) self.storageSyncer = LoopingCall(self.storage.sync) self.storageSyncer.start(self.config.get("storage_sync_interval", 5), now=False) self.log.info("Starting processes...") self.pruneRecentlyQuit = LoopingCall(self.pruneQuit) self.pruneRecentlyQuit.start(10, now=False) self.pruneRecentChannels = LoopingCall(self.pruneChannels) self.pruneRecentChannels.start(15, now=False) self.log.info("Loading modules...") self._loadModules() self.log.info("Binding ports...") self._bindPorts() self.log.info("txircd started!") try: self._logFilter.setLogLevelForNamespace("txircd", LogLevel.levelWithName(self.config["log_level"])) except (KeyError, InvalidLogLevelError): self._logFilter.setLogLevelForNamespace("txircd", LogLevel.warn) self.runActionStandard("startup")
def wait_for_volume(self, name): """ Wait for a volume by the given name, owned by thus service, to exist. Polls the storage pool for the specified volume to appear. :param VolumeName name: The name of the volume. :return: A ``Deferred`` that fires with a :class:`Volume`. """ # Change this to not create the Volume instance right away. Instead, # try to find it by uuid/name in `check_for_volume`. If a Volume # instance there matches, use that object as the final result of the # Deferred returned by this method (it will have its other attributes # set correctly because they will be set correctly by enumerate). # FLOC-976 volume = Volume(uuid=self.uuid, name=name, service=self) def check_for_volume(volumes): if volume in volumes: call.stop() def loop(): d = self.enumerate() d.addCallback(check_for_volume) return d call = LoopingCall(loop) call.clock = self._reactor d = call.start(WAIT_FOR_VOLUME_INTERVAL) d.addCallback(lambda _: volume) return d
class WebSocketConnection(WebSocketProtocol): def __init__(self, stateObject): self.state = stateObject self.opcode = TEXT WebSocketProtocol.__init__(self, WebSocketReceiver(self)) self.finished = Deferred() self.pingLoop = LoopingCall(self.doPing) def doPing(self): self.receiver.transport.sendFrame(PING, '') def write(self, data): self.receiver.transport.sendFrame(self.opcode, data) def sendFrame(self, opcode, data): self.receiver.transport.sendFrame(opcode, data) def writeSequence(self, data): for chunk in data: self.write(chunk) def connectionLost(self, reason): if self.pingLoop.running: self.pingLoop.stop() self.finished.callback(self)
def __init__(self): """ Initialize the model. This doesn't add any documents yet. """ silenceGensim() self.dictionaries = dict() self.preprocessor = TokenizingPorter2Stemmer() #this dict keeps a model for every source type # (since e.g. RSS feeds should be treated separately from twitter feeds) self.models = dict() #this dict keeps a dictionary for every source type self.dictionaries = dict() self.queue = Queue() self.modelQueue = Queue() self.nodeCommunicator = NodeCommunicator(self, LISTEN_PORT) self.nodeCommunicator.registerWithNode(CORE_IP, REGISTER_PORT) # register this node with the core ln.info("LSA Initialized") self.updating = False loop = LoopingCall(self.update) loop.start(5) reactor.run()
class CacheService(Service): def __init__(self, *args, **kwargs): self.call = None self.node_updater = NodeCacheUpdater() self.cluster_updater = ClusterCacheUpdater() self.vm_updater = VirtualMachineCacheUpdater() self.job_updater = JobCacheUpdater() def update_cache(self): """ a single run of all update classes """ return DeferredList( [ self.vm_updater.update(), self.job_updater.update(), self.node_updater.update(), self.cluster_updater.update(), ] ) def startService(self): self.call = LoopingCall(self.update_cache) self.call.start(settings.PERIODIC_CACHE_REFRESH) def stopService(self): if self.call is not None: self.call.stop()
class Hoster: def __init__(self, screen, port): self.screen = screen self.port = port def start(self): # Set up the connection between the state and the network; # using queues gives a degree of seperation between the # communication and game logic, and making them be deferred # keeps it all asynchronous. inqueue = DeferredQueue() outqueue = DeferredQueue() #Initialize GameState gs = GameState(self,self.screen,1,inqueue,outqueue) #Create Looping Call self.lc = LoopingCall(gs.gameLoop) self.lc.start(.0166666666) #Begin Listening connfactory = CommandConnFactory(inqueue,outqueue) self.listening = reactor.listenTCP(self.port,connfactory) def stop(self, nextscreen=None): # Stop the GameState logic, and let go of the port on which we're listening self.lc.stop() self.listening.stopListening() # Start up the next screen, if there is one if nextscreen: nextscreen.start() else: reactor.stop()
def onJoin(self, details): log.msg("PiMonitor connected") extra = self.config.extra self._id = extra['id'] self.publish_temperature = True self.threshold = 0 self._tick = 0 self._cpu_temp_celsius = None def scanTemperature(): self._cpu_temp_celsius = float(open("/sys/class/thermal/thermal_zone0/temp").read()) / 1000. if self.publish_temperature: self.publish(u'io.crossbar.examples.pi.{}.tempmon.on_temperature'.format(self._id), self._tick, self._cpu_temp_celsius) self._tick += 1 if self.threshold > 0 and self._cpu_temp_celsius > self.threshold: self.publish(u'io.crossbar.examples.pi.{}.tempmon.on_threshold_exceeded'.format(self._id), self._tick, self._cpu_temp_celsius) scan = LoopingCall(scanTemperature) scan.start(1) for proc in [self.get_temperature, self.impose_stress, self.toggle_publish, self.set_threshold]: uri = u'io.crossbar.examples.pi.{}.tempmon.{}'.format(self._id, proc.__name__) yield self.register(proc, uri) log.msg("Registered {}".format(uri)) log.msg("PiMonitor ready.")
class EventGenerator(Node): """ (TEST) A timer-based event generator """ # --- Initialization def __init__(self): self.count = 0 # --- Interfaces def configure(self): self.events = ( 'event1', 'event2', ) def setup(self): self.status.set_GREEN() self.loop1 = LoopingCall(self.loop_cb1) self.loop1.start(3, False) self.loop2 = LoopingCall(self.loop_cb2) self.loop2.start(4, False) # --- Workers def loop_cb1(self): self.count += 1 self.sendEvent('event1', self.count) def loop_cb2(self): self.count += 1 self.sendEvent('event2', self.count)
def delayed_setup(self): self.feeds = [] self.failures.clear() self.feed_times.clear() self.targets.clear() self.tasks.clear() for name, target in self.config["targets"].items(): proto = target["protocol"] if proto in self.factory_manager.factories: self.targets[name] = target else: self.logger.warn("Unknown protocol '%s' in target '%s'" % (proto, name)) for feed in self.config["feeds"]: append = True for target in feed["targets"]: if target["name"] not in self.targets: self.logger.warn("Unknown target '%s' for feed '%s'" % (target["name"], feed["name"])) append = False break if append: self.feeds.append(feed) for feed in self.feeds: task = LoopingCall(self.check_feed, feed) self.tasks[feed["name"]] = task task.start(feed["frequency"])
def reconfigServiceWithBuildbotConfig(self, new_config): # first, enable or disable if new_config.metrics is None: self.disable() else: self.enable() metrics_config = new_config.metrics # Start up periodic logging log_interval = metrics_config.get('log_interval', 60) if log_interval != self.log_interval: if self.log_task: self.log_task.stop() self.log_task = None if log_interval: self.log_task = LoopingCall(self.report) self.log_task.clock = self._reactor self.log_task.start(log_interval) # same for the periodic task periodic_interval = metrics_config.get('periodic_interval', 10) if periodic_interval != self.periodic_interval: if self.periodic_task: self.periodic_task.stop() self.periodic_task = None if periodic_interval: self.periodic_task = LoopingCall(periodicCheck, self._reactor) self.periodic_task.clock = self._reactor self.periodic_task.start(periodic_interval) # upcall return util_service.ReconfigurableServiceMixin.reconfigServiceWithBuildbotConfig(self, new_config)
def __init__(self, initial_name, shared_path='/tmp/mark2'): self.socket_to = lambda n: os.path.join(shared_path, n + ".sock") self.socket_from = lambda p: os.path.splitext(os.path.basename(p))[0] self.client = None self.stats = {} self.system_users = SystemUsers() #read the config self.config = properties.load( properties.ClientProperties, open_resource('resources/mark2rc.default.properties'), os.path.expanduser('~/.mark2rc.properties')) assert self.config is not None self.stats_template = Template(self.config['stats']) #start apps self.apps = [] #start ui self.ui = UI(self.config.get_palette(), self.get_players, self.run_command, self.switch_server, self.connect_to_server, self.config.get_player_actions(), self.config.get_player_reasons()) for name, command in self.config.get_apps(): app = App(name, self.config.get_interval('apps'), self.app_update, self.config['stats.app_shell'], command) self.apps.append(app) #tasks t = LoopingCall(self.update_servers) t.start(self.config.get_interval('servers')) t = LoopingCall(self.update_users) t.start(self.config.get_interval('users')) t = LoopingCall(self.update_players) t.start(self.config.get_interval('players')) t = LoopingCall(self.update_stats) t.start(self.config.get_interval('stats')) self.connect_to_server(initial_name)
class WebWallet(object): """Represents a wallet, usable within the explorer web app.""" def __init__(self): self.wallet_name = None self.wallets = {} self.taskq = queue.Queue() # load wallets from disk self.load_wallets() self.looping_call = LoopingCall(self.miner_tick) tick_seconds = 1 self.looping_call.start(tick_seconds) def is_mining(self): """Returns True if mining is in progress. False otherwise.""" return self.taskq.qsize() > 0 def mine(self, block): """Compute proof of work.""" i = 0 print(f"target: {block.target}") print(f"block initial proof: {block.proof}") while True: proof = str(i) if block.validate_proof(proof=proof): block.proof = proof return block i += 1 def miner_tick(self): """Check whether any mining tasks were added, and if so get them from the task queue and execute them.""" while True: try: wallet, p2p = self.taskq.get(block=False, timeout=1) magic = p2p.bc.magic coinbase = Transaction("0", wallet.get_address(), 1, magic=magic) wallet.sign_transaction(coinbase) b = p2p.bc.new_block(tx=coinbase) self.mine(b) success = p2p.bc.discard_block(b) if success: p2p.broadcast_block(b) logger.debug("Successfully mined and broadcasted block.") else: logger.debug("Failed to discard block.") except queue.Empty: break def wallet_names(self): """Returns a list of names of generated wallets""" return self.wallets.keys() def load_wallets(self): """Load existing wallets from disk.""" count = 0 for path in glob("webwallet_*.wallet"): w = Wallet.load_keys(path) self.wallets[path] = w count += 1 logging.debug(f"Successfully loaded {count} wallets from disk") def generate_wallet(self): """Generate a new wallet.""" w = Wallet() w.create_keys() first_free_count = 1 while True: path = f"webwallet_{first_free_count}.wallet" if not os.path.exists(path): break first_free_count += 1 # save wallet to disk w.save_key(path) # save wallet to memory self.wallets[path] = w def set_active_wallet(self, wallet_name): """Set the wallet with name `wallet_name` as the active wallet.""" if wallet_name in self.wallets: self.wallet_name = wallet_name def active_wallet(self): """Returns the name of the active wallet.""" return self.wallet_name def get_active_wallet(self): """Returns the active wallet or None if there is no active wallet.""" return self.wallets.get(self.wallet_name, None) def send_tx(self, destination, quantity, p2p): """Perform a transaction.""" w = self.get_active_wallet() magic = p2p.bc.magic tx = Transaction(w.get_address(), destination, quantity, magic=magic) w.sign_transaction(tx) success = p2p.bc.add_transaction(tx) if success: # broadcast transaction to p2p network p2p.broadcast_tx(tx) return True else: logging.debug("Failed to add transaction.") return False def start_mining(self, p2p): """Start mining a block (add task to queue).""" wallet = self.get_active_wallet() self.taskq.put((wallet, p2p))
class AdapterPmMetrics: def __init__(self, device): self.pm_names = {'tx_64_pkts', 'tx_65_127_pkts', 'tx_128_255_pkts', 'tx_256_511_pkts', 'tx_512_1023_pkts', 'tx_1024_1518_pkts', 'tx_1519_9k_pkts', 'rx_64_pkts', 'rx_65_127_pkts', 'rx_128_255_pkts', 'rx_256_511_pkts', 'rx_512_1023_pkts', 'rx_1024_1518_pkts', 'rx_1519_9k_pkts'} self.device = device self.id = device.id self.name = 'ponsim_olt' # self.id = "abc" self.default_freq = 150 self.grouped = False self.freq_override = False self.pon_metrics_config = dict() self.nni_metrics_config = dict() self.lc = None for m in self.pm_names: self.pon_metrics_config[m] = PmConfig(name=m, type=PmConfig.COUNTER, enabled=True) self.nni_metrics_config[m] = PmConfig(name=m, type=PmConfig.COUNTER, enabled=True) def update(self, pm_config): if self.default_freq != pm_config.default_freq: # Update the callback to the new frequency. self.default_freq = pm_config.default_freq self.lc.stop() self.lc.start(interval=self.default_freq / 10) for m in pm_config.metrics: self.pon_metrics_config[m.name].enabled = m.enabled self.nni_metrics_config[m.name].enabled = m.enabled def make_proto(self): pm_config = PmConfigs( id=self.id, default_freq=self.default_freq, grouped=False, freq_override=False) for m in sorted(self.pon_metrics_config): pm = self.pon_metrics_config[m] # Either will do they're the same pm_config.metrics.extend([PmConfig(name=pm.name, type=pm.type, enabled=pm.enabled)]) return pm_config def collect_port_metrics(self, channel): rtrn_port_metrics = dict() stub = ponsim_pb2_grpc.PonSimStub(channel) stats = stub.GetStats(ponsim_pb2.PonSimMetricsRequest(port=0)) rtrn_port_metrics['pon'] = self.extract_pon_metrics(stats) rtrn_port_metrics['nni'] = self.extract_nni_metrics(stats) return rtrn_port_metrics def extract_pon_metrics(self, stats): rtrn_pon_metrics = dict() for m in stats.metrics: if m.port_name == "pon": for p in m.packets: if self.pon_metrics_config[p.name].enabled: rtrn_pon_metrics[p.name] = p.value return rtrn_pon_metrics def extract_nni_metrics(self, stats): rtrn_pon_metrics = dict() for m in stats.metrics: if m.port_name == "nni": for p in m.packets: if self.pon_metrics_config[p.name].enabled: rtrn_pon_metrics[p.name] = p.value return rtrn_pon_metrics def start_collector(self, callback): log.info("starting-pm-collection", device_name=self.name, device_id=self.device.id) prefix = 'voltha.{}.{}'.format(self.name, self.device.id) self.lc = LoopingCall(callback, self.device.id, prefix) self.lc.start(interval=self.default_freq / 10) def stop_collector(self): log.info("stopping-pm-collection", device_name=self.name, device_id=self.device.id) self.lc.stop()
def start_collector(self, callback): log.info("starting-pm-collection", device_name=self.name, device_id=self.device.id) prefix = 'voltha.{}.{}'.format(self.name, self.device.id) self.lc = LoopingCall(callback, self.device.id, prefix) self.lc.start(interval=self.default_freq / 10)
def __init__(self): self.storage_reload_task = LoopingCall(reloadStorageSchemas) self.aggregation_reload_task = LoopingCall(reloadAggregationSchemas)
class Window(object): def __init__(self, environment): self.environment = environment self.images = Images(FilePath("data").child("img2")) self.images.load() self.actions = deque() self.action = None self.center = Vector2D((0, 0)) def addAction(self, action): # TODO We no longer have actions? pass def startAction(self): if self.action == None: try: self.action = self.actions.popleft() self.action.start(5).addCallback(self.stopAction) except: pass def stopAction(self, ign): self.action = None self.startAction() def paint(self): """ Call C{paint} on all views which have been directly added to this Window. """ bg = self.images.images["background"] bgWidth = bg.width bgHeight = bg.height x = -(self.center * 20).x y = -(self.center * 20).y while x > 0: x -= bgWidth while y > 0: y -= bgHeight while x < self.screen.get_width(): #800:#480: j = y while j < self.screen.get_height(): #480:#800: self.screen.blit(bg._image, pygame.Rect(x, j, bgWidth, bgHeight)) j += bgHeight x += bgWidth self.environment.paint(self) if self.action: # self.action.draw(self.screen, Vector2D((240, 400))) pass pygame.display.flip() def setCenter(self, position): self.center = position def worldCoord(self, p): width = self.screen.get_width() height = self.screen.get_height() (cx, cy) = self.screen.get_rect().center return Vector2D(((p.x - cx) * self.environment.width) / width, ((p.y - cy) * self.environment.height) / height) def screenCoord(self, p): width = self.screen.get_width() height = self.screen.get_height() (cx, cy) = self.screen.get_rect().center return Vector2D((((p.x - self.center[0]) / (self.environment.width / 2)) * width) + cx, (((p.y - self.center[1]) / (self.environment.height / 2)) * height) + cy) def start(self, title): self.screen = pygame.display.get_surface() pygame.display.set_caption(title) self._renderCall = LoopingCall(self.paint) self._renderCall.start(0.03) (cx, cy) = self.screen.get_rect().center # print "envirn: " + str(self.environment.width) + " x " + str(self.environment.height) # print "screen: " + str(self.screen.get_width()) + " x " + str(self.screen.get_height()) # print "scrctr: " + str(cx) + ", " + str(cy) def stop(self): self._renderCall.stop()
def loop_call(self, delay, func, *arg, **kw): loop = LoopingCall(func, *arg, **kw) loop.start(delay, False) self.loops.add(loop) return loop
import time from twisted.internet import reactor, threads from twisted.internet.task import LoopingCall def blockingApiCall(arg): time.sleep(1) return arg def nonblockingCall(arg): print arg def printResult(result): print result def finish(result): reactor.stop() d = threads.deferToThread(blockingApiCall, "Goose") d.addCallback(printResult) d.addCallback(finish) LoopingCall(nonblockingCall, "Duck").start(.25) reactor.run()
class RewriteRuleManager: def __init__(self): self.rulesets = defaultdict(list) self.rules_file = None self.read_task = LoopingCall(self.read_rules) self.rules_last_read = 0.0 def clear(self, ruleset=None): if ruleset: self.rulesets[ruleset] = [] else: self.rulesets.clear() def rules(self, ruleset): return self.rulesets[ruleset] def read_from(self, rules_file): self.rules_file = rules_file self.read_rules() if not self.read_task.running: self.read_task.start(10, now=False) def read_rules(self): if not exists(self.rules_file): self.clear() return # Only read if the rules file has been modified try: mtime = getmtime(self.rules_file) except (OSError, IOError): log.err("Failed to get mtime of %s" % self.rules_file) return if mtime <= self.rules_last_read: return section = None for line in open(self.rules_file): line = line.strip() if line.startswith('#') or not line: continue if line.startswith('[') and line.endswith(']'): section = line[1:-1].lower() self.clear(section) elif '=' in line: pattern, replacement = line.split('=', 1) pattern, replacement = pattern.strip(), replacement.strip() try: rule = RewriteRule(pattern, replacement) except re.error: log.err( "Invalid regular expression in rewrite rule: '{0}'". format(pattern)) continue self.rulesets[section].append(rule) else: log.err("Invalid syntax: not a section heading or rule: '{0}'". format(line)) self.rules_last_read = mtime
# check cache size every 5 minutes _FLUSH_CACHE(_IDMAPPER_CACHE_MAXSIZE) if _MAINTENANCE_COUNT % 3600 == 0: # validate scripts every hour evennia.ScriptDB.objects.validate() if _MAINTENANCE_COUNT % 3700 == 0: # validate channels off-sync with scripts evennia.CHANNEL_HANDLER.update() ## Commenting this out, it is probably not needed ## with CONN_MAX_AGE set. Keeping it as a reminder ## if database-gone-away errors appears again /Griatch #if _MAINTENANCE_COUNT % 18000 == 0: # connection.close() maintenance_task = LoopingCall(_server_maintenance) maintenance_task.start(60, now=True) # call every minute #------------------------------------------------------------ # Evennia Main Server object #------------------------------------------------------------ class Evennia(object): """ The main Evennia server handler. This object sets up the database and tracks and interlinks all the twisted network services that make up evennia. """ def __init__(self, application): """ Setup the server.
class PahoMqttAdapter(BaseMqttAdapter, Service): # If connection fails at first, retry connecting each X seconds retry_interval = 5 def connect(self): """ Connect to MQTT broker. """ # TODO: Check if we can do asynchronous connection establishment. # Currently, this is done synchronously which could harm # other subsystems in timeout or otherwise blocking situations. # Make MQTT client identifier even more unique by adding process id pid = os.getpid() client_id = '{}:{}'.format(self.name, str(pid)) # Connection establishment self.client = mqtt.Client(client_id=client_id, clean_session=True) # Optionally authenticate connection if self.broker_username: self.client.username_pw_set(self.broker_username, self.broker_password) # Set event handlers self.client.on_connect = lambda *args: reactor.callFromThread( self.on_connect, *args) self.client.on_message = lambda *args: reactor.callFromThread( self.on_message, *args) self.client.on_log = lambda *args: reactor.callFromThread( self.on_log, *args) # Connect with retry self.connect_loop = LoopingCall(self.connect_with_retry) self.connect_loop.start(self.retry_interval, now=True) def connect_with_retry(self): try: self.client.connect(self.broker_host, port=self.broker_port, keepalive=60) self.connect_loop.stop() except: log.failure( u'Error connecting to MQTT broker but retrying each {retry_interval} seconds', retry_interval=self.retry_interval) return """ This is part of the threaded client interface. Call this once to start a new thread to process network traffic. This provides an alternative to repeatedly calling loop() yourself. """ self.client.loop_start() reactor.addSystemEventTrigger('before', 'shutdown', self.client.loop_stop, True) # The callback for when the client receives a CONNACK response from the server. def on_connect(self, client, userdata, flags, rc): """ on_connect(client, userdata, flags, rc): called when the broker responds to our connection request. flags is a dict that contains response flags from the broker: flags['session present'] - this flag is useful for clients that are using clean session set to 0 only. If a client with clean session=0, that reconnects to a broker that it has previously connected to, this flag indicates whether the broker still has the session information for the client. If 1, the session still exists. The value of rc determines success or not: 0: Connection successful 1: Connection refused - incorrect protocol version 2: Connection refused - invalid client identifier 3: Connection refused - server unavailable 4: Connection refused - bad username or password 5: Connection refused - not authorised 6-255: Currently unused. """ log.debug( "Connected to MQTT. userdata={userdata}, flags={flags}, rc={rc}", userdata=userdata, flags=flags, rc=rc) # Subscribing in on_connect() means that if we lose the connection and # reconnect then subscriptions will be renewed. #client.subscribe("$SYS/#") self.subscribe() # The callback for when a PUBLISH message is received from the server. def on_message(self, client, userdata, message): """ on_message(client, userdata, message): called when a message has been received on a topic that the client subscribes to. The message variable is a MQTTMessage that describes all of the message parameters. """ # TODO: Do something with "client" object (paho.mqtt.client.Client) topic = message.topic payload = message.payload # Make metadata dictionary to be passed as kwargs later # Mungle topic and payload out of metadata metadata = message.__dict__.copy() del metadata['topic'] del metadata['payload'] # Mungle userdata into message metadata['userdata'] = userdata if not topic.endswith('error.json'): log.debug( 'on_message: name={name}, topic={topic}, payload={payload}, kwargs={kwargs}', name=self.name, topic=topic, payload=payload, kwargs=metadata) return self.callback(topic=topic, payload=payload, **metadata) def publish(self, topic, payload): log.debug(u'Publishing to topic={topic}, payload={payload}', topic=topic, payload=payload) return self.client.publish(topic, payload) def subscribe(self, *args): #d = self.protocol.subscribe("foo/bar/baz", 0) log.info(u"Subscribing to topics {subscriptions}. client={client}", subscriptions=self.subscriptions, client=self.client) for topic in self.subscriptions: log.info(u"Subscribing to topic '{topic}'", topic=topic) # Topic name **must not** be unicode, so casting to string e = self.client.subscribe(str(topic), qos=0) def on_log(self, client, userdata, level, buf): """ on_log(client, userdata, level, buf): called when the client has log information. Define to allow debugging. The level variable gives the severity of the message and will be one of MQTT_LOG_INFO, MQTT_LOG_NOTICE, MQTT_LOG_WARNING, MQTT_LOG_ERR, and MQTT_LOG_DEBUG. The message itself is in buf. """ log.debug(u'{message}. level={level_mqtt}, userdata={userdata}', message=buf, level_mqtt=level, userdata=userdata)
def __init__(self): self.rulesets = defaultdict(list) self.rules_file = None self.read_task = LoopingCall(self.read_rules) self.rules_last_read = 0.0
def __init__(self, interface: bytes, config_dict: Dict[str, Any]) -> None: # logfile path relative to config dir if not abs path log_filename = logfile.get() if log_filename.strip(): # catches empty filename if not os.path.isabs(log_filename): log_filename = os.path.join(config.config_dir, log_filename) ensure_dir_exists(log_filename) if logging_rotate_daily.get(): logging_file = DailyLogFile(log_filename, '.') else: logging_file = open(log_filename, 'a') predicate = LogLevelFilterPredicate( LogLevel.levelWithName(loglevel.get())) observers = [ FilteringLogObserver(textFileLogObserver(sys.stderr), [predicate]), FilteringLogObserver(textFileLogObserver(logging_file), [predicate]) ] globalLogBeginner.beginLoggingTo(observers) log.info('piqueserver started on %s' % time.strftime('%c')) self.config = config_dict if random_rotation.get(): self.map_rotator_type = random_choice_cycle else: self.map_rotator_type = itertools.cycle self.default_time_limit = default_time_limit.get() self.default_cap_limit = cap_limit.get() self.advance_on_win = int(advance_on_win.get()) self.win_count = itertools.count(1) self.bans = NetworkDict() # attempt to load a saved bans list try: with open(os.path.join(config.config_dir, bans_file.get()), 'r') as f: self.bans.read_list(json.load(f)) log.debug("loaded {count} bans", count=len(self.bans)) except FileNotFoundError: log.debug("skip loading bans: file unavailable", count=len(self.bans)) except IOError as e: log.error('Could not read bans file ({}): {}'.format( bans_file.get(), e)) except ValueError as e: log.error('Could not parse bans file ({}): {}'.format( bans_file.get(), e)) self.hard_bans = set() # possible DDoS'ers are added here self.player_memory = deque(maxlen=100) if len(self.name) > MAX_SERVER_NAME_SIZE: log.warn('(server name too long; it will be truncated to "%s")' % (self.name[:MAX_SERVER_NAME_SIZE])) self.respawn_time = respawn_time_option.get() self.respawn_waves = respawn_waves.get() # since AoS only supports CTF and TC at a protocol level, we need to get # the base game mode if we are using a custom game mode. game_mode_name = game_mode.get() if game_mode_name == 'ctf': self.game_mode = CTF_MODE elif game_mode.get() == 'tc': self.game_mode = TC_MODE elif self.game_mode not in [CTF_MODE, TC_MODE]: raise ValueError( 'invalid game mode: custom game mode "{}" does not set ' 'protocol.game_mode to one of TC_MODE or CTF_MODE. Are ' 'you sure the thing you have specified is a game mode?'.format( game_mode_name)) self.game_mode_name = game_mode.get().split('.')[-1] self.team1_name = team1_name.get()[:9] self.team2_name = team2_name.get()[:9] self.team1_color = tuple(team1_color.get()) self.team2_color = tuple(team2_color.get()) self.friendly_fire = friendly_fire.get() self.friendly_fire_on_grief = friendly_fire_on_grief.get() self.friendly_fire_time = grief_friendly_fire_time.get() self.spade_teamkills_on_grief = spade_teamkills_on_grief.get() self.fall_damage = fall_damage.get() self.teamswitch_interval = teamswitch_interval.get() self.teamswitch_allowed = teamswitch_allowed.get() self.max_players = max_players.get() self.melee_damage = melee_damage.get() self.max_connections_per_ip = max_connections_per_ip.get() self.passwords = passwords.get() self.server_prefix = server_prefix.get() self.time_announcements = time_announcements.get() self.balanced_teams = balanced_teams.get() self.login_retries = login_retries.get() # voting configuration self.default_ban_time = default_ban_duration.get() self.speedhack_detect = speedhack_detect.get() self.rubberband_distance = rubberband_distance.get() if user_blocks_only.get(): self.user_blocks = set() self.set_god_build = set_god_build.get() if ssh_enabled.get(): from piqueserver.ssh import RemoteConsole self.remote_console = RemoteConsole(self) irc = irc_options.get() if irc.get('enabled', False): from piqueserver.irc import IRCRelay self.irc_relay = IRCRelay(self, irc) if status_server_enabled.get(): from piqueserver.statusserver import StatusServer self.status_server = StatusServer(self) ensureDeferred(self.status_server.listen()) if ban_publish.get(): from piqueserver.banpublish import PublishServer self.ban_publish = PublishServer(self, ban_publish_port.get()) if bans_config_urls.get(): from piqueserver import bansubscribe self.ban_manager = bansubscribe.BanManager(self) ensureDeferred(as_deferred(self.ban_manager.start())) self.start_time = time.time() self.end_calls = [] # TODO: why is this here? create_console(self) for user_type, func_names in rights.get().items(): for func_name in func_names: commands.add_rights(user_type, func_name) if everyone_is_admin.get(): self.everyone_is_admin = True self.port = port_option.get() ServerProtocol.__init__(self, self.port, interface) self.host.intercept = self.receive_callback try: self.set_map_rotation(self.config['rotation']) except MapNotFound as e: log.critical('Invalid map in map rotation (%s), exiting.' % e.map) raise SystemExit map_load_d = self.advance_rotation() # discard the result of the map advance for now map_load_d.addCallback(lambda x: self._post_init()) ip_getter = ip_getter_option.get() if ip_getter: ensureDeferred(as_deferred(self.get_external_ip(ip_getter))) self.new_release = None notify_new_releases = config.option("release_notifications", default=True) if notify_new_releases.get(): ensureDeferred(as_deferred(self.watch_for_releases())) self.vacuum_loop = LoopingCall(self.vacuum_bans) # Run the vacuum every 6 hours, and kick it off it right now self.vacuum_loop.start(60 * 60 * 6, True) reactor.addSystemEventTrigger('before', 'shutdown', lambda: ensureDeferred(self.shutdown()))
class RollbackProtocol(protocol): rollback_in_progress = False rollback_max_rows = 10 # per 'cycle', intended to cap cpu usage rollback_max_packets = 180 # per 'cycle' cap for (unique packets * players) rollback_max_unique_packets = 12 # per 'cycle', each block op is at least 1 rollback_time_between_cycles = 0.06 rollback_time_between_progress_updates = 10.0 rollback_start_time = None rollback_last_chat = None rollback_rows = None rollback_total_rows = None # rollback def start_rollback(self, connection, mapname, start_x, start_y, end_x, end_y, ignore_indestructable = True): if self.rollback_in_progress: return S_ROLLBACK_IN_PROGRESS if mapname is None: map = self.rollback_map else: try: maps = check_rotation([mapname]) if not maps: return S_INVALID_MAP_NAME map = Map(maps[0]).data except MapNotFound as error: return error.message name = (connection.name if connection is not None else S_AUTOMATIC_ROLLBACK_PLAYER_NAME) message = S_ROLLBACK_COMMENCED.format(player = name) self.send_chat(message, irc = True) self.packet_generator = self.create_rollback_generator(self.map, map, start_x, start_y, end_x, end_y, ignore_indestructable) self.rollback_in_progress = True self.rollback_start_time = time.time() self.rollback_last_chat = self.rollback_start_time self.rollback_rows = 0 self.rollback_total_rows = end_x - start_x self.cycle_call = LoopingCall(self.rollback_cycle) self.cycle_call.start(self.rollback_time_between_cycles) def cancel_rollback(self, connection): if not self.rollback_in_progress: return S_NO_ROLLBACK_IN_PROGRESS result = S_ROLLBACK_CANCELLED.format(player = connection.name) self.end_rollback(result) def end_rollback(self, result): self.rollback_in_progress = False self.cycle_call.stop() self.cycle_call = None self.packet_generator = None self.update_entities() message = S_ROLLBACK_ENDED.format(result = result) self.send_chat(message, irc = True) def rollback_cycle(self): if not self.rollback_in_progress: return try: sent_unique = sent_total = rows = 0 while 1: if rows > self.rollback_max_rows: break if sent_unique > self.rollback_max_unique_packets: break if sent_total > self.rollback_max_packets: break sent = self.packet_generator.next() sent_unique += sent sent_total += sent * len(self.connections) rows += (sent == 0) self.rollback_rows += rows if (time.time() - self.rollback_last_chat > self.rollback_time_between_progress_updates): self.rollback_last_chat = time.time() progress = float(self.rollback_rows) / self.rollback_total_rows if progress < 1.0: message = S_ROLLBACK_PROGRESS.format(percent = progress) self.send_chat(message) else: self.send_chat(S_ROLLBACK_COLOR_PASS) except (StopIteration): elapsed = time.time() - self.rollback_start_time message = S_ROLLBACK_TIME_TAKEN.format(seconds = elapsed) self.end_rollback(message) def create_rollback_generator(self, cur, new, start_x, start_y, end_x, end_y, ignore_indestructable): surface = {} block_action = BlockAction() block_action.player_id = 31 set_color = SetColor() set_color.value = make_color(*NON_SURFACE_COLOR) set_color.player_id = 31 self.send_contained(set_color, save = True) old = cur.copy() check_protected = hasattr(protocol, 'protected') for x in xrange(start_x, end_x): block_action.x = x for y in xrange(start_y, end_y): block_action.y = y if check_protected and self.is_protected(x, y, 0): continue for z in xrange(63): action = None cur_solid = cur.get_solid(x, y, z) new_solid = new.get_solid(x, y, z) if cur_solid and not new_solid: if (not ignore_indestructable and self.is_indestructable(x, y, z)): continue else: action = DESTROY_BLOCK cur.remove_point(x, y, z) elif new_solid: new_is_surface = new.is_surface(x, y, z) if new_is_surface: new_color = new.get_color(x, y, z) if not cur_solid and new_is_surface: surface[(x, y, z)] = new_color elif not cur_solid and not new_is_surface: action = BUILD_BLOCK cur.set_point(x, y, z, NON_SURFACE_COLOR) elif cur_solid and new_is_surface: old_is_surface = old.is_surface(x, y, z) if old_is_surface: old_color = old.get_color(x, y, z) if not old_is_surface or old_color != new_color: surface[(x, y, z)] = new_color action = DESTROY_BLOCK cur.remove_point(x, y, z) if action is not None: block_action.z = z block_action.value = action self.send_contained(block_action, save = True) yield 1 yield 0 last_color = None block_action.value = BUILD_BLOCK for pos, color in sorted(surface.iteritems(), key = operator.itemgetter(1)): x, y, z = pos packets_sent = 0 if color != last_color: set_color.value = make_color(*color) self.send_contained(set_color, save = True) packets_sent += 1 last_color = color cur.set_point(x, y, z, color) block_action.x = x block_action.y = y block_action.z = z self.send_contained(block_action, save = True) packets_sent += 1 yield packets_sent def on_map_change(self, map): self.rollback_map = map.copy() protocol.on_map_change(self, map) def on_map_leave(self): if self.rollback_in_progress: self.end_rollback(S_MAP_CHANGED) protocol.on_map_leave(self) def on_game_end(self): if rollback_on_game_end: self.start_rollback(None, None, 0, 0, 512, 512, False) protocol.on_game_end(self)
class WebInterface(YomboLibrary): """ Web interface framework. """ webapp = Klein() # Like Flask, but for twisted visits = 0 alerts = OrderedDict() starting = True already_starting_web_servers = False hook_listeners = {} # special way to toss hook calls to routes. def __str__(self): """ Returns the name of the library. :return: Name of the library :rtype: string """ return "Yombo web interface library" def _init_(self, **kwargs): self.enabled = self._Configs.get('webinterface', 'enabled', True) if not self.enabled: return self.gateway_id = self._Configs.get2('core', 'gwid', 'local', False) # self._LocalDB = self._Loader.loadedLibraries['localdb'] self._current_dir = self._Atoms.get('yombo.path') + "/yombo" self._dir = '/lib/webinterface/' self._build_dist() # Make all the JS and CSS files self.secret_pin_totp = self._Configs.get2( 'webinterface', 'auth_pin_totp', yombo.utils.random_string( length=16, letters='ABCDEFGHIJKLMNOPQRSTUVWXYZ234567')) self._VoiceCmds = self._Loader.loadedLibraries['voicecmds'] self.misc_wi_data = {} self.wi_port_nonsecure = self._Configs.get2('webinterface', 'nonsecure_port', 8080) self.wi_port_secure = self._Configs.get2('webinterface', 'secure_port', 8443) self.webapp.templates = jinja2.Environment( loader=jinja2.FileSystemLoader(self._current_dir)) self.setup_basic_filters() self.web_interface_listener = None self.web_interface_ssl_listener = None self.api_stream_spectators = {} # Load API routes route_api_v1_automation(self.webapp) route_api_v1_command(self.webapp) route_api_v1_device(self.webapp) route_api_v1_device_command(self.webapp) route_api_v1_gateway(self.webapp) route_api_v1_module(self.webapp) route_api_v1_notification(self.webapp) route_api_v1_server(self.webapp) route_api_v1_statistics(self.webapp) route_api_v1_stream(self.webapp, self) route_api_v1_system(self.webapp) # Load devtool routes route_devtools_config(self.webapp) route_devtools_config_commands(self.webapp) route_devtools_config_device_types(self.webapp) route_devtools_config_input_types(self.webapp) route_devtools_config_modules(self.webapp) route_devtools_config_variables(self.webapp) route_devtools_debug(self.webapp) # Load web server routes route_apiauth(self.webapp) route_atoms(self.webapp) route_automation(self.webapp) route_configs(self.webapp) route_devices(self.webapp) route_locations(self.webapp) route_devtools_config(self.webapp) route_gateways(self.webapp) route_home(self.webapp) route_misc(self.webapp) route_modules(self.webapp) route_notices(self.webapp) route_panel(self.webapp) route_setup_wizard(self.webapp) route_statistics(self.webapp) route_states(self.webapp) route_system(self.webapp) route_voicecmds(self.webapp) self.temp_data = ExpiringDict(max_age_seconds=1800) self.web_server_started = False self.web_server_ssl_started = False self.web_factory = None # just here to set a password if it doesn't exist. mqtt_password = self._Configs.get('mqtt_users', 'panel.webinterface', yombo.utils.random_string()) # def _start_(self): # self.webapp.templates.globals['_'] = _ # i18n @property def operating_mode(self): return self._Loader.operating_mode def _load_(self, **kwargs): if not self.enabled: return self.module_config_links = {} self.auth_pin = self._Configs.get2( 'webinterface', 'auth_pin', yombo.utils.random_string( length=4, letters=yombo.utils.human_alpabet()).lower()) self.auth_pin_totp = self._Configs.get2( 'webinterface', 'auth_pin_totp', yombo.utils.random_string(length=16)) self.auth_pin_type = self._Configs.get2('webinterface', 'auth_pin_type', 'pin') self.auth_pin_required = self._Configs.get2('webinterface', 'auth_pin_required', True) # self.web_factory = Yombo_Site(self.webapp.resource(), None, logPath='/dev/null') self.web_factory = Yombo_Site(self.webapp.resource(), None, logPath=None) self.web_factory.setup_log_queue(self) self.web_factory.noisy = False # turn off Starting/stopping message # self.web_factory.sessionFactory = YomboSession self.displayTracebacks = False self._display_pin_console_at = 0 self.misc_wi_data[ 'gateway_configured'] = self._home_gateway_configured() self.misc_wi_data['gateway_label'] = self._Configs.get2( 'core', 'label', 'Yombo Gateway', False) self.misc_wi_data['operating_mode'] = self.operating_mode self.misc_wi_data['notifications'] = self._Notifications self.misc_wi_data[ 'notification_priority_map_css'] = NOTIFICATION_PRIORITY_MAP_CSS self.misc_wi_data['breadcrumb'] = [] # self.functions = { # 'yes_no': yombo.utils.is_yes_no, # } self.webapp.templates.globals[ 'local_gateway'] = self._Gateways.get_local() self.webapp.templates.globals['commands'] = self._Commands self.webapp.templates.globals['devices'] = self._Devices self.webapp.templates.globals['gateways'] = self._Gateways self.webapp.templates.globals['misc_wi_data'] = self.misc_wi_data self.webapp.templates.globals['webinterface'] = self # self.webapp.templates.globals['func'] = self.functions self.starting = False self.start_web_servers() @inlineCallbacks def _start_(self, **kwargs): self._Notifications.add({ 'title': 'System still starting', 'message': 'Still starting up. Please wait.', 'source': 'Web Interface Library', 'persist': True, 'priority': 'high', 'always_show': True, 'always_show_allow_clear': False, 'id': 'webinterface:starting', }) added_notification = True self._get_nav_side_items() module_configs = yield yombo.utils.global_invoke_all( '_webinterface_module_config_', called_by=self, ) for component_label, link in module_configs.items(): if link is None: continue component = self._Loader.get_loaded_component(component_label) self.module_config_links[component._module_id] = link print("Module_configs: %s" % self.module_config_links) def _started_(self, **kwargs): # if self.operating_mode != 'run': self._display_pin_console_at = int(time()) self.display_pin_console() self._Notifications.delete('webinterface:starting') self.send_hook_listeners_ping_loop = LoopingCall( self.send_hook_listeners_ping_loop) self.send_hook_listeners_ping_loop.start(55, True) def send_hook_listeners_ping_loop(self): route_api_v1_stream_broadcast(self, 'ping', int(time())) def register_hook(self, name, thecallback): if name not in self.hook_listeners: self.hook_listeners[name] = [] self.hook_listeners[name].append(thecallback) @inlineCallbacks def _yombo_universal_hook_(self, hook_name=None, **kwargs): """ Implements the universal hook. :param kwargs: :return: """ # print("web uni hook: %s" % hook_name) if hook_name in self.hook_listeners: for a_callback in self.hook_listeners[hook_name]: # print("web uni hook a_callback: %s" % a_callback) d = Deferred() d.addCallback(lambda ignored: maybeDeferred( a_callback, self, hook_name=hook_name, **kwargs)) d.addErrback(self.yombo_universal_hook_failure, hook_name, a_callback) d.callback(1) yield d def yombo_universal_hook_failure(self, failure, hook_name, acallback): logger.warn( "---==(failure WI:universal hook for hook ({hook_name})==----", hook_name=hook_name) logger.warn("--------------------------------------------------------") logger.warn("{acallback}", acallback=acallback) logger.warn("{failure}", failure=failure) logger.warn("--------------------------------------------------------") raise RuntimeError("failure during module invoke for hook: %s" % failure) def check_have_required_nodes(self): try: node = yield self._Nodes.get('main_page', 'webinterface_page') except KeyError as e: pass # add base node... @inlineCallbacks def change_ports(self, port_nonsecure=None, port_secure=None): if port_nonsecure is None and port_secure is None: logger.info("Asked to change ports, but nothing has changed.") return if port_nonsecure is not None: if port_nonsecure != self.wi_port_nonsecure(): self.wi_port_nonsecure(set=port_nonsecure) logger.info( "Changing port for the non-secure web interface: {port}", port=port_nonsecure) if self.web_server_started: yield self.web_interface_listener.stopListening() self.web_server_started = False if port_secure is not None: if port_secure != self.wi_port_secure(): self.wi_port_secure(set=port_secure) logger.info( "Changing port for the secure web interface: {port}", port=port_secure) if self.web_server_ssl_started: yield self.web_interface_ssl_listener.stopListening() self.web_server_ssl_started = False self.start_web_servers() # @inlineCallbacks def start_web_servers(self): if self.already_starting_web_servers is True: return self.already_starting_web_servers = True logger.debug("starting web servers") if self.web_server_started is False: if self.wi_port_nonsecure() == 0: logger.warn( "Non secure port has been disabled. With gateway stopped, edit yomobo.ini and change: webinterface->nonsecure_port" ) else: self.web_server_started = True port_attempts = 0 while port_attempts < 100: try: self.web_interface_listener = reactor.listenTCP( self.wi_port_nonsecure() + port_attempts, self.web_factory) break except Exception as e: port_attempts += 1 if port_attempts >= 100: logger.warn( "Unable to start web server, no available port could be found. Tried: {starting} - {ending}", starting=self.wi_port_secure(), ending=self.wi_port_secure() + port_attempts) elif port_attempts > 0: self._Configs.set('webinterface', 'nonsecure_port', self.wi_port_nonsecure() + port_attempts) logger.warn("Web interface is on a new port: {new_port}", new_port=self.wi_port_nonsecure() + port_attempts) if self.web_server_ssl_started is False: if self.wi_port_secure() == 0: logger.warn( "Secure port has been disabled. With gateway stopped, edit yomobo.ini and change: webinterface->secure_port" ) else: self.web_server_ssl_started = True cert = self._SSLCerts.get('lib_webinterface') privkeypyssl = crypto.load_privatekey(crypto.FILETYPE_PEM, cert['key']) certpyssl = crypto.load_certificate(crypto.FILETYPE_PEM, cert['cert']) if cert['chain'] is not None: chainpyssl = [ crypto.load_certificate(crypto.FILETYPE_PEM, cert['chain']) ] else: chainpyssl = None # print("web ssl cert: key_crypt %s" % cert['key_crypt']) # print("web ssl cert: key_crypt type %s" % type(cert['key_crypt'])) # # print("web ssl cert: key_crypt type %s" % type(cert['key_crypt'][0])) # # print("web ssl cert: key_crypt type %s" % type(cert['key_crypt'][1])) # print("web ssl cert: privkeypyssl %s" % privkeypyssl) # print("web ssl cert: privkeypyssl type: %s" % type(privkeypyssl)) # print("web ssl cert: privkeypyssl.serial: %s" % privkeypyssl.__dict__) # # print("web ssl cert: cert_crypt %s" % cert['cert_crypt']) # print("web ssl cert: cert_crypt.type %s" % type(cert['cert_crypt'])) # print("web ssl cert: certpyssl %s" % certpyssl) # print("web ssl cert: certpyssl.type %s" % type(certpyssl)) # print("web ssl cert: chain_crypt %s" % cert['chain_crypt']) # print("web ssl cert: chain_crypt.type %s" % type(cert['chain_crypt'])) # print("web ssl cert: chainpyssl %s" % certpyssl) # print("web ssl cert: chainpyssl.type %s" % type(certpyssl)) # # # chainpyssl = [crypto.load_certificate(crypto.FILETYPE_PEM, cert['chain'])] # # chainpyssl = None contextFactory = ssl.CertificateOptions( privateKey=cert['key_crypt'], certificate=cert['cert_crypt'], extraCertChain=cert['chain_crypt']) port_attempts = 0 while port_attempts < 100: try: # print("about to start ssl listener on port: %s" % self.wi_port_secure()) self.web_interface_ssl_listener = reactor.listenSSL( self.wi_port_secure(), self.web_factory, contextFactory) break except Exception as e: port_attempts += 1 if port_attempts >= 100: logger.warn( "Unable to start secure web server, no available port could be found. Tried: {starting} - {ending}", starting=self.wi_port_secure(), ending=self.wi_port_secure() + port_attempts) elif port_attempts > 0: self._Configs.set('webinterface', 'secure_port', self.wi_port_secure() + port_attempts) logger.warn( "Secure (tls/ssl) web interface is on a new port: {new_port}", new_port=self.wi_port_secure() + port_attempts) logger.debug("done starting web servers") self.already_starting_web_servers = False def _configuration_set_(self, **kwargs): """ Receive configuruation updates and adjust as needed. :param kwargs: section, option(key), value :return: """ section = kwargs['section'] option = kwargs['option'] value = kwargs['value'] # if section == 'core': # if option == 'label': # self.misc_wi_data['gateway_label'] = value if self.starting is True: return if section == 'webinterface': if option == 'nonsecure_port': self.change_ports(port_nonsecure=value) elif option == 'secure_port': self.change_ports(port_secure=value) def _sslcerts_(self, **kwargs): """ Called to collect to ssl cert requirements. :param kwargs: :return: """ fqdn = self._Configs.get('dns', 'fqdn', None, False) if fqdn is None: logger.warn( "Unable to create webinterface SSL cert: DNS not set properly." ) return cert = {} cert['sslname'] = "lib_webinterface" cert['sans'] = [ 'localhost', 'l', 'local', 'i', 'e', 'internal', 'external', str(int(time())) ] cert['cn'] = cert['sans'][0] cert['callback'] = self.new_ssl_cert return cert @inlineCallbacks def new_ssl_cert(self, newcert, **kwargs): """ Called when a requested certificate has been signed or updated. If needed, this function will function will restart the SSL service if the current certificate has expired or is a self-signed cert. :param kwargs: :return: """ logger.info("Got a new cert! About to install it.") if self.web_server_ssl_started is not None: yield self.web_interface_ssl_listener.stopListening() self.web_server_ssl_started = False self.start_web_servers() @inlineCallbacks def _unload_(self, **kwargs): if hasattr(self, 'web_factory'): if self.web_factory is not None: yield self.web_factory.save_log_queue() # def WebInterface_configuration_details(self, **kwargs): # return [{'webinterface': { # 'enabled': { # 'description': { # 'en': 'Enables/disables the web interface.', # } # }, # 'port': { # 'description': { # 'en': 'Port number for the web interface to listen on.' # } # } # }, # }] @webapp.route('/<path:catchall>') @require_auth() def page_404(self, request, session, catchall): request.setResponseCode(404) page = self.get_template(request, self._dir + 'pages/404.html') return page.render() @webapp.handle_errors(NotFound) @require_auth() def notfound(self, request, failure): request.setResponseCode(404) return 'Not found, I say' def display_pin_console(self): print("###########################################################") print("# #") if self.operating_mode != 'run': print( "# The Yombo Gateway website is running in #") print( "# configuration only mode. #") print( "# #") dns_fqdn = self._Configs.get('dns', 'fqdn', None, False) if dns_fqdn is None: local_hostname = "127.0.0.1" internal_hostname = self._Configs.get('core', 'localipaddress_v4') external_hostname = self._Configs.get('core', 'externalipaddress_v4') local = "http://%s:%s" % (local_hostname, self.wi_port_nonsecure()) internal = "http://%s:%s" % (internal_hostname, self.wi_port_nonsecure()) external = "https://%s:%s" % (external_hostname, self.wi_port_secure()) print( "# The gateway can be accessed from the following urls: #") print( "# #") print( "# On local machine: #") print("# %-54s #" % local) print( "# #") print( "# On local network: #") print("# %-54s #" % internal) print( "# #") print( "# From external network (check port forwarding): #") print("# %-54s #" % external) else: website_url = "http://%s" % dns_fqdn print( "# The gateway can be accessed from the following url: #") print( "# #") print( "# From anywhere: #") print("# %-54s #" % website_url) print("# #") print("# #") print("# Web Interface access pin code: #") print("# %-25s #" % self.auth_pin()) print("# #") print("###########################################################") def i18n(self, request): """ Gets a translator based on the language the browser provides us. :param request: The browser request. :return: """ return web_translator(self, request) @inlineCallbacks def _get_nav_side_items(self, **kwargs): """ Called before modules have their _prestart_ function called (after _load_). This implements the hook "webinterface_add_routes" and calls all libraries and modules. It allows libs and modules to add menus to the web interface and provide additional funcationality. **Usage**: .. code-block:: python def ModuleName_webinterface_add_routes(self, **kwargs): return { 'nav_side': [ { 'label1': 'Tools', 'label2': 'MQTT', 'priority1': 3000, 'priority2': 10000, 'icon': 'fa fa-wrench fa-fw', 'url': '/tools/mqtt', 'tooltip': '', 'opmode': 'run', }, ], 'routes': [ self.web_interface_routes, ], } """ # first, lets get the top levels already defined so children don't re-arrange ours. top_levels = {} temp_list = sorted(NAV_SIDE_MENU, key=itemgetter('priority1', 'priority2')) for item in temp_list: label1 = item['label1'] if label1 not in temp_list: top_levels[label1] = item['priority1'] nav_side_menu = NAV_SIDE_MENU.copy() add_on_menus = yield yombo.utils.global_invoke_all( '_webinterface_add_routes_', called_by=self, ) for component, options in add_on_menus.items(): if 'nav_side' in options: for new_nav in options['nav_side']: if isinstance(new_nav['priority1'], int) is False: new_nav['priority1'] = top_levels[new_nav['label1']] nav_side_menu.append(new_nav) if 'menu_priorities' in options: # allow modules to change the ording of top level menus for label, priority in options['menu_priorities'].items(): top_levels[label] = priority if 'routes' in options: for new_route in options['routes']: new_route(self.webapp) # build menu tree self.misc_wi_data['nav_side'] = OrderedDict() temp_list = sorted(nav_side_menu, key=itemgetter('priority1', 'priority2')) for item in temp_list: label1 = item['label1'] if label1 not in self.misc_wi_data['nav_side']: self.misc_wi_data['nav_side'][label1] = [] self.misc_wi_data['nav_side'][label1].append(item) self.starting = False def add_alert(self, message, level='info', dismissable=True, type='session', deletable=True): """ Add an alert to the stack. :param level: info, warning, error :param message: :return: """ rand = yombo.utils.random_string(length=12) self.alerts[rand] = { 'type': type, 'level': level, 'message': message, 'dismissable': dismissable, 'deletable': deletable, } return rand def make_alert(self, message, level='info', type='session', dismissable=False): """ Add an alert to the stack. :param level: info, warning, error :param message: :return: """ return { 'level': level, 'message': message, 'dismissable': dismissable, } def get_alerts(self, type=None, session=None): """ Retrieve a list of alerts for display. """ if type is None: type = 'session' show_alerts = OrderedDict() for keyid in list(self.alerts.keys()): if self.alerts[keyid]['type'] == type: show_alerts[keyid] = self.alerts[keyid] if type == 'session': del self.alerts[keyid] return show_alerts def get_template(self, request, template_path): request.setHeader('server', 'Yombo/1.0') return self.webapp.templates.get_template(template_path) def redirect(self, request, redirect_path): request.setHeader('server', 'Yombo/1.0') request.redirect(redirect_path) def _tpl_home_gateway_configured(self): if not self._home_gateway_configured(): return "This gateway is not properly configured. Click _here_ to run the configuration wizard." else: return "" def _home_gateway_configured(self): gwuuid = self._Configs.get("core", "gwuuid", None, False) gwhash = self._Configs.get("core", "gwhash", None, False) gpgkeyid = self._Configs.get('gpg', 'keyid', None, False) if gwuuid is None or gwhash is None or gpgkeyid is None: return False else: return True def _get_parms(self, request): return parse_qs(urlparse(request.uri).query) def format_markdown(self, input_text, formatting=None): if formatting == 'restructured' or formatting is None: return publish_parts(input_text, writer_name='html')['html_body'] elif formatting == 'markdown': return markdown.markdown(input_text, extensions=[ 'markdown.extensions.nl2br', 'markdown.extensions.codehilite' ]) return input_text def make_link(self, link, link_text, target=None): if link == '' or link is None or link.lower() == "None": return "None" if target is None: target = "_self" return '<a href="%s" target="%s">%s</a>' % (link, target, link_text) def request_get_default(self, request, name, default, offset=None): if offset == None: offset = 0 try: return request.args.get(name)[offset] except: return default def home_breadcrumb(self, request): self.add_breadcrumb(request, "/?", "Home") def add_breadcrumb(self, request, url=None, text=None, show=None, style=None, data=None): if hasattr(request, 'breadcrumb') is False: request.breadcrumb = [] self.misc_wi_data['breadcrumb'] = request.breadcrumb if show is None: show = True if style is None: style = 'link' elif style == 'select_groups': items = {} for option_label, option_data in data.items(): items[option_label] = [] for select_text, select_url in option_data.items(): selected = '' option_style = 'None' if select_url.startswith("$"): selected = 'selected' select_url = select_url[1:] elif select_url.startswith("#"): option_style = 'divider' items[option_label].append({ 'option_style': option_style, 'text': select_text, 'url': select_url, 'selected': selected, }) data = items elif style == 'select': items = [] for select_text, select_url in data.items(): selected = '' option_style = 'None' if select_url.startswith("$"): selected = 'selected' select_url = select_url[1:] elif select_url.startswith("#"): option_style = 'divider' items.append({ 'option_style': option_style, 'text': select_text, 'url': select_url, 'selected': selected, }) data = items hash = sha256( str( str(url) + str(text) + str(show) + str(style) + json.dumps(data)).encode()).hexdigest() breadcrumb = { 'hash': hash, 'url': url, 'text': text, 'show': show, 'style': style, 'data': data, } request.breadcrumb.append(breadcrumb) def setup_basic_filters(self): self.webapp.templates.filters['yes_no'] = yombo.utils.is_yes_no self.webapp.templates.filters['make_link'] = self.make_link self.webapp.templates.filters[ 'status_to_string'] = yombo.utils.status_to_string self.webapp.templates.filters[ 'public_to_string'] = yombo.utils.public_to_string self.webapp.templates.filters[ 'epoch_to_human'] = yombo.utils.epoch_to_string self.webapp.templates.filters[ 'epoch_to_pretty_date'] = self._Times.get_age # yesterday, 5 minutes ago, etc. self.webapp.templates.filters['format_markdown'] = self.format_markdown self.webapp.templates.filters['hide_none'] = self.dispay_hide_none self.webapp.templates.filters[ 'display_encrypted'] = self._GPG.display_encrypted self.webapp.templates.filters[ 'display_temperature'] = self._Localize.display_temperature def dispay_hide_none(self, input): if input is None: return "" if isinstance(input, str): if input.lower() == "none": return "" return input def restart(self, request, message=None, redirect=None): if message is None: message = "Web interface requested restart." if redirect is None: redirect = "/?" page = self.get_template(request, self._dir + 'pages/restart.html') reactor.callLater(0.3, self.do_restart) return page.render(message=message, redirect=redirect, uptime=str(self._Atoms['running_since'])) def do_restart(self): try: raise YomboRestart("Web Interface setup wizard complete.") except: pass def shutdown(self, request): page = self.get_template(request, self._dir + 'pages/shutdown.html') # reactor.callLater(0.3, self.do_shutdown) return page.render() def do_shutdown(self): raise YomboCritical("Web Interface setup wizard complete.") # def WebInterface_configuration_set(self, **kwargs): # """ # Hook from configuration library. Get any configuration changes. # # :param kwargs: 'section', 'option', and 'value' are sent here. # :return: # """ # if kwargs['section'] == 'webinterface': # option = kwargs['option'] # if option == 'auth_pin': # self.auth_pin(set=kwargs['value']) # elif option == 'auth_pin_totp': # self.auth_pin_totp(set=kwargs['value']) # elif option == 'auth_pin_type': # self.auth_pin_type(set=kwargs['value']) # elif option == 'auth_pin_required': # self.auth_pin_required(set=kwargs['value']) def _build_dist(self): """ This is blocking code. Doesn't really matter, it only does it on startup. Builds the 'dist' directory from the 'build' directory. Easy way to update the source css/js files and update the webinterface JS and CSS files. :return: """ if not path.exists('yombo/lib/webinterface/static/dist'): mkdir('yombo/lib/webinterface/static/dist') if not path.exists('yombo/lib/webinterface/static/dist/css'): mkdir('yombo/lib/webinterface/static/dist/css') if not path.exists('yombo/lib/webinterface/static/dist/js'): mkdir('yombo/lib/webinterface/static/dist/js') if not path.exists('yombo/lib/webinterface/static/dist/fonts'): mkdir('yombo/lib/webinterface/static/dist/fonts') def do_cat(inputs, output): output = 'yombo/lib/webinterface/static/' + output with open(output, 'w') as outfile: for fname in inputs: fname = 'yombo/lib/webinterface/static/' + fname with open(fname) as infile: outfile.write(infile.read()) def copytree(src, dst, symlinks=False, ignore=None): src = 'yombo/lib/webinterface/static/' + src dst = 'yombo/lib/webinterface/static/' + dst if path.isdir(src): if not path.exists(dst): mkdir(dst) for item in listdir(src): s = path.join(src, item) d = path.join(dst, item) if path.isdir(s): shutil.copytree(s, d, symlinks, ignore) else: shutil.copy2(s, d) CAT_SCRIPTS = [ 'source/jquery/jquery-2.2.4.min.js', 'source/sb-admin/js/js.cookie.min.js', 'source/bootstrap/dist/js/bootstrap.min.js', 'source/metisMenu/metisMenu.min.js', ] CAT_SCRIPTS_OUT = 'dist/js/jquery-cookie-bootstrap-metismenu.min.js' do_cat(CAT_SCRIPTS, CAT_SCRIPTS_OUT) CAT_SCRIPTS = [ 'source/jquery/jquery.validate.min.js', ] CAT_SCRIPTS_OUT = 'dist/js/jquery.validate.min.js' do_cat(CAT_SCRIPTS, CAT_SCRIPTS_OUT) CAT_SCRIPTS = [ 'source/bootstrap/dist/css/bootstrap.min.css', 'source/metisMenu/metisMenu.min.css', ] CAT_SCRIPTS_OUT = 'dist/css/bootstrap-metisMenu.min.css' do_cat(CAT_SCRIPTS, CAT_SCRIPTS_OUT) CAT_SCRIPTS = [ 'source/bootstrap/dist/css/bootstrap.min.css', ] CAT_SCRIPTS_OUT = 'dist/css/bootstrap.min.css' do_cat(CAT_SCRIPTS, CAT_SCRIPTS_OUT) CAT_SCRIPTS = [ 'source/sb-admin/js/sb-admin-2.min.js', 'source/sb-admin/js/yombo.js', ] CAT_SCRIPTS_OUT = 'dist/js/sb-admin2.min.js' do_cat(CAT_SCRIPTS, CAT_SCRIPTS_OUT) CAT_SCRIPTS = [ 'source/sb-admin/css/sb-admin-2.css', 'source/sb-admin/css/yombo.css', ] CAT_SCRIPTS_OUT = 'dist/css/admin2.min.css' do_cat(CAT_SCRIPTS, CAT_SCRIPTS_OUT) CAT_SCRIPTS = [ 'source/font-awesome/css/font-awesome.min.css', ] CAT_SCRIPTS_OUT = 'dist/css/font_awesome.min.css' do_cat(CAT_SCRIPTS, CAT_SCRIPTS_OUT) CAT_SCRIPTS = [ 'source/datatables-plugins/integration/bootstrap/3/dataTables.bootstrap.css', 'source/datatables-responsive/css/responsive.dataTables.min.css', ] CAT_SCRIPTS_OUT = 'dist/css/datatables.min.css' do_cat(CAT_SCRIPTS, CAT_SCRIPTS_OUT) CAT_SCRIPTS = [ 'source/datatables/js/jquery.dataTables.min.js', 'source/datatables-plugins/integration/bootstrap/3/dataTables.bootstrap.min.js', 'source/datatables-responsive/js/dataTables.responsive.min.js', ] CAT_SCRIPTS_OUT = 'dist/js/datatables.min.js' do_cat(CAT_SCRIPTS, CAT_SCRIPTS_OUT) CAT_SCRIPTS = [ 'source/jrcode/jquery-qrcode.min.js', ] CAT_SCRIPTS_OUT = 'dist/js/jquery-qrcode.min.js' do_cat(CAT_SCRIPTS, CAT_SCRIPTS_OUT) CAT_SCRIPTS = [ 'source/creative/js/jquery.easing.min.js', 'source/creative/js/scrollreveal.min.js', 'source/creative/js/creative.min.js', ] CAT_SCRIPTS_OUT = 'dist/js/creative.min.js' do_cat(CAT_SCRIPTS, CAT_SCRIPTS_OUT) CAT_SCRIPTS = [ 'source/creative/css/creative.css', ] CAT_SCRIPTS_OUT = 'dist/css/creative.css' do_cat(CAT_SCRIPTS, CAT_SCRIPTS_OUT) CAT_SCRIPTS = [ 'source/echarts/echarts.min.js', ] CAT_SCRIPTS_OUT = 'dist/js/echarts.min.js' do_cat(CAT_SCRIPTS, CAT_SCRIPTS_OUT) CAT_SCRIPTS = [ 'source/sb-admin/js/mappicker.js', ] CAT_SCRIPTS_OUT = 'dist/js/mappicker.js' do_cat(CAT_SCRIPTS, CAT_SCRIPTS_OUT) CAT_SCRIPTS = [ 'source/sb-admin/css/mappicker.css', ] CAT_SCRIPTS_OUT = 'dist/css/mappicker.css' do_cat(CAT_SCRIPTS, CAT_SCRIPTS_OUT) CAT_SCRIPTS = [ 'source/mqtt/mqttws31.min.js', ] CAT_SCRIPTS_OUT = 'dist/js/mqttws31.min.js' do_cat(CAT_SCRIPTS, CAT_SCRIPTS_OUT) CAT_SCRIPTS = [ 'source/sb-admin/js/jquery.serializejson.min.js', ] CAT_SCRIPTS_OUT = 'dist/js/jquery.serializejson.min.js' do_cat(CAT_SCRIPTS, CAT_SCRIPTS_OUT) CAT_SCRIPTS = [ 'source/sb-admin/js/sha256.js', ] CAT_SCRIPTS_OUT = 'dist/js/sha256.js' do_cat(CAT_SCRIPTS, CAT_SCRIPTS_OUT) CAT_SCRIPTS = [ 'source/yombo/jquery.are-you-sure.js', ] CAT_SCRIPTS_OUT = 'dist/js/jquery.are-you-sure.js' do_cat(CAT_SCRIPTS, CAT_SCRIPTS_OUT) # Just copy files copytree('source/font-awesome/fonts/', 'dist/fonts/') copytree('source/bootstrap/dist/fonts/', 'dist/fonts/')
class MetricLogObserver(config.ReconfigurableServiceMixin, service.MultiService): _reactor = reactor def __init__(self): service.MultiService.__init__(self) self.setName('metrics') self.enabled = False self.periodic_task = None self.periodic_interval = None self.log_task = None self.log_interval = None # Mapping of metric type to handlers for that type self.handlers = {} # Register our default handlers self.registerHandler(MetricCountEvent, MetricCountHandler(self)) self.registerHandler(MetricTimeEvent, MetricTimeHandler(self)) self.registerHandler(MetricAlarmEvent, MetricAlarmHandler(self)) # Make sure our changes poller is behaving self.getHandler(MetricTimeEvent).addWatcher(PollerWatcher(self)) self.getHandler(MetricCountEvent).addWatcher( AttachedSlavesWatcher(self)) def reconfigService(self, new_config): # first, enable or disable if new_config.metrics is None: self.disable() else: self.enable() metrics_config = new_config.metrics # Start up periodic logging log_interval = metrics_config.get('log_interval', 60) if log_interval != self.log_interval: if self.log_task: self.log_task.stop() self.log_task = None if log_interval: self.log_task = LoopingCall(self.report) self.log_task.clock = self._reactor self.log_task.start(log_interval) # same for the periodic task periodic_interval = metrics_config.get('periodic_interval', 10) if periodic_interval != self.periodic_interval: if self.periodic_task: self.periodic_task.stop() self.periodic_task = None if periodic_interval: self.periodic_task = LoopingCall(periodicCheck, self._reactor) self.periodic_task.clock = self._reactor self.periodic_task.start(periodic_interval) # upcall return config.ReconfigurableServiceMixin.reconfigService( self, new_config) def stopService(self): self.disable() service.MultiService.stopService(self) def enable(self): if self.enabled: return log.addObserver(self.emit) self.enabled = True def disable(self): if not self.enabled: return if self.periodic_task: self.periodic_task.stop() self.periodic_task = None if self.log_task: self.log_task.stop() self.log_task = None log.removeObserver(self.emit) self.enabled = False def registerHandler(self, interface, handler): old = self.getHandler(interface) self.handlers[interface] = handler return old def getHandler(self, interface): return self.handlers.get(interface) def emit(self, eventDict): # Ignore non-statistic events metric = eventDict.get('metric') if not metric or not isinstance(metric, MetricEvent): return if metric.__class__ not in self.handlers: return h = self.handlers[metric.__class__] h.handle(eventDict, metric) for w in h.watchers: w.run() def asDict(self): retval = {} for interface, handler in self.handlers.iteritems(): retval.update(handler.asDict()) return retval def report(self): try: for interface, handler in self.handlers.iteritems(): report = handler.report() if not report: continue for line in report.split("\n"): log.msg(line) except: klog.err_json(None, "generating metric report")
def __init__(self, factory): self.factory = factory self.loop = LoopingCall(self.check_log) self.log_provider = None
class FeatureProtocol(ServerProtocol): connection_class = FeatureConnection bans = None ban_publish = None ban_manager = None everyone_is_admin = False player_memory = None irc_relay = None balanced_teams = None timestamps = None building = True killing = True global_chat = True remote_console = None advance_call = None master_reconnect_call = None master = False ip = None identifier = None planned_map = None map_info = None spawns = None user_blocks = None god_blocks = None last_time = None interface = None team_class = FeatureTeam game_mode = None # default to None so we can check time_announce_schedule = None default_fog = (128, 232, 255) def __init__(self, interface: bytes, config_dict: Dict[str, Any]) -> None: # logfile path relative to config dir if not abs path log_filename = logfile.get() if log_filename.strip(): # catches empty filename if not os.path.isabs(log_filename): log_filename = os.path.join(config.config_dir, log_filename) ensure_dir_exists(log_filename) if logging_rotate_daily.get(): logging_file = DailyLogFile(log_filename, '.') else: logging_file = open(log_filename, 'a') predicate = LogLevelFilterPredicate( LogLevel.levelWithName(loglevel.get())) observers = [ FilteringLogObserver(textFileLogObserver(sys.stderr), [predicate]), FilteringLogObserver(textFileLogObserver(logging_file), [predicate]) ] globalLogBeginner.beginLoggingTo(observers) log.info('piqueserver started on %s' % time.strftime('%c')) self.config = config_dict if random_rotation.get(): self.map_rotator_type = random_choice_cycle else: self.map_rotator_type = itertools.cycle self.default_time_limit = default_time_limit.get() self.default_cap_limit = cap_limit.get() self.advance_on_win = int(advance_on_win.get()) self.win_count = itertools.count(1) self.bans = NetworkDict() # attempt to load a saved bans list try: with open(os.path.join(config.config_dir, bans_file.get()), 'r') as f: self.bans.read_list(json.load(f)) log.debug("loaded {count} bans", count=len(self.bans)) except FileNotFoundError: log.debug("skip loading bans: file unavailable", count=len(self.bans)) except IOError as e: log.error('Could not read bans file ({}): {}'.format( bans_file.get(), e)) except ValueError as e: log.error('Could not parse bans file ({}): {}'.format( bans_file.get(), e)) self.hard_bans = set() # possible DDoS'ers are added here self.player_memory = deque(maxlen=100) if len(self.name) > MAX_SERVER_NAME_SIZE: log.warn('(server name too long; it will be truncated to "%s")' % (self.name[:MAX_SERVER_NAME_SIZE])) self.respawn_time = respawn_time_option.get() self.respawn_waves = respawn_waves.get() # since AoS only supports CTF and TC at a protocol level, we need to get # the base game mode if we are using a custom game mode. game_mode_name = game_mode.get() if game_mode_name == 'ctf': self.game_mode = CTF_MODE elif game_mode.get() == 'tc': self.game_mode = TC_MODE elif self.game_mode not in [CTF_MODE, TC_MODE]: raise ValueError( 'invalid game mode: custom game mode "{}" does not set ' 'protocol.game_mode to one of TC_MODE or CTF_MODE. Are ' 'you sure the thing you have specified is a game mode?'.format( game_mode_name)) self.game_mode_name = game_mode.get().split('.')[-1] self.team1_name = team1_name.get()[:9] self.team2_name = team2_name.get()[:9] self.team1_color = tuple(team1_color.get()) self.team2_color = tuple(team2_color.get()) self.friendly_fire = friendly_fire.get() self.friendly_fire_on_grief = friendly_fire_on_grief.get() self.friendly_fire_time = grief_friendly_fire_time.get() self.spade_teamkills_on_grief = spade_teamkills_on_grief.get() self.fall_damage = fall_damage.get() self.teamswitch_interval = teamswitch_interval.get() self.teamswitch_allowed = teamswitch_allowed.get() self.max_players = max_players.get() self.melee_damage = melee_damage.get() self.max_connections_per_ip = max_connections_per_ip.get() self.passwords = passwords.get() self.server_prefix = server_prefix.get() self.time_announcements = time_announcements.get() self.balanced_teams = balanced_teams.get() self.login_retries = login_retries.get() # voting configuration self.default_ban_time = default_ban_duration.get() self.speedhack_detect = speedhack_detect.get() self.rubberband_distance = rubberband_distance.get() if user_blocks_only.get(): self.user_blocks = set() self.set_god_build = set_god_build.get() if ssh_enabled.get(): from piqueserver.ssh import RemoteConsole self.remote_console = RemoteConsole(self) irc = irc_options.get() if irc.get('enabled', False): from piqueserver.irc import IRCRelay self.irc_relay = IRCRelay(self, irc) if status_server_enabled.get(): from piqueserver.statusserver import StatusServer self.status_server = StatusServer(self) ensureDeferred(self.status_server.listen()) if ban_publish.get(): from piqueserver.banpublish import PublishServer self.ban_publish = PublishServer(self, ban_publish_port.get()) if bans_config_urls.get(): from piqueserver import bansubscribe self.ban_manager = bansubscribe.BanManager(self) ensureDeferred(as_deferred(self.ban_manager.start())) self.start_time = time.time() self.end_calls = [] # TODO: why is this here? create_console(self) for user_type, func_names in rights.get().items(): for func_name in func_names: commands.add_rights(user_type, func_name) if everyone_is_admin.get(): self.everyone_is_admin = True self.port = port_option.get() ServerProtocol.__init__(self, self.port, interface) self.host.intercept = self.receive_callback try: self.set_map_rotation(self.config['rotation']) except MapNotFound as e: log.critical('Invalid map in map rotation (%s), exiting.' % e.map) raise SystemExit map_load_d = self.advance_rotation() # discard the result of the map advance for now map_load_d.addCallback(lambda x: self._post_init()) ip_getter = ip_getter_option.get() if ip_getter: ensureDeferred(as_deferred(self.get_external_ip(ip_getter))) self.new_release = None notify_new_releases = config.option("release_notifications", default=True) if notify_new_releases.get(): ensureDeferred(as_deferred(self.watch_for_releases())) self.vacuum_loop = LoopingCall(self.vacuum_bans) # Run the vacuum every 6 hours, and kick it off it right now self.vacuum_loop.start(60 * 60 * 6, True) reactor.addSystemEventTrigger('before', 'shutdown', lambda: ensureDeferred(self.shutdown())) def _post_init(self): """called after the map has been loaded""" self.update_format() self.tip_frequency = tip_frequency.get() if self.tips and self.tip_frequency > 0: reactor.callLater(self.tip_frequency * 60, self.send_tip) self.master = register_master_option.get() self.set_master() async def get_external_ip(self, ip_getter: str) -> Iterator[Deferred]: log.info( 'Retrieving external IP from {!r} to generate server identifier.'. format(ip_getter)) try: async with aiohttp.ClientSession() as session: async with session.get(ip_getter) as response: ip = await response.text() ip = IPv4Address(ip.strip()) except AddressValueError as e: log.warn('External IP getter service returned invalid data.\n' 'Please check the "ip_getter" setting in your config.') return except Exception as e: # pylint: disable=broad-except log.warn("Getting external IP failed: {reason}", reason=e) return self.ip = ip self.identifier = make_server_identifier(ip, self.port) log.info('Server public ip address: {}:{}'.format(ip, self.port)) log.info('Public aos identifier: {}'.format(self.identifier)) def set_time_limit(self, time_limit: Optional[int] = None, additive: bool = False) -> Optional[int]: advance_call = self.advance_call add_time = 0.0 if advance_call is not None: add_time = ((advance_call.getTime() - reactor.seconds()) / 60.0) advance_call.cancel() self.advance_call = None time_limit = time_limit or self.default_time_limit if not time_limit: for call in self.end_calls[:]: call.set(None) return None if additive: time_limit = min(time_limit + add_time, self.default_time_limit) seconds = time_limit * 60 self.advance_call = reactor.callLater(seconds, self._time_up) for call in self.end_calls[:]: call.set(seconds) if self.time_announce_schedule is not None: self.time_announce_schedule.reset() self.time_announce_schedule = Scheduler(self) for seconds in self.time_announcements: self.time_announce_schedule.call_end(seconds, self._next_time_announce) return time_limit def _next_time_announce(self): remaining = self.advance_call.getTime() - reactor.seconds() if remaining < 60.001: if remaining < 10.001: self.broadcast_chat('%s...' % int(round(remaining))) else: self.broadcast_chat('%s seconds remaining.' % int(round(remaining))) else: self.broadcast_chat('%s minutes remaining.' % int(round(remaining / 60))) def _time_up(self): self.advance_call = None self.advance_rotation('Time up!') def advance_rotation(self, message: Optional[str] = None) -> Deferred: """ Advances to the next map in the rotation. If message is provided it will send it to the chat, waits for 10 seconds and then advances. Returns: Deferred that fires when the map has been loaded """ self.set_time_limit(False) if self.planned_map is None: self.planned_map = next(self.map_rotator) planned_map = self.planned_map self.planned_map = None self.on_advance(planned_map) async def do_advance(): if message is not None: log.info("advancing to map '{name}' ({reason}) in 10 seconds", name=planned_map.full_name, reason=message) self.broadcast_chat('{} Next map: {}.'.format( message, planned_map.full_name), irc=True) await sleep(10) else: log.info("advancing to map '{name}'", name=planned_map.full_name) await self.set_map_name(planned_map) return ensureDeferred(do_advance()) def get_mode_name(self) -> str: return self.game_mode_name async def set_map_name(self, rot_info: RotationInfo) -> None: """ Sets the map by its name. """ map_info = await self.make_map(rot_info) if self.map_info: self.on_map_leave() self.map_info = map_info self.max_score = self.map_info.cap_limit or self.default_cap_limit self.set_map(self.map_info.data) self.set_time_limit(self.map_info.time_limit) self.update_format() def set_server_name(self, name: str) -> None: name_option.set(name) self.update_format() def make_map(self, rot_info: RotationInfo) -> Deferred: """ Creates and returns a Map object from rotation info in a new thread Returns: Deferred that resolves to a `Map` object. """ # we must do this in a new thread, since map generation might take so # long that clients time out. return threads.deferToThread(Map, rot_info, os.path.join(config.config_dir, 'maps')) def set_map_rotation(self, maps: List[str]) -> None: """ Over-writes the current map rotation with provided one. `FeatureProtocol.advance_rotation` still needs to be called to actually change the map, """ maps = check_rotation(maps, os.path.join(config.config_dir, 'maps')) self.maps = maps self.map_rotator = self.map_rotator_type(maps) def get_map_rotation(self): return [map_item.full_name for map_item in self.maps] def is_indestructable(self, x: int, y: int, z: int) -> bool: if self.user_blocks is not None: if (x, y, z) not in self.user_blocks: return True if self.god_blocks is not None: if (x, y, z) in self.god_blocks: # pylint: disable=unsupported-membership-test return True map_is_indestructable = self.map_info.is_indestructable if map_is_indestructable is not None: if map_is_indestructable(self, x, y, z): return True return False def update_format(self) -> None: """ Called when the map (or other variables) have been updated """ self.name = self.format(name_option.get()) self.motd = self.format_lines(motd_option.get()) self.help = self.format_lines(help_option.get()) self.tips = self.format_lines(tips_option.get()) self.rules = self.format_lines(rules_option.get()) if self.master_connection is not None: self.master_connection.send_server() def format(self, value: str, extra: Optional[Dict[str, str]] = None) -> str: map_info = self.map_info format_dict = { 'map_name': map_info.name, 'map_author': map_info.author, 'map_description': map_info.description, 'game_mode': self.get_mode_name(), 'server_name': self.name, } if extra: format_dict.update(extra) # format with both old-style and new string formatting to stay # compatible with older configs return value.format(**format_dict) % format_dict def format_lines(self, value: List[str]) -> List[str]: if value is None: return return [self.format(line) for line in value] def got_master_connection(self, client): log.info('Master connection established.') ServerProtocol.got_master_connection(self, client) def master_disconnected(self, client=None): ServerProtocol.master_disconnected(self, client) if self.master and self.master_reconnect_call is None: if client: message = 'Master connection could not be established' else: message = 'Master connection lost' log.info('%s, reconnecting in 60 seconds...' % message) self.master_reconnect_call = reactor.callLater( 60, self.reconnect_master) def reconnect_master(self): self.master_reconnect_call = None self.set_master() def set_master_state(self, value): if value == self.master: return self.master = value has_connection = self.master_connection is not None has_reconnect = self.master_reconnect_call is not None if value: if not has_connection and not has_reconnect: self.set_master() else: if has_reconnect: self.master_reconnect_call.cancel() self.master_reconnect_call = None if has_connection: self.master_connection.disconnect() async def shutdown(self): """ Notifies players and disconnects them before a shutdown. """ if not self.connections: # exit instantly if nobody is connected anyway return # send shutdown notification log.info("disconnecting players") self.broadcast_chat("Server shutting down in 3sec.") for i in range(3, 0, -1): self.broadcast_chat(str(i) + "...") await sleep(1) # disconnect all players for connection in list(self.connections.values()): connection.disconnect(ERROR_SHUTDOWN) # give the connections some time to terminate await sleep(0.2) def add_ban(self, ip, reason, duration, name=None): """ Ban an ip with an optional reason and duration in seconds. If duration is None, ban is permanent. """ network = ip_network(str(ip), strict=False) for connection in list(self.connections.values()): if ip_address(connection.address[0]) in network: name = connection.name connection.kick(silent=True) if duration: duration = time.time() + duration else: duration = None self.bans[ip] = (name or '(unknown)', reason, duration) self.save_bans() def remove_ban(self, ip): results = self.bans.remove(ip) log.info('Removing ban: {ip} {results}', ip=ip, results=results) self.save_bans() async def watch_for_releases(self): """Starts a loop for `check_for_releases` and updates `self.new_release`.""" while True: self.new_release = await check_for_releases() if self.new_release: log.info("#" * 60) log.info(format_release(self.new_release)) log.info("#" * 60) await asyncio.sleep(86400) # 24 hrs def vacuum_bans(self): """remove any bans that might have expired. This takes a while, so it is split up over the event loop""" def do_vacuum_bans(): """do the actual clearing of bans""" bans_count = len(self.bans) log.info("starting ban vacuum with {count} bans", count=bans_count) start_time = time.time() # create a copy of the items, so we don't have issues modifying # while iteraing for ban in list(self.bans.iteritems()): ban_exipry = ban[1][2] if ban_exipry is None: # entry never expires continue if ban[1][2] < start_time: # expired del self.bans[ban[0]] yield log.debug( "ban vacuum took {time:.2f} seconds, removed {count} bans", count=bans_count - len(self.bans), time=time.time() - start_time) self.save_bans() # TODO: use cooperate() here instead, once you figure out why it's # swallowing errors. Perhaps try add an errback? coiterate(do_vacuum_bans()) def undo_last_ban(self): result = self.bans.pop() self.save_bans() return result def save_bans(self): ban_file = os.path.join(config.config_dir, bans_file.get()) ensure_dir_exists(ban_file) start_time = reactor.seconds() with open(ban_file, 'w') as f: json.dump(self.bans.make_list(), f, indent=2) log.debug("saving {count} bans took {time:.2f} seconds", count=len(self.bans), time=reactor.seconds() - start_time) if self.ban_publish is not None: self.ban_publish.update() def receive_callback(self, address: Address, data: bytes) -> int: """This hook receives the raw UDP data before it is processed by enet""" # exceptions get swallowed in the pyenet C stuff, so we catch anything # for now. This should ideally get fixed in pyenet instead. try: # reply to ASCII HELLO messages with HI so that clients can measure the # connection latency if data == b'HELLO': self.host.socket.send(address, b'HI') return 1 # reply to ASCII HELLOLAN messages with server data for LAN discovery elif data == b'HELLOLAN': # we might receive a HELLOLAN before the map has been loaded # if so, return a dummy string instead if self.map_info: map_name = self.map_info.short_name else: map_name = "loading..." entry = { "name": self.name, "players_current": self.get_player_count(), "players_max": self.max_players, "map": map_name, "game_mode": self.get_mode_name(), "game_version": "0.75" } payload = json.dumps(entry).encode() self.host.socket.send(address, payload) return 1 # This drop the connection of any ip in hard_bans if address.host in self.hard_bans: return 1 except Exception: import traceback traceback.print_exc() return 0 def data_received(self, peer: Peer, packet: Packet) -> None: ip = peer.address.host current_time = reactor.seconds() try: ServerProtocol.data_received(self, peer, packet) except (NoDataLeft, ValueError): import traceback traceback.print_exc() log.info( 'IP %s was hardbanned for invalid data or possibly DDoS.' % ip) self.hard_bans.add(ip) return dt = reactor.seconds() - current_time if dt > 1.0: log.warn('processing {!r} from {} took {}'.format( packet.data, ip, dt)) def irc_say(self, msg: str, me: bool = False) -> None: if self.irc_relay: if me: self.irc_relay.me(msg, do_filter=True) else: self.irc_relay.send(msg, do_filter=True) def send_tip(self): line = self.tips[random.randrange(len(self.tips))] self.broadcast_chat(line) reactor.callLater(self.tip_frequency * 60, self.send_tip) # pylint: disable=arguments-differ def broadcast_chat(self, value, global_message=True, sender=None, team=None, irc=False): """ Send a chat message to many users """ if irc: self.irc_say('* %s' % value) ServerProtocol.broadcast_chat(self, value, global_message, sender, team) # backwards compatability def send_chat(self, *args, **kwargs): """Deprecated: see broadcast_chat""" warnings.warn( "use of deprecated send_chat, use broadcast_chat instead", DeprecationWarning, stacklevel=2) self.broadcast_chat(*args, **kwargs) # log high CPU usage def update_world(self): last_time = self.last_time current_time = reactor.seconds() if last_time is not None: dt = current_time - last_time if dt > 1.0: log.warn('high CPU usage detected - %s' % dt) self.last_time = current_time ServerProtocol.update_world(self) time_taken = reactor.seconds() - current_time if time_taken > 1.0: log.warn('World update iteration took %s, objects: %s' % (time_taken, self.world.objects)) # events def on_map_change(self, the_map: VXLData) -> None: self.set_fog_color(getattr(self.map_info.info, 'fog', self.default_fog)) map_on_map_change = self.map_info.on_map_change if map_on_map_change is not None: map_on_map_change(self, the_map) def on_map_leave(self): map_on_map_leave = self.map_info.on_map_leave if map_on_map_leave is not None: map_on_map_leave(self) def on_game_end(self): if self.advance_on_win <= 0: self.irc_say('Round ended!', me=True) elif next(self.win_count) % self.advance_on_win == 0: self.advance_rotation('Game finished!') def on_advance(self, map_name: str) -> None: pass def on_ban_attempt(self, connection, reason, duration): return True def on_ban(self, connection, reason, duration): pass # voting def cancel_vote(self, connection=None): return 'No vote in progress.' # useful twisted wrappers def listenTCP(self, *arg, **kw) -> Port: return reactor.listenTCP(*arg, interface=network_interface.get(), **kw) def connectTCP(self, *arg, **kw): return reactor.connectTCP(*arg, bindAddress=(network_interface.get(), 0), **kw) # before-end calls def call_end(self, delay: int, func: Callable, *arg, **kw) -> EndCall: call = EndCall(self, delay, func, *arg, **kw) call.set(self.get_advance_time()) return call def get_advance_time(self) -> float: if not self.advance_call: return None return self.advance_call.getTime() - self.advance_call.seconds()
def init(self): self._periodic_clean_sessions = LoopingCall(self.clean_sessions) self._periodic_clean_sessions.start(random_int( 60 * 60 * 2, .7)) # Every 60-ish seconds. Save to disk, or remove from memory.
class Yombo_Site(Site): def setup_log_queue(self, webinterface): self.save_log_queue_loop = LoopingCall(self.save_log_queue) self.save_log_queue_loop.start(8.7, False) self.log_queue = [] self.webinterface = webinterface self.db_save_log = self.webinterface._LocalDB.webinterface_save_logs def _escape(self, s): """ Return a string like python repr, but always escaped as if surrounding quotes were double quotes. @param s: The string to escape. @type s: L{bytes} or L{unicode} @return: An escaped string. @rtype: L{unicode} """ if not isinstance(s, bytes): s = s.encode("ascii") r = repr(s) if not isinstance(r, str): r = r.decode("ascii") if r.startswith(u"b"): r = r[1:] if r.startswith(u"'"): return r[1:-1].replace(u'"', u'\\"').replace(u"\\'", u"'") return r[1:-1] def log(self, request): ignored_extensions = ('.js', '.css', '.jpg', '.jpeg', '.gif', '.ico', '.woff2', '.map') url_path = request.path.decode().strip() if any(url_path.endswith(ext) for ext in ignored_extensions): return od = OrderedDict({ 'request_at': time(), 'request_protocol': request.clientproto.decode().strip(), 'referrer': self._escape(request.getHeader(b"referer") or b"-").strip(), 'agent': self._escape(request.getHeader(b"user-agent") or b"-").strip(), 'ip': request.getClientIP(), 'hostname': request.getRequestHostname().decode().strip(), 'method': request.method.decode().strip(), 'path': url_path, 'secure': request.isSecure(), 'response_code': request.code, 'response_size': request.sentLength, 'uploadable': 1, 'uploaded': 0, }) self.log_queue.append(od) def save_log_queue(self): if len(self.log_queue) > 0: queue = self.log_queue self.log_queue = [] self.db_save_log(queue)
from hl7parser import patient, measure, channel, oru_wav from monitorapi import ICUMonitor, ICUMonitorFactory from monitor_multi import monitor_multi from twisted.internet import reactor from twisted.internet.task import LoopingCall ip = "localhost" port = 60000 smsg = None def setmsg(): global smsg return smsg def createmsg(m): m.preenche() orw = mon.get_orw((ip, 'CEN')) global smsg smsg = orw.to_str() if __name__ == '__main__': mon = monitor_multi(ip) monit = ICUMonitorFactory(setmsg, ip, port) LoopingCall(createmsg, mon).start(0.5) monit.startsend(reactor, 0.5) reactor.run()
class Relay(object): def __init__(self, config): self.config = config # TODO - this could/should be extracted into a Stack class, though # the relay app is small and straightforward like this, so no rush. self.websocket_layer = self.setup_websocket_layer() self.rendezvous_layer = self.setup_rendezvous_layer( self.websocket_layer) self.relay_layer = self.setup_relay_layer(self.rendezvous_layer) self.listen_url = self.get_listen_url(self.config['Relay']) self.tls_info = self.get_tls_info(self.config['Relay']) self.info_loop = LoopingCall(self.output_info) self.info_loop.start(2.0, now=False) ########################################################################### def setup_relay_layer(self, rendezvous_layer): l = RelayLayer() l.register_above_layer(rendezvous_layer) l.register_layer_event(self.on_stack_event, "RELAY") l.set_rendezvous_layer(rendezvous_layer) return l def setup_rendezvous_layer(self, below_layer): l = IncomingRendezvousLayer() l.register_above_layer(below_layer) l.register_layer_event(self.on_stack_event, "INCOMING_RENDEZVOUS") return l def setup_websocket_layer(self): l = IncomingWebsocketLayer() l.register_layer_event(self.on_stack_event, "INCOMING_WEBSOCKET") return l ########################################################################### def on_stack_event(self, layer_name, nexus, status): pass ########################################################################### def output_info(self): logging.info(str(self.rendezvous_layer)) def get_listen_url(self, config_section): bind = config_section['ListenBind'] port = int(config_section['ListenPort']) prefix = "wss" if config_section['UseTLS'] == "True" else "ws" return "%s://%s:%d" % (prefix, bind, port) def get_tls_info(self, config_section): if config_section['UseTLS'] != "True": return None tls_info = {'sslmethod': SSL.TLSv1_2_METHOD, 'cert_file': os.path.abspath(config_section['CertFile']), 'key_file': os.path.abspath(config_section['CertKey'])} tls_info['cert_chain_file'] = ( os.path.abspath(config_section['CertChainFile']) if config_section['SelfSignedCert'] == "False" else None) return tls_info ########################################################################### def run_app(self): print("listening at: %s" % self.listen_url) self.websocket_layer.listen(self.listen_url, tls_info=self.tls_info)
class IPCMasterService(service.Service): """ IPC master service. Provides the master side of the IPC communication between the workers. """ UPDATE_INTERVAL = 60 # 60 seconds. REMOVE_INTERVAL = 90 # 90 seconds. connections = None def __init__(self, reactor, workers=None, socket_path=None): super().__init__() self.reactor = reactor self.workers = workers self.socket_path = socket_path if self.socket_path is None: self.socket_path = get_ipc_socket_path() if os.path.exists(self.socket_path): os.remove(self.socket_path) self.endpoint = UNIXServerEndpoint(reactor, self.socket_path) self.port = None self.connections = {} self.factory = Factory.forProtocol(IPCMaster) self.factory.service = self self.updateLoop = LoopingCall(self.update) @asynchronous def startService(self): """Start listening on UNIX socket and create the region controller.""" super().startService() self.starting = self.endpoint.listen(self.factory) def save_port(port): self.port = port @transactional def create_region(result): RegionController.objects.get_or_create_running_controller() def start_update_loop(result): self.updateLoopDone = self.updateLoop.start(self.UPDATE_INTERVAL) def log_failure(failure): if failure.check(CancelledError): log.msg("IPCMasterService start-up has been cancelled.") else: log.err(failure, "IPCMasterService start-up failed.") self.starting.addCallback(save_port) self.starting.addCallback(partial(deferToDatabase, create_region)) self.starting.addCallback(start_update_loop) self.starting.addErrback(log_failure) # Twisted's service framework does not track start-up progress, i.e. # it does not check for Deferreds returned by startService(). Here we # return a Deferred anyway so that direct callers (esp. those from # tests) can easily wait for start-up. return self.starting @asynchronous @inlineCallbacks def stopService(self): """Stop listening.""" self.starting.cancel() if self.port: self.port, port = None, self.port yield port.stopListening() for data in self.connections.values(): try: yield data["connection"].transport.loseConnection() except Exception: log.err(None, "Failure when closing IPC connection.") @transactional def delete_all_processes(): region = RegionController.objects.get_running_controller() region.processes.all().delete() @asynchronous def stop_update_loop(): if self.updateLoop.running: self.updateLoop.stop() return self.updateLoopDone yield deferToDatabase(delete_all_processes) yield stop_update_loop() yield super().stopService() @asynchronous def registerWorker(self, pid, conn): """Register the worker with `pid` using `conn`.""" @transactional def create_process(pid): region = RegionController.objects.get_running_controller() process, _ = RegionControllerProcess.objects.get_or_create( region=region, pid=pid ) return (pid, process.id) def log_connected(result): pid, process_id = result log.msg("Worker pid:%d IPC connected." % pid) return result def add_to_connections(result): pid, process_id = result self.connections[pid] = { "process_id": process_id, "connection": conn, "rpc": {"port": None, "connections": set()}, } return process_id @transactional def update_service(process_id): region = RegionController.objects.get_running_controller() self._updateService(region) return process_id def return_result(process_id): return {"process_id": process_id} d = deferToDatabase(create_process, pid) d.addCallback(log_connected) d.addCallback(add_to_connections) d.addCallback(partial(deferToDatabase, update_service)) d.addCallback(return_result) return d def getPIDFromConnection(self, conn): """Get the PID from the connection.""" for pid, data in self.connections.items(): if data["connection"] == conn: return pid @asynchronous def unregisterWorker(self, conn, reason): """Unregister the worker with `pid` because of `reason`.""" pid = self.getPIDFromConnection(conn) if pid: @transactional def delete_process(pid): process_id = self.connections[pid]["process_id"] RegionControllerProcess.objects.filter(id=process_id).delete() return pid def remove_conn_kill_worker(pid): del self.connections[pid] if self.workers: self.workers.killWorker(pid) return pid def log_disconnected(pid): log.msg("Worker pid:%d IPC disconnected." % pid) d = deferToDatabase(delete_process, pid) d.addCallback(remove_conn_kill_worker) d.addCallback(log_disconnected) return d def _getListenAddresses(self, port): """Return list of tuple (address, port) for the addresses the worker is listening on.""" addresses = get_all_interface_source_addresses() if addresses: return set((addr, port) for addr in addresses) # There are no non-loopback addresses, so return loopback # address as a fallback. loopback_addresses = set() for addr in get_all_interface_addresses(): ipaddr = IPAddress(addr) if ipaddr.is_link_local(): continue # Don't advertise link-local addresses. if ipaddr.is_loopback(): loopback_addresses.add((addr, port)) return loopback_addresses @synchronous @transactional def _updateEndpoints(self, process, addresses): """Update the endpoints for `pid` and `port`.""" previous_endpoint_ids = set( RegionControllerProcessEndpoint.objects.filter( process=process ).values_list("id", flat=True) ) if addresses: for addr, port in addresses: ( endpoint, created, ) = RegionControllerProcessEndpoint.objects.get_or_create( process=process, address=addr, port=port ) if not created: previous_endpoint_ids.remove(endpoint.id) RegionControllerProcessEndpoint.objects.filter( id__in=previous_endpoint_ids ).delete() @synchronous def _getProcessObjFor(self, pid): """Return `RegionControllerProcess` for `pid`.""" process_id = self.connections[pid]["process_id"] try: return RegionControllerProcess.objects.get(id=process_id) except RegionControllerProcess.DoesNotExist: region_obj = RegionController.objects.get_running_controller() return RegionControllerProcess.objects.create( id=process_id, region=region_obj, pid=pid ) @asynchronous def registerWorkerRPC(self, pid, port): """Register the worker with `pid` has RPC `port` open.""" if pid in self.connections: @transactional def create_endpoints(result): pid, port = result process = self._getProcessObjFor(pid) self._updateEndpoints(process, self._getListenAddresses(port)) return result def set_result(result): pid, port = result self.connections[pid]["rpc"]["port"] = port self.connections[pid]["rpc"]["connections"] = {} return result def log_rpc_open(result): log.msg( "Worker pid:%d opened RPC listener on port:%s." % result ) d = deferToDatabase(create_endpoints, (pid, port)) d.addCallback(set_result) d.addCallback(log_rpc_open) return d @synchronous def _registerConnection(self, process, ident, host, port, force_save=True): rackd = RackController.objects.get(system_id=ident) endpoint, _ = RegionControllerProcessEndpoint.objects.get_or_create( process=process, address=host, port=port ) connection, created = RegionRackRPCConnection.objects.get_or_create( endpoint=endpoint, rack_controller=rackd ) if not created and force_save: # Force the save so that signals connected to the # RegionRackRPCConnection are performed. connection.save(force_update=True) return connection def registerWorkerRPCConnection(self, pid, connid, ident, host, port): """Register the worker with `pid` has RPC an RPC connection.""" if pid in self.connections: @transactional def register_connection(pid, connid, ident, host, port): process = self._getProcessObjFor(pid) self._registerConnection(process, ident, host, port) return (pid, connid, ident, host, port) def log_connection(result): pid, conn = result[0], result[1:] log.msg( "Worker pid:%d registered RPC connection to %s." % (pid, conn[1:]) ) return conn def set_result(conn): connid, conn = conn[0], conn[1:] self.connections[pid]["rpc"]["connections"][connid] = conn d = deferToDatabase( register_connection, pid, connid, ident, host, port ) d.addCallback(log_connection) d.addCallback(set_result) return d @transactional def _unregisterConnection(self, process, ident, host, port): """Unregister the connection into the database.""" try: endpoint = RegionControllerProcessEndpoint.objects.get( process=process, address=host, port=port ) except RegionControllerProcessEndpoint.DoesNotExist: # Endpoint no longer exists, nothing to do. pass else: try: rackd = RackController.objects.get(system_id=ident) except RackController.DoesNotExist: # No rack controller, nothing to do. pass else: RegionRackRPCConnection.objects.filter( endpoint=endpoint, rack_controller=rackd ).delete() def unregisterWorkerRPCConnection(self, pid, connid): """Unregister connection for worker with `pid`.""" if pid in self.connections: connections = self.connections[pid]["rpc"]["connections"] conn = connections.get(connid, None) if conn is not None: @transactional def unregister_connection(pid, connid, ident, host, port): process = self._getProcessObjFor(pid) self._unregisterConnection(process, ident, host, port) return (pid, connid, ident, host, port) def log_disconnect(result): pid, conn = result[0], result[1:] log.msg( "Worker pid:%d lost RPC connection to %s." % (pid, conn[1:]) ) return conn def set_result(conn): connid = conn[0] connections.pop(connid, None) d = deferToDatabase(unregister_connection, pid, connid, *conn) d.addCallback(log_disconnect) d.addCallback(set_result) return d @synchronous def _updateConnections(self, process, connections): """Update the existing RPC connections into this region. This is needed because the database could get in an incorrect state because another process removed its references in the database and the existing connections need to be re-created. """ if not connections: RegionRackRPCConnection.objects.filter( endpoint__process=process ).delete() else: previous_connection_ids = set( RegionRackRPCConnection.objects.filter( endpoint__process=process ).values_list("id", flat=True) ) for _, (ident, host, port) in connections.items(): db_conn = self._registerConnection( process, ident, host, port, force_save=False ) previous_connection_ids.discard(db_conn.id) if previous_connection_ids: RegionRackRPCConnection.objects.filter( id__in=previous_connection_ids ).delete() @synchronous def _updateService(self, region_obj): """Update the service status for this region.""" Service.objects.create_services_for(region_obj) number_of_processes = len(self.connections) not_running_count = workers.MAX_WORKERS_COUNT - number_of_processes if not_running_count > 0: if number_of_processes == 1: process_text = "process" else: process_text = "processes" Service.objects.update_service_for( region_obj, "regiond", SERVICE_STATUS.DEGRADED, "%d %s running but %d were expected." % ( number_of_processes, process_text, workers.MAX_WORKERS_COUNT, ), ) else: Service.objects.update_service_for( region_obj, "regiond", SERVICE_STATUS.RUNNING, "" ) @synchronous @transactional def _update(self): """Repopulate the database with process, endpoint, and connection information.""" # Get the region controller and update its hostname and last # updated time. region_obj = RegionController.objects.get_running_controller() hostname = gethostname() if region_obj.hostname != hostname: region_obj.hostname = hostname region_obj.save() # Get all the existing processes for the region controller. This is # used to remove the old processes that we did not update. previous_process_ids = set( RegionControllerProcess.objects.filter( region=region_obj ).values_list("id", flat=True) ) # Loop through all the current workers to update the records in the # database. Caution is needed because other region controllers can # remove expired processes. for pid, conn in self.connections.items(): process = self._getProcessObjFor(pid) process.updated = now() process.save() if conn["rpc"]["port"]: # Update the endpoints for the provided port. self._updateEndpoints( process, self._getListenAddresses(conn["rpc"]["port"]) ) else: # RPC is not running, no endpoints. self._updateEndpoints(process, []) self._updateConnections(process, conn["rpc"]["connections"]) previous_process_ids.discard(process.id) # Delete all the old processes that are dead. if previous_process_ids: RegionControllerProcess.objects.filter( id__in=previous_process_ids ).delete() # Remove any old processes not owned by this controller. Every # controller should update its processes based on the `UPDATE_INTERVAL` # any that are older than `REMOVE_INTERVAL` are dropped. remove_before_time = now() - timedelta(seconds=self.REMOVE_INTERVAL) RegionControllerProcess.objects.exclude(region=region_obj).filter( updated__lte=remove_before_time ).delete() # Update the status of this regiond service for this region based on # the number of running processes. self._updateService(region_obj) # Update the status of all regions that have no processes running. for other_region in RegionController.objects.exclude( system_id=region_obj.id ).prefetch_related("processes"): # Use len with `all` so the prefetch cache is used. if len(other_region.processes.all()) == 0: Service.objects.mark_dead(other_region, dead_region=True) @asynchronous def update(self): def ignore_cancel(failure): failure.trap(CancelledError) d = deferToDatabase(self._update) d.addErrback(ignore_cancel) d.addErrback( log.err, "Failed to update regiond's processes and endpoints; " "%s record's may be out of date" % (eventloop.loop.name,), ) return d
class ApiAuths(YomboLibrary): """ Session management. """ def __init__(self, loader): # we do some simulation of a Yombo Library... self.loader = loader self._FullName = "yombo.gateway.lib.webinterface.api_auth" self._Configs = self.loader.loadedLibraries['configuration'] self._LocalDB = self.loader.loadedLibraries['localdb'] self._Gateways = self.loader.loadedLibraries['gateways'] self.gateway_id = self._Configs.get('core', 'gwid', 'local', False) self.active_api_auth = {} self.session_type = "apiauth" # self.active_api_auth_cache = ExpiringDict(200, 5) # keep 200 entries, for at most 1 second...??? def __delitem__(self, key): if key in self.active_api_auth: self.active_api_auth[key].expire_session() return def __getitem__(self, key): if key in self.active_api_auth: return self.active_api_auth[key] else: raise KeyError("Cannot find api auth key 2: %s" % key) def __len__(self): return len(self.active_api_auth) def __setitem__(self, key, value): raise YomboWarning("Cannot set a session using this method.") def __contains__(self, key): if key in self.active_api_auth: return True return False def keys(self): """ Returns the keys (command ID's) that are configured. :return: A list of command IDs. :rtype: list """ return list(self.active_api_auth.keys()) def items(self): """ Gets a list of tuples representing the commands configured. :return: A list of tuples. :rtype: list """ return list(self.active_api_auth.items()) # @inlineCallbacks def init(self): self._periodic_clean_sessions = LoopingCall(self.clean_sessions) self._periodic_clean_sessions.start(random_int( 60 * 60 * 2, .7)) # Every 60-ish seconds. Save to disk, or remove from memory. def _unload_(self): self.unload_deferred = Deferred() self.clean_sessions(self.unload_deferred) return self.unload_deferred def close(self, request): api_auth = self.get(request) api_auth.expire_session() @inlineCallbacks def get_all(self): """ Returns the api auths from DB. :return: A list of command IDs. :rtype: list """ yield self.clean_sessions(True) api_auths = yield self._LocalDB.get_api_auth() return api_auths @inlineCallbacks def get(self, request=None, api_auth_id=None): """ Checks the request for an x-api-auth header and then tries to validate it. Returns True if everything is good, otherwise raises YomboWarning with status reason. :param request: The request instance. :return: bool """ if api_auth_id is None and request is not None: # print("request: %s" % request) api_auth_id = bytes_to_unicode(request.getHeader(b'x-api-auth')) if api_auth_id is None: try: api_auth_id = request.args.get('_api_auth')[0] except: api_auth_id = None if api_auth_id is None: raise YomboWarning("x-api-auth header is missing or blank.") if self.validate_api_auth_id(api_auth_id) is False: raise YomboWarning("x-api-auth header has invalid characters.") if api_auth_id in self.active_api_auth: if self.active_api_auth[api_auth_id].is_valid is True: return self.active_api_auth[api_auth_id] else: raise YomboWarning("x-api-auth header is no longer valid.") else: logger.debug("has_session is looking in database for session...") try: db_api_auth = yield self._LocalDB.get_api_auth(api_auth_id) except Exception as e: raise YomboWarning("x-api-auth is not valid") # logger.debug("has_session - found in DB! {db_session}", db_session=db_session) self.active_api_auth[api_auth_id] = ApiAuth(self, db_api_auth, source='database') return self.active_api_auth[api_auth_id] raise YomboWarning("x-api-auth header is invalid, other reasons.") def create(self, request=None, data=None): """ Creates a new session. :param request: :param make_active: If True or None (default), then store sesion in memory. :return: """ if data is None: data = {} if 'gateway_id' not in data or data['gateway_id'] is None: data['gateway_id'] = self.gateway_id if 'api_auth_id' not in data or data['api_auth_id'] is None: data['api_auth_id'] = random_string(length=randint(50, 55)) if 'api_auth_data' not in data: data['api_auth_data'] = {} self.active_api_auth[data['api_auth_id']] = ApiAuth(self, data) return self.active_api_auth[data['api_auth_id']] @inlineCallbacks def update(self, id=None, data=None): """ Updates an API Auth key :param request: :return: """ if id is None or data is None: return api_auth = yield self.get(api_auth_id=id) if api_auth is None or api_auth is False: raise YomboWarning("API Auth ID doesn't exist") api_auth.update_attributes(data) def validate_api_auth_id(self, api_auth_id): """ Validate the session id to make sure it's reasonable. :param api_auth_id: :return: """ if api_auth_id == "LOGOFF": # lets not raise an error. return True if api_auth_id.isalnum() is False: return False if len(api_auth_id) < 30: return False if len(api_auth_id) > 60: return False return True @inlineCallbacks def clean_sessions(self, close_deferred=None): """ Called by loopingcall. Cleanup the stored sessions """ for api_auth_id in list(self.active_api_auth.keys()): if self.active_api_auth[api_auth_id].check_valid( ) is False or self.active_api_auth[api_auth_id].is_valid is False: logger.debug("Removing invalid api auth: %s" % api_auth_id) del self.active_api_auth[api_auth_id] yield self._LocalDB.delete_api_auth(api_auth_id) for api_auth_id in list(self.active_api_auth): session = self.active_api_auth[api_auth_id] if session.is_dirty >= 200 or close_deferred is not None or session.last_access < int( time() - (60 * 5)): if session.in_db: logger.debug("updating old db api auth record: {id}", id=api_auth_id) yield self._LocalDB.update_api_auth(session) else: logger.debug("creating new db api auth record: {id}", id=api_auth_id) yield self._LocalDB.save_api_auth(session) session.in_db = True session.is_dirty = 0 if session.last_access < int( time() - (60 * 60 * 3)): # delete session from memory after 3 hours logger.debug("Deleting session from memory: {api_auth_id}", api_auth_id=api_auth_id) del self.active_api_auth[api_auth_id] # print("api auth clean sessions...: %s" % close_deferred) if close_deferred is not None and close_deferred is not True and close_deferred is not False: yield sleep(0.1) close_deferred.callback(1)
def __init__(self, svnurl, split_file=None, svnuser=None, svnpasswd=None, pollinterval=10 * 60, histmax=100, svnbin='svn'): """ @type svnurl: string @param svnurl: the SVN URL that describes the repository and subdirectory to watch. If this ChangeSource should only pay attention to a single branch, this should point at the repository for that branch, like svn://svn.twistedmatrix.com/svn/Twisted/trunk . If it should follow multiple branches, point it at the repository directory that contains all the branches like svn://svn.twistedmatrix.com/svn/Twisted and also provide a branch-determining function. Each file in the repository has a SVN URL in the form (SVNURL)/(BRANCH)/(FILEPATH), where (BRANCH) could be empty or not, depending upon your branch-determining function. Only files that start with (SVNURL)/(BRANCH) will be monitored. The Change objects that are sent to the Schedulers will see (FILEPATH) for each modified file. @type split_file: callable or None @param split_file: a function that is called with a string of the form (BRANCH)/(FILEPATH) and should return a tuple (BRANCH, FILEPATH). This function should match your repository's branch-naming policy. Each changed file has a fully-qualified URL that can be split into a prefix (which equals the value of the 'svnurl' argument) and a suffix; it is this suffix which is passed to the split_file function. If the function returns None, the file is ignored. Use this to indicate that the file is not a part of this project. For example, if your repository puts the trunk in trunk/... and branches are in places like branches/1.5/..., your split_file function could look like the following (this function is available as svnpoller.split_file_branches):: pieces = path.split('/') if pieces[0] == 'trunk': return (None, '/'.join(pieces[1:])) elif pieces[0] == 'branches': return ('/'.join(pieces[0:2]), '/'.join(pieces[2:])) else: return None If instead your repository layout puts the trunk for ProjectA in trunk/ProjectA/... and the 1.5 branch in branches/1.5/ProjectA/..., your split_file function could look like:: pieces = path.split('/') if pieces[0] == 'trunk': branch = None pieces.pop(0) # remove 'trunk' elif pieces[0] == 'branches': pieces.pop(0) # remove 'branches' # grab branch name branch = 'branches/' + pieces.pop(0) else: return None # something weird projectname = pieces.pop(0) if projectname != 'ProjectA': return None # wrong project return (branch, '/'.join(pieces)) The default of split_file= is None, which indicates that no splitting should be done. This is equivalent to the following function:: return (None, path) If you wish, you can override the split_file method with the same sort of function instead of passing in a split_file= argument. @type svnuser: string @param svnuser: If set, the --username option will be added to the 'svn log' command. You may need this to get access to a private repository. @type svnpasswd: string @param svnpasswd: If set, the --password option will be added. @type pollinterval: int @param pollinterval: interval in seconds between polls. The default is 600 seconds (10 minutes). Smaller values decrease the latency between the time a change is recorded and the time the buildbot notices it, but it also increases the system load. @type histmax: int @param histmax: maximum number of changes to look back through. The default is 100. Smaller values decrease system load, but if more than histmax changes are recorded between polls, the extra ones will be silently lost. @type svnbin: string @param svnbin: path to svn binary, defaults to just 'svn'. Use this if your subversion command lives in an unusual location. """ if svnurl.endswith("/"): svnurl = svnurl[:-1] # strip the trailing slash self.svnurl = svnurl self.split_file_function = split_file or split_file_alwaystrunk self.svnuser = svnuser self.svnpasswd = svnpasswd self.svnbin = svnbin self.pollinterval = pollinterval self.histmax = histmax self._prefix = None self.overrun_counter = 0 self.loop = LoopingCall(self.checksvn)
class AirsharkManager(object): def __init__(self): self.loop = LoopingCall(self.check_spectrum) self.wifi_interface = None self.scanner = None self.analyzer_process = AnalyzerProcessProtocol(self) self.spectrum_observers = [] self.analyzer_observers = [] airshark_interface_manager.add_observer(self) def status(self): hardware_ready = False software_ready = False airshark_running = False if self.wifi_interface: hardware_ready = True if pdos.exists(settings.AIRSHARK_INSTALL_DIR): software_ready = True if self.analyzer_process.isRunning(): airshark_running = True return (hardware_ready, software_ready, airshark_running) def on_interface_up(self, interface): self._stop() self.wifi_interface = interface self.scanner = Scanner(self.wifi_interface) self._start() def on_interface_down(self, interface): self._stop() self.wifi_interface = None self.scanner = None def _start(self): self.scanner.cmd_chanscan() self.scanner.start() if not self.analyzer_process.isRunning(): out.info("Launching airshark analyzer") cmd = [settings.AIRSHARK_INSTALL_DIR + "/analyzer", settings.AIRSHARK_INSTALL_DIR + "/specshape/specshape_v1_722N_5m.txt", "--spectrum-fd=3", "--output-fd=4"] spawnProcess(self.analyzer_process,\ cmd[0], cmd, env=None, \ childFDs={0:"w", 1:"r", 2:2, 3:"w", 4:"r"}) self.loop.start(0.2) return True def _stop(self): if self.scanner: self.scanner.stop() if self.analyzer_process.isRunning(): self.analyzer_process.stop() if self.loop.running: self.loop.stop() def check_spectrum(self): # The bandwidth of the data is about 160k Bytes per second ts, data = self.scanner.spectrum_reader.read_samples() if ((data is not None) and (len(self.spectrum_observers) > 0 or self.analyzer_process.isRunning())): if len(self.spectrum_observers) > 0: #for (tsf, max_exp, freq, rssi, noise, max_mag, max_index, bitmap_weight, sdata) in SpectrumReader.decode(data): # for observer in self.spectrum_observers: # observer.on_spectrum_data(tsf, max_exp, freq, rssi, noise, max_mag, max_index, bitmap_weight, sdata) # Due to performance issue, we have to delegate the packet decoding task to clients # for packet in SpectrumReader.decode(data): # for observer in self.spectrum_observers: # observer.on_spectrum_data(packet) for observer in self.spectrum_observers: observer.on_spectrum_data(data) if self.analyzer_process.isRunning(): # Forward spectrum data to the airshark analyzer self.analyzer_process.feedSpectrumData(data) # TODO: Not sure we need it or not def read_raw_samples(self): if (self.scanner): return self.scanner.spectrum_reader.read_samples() else: return None, None def on_analyzer_message(self, message): for observer in self.analyzer_observers: observer.on_analyzer_message(message) def add_spectrum_observer(self, observer): if (self.spectrum_observers.count(observer) == 0): self.spectrum_observers.append(observer) def remove_spectrum_observer(self, observer): if (self.spectrum_observers.count(observer) == 1): self.spectrum_observers.remove(observer) def add_analyzer_observer(self, observer): if (self.analyzer_observers.count(observer) == 0): self.analyzer_observers.append(observer) def remove_analyzer_observer(self, observer): if (self.analyzer_observers.count(observer) == 1): self.analyzer_observers.remove(observer)
def openDev(self): from twisted.internet.task import LoopingCall self.LC = LoopingCall(self._push_up_some_data) self.LC.start(0.020) self._data = ''
class NormalPMTFlow(LabradServer): name = 'NormalPMTFlow Dual' onNewCount = Signal(SIGNALID, 'signal: new count', 'v') onNewCount2 = Signal(SIGNALID + 10, 'signal: new count 2', 'v') onNewSetting = Signal(SIGNALID + 1, 'signal: new setting', '(ss)') onNewState = Signal(SIGNALID + 2, 'signal: new state', '(ss)') # onNewState2 = Signal(SIGNALID+12, 'signal: new state 2', '(ss)') @inlineCallbacks def initServer(self): self.saveFolder = ['', 'PMT Counts'] self.dataSetName = 'PMT Counts' self.modes = ['Normal', 'Differential'] self.collection_period = T.Value(0.100, 's') self.lastDifferential = {'ON': 0, 'OFF': 0} self.currentMode = 'Normal' self.dv = None self.pulser = None self.collectTimeRange = None self.openDataSet = None self.recordingInterrupted = False self.requestList = [] self.listeners = set() self.recording = LoopingCall(self._record) yield self.connect_data_vault() yield self.connect_pulser() yield self.setupListeners() @inlineCallbacks def setupListeners(self): yield self.client.manager.subscribe_to_named_message( 'Server Connect', 9898989, True) yield self.client.manager.subscribe_to_named_message( 'Server Disconnect', 9898989 + 1, True) yield self.client.manager.addListener( listener=self.followServerConnect, source=None, ID=9898989) yield self.client.manager.addListener( listener=self.followServerDisconnect, source=None, ID=9898989 + 1) @inlineCallbacks def followServerConnect(self, cntx, serverName): serverName = serverName[1] if serverName == 'Pulser': yield self.connect_pulser() elif serverName == 'Data Vault': yield self.connect_data_vault() @inlineCallbacks def followServerDisconnect(self, cntx, serverName): serverName = serverName[1] if serverName == 'Pulser': yield self.disconnect_pulser() elif serverName == 'Data Vault': yield self.disconnect_data_vault() @inlineCallbacks def connect_data_vault(self): try: #reconnect to data vault and navigate to the directory self.dv = yield self.client.data_vault yield self.dv.cd(self.saveFolder, True) if self.openDataSet is not None: self.openDataSet = yield self.makeNewDataSet( self.saveFolder, self.dataSetName) self.onNewSetting(('dataset', self.openDataSet)) print 'Connected: Data Vault' except AttributeError: self.dv = None print 'Not Connected: Data Vault' @inlineCallbacks def disconnect_data_vault(self): print 'Not Connected: Data Vault' self.dv = None yield None @inlineCallbacks def connect_pulser(self): try: self.pulser = yield self.client.pulser self.collectTimeRange = yield self.pulser.get_collection_time() pmt_id_list = yield self.pulser.get_pmt_id_list() self.pmt_list = [PMT(pmt_id) for pmt_id in pmt_id_list] self.pmt_list[0].enable() if self.recordingInterrupted: yield self.dorecordData() self.onNewState('on') self.recordingInterrupted = False print 'Connected: Pulser' except AttributeError: self.pulser = None print 'Not Connected: Pulser' @inlineCallbacks def disconnect_pulser(self): print 'Not Connected: Pulser' self.pulser = None if self.recording.running: yield self.recording.stop() self.onNewState('off') self.recordingInterrupted = True def initContext(self, c): """Initialize a new context object.""" self.listeners.add(c.ID) def expireContext(self, c): self.listeners.remove(c.ID) def getOtherListeners(self, c): notified = self.listeners.copy() notified.remove(c.ID) return notified @inlineCallbacks def makeNewDataSet(self, folder, name): for pmt in self.pmt_list: pmt.context = self.dv.context() yield self.dv.cd(folder, True, context=pmt.context) newSet = yield self.dv.new( name + pmt.suffix, [('t', 'num')], [('KiloCounts/sec', '866 ON', 'num'), ('KiloCounts/sec', '866 OFF', 'num'), ('KiloCounts/sec', 'Differential Signal', 'num')], context=pmt.context) self.startTime = time.time() yield self.addParameters(self.startTime) name = newSet[1][:-2] returnValue(name) @inlineCallbacks def addParameters(self, start): for pmt in self.pmt_list: yield self.dv.add_parameter("Window", ["PMT Counts"], context=pmt.context) yield self.dv.add_parameter('plotLive', True, context=pmt.context) yield self.dv.add_parameter('startTime', start, context=pmt.context) @setting(0, 'Set Save Folder', folder='*s', returns='') def setSaveFolder(self, c, folder): yield self.dv.cd(folder, True) self.saveFolder = folder @setting(1, 'Start New Dataset', setName='s', returns='s') def setNewDataSet(self, c, setName=None): """Starts new dataset, if name not provided, it will be the same""" if setName is not None: self.dataSetName = setName self.openDataSet = yield self.makeNewDataSet(self.saveFolder, self.dataSetName) otherListeners = self.getOtherListeners(c) self.onNewSetting(('dataset', self.openDataSet), otherListeners) returnValue(self.openDataSet) @setting(2, "Set Mode", mode='s', returns='') def setMode(self, c, mode): """ Start recording Time Resolved Counts into Data Vault """ if mode not in self.modes: raise Exception('Incorrect Mode') if not self.recording.running: self.currentMode = mode yield self.pulser.set_mode(mode) else: yield self.dostopRecording() self.currentMode = mode yield self.dorecordData() otherListeners = self.getOtherListeners(c) self.onNewSetting(('mode', mode), otherListeners) @setting(3, 'getCurrentMode', returns='s') def getCurrentMode(self, c): """ Returns the currently running mode """ return self.currentMode @setting(4, 'Record Data', returns='') def recordData(self, c): """ Starts recording data of the current PMT mode into datavault """ setname = yield self.dorecordData() otherListeners = self.getOtherListeners(c) if setname is not None: setname = setname[1] self.onNewSetting(('dataset', setname), otherListeners) self.onNewState('on', otherListeners) @inlineCallbacks def dorecordData(self): #begins the process of data record #sets the collection time and mode, programs the pulser if necessary and opens the dataset if necessasry #then starts the recording loop newSet = None self.keepRunning = True yield self.pulser.set_collection_time(self.collection_period, self.currentMode) yield self.pulser.set_mode(self.currentMode) if self.currentMode == 'Differential': yield self._programPulserDiff() if self.openDataSet is None: self.openDataSet = yield self.makeNewDataSet( self.saveFolder, self.dataSetName) self.recording.start(self.collection_period['s'] / 2.0) returnValue(newSet) @setting(5, returns='') def stopRecording(self, c): """ Stop recording counts into Data Vault """ yield self.dostopRecording() otherListeners = self.getOtherListeners(c) self.onNewState('off', otherListeners) @inlineCallbacks def dostopRecording(self): yield self.recording.stop() if self.currentMode == 'Differential': yield self._stopPulserDiff() @setting(6, returns='b') def isRunning(self, c): """ Returns whether or not currently recording """ return self.recording.running @setting(7, returns='s') def currentDataSet(self, c): if self.openDataSet is None: return '' return self.openDataSet @setting(8, 'Set Time Length', timelength='v[s]') def setTimeLength(self, c, timelength): """Sets the time length for the current mode""" mode = self.currentMode if not self.collectTimeRange[0] <= timelength[ 's'] <= self.collectTimeRange[1]: raise Exception('Incorrect Recording Time') self.collection_period = timelength initrunning = self.recording.running #if recording when the call is made, need to stop and restart if initrunning: yield self.recording.stop() yield self.pulser.set_collection_time(timelength['s'], mode) if initrunning: if mode == 'Differential': yield self._stopPulserDiff() yield self._programPulserDiff() self.recording.start(timelength['s'] / 2.0) otherListeners = self.getOtherListeners(c) self.onNewSetting(('timelength', str(timelength['s'])), otherListeners) @setting(9, 'Get Next Counts', kind='s', number='w', average='b', returns=['*v', 'v']) def getNextCounts(self, c, kind, number, average=False): """ Acquires next number of counts, where type can be 'ON' or 'OFF' or 'DIFF' Average is optionally True if the counts should be averaged Note in differential mode, Diff counts get updates every time, but ON and OFF get updated every 2 times. """ if kind not in ['ON', 'OFF', 'DIFF']: raise Exception('Incorrect type') if kind in ['OFF', 'DIFF'] and self.currentMode == 'Normal': raise Exception('in the wrong mode to process this request') if not 0 < number < 1000: raise Exception('Incorrect Number') if not self.recording.running: raise Exception('Not currently recording') d = Deferred() self.requestList.append(self.readingRequest(d, kind, number)) data = yield d if average: data = sum(data) / len(data) returnValue(data) @setting(10, 'Get Time Length', returns='v') def getMode(self, c): """ Returns the current timelength of in the current mode """ return self.collection_period @setting(11, 'Get Time Length Range', returns='(vv)') def get_time_length_range(self, c): if self.collectTimeRange is not None: return self.collectTimeRange else: raise Exception( "Not available because Pulser Server is not available") @setting(12, 'Set PMT State', pmt_id='i', state='b', returns='') def enablePMT(self, c, pmt_id, state): self.pmt_list[pmt_id - 1].enabled = state @inlineCallbacks def _programPulserDiff(self): yield self.pulser.new_sequence() yield self.pulser.add_ttl_pulse('DiffCountTrigger', T.Value(0.0, 'us'), T.Value(10.0, 'us')) yield self.pulser.add_ttl_pulse('DiffCountTrigger', self.collection_period, T.Value(10.0, 'us')) yield self.pulser.add_ttl_pulse('866DP', T.Value(0.0, 'us'), self.collection_period) yield self.pulser.extend_sequence_length(2 * self.collection_period) yield self.pulser.program_sequence() yield self.pulser.start_infinite() @inlineCallbacks def _stopPulserDiff(self): yield self.pulser.complete_infinite_iteration() yield self.pulser.wait_sequence_done() yield self.pulser.stop_sequence() class readingRequest(): def __init__(self, d, kind, count): self.d = d self.count = count self.kind = kind self.data = [] def is_fulfilled(self): return len(self.data) == self.count def processRequests(self, data): if not len(self.requestList): return for dataPoint in data: for item, req in enumerate(self.requestList): if dataPoint[1] != 0 and req.kind == 'ON': req.data.append(dataPoint[1]) if dataPoint[2] != 0 and req.kind == 'OFF': req.data.append(dataPoint[2]) if dataPoint[3] != 0 and req.kind == 'DIFF': req.data.append(dataPoint[3]) if req.is_fulfilled(): req.d.callback(req.data) del (self.requestList[item]) @inlineCallbacks def _record(self): for pmt in self.pmt_list: if not pmt.enabled: return try: yield self.getPMTCounts(pmt.id) except: print 'Not Able to Get PMT Counts' self.rawdata = [] if len(self.rawdata) != 0: if self.currentMode == 'Normal': toDataVault = [ [elem[2] - self.startTime, elem[0], 0, 0] for elem in self.rawdata ] # converting to format [time, normal count, 0 , 0] elif self.currentMode == 'Differential': toDataVault = self.convertDifferential(self.rawdata) self.processRequests( toDataVault) #if we have any requests, process them self.processSignals(toDataVault, pmt.id) try: yield self.dv.add(toDataVault, context=pmt.context) except: print 'Not Able to Save To Data Vault' @inlineCallbacks def getPMTCounts(self, pmt_id): if pmt_id == 2: self.rawdata = yield self.pulser.get_secondary_pmt_counts() else: self.rawdata = yield self.pulser.get_pmt_counts() def processSignals(self, data, pmt_id): lastPt = data[-1] NormalCount = lastPt[1] if pmt_id == 2: self.onNewCount2(NormalCount) else: self.onNewCount(NormalCount) def convertDifferential(self, rawdata): totalData = [] for dataPoint in rawdata: t = str(dataPoint[1]) self.lastDifferential[t] = float(dataPoint[0]) diff = self.lastDifferential['ON'] - self.lastDifferential['OFF'] totalData.append([ dataPoint[2] - self.startTime, self.lastDifferential['ON'], self.lastDifferential['OFF'], diff ]) return totalData
class SVNPoller(base.ChangeSource, util.ComparableMixin): """This source will poll a Subversion repository for changes and submit them to the change master.""" compare_attrs = [ "svnurl", "split_file_function", "svnuser", "svnpasswd", "pollinterval", "histmax", "svnbin" ] parent = None # filled in when we're added last_change = None loop = None working = False def __init__(self, svnurl, split_file=None, svnuser=None, svnpasswd=None, pollinterval=10 * 60, histmax=100, svnbin='svn'): """ @type svnurl: string @param svnurl: the SVN URL that describes the repository and subdirectory to watch. If this ChangeSource should only pay attention to a single branch, this should point at the repository for that branch, like svn://svn.twistedmatrix.com/svn/Twisted/trunk . If it should follow multiple branches, point it at the repository directory that contains all the branches like svn://svn.twistedmatrix.com/svn/Twisted and also provide a branch-determining function. Each file in the repository has a SVN URL in the form (SVNURL)/(BRANCH)/(FILEPATH), where (BRANCH) could be empty or not, depending upon your branch-determining function. Only files that start with (SVNURL)/(BRANCH) will be monitored. The Change objects that are sent to the Schedulers will see (FILEPATH) for each modified file. @type split_file: callable or None @param split_file: a function that is called with a string of the form (BRANCH)/(FILEPATH) and should return a tuple (BRANCH, FILEPATH). This function should match your repository's branch-naming policy. Each changed file has a fully-qualified URL that can be split into a prefix (which equals the value of the 'svnurl' argument) and a suffix; it is this suffix which is passed to the split_file function. If the function returns None, the file is ignored. Use this to indicate that the file is not a part of this project. For example, if your repository puts the trunk in trunk/... and branches are in places like branches/1.5/..., your split_file function could look like the following (this function is available as svnpoller.split_file_branches):: pieces = path.split('/') if pieces[0] == 'trunk': return (None, '/'.join(pieces[1:])) elif pieces[0] == 'branches': return ('/'.join(pieces[0:2]), '/'.join(pieces[2:])) else: return None If instead your repository layout puts the trunk for ProjectA in trunk/ProjectA/... and the 1.5 branch in branches/1.5/ProjectA/..., your split_file function could look like:: pieces = path.split('/') if pieces[0] == 'trunk': branch = None pieces.pop(0) # remove 'trunk' elif pieces[0] == 'branches': pieces.pop(0) # remove 'branches' # grab branch name branch = 'branches/' + pieces.pop(0) else: return None # something weird projectname = pieces.pop(0) if projectname != 'ProjectA': return None # wrong project return (branch, '/'.join(pieces)) The default of split_file= is None, which indicates that no splitting should be done. This is equivalent to the following function:: return (None, path) If you wish, you can override the split_file method with the same sort of function instead of passing in a split_file= argument. @type svnuser: string @param svnuser: If set, the --username option will be added to the 'svn log' command. You may need this to get access to a private repository. @type svnpasswd: string @param svnpasswd: If set, the --password option will be added. @type pollinterval: int @param pollinterval: interval in seconds between polls. The default is 600 seconds (10 minutes). Smaller values decrease the latency between the time a change is recorded and the time the buildbot notices it, but it also increases the system load. @type histmax: int @param histmax: maximum number of changes to look back through. The default is 100. Smaller values decrease system load, but if more than histmax changes are recorded between polls, the extra ones will be silently lost. @type svnbin: string @param svnbin: path to svn binary, defaults to just 'svn'. Use this if your subversion command lives in an unusual location. """ if svnurl.endswith("/"): svnurl = svnurl[:-1] # strip the trailing slash self.svnurl = svnurl self.split_file_function = split_file or split_file_alwaystrunk self.svnuser = svnuser self.svnpasswd = svnpasswd self.svnbin = svnbin self.pollinterval = pollinterval self.histmax = histmax self._prefix = None self.overrun_counter = 0 self.loop = LoopingCall(self.checksvn) def split_file(self, path): # use getattr() to avoid turning this function into a bound method, # which would require it to have an extra 'self' argument f = getattr(self, "split_file_function") return f(path) def startService(self): log.msg("SVNPoller(%s) starting" % self.svnurl) base.ChangeSource.startService(self) # Don't start the loop just yet because the reactor isn't running. # Give it a chance to go and install our SIGCHLD handler before # spawning processes. reactor.callLater(0, self.loop.start, self.pollinterval) def stopService(self): log.msg("SVNPoller(%s) shutting down" % self.svnurl) self.loop.stop() return base.ChangeSource.stopService(self) def describe(self): return "SVNPoller watching %s" % self.svnurl def checksvn(self): # Our return value is only used for unit testing. # we need to figure out the repository root, so we can figure out # repository-relative pathnames later. Each SVNURL is in the form # (ROOT)/(PROJECT)/(BRANCH)/(FILEPATH), where (ROOT) is something # like svn://svn.twistedmatrix.com/svn/Twisted (i.e. there is a # physical repository at /svn/Twisted on that host), (PROJECT) is # something like Projects/Twisted (i.e. within the repository's # internal namespace, everything under Projects/Twisted/ has # something to do with Twisted, but these directory names do not # actually appear on the repository host), (BRANCH) is something like # "trunk" or "branches/2.0.x", and (FILEPATH) is a tree-relative # filename like "twisted/internet/defer.py". # our self.svnurl attribute contains (ROOT)/(PROJECT) combined # together in a way that we can't separate without svn's help. If the # user is not using the split_file= argument, then self.svnurl might # be (ROOT)/(PROJECT)/(BRANCH) . In any case, the filenames we will # get back from 'svn log' will be of the form # (PROJECT)/(BRANCH)/(FILEPATH), but we want to be able to remove # that (PROJECT) prefix from them. To do this without requiring the # user to tell us how svnurl is split into ROOT and PROJECT, we do an # 'svn info --xml' command at startup. This command will include a # <root> element that tells us ROOT. We then strip this prefix from # self.svnurl to determine PROJECT, and then later we strip the # PROJECT prefix from the filenames reported by 'svn log --xml' to # get a (BRANCH)/(FILEPATH) that can be passed to split_file() to # turn into separate BRANCH and FILEPATH values. # whew. if self.working: log.msg("SVNPoller(%s) overrun: timer fired but the previous " "poll had not yet finished.") self.overrun_counter += 1 return defer.succeed(None) self.working = True log.msg("SVNPoller polling") if not self._prefix: # this sets self._prefix when it finishes. It fires with # self._prefix as well, because that makes the unit tests easier # to write. d = self.get_root() d.addCallback(self.determine_prefix) else: d = defer.succeed(self._prefix) d.addCallback(self.get_logs) d.addCallback(self.parse_logs) d.addCallback(self.get_new_logentries) d.addCallback(self.create_changes) d.addCallback(self.submit_changes) d.addCallbacks(self.finished_ok, self.finished_failure) return d def getProcessOutput(self, args): # this exists so we can override it during the unit tests d = utils.getProcessOutput(self.svnbin, args, {}) return d def get_root(self): args = ["info", "--xml", "--non-interactive", self.svnurl] if self.svnuser: args.extend(["--username=%s" % self.svnuser]) if self.svnpasswd: args.extend(["--password=%s" % self.svnpasswd]) d = self.getProcessOutput(args) return d def determine_prefix(self, output): try: doc = xml.dom.minidom.parseString(output) except xml.parsers.expat.ExpatError: dbgMsg("_process_changes: ExpatError in %s" % output) log.msg("SVNPoller._determine_prefix_2: ExpatError in '%s'" % output) raise rootnodes = doc.getElementsByTagName("root") if not rootnodes: # this happens if the URL we gave was already the root. In this # case, our prefix is empty. self._prefix = "" return self._prefix rootnode = rootnodes[0] root = "".join([c.data for c in rootnode.childNodes]) # root will be a unicode string _assert( self.svnurl.startswith(root), "svnurl='%s' doesn't start with <root>='%s'" % (self.svnurl, root)) self._prefix = self.svnurl[len(root):] if self._prefix.startswith("/"): self._prefix = self._prefix[1:] log.msg("SVNPoller: svnurl=%s, root=%s, so prefix=%s" % (self.svnurl, root, self._prefix)) return self._prefix def get_logs(self, ignored_prefix=None): args = [] args.extend(["log", "--xml", "--verbose", "--non-interactive"]) if self.svnuser: args.extend(["--username=%s" % self.svnuser]) if self.svnpasswd: args.extend(["--password=%s" % self.svnpasswd]) args.extend(["--limit=%d" % (self.histmax), self.svnurl]) d = self.getProcessOutput(args) return d def parse_logs(self, output): # parse the XML output, return a list of <logentry> nodes try: doc = xml.dom.minidom.parseString(output) except xml.parsers.expat.ExpatError: dbgMsg("_process_changes: ExpatError in %s" % output) log.msg("SVNPoller._parse_changes: ExpatError in '%s'" % output) raise logentries = doc.getElementsByTagName("logentry") return logentries def _filter_new_logentries(self, logentries, last_change): # given a list of logentries, return a tuple of (new_last_change, # new_logentries), where new_logentries contains only the ones after # last_change if not logentries: # no entries, so last_change must stay at None return (None, []) mostRecent = int(logentries[0].getAttribute("revision")) if last_change is None: # if this is the first time we've been run, ignore any changes # that occurred before now. This prevents a build at every # startup. log.msg('svnPoller: starting at change %s' % mostRecent) return (mostRecent, []) if last_change == mostRecent: # an unmodified repository will hit this case log.msg('svnPoller: _process_changes last %s mostRecent %s' % (last_change, mostRecent)) return (mostRecent, []) new_logentries = [] for el in logentries: if last_change == int(el.getAttribute("revision")): break new_logentries.append(el) new_logentries.reverse() # return oldest first return (mostRecent, new_logentries) def get_new_logentries(self, logentries): last_change = self.last_change (new_last_change, new_logentries) = self._filter_new_logentries(logentries, self.last_change) self.last_change = new_last_change log.msg('svnPoller: _process_changes %s .. %s' % (last_change, new_last_change)) return new_logentries def _get_text(self, element, tag_name): child_nodes = element.getElementsByTagName(tag_name)[0].childNodes text = "".join([t.data for t in child_nodes]) return text def _transform_path(self, path): _assert( path.startswith(self._prefix), "filepath '%s' should start with prefix '%s'" % (path, self._prefix)) relative_path = path[len(self._prefix):] if relative_path.startswith("/"): relative_path = relative_path[1:] where = self.split_file(relative_path) # 'where' is either None or (branch, final_path) return where def create_changes(self, new_logentries): changes = [] for el in new_logentries: branch_files = [] # get oldest change first # TODO: revisit this, I think I've settled on Change.revision # being a string everywhere, and leaving the interpretation # of that string up to b.s.source.SVN methods revision = int(el.getAttribute("revision")) dbgMsg("Adding change revision %s" % (revision, )) # TODO: the rest of buildbot may not be ready for unicode 'who' # values author = self._get_text(el, "author") comments = self._get_text(el, "msg") # there is a "date" field, but it provides localtime in the # repository's timezone, whereas we care about buildmaster's # localtime (since this will get used to position the boxes on # the Waterfall display, etc). So ignore the date field and use # our local clock instead. #when = self._get_text(el, "date") #when = time.mktime(time.strptime("%.19s" % when, # "%Y-%m-%dT%H:%M:%S")) branches = {} pathlist = el.getElementsByTagName("paths")[0] for p in pathlist.getElementsByTagName("path"): path = "".join([t.data for t in p.childNodes]) # the rest of buildbot is certaily not yet ready to handle # unicode filenames, because they get put in RemoteCommands # which get sent via PB to the buildslave, and PB doesn't # handle unicode. path = path.encode("ascii") if path.startswith("/"): path = path[1:] where = self._transform_path(path) # if 'where' is None, the file was outside any project that # we care about and we should ignore it if where: branch, filename = where if not branch in branches: branches[branch] = [] branches[branch].append(filename) for branch in branches: c = Change(who=author, files=branches[branch], comments=comments, revision=revision, branch=branch) changes.append(c) return changes def submit_changes(self, changes): for c in changes: self.parent.addChange(c) def finished_ok(self, res): log.msg("SVNPoller finished polling") dbgMsg('_finished : %s' % res) assert self.working self.working = False return res def finished_failure(self, f): log.msg("SVNPoller failed") dbgMsg('_finished : %s' % f) assert self.working self.working = False return None # eat the failure
class TelnetProtocol(Telnet, StatefulTelnetProtocol, Session): """ Each player connecting over telnet (ie using most traditional mud clients) gets a telnet protocol instance assigned to them. All communication between game and player goes through here. """ def __init__(self, *args, **kwargs): self.protocol_name = "telnet" super(TelnetProtocol, self).__init__(*args, **kwargs) def connectionMade(self): """ This is called when the connection is first established. """ # initialize the session self.line_buffer = [] self.iaw_mode = False self.no_lb_mode = False client_address = self.transport.client client_address = client_address[0] if client_address else None # this number is counted down for every handshake that completes. # when it reaches 0 the portal/server syncs their data self.handshakes = 7 # naws, ttype, mccp, mssp, msdp, gmcp, mxp self.init_session(self.protocol_name, client_address, self.factory.sessionhandler) # negotiate client size self.naws = naws.Naws(self) # negotiate ttype (client info) # Obs: mudlet ttype does not seem to work if we start mccp before ttype. /Griatch self.ttype = ttype.Ttype(self) # negotiate mccp (data compression) - turn this off for wireshark analysis self.mccp = Mccp(self) # negotiate mssp (crawler communication) self.mssp = mssp.Mssp(self) # oob communication (MSDP, GMCP) - two handshake calls! self.oob = telnet_oob.TelnetOOB(self) # mxp support self.mxp = Mxp(self) # add this new connection to sessionhandler so # the Server becomes aware of it. self.sessionhandler.connect(self) # timeout the handshakes in case the client doesn't reply at all from evennia.utils.utils import delay delay(2, callback=self.handshake_done, retval=True) # TCP/IP keepalive watches for dead links self.transport.setTcpKeepAlive(1) # The TCP/IP keepalive is not enough for some networks; # we have to complement it with a NOP keep-alive. self.protocol_flags["NOPKEEPALIVE"] = True self.nop_keep_alive = None self.toggle_nop_keepalive() def _send_nop_keepalive(self): "Send NOP keepalive unless flag is set" if self.protocol_flags.get("NOPKEEPALIVE"): self._write(IAC + NOP) def toggle_nop_keepalive(self): """ Allow to toggle the NOP keepalive for those sad clients that can't even handle a NOP instruction. This is turned off by the protocol_flag NOPKEEPALIVE (settable e.g. by the default `@option` command). """ if self.nop_keep_alive and self.nop_keep_alive.running: self.nop_keep_alive.stop() else: self.nop_keep_alive = LoopingCall(self._send_nop_keepalive) self.nop_keep_alive.start(30, now=False) def handshake_done(self, force=False): """ This is called by all telnet extensions once they are finished. When all have reported, a sync with the server is performed. The system will force-call this sync after a small time to handle clients that don't reply to handshakes at all. """ if self.handshakes > 0: if force: self.sessionhandler.sync(self) return self.handshakes -= 1 if self.handshakes <= 0: # do the sync self.sessionhandler.sync(self) def enableRemote(self, option): """ This sets up the remote-activated options we allow for this protocol. Args: option (char): The telnet option to enable. Returns: enable (bool): If this option should be enabled. """ return (option == LINEMODE or option == ttype.TTYPE or option == naws.NAWS or option == MCCP or option == mssp.MSSP) def enableLocal(self, option): """ Call to allow the activation of options for this protocol Args: option (char): The telnet option to enable locally. Returns: enable (bool): If this option should be enabled. """ return (option == MCCP or option==ECHO) def disableLocal(self, option): """ Disable a given option locally. Args: option (char): The telnet option to disable locally. """ if option == ECHO: return True if option == MCCP: self.mccp.no_mccp(option) return True else: return super(TelnetProtocol, self).disableLocal(option) def connectionLost(self, reason): """ this is executed when the connection is lost for whatever reason. it can also be called directly, from the disconnect method Args: reason (str): Motivation for losing connection. """ self.sessionhandler.disconnect(self) self.transport.loseConnection() def dataReceived(self, data): """ Handle incoming data over the wire. This method will split the incoming data depending on if it starts with IAC (a telnet command) or not. All other data will be handled in line mode. Some clients also sends an erroneous line break after IAC, which we must watch out for. Args: data (str): Incoming data. Notes: OOB protocols (MSDP etc) already intercept subnegotiations on their own, never entering this method. They will relay their parsed data directly to self.data_in. """ if data and data[0] == IAC or self.iaw_mode: try: super(TelnetProtocol, self).dataReceived(data) if len(data) == 1: self.iaw_mode = True else: self.iaw_mode = False return except Exception as err1: conv = "" try: for b in data: conv += " " + repr(ord(b)) except Exception as err2: conv = str(err2) + ":", str(data) out = "Telnet Error (%s): %s (%s)" % (err1, data, conv) logger.log_trace(out) return if self.no_lb_mode and _RE_LEND.search(data): # we are in no_lb_mode and receive a line break; # this means we should empty the buffer and send # the command. data = "".join(self.line_buffer) + data data = data.rstrip("\r\n") + "\n" self.line_buffer = [] self.no_lb_mode = False elif not _RE_LEND.search(data): # no line break at the end of the command, buffer instead. self.line_buffer.append(data) self.no_lb_mode = True return # if we get to this point the command should end with a linebreak. # We make sure to add it, to fix some clients messing this up. StatefulTelnetProtocol.dataReceived(self, data) def _write(self, data): "hook overloading the one used in plain telnet" data = data.replace('\n', '\r\n').replace('\r\r\n', '\r\n') #data = data.replace('\n', '\r\n') super(TelnetProtocol, self)._write(mccp_compress(self, data)) def sendLine(self, line): """ Hook overloading the one used by linereceiver. Args: line (str): Line to send. """ #escape IAC in line mode, and correctly add \r\n line += self.delimiter line = line.replace(IAC, IAC + IAC).replace('\n', '\r\n') return self.transport.write(mccp_compress(self, line)) def lineReceived(self, string): """ Telnet method called when data is coming in over the telnet connection. We pass it on to the game engine directly. Args: string (str): Incoming data. """ self.data_in(text=string) # Session hooks def disconnect(self, reason=None): """ generic hook for the engine to call in order to disconnect this protocol. Args: reason (str): Reason for disconnecting. """ self.data_out(text=((reason or "",), {})) self.connectionLost(reason) def data_in(self, **kwargs): """ Data User -> Evennia Kwargs: kwargs (any): Options from the protocol. """ #from evennia.server.profiling.timetrace import timetrace #text = timetrace(text, "telnet.data_in") self.sessionhandler.data_in(self, **kwargs) def data_out(self, **kwargs): """ Data Evennia -> User Kwargs: kwargs (any): Options to the protocol """ self.sessionhandler.data_out(self, **kwargs) # send_* methods def send_text(self, *args, **kwargs): """ Send text data. This is an in-band telnet operation. Args: text (str): The first argument is always the text string to send. No other arguments are considered. Kwargs: options (dict): Send-option flags - mxp: Enforce MXP link support. - ansi: Enforce no ANSI colors. - xterm256: Enforce xterm256 colors, regardless of TTYPE. - noxterm256: Enforce no xterm256 color support, regardless of TTYPE. - nomarkup: Strip all ANSI markup. This is the same as noxterm256,noansi - raw: Pass string through without any ansi processing (i.e. include Evennia ansi markers but do not convert them into ansi tokens) - echo: Turn on/off line echo on the client. Turn off line echo for client, for example for password. Note that it must be actively turned back on again! """ text = args[0] if args else "" if text is None: return text = to_str(text, force_string=True) # handle arguments options = kwargs.get("options", {}) flags = self.protocol_flags xterm256 = options.get("xterm256", flags.get('XTERM256', False) if flags["TTYPE"] else True) useansi = options.get("ansi", flags.get('ANSI', False) if flags["TTYPE"] else True) raw = options.get("raw", flags.get("RAW", False)) nomarkup = options.get("nomarkup", flags.get("NOMARKUP", not (xterm256 or useansi))) echo = options.get("echo", None) mxp = options.get("mxp", flags.get("MXP", False)) screenreader = options.get("screenreader", flags.get("SCREENREADER", False)) if screenreader: # screenreader mode cleans up output text = ansi.parse_ansi(text, strip_ansi=True, xterm256=False, mxp=False) text = _RE_SCREENREADER_REGEX.sub("", text) if options.get("send_prompt"): # send a prompt instead. if not raw: # processing prompt = ansi.parse_ansi(_RE_N.sub("", text) + "{n", strip_ansi=nomarkup, xterm256=xterm256) if mxp: prompt = mxp_parse(prompt) prompt = prompt.replace(IAC, IAC + IAC).replace('\n', '\r\n') prompt += IAC + GA self.transport.write(mccp_compress(self, prompt)) else: if echo is not None: # turn on/off echo. Note that this is a bit turned around since we use # echo as if we are "turning off the client's echo" when telnet really # handles it the other way around. if echo: # by telling the client that WE WON'T echo, the client knows # that IT should echo. This is the expected behavior from # our perspective. self.transport.write(mccp_compress(self, IAC+WONT+ECHO)) else: # by telling the client that WE WILL echo, the client can # safely turn OFF its OWN echo. self.transport.write(mccp_compress(self, IAC+WILL+ECHO)) if raw: # no processing self.sendLine(text) return else: # we need to make sure to kill the color at the end in order # to match the webclient output. linetosend = ansi.parse_ansi(_RE_N.sub("", text) + "{n", strip_ansi=nomarkup, xterm256=xterm256, mxp=mxp) if mxp: linetosend = mxp_parse(linetosend) self.sendLine(linetosend) def send_prompt(self, *args, **kwargs): """ Send a prompt - a text without a line end. See send_text for argument options. """ kwargs["options"].update({"send_prompt": True}) self.send_text(*args, **kwargs) def send_default(self, cmdname, *args, **kwargs): """ Send other oob data """ if not cmdname == "options": self.oob.data_out(cmdname, *args, **kwargs)
class ServerController(object): def __init__(self, realm, view): print('server cont!') self.realm = realm self.view = view def _handleInput(self): """ Handle currently available pygame input events. """ for event in pygame.event.get(): if (event.type == pygame.QUIT) or ((event.type == pygame.KEYDOWN) and (event.key == QUIT)): reactor.stop() sys.exit() if (event.type == pygame.KEYDOWN): if (event.key == START_GAME): self.realm.environment.startGame() elif (event.key == RESET_GAME): self.realm.environment.setPreGame() def go(self): self.previousTime = pygame.time.get_ticks() self._inputCall = LoopingCall(self._handleInput) d = self._inputCall.start(0.03) return d def stop(self): self._inputCall.stop()