Ejemplo n.º 1
0
 def start(lc):
     global state, update, client, cleanup
     state = 0
     client.addDispatcher(d)
     lc = LoopingCall(update)
     dd = lc.start(1.0 / 30)
     dd.addCallbacks(cleanup)
Ejemplo n.º 2
0
class Rain(object):
    """
    Make it rain.

    Rain only occurs during spring.
    """

    implements(IAutomaton)

    blocks = tuple()

    def __init__(self):
        self.season_loop = LoopingCall(self.check_season)

    def scan(self, chunk):
        pass

    def feed(self, coords):
        pass

    def start(self):
        self.season_loop.start(5 * 60)

    def stop(self):
        self.season_loop.stop()

    def check_season(self):
        if factory.world.season.name == "spring":
            factory.vane.weather = "rainy"
            reactor.callLater(1 * 60, setattr, factory.vane, "weather",
                "sunny")

    name = "rain"
Ejemplo n.º 3
0
    def __init__(
        self, reactor, request, request_rate=10,
        sample_size=DEFAULT_SAMPLE_SIZE, timeout=45,
        tolerance_percentage=0.2
    ):
        """
        ``RequestLoadScenario`` constructor.

        :param tolerance_percentage: error percentage in the rate that is
            considered valid. For example, if we request a ``request_rate``
            of 20, and we give a tolerance_percentage of 0.2 (20%), anything
            in [16,20] will be a valid rate.
        """
        self.reactor = reactor
        self.request = request
        self.request_rate = request_rate
        self.timeout = timeout
        self._maintained = Deferred()
        self.rate_measurer = RateMeasurer(sample_size)
        self.max_outstanding = 10 * request_rate
        self.tolerated_errors = 5 * request_rate
        # Send requests per second
        self.loop = LoopingCall.withCount(self._request_and_measure)
        self.loop.clock = self.reactor
        # Monitor the status of the scenario
        self.monitor_loop = LoopingCall(self.check_rate)
        self.monitor_loop.clock = self.reactor
        self.is_started = False
        self.rate_tolerated = (
            float(request_rate) - (request_rate*tolerance_percentage)
        )
Ejemplo n.º 4
0
 def add(self, request, save_time = 0):
     self.data[request] = time.time() + save_time * 60
     d = request.notifyFinish()
     d.addCallback(self.notifyFinished, request)
     d.addErrback(self.notifyFinished, request)
     call = LoopingCall(self.prune)
     call.start(60)
Ejemplo n.º 5
0
def modbus_master(module, properties):
    log.debug('Modbus master module : '  + str(module))
    # Modbus Master
    #--------------------------------------------------------------------------#
    # initialize your data store
    #--------------------------------------------------------------------------#
    store = ModbusSlaveContext(
        co = ModbusSequentialDataBlock(0, [0]*100),
        hr = ModbusSequentialDataBlock(0, [0]*100))
    context = ModbusServerContext(slaves=store, single=True)

    #--------------------------------------------------------------------------#
    # initialize the server information
    #--------------------------------------------------------------------------#
    identity = ModbusDeviceIdentification()
    identity.VendorName  = 'ASO+AKO'
    identity.ProductCode = 'DYODE'
    identity.VendorUrl   = 'yoloswag'
    identity.ProductName = 'DYODE'
    identity.ModelName   = 'BSides LV release'
    identity.MajorMinorRevision = '0.9'

    #--------------------------------------------------------------------------#
    # run the server you want
    #--------------------------------------------------------------------------#
    time = 1 # 5 seconds delay
    loop = LoopingCall(f=modbus_master_update, a=(module, properties, context))
    loop.start(time, now=False) # initially delay by time
    StartTcpServer(context, identity=identity, address=("0.0.0.0", \
                   properties['port_out']))
Ejemplo n.º 6
0
class Pinger(object):
    """
    An periodic AMP ping helper.
    """
    def __init__(self, reactor):
        """
        :param IReactorTime reactor: The reactor to use to schedule the pings.
        """
        self.reactor = reactor

    def start(self, protocol, interval):
        """
        Start sending some pings.

        :param AMP protocol: The protocol over which to send the pings.
        :param timedelta interval: The interval at which to send the pings.
        """
        def ping():
            protocol.callRemote(NoOp)
        self._pinging = LoopingCall(ping)
        self._pinging.clock = self.reactor
        self._pinging.start(interval.total_seconds(), now=False)

    def stop(self):
        """
        Stop sending the pings.
        """
        self._pinging.stop()
Ejemplo n.º 7
0
class LeaseService(Service):
    """
    Manage leases.
    In particular, clear out expired leases once a second.

    :ivar _reactor: A ``IReactorTime`` provider.
    :ivar _persistence_service: The persistence service to act with.
    :ivar _lc: A ``twisted.internet.task.LoopingCall`` run every second
        to update the configured leases by releasing leases that have
        expired.
    """
    def __init__(self, reactor, persistence_service):
        self._reactor = reactor
        self._persistence_service = persistence_service

    def startService(self):
        self._lc = LoopingCall(self._expire)
        self._lc.clock = self._reactor
        self._lc.start(1)

    def stopService(self):
        self._lc.stop()

    def _expire(self):
        now = datetime.fromtimestamp(self._reactor.seconds(), tz=UTC)

        def expire(leases):
            updated_leases = leases.expire(now)
            for dataset_id in set(leases) - set(updated_leases):
                _LOG_EXPIRE(dataset_id=dataset_id,
                            node_id=leases[dataset_id].node_id).write()
            return updated_leases
        return update_leases(expire, self._persistence_service)
Ejemplo n.º 8
0
class BandwidthEstimator:
    bufsize = 20
    totalBytes = 0
    def __init__(self, message, length):
        self.length = length
        self.message = message
        self.estim = []
        self.bytes = 0
        self.call = LoopingCall(self.estimateBandwidth)
        self.call.start(1)

    def estimateBandwidth(self):
        bytes = self.bytes
        self.totalBytes += bytes
        self.estim.append(bytes)
        self.message("%0.2f k/s (%0.2d%%)"
                     % ((sum(self.estim) / len(self.estim)) / 1024.,
                        (float(self.totalBytes) / self.length) * 100))
        if len(self.estim) > self.bufsize:
            self.estim.pop(0)
        self.bytes = 0

    def stop(self):
        self.call.stop()
        self.estimateBandwidth()
        self.message("Finished receiving: %d bytes (%d%%)" % (
                self.totalBytes, (float(self.totalBytes) / self.length) * 100))
Ejemplo n.º 9
0
    def openShell(self, transport):
        """
        Write 60 lines of data to the transport, then exit.
        """
        proto = protocol.Protocol()
        proto.makeConnection(transport)
        transport.makeConnection(wrapProtocol(proto))

        # Send enough bytes to the connection so that a rekey is triggered in
        # the client.
        def write(counter):
            i = counter()
            if i == 60:
                call.stop()
                transport.session.conn.sendRequest(
                    transport.session, 'exit-status', '\x00\x00\x00\x00')
                transport.loseConnection()
            else:
                transport.write("line #%02d\n" % (i,))

        # The timing for this loop is an educated guess (and/or the result of
        # experimentation) to exercise the case where a packet is generated
        # mid-rekey.  Since the other side of the connection is (so far) the
        # OpenSSH command line client, there's no easy way to determine when the
        # rekey has been initiated.  If there were, then generating a packet
        # immediately at that time would be a better way to test the
        # functionality being tested here.
        call = LoopingCall(write, count().next)
        call.start(0.01)
Ejemplo n.º 10
0
    def __init__(self, settings, strategy_class):
        partition_id = settings.get('SCORING_PARTITION_ID')
        if partition_id is None or type(partition_id) != int:
            raise AttributeError("Scoring worker partition id isn't set.")

        messagebus = load_object(settings.get('MESSAGE_BUS'))
        mb = messagebus(settings)
        spider_log = mb.spider_log()
        scoring_log = mb.scoring_log()
        self.consumer = spider_log.consumer(partition_id=partition_id, type=b'sw')
        self.scoring_log_producer = scoring_log.producer()

        self._manager = FrontierManager.from_settings(settings, strategy_worker=True)
        codec_path = settings.get('MESSAGE_BUS_CODEC')
        encoder_cls = load_object(codec_path+".Encoder")
        decoder_cls = load_object(codec_path+".Decoder")
        self._decoder = decoder_cls(self._manager.request_model, self._manager.response_model)
        self._encoder = encoder_cls(self._manager.request_model)

        self.update_score = UpdateScoreStream(self._encoder, self.scoring_log_producer, 1024)
        self.states_context = StatesContext(self._manager.backend.states)

        self.consumer_batch_size = settings.get('SPIDER_LOG_CONSUMER_BATCH_SIZE')
        self.strategy = strategy_class.from_worker(self._manager, self.update_score, self.states_context)
        self.states = self._manager.backend.states
        self.stats = {
            'consumed_since_start': 0
        }
        self.job_id = 0
        self.task = LoopingCall(self.work)
        self._logging_task = LoopingCall(self.log_status)
        self._flush_states_task = LoopingCall(self.flush_states)
        logger.info("Strategy worker is initialized and consuming partition %d", partition_id)
Ejemplo n.º 11
0
class PeriodicTasksService(Service):
    def __init__(self, clock, taskFunction, interval=60):
        self.clock = clock
        self.taskFunction = taskFunction
        self.interval = interval
        self._delayedCall = None
        self._looper = None
        self._loopFinished = None

    def startService(self):
        Service.startService(self)
        now = time.time()
        startDelta = now // self.interval * self.interval + self.interval - now
        self._delayedCall = self.clock.callLater(startDelta, self._startLooping)

    def _startLooping(self):
        self._delayedCall = None
        self._looper = LoopingCall(self._call)
        self._looper.clock = self.clock
        self._loopFinished = self._looper.start(self.interval)

    def _call(self):
        d = deferToThread(self.taskFunction)
        d.addErrback(log.err, "error calling periodic tasks")
        return d

    def stopService(self):
        if self._delayedCall:
            self._delayedCall.cancel()
            self._delayedCall = None
            return
        self._looper.stop()
        self._loopFinished.addCallback(lambda _: Service.stopService(self))
        return self._loopFinished
Ejemplo n.º 12
0
 def __init__(self):
     self.output("TWS init")
     self.username = environ["TXTRADER_USERNAME"]
     self.password = environ["TXTRADER_PASSWORD"]
     self.xmlrpc_port = int(environ["TXTRADER_XMLRPC_PORT"])
     self.tcp_port = int(environ["TXTRADER_TCP_PORT"])
     self.callback_timeout = int(environ["TXTRADER_TWS_CALLBACK_TIMEOUT"])
     if not self.callback_timeout:
         self.callback_timeout = DEFAULT_TWS_CALLBACK_TIMEOUT
     self.output("callback_timeout=%d" % self.callback_timeout)
     self.enable_ticker = bool(int(environ["TXTRADER_ENABLE_TICKER"]))
     self.label = "TWS Gateway"
     self.channel = "tws"
     self.current_account = ""
     self.clients = set([])
     self.orders = {}
     self.pending_orders = {}
     self.openorder_callbacks = []
     self.accounts = []
     self.account_data = {}
     self.positions = {}
     self.position_callbacks = []
     self.executions = {}
     self.execution_callbacks = []
     self.bardata_callbacks = []
     self.cancel_callbacks = []
     self.order_callbacks = []
     self.addsymbol_callbacks = []
     self.accountdata_callbacks = []
     self.last_connection_status = ""
     self.connection_status = "Initializing"
     self.LastError = -1
     self.next_order_id = -1
     self.last_minute = -1
     self.handlers = {
         "error": self.error_handler,
         "tickSize": self.handle_tick_size,
         "tickPrice": self.handle_tick_price,
         "tickString": self.handle_tick_string,
         "nextValidId": self.handle_next_valid_id,
         "currentTime": self.handle_time,
         "managedAccounts": self.handle_accounts,
         "orderStatus": self.handle_order_status,
         "openOrder": self.handle_open_order,
         "openOrderEnd": self.handle_open_order_end,
         "execDetails": self.handle_exec_details,
         "execDetailsEnd": self.handle_exec_details_end,
         "position": self.handle_position,
         "positionEnd": self.handle_position_end,
         "historicalData": self.handle_historical_data,
         "updateAccountValue": self.handle_account_value,
         "accountDownloadEnd": self.handle_account_download_end,
     }
     self.ticker_ids = {}
     self.symbols = {}
     self.symbols_by_id = {}
     self.primary_exchange_map = {}
     self.tws_conn = None
     repeater = LoopingCall(self.EverySecond)
     repeater.start(1)
Ejemplo n.º 13
0
class _ExchangeRate(object):
    """Download an exchange rate from Yahoo Finance using Twisted."""

    def __init__(self, name):
        self._value = None
        self._name = name

    # External API:
    def latest_value(self):
        """Return the latest exchange rate value.

        May be None if no value is available.
        """
        return self._value

    def start(self):
        """Start the background process."""
        self._lc = LoopingCall(self._download)
        # Run immediately, and then every 30 seconds:
        self._lc.start(30, now=True)

    def _download(self):
        """Download the page."""
        print("Downloading!")
        def parse(result):
            print("Got %r back from Yahoo." % (result,))
            values = result.strip().split(",")
            self._value = float(values[1])
        d = getPage(
            "http://download.finance.yahoo.com/d/quotes.csv?e=.csv&f=c4l1&s=%s=X"
            % (self._name,))
        d.addCallback(parse)
        d.addErrback(log.err)
        return d
Ejemplo n.º 14
0
def make_lc(self, reactor, func):

    if DEBUG:
        self.stdout_length = 0
        self.stderr_length = 0

    def _(lc, reactor):
        if DEBUG:
            stdout = self.stdout.getvalue()
            stderr = self.stderr.getvalue()

            if self.stdout.getvalue()[self.stdout_length:]:
                print(self.stdout.getvalue()[self.stdout_length:],
                      file=sys.__stdout__)
            if self.stderr.getvalue()[self.stderr_length:]:
                print(self.stderr.getvalue()[self.stderr_length:],
                      file=sys.__stderr__)

            self.stdout_length = len(stdout)
            self.stderr_length = len(stderr)

        return func(lc, reactor)

    lc = LoopingCall(_)
    lc.a = (lc, reactor)
    lc.clock = reactor
    lc.start(0.1)
    return lc
Ejemplo n.º 15
0
class PollingDataStream(DataStream):
    """ A self-polling data stream.

    This class represents a data stream that wakes up at a given frequency,
    and calls the :meth:`poll` method.
    """

    frequency = None  # Either a timedelta object, or the number of seconds
    now = False

    def __init__(self):
        super(PollingDataStream, self).__init__()
        self.timer = LoopingCall(self.poll)
        if isinstance(self.frequency, timedelta):
            seconds = (
                self.frequency.seconds
                + (self.frequency.days * 24 * 60 * 60)
                + (self.frequency.microseconds / 1000000.0)
            )
        else:
            seconds = self.frequency
        log.debug("Setting a %s second timer" % seconds)
        self.timer.start(seconds, now=self.now)

    def poll(self):
        raise NotImplementedError

    def stop(self):
        super(PollingDataStream, self).stop()
        try:
            if hasattr(self, "timer"):
                self.timer.stop()
        except Exception, e:
            self.log.warn(e)
Ejemplo n.º 16
0
        class SendsManyFileDescriptors(ConnectableProtocol):
            paused = False

            def connectionMade(self):
                self.socket = socket()
                self.transport.registerProducer(self, True)
                def sender():
                    self.transport.sendFileDescriptor(self.socket.fileno())
                    self.transport.write(b"x")
                self.task = LoopingCall(sender)
                self.task.clock = self.transport.reactor
                self.task.start(0).addErrback(err, "Send loop failure")

            def stopProducing(self):
                self._disconnect()

            def resumeProducing(self):
                self._disconnect()

            def pauseProducing(self):
                self.paused = True
                self.transport.unregisterProducer()
                self._disconnect()

            def _disconnect(self):
                self.task.stop()
                self.transport.abortConnection()
                self.other.transport.abortConnection()
Ejemplo n.º 17
0
    def __init__(self, protocol):
        # info from hamlib
        self.__cache = {}
        self.__caps = {}
        self.__levels = []
        
        # invert command table
        # TODO: we only need to do this once per class, really
        self._how_to_command = {key: command
            for command, keys in self._commands.iteritems()
            for key in keys}
        
        # keys are same as __cache, values are functions to call with new values from rig
        self._cell_updaters = {}
        
        self.__communication_error = False
        self.__last_error = (-1e9, '', 0)
        
        self.__protocol = protocol
        self.__disconnect_deferred = defer.Deferred()
        protocol._set_proxy(self)

        # TODO: If hamlib backend supports "transceive mode", use it in lieu of polling
        self.__poller_slow = LoopingCall(self.__poll_slow)
        self.__poller_fast = LoopingCall(self.__poll_fast)
        self.__poller_slow.start(2.0)
        self.__poller_fast.start(0.2)
        
        self._ready_deferred = protocol.rc_send('dump_caps')
class ICUMonitorFactory(protocol.ClientFactory):
        def __init__(self, gui_msg_callback, gui_ip, gui_port):
                self.ip = gui_ip
                self.port = gui_port
                self.msg_callback = gui_msg_callback
                self.objmonit = None

        def buildProtocol(self, addr):
                return ICUMonitor(self.msg_callback)

        def clientConnectionLost(self, connector, reason):
                pass
        
        def clientConnectionFailed(self, connector, reason):
                pass

        def sendmsg(self, gui_reactor):
                gui_reactor.connectTCP(self.ip, self.port, self)

        def startsend(self, gui_reactor, timerep):
                self.loop = LoopingCall(self.sendmsg, gui_reactor)
                self.loop.start(timerep)

        def stopsend(self):
                self.loop.stop()
Ejemplo n.º 19
0
class ProvisionerQueryService(ServiceProcess):
    """Provisioner querying service
    """

    declare = ServiceProcess.service_declare(name='provisioner_query',
                                             version='0.1.0',
                                             dependencies=[])

    def slc_init(self):
        interval = float(self.spawn_args.get("interval_seconds",
                                             DEFAULT_QUERY_INTERVAL))

        self.client = ProvisionerClient(self)

        log.debug('Starting provisioner query loop - %s second interval',
                  interval)
        self.loop = LoopingCall(self.query)
        self.loop.start(interval)

    def slc_terminate(self):
        if self.loop:
            self.loop.stop()

    @defer.inlineCallbacks
    def query(self):
        try:
            yield self._do_query()
        except Exception,e:
            log.error("Error sending provisioner query request: %s", e,
                      exc_info=True)
Ejemplo n.º 20
0
	def startService(self):
		self.log.info("Starting up...")
		self.startupTime = now()
		self.log.info("Loading configuration...")
		self.config.reload()
		self.name = self.config["server_name"]
		self.serverID = self.config["server_id"]
		self.log.info("Loading storage...")
		self.storage = shelve.open(self.config["datastore_path"], writeback=True)
		self.storageSyncer = LoopingCall(self.storage.sync)
		self.storageSyncer.start(self.config.get("storage_sync_interval", 5), now=False)
		self.log.info("Starting processes...")
		self.pruneRecentlyQuit = LoopingCall(self.pruneQuit)
		self.pruneRecentlyQuit.start(10, now=False)
		self.pruneRecentChannels = LoopingCall(self.pruneChannels)
		self.pruneRecentChannels.start(15, now=False)
		self.log.info("Loading modules...")
		self._loadModules()
		self.log.info("Binding ports...")
		self._bindPorts()
		self.log.info("txircd started!")
		try:
			self._logFilter.setLogLevelForNamespace("txircd", LogLevel.levelWithName(self.config["log_level"]))
		except (KeyError, InvalidLogLevelError):
			self._logFilter.setLogLevelForNamespace("txircd", LogLevel.warn)
		self.runActionStandard("startup")
Ejemplo n.º 21
0
    def wait_for_volume(self, name):
        """
        Wait for a volume by the given name, owned by thus service, to exist.

        Polls the storage pool for the specified volume to appear.

        :param VolumeName name: The name of the volume.

        :return: A ``Deferred`` that fires with a :class:`Volume`.
        """
        # Change this to not create the Volume instance right away.  Instead,
        # try to find it by uuid/name in `check_for_volume`.  If a Volume
        # instance there matches, use that object as the final result of the
        # Deferred returned by this method (it will have its other attributes
        # set correctly because they will be set correctly by enumerate).
        # FLOC-976
        volume = Volume(uuid=self.uuid, name=name, service=self)

        def check_for_volume(volumes):
            if volume in volumes:
                call.stop()

        def loop():
            d = self.enumerate()
            d.addCallback(check_for_volume)
            return d

        call = LoopingCall(loop)
        call.clock = self._reactor
        d = call.start(WAIT_FOR_VOLUME_INTERVAL)
        d.addCallback(lambda _: volume)
        return d
Ejemplo n.º 22
0
class WebSocketConnection(WebSocketProtocol):
	def __init__(self, stateObject):
		self.state = stateObject
		self.opcode = TEXT
		WebSocketProtocol.__init__(self, WebSocketReceiver(self))
		self.finished = Deferred()
		self.pingLoop = LoopingCall(self.doPing)

	def doPing(self):
		self.receiver.transport.sendFrame(PING, '')

	def write(self, data):
		self.receiver.transport.sendFrame(self.opcode, data)

	def sendFrame(self, opcode, data):
		self.receiver.transport.sendFrame(opcode, data)

	def writeSequence(self, data):
		for chunk in data:
			self.write(chunk)

	def connectionLost(self, reason):
		if self.pingLoop.running:
			self.pingLoop.stop()
		self.finished.callback(self)
Ejemplo n.º 23
0
    def __init__(self):
        """
        Initialize the model. This doesn't add any documents yet.
        """
        silenceGensim()

        self.dictionaries = dict()
        self.preprocessor = TokenizingPorter2Stemmer()

        #this dict keeps a model for every source type
        #  (since e.g. RSS feeds should be treated separately from twitter feeds)
        self.models = dict()

        #this dict keeps a dictionary for every source type 
        self.dictionaries = dict()

        self.queue = Queue()
        self.modelQueue = Queue()
        self.nodeCommunicator = NodeCommunicator(self, LISTEN_PORT)
        self.nodeCommunicator.registerWithNode(CORE_IP, REGISTER_PORT)  # register this node with the core

        ln.info("LSA Initialized")
        self.updating = False
        loop = LoopingCall(self.update)
        loop.start(5)

        reactor.run()
Ejemplo n.º 24
0
class CacheService(Service):
    def __init__(self, *args, **kwargs):
        self.call = None

        self.node_updater = NodeCacheUpdater()
        self.cluster_updater = ClusterCacheUpdater()
        self.vm_updater = VirtualMachineCacheUpdater()
        self.job_updater = JobCacheUpdater()

    def update_cache(self):
        """ a single run of all update classes """
        return DeferredList(
            [
                self.vm_updater.update(),
                self.job_updater.update(),
                self.node_updater.update(),
                self.cluster_updater.update(),
            ]
        )

    def startService(self):
        self.call = LoopingCall(self.update_cache)
        self.call.start(settings.PERIODIC_CACHE_REFRESH)

    def stopService(self):
        if self.call is not None:
            self.call.stop()
Ejemplo n.º 25
0
Archivo: home.py Proyecto: mbau/memory
class Hoster:
	def __init__(self, screen, port):
		self.screen = screen
		self.port = port

	def start(self):
		# Set up the connection between the state and the network;
		# using queues gives a degree of seperation between the
		# communication and game logic, and making them be deferred
		# keeps it all asynchronous.
		inqueue = DeferredQueue()
		outqueue = DeferredQueue()

		#Initialize GameState
		gs = GameState(self,self.screen,1,inqueue,outqueue)

		#Create Looping Call
		self.lc = LoopingCall(gs.gameLoop)
		self.lc.start(.0166666666)
		
		#Begin Listening
		connfactory = CommandConnFactory(inqueue,outqueue)
		self.listening = reactor.listenTCP(self.port,connfactory)

	def stop(self, nextscreen=None):
		# Stop the GameState logic, and let go of the port on which we're listening
		self.lc.stop()
		self.listening.stopListening()

		# Start up the next screen, if there is one
		if nextscreen:
			nextscreen.start()
		else: reactor.stop()
Ejemplo n.º 26
0
    def onJoin(self, details):
        log.msg("PiMonitor connected")

        extra = self.config.extra

        self._id = extra['id']
        self.publish_temperature = True
        self.threshold = 0

        self._tick = 0
        self._cpu_temp_celsius = None

        def scanTemperature():
            self._cpu_temp_celsius = float(open("/sys/class/thermal/thermal_zone0/temp").read()) / 1000.
            
            if self.publish_temperature:
                self.publish(u'io.crossbar.examples.pi.{}.tempmon.on_temperature'.format(self._id), self._tick, self._cpu_temp_celsius)
                self._tick += 1

            if self.threshold > 0 and self._cpu_temp_celsius > self.threshold:
                self.publish(u'io.crossbar.examples.pi.{}.tempmon.on_threshold_exceeded'.format(self._id), self._tick, self._cpu_temp_celsius)
          
        scan = LoopingCall(scanTemperature)
        scan.start(1)

        for proc in [self.get_temperature, self.impose_stress, self.toggle_publish, self.set_threshold]:
            uri = u'io.crossbar.examples.pi.{}.tempmon.{}'.format(self._id, proc.__name__)
            yield self.register(proc, uri)
            log.msg("Registered {}".format(uri))

        log.msg("PiMonitor ready.")
Ejemplo n.º 27
0
class EventGenerator(Node):
    """ (TEST) A timer-based event generator """

    # --- Initialization
    def __init__(self):
        self.count = 0

    # --- Interfaces
    def configure(self):

        self.events = (
            'event1',
            'event2',
        )

    def setup(self):
        self.status.set_GREEN()

        self.loop1 = LoopingCall(self.loop_cb1)
        self.loop1.start(3, False)

        self.loop2 = LoopingCall(self.loop_cb2)
        self.loop2.start(4, False)

    # --- Workers
    def loop_cb1(self):
        self.count += 1
        self.sendEvent('event1', self.count)

    def loop_cb2(self):
        self.count += 1
        self.sendEvent('event2', self.count)
Ejemplo n.º 28
0
    def delayed_setup(self):
        self.feeds = []

        self.failures.clear()
        self.feed_times.clear()
        self.targets.clear()
        self.tasks.clear()

        for name, target in self.config["targets"].items():
            proto = target["protocol"]
            if proto in self.factory_manager.factories:
                self.targets[name] = target
            else:
                self.logger.warn("Unknown protocol '%s' in target '%s'"
                                 % (proto, name))

        for feed in self.config["feeds"]:
            append = True
            for target in feed["targets"]:
                if target["name"] not in self.targets:
                    self.logger.warn("Unknown target '%s' for feed '%s'"
                                     % (target["name"], feed["name"]))
                    append = False
                    break

            if append:
                self.feeds.append(feed)

        for feed in self.feeds:
            task = LoopingCall(self.check_feed, feed)
            self.tasks[feed["name"]] = task
            task.start(feed["frequency"])
Ejemplo n.º 29
0
    def reconfigServiceWithBuildbotConfig(self, new_config):
        # first, enable or disable
        if new_config.metrics is None:
            self.disable()
        else:
            self.enable()

            metrics_config = new_config.metrics

            # Start up periodic logging
            log_interval = metrics_config.get('log_interval', 60)
            if log_interval != self.log_interval:
                if self.log_task:
                    self.log_task.stop()
                    self.log_task = None
                if log_interval:
                    self.log_task = LoopingCall(self.report)
                    self.log_task.clock = self._reactor
                    self.log_task.start(log_interval)

            # same for the periodic task
            periodic_interval = metrics_config.get('periodic_interval', 10)
            if periodic_interval != self.periodic_interval:
                if self.periodic_task:
                    self.periodic_task.stop()
                    self.periodic_task = None
                if periodic_interval:
                    self.periodic_task = LoopingCall(periodicCheck,
                                                     self._reactor)
                    self.periodic_task.clock = self._reactor
                    self.periodic_task.start(periodic_interval)

        # upcall
        return util_service.ReconfigurableServiceMixin.reconfigServiceWithBuildbotConfig(self,
                                                                                         new_config)
Ejemplo n.º 30
0
    def __init__(self, initial_name, shared_path='/tmp/mark2'):
        self.socket_to = lambda n: os.path.join(shared_path, n + ".sock")
        self.socket_from = lambda p: os.path.splitext(os.path.basename(p))[0]

        self.client = None
        self.stats = {}
        self.system_users = SystemUsers()

        #read the config
        self.config = properties.load(
            properties.ClientProperties,
            open_resource('resources/mark2rc.default.properties'),
            os.path.expanduser('~/.mark2rc.properties'))
        assert self.config is not None
        self.stats_template = Template(self.config['stats'])

        #start apps
        self.apps = []

        #start ui
        self.ui = UI(self.config.get_palette(), self.get_players,
                     self.run_command, self.switch_server,
                     self.connect_to_server, self.config.get_player_actions(),
                     self.config.get_player_reasons())
        for name, command in self.config.get_apps():
            app = App(name, self.config.get_interval('apps'), self.app_update,
                      self.config['stats.app_shell'], command)
            self.apps.append(app)

        #tasks
        t = LoopingCall(self.update_servers)
        t.start(self.config.get_interval('servers'))

        t = LoopingCall(self.update_users)
        t.start(self.config.get_interval('users'))

        t = LoopingCall(self.update_players)
        t.start(self.config.get_interval('players'))

        t = LoopingCall(self.update_stats)
        t.start(self.config.get_interval('stats'))

        self.connect_to_server(initial_name)
Ejemplo n.º 31
0
class WebWallet(object):
    """Represents a wallet, usable within the explorer web app."""
    def __init__(self):
        self.wallet_name = None
        self.wallets = {}
        self.taskq = queue.Queue()

        # load wallets from disk
        self.load_wallets()

        self.looping_call = LoopingCall(self.miner_tick)
        tick_seconds = 1
        self.looping_call.start(tick_seconds)

    def is_mining(self):
        """Returns True if mining is in progress. False otherwise."""
        return self.taskq.qsize() > 0

    def mine(self, block):
        """Compute proof of work."""
        i = 0
        print(f"target: {block.target}")
        print(f"block initial proof: {block.proof}")
        while True:
            proof = str(i)
            if block.validate_proof(proof=proof):
                block.proof = proof
                return block
            i += 1

    def miner_tick(self):
        """Check whether any mining tasks were added, and if so
        get them from the task queue and execute them."""
        while True:
            try:
                wallet, p2p = self.taskq.get(block=False, timeout=1)
                magic = p2p.bc.magic
                coinbase = Transaction("0",
                                       wallet.get_address(),
                                       1,
                                       magic=magic)
                wallet.sign_transaction(coinbase)

                b = p2p.bc.new_block(tx=coinbase)
                self.mine(b)
                success = p2p.bc.discard_block(b)
                if success:
                    p2p.broadcast_block(b)
                    logger.debug("Successfully mined and broadcasted block.")
                else:
                    logger.debug("Failed to discard block.")
            except queue.Empty:
                break

    def wallet_names(self):
        """Returns a list of names of generated wallets"""
        return self.wallets.keys()

    def load_wallets(self):
        """Load existing wallets from disk."""
        count = 0
        for path in glob("webwallet_*.wallet"):
            w = Wallet.load_keys(path)
            self.wallets[path] = w
            count += 1
        logging.debug(f"Successfully loaded {count} wallets from disk")

    def generate_wallet(self):
        """Generate a new wallet."""
        w = Wallet()
        w.create_keys()
        first_free_count = 1
        while True:
            path = f"webwallet_{first_free_count}.wallet"
            if not os.path.exists(path):
                break
            first_free_count += 1

        # save wallet to disk
        w.save_key(path)

        # save wallet to memory
        self.wallets[path] = w

    def set_active_wallet(self, wallet_name):
        """Set the wallet with name `wallet_name` as the active wallet."""
        if wallet_name in self.wallets:
            self.wallet_name = wallet_name

    def active_wallet(self):
        """Returns the name of the active wallet."""
        return self.wallet_name

    def get_active_wallet(self):
        """Returns the active wallet or None if there is no active wallet."""
        return self.wallets.get(self.wallet_name, None)

    def send_tx(self, destination, quantity, p2p):
        """Perform a transaction."""
        w = self.get_active_wallet()
        magic = p2p.bc.magic
        tx = Transaction(w.get_address(), destination, quantity, magic=magic)
        w.sign_transaction(tx)

        success = p2p.bc.add_transaction(tx)
        if success:
            # broadcast transaction to p2p network
            p2p.broadcast_tx(tx)
            return True
        else:
            logging.debug("Failed to add transaction.")
            return False

    def start_mining(self, p2p):
        """Start mining a block (add task to queue)."""
        wallet = self.get_active_wallet()
        self.taskq.put((wallet, p2p))
Ejemplo n.º 32
0
class AdapterPmMetrics:
    def __init__(self, device):
        self.pm_names = {'tx_64_pkts', 'tx_65_127_pkts', 'tx_128_255_pkts',
                         'tx_256_511_pkts', 'tx_512_1023_pkts',
                         'tx_1024_1518_pkts', 'tx_1519_9k_pkts',
                         'rx_64_pkts', 'rx_65_127_pkts',
                         'rx_128_255_pkts', 'rx_256_511_pkts',
                         'rx_512_1023_pkts', 'rx_1024_1518_pkts',
                         'rx_1519_9k_pkts'}
        self.device = device
        self.id = device.id
        self.name = 'ponsim_olt'
        # self.id = "abc"
        self.default_freq = 150
        self.grouped = False
        self.freq_override = False
        self.pon_metrics_config = dict()
        self.nni_metrics_config = dict()
        self.lc = None
        for m in self.pm_names:
            self.pon_metrics_config[m] = PmConfig(name=m,
                                                  type=PmConfig.COUNTER,
                                                  enabled=True)
            self.nni_metrics_config[m] = PmConfig(name=m,
                                                  type=PmConfig.COUNTER,
                                                  enabled=True)

    def update(self, pm_config):
        if self.default_freq != pm_config.default_freq:
            # Update the callback to the new frequency.
            self.default_freq = pm_config.default_freq
            self.lc.stop()
            self.lc.start(interval=self.default_freq / 10)
        for m in pm_config.metrics:
            self.pon_metrics_config[m.name].enabled = m.enabled
            self.nni_metrics_config[m.name].enabled = m.enabled

    def make_proto(self):
        pm_config = PmConfigs(
            id=self.id,
            default_freq=self.default_freq,
            grouped=False,
            freq_override=False)
        for m in sorted(self.pon_metrics_config):
            pm = self.pon_metrics_config[m]  # Either will do they're the same
            pm_config.metrics.extend([PmConfig(name=pm.name,
                                               type=pm.type,
                                               enabled=pm.enabled)])
        return pm_config

    def collect_port_metrics(self, channel):
        rtrn_port_metrics = dict()
        stub = ponsim_pb2_grpc.PonSimStub(channel)
        stats = stub.GetStats(ponsim_pb2.PonSimMetricsRequest(port=0))
        rtrn_port_metrics['pon'] = self.extract_pon_metrics(stats)
        rtrn_port_metrics['nni'] = self.extract_nni_metrics(stats)
        return rtrn_port_metrics

    def extract_pon_metrics(self, stats):
        rtrn_pon_metrics = dict()
        for m in stats.metrics:
            if m.port_name == "pon":
                for p in m.packets:
                    if self.pon_metrics_config[p.name].enabled:
                        rtrn_pon_metrics[p.name] = p.value
                return rtrn_pon_metrics

    def extract_nni_metrics(self, stats):
        rtrn_pon_metrics = dict()
        for m in stats.metrics:
            if m.port_name == "nni":
                for p in m.packets:
                    if self.pon_metrics_config[p.name].enabled:
                        rtrn_pon_metrics[p.name] = p.value
                return rtrn_pon_metrics

    def start_collector(self, callback):
        log.info("starting-pm-collection", device_name=self.name,
                 device_id=self.device.id)
        prefix = 'voltha.{}.{}'.format(self.name, self.device.id)
        self.lc = LoopingCall(callback, self.device.id, prefix)
        self.lc.start(interval=self.default_freq / 10)

    def stop_collector(self):
        log.info("stopping-pm-collection", device_name=self.name,
                 device_id=self.device.id)
        self.lc.stop()
Ejemplo n.º 33
0
 def start_collector(self, callback):
     log.info("starting-pm-collection", device_name=self.name,
              device_id=self.device.id)
     prefix = 'voltha.{}.{}'.format(self.name, self.device.id)
     self.lc = LoopingCall(callback, self.device.id, prefix)
     self.lc.start(interval=self.default_freq / 10)
Ejemplo n.º 34
0
 def __init__(self):
     self.storage_reload_task = LoopingCall(reloadStorageSchemas)
     self.aggregation_reload_task = LoopingCall(reloadAggregationSchemas)
Ejemplo n.º 35
0
class Window(object):
    def __init__(self, environment):
        self.environment = environment
        self.images = Images(FilePath("data").child("img2"))
        self.images.load()
        self.actions = deque()
        self.action = None
        self.center = Vector2D((0, 0))

    def addAction(self, action):
        # TODO We no longer have actions?
        pass

    def startAction(self):
        if self.action == None:
            try:
                self.action = self.actions.popleft()
                self.action.start(5).addCallback(self.stopAction)
            except:
                pass

    def stopAction(self, ign):
        self.action = None
        self.startAction()

    def paint(self):
        """
        Call C{paint} on all views which have been directly added to
        this Window.
        """
        bg = self.images.images["background"]
        bgWidth = bg.width
        bgHeight = bg.height
        x = -(self.center * 20).x
        y = -(self.center * 20).y
        while x > 0:
            x -= bgWidth
        while y > 0:
            y -= bgHeight
        while x < self.screen.get_width():  #800:#480:
            j = y
            while j < self.screen.get_height():  #480:#800:
                self.screen.blit(bg._image,
                                 pygame.Rect(x, j, bgWidth, bgHeight))
                j += bgHeight
            x += bgWidth
        self.environment.paint(self)
        if self.action:
            #   self.action.draw(self.screen, Vector2D((240, 400)))
            pass
        pygame.display.flip()

    def setCenter(self, position):
        self.center = position

    def worldCoord(self, p):
        width = self.screen.get_width()
        height = self.screen.get_height()
        (cx, cy) = self.screen.get_rect().center
        return Vector2D(((p.x - cx) * self.environment.width) / width,
                        ((p.y - cy) * self.environment.height) / height)

    def screenCoord(self, p):
        width = self.screen.get_width()
        height = self.screen.get_height()
        (cx, cy) = self.screen.get_rect().center
        return Vector2D((((p.x - self.center[0]) /
                          (self.environment.width / 2)) * width) + cx,
                        (((p.y - self.center[1]) /
                          (self.environment.height / 2)) * height) + cy)

    def start(self, title):
        self.screen = pygame.display.get_surface()
        pygame.display.set_caption(title)
        self._renderCall = LoopingCall(self.paint)
        self._renderCall.start(0.03)

        (cx, cy) = self.screen.get_rect().center
#        print "envirn: " + str(self.environment.width) + " x " + str(self.environment.height)
#        print "screen: " + str(self.screen.get_width()) + " x " + str(self.screen.get_height())
#        print "scrctr: " + str(cx) + ", " + str(cy)

    def stop(self):
        self._renderCall.stop()
Ejemplo n.º 36
0
 def loop_call(self, delay, func, *arg, **kw):
     loop = LoopingCall(func, *arg, **kw)
     loop.start(delay, False)
     self.loops.add(loop)
     return loop
Ejemplo n.º 37
0
import time

from twisted.internet import reactor, threads
from twisted.internet.task import LoopingCall


def blockingApiCall(arg):
    time.sleep(1)
    return arg


def nonblockingCall(arg):
    print arg


def printResult(result):
    print result


def finish(result):
    reactor.stop()


d = threads.deferToThread(blockingApiCall, "Goose")
d.addCallback(printResult)
d.addCallback(finish)

LoopingCall(nonblockingCall, "Duck").start(.25)

reactor.run()
Ejemplo n.º 38
0
class RewriteRuleManager:
    def __init__(self):
        self.rulesets = defaultdict(list)
        self.rules_file = None
        self.read_task = LoopingCall(self.read_rules)
        self.rules_last_read = 0.0

    def clear(self, ruleset=None):
        if ruleset:
            self.rulesets[ruleset] = []
        else:
            self.rulesets.clear()

    def rules(self, ruleset):
        return self.rulesets[ruleset]

    def read_from(self, rules_file):
        self.rules_file = rules_file
        self.read_rules()
        if not self.read_task.running:
            self.read_task.start(10, now=False)

    def read_rules(self):
        if not exists(self.rules_file):
            self.clear()
            return

        # Only read if the rules file has been modified
        try:
            mtime = getmtime(self.rules_file)
        except (OSError, IOError):
            log.err("Failed to get mtime of %s" % self.rules_file)
            return
        if mtime <= self.rules_last_read:
            return

        section = None
        for line in open(self.rules_file):
            line = line.strip()
            if line.startswith('#') or not line:
                continue

            if line.startswith('[') and line.endswith(']'):
                section = line[1:-1].lower()
                self.clear(section)
            elif '=' in line:
                pattern, replacement = line.split('=', 1)
                pattern, replacement = pattern.strip(), replacement.strip()
                try:
                    rule = RewriteRule(pattern, replacement)
                except re.error:
                    log.err(
                        "Invalid regular expression in rewrite rule: '{0}'".
                        format(pattern))
                    continue

                self.rulesets[section].append(rule)
            else:
                log.err("Invalid syntax: not a section heading or rule: '{0}'".
                        format(line))

        self.rules_last_read = mtime
Ejemplo n.º 39
0
        # check cache size every 5 minutes
        _FLUSH_CACHE(_IDMAPPER_CACHE_MAXSIZE)
    if _MAINTENANCE_COUNT % 3600 == 0:
        # validate scripts every hour
        evennia.ScriptDB.objects.validate()
    if _MAINTENANCE_COUNT % 3700 == 0:
        # validate channels off-sync with scripts
        evennia.CHANNEL_HANDLER.update()
    ## Commenting this out, it is probably not needed
    ## with CONN_MAX_AGE set. Keeping it as a reminder
    ## if database-gone-away errors appears again /Griatch
    #if _MAINTENANCE_COUNT % 18000 == 0:
    #    connection.close()


maintenance_task = LoopingCall(_server_maintenance)
maintenance_task.start(60, now=True)  # call every minute


#------------------------------------------------------------
# Evennia Main Server object
#------------------------------------------------------------
class Evennia(object):
    """
    The main Evennia server handler. This object sets up the database and
    tracks and interlinks all the twisted network services that make up
    evennia.
    """
    def __init__(self, application):
        """
        Setup the server.
Ejemplo n.º 40
0
class PahoMqttAdapter(BaseMqttAdapter, Service):

    # If connection fails at first, retry connecting each X seconds
    retry_interval = 5

    def connect(self):
        """
        Connect to MQTT broker.
        """
        # TODO: Check if we can do asynchronous connection establishment.
        #       Currently, this is done synchronously which could harm
        #       other subsystems in timeout or otherwise blocking situations.

        # Make MQTT client identifier even more unique by adding process id
        pid = os.getpid()
        client_id = '{}:{}'.format(self.name, str(pid))

        # Connection establishment
        self.client = mqtt.Client(client_id=client_id, clean_session=True)

        # Optionally authenticate connection
        if self.broker_username:
            self.client.username_pw_set(self.broker_username,
                                        self.broker_password)

        # Set event handlers
        self.client.on_connect = lambda *args: reactor.callFromThread(
            self.on_connect, *args)
        self.client.on_message = lambda *args: reactor.callFromThread(
            self.on_message, *args)
        self.client.on_log = lambda *args: reactor.callFromThread(
            self.on_log, *args)

        # Connect with retry
        self.connect_loop = LoopingCall(self.connect_with_retry)
        self.connect_loop.start(self.retry_interval, now=True)

    def connect_with_retry(self):
        try:
            self.client.connect(self.broker_host,
                                port=self.broker_port,
                                keepalive=60)
            self.connect_loop.stop()
        except:
            log.failure(
                u'Error connecting to MQTT broker but retrying each {retry_interval} seconds',
                retry_interval=self.retry_interval)
            return
        """
        This is part of the threaded client interface. Call this once to
        start a new thread to process network traffic. This provides an
        alternative to repeatedly calling loop() yourself.
        """
        self.client.loop_start()
        reactor.addSystemEventTrigger('before', 'shutdown',
                                      self.client.loop_stop, True)

    # The callback for when the client receives a CONNACK response from the server.
    def on_connect(self, client, userdata, flags, rc):
        """
        on_connect(client, userdata, flags, rc): called when the broker responds to our connection
          request.
          flags is a dict that contains response flags from the broker:
            flags['session present'] - this flag is useful for clients that are
                using clean session set to 0 only. If a client with clean
                session=0, that reconnects to a broker that it has previously
                connected to, this flag indicates whether the broker still has the
                session information for the client. If 1, the session still exists.
          The value of rc determines success or not:
            0: Connection successful
            1: Connection refused - incorrect protocol version
            2: Connection refused - invalid client identifier
            3: Connection refused - server unavailable
            4: Connection refused - bad username or password
            5: Connection refused - not authorised
            6-255: Currently unused.
        """
        log.debug(
            "Connected to MQTT. userdata={userdata}, flags={flags}, rc={rc}",
            userdata=userdata,
            flags=flags,
            rc=rc)

        # Subscribing in on_connect() means that if we lose the connection and
        # reconnect then subscriptions will be renewed.
        #client.subscribe("$SYS/#")
        self.subscribe()

    # The callback for when a PUBLISH message is received from the server.
    def on_message(self, client, userdata, message):
        """
        on_message(client, userdata, message): called when a message has been received on a
          topic that the client subscribes to. The message variable is a
          MQTTMessage that describes all of the message parameters.
        """

        # TODO: Do something with "client" object (paho.mqtt.client.Client)

        topic = message.topic
        payload = message.payload

        # Make metadata dictionary to be passed as kwargs later

        # Mungle topic and payload out of metadata
        metadata = message.__dict__.copy()
        del metadata['topic']
        del metadata['payload']

        # Mungle userdata into message
        metadata['userdata'] = userdata

        if not topic.endswith('error.json'):
            log.debug(
                'on_message: name={name}, topic={topic}, payload={payload}, kwargs={kwargs}',
                name=self.name,
                topic=topic,
                payload=payload,
                kwargs=metadata)

        return self.callback(topic=topic, payload=payload, **metadata)

    def publish(self, topic, payload):
        log.debug(u'Publishing to topic={topic}, payload={payload}',
                  topic=topic,
                  payload=payload)
        return self.client.publish(topic, payload)

    def subscribe(self, *args):
        #d = self.protocol.subscribe("foo/bar/baz", 0)
        log.info(u"Subscribing to topics {subscriptions}. client={client}",
                 subscriptions=self.subscriptions,
                 client=self.client)
        for topic in self.subscriptions:
            log.info(u"Subscribing to topic '{topic}'", topic=topic)
            # Topic name **must not** be unicode, so casting to string
            e = self.client.subscribe(str(topic), qos=0)

    def on_log(self, client, userdata, level, buf):
        """
        on_log(client, userdata, level, buf): called when the client has log information. Define
          to allow debugging. The level variable gives the severity of the message
          and will be one of MQTT_LOG_INFO, MQTT_LOG_NOTICE, MQTT_LOG_WARNING,
          MQTT_LOG_ERR, and MQTT_LOG_DEBUG. The message itself is in buf.
        """
        log.debug(u'{message}. level={level_mqtt}, userdata={userdata}',
                  message=buf,
                  level_mqtt=level,
                  userdata=userdata)
Ejemplo n.º 41
0
 def __init__(self):
     self.rulesets = defaultdict(list)
     self.rules_file = None
     self.read_task = LoopingCall(self.read_rules)
     self.rules_last_read = 0.0
Ejemplo n.º 42
0
    def __init__(self, interface: bytes, config_dict: Dict[str, Any]) -> None:
        # logfile path relative to config dir if not abs path
        log_filename = logfile.get()
        if log_filename.strip():  # catches empty filename
            if not os.path.isabs(log_filename):
                log_filename = os.path.join(config.config_dir, log_filename)
            ensure_dir_exists(log_filename)
            if logging_rotate_daily.get():
                logging_file = DailyLogFile(log_filename, '.')
            else:
                logging_file = open(log_filename, 'a')
            predicate = LogLevelFilterPredicate(
                LogLevel.levelWithName(loglevel.get()))
            observers = [
                FilteringLogObserver(textFileLogObserver(sys.stderr),
                                     [predicate]),
                FilteringLogObserver(textFileLogObserver(logging_file),
                                     [predicate])
            ]
            globalLogBeginner.beginLoggingTo(observers)
            log.info('piqueserver started on %s' % time.strftime('%c'))

        self.config = config_dict
        if random_rotation.get():
            self.map_rotator_type = random_choice_cycle
        else:
            self.map_rotator_type = itertools.cycle
        self.default_time_limit = default_time_limit.get()
        self.default_cap_limit = cap_limit.get()
        self.advance_on_win = int(advance_on_win.get())
        self.win_count = itertools.count(1)
        self.bans = NetworkDict()

        # attempt to load a saved bans list
        try:
            with open(os.path.join(config.config_dir, bans_file.get()),
                      'r') as f:
                self.bans.read_list(json.load(f))
            log.debug("loaded {count} bans", count=len(self.bans))
        except FileNotFoundError:
            log.debug("skip loading bans: file unavailable",
                      count=len(self.bans))
        except IOError as e:
            log.error('Could not read bans file ({}): {}'.format(
                bans_file.get(), e))
        except ValueError as e:
            log.error('Could not parse bans file ({}): {}'.format(
                bans_file.get(), e))

        self.hard_bans = set()  # possible DDoS'ers are added here
        self.player_memory = deque(maxlen=100)
        if len(self.name) > MAX_SERVER_NAME_SIZE:
            log.warn('(server name too long; it will be truncated to "%s")' %
                     (self.name[:MAX_SERVER_NAME_SIZE]))
        self.respawn_time = respawn_time_option.get()
        self.respawn_waves = respawn_waves.get()

        # since AoS only supports CTF and TC at a protocol level, we need to get
        # the base game mode if we are using a custom game mode.
        game_mode_name = game_mode.get()
        if game_mode_name == 'ctf':
            self.game_mode = CTF_MODE
        elif game_mode.get() == 'tc':
            self.game_mode = TC_MODE
        elif self.game_mode not in [CTF_MODE, TC_MODE]:
            raise ValueError(
                'invalid game mode: custom game mode "{}" does not set '
                'protocol.game_mode to one of TC_MODE or CTF_MODE. Are '
                'you sure the thing you have specified is a game mode?'.format(
                    game_mode_name))

        self.game_mode_name = game_mode.get().split('.')[-1]
        self.team1_name = team1_name.get()[:9]
        self.team2_name = team2_name.get()[:9]
        self.team1_color = tuple(team1_color.get())
        self.team2_color = tuple(team2_color.get())
        self.friendly_fire = friendly_fire.get()
        self.friendly_fire_on_grief = friendly_fire_on_grief.get()
        self.friendly_fire_time = grief_friendly_fire_time.get()
        self.spade_teamkills_on_grief = spade_teamkills_on_grief.get()
        self.fall_damage = fall_damage.get()
        self.teamswitch_interval = teamswitch_interval.get()
        self.teamswitch_allowed = teamswitch_allowed.get()
        self.max_players = max_players.get()
        self.melee_damage = melee_damage.get()
        self.max_connections_per_ip = max_connections_per_ip.get()
        self.passwords = passwords.get()
        self.server_prefix = server_prefix.get()
        self.time_announcements = time_announcements.get()
        self.balanced_teams = balanced_teams.get()
        self.login_retries = login_retries.get()

        # voting configuration
        self.default_ban_time = default_ban_duration.get()

        self.speedhack_detect = speedhack_detect.get()
        self.rubberband_distance = rubberband_distance.get()
        if user_blocks_only.get():
            self.user_blocks = set()
        self.set_god_build = set_god_build.get()
        if ssh_enabled.get():
            from piqueserver.ssh import RemoteConsole
            self.remote_console = RemoteConsole(self)
        irc = irc_options.get()
        if irc.get('enabled', False):
            from piqueserver.irc import IRCRelay
            self.irc_relay = IRCRelay(self, irc)
        if status_server_enabled.get():
            from piqueserver.statusserver import StatusServer
            self.status_server = StatusServer(self)
            ensureDeferred(self.status_server.listen())
        if ban_publish.get():
            from piqueserver.banpublish import PublishServer
            self.ban_publish = PublishServer(self, ban_publish_port.get())
        if bans_config_urls.get():
            from piqueserver import bansubscribe
            self.ban_manager = bansubscribe.BanManager(self)
            ensureDeferred(as_deferred(self.ban_manager.start()))
        self.start_time = time.time()
        self.end_calls = []
        # TODO: why is this here?
        create_console(self)

        for user_type, func_names in rights.get().items():
            for func_name in func_names:
                commands.add_rights(user_type, func_name)

        if everyone_is_admin.get():
            self.everyone_is_admin = True

        self.port = port_option.get()
        ServerProtocol.__init__(self, self.port, interface)
        self.host.intercept = self.receive_callback

        try:
            self.set_map_rotation(self.config['rotation'])
        except MapNotFound as e:
            log.critical('Invalid map in map rotation (%s), exiting.' % e.map)
            raise SystemExit

        map_load_d = self.advance_rotation()
        # discard the result of the map advance for now
        map_load_d.addCallback(lambda x: self._post_init())

        ip_getter = ip_getter_option.get()
        if ip_getter:
            ensureDeferred(as_deferred(self.get_external_ip(ip_getter)))

        self.new_release = None
        notify_new_releases = config.option("release_notifications",
                                            default=True)
        if notify_new_releases.get():
            ensureDeferred(as_deferred(self.watch_for_releases()))

        self.vacuum_loop = LoopingCall(self.vacuum_bans)
        # Run the vacuum every 6 hours, and kick it off it right now
        self.vacuum_loop.start(60 * 60 * 6, True)

        reactor.addSystemEventTrigger('before', 'shutdown',
                                      lambda: ensureDeferred(self.shutdown()))
Ejemplo n.º 43
0
 class RollbackProtocol(protocol):
     rollback_in_progress = False
     rollback_max_rows = 10 # per 'cycle', intended to cap cpu usage
     rollback_max_packets = 180 # per 'cycle' cap for (unique packets * players)
     rollback_max_unique_packets = 12 # per 'cycle', each block op is at least 1
     rollback_time_between_cycles = 0.06
     rollback_time_between_progress_updates = 10.0
     rollback_start_time = None
     rollback_last_chat = None
     rollback_rows = None
     rollback_total_rows = None
     
     # rollback
     
     def start_rollback(self, connection, mapname, start_x, start_y,
         end_x, end_y, ignore_indestructable = True):
         if self.rollback_in_progress:
             return S_ROLLBACK_IN_PROGRESS
         if mapname is None:
             map = self.rollback_map
         else:
             try:
                 maps = check_rotation([mapname])
                 if not maps:
                     return S_INVALID_MAP_NAME
                 map = Map(maps[0]).data
             except MapNotFound as error:
                 return error.message
         name = (connection.name if connection is not None
             else S_AUTOMATIC_ROLLBACK_PLAYER_NAME)
         message = S_ROLLBACK_COMMENCED.format(player = name)
         self.send_chat(message, irc = True)
         self.packet_generator = self.create_rollback_generator(self.map,
             map, start_x, start_y, end_x, end_y, ignore_indestructable)
         self.rollback_in_progress = True
         self.rollback_start_time = time.time()
         self.rollback_last_chat = self.rollback_start_time
         self.rollback_rows = 0
         self.rollback_total_rows = end_x - start_x
         self.cycle_call = LoopingCall(self.rollback_cycle)
         self.cycle_call.start(self.rollback_time_between_cycles)
     
     def cancel_rollback(self, connection):
         if not self.rollback_in_progress:
             return S_NO_ROLLBACK_IN_PROGRESS
         result = S_ROLLBACK_CANCELLED.format(player = connection.name)
         self.end_rollback(result)
     
     def end_rollback(self, result):
         self.rollback_in_progress = False
         self.cycle_call.stop()
         self.cycle_call = None
         self.packet_generator = None
         self.update_entities()
         message = S_ROLLBACK_ENDED.format(result = result)
         self.send_chat(message, irc = True)
     
     def rollback_cycle(self):
         if not self.rollback_in_progress:
             return
         try:
             sent_unique = sent_total = rows = 0
             while 1:
                 if rows > self.rollback_max_rows:
                     break
                 if sent_unique > self.rollback_max_unique_packets:
                     break
                 if sent_total > self.rollback_max_packets:
                     break
                 sent = self.packet_generator.next()
                 sent_unique += sent
                 sent_total += sent * len(self.connections)
                 rows += (sent == 0)
             self.rollback_rows += rows
             if (time.time() - self.rollback_last_chat >
                 self.rollback_time_between_progress_updates):
                 self.rollback_last_chat = time.time()
                 progress = float(self.rollback_rows) / self.rollback_total_rows
                 if progress < 1.0:
                     message = S_ROLLBACK_PROGRESS.format(percent = progress)
                     self.send_chat(message)
                 else:
                     self.send_chat(S_ROLLBACK_COLOR_PASS)
         except (StopIteration):
             elapsed = time.time() - self.rollback_start_time
             message = S_ROLLBACK_TIME_TAKEN.format(seconds = elapsed)
             self.end_rollback(message)
     
     def create_rollback_generator(self, cur, new, start_x, start_y,
         end_x, end_y, ignore_indestructable):
         surface = {}
         block_action = BlockAction()
         block_action.player_id = 31
         set_color = SetColor()
         set_color.value = make_color(*NON_SURFACE_COLOR)
         set_color.player_id = 31
         self.send_contained(set_color, save = True)
         old = cur.copy()
         check_protected = hasattr(protocol, 'protected')
         for x in xrange(start_x, end_x):
             block_action.x = x
             for y in xrange(start_y, end_y):
                 block_action.y = y
                 if check_protected and self.is_protected(x, y, 0):
                     continue
                 for z in xrange(63):
                     action = None
                     cur_solid = cur.get_solid(x, y, z)
                     new_solid = new.get_solid(x, y, z)
                     if cur_solid and not new_solid:
                         if (not ignore_indestructable and
                             self.is_indestructable(x, y, z)):
                             continue
                         else:
                             action = DESTROY_BLOCK
                             cur.remove_point(x, y, z)
                     elif new_solid:
                         new_is_surface = new.is_surface(x, y, z)
                         if new_is_surface:
                             new_color = new.get_color(x, y, z)
                         if not cur_solid and new_is_surface:
                             surface[(x, y, z)] = new_color
                         elif not cur_solid and not new_is_surface:
                             action = BUILD_BLOCK
                             cur.set_point(x, y, z, NON_SURFACE_COLOR)
                         elif cur_solid and new_is_surface:
                             old_is_surface = old.is_surface(x, y, z)
                             if old_is_surface:
                                 old_color = old.get_color(x, y, z)
                             if not old_is_surface or old_color != new_color:
                                 surface[(x, y, z)] = new_color
                                 action = DESTROY_BLOCK
                                 cur.remove_point(x, y, z)
                     if action is not None:
                         block_action.z = z
                         block_action.value = action
                         self.send_contained(block_action, save = True)
                         yield 1
             yield 0
         last_color = None
         block_action.value = BUILD_BLOCK
         for pos, color in sorted(surface.iteritems(),
             key = operator.itemgetter(1)):
             x, y, z = pos
             packets_sent = 0
             if color != last_color:
                 set_color.value = make_color(*color)
                 self.send_contained(set_color, save = True)
                 packets_sent += 1
                 last_color = color
             cur.set_point(x, y, z, color)
             block_action.x = x
             block_action.y = y
             block_action.z = z
             self.send_contained(block_action, save = True)
             packets_sent += 1
             yield packets_sent
     
     def on_map_change(self, map):
         self.rollback_map = map.copy()
         protocol.on_map_change(self, map)
     
     def on_map_leave(self):
         if self.rollback_in_progress:
             self.end_rollback(S_MAP_CHANGED)
         protocol.on_map_leave(self)
     
     def on_game_end(self):
         if rollback_on_game_end:
             self.start_rollback(None, None, 0, 0, 512, 512, False)
         protocol.on_game_end(self)
Ejemplo n.º 44
0
class WebInterface(YomboLibrary):
    """
    Web interface framework.
    """
    webapp = Klein()  # Like Flask, but for twisted

    visits = 0
    alerts = OrderedDict()
    starting = True
    already_starting_web_servers = False
    hook_listeners = {}  # special way to toss hook calls to routes.

    def __str__(self):
        """
        Returns the name of the library.
        :return: Name of the library
        :rtype: string
        """
        return "Yombo web interface library"

    def _init_(self, **kwargs):
        self.enabled = self._Configs.get('webinterface', 'enabled', True)
        if not self.enabled:
            return

        self.gateway_id = self._Configs.get2('core', 'gwid', 'local', False)
        # self._LocalDB = self._Loader.loadedLibraries['localdb']
        self._current_dir = self._Atoms.get('yombo.path') + "/yombo"
        self._dir = '/lib/webinterface/'
        self._build_dist()  # Make all the JS and CSS files
        self.secret_pin_totp = self._Configs.get2(
            'webinterface', 'auth_pin_totp',
            yombo.utils.random_string(
                length=16, letters='ABCDEFGHIJKLMNOPQRSTUVWXYZ234567'))
        self._VoiceCmds = self._Loader.loadedLibraries['voicecmds']
        self.misc_wi_data = {}

        self.wi_port_nonsecure = self._Configs.get2('webinterface',
                                                    'nonsecure_port', 8080)
        self.wi_port_secure = self._Configs.get2('webinterface', 'secure_port',
                                                 8443)

        self.webapp.templates = jinja2.Environment(
            loader=jinja2.FileSystemLoader(self._current_dir))
        self.setup_basic_filters()

        self.web_interface_listener = None
        self.web_interface_ssl_listener = None

        self.api_stream_spectators = {}

        # Load API routes
        route_api_v1_automation(self.webapp)
        route_api_v1_command(self.webapp)
        route_api_v1_device(self.webapp)
        route_api_v1_device_command(self.webapp)
        route_api_v1_gateway(self.webapp)
        route_api_v1_module(self.webapp)
        route_api_v1_notification(self.webapp)
        route_api_v1_server(self.webapp)
        route_api_v1_statistics(self.webapp)
        route_api_v1_stream(self.webapp, self)
        route_api_v1_system(self.webapp)

        # Load devtool routes
        route_devtools_config(self.webapp)
        route_devtools_config_commands(self.webapp)
        route_devtools_config_device_types(self.webapp)
        route_devtools_config_input_types(self.webapp)
        route_devtools_config_modules(self.webapp)
        route_devtools_config_variables(self.webapp)
        route_devtools_debug(self.webapp)

        # Load web server routes
        route_apiauth(self.webapp)
        route_atoms(self.webapp)
        route_automation(self.webapp)
        route_configs(self.webapp)
        route_devices(self.webapp)
        route_locations(self.webapp)
        route_devtools_config(self.webapp)
        route_gateways(self.webapp)
        route_home(self.webapp)
        route_misc(self.webapp)
        route_modules(self.webapp)
        route_notices(self.webapp)
        route_panel(self.webapp)
        route_setup_wizard(self.webapp)
        route_statistics(self.webapp)
        route_states(self.webapp)
        route_system(self.webapp)
        route_voicecmds(self.webapp)

        self.temp_data = ExpiringDict(max_age_seconds=1800)
        self.web_server_started = False
        self.web_server_ssl_started = False

        self.web_factory = None

        # just here to set a password if it doesn't exist.
        mqtt_password = self._Configs.get('mqtt_users', 'panel.webinterface',
                                          yombo.utils.random_string())

    # def _start_(self):
    #     self.webapp.templates.globals['_'] = _  # i18n

    @property
    def operating_mode(self):
        return self._Loader.operating_mode

    def _load_(self, **kwargs):
        if not self.enabled:
            return

        self.module_config_links = {}

        self.auth_pin = self._Configs.get2(
            'webinterface', 'auth_pin',
            yombo.utils.random_string(
                length=4, letters=yombo.utils.human_alpabet()).lower())
        self.auth_pin_totp = self._Configs.get2(
            'webinterface', 'auth_pin_totp',
            yombo.utils.random_string(length=16))
        self.auth_pin_type = self._Configs.get2('webinterface',
                                                'auth_pin_type', 'pin')
        self.auth_pin_required = self._Configs.get2('webinterface',
                                                    'auth_pin_required', True)

        # self.web_factory = Yombo_Site(self.webapp.resource(), None, logPath='/dev/null')
        self.web_factory = Yombo_Site(self.webapp.resource(),
                                      None,
                                      logPath=None)
        self.web_factory.setup_log_queue(self)
        self.web_factory.noisy = False  # turn off Starting/stopping message
        #        self.web_factory.sessionFactory = YomboSession
        self.displayTracebacks = False

        self._display_pin_console_at = 0

        self.misc_wi_data[
            'gateway_configured'] = self._home_gateway_configured()
        self.misc_wi_data['gateway_label'] = self._Configs.get2(
            'core', 'label', 'Yombo Gateway', False)
        self.misc_wi_data['operating_mode'] = self.operating_mode
        self.misc_wi_data['notifications'] = self._Notifications
        self.misc_wi_data[
            'notification_priority_map_css'] = NOTIFICATION_PRIORITY_MAP_CSS
        self.misc_wi_data['breadcrumb'] = []

        # self.functions = {
        #     'yes_no': yombo.utils.is_yes_no,
        # }
        self.webapp.templates.globals[
            'local_gateway'] = self._Gateways.get_local()
        self.webapp.templates.globals['commands'] = self._Commands
        self.webapp.templates.globals['devices'] = self._Devices
        self.webapp.templates.globals['gateways'] = self._Gateways
        self.webapp.templates.globals['misc_wi_data'] = self.misc_wi_data
        self.webapp.templates.globals['webinterface'] = self
        # self.webapp.templates.globals['func'] = self.functions

        self.starting = False
        self.start_web_servers()

    @inlineCallbacks
    def _start_(self, **kwargs):
        self._Notifications.add({
            'title': 'System still starting',
            'message': 'Still starting up. Please wait.',
            'source': 'Web Interface Library',
            'persist': True,
            'priority': 'high',
            'always_show': True,
            'always_show_allow_clear': False,
            'id': 'webinterface:starting',
        })
        added_notification = True
        self._get_nav_side_items()
        module_configs = yield yombo.utils.global_invoke_all(
            '_webinterface_module_config_',
            called_by=self,
        )
        for component_label, link in module_configs.items():
            if link is None:
                continue
            component = self._Loader.get_loaded_component(component_label)
            self.module_config_links[component._module_id] = link
        print("Module_configs: %s" % self.module_config_links)

    def _started_(self, **kwargs):
        # if self.operating_mode != 'run':
        self._display_pin_console_at = int(time())
        self.display_pin_console()
        self._Notifications.delete('webinterface:starting')
        self.send_hook_listeners_ping_loop = LoopingCall(
            self.send_hook_listeners_ping_loop)
        self.send_hook_listeners_ping_loop.start(55, True)

    def send_hook_listeners_ping_loop(self):
        route_api_v1_stream_broadcast(self, 'ping', int(time()))

    def register_hook(self, name, thecallback):
        if name not in self.hook_listeners:
            self.hook_listeners[name] = []
        self.hook_listeners[name].append(thecallback)

    @inlineCallbacks
    def _yombo_universal_hook_(self, hook_name=None, **kwargs):
        """
        Implements the universal hook.

        :param kwargs:
        :return:
        """
        # print("web uni hook: %s" % hook_name)
        if hook_name in self.hook_listeners:
            for a_callback in self.hook_listeners[hook_name]:
                # print("web uni hook a_callback: %s" % a_callback)
                d = Deferred()
                d.addCallback(lambda ignored: maybeDeferred(
                    a_callback, self, hook_name=hook_name, **kwargs))
                d.addErrback(self.yombo_universal_hook_failure, hook_name,
                             a_callback)
                d.callback(1)
                yield d

    def yombo_universal_hook_failure(self, failure, hook_name, acallback):
        logger.warn(
            "---==(failure WI:universal hook for hook ({hook_name})==----",
            hook_name=hook_name)
        logger.warn("--------------------------------------------------------")
        logger.warn("{acallback}", acallback=acallback)
        logger.warn("{failure}", failure=failure)
        logger.warn("--------------------------------------------------------")
        raise RuntimeError("failure during module invoke for hook: %s" %
                           failure)

    def check_have_required_nodes(self):
        try:
            node = yield self._Nodes.get('main_page', 'webinterface_page')
        except KeyError as e:
            pass
            # add base node...

    @inlineCallbacks
    def change_ports(self, port_nonsecure=None, port_secure=None):
        if port_nonsecure is None and port_secure is None:
            logger.info("Asked to change ports, but nothing has changed.")
            return

        if port_nonsecure is not None:
            if port_nonsecure != self.wi_port_nonsecure():
                self.wi_port_nonsecure(set=port_nonsecure)
                logger.info(
                    "Changing port for the non-secure web interface: {port}",
                    port=port_nonsecure)
                if self.web_server_started:
                    yield self.web_interface_listener.stopListening()
                    self.web_server_started = False

        if port_secure is not None:
            if port_secure != self.wi_port_secure():
                self.wi_port_secure(set=port_secure)
                logger.info(
                    "Changing port for the secure web interface: {port}",
                    port=port_secure)
                if self.web_server_ssl_started:
                    yield self.web_interface_ssl_listener.stopListening()
                    self.web_server_ssl_started = False

        self.start_web_servers()

    # @inlineCallbacks
    def start_web_servers(self):
        if self.already_starting_web_servers is True:
            return
        self.already_starting_web_servers = True
        logger.debug("starting web servers")
        if self.web_server_started is False:
            if self.wi_port_nonsecure() == 0:
                logger.warn(
                    "Non secure port has been disabled. With gateway stopped, edit yomobo.ini and change: webinterface->nonsecure_port"
                )
            else:
                self.web_server_started = True
                port_attempts = 0
                while port_attempts < 100:
                    try:
                        self.web_interface_listener = reactor.listenTCP(
                            self.wi_port_nonsecure() + port_attempts,
                            self.web_factory)
                        break
                    except Exception as e:
                        port_attempts += 1
                if port_attempts >= 100:
                    logger.warn(
                        "Unable to start web server, no available port could be found. Tried: {starting} - {ending}",
                        starting=self.wi_port_secure(),
                        ending=self.wi_port_secure() + port_attempts)
                elif port_attempts > 0:
                    self._Configs.set('webinterface', 'nonsecure_port',
                                      self.wi_port_nonsecure() + port_attempts)
                    logger.warn("Web interface is on a new port: {new_port}",
                                new_port=self.wi_port_nonsecure() +
                                port_attempts)

        if self.web_server_ssl_started is False:
            if self.wi_port_secure() == 0:
                logger.warn(
                    "Secure port has been disabled. With gateway stopped, edit yomobo.ini and change: webinterface->secure_port"
                )
            else:
                self.web_server_ssl_started = True
                cert = self._SSLCerts.get('lib_webinterface')
                privkeypyssl = crypto.load_privatekey(crypto.FILETYPE_PEM,
                                                      cert['key'])
                certpyssl = crypto.load_certificate(crypto.FILETYPE_PEM,
                                                    cert['cert'])
                if cert['chain'] is not None:
                    chainpyssl = [
                        crypto.load_certificate(crypto.FILETYPE_PEM,
                                                cert['chain'])
                    ]
                else:
                    chainpyssl = None
                # print("web ssl cert: key_crypt %s" % cert['key_crypt'])
                # print("web ssl cert: key_crypt type %s" % type(cert['key_crypt']))
                # # print("web ssl cert: key_crypt type %s" % type(cert['key_crypt'][0]))
                # # print("web ssl cert: key_crypt type %s" % type(cert['key_crypt'][1]))
                # print("web ssl cert: privkeypyssl %s" % privkeypyssl)
                # print("web ssl cert: privkeypyssl type: %s" % type(privkeypyssl))
                # print("web ssl cert: privkeypyssl.serial: %s" % privkeypyssl.__dict__)
                #
                # print("web ssl cert: cert_crypt %s" % cert['cert_crypt'])
                # print("web ssl cert: cert_crypt.type %s" % type(cert['cert_crypt']))
                # print("web ssl cert: certpyssl %s" % certpyssl)
                # print("web ssl cert: certpyssl.type %s" % type(certpyssl))
                # print("web ssl cert: chain_crypt %s" % cert['chain_crypt'])
                # print("web ssl cert: chain_crypt.type %s" % type(cert['chain_crypt']))
                # print("web ssl cert: chainpyssl %s" % certpyssl)
                # print("web ssl cert: chainpyssl.type %s" % type(certpyssl))
                #
                #     # chainpyssl = [crypto.load_certificate(crypto.FILETYPE_PEM, cert['chain'])]
                # # chainpyssl = None
                contextFactory = ssl.CertificateOptions(
                    privateKey=cert['key_crypt'],
                    certificate=cert['cert_crypt'],
                    extraCertChain=cert['chain_crypt'])
                port_attempts = 0
                while port_attempts < 100:
                    try:
                        # print("about to start ssl listener on port: %s" % self.wi_port_secure())
                        self.web_interface_ssl_listener = reactor.listenSSL(
                            self.wi_port_secure(), self.web_factory,
                            contextFactory)
                        break
                    except Exception as e:
                        port_attempts += 1
                if port_attempts >= 100:
                    logger.warn(
                        "Unable to start secure web server, no available port could be found. Tried: {starting} - {ending}",
                        starting=self.wi_port_secure(),
                        ending=self.wi_port_secure() + port_attempts)
                elif port_attempts > 0:
                    self._Configs.set('webinterface', 'secure_port',
                                      self.wi_port_secure() + port_attempts)
                    logger.warn(
                        "Secure (tls/ssl) web interface is on a new port: {new_port}",
                        new_port=self.wi_port_secure() + port_attempts)

        logger.debug("done starting web servers")
        self.already_starting_web_servers = False

    def _configuration_set_(self, **kwargs):
        """
        Receive configuruation updates and adjust as needed.

        :param kwargs: section, option(key), value
        :return:
        """
        section = kwargs['section']
        option = kwargs['option']
        value = kwargs['value']

        # if section == 'core':
        #     if option == 'label':
        #         self.misc_wi_data['gateway_label'] = value

        if self.starting is True:
            return

        if section == 'webinterface':
            if option == 'nonsecure_port':
                self.change_ports(port_nonsecure=value)
            elif option == 'secure_port':
                self.change_ports(port_secure=value)

    def _sslcerts_(self, **kwargs):
        """
        Called to collect to ssl cert requirements.

        :param kwargs:
        :return:
        """
        fqdn = self._Configs.get('dns', 'fqdn', None, False)
        if fqdn is None:
            logger.warn(
                "Unable to create webinterface SSL cert: DNS not set properly."
            )
            return
        cert = {}
        cert['sslname'] = "lib_webinterface"
        cert['sans'] = [
            'localhost', 'l', 'local', 'i', 'e', 'internal', 'external',
            str(int(time()))
        ]
        cert['cn'] = cert['sans'][0]
        cert['callback'] = self.new_ssl_cert
        return cert

    @inlineCallbacks
    def new_ssl_cert(self, newcert, **kwargs):
        """
        Called when a requested certificate has been signed or updated. If needed, this function
        will function will restart the SSL service if the current certificate has expired or is
        a self-signed cert.

        :param kwargs:
        :return:
        """
        logger.info("Got a new cert! About to install it.")
        if self.web_server_ssl_started is not None:
            yield self.web_interface_ssl_listener.stopListening()
            self.web_server_ssl_started = False
        self.start_web_servers()

    @inlineCallbacks
    def _unload_(self, **kwargs):
        if hasattr(self, 'web_factory'):
            if self.web_factory is not None:
                yield self.web_factory.save_log_queue()

    # def WebInterface_configuration_details(self, **kwargs):
    #     return [{'webinterface': {
    #                 'enabled': {
    #                     'description': {
    #                         'en': 'Enables/disables the web interface.',
    #                     }
    #                 },
    #                 'port': {
    #                     'description': {
    #                         'en': 'Port number for the web interface to listen on.'
    #                     }
    #                 }
    #             },
    #     }]

    @webapp.route('/<path:catchall>')
    @require_auth()
    def page_404(self, request, session, catchall):
        request.setResponseCode(404)
        page = self.get_template(request, self._dir + 'pages/404.html')
        return page.render()

    @webapp.handle_errors(NotFound)
    @require_auth()
    def notfound(self, request, failure):
        request.setResponseCode(404)
        return 'Not found, I say'

    def display_pin_console(self):
        print("###########################################################")
        print("#                                                         #")
        if self.operating_mode != 'run':
            print(
                "# The Yombo Gateway website is running in                 #")
            print(
                "# configuration only mode.                                #")
            print(
                "#                                                         #")

        dns_fqdn = self._Configs.get('dns', 'fqdn', None, False)
        if dns_fqdn is None:
            local_hostname = "127.0.0.1"
            internal_hostname = self._Configs.get('core', 'localipaddress_v4')
            external_hostname = self._Configs.get('core',
                                                  'externalipaddress_v4')
            local = "http://%s:%s" % (local_hostname, self.wi_port_nonsecure())
            internal = "http://%s:%s" % (internal_hostname,
                                         self.wi_port_nonsecure())
            external = "https://%s:%s" % (external_hostname,
                                          self.wi_port_secure())
            print(
                "# The gateway can be accessed from the following urls:    #")
            print(
                "#                                                         #")
            print(
                "# On local machine:                                       #")
            print("#  %-54s #" % local)
            print(
                "#                                                         #")
            print(
                "# On local network:                                       #")
            print("#  %-54s #" % internal)
            print(
                "#                                                         #")
            print(
                "# From external network (check port forwarding):          #")
            print("#  %-54s #" % external)
        else:
            website_url = "http://%s" % dns_fqdn
            print(
                "# The gateway can be accessed from the following url:     #")
            print(
                "#                                                         #")
            print(
                "# From anywhere:                                          #")
            print("#  %-54s #" % website_url)

        print("#                                                         #")
        print("#                                                         #")
        print("# Web Interface access pin code:                          #")
        print("#  %-25s                              #" % self.auth_pin())
        print("#                                                         #")
        print("###########################################################")

    def i18n(self, request):
        """
        Gets a translator based on the language the browser provides us.

        :param request: The browser request.
        :return:
        """
        return web_translator(self, request)

    @inlineCallbacks
    def _get_nav_side_items(self, **kwargs):
        """
        Called before modules have their _prestart_ function called (after _load_).

        This implements the hook "webinterface_add_routes" and calls all libraries and modules. It allows libs and
        modules to add menus to the web interface and provide additional funcationality.

        **Usage**:

        .. code-block:: python

           def ModuleName_webinterface_add_routes(self, **kwargs):
               return {
                   'nav_side': [
                       {
                       'label1': 'Tools',
                       'label2': 'MQTT',
                       'priority1': 3000,
                       'priority2': 10000,
                       'icon': 'fa fa-wrench fa-fw',
                       'url': '/tools/mqtt',
                       'tooltip': '',
                       'opmode': 'run',
                       },
                   ],
                   'routes': [
                       self.web_interface_routes,
                  ],
               }

        """
        # first, lets get the top levels already defined so children don't re-arrange ours.
        top_levels = {}
        temp_list = sorted(NAV_SIDE_MENU,
                           key=itemgetter('priority1', 'priority2'))
        for item in temp_list:
            label1 = item['label1']
            if label1 not in temp_list:
                top_levels[label1] = item['priority1']

        nav_side_menu = NAV_SIDE_MENU.copy()

        add_on_menus = yield yombo.utils.global_invoke_all(
            '_webinterface_add_routes_',
            called_by=self,
        )
        for component, options in add_on_menus.items():
            if 'nav_side' in options:
                for new_nav in options['nav_side']:
                    if isinstance(new_nav['priority1'], int) is False:
                        new_nav['priority1'] = top_levels[new_nav['label1']]
                    nav_side_menu.append(new_nav)
            if 'menu_priorities' in options:  # allow modules to change the ording of top level menus
                for label, priority in options['menu_priorities'].items():
                    top_levels[label] = priority
            if 'routes' in options:
                for new_route in options['routes']:
                    new_route(self.webapp)

        # build menu tree
        self.misc_wi_data['nav_side'] = OrderedDict()

        temp_list = sorted(nav_side_menu,
                           key=itemgetter('priority1', 'priority2'))
        for item in temp_list:
            label1 = item['label1']
            if label1 not in self.misc_wi_data['nav_side']:
                self.misc_wi_data['nav_side'][label1] = []
            self.misc_wi_data['nav_side'][label1].append(item)
        self.starting = False

    def add_alert(self,
                  message,
                  level='info',
                  dismissable=True,
                  type='session',
                  deletable=True):
        """
        Add an alert to the stack.
        :param level: info, warning, error
        :param message:
        :return:
        """
        rand = yombo.utils.random_string(length=12)
        self.alerts[rand] = {
            'type': type,
            'level': level,
            'message': message,
            'dismissable': dismissable,
            'deletable': deletable,
        }
        return rand

    def make_alert(self,
                   message,
                   level='info',
                   type='session',
                   dismissable=False):
        """
        Add an alert to the stack.
        :param level: info, warning, error
        :param message:
        :return:
        """
        return {
            'level': level,
            'message': message,
            'dismissable': dismissable,
        }

    def get_alerts(self, type=None, session=None):
        """
        Retrieve a list of alerts for display.
        """
        if type is None:
            type = 'session'

        show_alerts = OrderedDict()
        for keyid in list(self.alerts.keys()):
            if self.alerts[keyid]['type'] == type:
                show_alerts[keyid] = self.alerts[keyid]
                if type == 'session':
                    del self.alerts[keyid]
        return show_alerts

    def get_template(self, request, template_path):
        request.setHeader('server', 'Yombo/1.0')
        return self.webapp.templates.get_template(template_path)

    def redirect(self, request, redirect_path):
        request.setHeader('server', 'Yombo/1.0')
        request.redirect(redirect_path)

    def _tpl_home_gateway_configured(self):
        if not self._home_gateway_configured():
            return "This gateway is not properly configured. Click _here_ to run the configuration wizard."
        else:
            return ""

    def _home_gateway_configured(self):
        gwuuid = self._Configs.get("core", "gwuuid", None, False)
        gwhash = self._Configs.get("core", "gwhash", None, False)
        gpgkeyid = self._Configs.get('gpg', 'keyid', None, False)

        if gwuuid is None or gwhash is None or gpgkeyid is None:
            return False
        else:
            return True

    def _get_parms(self, request):
        return parse_qs(urlparse(request.uri).query)

    def format_markdown(self, input_text, formatting=None):
        if formatting == 'restructured' or formatting is None:
            return publish_parts(input_text, writer_name='html')['html_body']
        elif formatting == 'markdown':
            return markdown.markdown(input_text,
                                     extensions=[
                                         'markdown.extensions.nl2br',
                                         'markdown.extensions.codehilite'
                                     ])
        return input_text

    def make_link(self, link, link_text, target=None):
        if link == '' or link is None or link.lower() == "None":
            return "None"
        if target is None:
            target = "_self"
        return '<a href="%s" target="%s">%s</a>' % (link, target, link_text)

    def request_get_default(self, request, name, default, offset=None):
        if offset == None:
            offset = 0
        try:
            return request.args.get(name)[offset]
        except:
            return default

    def home_breadcrumb(self, request):
        self.add_breadcrumb(request, "/?", "Home")

    def add_breadcrumb(self,
                       request,
                       url=None,
                       text=None,
                       show=None,
                       style=None,
                       data=None):
        if hasattr(request, 'breadcrumb') is False:
            request.breadcrumb = []
            self.misc_wi_data['breadcrumb'] = request.breadcrumb

        if show is None:
            show = True

        if style is None:
            style = 'link'
        elif style == 'select_groups':
            items = {}
            for option_label, option_data in data.items():
                items[option_label] = []
                for select_text, select_url in option_data.items():
                    selected = ''
                    option_style = 'None'
                    if select_url.startswith("$"):
                        selected = 'selected'
                        select_url = select_url[1:]
                    elif select_url.startswith("#"):
                        option_style = 'divider'

                    items[option_label].append({
                        'option_style': option_style,
                        'text': select_text,
                        'url': select_url,
                        'selected': selected,
                    })
            data = items
        elif style == 'select':
            items = []
            for select_text, select_url in data.items():
                selected = ''
                option_style = 'None'
                if select_url.startswith("$"):
                    selected = 'selected'
                    select_url = select_url[1:]
                elif select_url.startswith("#"):
                    option_style = 'divider'

                items.append({
                    'option_style': option_style,
                    'text': select_text,
                    'url': select_url,
                    'selected': selected,
                })
            data = items

        hash = sha256(
            str(
                str(url) + str(text) + str(show) + str(style) +
                json.dumps(data)).encode()).hexdigest()
        breadcrumb = {
            'hash': hash,
            'url': url,
            'text': text,
            'show': show,
            'style': style,
            'data': data,
        }
        request.breadcrumb.append(breadcrumb)

    def setup_basic_filters(self):
        self.webapp.templates.filters['yes_no'] = yombo.utils.is_yes_no
        self.webapp.templates.filters['make_link'] = self.make_link
        self.webapp.templates.filters[
            'status_to_string'] = yombo.utils.status_to_string
        self.webapp.templates.filters[
            'public_to_string'] = yombo.utils.public_to_string
        self.webapp.templates.filters[
            'epoch_to_human'] = yombo.utils.epoch_to_string
        self.webapp.templates.filters[
            'epoch_to_pretty_date'] = self._Times.get_age  # yesterday, 5 minutes ago, etc.
        self.webapp.templates.filters['format_markdown'] = self.format_markdown
        self.webapp.templates.filters['hide_none'] = self.dispay_hide_none
        self.webapp.templates.filters[
            'display_encrypted'] = self._GPG.display_encrypted
        self.webapp.templates.filters[
            'display_temperature'] = self._Localize.display_temperature

    def dispay_hide_none(self, input):
        if input is None:
            return ""
        if isinstance(input, str):
            if input.lower() == "none":
                return ""
        return input

    def restart(self, request, message=None, redirect=None):
        if message is None:
            message = "Web interface requested restart."
        if redirect is None:
            redirect = "/?"

        page = self.get_template(request, self._dir + 'pages/restart.html')
        reactor.callLater(0.3, self.do_restart)
        return page.render(message=message,
                           redirect=redirect,
                           uptime=str(self._Atoms['running_since']))

    def do_restart(self):
        try:
            raise YomboRestart("Web Interface setup wizard complete.")
        except:
            pass

    def shutdown(self, request):
        page = self.get_template(request, self._dir + 'pages/shutdown.html')
        # reactor.callLater(0.3, self.do_shutdown)
        return page.render()

    def do_shutdown(self):
        raise YomboCritical("Web Interface setup wizard complete.")

    # def WebInterface_configuration_set(self, **kwargs):
    #     """
    #     Hook from configuration library. Get any configuration changes.
    #
    #     :param kwargs: 'section', 'option', and 'value' are sent here.
    #     :return:
    #     """
    #     if kwargs['section'] == 'webinterface':
    #         option = kwargs['option']
    #         if option == 'auth_pin':
    #             self.auth_pin(set=kwargs['value'])
    #         elif option == 'auth_pin_totp':
    #             self.auth_pin_totp(set=kwargs['value'])
    #         elif option == 'auth_pin_type':
    #             self.auth_pin_type(set=kwargs['value'])
    #         elif option == 'auth_pin_required':
    #             self.auth_pin_required(set=kwargs['value'])

    def _build_dist(self):
        """
        This is blocking code. Doesn't really matter, it only does it on startup.

        Builds the 'dist' directory from the 'build' directory. Easy way to update the source css/js files and update
        the webinterface JS and CSS files.
        :return:
        """
        if not path.exists('yombo/lib/webinterface/static/dist'):
            mkdir('yombo/lib/webinterface/static/dist')
        if not path.exists('yombo/lib/webinterface/static/dist/css'):
            mkdir('yombo/lib/webinterface/static/dist/css')
        if not path.exists('yombo/lib/webinterface/static/dist/js'):
            mkdir('yombo/lib/webinterface/static/dist/js')
        if not path.exists('yombo/lib/webinterface/static/dist/fonts'):
            mkdir('yombo/lib/webinterface/static/dist/fonts')

        def do_cat(inputs, output):
            output = 'yombo/lib/webinterface/static/' + output
            with open(output, 'w') as outfile:
                for fname in inputs:
                    fname = 'yombo/lib/webinterface/static/' + fname
                    with open(fname) as infile:
                        outfile.write(infile.read())

        def copytree(src, dst, symlinks=False, ignore=None):
            src = 'yombo/lib/webinterface/static/' + src
            dst = 'yombo/lib/webinterface/static/' + dst
            if path.isdir(src):
                if not path.exists(dst):
                    mkdir(dst)
            for item in listdir(src):
                s = path.join(src, item)
                d = path.join(dst, item)
                if path.isdir(s):
                    shutil.copytree(s, d, symlinks, ignore)
                else:
                    shutil.copy2(s, d)

        CAT_SCRIPTS = [
            'source/jquery/jquery-2.2.4.min.js',
            'source/sb-admin/js/js.cookie.min.js',
            'source/bootstrap/dist/js/bootstrap.min.js',
            'source/metisMenu/metisMenu.min.js',
        ]
        CAT_SCRIPTS_OUT = 'dist/js/jquery-cookie-bootstrap-metismenu.min.js'
        do_cat(CAT_SCRIPTS, CAT_SCRIPTS_OUT)

        CAT_SCRIPTS = [
            'source/jquery/jquery.validate.min.js',
        ]
        CAT_SCRIPTS_OUT = 'dist/js/jquery.validate.min.js'
        do_cat(CAT_SCRIPTS, CAT_SCRIPTS_OUT)

        CAT_SCRIPTS = [
            'source/bootstrap/dist/css/bootstrap.min.css',
            'source/metisMenu/metisMenu.min.css',
        ]
        CAT_SCRIPTS_OUT = 'dist/css/bootstrap-metisMenu.min.css'
        do_cat(CAT_SCRIPTS, CAT_SCRIPTS_OUT)

        CAT_SCRIPTS = [
            'source/bootstrap/dist/css/bootstrap.min.css',
        ]
        CAT_SCRIPTS_OUT = 'dist/css/bootstrap.min.css'
        do_cat(CAT_SCRIPTS, CAT_SCRIPTS_OUT)

        CAT_SCRIPTS = [
            'source/sb-admin/js/sb-admin-2.min.js',
            'source/sb-admin/js/yombo.js',
        ]
        CAT_SCRIPTS_OUT = 'dist/js/sb-admin2.min.js'
        do_cat(CAT_SCRIPTS, CAT_SCRIPTS_OUT)

        CAT_SCRIPTS = [
            'source/sb-admin/css/sb-admin-2.css',
            'source/sb-admin/css/yombo.css',
        ]
        CAT_SCRIPTS_OUT = 'dist/css/admin2.min.css'
        do_cat(CAT_SCRIPTS, CAT_SCRIPTS_OUT)

        CAT_SCRIPTS = [
            'source/font-awesome/css/font-awesome.min.css',
        ]
        CAT_SCRIPTS_OUT = 'dist/css/font_awesome.min.css'
        do_cat(CAT_SCRIPTS, CAT_SCRIPTS_OUT)

        CAT_SCRIPTS = [
            'source/datatables-plugins/integration/bootstrap/3/dataTables.bootstrap.css',
            'source/datatables-responsive/css/responsive.dataTables.min.css',
        ]
        CAT_SCRIPTS_OUT = 'dist/css/datatables.min.css'
        do_cat(CAT_SCRIPTS, CAT_SCRIPTS_OUT)

        CAT_SCRIPTS = [
            'source/datatables/js/jquery.dataTables.min.js',
            'source/datatables-plugins/integration/bootstrap/3/dataTables.bootstrap.min.js',
            'source/datatables-responsive/js/dataTables.responsive.min.js',
        ]
        CAT_SCRIPTS_OUT = 'dist/js/datatables.min.js'
        do_cat(CAT_SCRIPTS, CAT_SCRIPTS_OUT)

        CAT_SCRIPTS = [
            'source/jrcode/jquery-qrcode.min.js',
        ]
        CAT_SCRIPTS_OUT = 'dist/js/jquery-qrcode.min.js'
        do_cat(CAT_SCRIPTS, CAT_SCRIPTS_OUT)

        CAT_SCRIPTS = [
            'source/creative/js/jquery.easing.min.js',
            'source/creative/js/scrollreveal.min.js',
            'source/creative/js/creative.min.js',
        ]
        CAT_SCRIPTS_OUT = 'dist/js/creative.min.js'
        do_cat(CAT_SCRIPTS, CAT_SCRIPTS_OUT)

        CAT_SCRIPTS = [
            'source/creative/css/creative.css',
        ]
        CAT_SCRIPTS_OUT = 'dist/css/creative.css'
        do_cat(CAT_SCRIPTS, CAT_SCRIPTS_OUT)

        CAT_SCRIPTS = [
            'source/echarts/echarts.min.js',
        ]
        CAT_SCRIPTS_OUT = 'dist/js/echarts.min.js'
        do_cat(CAT_SCRIPTS, CAT_SCRIPTS_OUT)

        CAT_SCRIPTS = [
            'source/sb-admin/js/mappicker.js',
        ]
        CAT_SCRIPTS_OUT = 'dist/js/mappicker.js'
        do_cat(CAT_SCRIPTS, CAT_SCRIPTS_OUT)
        CAT_SCRIPTS = [
            'source/sb-admin/css/mappicker.css',
        ]
        CAT_SCRIPTS_OUT = 'dist/css/mappicker.css'
        do_cat(CAT_SCRIPTS, CAT_SCRIPTS_OUT)

        CAT_SCRIPTS = [
            'source/mqtt/mqttws31.min.js',
        ]
        CAT_SCRIPTS_OUT = 'dist/js/mqttws31.min.js'
        do_cat(CAT_SCRIPTS, CAT_SCRIPTS_OUT)

        CAT_SCRIPTS = [
            'source/sb-admin/js/jquery.serializejson.min.js',
        ]
        CAT_SCRIPTS_OUT = 'dist/js/jquery.serializejson.min.js'
        do_cat(CAT_SCRIPTS, CAT_SCRIPTS_OUT)

        CAT_SCRIPTS = [
            'source/sb-admin/js/sha256.js',
        ]
        CAT_SCRIPTS_OUT = 'dist/js/sha256.js'
        do_cat(CAT_SCRIPTS, CAT_SCRIPTS_OUT)

        CAT_SCRIPTS = [
            'source/yombo/jquery.are-you-sure.js',
        ]
        CAT_SCRIPTS_OUT = 'dist/js/jquery.are-you-sure.js'
        do_cat(CAT_SCRIPTS, CAT_SCRIPTS_OUT)

        # Just copy files
        copytree('source/font-awesome/fonts/', 'dist/fonts/')

        copytree('source/bootstrap/dist/fonts/', 'dist/fonts/')
Ejemplo n.º 45
0
class MetricLogObserver(config.ReconfigurableServiceMixin,
                        service.MultiService):
    _reactor = reactor

    def __init__(self):
        service.MultiService.__init__(self)
        self.setName('metrics')

        self.enabled = False
        self.periodic_task = None
        self.periodic_interval = None
        self.log_task = None
        self.log_interval = None

        # Mapping of metric type to handlers for that type
        self.handlers = {}

        # Register our default handlers
        self.registerHandler(MetricCountEvent, MetricCountHandler(self))
        self.registerHandler(MetricTimeEvent, MetricTimeHandler(self))
        self.registerHandler(MetricAlarmEvent, MetricAlarmHandler(self))

        # Make sure our changes poller is behaving
        self.getHandler(MetricTimeEvent).addWatcher(PollerWatcher(self))
        self.getHandler(MetricCountEvent).addWatcher(
            AttachedSlavesWatcher(self))

    def reconfigService(self, new_config):
        # first, enable or disable
        if new_config.metrics is None:
            self.disable()
        else:
            self.enable()

            metrics_config = new_config.metrics

            # Start up periodic logging
            log_interval = metrics_config.get('log_interval', 60)
            if log_interval != self.log_interval:
                if self.log_task:
                    self.log_task.stop()
                    self.log_task = None
                if log_interval:
                    self.log_task = LoopingCall(self.report)
                    self.log_task.clock = self._reactor
                    self.log_task.start(log_interval)

            # same for the periodic task
            periodic_interval = metrics_config.get('periodic_interval', 10)
            if periodic_interval != self.periodic_interval:
                if self.periodic_task:
                    self.periodic_task.stop()
                    self.periodic_task = None
                if periodic_interval:
                    self.periodic_task = LoopingCall(periodicCheck,
                                                     self._reactor)
                    self.periodic_task.clock = self._reactor
                    self.periodic_task.start(periodic_interval)

        # upcall
        return config.ReconfigurableServiceMixin.reconfigService(
            self, new_config)

    def stopService(self):
        self.disable()
        service.MultiService.stopService(self)

    def enable(self):
        if self.enabled:
            return
        log.addObserver(self.emit)
        self.enabled = True

    def disable(self):
        if not self.enabled:
            return

        if self.periodic_task:
            self.periodic_task.stop()
            self.periodic_task = None

        if self.log_task:
            self.log_task.stop()
            self.log_task = None

        log.removeObserver(self.emit)
        self.enabled = False

    def registerHandler(self, interface, handler):
        old = self.getHandler(interface)
        self.handlers[interface] = handler
        return old

    def getHandler(self, interface):
        return self.handlers.get(interface)

    def emit(self, eventDict):
        # Ignore non-statistic events
        metric = eventDict.get('metric')
        if not metric or not isinstance(metric, MetricEvent):
            return

        if metric.__class__ not in self.handlers:
            return

        h = self.handlers[metric.__class__]
        h.handle(eventDict, metric)
        for w in h.watchers:
            w.run()

    def asDict(self):
        retval = {}
        for interface, handler in self.handlers.iteritems():
            retval.update(handler.asDict())
        return retval

    def report(self):
        try:
            for interface, handler in self.handlers.iteritems():
                report = handler.report()
                if not report:
                    continue
                for line in report.split("\n"):
                    log.msg(line)
        except:
            klog.err_json(None, "generating metric report")
Ejemplo n.º 46
0
 def __init__(self, factory):
     self.factory = factory
     self.loop = LoopingCall(self.check_log)
     self.log_provider = None
Ejemplo n.º 47
0
class FeatureProtocol(ServerProtocol):
    connection_class = FeatureConnection
    bans = None
    ban_publish = None
    ban_manager = None
    everyone_is_admin = False
    player_memory = None
    irc_relay = None
    balanced_teams = None
    timestamps = None
    building = True
    killing = True
    global_chat = True
    remote_console = None
    advance_call = None
    master_reconnect_call = None
    master = False
    ip = None
    identifier = None

    planned_map = None

    map_info = None
    spawns = None
    user_blocks = None
    god_blocks = None

    last_time = None
    interface = None

    team_class = FeatureTeam

    game_mode = None  # default to None so we can check
    time_announce_schedule = None

    default_fog = (128, 232, 255)

    def __init__(self, interface: bytes, config_dict: Dict[str, Any]) -> None:
        # logfile path relative to config dir if not abs path
        log_filename = logfile.get()
        if log_filename.strip():  # catches empty filename
            if not os.path.isabs(log_filename):
                log_filename = os.path.join(config.config_dir, log_filename)
            ensure_dir_exists(log_filename)
            if logging_rotate_daily.get():
                logging_file = DailyLogFile(log_filename, '.')
            else:
                logging_file = open(log_filename, 'a')
            predicate = LogLevelFilterPredicate(
                LogLevel.levelWithName(loglevel.get()))
            observers = [
                FilteringLogObserver(textFileLogObserver(sys.stderr),
                                     [predicate]),
                FilteringLogObserver(textFileLogObserver(logging_file),
                                     [predicate])
            ]
            globalLogBeginner.beginLoggingTo(observers)
            log.info('piqueserver started on %s' % time.strftime('%c'))

        self.config = config_dict
        if random_rotation.get():
            self.map_rotator_type = random_choice_cycle
        else:
            self.map_rotator_type = itertools.cycle
        self.default_time_limit = default_time_limit.get()
        self.default_cap_limit = cap_limit.get()
        self.advance_on_win = int(advance_on_win.get())
        self.win_count = itertools.count(1)
        self.bans = NetworkDict()

        # attempt to load a saved bans list
        try:
            with open(os.path.join(config.config_dir, bans_file.get()),
                      'r') as f:
                self.bans.read_list(json.load(f))
            log.debug("loaded {count} bans", count=len(self.bans))
        except FileNotFoundError:
            log.debug("skip loading bans: file unavailable",
                      count=len(self.bans))
        except IOError as e:
            log.error('Could not read bans file ({}): {}'.format(
                bans_file.get(), e))
        except ValueError as e:
            log.error('Could not parse bans file ({}): {}'.format(
                bans_file.get(), e))

        self.hard_bans = set()  # possible DDoS'ers are added here
        self.player_memory = deque(maxlen=100)
        if len(self.name) > MAX_SERVER_NAME_SIZE:
            log.warn('(server name too long; it will be truncated to "%s")' %
                     (self.name[:MAX_SERVER_NAME_SIZE]))
        self.respawn_time = respawn_time_option.get()
        self.respawn_waves = respawn_waves.get()

        # since AoS only supports CTF and TC at a protocol level, we need to get
        # the base game mode if we are using a custom game mode.
        game_mode_name = game_mode.get()
        if game_mode_name == 'ctf':
            self.game_mode = CTF_MODE
        elif game_mode.get() == 'tc':
            self.game_mode = TC_MODE
        elif self.game_mode not in [CTF_MODE, TC_MODE]:
            raise ValueError(
                'invalid game mode: custom game mode "{}" does not set '
                'protocol.game_mode to one of TC_MODE or CTF_MODE. Are '
                'you sure the thing you have specified is a game mode?'.format(
                    game_mode_name))

        self.game_mode_name = game_mode.get().split('.')[-1]
        self.team1_name = team1_name.get()[:9]
        self.team2_name = team2_name.get()[:9]
        self.team1_color = tuple(team1_color.get())
        self.team2_color = tuple(team2_color.get())
        self.friendly_fire = friendly_fire.get()
        self.friendly_fire_on_grief = friendly_fire_on_grief.get()
        self.friendly_fire_time = grief_friendly_fire_time.get()
        self.spade_teamkills_on_grief = spade_teamkills_on_grief.get()
        self.fall_damage = fall_damage.get()
        self.teamswitch_interval = teamswitch_interval.get()
        self.teamswitch_allowed = teamswitch_allowed.get()
        self.max_players = max_players.get()
        self.melee_damage = melee_damage.get()
        self.max_connections_per_ip = max_connections_per_ip.get()
        self.passwords = passwords.get()
        self.server_prefix = server_prefix.get()
        self.time_announcements = time_announcements.get()
        self.balanced_teams = balanced_teams.get()
        self.login_retries = login_retries.get()

        # voting configuration
        self.default_ban_time = default_ban_duration.get()

        self.speedhack_detect = speedhack_detect.get()
        self.rubberband_distance = rubberband_distance.get()
        if user_blocks_only.get():
            self.user_blocks = set()
        self.set_god_build = set_god_build.get()
        if ssh_enabled.get():
            from piqueserver.ssh import RemoteConsole
            self.remote_console = RemoteConsole(self)
        irc = irc_options.get()
        if irc.get('enabled', False):
            from piqueserver.irc import IRCRelay
            self.irc_relay = IRCRelay(self, irc)
        if status_server_enabled.get():
            from piqueserver.statusserver import StatusServer
            self.status_server = StatusServer(self)
            ensureDeferred(self.status_server.listen())
        if ban_publish.get():
            from piqueserver.banpublish import PublishServer
            self.ban_publish = PublishServer(self, ban_publish_port.get())
        if bans_config_urls.get():
            from piqueserver import bansubscribe
            self.ban_manager = bansubscribe.BanManager(self)
            ensureDeferred(as_deferred(self.ban_manager.start()))
        self.start_time = time.time()
        self.end_calls = []
        # TODO: why is this here?
        create_console(self)

        for user_type, func_names in rights.get().items():
            for func_name in func_names:
                commands.add_rights(user_type, func_name)

        if everyone_is_admin.get():
            self.everyone_is_admin = True

        self.port = port_option.get()
        ServerProtocol.__init__(self, self.port, interface)
        self.host.intercept = self.receive_callback

        try:
            self.set_map_rotation(self.config['rotation'])
        except MapNotFound as e:
            log.critical('Invalid map in map rotation (%s), exiting.' % e.map)
            raise SystemExit

        map_load_d = self.advance_rotation()
        # discard the result of the map advance for now
        map_load_d.addCallback(lambda x: self._post_init())

        ip_getter = ip_getter_option.get()
        if ip_getter:
            ensureDeferred(as_deferred(self.get_external_ip(ip_getter)))

        self.new_release = None
        notify_new_releases = config.option("release_notifications",
                                            default=True)
        if notify_new_releases.get():
            ensureDeferred(as_deferred(self.watch_for_releases()))

        self.vacuum_loop = LoopingCall(self.vacuum_bans)
        # Run the vacuum every 6 hours, and kick it off it right now
        self.vacuum_loop.start(60 * 60 * 6, True)

        reactor.addSystemEventTrigger('before', 'shutdown',
                                      lambda: ensureDeferred(self.shutdown()))

    def _post_init(self):
        """called after the map has been loaded"""
        self.update_format()
        self.tip_frequency = tip_frequency.get()
        if self.tips and self.tip_frequency > 0:
            reactor.callLater(self.tip_frequency * 60, self.send_tip)

        self.master = register_master_option.get()
        self.set_master()

    async def get_external_ip(self, ip_getter: str) -> Iterator[Deferred]:
        log.info(
            'Retrieving external IP from {!r} to generate server identifier.'.
            format(ip_getter))
        try:
            async with aiohttp.ClientSession() as session:
                async with session.get(ip_getter) as response:
                    ip = await response.text()
                    ip = IPv4Address(ip.strip())
        except AddressValueError as e:
            log.warn('External IP getter service returned invalid data.\n'
                     'Please check the "ip_getter" setting in your config.')
            return
        except Exception as e:  # pylint: disable=broad-except
            log.warn("Getting external IP failed: {reason}", reason=e)
            return

        self.ip = ip
        self.identifier = make_server_identifier(ip, self.port)
        log.info('Server public ip address: {}:{}'.format(ip, self.port))
        log.info('Public aos identifier: {}'.format(self.identifier))

    def set_time_limit(self,
                       time_limit: Optional[int] = None,
                       additive: bool = False) -> Optional[int]:
        advance_call = self.advance_call
        add_time = 0.0
        if advance_call is not None:
            add_time = ((advance_call.getTime() - reactor.seconds()) / 60.0)
            advance_call.cancel()
            self.advance_call = None
        time_limit = time_limit or self.default_time_limit
        if not time_limit:
            for call in self.end_calls[:]:
                call.set(None)
            return None

        if additive:
            time_limit = min(time_limit + add_time, self.default_time_limit)

        seconds = time_limit * 60
        self.advance_call = reactor.callLater(seconds, self._time_up)

        for call in self.end_calls[:]:
            call.set(seconds)

        if self.time_announce_schedule is not None:
            self.time_announce_schedule.reset()
        self.time_announce_schedule = Scheduler(self)
        for seconds in self.time_announcements:
            self.time_announce_schedule.call_end(seconds,
                                                 self._next_time_announce)

        return time_limit

    def _next_time_announce(self):
        remaining = self.advance_call.getTime() - reactor.seconds()
        if remaining < 60.001:
            if remaining < 10.001:
                self.broadcast_chat('%s...' % int(round(remaining)))
            else:
                self.broadcast_chat('%s seconds remaining.' %
                                    int(round(remaining)))
        else:
            self.broadcast_chat('%s minutes remaining.' %
                                int(round(remaining / 60)))

    def _time_up(self):
        self.advance_call = None
        self.advance_rotation('Time up!')

    def advance_rotation(self, message: Optional[str] = None) -> Deferred:
        """
        Advances to the next map in the rotation. If message is provided
        it will send it to the chat, waits for 10 seconds and then advances.

        Returns:
            Deferred that fires when the map has been loaded
        """
        self.set_time_limit(False)
        if self.planned_map is None:
            self.planned_map = next(self.map_rotator)
        planned_map = self.planned_map
        self.planned_map = None
        self.on_advance(planned_map)

        async def do_advance():
            if message is not None:
                log.info("advancing to map '{name}' ({reason}) in 10 seconds",
                         name=planned_map.full_name,
                         reason=message)
                self.broadcast_chat('{} Next map: {}.'.format(
                    message, planned_map.full_name),
                                    irc=True)
                await sleep(10)
            else:
                log.info("advancing to map '{name}'",
                         name=planned_map.full_name)

            await self.set_map_name(planned_map)

        return ensureDeferred(do_advance())

    def get_mode_name(self) -> str:
        return self.game_mode_name

    async def set_map_name(self, rot_info: RotationInfo) -> None:
        """
        Sets the map by its name.
        """
        map_info = await self.make_map(rot_info)
        if self.map_info:
            self.on_map_leave()
        self.map_info = map_info
        self.max_score = self.map_info.cap_limit or self.default_cap_limit
        self.set_map(self.map_info.data)
        self.set_time_limit(self.map_info.time_limit)
        self.update_format()

    def set_server_name(self, name: str) -> None:
        name_option.set(name)
        self.update_format()

    def make_map(self, rot_info: RotationInfo) -> Deferred:
        """
        Creates and returns a Map object from rotation info in a new thread

        Returns:
            Deferred that resolves to a `Map` object.
        """
        # we must do this in a new thread, since map generation might take so
        # long that clients time out.
        return threads.deferToThread(Map, rot_info,
                                     os.path.join(config.config_dir, 'maps'))

    def set_map_rotation(self, maps: List[str]) -> None:
        """
        Over-writes the current map rotation with provided one.
        `FeatureProtocol.advance_rotation` still needs to be called to actually
        change the map,
        """
        maps = check_rotation(maps, os.path.join(config.config_dir, 'maps'))
        self.maps = maps
        self.map_rotator = self.map_rotator_type(maps)

    def get_map_rotation(self):
        return [map_item.full_name for map_item in self.maps]

    def is_indestructable(self, x: int, y: int, z: int) -> bool:
        if self.user_blocks is not None:
            if (x, y, z) not in self.user_blocks:
                return True
        if self.god_blocks is not None:
            if (x, y, z) in self.god_blocks:  # pylint: disable=unsupported-membership-test
                return True
        map_is_indestructable = self.map_info.is_indestructable
        if map_is_indestructable is not None:
            if map_is_indestructable(self, x, y, z):
                return True
        return False

    def update_format(self) -> None:
        """
        Called when the map (or other variables) have been updated
        """
        self.name = self.format(name_option.get())
        self.motd = self.format_lines(motd_option.get())
        self.help = self.format_lines(help_option.get())
        self.tips = self.format_lines(tips_option.get())
        self.rules = self.format_lines(rules_option.get())
        if self.master_connection is not None:
            self.master_connection.send_server()

    def format(self,
               value: str,
               extra: Optional[Dict[str, str]] = None) -> str:
        map_info = self.map_info
        format_dict = {
            'map_name': map_info.name,
            'map_author': map_info.author,
            'map_description': map_info.description,
            'game_mode': self.get_mode_name(),
            'server_name': self.name,
        }

        if extra:
            format_dict.update(extra)
        # format with both old-style and new string formatting to stay
        # compatible with older configs
        return value.format(**format_dict) % format_dict

    def format_lines(self, value: List[str]) -> List[str]:
        if value is None:
            return
        return [self.format(line) for line in value]

    def got_master_connection(self, client):
        log.info('Master connection established.')
        ServerProtocol.got_master_connection(self, client)

    def master_disconnected(self, client=None):
        ServerProtocol.master_disconnected(self, client)
        if self.master and self.master_reconnect_call is None:
            if client:
                message = 'Master connection could not be established'
            else:
                message = 'Master connection lost'
            log.info('%s, reconnecting in 60 seconds...' % message)
            self.master_reconnect_call = reactor.callLater(
                60, self.reconnect_master)

    def reconnect_master(self):
        self.master_reconnect_call = None
        self.set_master()

    def set_master_state(self, value):
        if value == self.master:
            return
        self.master = value
        has_connection = self.master_connection is not None
        has_reconnect = self.master_reconnect_call is not None
        if value:
            if not has_connection and not has_reconnect:
                self.set_master()
        else:
            if has_reconnect:
                self.master_reconnect_call.cancel()
                self.master_reconnect_call = None
            if has_connection:
                self.master_connection.disconnect()

    async def shutdown(self):
        """
        Notifies players and disconnects them before a shutdown.
        """
        if not self.connections:
            # exit instantly if nobody is connected anyway
            return

        # send shutdown notification
        log.info("disconnecting players")
        self.broadcast_chat("Server shutting down in 3sec.")
        for i in range(3, 0, -1):
            self.broadcast_chat(str(i) + "...")
            await sleep(1)

        # disconnect all players
        for connection in list(self.connections.values()):
            connection.disconnect(ERROR_SHUTDOWN)

        # give the connections some time to terminate
        await sleep(0.2)

    def add_ban(self, ip, reason, duration, name=None):
        """
        Ban an ip with an optional reason and duration in seconds. If duration
        is None, ban is permanent.
        """
        network = ip_network(str(ip), strict=False)
        for connection in list(self.connections.values()):
            if ip_address(connection.address[0]) in network:
                name = connection.name
                connection.kick(silent=True)
        if duration:
            duration = time.time() + duration
        else:
            duration = None
        self.bans[ip] = (name or '(unknown)', reason, duration)
        self.save_bans()

    def remove_ban(self, ip):
        results = self.bans.remove(ip)
        log.info('Removing ban: {ip} {results}', ip=ip, results=results)
        self.save_bans()

    async def watch_for_releases(self):
        """Starts a loop for `check_for_releases` and updates `self.new_release`."""
        while True:
            self.new_release = await check_for_releases()
            if self.new_release:
                log.info("#" * 60)
                log.info(format_release(self.new_release))
                log.info("#" * 60)
            await asyncio.sleep(86400)  # 24 hrs

    def vacuum_bans(self):
        """remove any bans that might have expired. This takes a while, so it is
        split up over the event loop"""
        def do_vacuum_bans():
            """do the actual clearing of bans"""

            bans_count = len(self.bans)
            log.info("starting ban vacuum with {count} bans", count=bans_count)
            start_time = time.time()

            # create a copy of the items, so we don't have issues modifying
            # while iteraing
            for ban in list(self.bans.iteritems()):
                ban_exipry = ban[1][2]
                if ban_exipry is None:
                    # entry never expires
                    continue
                if ban[1][2] < start_time:
                    # expired
                    del self.bans[ban[0]]
                yield
            log.debug(
                "ban vacuum took {time:.2f} seconds, removed {count} bans",
                count=bans_count - len(self.bans),
                time=time.time() - start_time)
            self.save_bans()

        # TODO: use cooperate() here instead, once you figure out why it's
        # swallowing errors. Perhaps try add an errback?
        coiterate(do_vacuum_bans())

    def undo_last_ban(self):
        result = self.bans.pop()
        self.save_bans()
        return result

    def save_bans(self):
        ban_file = os.path.join(config.config_dir, bans_file.get())
        ensure_dir_exists(ban_file)

        start_time = reactor.seconds()
        with open(ban_file, 'w') as f:
            json.dump(self.bans.make_list(), f, indent=2)
        log.debug("saving {count} bans took {time:.2f} seconds",
                  count=len(self.bans),
                  time=reactor.seconds() - start_time)

        if self.ban_publish is not None:
            self.ban_publish.update()

    def receive_callback(self, address: Address, data: bytes) -> int:
        """This hook receives the raw UDP data before it is processed by enet"""

        # exceptions get swallowed in the pyenet C stuff, so we catch anything
        # for now. This should ideally get fixed in pyenet instead.
        try:
            # reply to ASCII HELLO messages with HI so that clients can measure the
            # connection latency
            if data == b'HELLO':
                self.host.socket.send(address, b'HI')
                return 1
            # reply to ASCII HELLOLAN messages with server data for LAN discovery
            elif data == b'HELLOLAN':
                # we might receive a HELLOLAN before the map has been loaded
                # if so, return a dummy string instead
                if self.map_info:
                    map_name = self.map_info.short_name
                else:
                    map_name = "loading..."
                entry = {
                    "name": self.name,
                    "players_current": self.get_player_count(),
                    "players_max": self.max_players,
                    "map": map_name,
                    "game_mode": self.get_mode_name(),
                    "game_version": "0.75"
                }
                payload = json.dumps(entry).encode()
                self.host.socket.send(address, payload)
                return 1

            # This drop the connection of any ip in hard_bans
            if address.host in self.hard_bans:
                return 1
        except Exception:
            import traceback
            traceback.print_exc()

        return 0

    def data_received(self, peer: Peer, packet: Packet) -> None:
        ip = peer.address.host
        current_time = reactor.seconds()
        try:
            ServerProtocol.data_received(self, peer, packet)
        except (NoDataLeft, ValueError):
            import traceback
            traceback.print_exc()
            log.info(
                'IP %s was hardbanned for invalid data or possibly DDoS.' % ip)
            self.hard_bans.add(ip)
            return
        dt = reactor.seconds() - current_time
        if dt > 1.0:
            log.warn('processing {!r} from {} took {}'.format(
                packet.data, ip, dt))

    def irc_say(self, msg: str, me: bool = False) -> None:
        if self.irc_relay:
            if me:
                self.irc_relay.me(msg, do_filter=True)
            else:
                self.irc_relay.send(msg, do_filter=True)

    def send_tip(self):
        line = self.tips[random.randrange(len(self.tips))]
        self.broadcast_chat(line)
        reactor.callLater(self.tip_frequency * 60, self.send_tip)

    # pylint: disable=arguments-differ
    def broadcast_chat(self,
                       value,
                       global_message=True,
                       sender=None,
                       team=None,
                       irc=False):
        """
        Send a chat message to many users
        """
        if irc:
            self.irc_say('* %s' % value)
        ServerProtocol.broadcast_chat(self, value, global_message, sender,
                                      team)

    # backwards compatability
    def send_chat(self, *args, **kwargs):
        """Deprecated: see broadcast_chat"""
        warnings.warn(
            "use of deprecated send_chat, use broadcast_chat instead",
            DeprecationWarning,
            stacklevel=2)
        self.broadcast_chat(*args, **kwargs)

    # log high CPU usage

    def update_world(self):
        last_time = self.last_time
        current_time = reactor.seconds()
        if last_time is not None:
            dt = current_time - last_time
            if dt > 1.0:
                log.warn('high CPU usage detected - %s' % dt)
        self.last_time = current_time
        ServerProtocol.update_world(self)
        time_taken = reactor.seconds() - current_time
        if time_taken > 1.0:
            log.warn('World update iteration took %s, objects: %s' %
                     (time_taken, self.world.objects))

    # events

    def on_map_change(self, the_map: VXLData) -> None:
        self.set_fog_color(getattr(self.map_info.info, 'fog',
                                   self.default_fog))

        map_on_map_change = self.map_info.on_map_change
        if map_on_map_change is not None:
            map_on_map_change(self, the_map)

    def on_map_leave(self):
        map_on_map_leave = self.map_info.on_map_leave
        if map_on_map_leave is not None:
            map_on_map_leave(self)

    def on_game_end(self):
        if self.advance_on_win <= 0:
            self.irc_say('Round ended!', me=True)
        elif next(self.win_count) % self.advance_on_win == 0:
            self.advance_rotation('Game finished!')

    def on_advance(self, map_name: str) -> None:
        pass

    def on_ban_attempt(self, connection, reason, duration):
        return True

    def on_ban(self, connection, reason, duration):
        pass

    # voting

    def cancel_vote(self, connection=None):
        return 'No vote in progress.'

    # useful twisted wrappers

    def listenTCP(self, *arg, **kw) -> Port:
        return reactor.listenTCP(*arg, interface=network_interface.get(), **kw)

    def connectTCP(self, *arg, **kw):
        return reactor.connectTCP(*arg,
                                  bindAddress=(network_interface.get(), 0),
                                  **kw)

    # before-end calls

    def call_end(self, delay: int, func: Callable, *arg, **kw) -> EndCall:
        call = EndCall(self, delay, func, *arg, **kw)
        call.set(self.get_advance_time())
        return call

    def get_advance_time(self) -> float:
        if not self.advance_call:
            return None
        return self.advance_call.getTime() - self.advance_call.seconds()
Ejemplo n.º 48
0
 def init(self):
     self._periodic_clean_sessions = LoopingCall(self.clean_sessions)
     self._periodic_clean_sessions.start(random_int(
         60 * 60 * 2,
         .7))  # Every 60-ish seconds. Save to disk, or remove from memory.
Ejemplo n.º 49
0
class Yombo_Site(Site):
    def setup_log_queue(self, webinterface):
        self.save_log_queue_loop = LoopingCall(self.save_log_queue)
        self.save_log_queue_loop.start(8.7, False)

        self.log_queue = []

        self.webinterface = webinterface
        self.db_save_log = self.webinterface._LocalDB.webinterface_save_logs

    def _escape(self, s):
        """
        Return a string like python repr, but always escaped as if surrounding
        quotes were double quotes.
        @param s: The string to escape.
        @type s: L{bytes} or L{unicode}
        @return: An escaped string.
        @rtype: L{unicode}
        """
        if not isinstance(s, bytes):
            s = s.encode("ascii")

        r = repr(s)
        if not isinstance(r, str):
            r = r.decode("ascii")
        if r.startswith(u"b"):
            r = r[1:]
        if r.startswith(u"'"):
            return r[1:-1].replace(u'"', u'\\"').replace(u"\\'", u"'")
        return r[1:-1]

    def log(self, request):
        ignored_extensions = ('.js', '.css', '.jpg', '.jpeg', '.gif', '.ico',
                              '.woff2', '.map')
        url_path = request.path.decode().strip()

        if any(url_path.endswith(ext) for ext in ignored_extensions):
            return

        od = OrderedDict({
            'request_at':
            time(),
            'request_protocol':
            request.clientproto.decode().strip(),
            'referrer':
            self._escape(request.getHeader(b"referer") or b"-").strip(),
            'agent':
            self._escape(request.getHeader(b"user-agent") or b"-").strip(),
            'ip':
            request.getClientIP(),
            'hostname':
            request.getRequestHostname().decode().strip(),
            'method':
            request.method.decode().strip(),
            'path':
            url_path,
            'secure':
            request.isSecure(),
            'response_code':
            request.code,
            'response_size':
            request.sentLength,
            'uploadable':
            1,
            'uploaded':
            0,
        })

        self.log_queue.append(od)

    def save_log_queue(self):
        if len(self.log_queue) > 0:
            queue = self.log_queue
            self.log_queue = []
            self.db_save_log(queue)
Ejemplo n.º 50
0
from hl7parser import patient, measure, channel, oru_wav
from monitorapi import ICUMonitor, ICUMonitorFactory
from monitor_multi import monitor_multi
from twisted.internet import reactor
from twisted.internet.task import LoopingCall

ip = "localhost"
port = 60000

smsg = None


def setmsg():
    global smsg
    return smsg


def createmsg(m):
    m.preenche()
    orw = mon.get_orw((ip, 'CEN'))
    global smsg
    smsg = orw.to_str()


if __name__ == '__main__':
    mon = monitor_multi(ip)
    monit = ICUMonitorFactory(setmsg, ip, port)
    LoopingCall(createmsg, mon).start(0.5)
    monit.startsend(reactor, 0.5)
    reactor.run()
Ejemplo n.º 51
0
Archivo: app.py Proyecto: jarret/relay
class Relay(object):
    def __init__(self, config):
        self.config = config

        # TODO - this could/should be extracted into a Stack class, though
        # the relay app is small and straightforward like this, so no rush.
        self.websocket_layer = self.setup_websocket_layer()
        self.rendezvous_layer = self.setup_rendezvous_layer(
            self.websocket_layer)
        self.relay_layer = self.setup_relay_layer(self.rendezvous_layer)

        self.listen_url = self.get_listen_url(self.config['Relay'])
        self.tls_info = self.get_tls_info(self.config['Relay'])

        self.info_loop = LoopingCall(self.output_info)
        self.info_loop.start(2.0, now=False)

    ###########################################################################

    def setup_relay_layer(self, rendezvous_layer):
        l = RelayLayer()
        l.register_above_layer(rendezvous_layer)
        l.register_layer_event(self.on_stack_event, "RELAY")
        l.set_rendezvous_layer(rendezvous_layer)
        return l

    def setup_rendezvous_layer(self, below_layer):
        l = IncomingRendezvousLayer()
        l.register_above_layer(below_layer)
        l.register_layer_event(self.on_stack_event, "INCOMING_RENDEZVOUS")
        return l

    def setup_websocket_layer(self):
        l = IncomingWebsocketLayer()
        l.register_layer_event(self.on_stack_event, "INCOMING_WEBSOCKET")
        return l

    ###########################################################################

    def on_stack_event(self, layer_name, nexus, status):
        pass

    ###########################################################################

    def output_info(self):
        logging.info(str(self.rendezvous_layer))

    def get_listen_url(self, config_section):
        bind = config_section['ListenBind']
        port = int(config_section['ListenPort'])
        prefix = "wss" if config_section['UseTLS'] == "True" else "ws"
        return "%s://%s:%d" % (prefix, bind, port)

    def get_tls_info(self, config_section):
        if config_section['UseTLS'] != "True":
            return None
        tls_info = {'sslmethod': SSL.TLSv1_2_METHOD,
                    'cert_file': os.path.abspath(config_section['CertFile']),
                    'key_file':  os.path.abspath(config_section['CertKey'])}
        tls_info['cert_chain_file'] = (
            os.path.abspath(config_section['CertChainFile']) if
            config_section['SelfSignedCert'] == "False" else None)
        return tls_info

    ###########################################################################

    def run_app(self):
        print("listening at: %s" % self.listen_url)
        self.websocket_layer.listen(self.listen_url, tls_info=self.tls_info)
Ejemplo n.º 52
0
class IPCMasterService(service.Service):
    """
    IPC master service.

    Provides the master side of the IPC communication between the workers.
    """

    UPDATE_INTERVAL = 60  # 60 seconds.

    REMOVE_INTERVAL = 90  # 90 seconds.

    connections = None

    def __init__(self, reactor, workers=None, socket_path=None):
        super().__init__()
        self.reactor = reactor
        self.workers = workers
        self.socket_path = socket_path
        if self.socket_path is None:
            self.socket_path = get_ipc_socket_path()
        if os.path.exists(self.socket_path):
            os.remove(self.socket_path)
        self.endpoint = UNIXServerEndpoint(reactor, self.socket_path)
        self.port = None
        self.connections = {}
        self.factory = Factory.forProtocol(IPCMaster)
        self.factory.service = self
        self.updateLoop = LoopingCall(self.update)

    @asynchronous
    def startService(self):
        """Start listening on UNIX socket and create the region controller."""
        super().startService()
        self.starting = self.endpoint.listen(self.factory)

        def save_port(port):
            self.port = port

        @transactional
        def create_region(result):
            RegionController.objects.get_or_create_running_controller()

        def start_update_loop(result):
            self.updateLoopDone = self.updateLoop.start(self.UPDATE_INTERVAL)

        def log_failure(failure):
            if failure.check(CancelledError):
                log.msg("IPCMasterService start-up has been cancelled.")
            else:
                log.err(failure, "IPCMasterService start-up failed.")

        self.starting.addCallback(save_port)
        self.starting.addCallback(partial(deferToDatabase, create_region))
        self.starting.addCallback(start_update_loop)
        self.starting.addErrback(log_failure)

        # Twisted's service framework does not track start-up progress, i.e.
        # it does not check for Deferreds returned by startService(). Here we
        # return a Deferred anyway so that direct callers (esp. those from
        # tests) can easily wait for start-up.
        return self.starting

    @asynchronous
    @inlineCallbacks
    def stopService(self):
        """Stop listening."""
        self.starting.cancel()
        if self.port:
            self.port, port = None, self.port
            yield port.stopListening()
        for data in self.connections.values():
            try:
                yield data["connection"].transport.loseConnection()
            except Exception:
                log.err(None, "Failure when closing IPC connection.")

        @transactional
        def delete_all_processes():
            region = RegionController.objects.get_running_controller()
            region.processes.all().delete()

        @asynchronous
        def stop_update_loop():
            if self.updateLoop.running:
                self.updateLoop.stop()
                return self.updateLoopDone

        yield deferToDatabase(delete_all_processes)
        yield stop_update_loop()
        yield super().stopService()

    @asynchronous
    def registerWorker(self, pid, conn):
        """Register the worker with `pid` using `conn`."""

        @transactional
        def create_process(pid):
            region = RegionController.objects.get_running_controller()
            process, _ = RegionControllerProcess.objects.get_or_create(
                region=region, pid=pid
            )
            return (pid, process.id)

        def log_connected(result):
            pid, process_id = result
            log.msg("Worker pid:%d IPC connected." % pid)
            return result

        def add_to_connections(result):
            pid, process_id = result
            self.connections[pid] = {
                "process_id": process_id,
                "connection": conn,
                "rpc": {"port": None, "connections": set()},
            }
            return process_id

        @transactional
        def update_service(process_id):
            region = RegionController.objects.get_running_controller()
            self._updateService(region)
            return process_id

        def return_result(process_id):
            return {"process_id": process_id}

        d = deferToDatabase(create_process, pid)
        d.addCallback(log_connected)
        d.addCallback(add_to_connections)
        d.addCallback(partial(deferToDatabase, update_service))
        d.addCallback(return_result)
        return d

    def getPIDFromConnection(self, conn):
        """Get the PID from the connection."""
        for pid, data in self.connections.items():
            if data["connection"] == conn:
                return pid

    @asynchronous
    def unregisterWorker(self, conn, reason):
        """Unregister the worker with `pid` because of `reason`."""
        pid = self.getPIDFromConnection(conn)
        if pid:

            @transactional
            def delete_process(pid):
                process_id = self.connections[pid]["process_id"]
                RegionControllerProcess.objects.filter(id=process_id).delete()
                return pid

            def remove_conn_kill_worker(pid):
                del self.connections[pid]
                if self.workers:
                    self.workers.killWorker(pid)
                return pid

            def log_disconnected(pid):
                log.msg("Worker pid:%d IPC disconnected." % pid)

            d = deferToDatabase(delete_process, pid)
            d.addCallback(remove_conn_kill_worker)
            d.addCallback(log_disconnected)
            return d

    def _getListenAddresses(self, port):
        """Return list of tuple (address, port) for the addresses the worker
        is listening on."""
        addresses = get_all_interface_source_addresses()
        if addresses:
            return set((addr, port) for addr in addresses)
        # There are no non-loopback addresses, so return loopback
        # address as a fallback.
        loopback_addresses = set()
        for addr in get_all_interface_addresses():
            ipaddr = IPAddress(addr)
            if ipaddr.is_link_local():
                continue  # Don't advertise link-local addresses.
            if ipaddr.is_loopback():
                loopback_addresses.add((addr, port))
        return loopback_addresses

    @synchronous
    @transactional
    def _updateEndpoints(self, process, addresses):
        """Update the endpoints for `pid` and `port`."""
        previous_endpoint_ids = set(
            RegionControllerProcessEndpoint.objects.filter(
                process=process
            ).values_list("id", flat=True)
        )
        if addresses:
            for addr, port in addresses:
                (
                    endpoint,
                    created,
                ) = RegionControllerProcessEndpoint.objects.get_or_create(
                    process=process, address=addr, port=port
                )
                if not created:
                    previous_endpoint_ids.remove(endpoint.id)
        RegionControllerProcessEndpoint.objects.filter(
            id__in=previous_endpoint_ids
        ).delete()

    @synchronous
    def _getProcessObjFor(self, pid):
        """Return `RegionControllerProcess` for `pid`."""
        process_id = self.connections[pid]["process_id"]
        try:
            return RegionControllerProcess.objects.get(id=process_id)
        except RegionControllerProcess.DoesNotExist:
            region_obj = RegionController.objects.get_running_controller()
            return RegionControllerProcess.objects.create(
                id=process_id, region=region_obj, pid=pid
            )

    @asynchronous
    def registerWorkerRPC(self, pid, port):
        """Register the worker with `pid` has RPC `port` open."""
        if pid in self.connections:

            @transactional
            def create_endpoints(result):
                pid, port = result
                process = self._getProcessObjFor(pid)
                self._updateEndpoints(process, self._getListenAddresses(port))
                return result

            def set_result(result):
                pid, port = result
                self.connections[pid]["rpc"]["port"] = port
                self.connections[pid]["rpc"]["connections"] = {}
                return result

            def log_rpc_open(result):
                log.msg(
                    "Worker pid:%d opened RPC listener on port:%s." % result
                )

            d = deferToDatabase(create_endpoints, (pid, port))
            d.addCallback(set_result)
            d.addCallback(log_rpc_open)
            return d

    @synchronous
    def _registerConnection(self, process, ident, host, port, force_save=True):
        rackd = RackController.objects.get(system_id=ident)
        endpoint, _ = RegionControllerProcessEndpoint.objects.get_or_create(
            process=process, address=host, port=port
        )
        connection, created = RegionRackRPCConnection.objects.get_or_create(
            endpoint=endpoint, rack_controller=rackd
        )
        if not created and force_save:
            # Force the save so that signals connected to the
            # RegionRackRPCConnection are performed.
            connection.save(force_update=True)
        return connection

    def registerWorkerRPCConnection(self, pid, connid, ident, host, port):
        """Register the worker with `pid` has RPC an RPC connection."""
        if pid in self.connections:

            @transactional
            def register_connection(pid, connid, ident, host, port):
                process = self._getProcessObjFor(pid)
                self._registerConnection(process, ident, host, port)
                return (pid, connid, ident, host, port)

            def log_connection(result):
                pid, conn = result[0], result[1:]
                log.msg(
                    "Worker pid:%d registered RPC connection to %s."
                    % (pid, conn[1:])
                )
                return conn

            def set_result(conn):
                connid, conn = conn[0], conn[1:]
                self.connections[pid]["rpc"]["connections"][connid] = conn

            d = deferToDatabase(
                register_connection, pid, connid, ident, host, port
            )
            d.addCallback(log_connection)
            d.addCallback(set_result)
            return d

    @transactional
    def _unregisterConnection(self, process, ident, host, port):
        """Unregister the connection into the database."""
        try:
            endpoint = RegionControllerProcessEndpoint.objects.get(
                process=process, address=host, port=port
            )
        except RegionControllerProcessEndpoint.DoesNotExist:
            # Endpoint no longer exists, nothing to do.
            pass
        else:
            try:
                rackd = RackController.objects.get(system_id=ident)
            except RackController.DoesNotExist:
                # No rack controller, nothing to do.
                pass
            else:
                RegionRackRPCConnection.objects.filter(
                    endpoint=endpoint, rack_controller=rackd
                ).delete()

    def unregisterWorkerRPCConnection(self, pid, connid):
        """Unregister connection for worker with `pid`."""
        if pid in self.connections:
            connections = self.connections[pid]["rpc"]["connections"]
            conn = connections.get(connid, None)
            if conn is not None:

                @transactional
                def unregister_connection(pid, connid, ident, host, port):
                    process = self._getProcessObjFor(pid)
                    self._unregisterConnection(process, ident, host, port)
                    return (pid, connid, ident, host, port)

                def log_disconnect(result):
                    pid, conn = result[0], result[1:]
                    log.msg(
                        "Worker pid:%d lost RPC connection to %s."
                        % (pid, conn[1:])
                    )
                    return conn

                def set_result(conn):
                    connid = conn[0]
                    connections.pop(connid, None)

                d = deferToDatabase(unregister_connection, pid, connid, *conn)
                d.addCallback(log_disconnect)
                d.addCallback(set_result)
                return d

    @synchronous
    def _updateConnections(self, process, connections):
        """Update the existing RPC connections into this region.

        This is needed because the database could get in an incorrect state
        because another process removed its references in the database and
        the existing connections need to be re-created.
        """
        if not connections:
            RegionRackRPCConnection.objects.filter(
                endpoint__process=process
            ).delete()
        else:
            previous_connection_ids = set(
                RegionRackRPCConnection.objects.filter(
                    endpoint__process=process
                ).values_list("id", flat=True)
            )
            for _, (ident, host, port) in connections.items():
                db_conn = self._registerConnection(
                    process, ident, host, port, force_save=False
                )
                previous_connection_ids.discard(db_conn.id)
            if previous_connection_ids:
                RegionRackRPCConnection.objects.filter(
                    id__in=previous_connection_ids
                ).delete()

    @synchronous
    def _updateService(self, region_obj):
        """Update the service status for this region."""
        Service.objects.create_services_for(region_obj)
        number_of_processes = len(self.connections)
        not_running_count = workers.MAX_WORKERS_COUNT - number_of_processes
        if not_running_count > 0:
            if number_of_processes == 1:
                process_text = "process"
            else:
                process_text = "processes"
            Service.objects.update_service_for(
                region_obj,
                "regiond",
                SERVICE_STATUS.DEGRADED,
                "%d %s running but %d were expected."
                % (
                    number_of_processes,
                    process_text,
                    workers.MAX_WORKERS_COUNT,
                ),
            )
        else:
            Service.objects.update_service_for(
                region_obj, "regiond", SERVICE_STATUS.RUNNING, ""
            )

    @synchronous
    @transactional
    def _update(self):
        """Repopulate the database with process, endpoint, and connection
        information."""
        # Get the region controller and update its hostname and last
        # updated time.
        region_obj = RegionController.objects.get_running_controller()
        hostname = gethostname()
        if region_obj.hostname != hostname:
            region_obj.hostname = hostname
            region_obj.save()

        # Get all the existing processes for the region controller. This is
        # used to remove the old processes that we did not update.
        previous_process_ids = set(
            RegionControllerProcess.objects.filter(
                region=region_obj
            ).values_list("id", flat=True)
        )

        # Loop through all the current workers to update the records in the
        # database. Caution is needed because other region controllers can
        # remove expired processes.
        for pid, conn in self.connections.items():
            process = self._getProcessObjFor(pid)
            process.updated = now()
            process.save()
            if conn["rpc"]["port"]:
                # Update the endpoints for the provided port.
                self._updateEndpoints(
                    process, self._getListenAddresses(conn["rpc"]["port"])
                )
            else:
                # RPC is not running, no endpoints.
                self._updateEndpoints(process, [])
            self._updateConnections(process, conn["rpc"]["connections"])
            previous_process_ids.discard(process.id)

        # Delete all the old processes that are dead.
        if previous_process_ids:
            RegionControllerProcess.objects.filter(
                id__in=previous_process_ids
            ).delete()

        # Remove any old processes not owned by this controller. Every
        # controller should update its processes based on the `UPDATE_INTERVAL`
        # any that are older than `REMOVE_INTERVAL` are dropped.
        remove_before_time = now() - timedelta(seconds=self.REMOVE_INTERVAL)
        RegionControllerProcess.objects.exclude(region=region_obj).filter(
            updated__lte=remove_before_time
        ).delete()

        # Update the status of this regiond service for this region based on
        # the number of running processes.
        self._updateService(region_obj)

        # Update the status of all regions that have no processes running.
        for other_region in RegionController.objects.exclude(
            system_id=region_obj.id
        ).prefetch_related("processes"):
            # Use len with `all` so the prefetch cache is used.
            if len(other_region.processes.all()) == 0:
                Service.objects.mark_dead(other_region, dead_region=True)

    @asynchronous
    def update(self):
        def ignore_cancel(failure):
            failure.trap(CancelledError)

        d = deferToDatabase(self._update)
        d.addErrback(ignore_cancel)
        d.addErrback(
            log.err,
            "Failed to update regiond's processes and endpoints; "
            "%s record's may be out of date" % (eventloop.loop.name,),
        )
        return d
Ejemplo n.º 53
0
class ApiAuths(YomboLibrary):
    """
    Session management.
    """
    def __init__(self, loader):  # we do some simulation of a Yombo Library...
        self.loader = loader
        self._FullName = "yombo.gateway.lib.webinterface.api_auth"
        self._Configs = self.loader.loadedLibraries['configuration']
        self._LocalDB = self.loader.loadedLibraries['localdb']
        self._Gateways = self.loader.loadedLibraries['gateways']
        self.gateway_id = self._Configs.get('core', 'gwid', 'local', False)
        self.active_api_auth = {}
        self.session_type = "apiauth"
        # self.active_api_auth_cache = ExpiringDict(200, 5)  # keep 200 entries, for at most 1 second...???

    def __delitem__(self, key):
        if key in self.active_api_auth:
            self.active_api_auth[key].expire_session()
        return

    def __getitem__(self, key):
        if key in self.active_api_auth:
            return self.active_api_auth[key]
        else:
            raise KeyError("Cannot find api auth key 2: %s" % key)

    def __len__(self):
        return len(self.active_api_auth)

    def __setitem__(self, key, value):
        raise YomboWarning("Cannot set a session using this method.")

    def __contains__(self, key):
        if key in self.active_api_auth:
            return True
        return False

    def keys(self):
        """
        Returns the keys (command ID's) that are configured.

        :return: A list of command IDs.
        :rtype: list
        """
        return list(self.active_api_auth.keys())

    def items(self):
        """
        Gets a list of tuples representing the commands configured.

        :return: A list of tuples.
        :rtype: list
        """
        return list(self.active_api_auth.items())

    # @inlineCallbacks
    def init(self):
        self._periodic_clean_sessions = LoopingCall(self.clean_sessions)
        self._periodic_clean_sessions.start(random_int(
            60 * 60 * 2,
            .7))  # Every 60-ish seconds. Save to disk, or remove from memory.

    def _unload_(self):
        self.unload_deferred = Deferred()
        self.clean_sessions(self.unload_deferred)
        return self.unload_deferred

    def close(self, request):
        api_auth = self.get(request)
        api_auth.expire_session()

    @inlineCallbacks
    def get_all(self):
        """
        Returns the api auths from DB.

        :return: A list of command IDs.
        :rtype: list
        """
        yield self.clean_sessions(True)
        api_auths = yield self._LocalDB.get_api_auth()
        return api_auths

    @inlineCallbacks
    def get(self, request=None, api_auth_id=None):
        """
        Checks the request for an x-api-auth header and then tries to validate it.

        Returns True if everything is good, otherwise raises YomboWarning with
        status reason.

        :param request: The request instance.
        :return: bool
        """
        if api_auth_id is None and request is not None:
            # print("request: %s" % request)
            api_auth_id = bytes_to_unicode(request.getHeader(b'x-api-auth'))
            if api_auth_id is None:
                try:
                    api_auth_id = request.args.get('_api_auth')[0]
                except:
                    api_auth_id = None
        if api_auth_id is None:
            raise YomboWarning("x-api-auth header is missing or blank.")
        if self.validate_api_auth_id(api_auth_id) is False:
            raise YomboWarning("x-api-auth header has invalid characters.")

        if api_auth_id in self.active_api_auth:
            if self.active_api_auth[api_auth_id].is_valid is True:
                return self.active_api_auth[api_auth_id]
            else:
                raise YomboWarning("x-api-auth header is no longer valid.")
        else:
            logger.debug("has_session is looking in database for session...")
            try:
                db_api_auth = yield self._LocalDB.get_api_auth(api_auth_id)
            except Exception as e:
                raise YomboWarning("x-api-auth is not valid")
            # logger.debug("has_session - found in DB! {db_session}", db_session=db_session)
            self.active_api_auth[api_auth_id] = ApiAuth(self,
                                                        db_api_auth,
                                                        source='database')
            return self.active_api_auth[api_auth_id]
        raise YomboWarning("x-api-auth header is invalid, other reasons.")

    def create(self, request=None, data=None):
        """
        Creates a new session.
        :param request:
        :param make_active: If True or None (default), then store sesion in memory.
        :return:
        """
        if data is None:
            data = {}
        if 'gateway_id' not in data or data['gateway_id'] is None:
            data['gateway_id'] = self.gateway_id
        if 'api_auth_id' not in data or data['api_auth_id'] is None:
            data['api_auth_id'] = random_string(length=randint(50, 55))
        if 'api_auth_data' not in data:
            data['api_auth_data'] = {}

        self.active_api_auth[data['api_auth_id']] = ApiAuth(self, data)
        return self.active_api_auth[data['api_auth_id']]

    @inlineCallbacks
    def update(self, id=None, data=None):
        """
        Updates an API Auth key

        :param request:
        :return:
        """
        if id is None or data is None:
            return
        api_auth = yield self.get(api_auth_id=id)
        if api_auth is None or api_auth is False:
            raise YomboWarning("API Auth ID doesn't exist")
        api_auth.update_attributes(data)

    def validate_api_auth_id(self, api_auth_id):
        """
        Validate the session id to make sure it's reasonable.
        :param api_auth_id: 
        :return: 
        """
        if api_auth_id == "LOGOFF":  # lets not raise an error.
            return True
        if api_auth_id.isalnum() is False:
            return False
        if len(api_auth_id) < 30:
            return False
        if len(api_auth_id) > 60:
            return False
        return True

    @inlineCallbacks
    def clean_sessions(self, close_deferred=None):
        """
        Called by loopingcall.

        Cleanup the stored sessions
        """
        for api_auth_id in list(self.active_api_auth.keys()):
            if self.active_api_auth[api_auth_id].check_valid(
            ) is False or self.active_api_auth[api_auth_id].is_valid is False:
                logger.debug("Removing invalid api auth: %s" % api_auth_id)
                del self.active_api_auth[api_auth_id]
                yield self._LocalDB.delete_api_auth(api_auth_id)

        for api_auth_id in list(self.active_api_auth):
            session = self.active_api_auth[api_auth_id]
            if session.is_dirty >= 200 or close_deferred is not None or session.last_access < int(
                    time() - (60 * 5)):
                if session.in_db:
                    logger.debug("updating old db api auth record: {id}",
                                 id=api_auth_id)
                    yield self._LocalDB.update_api_auth(session)
                else:
                    logger.debug("creating new db api auth record: {id}",
                                 id=api_auth_id)
                    yield self._LocalDB.save_api_auth(session)
                    session.in_db = True
                session.is_dirty = 0
                if session.last_access < int(
                        time() -
                    (60 * 60 * 3)):  # delete session from memory after 3 hours
                    logger.debug("Deleting session from memory: {api_auth_id}",
                                 api_auth_id=api_auth_id)
                    del self.active_api_auth[api_auth_id]

        # print("api auth clean sessions...: %s" % close_deferred)
        if close_deferred is not None and close_deferred is not True and close_deferred is not False:
            yield sleep(0.1)
            close_deferred.callback(1)
Ejemplo n.º 54
0
    def __init__(self,
                 svnurl,
                 split_file=None,
                 svnuser=None,
                 svnpasswd=None,
                 pollinterval=10 * 60,
                 histmax=100,
                 svnbin='svn'):
        """
        @type  svnurl: string
        @param svnurl: the SVN URL that describes the repository and
                       subdirectory to watch. If this ChangeSource should
                       only pay attention to a single branch, this should
                       point at the repository for that branch, like
                       svn://svn.twistedmatrix.com/svn/Twisted/trunk . If it
                       should follow multiple branches, point it at the
                       repository directory that contains all the branches
                       like svn://svn.twistedmatrix.com/svn/Twisted and also
                       provide a branch-determining function.

                       Each file in the repository has a SVN URL in the form
                       (SVNURL)/(BRANCH)/(FILEPATH), where (BRANCH) could be
                       empty or not, depending upon your branch-determining
                       function. Only files that start with (SVNURL)/(BRANCH)
                       will be monitored. The Change objects that are sent to
                       the Schedulers will see (FILEPATH) for each modified
                       file.

        @type  split_file: callable or None
        @param split_file: a function that is called with a string of the
                           form (BRANCH)/(FILEPATH) and should return a tuple
                           (BRANCH, FILEPATH). This function should match
                           your repository's branch-naming policy. Each
                           changed file has a fully-qualified URL that can be
                           split into a prefix (which equals the value of the
                           'svnurl' argument) and a suffix; it is this suffix
                           which is passed to the split_file function.

                           If the function returns None, the file is ignored.
                           Use this to indicate that the file is not a part
                           of this project.
                           
                           For example, if your repository puts the trunk in
                           trunk/... and branches are in places like
                           branches/1.5/..., your split_file function could
                           look like the following (this function is
                           available as svnpoller.split_file_branches)::

                            pieces = path.split('/')
                            if pieces[0] == 'trunk':
                                return (None, '/'.join(pieces[1:]))
                            elif pieces[0] == 'branches':
                                return ('/'.join(pieces[0:2]),
                                        '/'.join(pieces[2:]))
                            else:
                                return None

                           If instead your repository layout puts the trunk
                           for ProjectA in trunk/ProjectA/... and the 1.5
                           branch in branches/1.5/ProjectA/..., your
                           split_file function could look like::

                            pieces = path.split('/')
                            if pieces[0] == 'trunk':
                                branch = None
                                pieces.pop(0) # remove 'trunk'
                            elif pieces[0] == 'branches':
                                pieces.pop(0) # remove 'branches'
                                # grab branch name
                                branch = 'branches/' + pieces.pop(0)
                            else:
                                return None # something weird
                            projectname = pieces.pop(0)
                            if projectname != 'ProjectA':
                                return None # wrong project
                            return (branch, '/'.join(pieces))

                           The default of split_file= is None, which
                           indicates that no splitting should be done. This
                           is equivalent to the following function::

                            return (None, path)

                           If you wish, you can override the split_file
                           method with the same sort of function instead of
                           passing in a split_file= argument.


        @type  svnuser:      string
        @param svnuser:      If set, the --username option will be added to
                             the 'svn log' command. You may need this to get
                             access to a private repository.
        @type  svnpasswd:    string
        @param svnpasswd:    If set, the --password option will be added.

        @type  pollinterval: int
        @param pollinterval: interval in seconds between polls. The default
                             is 600 seconds (10 minutes). Smaller values
                             decrease the latency between the time a change
                             is recorded and the time the buildbot notices
                             it, but it also increases the system load.

        @type  histmax:      int
        @param histmax:      maximum number of changes to look back through.
                             The default is 100. Smaller values decrease
                             system load, but if more than histmax changes
                             are recorded between polls, the extra ones will
                             be silently lost.

        @type  svnbin:       string
        @param svnbin:       path to svn binary, defaults to just 'svn'. Use
                             this if your subversion command lives in an
                             unusual location.
        """

        if svnurl.endswith("/"):
            svnurl = svnurl[:-1]  # strip the trailing slash
        self.svnurl = svnurl
        self.split_file_function = split_file or split_file_alwaystrunk
        self.svnuser = svnuser
        self.svnpasswd = svnpasswd

        self.svnbin = svnbin
        self.pollinterval = pollinterval
        self.histmax = histmax
        self._prefix = None
        self.overrun_counter = 0
        self.loop = LoopingCall(self.checksvn)
Ejemplo n.º 55
0
class AirsharkManager(object):
    def __init__(self):
        self.loop = LoopingCall(self.check_spectrum)
        self.wifi_interface = None
        self.scanner = None
        self.analyzer_process = AnalyzerProcessProtocol(self)
        self.spectrum_observers = []
        self.analyzer_observers = []

        airshark_interface_manager.add_observer(self)

    def status(self):
        hardware_ready = False
        software_ready = False
        airshark_running = False
        if self.wifi_interface:
            hardware_ready = True

        if pdos.exists(settings.AIRSHARK_INSTALL_DIR):
            software_ready = True

        if self.analyzer_process.isRunning():
            airshark_running = True

        return (hardware_ready, software_ready, airshark_running)

    def on_interface_up(self, interface):
        self._stop()
        self.wifi_interface = interface
        self.scanner = Scanner(self.wifi_interface)
        self._start()

    def on_interface_down(self, interface):
        self._stop()
        self.wifi_interface = None
        self.scanner = None

    def _start(self):
        self.scanner.cmd_chanscan()
        self.scanner.start()

        if not self.analyzer_process.isRunning():
            out.info("Launching airshark analyzer")
            cmd = [settings.AIRSHARK_INSTALL_DIR + "/analyzer", settings.AIRSHARK_INSTALL_DIR + "/specshape/specshape_v1_722N_5m.txt", "--spectrum-fd=3", "--output-fd=4"]
            spawnProcess(self.analyzer_process,\
                         cmd[0], cmd, env=None, \
                         childFDs={0:"w", 1:"r", 2:2, 3:"w", 4:"r"})

        self.loop.start(0.2)
        return True

    def _stop(self):
        if self.scanner:
            self.scanner.stop()

        if self.analyzer_process.isRunning():
            self.analyzer_process.stop()

        if self.loop.running:
            self.loop.stop()

    def check_spectrum(self):
        # The bandwidth of the data is about 160k Bytes per second
        ts, data = self.scanner.spectrum_reader.read_samples()
        if ((data is not None) and (len(self.spectrum_observers) > 0 or self.analyzer_process.isRunning())):
            if len(self.spectrum_observers) > 0:
                #for (tsf, max_exp, freq, rssi, noise, max_mag, max_index, bitmap_weight, sdata) in SpectrumReader.decode(data):
                #    for observer in self.spectrum_observers:
                #        observer.on_spectrum_data(tsf, max_exp, freq, rssi, noise, max_mag, max_index, bitmap_weight, sdata)

                # Due to performance issue, we have to delegate the packet decoding task to clients
                # for packet in SpectrumReader.decode(data):
                #    for observer in self.spectrum_observers:
                #        observer.on_spectrum_data(packet)
                for observer in self.spectrum_observers:
                    observer.on_spectrum_data(data)

            if self.analyzer_process.isRunning():
                # Forward spectrum data to the airshark analyzer
                self.analyzer_process.feedSpectrumData(data)

    # TODO: Not sure we need it or not
    def read_raw_samples(self):
        if (self.scanner):
            return self.scanner.spectrum_reader.read_samples()
        else:
            return None, None

    def on_analyzer_message(self, message):
        for observer in self.analyzer_observers:
            observer.on_analyzer_message(message)

    def add_spectrum_observer(self, observer):
        if (self.spectrum_observers.count(observer) == 0):
            self.spectrum_observers.append(observer)

    def remove_spectrum_observer(self, observer):
        if (self.spectrum_observers.count(observer) == 1):
            self.spectrum_observers.remove(observer)

    def add_analyzer_observer(self, observer):
        if (self.analyzer_observers.count(observer) == 0):
            self.analyzer_observers.append(observer)

    def remove_analyzer_observer(self, observer):
        if (self.analyzer_observers.count(observer) == 1):
            self.analyzer_observers.remove(observer)
Ejemplo n.º 56
0
 def openDev(self):
     from twisted.internet.task import LoopingCall
     self.LC = LoopingCall(self._push_up_some_data)
     self.LC.start(0.020)
     self._data = ''
Ejemplo n.º 57
0
class NormalPMTFlow(LabradServer):

    name = 'NormalPMTFlow Dual'
    onNewCount = Signal(SIGNALID, 'signal: new count', 'v')
    onNewCount2 = Signal(SIGNALID + 10, 'signal: new count 2', 'v')
    onNewSetting = Signal(SIGNALID + 1, 'signal: new setting', '(ss)')
    onNewState = Signal(SIGNALID + 2, 'signal: new state', '(ss)')
    # onNewState2 = Signal(SIGNALID+12, 'signal: new state 2', '(ss)')

    @inlineCallbacks
    def initServer(self):
        self.saveFolder = ['', 'PMT Counts']
        self.dataSetName = 'PMT Counts'
        self.modes = ['Normal', 'Differential']
        self.collection_period = T.Value(0.100, 's')
        self.lastDifferential = {'ON': 0, 'OFF': 0}
        self.currentMode = 'Normal'
        self.dv = None
        self.pulser = None
        self.collectTimeRange = None
        self.openDataSet = None
        self.recordingInterrupted = False
        self.requestList = []
        self.listeners = set()
        self.recording = LoopingCall(self._record)
        yield self.connect_data_vault()
        yield self.connect_pulser()
        yield self.setupListeners()

    @inlineCallbacks
    def setupListeners(self):
        yield self.client.manager.subscribe_to_named_message(
            'Server Connect', 9898989, True)
        yield self.client.manager.subscribe_to_named_message(
            'Server Disconnect', 9898989 + 1, True)
        yield self.client.manager.addListener(
            listener=self.followServerConnect, source=None, ID=9898989)
        yield self.client.manager.addListener(
            listener=self.followServerDisconnect, source=None, ID=9898989 + 1)

    @inlineCallbacks
    def followServerConnect(self, cntx, serverName):
        serverName = serverName[1]
        if serverName == 'Pulser':
            yield self.connect_pulser()
        elif serverName == 'Data Vault':
            yield self.connect_data_vault()

    @inlineCallbacks
    def followServerDisconnect(self, cntx, serverName):
        serverName = serverName[1]
        if serverName == 'Pulser':
            yield self.disconnect_pulser()
        elif serverName == 'Data Vault':
            yield self.disconnect_data_vault()

    @inlineCallbacks
    def connect_data_vault(self):
        try:
            #reconnect to data vault and navigate to the directory
            self.dv = yield self.client.data_vault
            yield self.dv.cd(self.saveFolder, True)
            if self.openDataSet is not None:
                self.openDataSet = yield self.makeNewDataSet(
                    self.saveFolder, self.dataSetName)
                self.onNewSetting(('dataset', self.openDataSet))
            print 'Connected: Data Vault'
        except AttributeError:
            self.dv = None
            print 'Not Connected: Data Vault'

    @inlineCallbacks
    def disconnect_data_vault(self):
        print 'Not Connected: Data Vault'
        self.dv = None
        yield None

    @inlineCallbacks
    def connect_pulser(self):
        try:
            self.pulser = yield self.client.pulser
            self.collectTimeRange = yield self.pulser.get_collection_time()
            pmt_id_list = yield self.pulser.get_pmt_id_list()
            self.pmt_list = [PMT(pmt_id) for pmt_id in pmt_id_list]
            self.pmt_list[0].enable()
            if self.recordingInterrupted:
                yield self.dorecordData()
                self.onNewState('on')
                self.recordingInterrupted = False
            print 'Connected: Pulser'
        except AttributeError:
            self.pulser = None
            print 'Not Connected: Pulser'

    @inlineCallbacks
    def disconnect_pulser(self):
        print 'Not Connected: Pulser'
        self.pulser = None
        if self.recording.running:
            yield self.recording.stop()
            self.onNewState('off')
            self.recordingInterrupted = True

    def initContext(self, c):
        """Initialize a new context object."""
        self.listeners.add(c.ID)

    def expireContext(self, c):
        self.listeners.remove(c.ID)

    def getOtherListeners(self, c):
        notified = self.listeners.copy()
        notified.remove(c.ID)
        return notified

    @inlineCallbacks
    def makeNewDataSet(self, folder, name):

        for pmt in self.pmt_list:
            pmt.context = self.dv.context()
            yield self.dv.cd(folder, True, context=pmt.context)
            newSet = yield self.dv.new(
                name + pmt.suffix, [('t', 'num')],
                [('KiloCounts/sec', '866 ON', 'num'),
                 ('KiloCounts/sec', '866 OFF', 'num'),
                 ('KiloCounts/sec', 'Differential Signal', 'num')],
                context=pmt.context)
        self.startTime = time.time()
        yield self.addParameters(self.startTime)
        name = newSet[1][:-2]
        returnValue(name)

    @inlineCallbacks
    def addParameters(self, start):
        for pmt in self.pmt_list:
            yield self.dv.add_parameter("Window", ["PMT Counts"],
                                        context=pmt.context)
            yield self.dv.add_parameter('plotLive', True, context=pmt.context)
            yield self.dv.add_parameter('startTime',
                                        start,
                                        context=pmt.context)

    @setting(0, 'Set Save Folder', folder='*s', returns='')
    def setSaveFolder(self, c, folder):
        yield self.dv.cd(folder, True)
        self.saveFolder = folder

    @setting(1, 'Start New Dataset', setName='s', returns='s')
    def setNewDataSet(self, c, setName=None):
        """Starts new dataset, if name not provided, it will be the same"""
        if setName is not None: self.dataSetName = setName
        self.openDataSet = yield self.makeNewDataSet(self.saveFolder,
                                                     self.dataSetName)
        otherListeners = self.getOtherListeners(c)
        self.onNewSetting(('dataset', self.openDataSet), otherListeners)
        returnValue(self.openDataSet)

    @setting(2, "Set Mode", mode='s', returns='')
    def setMode(self, c, mode):
        """
        Start recording Time Resolved Counts into Data Vault
        """
        if mode not in self.modes: raise Exception('Incorrect Mode')
        if not self.recording.running:
            self.currentMode = mode
            yield self.pulser.set_mode(mode)
        else:
            yield self.dostopRecording()
            self.currentMode = mode
            yield self.dorecordData()
        otherListeners = self.getOtherListeners(c)
        self.onNewSetting(('mode', mode), otherListeners)

    @setting(3, 'getCurrentMode', returns='s')
    def getCurrentMode(self, c):
        """
        Returns the currently running mode
        """
        return self.currentMode

    @setting(4, 'Record Data', returns='')
    def recordData(self, c):
        """
        Starts recording data of the current PMT mode into datavault
        """
        setname = yield self.dorecordData()
        otherListeners = self.getOtherListeners(c)
        if setname is not None:
            setname = setname[1]
            self.onNewSetting(('dataset', setname), otherListeners)
        self.onNewState('on', otherListeners)

    @inlineCallbacks
    def dorecordData(self):
        #begins the process of data record
        #sets the collection time and mode, programs the pulser if necessary and opens the dataset if necessasry
        #then starts the recording loop
        newSet = None
        self.keepRunning = True
        yield self.pulser.set_collection_time(self.collection_period,
                                              self.currentMode)
        yield self.pulser.set_mode(self.currentMode)
        if self.currentMode == 'Differential':
            yield self._programPulserDiff()
        if self.openDataSet is None:
            self.openDataSet = yield self.makeNewDataSet(
                self.saveFolder, self.dataSetName)
        self.recording.start(self.collection_period['s'] / 2.0)
        returnValue(newSet)

    @setting(5, returns='')
    def stopRecording(self, c):
        """
        Stop recording counts into Data Vault
        """
        yield self.dostopRecording()
        otherListeners = self.getOtherListeners(c)
        self.onNewState('off', otherListeners)

    @inlineCallbacks
    def dostopRecording(self):
        yield self.recording.stop()
        if self.currentMode == 'Differential':
            yield self._stopPulserDiff()

    @setting(6, returns='b')
    def isRunning(self, c):
        """
        Returns whether or not currently recording
        """
        return self.recording.running

    @setting(7, returns='s')
    def currentDataSet(self, c):
        if self.openDataSet is None: return ''
        return self.openDataSet

    @setting(8, 'Set Time Length', timelength='v[s]')
    def setTimeLength(self, c, timelength):
        """Sets the time length for the current mode"""
        mode = self.currentMode
        if not self.collectTimeRange[0] <= timelength[
                's'] <= self.collectTimeRange[1]:
            raise Exception('Incorrect Recording Time')
        self.collection_period = timelength
        initrunning = self.recording.running  #if recording when the call is made, need to stop and restart
        if initrunning:
            yield self.recording.stop()
        yield self.pulser.set_collection_time(timelength['s'], mode)
        if initrunning:
            if mode == 'Differential':
                yield self._stopPulserDiff()
                yield self._programPulserDiff()
            self.recording.start(timelength['s'] / 2.0)
        otherListeners = self.getOtherListeners(c)
        self.onNewSetting(('timelength', str(timelength['s'])), otherListeners)

    @setting(9,
             'Get Next Counts',
             kind='s',
             number='w',
             average='b',
             returns=['*v', 'v'])
    def getNextCounts(self, c, kind, number, average=False):
        """
        Acquires next number of counts, where type can be 'ON' or 'OFF' or 'DIFF'
        Average is optionally True if the counts should be averaged
        
        Note in differential mode, Diff counts get updates every time, but ON and OFF
        get updated every 2 times.
        """
        if kind not in ['ON', 'OFF', 'DIFF']: raise Exception('Incorrect type')
        if kind in ['OFF', 'DIFF'] and self.currentMode == 'Normal':
            raise Exception('in the wrong mode to process this request')
        if not 0 < number < 1000: raise Exception('Incorrect Number')
        if not self.recording.running:
            raise Exception('Not currently recording')
        d = Deferred()
        self.requestList.append(self.readingRequest(d, kind, number))
        data = yield d
        if average:
            data = sum(data) / len(data)
        returnValue(data)

    @setting(10, 'Get Time Length', returns='v')
    def getMode(self, c):
        """
        Returns the current timelength of in the current mode
        """
        return self.collection_period

    @setting(11, 'Get Time Length Range', returns='(vv)')
    def get_time_length_range(self, c):
        if self.collectTimeRange is not None:
            return self.collectTimeRange
        else:
            raise Exception(
                "Not available because Pulser Server is not available")

    @setting(12, 'Set PMT State', pmt_id='i', state='b', returns='')
    def enablePMT(self, c, pmt_id, state):
        self.pmt_list[pmt_id - 1].enabled = state

    @inlineCallbacks
    def _programPulserDiff(self):
        yield self.pulser.new_sequence()
        yield self.pulser.add_ttl_pulse('DiffCountTrigger', T.Value(0.0, 'us'),
                                        T.Value(10.0, 'us'))
        yield self.pulser.add_ttl_pulse('DiffCountTrigger',
                                        self.collection_period,
                                        T.Value(10.0, 'us'))
        yield self.pulser.add_ttl_pulse('866DP', T.Value(0.0, 'us'),
                                        self.collection_period)
        yield self.pulser.extend_sequence_length(2 * self.collection_period)
        yield self.pulser.program_sequence()
        yield self.pulser.start_infinite()

    @inlineCallbacks
    def _stopPulserDiff(self):
        yield self.pulser.complete_infinite_iteration()
        yield self.pulser.wait_sequence_done()
        yield self.pulser.stop_sequence()

    class readingRequest():
        def __init__(self, d, kind, count):
            self.d = d
            self.count = count
            self.kind = kind
            self.data = []

        def is_fulfilled(self):
            return len(self.data) == self.count

    def processRequests(self, data):
        if not len(self.requestList): return
        for dataPoint in data:
            for item, req in enumerate(self.requestList):
                if dataPoint[1] != 0 and req.kind == 'ON':
                    req.data.append(dataPoint[1])
                if dataPoint[2] != 0 and req.kind == 'OFF':
                    req.data.append(dataPoint[2])
                if dataPoint[3] != 0 and req.kind == 'DIFF':
                    req.data.append(dataPoint[3])
                if req.is_fulfilled():
                    req.d.callback(req.data)
                    del (self.requestList[item])

    @inlineCallbacks
    def _record(self):
        for pmt in self.pmt_list:
            if not pmt.enabled: return
            try:
                yield self.getPMTCounts(pmt.id)
            except:
                print 'Not Able to Get PMT Counts'
                self.rawdata = []
            if len(self.rawdata) != 0:
                if self.currentMode == 'Normal':
                    toDataVault = [
                        [elem[2] - self.startTime, elem[0], 0, 0]
                        for elem in self.rawdata
                    ]  # converting to format [time, normal count, 0 , 0]
                elif self.currentMode == 'Differential':
                    toDataVault = self.convertDifferential(self.rawdata)
                self.processRequests(
                    toDataVault)  #if we have any requests, process them
                self.processSignals(toDataVault, pmt.id)
                try:
                    yield self.dv.add(toDataVault, context=pmt.context)
                except:
                    print 'Not Able to Save To Data Vault'

    @inlineCallbacks
    def getPMTCounts(self, pmt_id):
        if pmt_id == 2:
            self.rawdata = yield self.pulser.get_secondary_pmt_counts()
        else:
            self.rawdata = yield self.pulser.get_pmt_counts()

    def processSignals(self, data, pmt_id):
        lastPt = data[-1]
        NormalCount = lastPt[1]
        if pmt_id == 2: self.onNewCount2(NormalCount)
        else: self.onNewCount(NormalCount)

    def convertDifferential(self, rawdata):
        totalData = []
        for dataPoint in rawdata:
            t = str(dataPoint[1])
            self.lastDifferential[t] = float(dataPoint[0])
            diff = self.lastDifferential['ON'] - self.lastDifferential['OFF']
            totalData.append([
                dataPoint[2] - self.startTime, self.lastDifferential['ON'],
                self.lastDifferential['OFF'], diff
            ])
        return totalData
Ejemplo n.º 58
0
class SVNPoller(base.ChangeSource, util.ComparableMixin):
    """This source will poll a Subversion repository for changes and submit
    them to the change master."""

    compare_attrs = [
        "svnurl", "split_file_function", "svnuser", "svnpasswd",
        "pollinterval", "histmax", "svnbin"
    ]

    parent = None  # filled in when we're added
    last_change = None
    loop = None
    working = False

    def __init__(self,
                 svnurl,
                 split_file=None,
                 svnuser=None,
                 svnpasswd=None,
                 pollinterval=10 * 60,
                 histmax=100,
                 svnbin='svn'):
        """
        @type  svnurl: string
        @param svnurl: the SVN URL that describes the repository and
                       subdirectory to watch. If this ChangeSource should
                       only pay attention to a single branch, this should
                       point at the repository for that branch, like
                       svn://svn.twistedmatrix.com/svn/Twisted/trunk . If it
                       should follow multiple branches, point it at the
                       repository directory that contains all the branches
                       like svn://svn.twistedmatrix.com/svn/Twisted and also
                       provide a branch-determining function.

                       Each file in the repository has a SVN URL in the form
                       (SVNURL)/(BRANCH)/(FILEPATH), where (BRANCH) could be
                       empty or not, depending upon your branch-determining
                       function. Only files that start with (SVNURL)/(BRANCH)
                       will be monitored. The Change objects that are sent to
                       the Schedulers will see (FILEPATH) for each modified
                       file.

        @type  split_file: callable or None
        @param split_file: a function that is called with a string of the
                           form (BRANCH)/(FILEPATH) and should return a tuple
                           (BRANCH, FILEPATH). This function should match
                           your repository's branch-naming policy. Each
                           changed file has a fully-qualified URL that can be
                           split into a prefix (which equals the value of the
                           'svnurl' argument) and a suffix; it is this suffix
                           which is passed to the split_file function.

                           If the function returns None, the file is ignored.
                           Use this to indicate that the file is not a part
                           of this project.
                           
                           For example, if your repository puts the trunk in
                           trunk/... and branches are in places like
                           branches/1.5/..., your split_file function could
                           look like the following (this function is
                           available as svnpoller.split_file_branches)::

                            pieces = path.split('/')
                            if pieces[0] == 'trunk':
                                return (None, '/'.join(pieces[1:]))
                            elif pieces[0] == 'branches':
                                return ('/'.join(pieces[0:2]),
                                        '/'.join(pieces[2:]))
                            else:
                                return None

                           If instead your repository layout puts the trunk
                           for ProjectA in trunk/ProjectA/... and the 1.5
                           branch in branches/1.5/ProjectA/..., your
                           split_file function could look like::

                            pieces = path.split('/')
                            if pieces[0] == 'trunk':
                                branch = None
                                pieces.pop(0) # remove 'trunk'
                            elif pieces[0] == 'branches':
                                pieces.pop(0) # remove 'branches'
                                # grab branch name
                                branch = 'branches/' + pieces.pop(0)
                            else:
                                return None # something weird
                            projectname = pieces.pop(0)
                            if projectname != 'ProjectA':
                                return None # wrong project
                            return (branch, '/'.join(pieces))

                           The default of split_file= is None, which
                           indicates that no splitting should be done. This
                           is equivalent to the following function::

                            return (None, path)

                           If you wish, you can override the split_file
                           method with the same sort of function instead of
                           passing in a split_file= argument.


        @type  svnuser:      string
        @param svnuser:      If set, the --username option will be added to
                             the 'svn log' command. You may need this to get
                             access to a private repository.
        @type  svnpasswd:    string
        @param svnpasswd:    If set, the --password option will be added.

        @type  pollinterval: int
        @param pollinterval: interval in seconds between polls. The default
                             is 600 seconds (10 minutes). Smaller values
                             decrease the latency between the time a change
                             is recorded and the time the buildbot notices
                             it, but it also increases the system load.

        @type  histmax:      int
        @param histmax:      maximum number of changes to look back through.
                             The default is 100. Smaller values decrease
                             system load, but if more than histmax changes
                             are recorded between polls, the extra ones will
                             be silently lost.

        @type  svnbin:       string
        @param svnbin:       path to svn binary, defaults to just 'svn'. Use
                             this if your subversion command lives in an
                             unusual location.
        """

        if svnurl.endswith("/"):
            svnurl = svnurl[:-1]  # strip the trailing slash
        self.svnurl = svnurl
        self.split_file_function = split_file or split_file_alwaystrunk
        self.svnuser = svnuser
        self.svnpasswd = svnpasswd

        self.svnbin = svnbin
        self.pollinterval = pollinterval
        self.histmax = histmax
        self._prefix = None
        self.overrun_counter = 0
        self.loop = LoopingCall(self.checksvn)

    def split_file(self, path):
        # use getattr() to avoid turning this function into a bound method,
        # which would require it to have an extra 'self' argument
        f = getattr(self, "split_file_function")
        return f(path)

    def startService(self):
        log.msg("SVNPoller(%s) starting" % self.svnurl)
        base.ChangeSource.startService(self)
        # Don't start the loop just yet because the reactor isn't running.
        # Give it a chance to go and install our SIGCHLD handler before
        # spawning processes.
        reactor.callLater(0, self.loop.start, self.pollinterval)

    def stopService(self):
        log.msg("SVNPoller(%s) shutting down" % self.svnurl)
        self.loop.stop()
        return base.ChangeSource.stopService(self)

    def describe(self):
        return "SVNPoller watching %s" % self.svnurl

    def checksvn(self):
        # Our return value is only used for unit testing.

        # we need to figure out the repository root, so we can figure out
        # repository-relative pathnames later. Each SVNURL is in the form
        # (ROOT)/(PROJECT)/(BRANCH)/(FILEPATH), where (ROOT) is something
        # like svn://svn.twistedmatrix.com/svn/Twisted (i.e. there is a
        # physical repository at /svn/Twisted on that host), (PROJECT) is
        # something like Projects/Twisted (i.e. within the repository's
        # internal namespace, everything under Projects/Twisted/ has
        # something to do with Twisted, but these directory names do not
        # actually appear on the repository host), (BRANCH) is something like
        # "trunk" or "branches/2.0.x", and (FILEPATH) is a tree-relative
        # filename like "twisted/internet/defer.py".

        # our self.svnurl attribute contains (ROOT)/(PROJECT) combined
        # together in a way that we can't separate without svn's help. If the
        # user is not using the split_file= argument, then self.svnurl might
        # be (ROOT)/(PROJECT)/(BRANCH) . In any case, the filenames we will
        # get back from 'svn log' will be of the form
        # (PROJECT)/(BRANCH)/(FILEPATH), but we want to be able to remove
        # that (PROJECT) prefix from them. To do this without requiring the
        # user to tell us how svnurl is split into ROOT and PROJECT, we do an
        # 'svn info --xml' command at startup. This command will include a
        # <root> element that tells us ROOT. We then strip this prefix from
        # self.svnurl to determine PROJECT, and then later we strip the
        # PROJECT prefix from the filenames reported by 'svn log --xml' to
        # get a (BRANCH)/(FILEPATH) that can be passed to split_file() to
        # turn into separate BRANCH and FILEPATH values.

        # whew.

        if self.working:
            log.msg("SVNPoller(%s) overrun: timer fired but the previous "
                    "poll had not yet finished.")
            self.overrun_counter += 1
            return defer.succeed(None)
        self.working = True

        log.msg("SVNPoller polling")
        if not self._prefix:
            # this sets self._prefix when it finishes. It fires with
            # self._prefix as well, because that makes the unit tests easier
            # to write.
            d = self.get_root()
            d.addCallback(self.determine_prefix)
        else:
            d = defer.succeed(self._prefix)

        d.addCallback(self.get_logs)
        d.addCallback(self.parse_logs)
        d.addCallback(self.get_new_logentries)
        d.addCallback(self.create_changes)
        d.addCallback(self.submit_changes)
        d.addCallbacks(self.finished_ok, self.finished_failure)
        return d

    def getProcessOutput(self, args):
        # this exists so we can override it during the unit tests
        d = utils.getProcessOutput(self.svnbin, args, {})
        return d

    def get_root(self):
        args = ["info", "--xml", "--non-interactive", self.svnurl]
        if self.svnuser:
            args.extend(["--username=%s" % self.svnuser])
        if self.svnpasswd:
            args.extend(["--password=%s" % self.svnpasswd])
        d = self.getProcessOutput(args)
        return d

    def determine_prefix(self, output):
        try:
            doc = xml.dom.minidom.parseString(output)
        except xml.parsers.expat.ExpatError:
            dbgMsg("_process_changes: ExpatError in %s" % output)
            log.msg("SVNPoller._determine_prefix_2: ExpatError in '%s'" %
                    output)
            raise
        rootnodes = doc.getElementsByTagName("root")
        if not rootnodes:
            # this happens if the URL we gave was already the root. In this
            # case, our prefix is empty.
            self._prefix = ""
            return self._prefix
        rootnode = rootnodes[0]
        root = "".join([c.data for c in rootnode.childNodes])
        # root will be a unicode string
        _assert(
            self.svnurl.startswith(root),
            "svnurl='%s' doesn't start with <root>='%s'" % (self.svnurl, root))
        self._prefix = self.svnurl[len(root):]
        if self._prefix.startswith("/"):
            self._prefix = self._prefix[1:]
        log.msg("SVNPoller: svnurl=%s, root=%s, so prefix=%s" %
                (self.svnurl, root, self._prefix))
        return self._prefix

    def get_logs(self, ignored_prefix=None):
        args = []
        args.extend(["log", "--xml", "--verbose", "--non-interactive"])
        if self.svnuser:
            args.extend(["--username=%s" % self.svnuser])
        if self.svnpasswd:
            args.extend(["--password=%s" % self.svnpasswd])
        args.extend(["--limit=%d" % (self.histmax), self.svnurl])
        d = self.getProcessOutput(args)
        return d

    def parse_logs(self, output):
        # parse the XML output, return a list of <logentry> nodes
        try:
            doc = xml.dom.minidom.parseString(output)
        except xml.parsers.expat.ExpatError:
            dbgMsg("_process_changes: ExpatError in %s" % output)
            log.msg("SVNPoller._parse_changes: ExpatError in '%s'" % output)
            raise
        logentries = doc.getElementsByTagName("logentry")
        return logentries

    def _filter_new_logentries(self, logentries, last_change):
        # given a list of logentries, return a tuple of (new_last_change,
        # new_logentries), where new_logentries contains only the ones after
        # last_change
        if not logentries:
            # no entries, so last_change must stay at None
            return (None, [])

        mostRecent = int(logentries[0].getAttribute("revision"))

        if last_change is None:
            # if this is the first time we've been run, ignore any changes
            # that occurred before now. This prevents a build at every
            # startup.
            log.msg('svnPoller: starting at change %s' % mostRecent)
            return (mostRecent, [])

        if last_change == mostRecent:
            # an unmodified repository will hit this case
            log.msg('svnPoller: _process_changes last %s mostRecent %s' %
                    (last_change, mostRecent))
            return (mostRecent, [])

        new_logentries = []
        for el in logentries:
            if last_change == int(el.getAttribute("revision")):
                break
            new_logentries.append(el)
        new_logentries.reverse()  # return oldest first
        return (mostRecent, new_logentries)

    def get_new_logentries(self, logentries):
        last_change = self.last_change
        (new_last_change,
         new_logentries) = self._filter_new_logentries(logentries,
                                                       self.last_change)
        self.last_change = new_last_change
        log.msg('svnPoller: _process_changes %s .. %s' %
                (last_change, new_last_change))
        return new_logentries

    def _get_text(self, element, tag_name):
        child_nodes = element.getElementsByTagName(tag_name)[0].childNodes
        text = "".join([t.data for t in child_nodes])
        return text

    def _transform_path(self, path):
        _assert(
            path.startswith(self._prefix),
            "filepath '%s' should start with prefix '%s'" %
            (path, self._prefix))
        relative_path = path[len(self._prefix):]
        if relative_path.startswith("/"):
            relative_path = relative_path[1:]
        where = self.split_file(relative_path)
        # 'where' is either None or (branch, final_path)
        return where

    def create_changes(self, new_logentries):
        changes = []

        for el in new_logentries:
            branch_files = []  # get oldest change first
            # TODO: revisit this, I think I've settled on Change.revision
            # being a string everywhere, and leaving the interpretation
            # of that string up to b.s.source.SVN methods
            revision = int(el.getAttribute("revision"))
            dbgMsg("Adding change revision %s" % (revision, ))
            # TODO: the rest of buildbot may not be ready for unicode 'who'
            # values
            author = self._get_text(el, "author")
            comments = self._get_text(el, "msg")
            # there is a "date" field, but it provides localtime in the
            # repository's timezone, whereas we care about buildmaster's
            # localtime (since this will get used to position the boxes on
            # the Waterfall display, etc). So ignore the date field and use
            # our local clock instead.
            #when     = self._get_text(el, "date")
            #when     = time.mktime(time.strptime("%.19s" % when,
            #                                     "%Y-%m-%dT%H:%M:%S"))
            branches = {}
            pathlist = el.getElementsByTagName("paths")[0]
            for p in pathlist.getElementsByTagName("path"):
                path = "".join([t.data for t in p.childNodes])
                # the rest of buildbot is certaily not yet ready to handle
                # unicode filenames, because they get put in RemoteCommands
                # which get sent via PB to the buildslave, and PB doesn't
                # handle unicode.
                path = path.encode("ascii")
                if path.startswith("/"):
                    path = path[1:]
                where = self._transform_path(path)
                # if 'where' is None, the file was outside any project that
                # we care about and we should ignore it
                if where:
                    branch, filename = where
                    if not branch in branches:
                        branches[branch] = []
                    branches[branch].append(filename)

            for branch in branches:
                c = Change(who=author,
                           files=branches[branch],
                           comments=comments,
                           revision=revision,
                           branch=branch)
                changes.append(c)

        return changes

    def submit_changes(self, changes):
        for c in changes:
            self.parent.addChange(c)

    def finished_ok(self, res):
        log.msg("SVNPoller finished polling")
        dbgMsg('_finished : %s' % res)
        assert self.working
        self.working = False
        return res

    def finished_failure(self, f):
        log.msg("SVNPoller failed")
        dbgMsg('_finished : %s' % f)
        assert self.working
        self.working = False
        return None  # eat the failure
Ejemplo n.º 59
0
class TelnetProtocol(Telnet, StatefulTelnetProtocol, Session):
    """
    Each player connecting over telnet (ie using most traditional mud
    clients) gets a telnet protocol instance assigned to them.  All
    communication between game and player goes through here.
    """
    def __init__(self, *args, **kwargs):
        self.protocol_name = "telnet"
        super(TelnetProtocol, self).__init__(*args, **kwargs)

    def connectionMade(self):
        """
        This is called when the connection is first established.

        """
        # initialize the session
        self.line_buffer = []
        self.iaw_mode = False
        self.no_lb_mode = False
        client_address = self.transport.client
        client_address = client_address[0] if client_address else None
        # this number is counted down for every handshake that completes.
        # when it reaches 0 the portal/server syncs their data
        self.handshakes = 7 # naws, ttype, mccp, mssp, msdp, gmcp, mxp
        self.init_session(self.protocol_name, client_address, self.factory.sessionhandler)

        # negotiate client size
        self.naws = naws.Naws(self)
        # negotiate ttype (client info)
        # Obs: mudlet ttype does not seem to work if we start mccp before ttype. /Griatch
        self.ttype = ttype.Ttype(self)
        # negotiate mccp (data compression) - turn this off for wireshark analysis
        self.mccp = Mccp(self)
        # negotiate mssp (crawler communication)
        self.mssp = mssp.Mssp(self)
        # oob communication (MSDP, GMCP) - two handshake calls!
        self.oob = telnet_oob.TelnetOOB(self)
        # mxp support
        self.mxp = Mxp(self)
        # add this new connection to sessionhandler so
        # the Server becomes aware of it.
        self.sessionhandler.connect(self)

        # timeout the handshakes in case the client doesn't reply at all
        from evennia.utils.utils import delay
        delay(2, callback=self.handshake_done, retval=True)

        # TCP/IP keepalive watches for dead links
        self.transport.setTcpKeepAlive(1)
        # The TCP/IP keepalive is not enough for some networks;
        # we have to complement it with a NOP keep-alive.
        self.protocol_flags["NOPKEEPALIVE"] = True
        self.nop_keep_alive = None
        self.toggle_nop_keepalive()

    def _send_nop_keepalive(self):
        "Send NOP keepalive unless flag is set"
        if self.protocol_flags.get("NOPKEEPALIVE"):
            self._write(IAC + NOP)

    def toggle_nop_keepalive(self):
        """
        Allow to toggle the NOP keepalive for those sad clients that
        can't even handle a NOP instruction. This is turned off by the
        protocol_flag NOPKEEPALIVE (settable e.g. by the default
        `@option` command).
        """
        if self.nop_keep_alive and self.nop_keep_alive.running:
            self.nop_keep_alive.stop()
        else:
            self.nop_keep_alive = LoopingCall(self._send_nop_keepalive)
            self.nop_keep_alive.start(30, now=False)

    def handshake_done(self, force=False):
        """
        This is called by all telnet extensions once they are finished.
        When all have reported, a sync with the server is performed.
        The system will force-call this sync after a small time to handle
        clients that don't reply to handshakes at all.
        """
        if self.handshakes > 0:
            if force:
                self.sessionhandler.sync(self)
                return
            self.handshakes -= 1
            if self.handshakes <= 0:
                # do the sync
                self.sessionhandler.sync(self)

    def enableRemote(self, option):
        """
        This sets up the remote-activated options we allow for this protocol.

        Args:
            option (char): The telnet option to enable.

        Returns:
            enable (bool): If this option should be enabled.

        """
        return (option == LINEMODE or
                option == ttype.TTYPE or
                option == naws.NAWS or
                option == MCCP or
                option == mssp.MSSP)

    def enableLocal(self, option):
        """
        Call to allow the activation of options for this protocol

        Args:
            option (char): The telnet option to enable locally.

        Returns:
            enable (bool): If this option should be enabled.

        """
        return (option == MCCP or option==ECHO)

    def disableLocal(self, option):
        """
        Disable a given option locally.

        Args:
            option (char): The telnet option to disable locally.

        """
        if option == ECHO:
            return True
        if option == MCCP:
            self.mccp.no_mccp(option)
            return True
        else:
            return super(TelnetProtocol, self).disableLocal(option)

    def connectionLost(self, reason):
        """
        this is executed when the connection is lost for whatever
        reason. it can also be called directly, from the disconnect
        method

        Args:
            reason (str): Motivation for losing connection.

        """
        self.sessionhandler.disconnect(self)
        self.transport.loseConnection()

    def dataReceived(self, data):
        """
        Handle incoming data over the wire.

        This method will split the incoming data depending on if it
        starts with IAC (a telnet command) or not. All other data will
        be handled in line mode. Some clients also sends an erroneous
        line break after IAC, which we must watch out for.

        Args:
            data (str): Incoming data.

        Notes:
            OOB protocols (MSDP etc) already intercept subnegotiations on
            their own, never entering this method. They will relay their
            parsed data directly to self.data_in.

        """
        if data and data[0] == IAC or self.iaw_mode:
            try:
                super(TelnetProtocol, self).dataReceived(data)
                if len(data) == 1:
                    self.iaw_mode = True
                else:
                    self.iaw_mode = False
                return
            except Exception as err1:
                conv = ""
                try:
                    for b in data:
                        conv += " " + repr(ord(b))
                except Exception as err2:
                    conv = str(err2) + ":", str(data)
                out = "Telnet Error (%s): %s (%s)" % (err1, data, conv)
                logger.log_trace(out)
                return

        if self.no_lb_mode and _RE_LEND.search(data):
            # we are in no_lb_mode and receive a line break;
            # this means we should empty the buffer and send
            # the command.
            data = "".join(self.line_buffer) + data
            data = data.rstrip("\r\n") + "\n"
            self.line_buffer = []
            self.no_lb_mode = False
        elif not _RE_LEND.search(data):
            # no line break at the end of the command, buffer instead.
            self.line_buffer.append(data)
            self.no_lb_mode = True
            return

        # if we get to this point the command should end with a linebreak.
        # We make sure to add it, to fix some clients messing this up.
        StatefulTelnetProtocol.dataReceived(self, data)

    def _write(self, data):
        "hook overloading the one used in plain telnet"
        data = data.replace('\n', '\r\n').replace('\r\r\n', '\r\n')
        #data = data.replace('\n', '\r\n')
        super(TelnetProtocol, self)._write(mccp_compress(self, data))

    def sendLine(self, line):
        """
        Hook overloading the one used by linereceiver.

        Args:
            line (str): Line to send.

        """
        #escape IAC in line mode, and correctly add \r\n
        line += self.delimiter
        line = line.replace(IAC, IAC + IAC).replace('\n', '\r\n')
        return self.transport.write(mccp_compress(self, line))

    def lineReceived(self, string):
        """
        Telnet method called when data is coming in over the telnet
        connection. We pass it on to the game engine directly.

        Args:
            string (str): Incoming data.

        """
        self.data_in(text=string)

    # Session hooks

    def disconnect(self, reason=None):
        """
        generic hook for the engine to call in order to
        disconnect this protocol.

        Args:
            reason (str): Reason for disconnecting.

        """
        self.data_out(text=((reason or "",), {}))
        self.connectionLost(reason)

    def data_in(self, **kwargs):
        """
        Data User -> Evennia

        Kwargs:
            kwargs (any): Options from the protocol.

        """
        #from evennia.server.profiling.timetrace import timetrace
        #text = timetrace(text, "telnet.data_in")

        self.sessionhandler.data_in(self, **kwargs)

    def data_out(self, **kwargs):
        """
        Data Evennia -> User

        Kwargs:
            kwargs (any): Options to the protocol
        """
        self.sessionhandler.data_out(self, **kwargs)

    # send_* methods

    def send_text(self, *args, **kwargs):
        """
        Send text data. This is an in-band telnet operation.

        Args:
            text (str): The first argument is always the text string to send. No other arguments
                are considered.
        Kwargs:
            options (dict): Send-option flags
                   - mxp: Enforce MXP link support.
                   - ansi: Enforce no ANSI colors.
                   - xterm256: Enforce xterm256 colors, regardless of TTYPE.
                   - noxterm256: Enforce no xterm256 color support, regardless of TTYPE.
                   - nomarkup: Strip all ANSI markup. This is the same as noxterm256,noansi
                   - raw: Pass string through without any ansi processing
                        (i.e. include Evennia ansi markers but do not
                        convert them into ansi tokens)
                   - echo: Turn on/off line echo on the client. Turn
                        off line echo for client, for example for password.
                        Note that it must be actively turned back on again!

        """
        text = args[0] if args else ""
        if text is None:
            return
        text = to_str(text, force_string=True)

        # handle arguments
        options = kwargs.get("options", {})
        flags = self.protocol_flags
        xterm256 = options.get("xterm256", flags.get('XTERM256', False) if flags["TTYPE"] else True)
        useansi = options.get("ansi", flags.get('ANSI', False) if flags["TTYPE"] else True)
        raw = options.get("raw", flags.get("RAW", False))
        nomarkup = options.get("nomarkup", flags.get("NOMARKUP", not (xterm256 or useansi)))
        echo = options.get("echo", None)
        mxp = options.get("mxp", flags.get("MXP", False))
        screenreader =  options.get("screenreader", flags.get("SCREENREADER", False))

        if screenreader:
            # screenreader mode cleans up output
            text = ansi.parse_ansi(text, strip_ansi=True, xterm256=False, mxp=False)
            text = _RE_SCREENREADER_REGEX.sub("", text)

        if options.get("send_prompt"):
            # send a prompt instead.
            if not raw:
                # processing
                prompt = ansi.parse_ansi(_RE_N.sub("", text) + "{n", strip_ansi=nomarkup, xterm256=xterm256)
                if mxp:
                    prompt = mxp_parse(prompt)
            prompt = prompt.replace(IAC, IAC + IAC).replace('\n', '\r\n')
            prompt += IAC + GA
            self.transport.write(mccp_compress(self, prompt))
        else:
            if echo is not None:
                # turn on/off echo. Note that this is a bit turned around since we use
                # echo as if we are "turning off the client's echo" when telnet really
                # handles it the other way around.
                if echo:
                    # by telling the client that WE WON'T echo, the client knows
                    # that IT should echo. This is the expected behavior from
                    # our perspective.
                    self.transport.write(mccp_compress(self, IAC+WONT+ECHO))
                else:
                    # by telling the client that WE WILL echo, the client can
                    # safely turn OFF its OWN echo.
                    self.transport.write(mccp_compress(self, IAC+WILL+ECHO))
            if raw:
                # no processing
                self.sendLine(text)
                return
            else:
                # we need to make sure to kill the color at the end in order
                # to match the webclient output.
                linetosend = ansi.parse_ansi(_RE_N.sub("", text) + "{n", strip_ansi=nomarkup, xterm256=xterm256, mxp=mxp)
                if mxp:
                    linetosend = mxp_parse(linetosend)
                self.sendLine(linetosend)

    def send_prompt(self, *args, **kwargs):
        """
        Send a prompt - a text without a line end. See send_text for argument options.

        """
        kwargs["options"].update({"send_prompt": True})
        self.send_text(*args, **kwargs)


    def send_default(self, cmdname, *args, **kwargs):
        """
        Send other oob data
        """
        if not cmdname == "options":
            self.oob.data_out(cmdname, *args, **kwargs)
Ejemplo n.º 60
-1
class ServerController(object):
    def __init__(self, realm, view):
        print('server cont!')
        self.realm = realm
        self.view = view
        
    def _handleInput(self):
        """
        Handle currently available pygame input events.
        """
        for event in pygame.event.get():
            if (event.type == pygame.QUIT) or ((event.type == pygame.KEYDOWN) and (event.key == QUIT)):
                reactor.stop()
                sys.exit()
                
            if (event.type == pygame.KEYDOWN):
                if (event.key == START_GAME):
                    self.realm.environment.startGame()
                elif (event.key == RESET_GAME):
                    self.realm.environment.setPreGame()
    
    def go(self):
        self.previousTime = pygame.time.get_ticks()
        self._inputCall = LoopingCall(self._handleInput)
        d = self._inputCall.start(0.03)
        return d


    def stop(self):
        self._inputCall.stop()