class USBClient(Protocol): def __init__(self, network): self.network = network self.usb_list = [] self.lc = LoopingCall(self.timerEvent) self.serialBuffer = bytes("") def timerEvent(self): self.lc.stop() for cli in client_list: cli.transport.write(self.serialBuffer) self.serialBuffer = bytes("") def connectionFailed(self): log.msg ("Connection Failed:", self) reactor.stop() def connectionMade(self): usb_list.append(self) log.msg("Connected to %s device" % device) def dataReceived(self, data): #log.msg("data resrvr") self.serialBuffer += data if self.lc.running: self.lc.stop() self.lc.start(0.2, False) def sendLine(self, cmd): print cmd self.transport.write(cmd + "\r\n") def outReceived(self, data): self.data = self.data + data
class Rain(object): """ Make it rain. Rain only occurs during spring. """ implements(IAutomaton) blocks = tuple() def __init__(self): self.season_loop = LoopingCall(self.check_season) def scan(self, chunk): pass def feed(self, coords): pass def start(self): self.season_loop.start(5 * 60) def stop(self): self.season_loop.stop() def check_season(self): if factory.world.season.name == "spring": factory.vane.weather = "rainy" reactor.callLater(1 * 60, setattr, factory.vane, "weather", "sunny") name = "rain"
def modbus_master(module, properties): log.debug('Modbus master module : ' + str(module)) # Modbus Master #--------------------------------------------------------------------------# # initialize your data store #--------------------------------------------------------------------------# store = ModbusSlaveContext( co = ModbusSequentialDataBlock(0, [0]*100), hr = ModbusSequentialDataBlock(0, [0]*100)) context = ModbusServerContext(slaves=store, single=True) #--------------------------------------------------------------------------# # initialize the server information #--------------------------------------------------------------------------# identity = ModbusDeviceIdentification() identity.VendorName = 'ASO+AKO' identity.ProductCode = 'DYODE' identity.VendorUrl = 'yoloswag' identity.ProductName = 'DYODE' identity.ModelName = 'BSides LV release' identity.MajorMinorRevision = '0.9' #--------------------------------------------------------------------------# # run the server you want #--------------------------------------------------------------------------# time = 1 # 5 seconds delay loop = LoopingCall(f=modbus_master_update, a=(module, properties, context)) loop.start(time, now=False) # initially delay by time StartTcpServer(context, identity=identity, address=("0.0.0.0", \ properties['port_out']))
def build_reactor(options, **kwargs): web_socket_instance = FilteredWebSocketFactory(**kwargs) subscriber = kwargs.pop("subscriber", None) if options.key and options.cert: with open(options.key) as keyFile: with open(options.cert) as certFile: cert = ssl.PrivateCertificate.loadPEM(keyFile.read() + certFile.read()) reactor.listenSSL(options.port, web_socket_instance, cert.options()) else: reactor.listenTCP( options.port, web_socket_instance ) if subscriber is not None: reactor.callInThread( subscriber.listener, web_socket_instance ) reactor.addSystemEventTrigger( "before", "shutdown", subscriber.kill ) # Start the consumer loop consumer_loop = LoopingCall( web_socket_instance.consumer ) consumer_loop.start(0.001, now=False) return web_socket_instance
class PollingDataStream(DataStream): """ A self-polling data stream. This class represents a data stream that wakes up at a given frequency, and calls the :meth:`poll` method. """ frequency = None # Either a timedelta object, or the number of seconds now = False def __init__(self): super(PollingDataStream, self).__init__() self.timer = LoopingCall(self.poll) if isinstance(self.frequency, timedelta): seconds = ( self.frequency.seconds + (self.frequency.days * 24 * 60 * 60) + (self.frequency.microseconds / 1000000.0) ) else: seconds = self.frequency log.debug("Setting a %s second timer" % seconds) self.timer.start(seconds, now=self.now) def poll(self): raise NotImplementedError def stop(self): super(PollingDataStream, self).stop() try: if hasattr(self, "timer"): self.timer.stop() except Exception, e: self.log.warn(e)
class Pinger(object): """ An periodic AMP ping helper. """ def __init__(self, reactor): """ :param IReactorTime reactor: The reactor to use to schedule the pings. """ self.reactor = reactor def start(self, protocol, interval): """ Start sending some pings. :param AMP protocol: The protocol over which to send the pings. :param timedelta interval: The interval at which to send the pings. """ def ping(): protocol.callRemote(NoOp) self._pinging = LoopingCall(ping) self._pinging.clock = self.reactor self._pinging.start(interval.total_seconds(), now=False) def stop(self): """ Stop sending the pings. """ self._pinging.stop()
class LeaseService(Service): """ Manage leases. In particular, clear out expired leases once a second. :ivar _reactor: A ``IReactorTime`` provider. :ivar _persistence_service: The persistence service to act with. :ivar _lc: A ``twisted.internet.task.LoopingCall`` run every second to update the configured leases by releasing leases that have expired. """ def __init__(self, reactor, persistence_service): self._reactor = reactor self._persistence_service = persistence_service def startService(self): self._lc = LoopingCall(self._expire) self._lc.clock = self._reactor self._lc.start(1) def stopService(self): self._lc.stop() def _expire(self): now = datetime.fromtimestamp(self._reactor.seconds(), tz=UTC) def expire(leases): updated_leases = leases.expire(now) for dataset_id in set(leases) - set(updated_leases): _LOG_EXPIRE(dataset_id=dataset_id, node_id=leases[dataset_id].node_id).write() return updated_leases return update_leases(expire, self._persistence_service)
class BandwidthEstimator: bufsize = 20 totalBytes = 0 def __init__(self, message, length): self.length = length self.message = message self.estim = [] self.bytes = 0 self.call = LoopingCall(self.estimateBandwidth) self.call.start(1) def estimateBandwidth(self): bytes = self.bytes self.totalBytes += bytes self.estim.append(bytes) self.message("%0.2f k/s (%0.2d%%)" % ((sum(self.estim) / len(self.estim)) / 1024., (float(self.totalBytes) / self.length) * 100)) if len(self.estim) > self.bufsize: self.estim.pop(0) self.bytes = 0 def stop(self): self.call.stop() self.estimateBandwidth() self.message("Finished receiving: %d bytes (%d%%)" % ( self.totalBytes, (float(self.totalBytes) / self.length) * 100))
def openShell(self, transport): """ Write 60 lines of data to the transport, then exit. """ proto = protocol.Protocol() proto.makeConnection(transport) transport.makeConnection(wrapProtocol(proto)) # Send enough bytes to the connection so that a rekey is triggered in # the client. def write(counter): i = counter() if i == 60: call.stop() transport.session.conn.sendRequest( transport.session, 'exit-status', '\x00\x00\x00\x00') transport.loseConnection() else: transport.write("line #%02d\n" % (i,)) # The timing for this loop is an educated guess (and/or the result of # experimentation) to exercise the case where a packet is generated # mid-rekey. Since the other side of the connection is (so far) the # OpenSSH command line client, there's no easy way to determine when the # rekey has been initiated. If there were, then generating a packet # immediately at that time would be a better way to test the # functionality being tested here. call = LoopingCall(write, count().next) call.start(0.01)
def __init__(self): self.output("TWS init") self.username = environ["TXTRADER_USERNAME"] self.password = environ["TXTRADER_PASSWORD"] self.xmlrpc_port = int(environ["TXTRADER_XMLRPC_PORT"]) self.tcp_port = int(environ["TXTRADER_TCP_PORT"]) self.callback_timeout = int(environ["TXTRADER_TWS_CALLBACK_TIMEOUT"]) if not self.callback_timeout: self.callback_timeout = DEFAULT_TWS_CALLBACK_TIMEOUT self.output("callback_timeout=%d" % self.callback_timeout) self.enable_ticker = bool(int(environ["TXTRADER_ENABLE_TICKER"])) self.label = "TWS Gateway" self.channel = "tws" self.current_account = "" self.clients = set([]) self.orders = {} self.pending_orders = {} self.openorder_callbacks = [] self.accounts = [] self.account_data = {} self.positions = {} self.position_callbacks = [] self.executions = {} self.execution_callbacks = [] self.bardata_callbacks = [] self.cancel_callbacks = [] self.order_callbacks = [] self.addsymbol_callbacks = [] self.accountdata_callbacks = [] self.last_connection_status = "" self.connection_status = "Initializing" self.LastError = -1 self.next_order_id = -1 self.last_minute = -1 self.handlers = { "error": self.error_handler, "tickSize": self.handle_tick_size, "tickPrice": self.handle_tick_price, "tickString": self.handle_tick_string, "nextValidId": self.handle_next_valid_id, "currentTime": self.handle_time, "managedAccounts": self.handle_accounts, "orderStatus": self.handle_order_status, "openOrder": self.handle_open_order, "openOrderEnd": self.handle_open_order_end, "execDetails": self.handle_exec_details, "execDetailsEnd": self.handle_exec_details_end, "position": self.handle_position, "positionEnd": self.handle_position_end, "historicalData": self.handle_historical_data, "updateAccountValue": self.handle_account_value, "accountDownloadEnd": self.handle_account_download_end, } self.ticker_ids = {} self.symbols = {} self.symbols_by_id = {} self.primary_exchange_map = {} self.tws_conn = None repeater = LoopingCall(self.EverySecond) repeater.start(1)
class _ExchangeRate(object): """Download an exchange rate from Yahoo Finance using Twisted.""" def __init__(self, name): self._value = None self._name = name # External API: def latest_value(self): """Return the latest exchange rate value. May be None if no value is available. """ return self._value def start(self): """Start the background process.""" self._lc = LoopingCall(self._download) # Run immediately, and then every 30 seconds: self._lc.start(30, now=True) def _download(self): """Download the page.""" print("Downloading!") def parse(result): print("Got %r back from Yahoo." % (result,)) values = result.strip().split(",") self._value = float(values[1]) d = getPage( "http://download.finance.yahoo.com/d/quotes.csv?e=.csv&f=c4l1&s=%s=X" % (self._name,)) d.addCallback(parse) d.addErrback(log.err) return d
class SendsManyFileDescriptors(ConnectableProtocol): paused = False def connectionMade(self): self.socket = socket() self.transport.registerProducer(self, True) def sender(): self.transport.sendFileDescriptor(self.socket.fileno()) self.transport.write(b"x") self.task = LoopingCall(sender) self.task.clock = self.transport.reactor self.task.start(0).addErrback(err, "Send loop failure") def stopProducing(self): self._disconnect() def resumeProducing(self): self._disconnect() def pauseProducing(self): self.paused = True self.transport.unregisterProducer() self._disconnect() def _disconnect(self): self.task.stop() self.transport.abortConnection() self.other.transport.abortConnection()
def make_lc(self, reactor, func): if DEBUG: self.stdout_length = 0 self.stderr_length = 0 def _(lc, reactor): if DEBUG: stdout = self.stdout.getvalue() stderr = self.stderr.getvalue() if self.stdout.getvalue()[self.stdout_length:]: print(self.stdout.getvalue()[self.stdout_length:], file=sys.__stdout__) if self.stderr.getvalue()[self.stderr_length:]: print(self.stderr.getvalue()[self.stderr_length:], file=sys.__stderr__) self.stdout_length = len(stdout) self.stderr_length = len(stderr) return func(lc, reactor) lc = LoopingCall(_) lc.a = (lc, reactor) lc.clock = reactor lc.start(0.1) return lc
class ICUMonitorFactory(protocol.ClientFactory): def __init__(self, gui_msg_callback, gui_ip, gui_port): self.ip = gui_ip self.port = gui_port self.msg_callback = gui_msg_callback self.objmonit = None def buildProtocol(self, addr): return ICUMonitor(self.msg_callback) def clientConnectionLost(self, connector, reason): pass def clientConnectionFailed(self, connector, reason): pass def sendmsg(self, gui_reactor): gui_reactor.connectTCP(self.ip, self.port, self) def startsend(self, gui_reactor, timerep): self.loop = LoopingCall(self.sendmsg, gui_reactor) self.loop.start(timerep) def stopsend(self): self.loop.stop()
class ProvisionerQueryService(ServiceProcess): """Provisioner querying service """ declare = ServiceProcess.service_declare(name='provisioner_query', version='0.1.0', dependencies=[]) def slc_init(self): interval = float(self.spawn_args.get("interval_seconds", DEFAULT_QUERY_INTERVAL)) self.client = ProvisionerClient(self) log.debug('Starting provisioner query loop - %s second interval', interval) self.loop = LoopingCall(self.query) self.loop.start(interval) def slc_terminate(self): if self.loop: self.loop.stop() @defer.inlineCallbacks def query(self): try: yield self._do_query() except Exception,e: log.error("Error sending provisioner query request: %s", e, exc_info=True)
class CacheService(Service): def __init__(self, *args, **kwargs): self.call = None self.node_updater = NodeCacheUpdater() self.cluster_updater = ClusterCacheUpdater() self.vm_updater = VirtualMachineCacheUpdater() self.job_updater = JobCacheUpdater() def update_cache(self): """ a single run of all update classes """ return DeferredList( [ self.vm_updater.update(), self.job_updater.update(), self.node_updater.update(), self.cluster_updater.update(), ] ) def startService(self): self.call = LoopingCall(self.update_cache) self.call.start(settings.PERIODIC_CACHE_REFRESH) def stopService(self): if self.call is not None: self.call.stop()
class EventGenerator(Node): """ (TEST) A timer-based event generator """ # --- Initialization def __init__(self): self.count = 0 # --- Interfaces def configure(self): self.events = ( 'event1', 'event2', ) def setup(self): self.status.set_GREEN() self.loop1 = LoopingCall(self.loop_cb1) self.loop1.start(3, False) self.loop2 = LoopingCall(self.loop_cb2) self.loop2.start(4, False) # --- Workers def loop_cb1(self): self.count += 1 self.sendEvent('event1', self.count) def loop_cb2(self): self.count += 1 self.sendEvent('event2', self.count)
def onJoin(self, details): log.msg("PiMonitor connected") extra = self.config.extra self._id = extra['id'] self.publish_temperature = True self.threshold = 0 self._tick = 0 self._cpu_temp_celsius = None def scanTemperature(): self._cpu_temp_celsius = float(open("/sys/class/thermal/thermal_zone0/temp").read()) / 1000. if self.publish_temperature: self.publish(u'io.crossbar.examples.pi.{}.tempmon.on_temperature'.format(self._id), self._tick, self._cpu_temp_celsius) self._tick += 1 if self.threshold > 0 and self._cpu_temp_celsius > self.threshold: self.publish(u'io.crossbar.examples.pi.{}.tempmon.on_threshold_exceeded'.format(self._id), self._tick, self._cpu_temp_celsius) scan = LoopingCall(scanTemperature) scan.start(1) for proc in [self.get_temperature, self.impose_stress, self.toggle_publish, self.set_threshold]: uri = u'io.crossbar.examples.pi.{}.tempmon.{}'.format(self._id, proc.__name__) yield self.register(proc, uri) log.msg("Registered {}".format(uri)) log.msg("PiMonitor ready.")
def __init__(self): """ Initialize the model. This doesn't add any documents yet. """ silenceGensim() self.dictionaries = dict() self.preprocessor = TokenizingPorter2Stemmer() #this dict keeps a model for every source type # (since e.g. RSS feeds should be treated separately from twitter feeds) self.models = dict() #this dict keeps a dictionary for every source type self.dictionaries = dict() self.queue = Queue() self.modelQueue = Queue() self.nodeCommunicator = NodeCommunicator(self, LISTEN_PORT) self.nodeCommunicator.registerWithNode(CORE_IP, REGISTER_PORT) # register this node with the core ln.info("LSA Initialized") self.updating = False loop = LoopingCall(self.update) loop.start(5) reactor.run()
def delayed_setup(self): self.feeds = [] self.failures.clear() self.feed_times.clear() self.targets.clear() self.tasks.clear() for name, target in self.config["targets"].items(): proto = target["protocol"] if proto in self.factory_manager.factories: self.targets[name] = target else: self.logger.warn("Unknown protocol '%s' in target '%s'" % (proto, name)) for feed in self.config["feeds"]: append = True for target in feed["targets"]: if target["name"] not in self.targets: self.logger.warn("Unknown target '%s' for feed '%s'" % (target["name"], feed["name"])) append = False break if append: self.feeds.append(feed) for feed in self.feeds: task = LoopingCall(self.check_feed, feed) self.tasks[feed["name"]] = task task.start(feed["frequency"])
class Hoster: def __init__(self, screen, port): self.screen = screen self.port = port def start(self): # Set up the connection between the state and the network; # using queues gives a degree of seperation between the # communication and game logic, and making them be deferred # keeps it all asynchronous. inqueue = DeferredQueue() outqueue = DeferredQueue() #Initialize GameState gs = GameState(self,self.screen,1,inqueue,outqueue) #Create Looping Call self.lc = LoopingCall(gs.gameLoop) self.lc.start(.0166666666) #Begin Listening connfactory = CommandConnFactory(inqueue,outqueue) self.listening = reactor.listenTCP(self.port,connfactory) def stop(self, nextscreen=None): # Stop the GameState logic, and let go of the port on which we're listening self.lc.stop() self.listening.stopListening() # Start up the next screen, if there is one if nextscreen: nextscreen.start() else: reactor.stop()
class List(set): def __init__(self, filename, url='', mode='MERGE', refreshPeriod=0): set.__init__(self) self.filename = filename self.url = url self.mode = mode self.load() if url != '' and refreshPeriod != 0: self.lc = LoopingCall(self.update) self.lc.start(refreshPeriod) def load(self): """ Load the list from the specified file. """ self.clear() #simple touch to create non existent files try: open(self.filename, 'a').close() with open(self.filename, 'r') as fh: for l in fh.readlines(): self.add(re.split("#", l)[0].rstrip("[ , \n,\t]")) except: pass def dump(self): """ Dump the list to the specified file. """ try: with open(self.filename, 'w') as fh: for l in self: fh.write(l + "\n") except: pass def handleData(self, data): if self.mode == 'REPLACE': self.clear() for elem in data.split('\n'): if elem != '': self.add(elem) def processData(self, data): try: if len(data) != 0: self.handleData(data) self.dump() except Exception: pass def update(self): pageFetchedDeferred = getPageCached(self.url) pageFetchedDeferred.addCallback(self.processData) return pageFetchedDeferred
class UdpSender(DatagramProtocol): """ Class that will send UDP packets to UDP Listener """ def __init__(self, host, count, size, duration): # LoopingCall does not know whether UDP socket is actually writable self.looper = None self.host = host self.count = count self.duration = duration self.start = time.time() self.sent = 0 self.data = array.array('c', 'X' * size) def startProtocol(self): self.looper = LoopingCall(self.sendData) period = self.duration / float(self.count) self.looper.start(period, now=False) def stopProtocol(self): if (self.looper is not None): self.looper.stop() self.looper = None def datagramReceived(self, data, (host, port)): pass
def begin_logging(self): def logging_callback(*args, **kwargs): print("number of players: " + str(len(self.clients))) print("number of lobbies: " + str(len(self.lobbys))) print("Total number of connects: " + str(self.total_clients)) looping_call = LoopingCall(logging_callback) looping_call.start(60.0*60.0) #every hour
def authenticate(self, twistedStanza): """authentication callback""" self._send = twistedStanza.send # add observers for incoming message / presence requests # twistedStanza.addObserver('/iq', self.iqHandler) # twistedStanza.addObserver('/presence', self.dialogHandler.presenceHandler) twistedStanza.addObserver("/presence", self.presenceHandler) twistedStanza.addObserver("//event/stream/error", self.streamErrorHandler) # twistedStanza.addObserver('//event/stream/error', self.streamErrorHandler) # twistedStanza.addObserver('/message', self.dialogHandler.messageHandler) twistedStanza.addObserver("/message", self.messageHandler) self.requestRoster() # let'em know we are online presence = domish.Element((JABBER_CLIENT_NS, "presence")) presence.addElement("status").addContent("Online") self.send(presence) keepalive = LoopingCall(self.loopEntry) keepalive.start(60) self.joinChatrooms(self.chatrooms)
def addSlave(self, slaveSpec): # try to catch a slave reconnection; else assign a new slave ID try: (connectedSlave, connectedSlaveStatus, connectedSlaveTask) = self._getSlaveBySlaveSpec(slaveSpec, asTuple=True) slaveId = self._getSlaveIdByObject(connectedSlave) log.warn("Slave %d (%s://%s:%d/%s) reconnecting. Resetting slave state!" % (slaveId, connectedSlave.slaveSpec.scheme, connectedSlave.slaveSpec.host, connectedSlave.slaveSpec.port, connectedSlave.slaveSpec.path)) except SlaveNotFound: slaveNo = self._getSlaveNo() else: slaveNo = slaveId slave = SlavePerspective(slaveSpec) status = SlaveState() status.state = SlaveState.CONNECTED # check that the slave is up and running request = self.checkHealth(slave) yield request if request.result != True: raise SlaveConnectionError else: status.state = SlaveState.IDLE # set up a heartbeat call task = LoopingCall(self.checkHealth, slave) task.start(60, now=False) self.slaves[slaveNo] = (slave, status, task) returnValue(slaveNo)
class LiveStream(WebSocketClientProtocol): # pragma: no cover """ Internal class used to call the websocket callbacks. """ sendqueue = [] sending = False def _sender(self): if len(self.sendqueue): while self.sendqueue: data = self.sendqueue.pop() self.sendMessage(data) def queueMessage(self, message): self.sendqueue.append(message) if not self.sending: self.sending = True self._loopingcall = LoopingCall(self._sender) self._loopingcall.start(0.1) def onOpen(self): self.factory.datasift['send_message'] = self.queueMessage self.factory.datasift['on_open']() def onClose(self, wasClean, code, reason): self.factory.datasift['on_close'](wasClean, code, reason) def onMessage(self, msg, binary): self.factory.datasift['on_message'](msg, binary) self.factory.resetDelay() def onPing(self, payload): self.factory.resetDelay()
class Blinker(object): def __init__(self, port, interval=0.5): self.port = port self.interval = interval self.state = None self.transition = None def set(self, *args, **kwargs): self.blink() def unset(self): self.transition_stop() self.port.unset() def blink(self): self.transition_stop() self.transition = LoopingCall(self.blink_task) self.transition.start(self.interval) def transition_stop(self): if self.transition and self.transition.running: self.transition.stop() def blink_task(self): if self.state: self.state = False self.port.set() else: self.state = True self.port.unset()
class NetworkSpeedDaemon(VMCDaemon): """ I enque fake SIG_SPEED UnsolicitedNotifications """ def __init__(self, frequency, device, notification_manager): super(NetworkSpeedDaemon, self).__init__(frequency, device, notification_manager) self.netspeed = NetworkSpeed() def start(self): log.msg("DAEMONS: DAEMON %s started..." % self.__class__.__name__) self.netspeed.start() self.loop = LoopingCall(self.function) self.loop.start(self.frequency) def stop(self): log.msg("DAEMONS: DAEMON %s stopped..." % self.__class__.__name__) self.loop.stop() self.netspeed.stop() def function(self): up, down = self.netspeed["up"], self.netspeed["down"] n = N.UnsolicitedNotification(N.SIG_SPEED, bps_to_human(up, down)) self.manager.on_notification_received(n)
def add(self, request, save_time = 0): self.data[request] = time.time() + save_time * 60 d = request.notifyFinish() d.addCallback(self.notifyFinished, request) d.addErrback(self.notifyFinished, request) call = LoopingCall(self.prune) call.start(60)
class TelnetProtocol(Telnet, StatefulTelnetProtocol, Session): """ Each player connecting over telnet (ie using most traditional mud clients) gets a telnet protocol instance assigned to them. All communication between game and player goes through here. """ def __init__(self, *args, **kwargs): self.protocol_name = "telnet" super(TelnetProtocol, self).__init__(*args, **kwargs) def connectionMade(self): """ This is called when the connection is first established. """ # initialize the session self.line_buffer = "" client_address = self.transport.client client_address = client_address[0] if client_address else None # this number is counted down for every handshake that completes. # when it reaches 0 the portal/server syncs their data self.handshakes = 7 # naws, ttype, mccp, mssp, msdp, gmcp, mxp self.init_session(self.protocol_name, client_address, self.factory.sessionhandler) # negotiate client size self.naws = naws.Naws(self) # negotiate ttype (client info) # Obs: mudlet ttype does not seem to work if we start mccp before ttype. /Griatch self.ttype = ttype.Ttype(self) # negotiate mccp (data compression) - turn this off for wireshark analysis self.mccp = Mccp(self) # negotiate mssp (crawler communication) self.mssp = mssp.Mssp(self) # oob communication (MSDP, GMCP) - two handshake calls! self.oob = telnet_oob.TelnetOOB(self) # mxp support self.mxp = Mxp(self) # add this new connection to sessionhandler so # the Server becomes aware of it. self.sessionhandler.connect(self) # timeout the handshakes in case the client doesn't reply at all from evennia.utils.utils import delay delay(2, callback=self.handshake_done, force=True) # TCP/IP keepalive watches for dead links self.transport.setTcpKeepAlive(1) # The TCP/IP keepalive is not enough for some networks; # we have to complement it with a NOP keep-alive. self.protocol_flags["NOPKEEPALIVE"] = True self.nop_keep_alive = None self.toggle_nop_keepalive() def _send_nop_keepalive(self): "Send NOP keepalive unless flag is set" if self.protocol_flags.get("NOPKEEPALIVE"): self._write(IAC + NOP) def toggle_nop_keepalive(self): """ Allow to toggle the NOP keepalive for those sad clients that can't even handle a NOP instruction. This is turned off by the protocol_flag NOPKEEPALIVE (settable e.g. by the default `@option` command). """ if self.nop_keep_alive and self.nop_keep_alive.running: self.nop_keep_alive.stop() else: self.nop_keep_alive = LoopingCall(self._send_nop_keepalive) self.nop_keep_alive.start(30, now=False) def handshake_done(self, force=False): """ This is called by all telnet extensions once they are finished. When all have reported, a sync with the server is performed. The system will force-call this sync after a small time to handle clients that don't reply to handshakes at all. """ if self.handshakes > 0: if force: self.sessionhandler.sync(self) return self.handshakes -= 1 if self.handshakes <= 0: # do the sync self.sessionhandler.sync(self) def enableRemote(self, option): """ This sets up the remote-activated options we allow for this protocol. Args: option (char): The telnet option to enable. Returns: enable (bool): If this option should be enabled. """ return (option == LINEMODE or option == ttype.TTYPE or option == naws.NAWS or option == MCCP or option == mssp.MSSP) def enableLocal(self, option): """ Call to allow the activation of options for this protocol Args: option (char): The telnet option to enable locally. Returns: enable (bool): If this option should be enabled. """ return (option == MCCP or option == ECHO) def disableLocal(self, option): """ Disable a given option locally. Args: option (char): The telnet option to disable locally. """ if option == ECHO: return True if option == MCCP: self.mccp.no_mccp(option) return True else: return super(TelnetProtocol, self).disableLocal(option) def connectionLost(self, reason): """ this is executed when the connection is lost for whatever reason. it can also be called directly, from the disconnect method Args: reason (str): Motivation for losing connection. """ self.sessionhandler.disconnect(self) self.transport.loseConnection() def applicationDataReceived(self, data): """ Telnet method called when non-telnet-command data is coming in over the telnet connection. We pass it on to the game engine directly. Args: string (str): Incoming data. """ if not data: data = [data] elif data.strip() == NULL: # this is an ancient type of keepalive used by some # legacy clients. There should never be a reason to send a # lone NULL character so this seems to be a safe thing to # support for backwards compatibility. It also stops the # NULL from continously popping up as an unknown command. data = [_IDLE_COMMAND] else: data = _RE_LINEBREAK.split(data) if self.line_buffer and len(data) > 1: # buffer exists, it is terminated by the first line feed data[0] = self.line_buffer + data[0] self.line_buffer = "" # if the last data split is empty, it means all splits have # line breaks, if not, it is unterminated and must be # buffered. self.line_buffer += data.pop() # send all data chunks for dat in data: self.data_in(text=dat + "\n") def _write(self, data): "hook overloading the one used in plain telnet" data = data.replace('\n', '\r\n').replace('\r\r\n', '\r\n') super(TelnetProtocol, self)._write(mccp_compress(self, data)) def sendLine(self, line): """ Hook overloading the one used by linereceiver. Args: line (str): Line to send. """ #escape IAC in line mode, and correctly add \r\n line += self.delimiter line = line.replace(IAC, IAC + IAC).replace('\n', '\r\n') return self.transport.write(mccp_compress(self, line)) # Session hooks def disconnect(self, reason=None): """ generic hook for the engine to call in order to disconnect this protocol. Args: reason (str): Reason for disconnecting. """ self.data_out(text=((reason or "", ), {})) self.connectionLost(reason) def data_in(self, **kwargs): """ Data User -> Evennia Kwargs: kwargs (any): Options from the protocol. """ #from evennia.server.profiling.timetrace import timetrace #text = timetrace(text, "telnet.data_in") self.sessionhandler.data_in(self, **kwargs) def data_out(self, **kwargs): """ Data Evennia -> User Kwargs: kwargs (any): Options to the protocol """ self.sessionhandler.data_out(self, **kwargs) # send_* methods def send_text(self, *args, **kwargs): """ Send text data. This is an in-band telnet operation. Args: text (str): The first argument is always the text string to send. No other arguments are considered. Kwargs: options (dict): Send-option flags - mxp: Enforce MXP link support. - ansi: Enforce no ANSI colors. - xterm256: Enforce xterm256 colors, regardless of TTYPE. - noxterm256: Enforce no xterm256 color support, regardless of TTYPE. - nomarkup: Strip all ANSI markup. This is the same as noxterm256,noansi - raw: Pass string through without any ansi processing (i.e. include Evennia ansi markers but do not convert them into ansi tokens) - echo: Turn on/off line echo on the client. Turn off line echo for client, for example for password. Note that it must be actively turned back on again! """ text = args[0] if args else "" if text is None: return text = to_str(text, force_string=True) # handle arguments options = kwargs.get("options", {}) flags = self.protocol_flags xterm256 = options.get( "xterm256", flags.get('XTERM256', False) if flags["TTYPE"] else True) useansi = options.get( "ansi", flags.get('ANSI', False) if flags["TTYPE"] else True) raw = options.get("raw", flags.get("RAW", False)) nomarkup = options.get( "nomarkup", flags.get("NOMARKUP", not (xterm256 or useansi))) echo = options.get("echo", None) mxp = options.get("mxp", flags.get("MXP", False)) screenreader = options.get("screenreader", flags.get("SCREENREADER", False)) if screenreader: # screenreader mode cleans up output text = ansi.parse_ansi(text, strip_ansi=True, xterm256=False, mxp=False) text = _RE_SCREENREADER_REGEX.sub("", text) if options.get("send_prompt"): # send a prompt instead. if not raw: # processing prompt = ansi.parse_ansi(_RE_N.sub("", text) + "{n", strip_ansi=nomarkup, xterm256=xterm256) if mxp: prompt = mxp_parse(prompt) prompt = prompt.replace(IAC, IAC + IAC).replace('\n', '\r\n') prompt += IAC + GA self.transport.write(mccp_compress(self, prompt)) else: if echo is not None: # turn on/off echo. Note that this is a bit turned around since we use # echo as if we are "turning off the client's echo" when telnet really # handles it the other way around. if echo: # by telling the client that WE WON'T echo, the client knows # that IT should echo. This is the expected behavior from # our perspective. self.transport.write(mccp_compress(self, IAC + WONT + ECHO)) else: # by telling the client that WE WILL echo, the client can # safely turn OFF its OWN echo. self.transport.write(mccp_compress(self, IAC + WILL + ECHO)) if raw: # no processing self.sendLine(text) return else: # we need to make sure to kill the color at the end in order # to match the webclient output. linetosend = ansi.parse_ansi(_RE_N.sub("", text) + "{n", strip_ansi=nomarkup, xterm256=xterm256, mxp=mxp) if mxp: linetosend = mxp_parse(linetosend) self.sendLine(linetosend) def send_prompt(self, *args, **kwargs): """ Send a prompt - a text without a line end. See send_text for argument options. """ kwargs["options"].update({"send_prompt": True}) self.send_text(*args, **kwargs) def send_default(self, cmdname, *args, **kwargs): """ Send other oob data """ if not cmdname == "options": self.oob.data_out(cmdname, *args, **kwargs)
class AdapterPmMetrics: class Metrics: def __init__(self, config, value): self.config = config self.value = value def __init__(self,device): self.pm_names = {'tx_64_pkts','tx_65_127_pkts', 'tx_128_255_pkts', 'tx_256_511_pkts', 'tx_512_1023_pkts', 'tx_1024_1518_pkts', 'tx_1519_9k_pkts', 'rx_64_pkts', 'rx_65_127_pkts', 'rx_128_255_pkts', 'rx_256_511_pkts', 'rx_512_1023_pkts', 'rx_1024_1518_pkts', 'rx_1519_9k_pkts', 'tx_pkts', 'rx_pkts', 'tx_bytes', 'rx_bytes'} # This is just to generate more realistic looking values. This would # not be implemented in a normal adapter. self.rand_ranges = dict ( tx_64_pkts=[50, 55], tx_65_127_pkts=[55,60], tx_128_255_pkts=[60,65], tx_256_511_pkts=[85,90], tx_512_1023_pkts=[90,95], tx_1024_1518_pkts=[60,65], tx_1519_9k_pkts=[50,55], rx_64_pkts=[50, 55], rx_65_127_pkts=[55,60], rx_128_255_pkts=[60,65], rx_256_511_pkts=[85,90], rx_512_1023_pkts=[90,95], rx_1024_1518_pkts=[60,65], rx_1519_9k_pkts=[50,55], tx_pkts=[90,100], rx_pkts=[90,100], rx_bytes=[90000,100000], tx_bytes=[90000,100000] ) self.device = device self.id = device.id self.default_freq = 150 self.grouped = False self.freq_override = False self.pon_metrics = dict() self.nni_metrics = dict() self.lc = None for m in self.pm_names: self.pon_metrics[m] = \ self.Metrics(config = PmConfig(name=m, type=PmConfig.COUNTER, enabled=True), value = 0) self.nni_metrics[m] = \ self.Metrics(config = PmConfig(name=m, type=PmConfig.COUNTER, enabled=True), value = 0) def update(self, pm_config): if self.default_freq != pm_config.default_freq: # Update the callback to the new frequency. self.default_freq = pm_config.default_freq self.lc.stop() self.lc.start(interval=self.default_freq/10) for m in pm_config.metrics: self.pon_metrics[m.name].config.enabled = m.enabled self.nni_metrics[m.name].config.enabled = m.enabled def make_proto(self): pm_config = PmConfigs( id=self.id, default_freq=self.default_freq, grouped = False, freq_override = False) for m in sorted(self.pon_metrics): pm=self.pon_metrics[m] pm_config.metrics.extend([PmConfig(name=pm.config.name, type=pm.config.type, enabled=pm.config.enabled)]) return pm_config def collect_pon_metrics(self): import random rtrn_pon_metrics = dict() for m in self.pm_names: if self.pon_metrics[m].config.enabled: self.pon_metrics[m].value += \ random.randint(self.rand_ranges[m][0], self.rand_ranges[m][1]) rtrn_pon_metrics[m] = self.pon_metrics[m].value return rtrn_pon_metrics def collect_nni_metrics(self): import random rtrn_nni_metrics = dict() for m in self.pm_names: if self.nni_metrics[m].config.enabled: self.nni_metrics[m].value += \ random.randint(self.rand_ranges[m][0], self.rand_ranges[m][1]) rtrn_nni_metrics[m] = self.nni_metrics[m].value return rtrn_nni_metrics def start_collector(self, device_name, device_id, callback): prefix = 'voltha.{}.{}'.format(device_name, device_id) self.lc = LoopingCall(callback, device_id, prefix) self.lc.start(interval=self.default_freq/10)
class ConsumerNexus(Nexus): def __init__(self, below_nexus, layer): super().__init__(below_nexus, layer) self.onping = None self.consumer_finished_cb = None self.handshake_finished = False self.ping_interval = None self.ping_start_time = None ########################################################################### def is_layer_message(self, msg): if msg['message_class'] != "NOTIFICATION": return False return msg['notification_name'] in { "NOTIFY_PROVIDER", "NOTIFY_PROVIDER_NOT_READY", "NOTIFY_PING" } def on_message(self, below_nexus, msg): logging.info("consumer nexus got msg") if not self.is_layer_message(msg): super().on_message(below_nexus, msg) return if msg['notification_name'] == "NOTIFY_PROVIDER": if not self.handshake_finished: self.handshake_finished = True # nexus is announced to the transact layer # TODO, this message might be better to handle in the transact # layer instead. Refactor might be needed. self.consumer_finished_cb(self) # pass the message above for the transact layer to process super().on_message(below_nexus, msg) elif msg['notification_name'] == "NOTIFY_PROVIDER_NOT_READY": logging.info("provider not ready, waiting") elif msg['notification_name'] == "NOTIFY_PONG": if not self.ping_start_time: return msecs = (time.time() - self.ping_start_time) * 1000 if self.onping: self.onping(self, round(msecs)) self.ping_start_time = None def on_bin_message(self, below_nexus, msg_bytes): logging.info("provider nexus got raw msg") super().on_bin_message(below_nexus, msg_bytes) ########################################################################### def start_handshake(self, consumer_finished_cb): self.consumer_finished_cb = consumer_finished_cb self.send(RequestProvider()) def send_ping(self): self.ping_start_time = time.time() self.send(RequestPing()) def start_pinging(self): logging.info("START PING") self.ping_interval = LoopingCall(self.send_ping) self.ping_interval.start(3.0, now=True) def stop_pinging(self): logging.info("STOP PING") if not self.ping_interval: return self.ping_interval.stop() self.ping_interval = None
class FabricNode(node.Node): """ Crossbar.io FX node personality. """ DEFAULT_CONFIG_PATH = 'edge/node/config/pairme.json' NODE_CONTROLLER = FabricNodeControllerSession def __init__(self, personality, cbdir=None, reactor=None, native_workers=None, options=None): node.Node.__init__(self, personality, cbdir, reactor, native_workers, options) # looping call that runs the node controller watchdog self._watchdog_looper = None # the node controller realm (where eg worker controller live). we explicitly use the # same realm as Crossbar.io OSS self._realm = 'crossbar' # enable docker daemon integration self._enable_docker = None # when running in managed mode, this will hold the bridge session # attached to the local management router self._bridge_session = None # if this node has a proper management uplink configured to connect to self._has_management_config = False # if this node was connected to its configured management uplink successfully at least once # during run-time (since last reboot) of this node self._was_management_connected = False # when we periodically check for a node activation file, the looping call for doing # the check - and automatically shutdown when an activation file was found (after boot) self._check_for_activation_file = None # when self._was_management_connected, the URL we've been connected to self._management_url = None # when running in managed mode, this will hold the management uplink session to # the crossbar master node self._manager = None self._manager_runner = None # the node's management realm when running in managed mode (this comes from CFC!) self._management_realm = None # the node's ID when running in managed mode (this comes from CFC!) self._node_id = None # node extra when running in managed mode (this comes from CFC!) self._node_extra = None # when the node starts up, it will connect to CFC, and then apply the # local node configuration, and this flag will be set. when the CFC connection # is lost, and then reestablished, the local node configuration should NOT # be applied a second time though - hence this flag self._local_config_applied = False # We really only need to see this once (?) self._displayed_pairing_message = False # for automatic ID assignment of "makers" within workers of type "xbrmm" self._maker_no = 1 def load_config(self, configfile=None): """ Check and load the node configuration from: * from ``.crossbar/config.json`` or * from built-in (empty) default configuration This is the _second_ function being called after the Node has been instantiated. IMPORTANT: this function is run _before_ start of Twisted reactor! """ config_source = None config_path = None # if the node hasn't been configured from XBR network, fallback to loading config from local config file if not self._config: default_filename = pkg_resources.resource_filename('crossbar', self.DEFAULT_CONFIG_PATH) with open(default_filename) as f: default_config = json.load(f) config_source, config_path = node.Node.load_config(self, configfile, default_config) self.log.info('Node configuration loaded from {config_source} ({config_path})', config_source=hlid(config_source), config_path=hlid(config_path)) # Docker host integration if _HAS_DOCKER and self._config and 'controller' in self._config: self._enable_docker = self._config['controller'].get('enable_docker', False) return config_source, config_path def _watchdog(self): # on Linux, check that we start with sufficient system entropy entropy_avail = None if sys.platform.startswith('linux'): try: with open('/proc/sys/kernel/random/entropy_avail', 'r') as ent: entropy_avail = int(ent.read()) # FIXME: my machine never has more than ~ 300 units available, 1000 seems a little optomistic! if entropy_avail < 64: self.log.warn('WARNING: insufficient entropy ({} bytes) available - try installing rng-tools'.format(entropy_avail)) except PermissionError: # this happens when packaged as a snap: the code prevented from reading a location # # that is not allowed to a confined snap package entropy_avail = -1 # check for at least 100MB free memory mem_avail = psutil.virtual_memory().available // 2 ** 20 if mem_avail < 100: self.log.warn('WARNING: available memory dropped to {mem_avail} MB', mem_avail=mem_avail) self.log.trace('WATCHDOG: entropy_avail {entropy_avail} bytes, mem_avail {mem_avail} MB', entropy_avail=entropy_avail, mem_avail=mem_avail) @inlineCallbacks def start(self, node_id=None): self.log.info('{note} [{method}]', note=hl('Starting node (initialize edge-node personality) ..', color='green', bold=True), method=hltype(FabricNode.start)) # run watchdog at 5Hz self._watchdog_looper = LoopingCall(self._watchdog) self._watchdog_looper.start(.2) res = yield node.Node.start(self, node_id or self._node_id) return res @inlineCallbacks def boot(self, use_activation_file=True, use_default_fabric=False): self.log.info('Booting node {method}', method=hltype(FabricNode.boot)) def reboot(): self.stop(restart=True) # determine the transport configuration of the management uplink for this node fabric_transport_config = compute_mgmt_uplink_config(self.log, self._cbdir, self._config, reboot, use_activation_file=use_activation_file, use_default_fabric=use_default_fabric) # now finally, if we do have a transport configuration for the management uplink at this point, # then start the management uplink .. if fabric_transport_config: self._has_management_config = True url = fabric_transport_config['url'] hostname = None if 'tls' in fabric_transport_config.get('endpoint', {}): hostname = fabric_transport_config['endpoint']['tls']['hostname'] self._manager_runner = ApplicationRunner( url=url, realm=None, extra=None, ssl=optionsForClientTLS(hostname) if hostname else None, ) def make(config): # extra info forwarded to CFC client session extra = { 'node': self, 'on_ready': Deferred(), 'on_exit': Deferred(), 'node_key': self._node_key, } @inlineCallbacks def on_ready_success(res): try: self._manager, self._management_realm, self._management_session_id, self._node_id, self._node_extra = res if self._bridge_session: try: yield self._bridge_session.attach_manager( self._manager, self._management_realm, self._node_id) except: self.log.failure() else: while True: try: # we actually test the management uplink by calling a procedure on the master yield self._manager.call('crossbarfabriccenter.mrealm.get_status') except ApplicationError as e: if e.error == 'wamp.error.no_such_procedure': self.log.info('Could not get master status ("wamp.error.no_such_procedure") - retrying in 5 secs ..') else: self.log.failure() yield sleep(5) except: self.log.failure() self.log.info('Could not get master status - retrying in 5 secs ..') yield sleep(5) else: self.log.info( click.style( 'Connected to Crossbar.io FX Master at management realm "{realm}", set node ID "{node_id}" (extra={node_extra}, session_id={session_id})', fg='green', bold=True), realm=self._management_realm, node_id=self._node_id, node_extra=self._node_extra, session_id=self._manager._session_id) # if the management uplink was successfully established and tested once, mark it so if not self._was_management_connected: self._was_management_connected = True self._management_url = url try: worker_ids = yield self._bridge_session.call( 'crossbar.get_workers') for worker_id in worker_ids: yield self._bridge_session.call( 'crossbar.worker.{}.set_node_id'.format(worker_id), self._node_id) except: self.log.warn( 'INTERNAL ERROR: could not set node_id "{node_id}" after CFC connection was established', node_id=self._node_id) self.log.failure() break else: self.log.warn( 'Uplink Crossbar.io FX Master session established, but no bridge session setup!' ) except Exception as e: self.log.warn('error in on_ready: {}'.format(e)) # ok, we are connected to CFC and normally will be configurated programmatically from there. # however, it is still possible to apply any local node configuration by setting # # node_extra: # { # "on_start_apply_config", true # } # # node_extra comes from CFC and has to be configured there (when the node is paired) # if self._node_extra: # by default, apply local config (from a node configuration file, if there is one) on_start_apply_config = self._node_extra.get('on_start_apply_config', True) if on_start_apply_config: if not self._local_config_applied: self.log.info('Applying local node configuration (on_start_apply_config is enabled)') yield self.boot_from_config(self._config) self._local_config_applied = True else: self.log.info('Local node configuration was already applied - skipping') else: self.log.info('Skipping any local node configuration (no local config or on_start_apply_config is "off")') def on_ready_error(err): if isinstance( err.value, ApplicationError) and err.value.error in ['fabric.auth-failed.node-unpaired', 'fabric.auth-failed.node-already-connected']: if not self._displayed_pairing_message: self._displayed_pairing_message = True self.log.error( click.style(err.value.error_message().upper(), fg='red', bold=True)) self.stop() else: self.log.error(click.style( 'Could not connect to CFC: {error}', fg='red', bold=True), error=err.value ) @inlineCallbacks def on_exit_success(reason): if self._bridge_session: try: yield self._bridge_session.detach_manager() except: self.log.failure() else: self.log.debug( 'Disconnected from Crossbar.io FX Master for management realm "{realm}"', realm=self._management_realm) else: self.log.warn( 'Uplink Crossbar.io FX Master session lost, but no bridge session setup!') self._manager, self._management_realm, self._management_session_id, self._node_id, self._node_extra = None, None, None, None, None def on_exit_error(err): print(err) extra['on_ready'].addCallbacks(on_ready_success, on_ready_error) extra['on_exit'].addCallbacks(on_exit_success, on_exit_error) config = ComponentConfig(extra=extra) session = NodeManagementSession(self._manager_runner, config) return session self.log.info('Connecting to Crossbar.io FX Master at {url} ..', url=url) yield self._manager_runner.run(make, start_reactor=False, auto_reconnect=True) else: # here, we don't have a management uplink :( self.log.info( hl('No management uplink configured (running unmanaged/single-node)', color='red', bold=True)) self._has_management_config = False # nevertheless, now boot from local node config! yield self.boot_from_config(self._config) self._local_config_applied = True def _add_extra_controller_components(self, controller_config): extra = { 'node_key': self._node_key, 'controller_config': controller_config, } cfg = ComponentConfig(self._realm, extra=extra) self._bridge_session = NodeManagementBridgeSession(cfg) router = self._router_factory.get(self._realm) self._router_session_factory.add(self._bridge_session, router, authrole='trusted') def _set_shutdown_triggers(self, controller_options): if 'shutdown' in controller_options: self._node_shutdown_triggers = controller_options['shutdown'] self.log.info("Using node shutdown triggers {triggers} from configuration", triggers=self._node_shutdown_triggers) else: # NODE_SHUTDOWN_ON_SHUTDOWN_REQUESTED # NODE_SHUTDOWN_ON_WORKER_EXIT # NODE_SHUTDOWN_ON_WORKER_EXIT_WITH_ERROR # NODE_SHUTDOWN_ON_LAST_WORKER_EXIT # in managed mode, a node - by default - only shuts down when explicitly asked to, # or upon a fatal error in the node controller self._node_shutdown_triggers = [checkconfig.NODE_SHUTDOWN_ON_SHUTDOWN_REQUESTED] self.log.info("Using default node shutdown triggers {triggers}", triggers=self._node_shutdown_triggers) def _add_worker_role(self, worker_auth_role, options): worker_role_config = { # each (native) worker is authenticated under a worker-specific authrole "name": worker_auth_role, "permissions": [ # the worker requires these permissions to work: { # management API provided by the worker. note that the worker management API is provided under # the URI prefix "crossbar.worker.<worker_id>". note that the worker is also authenticated # under authrole <worker_auth_role> on realm "crossbar" "uri": worker_auth_role, "match": "prefix", "allow": { "call": True, "register": True, "publish": True, "subscribe": True }, "disclose": { "caller": True, "publisher": True }, "cache": True }, { # controller procedure called by the worker (to check for controller status) "uri": "crossbar.get_status", "match": "exact", "allow": { "call": True, "register": False, "publish": False, "subscribe": False }, "disclose": { "caller": True, "publisher": True }, "cache": True } ] } # if configured to expose the controller connection within the worker (to make it available # in user code such as dynamic authenticators and router/container components), also add # permissions to actually use the (local) node management API if options.get('expose_controller', True): vendor_permissions = { "uri": "crossbar.", "match": "prefix", "allow": { "call": True, "register": False, "publish": False, "subscribe": True }, "disclose": { "caller": True, "publisher": True }, "cache": True } worker_role_config["permissions"].append(vendor_permissions) vendor_permissions = { "uri": "crossbarfabriccenter.", "match": "prefix", "allow": { "call": True, "register": True, "publish": True, "subscribe": True }, "disclose": { "caller": True, "publisher": True }, "cache": True } worker_role_config["permissions"].append(vendor_permissions) self._router_factory.add_role(self._realm, worker_role_config) self.log.info( 'worker-specific role "{authrole}" added on node management router realm "{realm}" {func}', func=hltype(self._add_worker_role), authrole=hlid(worker_role_config['name']), realm=hlid(self._realm)) def _extend_worker_args(self, args, options): if 'expose_shared' in options and options['expose_shared']: args.extend(['--expose_shared=true']) if 'expose_controller' in options and options['expose_controller']: args.extend(['--expose_controller=true']) @inlineCallbacks def _configure_native_worker_connections(self, worker_logname, worker_id, worker): # start connections (such as PostgreSQL database connection pools) # to run embedded in the router for connection in worker.get('connections', []): if 'id' in connection: connection_id = connection.pop('id') else: connection_id = 'connection{:03d}'.format(self._connection_no) self._connection_no += 1 yield self._controller.call('crossbar.worker.{}.start_connection'.format(worker_id), connection_id, connection, options=CallOptions()) self.log.info( "{logname}: connection '{connection}' started", logname=worker_logname, connection=connection_id, ) @inlineCallbacks def _configure_native_worker_router(self, worker_logname, worker_id, worker): # setup db connection pool yield self._configure_native_worker_connections(worker_logname, worker_id, worker) # in this case, call the base class method _after_ above - because we want db connections # to be available when router/container components might start .. yield node.Node._configure_native_worker_router(self, worker_logname, worker_id, worker) @inlineCallbacks def _configure_native_worker_container(self, worker_logname, worker_id, worker): # setup db connection pool yield self._configure_native_worker_connections(worker_logname, worker_id, worker) # in this case, call the base class method _after_ above - because we want db connections # to be available when router/container components might start .. yield node.Node._configure_native_worker_container(self, worker_logname, worker_id, worker) @inlineCallbacks def _configure_native_worker_hostmonitor(self, worker_logname, worker_id, worker): # after the native worker has started, and the HostMonitor controller session # has attached to the local node router, we need to do the following, if we # want it to work _also_ from a local node config.json. driving from CFC # at run-time always works (also without the bits here) # 1. the native workers' common options are configured by calling into the worker yield self._configure_native_worker_common(worker_logname, worker_id, worker) # 2. the host monitor specific actions need to be done, namely, starting the monitoring monitoring_config = worker.get('monitor', None) yield self._controller.call('crossbar.worker.{}.start_monitoring'.format(worker_id), monitoring_config, options=CallOptions()) @inlineCallbacks def _configure_native_worker_xbrmm(self, worker_logname, worker_id, worker): # 1. configure native workers' common options yield self._configure_native_worker_common(worker_logname, worker_id, worker) # 2. configure market makers (defined within the xbrmm worker) for maker in worker.get('makers', []): if 'id' in maker: maker_id = maker.pop('id') else: maker_id = 'maker{:03d}'.format(self._maker_no) self._maker_no += 1 maker['id'] = maker_id yield self._controller.call('crossbar.worker.{}.start_market_maker'.format(worker_id), maker_id, maker, options=CallOptions())
def compute_mgmt_uplink_config(log, cbdir, config, fn_reboot=None, use_activation_file=True, use_default_fabric=False): """ Determine the transport configuration of the management uplink for this node in the following order (using the first one that succeeds): * node activation file * management URL environment variable * node configuration file * built-in default ("master.xbr.network") :param cbdir: :param config: :param fn_reboot: :param use_activation_file: :param use_default_fabric: :return: """ fabric_transport_config = None # [1] pick up auto-activation files dropped by a master node (`${CBDIR}/key.activate`) if not fabric_transport_config and use_activation_file: def do_check_for_activation_file(activation_file, reboot_on_discover): if os.path.isfile(activation_file): tags = _parse_activation_file(activation_file) is_secure, hostname, port, _, _, _ = parse_url(tags['management-url']) config = { 'type': 'websocket', 'url': tags['management-url'], 'endpoint': { 'type': 'tcp', 'host': hostname, 'port': port, 'timeout': 5 } } if is_secure: config['endpoint']['tls'] = { 'hostname': hostname } _msg = 'Found auto-activation file "{}", using management URL "{}" - [1]'.format(activation_file, config['url']) log.info(click.style(_msg, fg='red', bold=True)) if reboot_on_discover and fn_reboot: # stop the node and enforce complete reboot - which will then pick up the new configuration fn_reboot() # return the management uplink transport configuration, as derived from the activation file return config # an activation file must be placed next to the node key pair (key.pub, key.priv) activation_file = os.path.join(cbdir, 'key.activate') # check and maybe load config from activation file fabric_transport_config = do_check_for_activation_file(activation_file, reboot_on_discover=False) # if there wasn't an activation file, periodically check for .. if not fabric_transport_config: lc = LoopingCall(do_check_for_activation_file, activation_file, reboot_on_discover=True) lc.start(1) log.info('Looping call to check for node activation file started! - [1]') # [2] management uplink configured via env var if not fabric_transport_config: url = os.environ['CROSSBAR_FABRIC_URL'].strip() if 'CROSSBAR_FABRIC_URL' in os.environ else '' if url != '': secure, host, tcp_port, _, _, _ = parse_url(url) fabric_transport_config = { 'type': 'websocket', 'url': url, 'endpoint': { 'type': 'tcp', 'host': host, 'port': tcp_port, 'timeout': 5 } } if secure: fabric_transport_config['endpoint']['tls'] = { 'hostname': host } log.info( click.style('Using management uplink at "{}" (from envvar) - [2]'.format(url), fg='red', bold=True)) # [3] user has configured a custom management uplink in the node configuration if not fabric_transport_config: if 'controller' in config and 'fabric' in config['controller'] and config['controller']['fabric']: fabric_config = config['controller']['fabric'] # allow to deactivate the management uplink connecting transport by setting "transport" to null fabric_transport_config = fabric_config.get('transport', None) if fabric_transport_config: url = fabric_transport_config.get('url', None) log.info( click.style('Using management uplink at "{}" (from node configuration) - [3]'.format(url), fg='red', bold=True)) else: log.info( click.style('Management uplink deactivated - [3]', fg='red', bold=True)) # [4] use hard-coded default management uplink if not fabric_transport_config and use_default_fabric: # default CFC (= our cloud hosted CFC service) fabric_transport_config = { 'type': 'websocket', 'url': 'wss://master.xbr.network/ws', 'endpoint': { 'type': 'tcp', 'host': 'master.xbr.network', 'port': 443, 'timeout': 5, 'tls': { 'hostname': 'master.xbr.network' } } } log.info( click.style( 'Using default fabric controller at URL "{}" (from envvar) - [4]'.format(fabric_transport_config['url']), fg='red', bold=True)) return fabric_transport_config
_FLUSH_CACHE(_IDMAPPER_CACHE_MAXSIZE) if _MAINTENANCE_COUNT % 3600 == 0: # validate scripts every hour evennia.ScriptDB.objects.validate() if _MAINTENANCE_COUNT % 3700 == 0: # validate channels off-sync with scripts evennia.CHANNEL_HANDLER.update() ## Commenting this out, it is probably not needed ## with CONN_MAX_AGE set. Keeping it as a reminder ## if database-gone-away errors appears again /Griatch #if _MAINTENANCE_COUNT % 18000 == 0: # connection.close() maintenance_task = LoopingCall(_server_maintenance) maintenance_task.start(60, now=True) # call every minute #------------------------------------------------------------ # Evennia Main Server object #------------------------------------------------------------ class Evennia(object): """ The main Evennia server handler. This object sets up the database and tracks and interlinks all the twisted network services that make up evennia. """ def __init__(self, application): """ Setup the server.
class Output(cowrie.core.output.Output): db = None def __init__(self): try: self.debug = CONFIG.getboolean('output_mysql', 'debug') self.apiKey = CONFIG.get('output_mysql', 'api_key') except Exception: self.debug = False cowrie.core.output.Output.__init__(self) def start(self): try: port = CONFIG.getint('output_mysql', 'port') except Exception: port = 3306 try: self.db = ReconnectingConnectionPool( 'MySQLdb', host=CONFIG.get('output_mysql', 'host'), db=CONFIG.get('output_mysql', 'database'), user=CONFIG.get('output_mysql', 'username'), passwd=CONFIG.get('output_mysql', 'password', raw=True), port=port, cp_min=1, cp_max=1 ) except MySQLdb.Error as e: log.msg("output_mysql: Error %d: %s" % (e.args[0], e.args[1])) self.lc = LoopingCall(self.check_wait) self.lc.start(30) self.versions = {} def stop(self): self.lc.stop() self.db.close() self.versions = {} def nowUnix(self): """return the current UTC time as an UNIX timestamp""" return int(time.time()) def sqlerror(self, error): log.err('output_mysql: MySQL Error: {}'.format(error.value)) def simpleQuery(self, sql, args): """ Just run a deferred sql query, only care about errors """ if self.debug: log.msg("output_mysql: MySQL query: {} {}".format(sql, repr(args))) d = self.db.runQuery(sql, args) d.addErrback(self.sqlerror) def simpleQueryWithCallback(self, callback, sql, args): if self.debug: log.msg("output_mysql: MySQL query: {} {}".format(sql, repr(args))) d = self.db.runQuery(sql, args) d.addCallbacks(callback, self.sqlerror) ############################ def createSession(self, peerIP, peerPort, hostIP, hostPort, timestamp, sessionId=None): if sessionId == None: sid = uuid.uuid4().hex else: sid = sessionId self.createSessionWhenever(sid, peerIP, hostIP, timestamp) return sid def createASNForIP(self, sid, peerIP, sensorId, timestamp): def addslashes(s): l = ["\\", '"', "'", "\0", ] for i in l: if i in s: s = s.replace(i, '\\'+i) return s def reverseIP(address): temp = address.split(".") convertedAddress = str(temp[3]) +'.' + str(temp[2]) + '.' + str(temp[1]) +'.' + str(temp[0]) return convertedAddress def onASNRecordTest(r): if r: createTheSession(sid, peerIP, sensorId, int(r[0][0]), timestamp) else: timeModified = datetime.datetime.fromtimestamp(timestamp).strftime('%Y-%m-%d %H:%M:%S') self.simpleQueryWithCallback(onASNRecordInsert, 'INSERT INTO `asinfo` (`asn`, `rir`, `country`, `asname`, `updatedTime`) VALUES (%s, %s, %s, %s, STR_TO_DATE(%s, %s)) ', (ASN, registry, country, isp, timeModified, '%Y-%m-%d %H:%i:%s')) def onASNRecordInsert(r): self.simpleQueryWithCallback(onASNRecordReady, 'SELECT `asnid` FROM `asinfo` WHERE `asn` = %s AND `rir` = %s AND `country` = %s AND `asname` = %s ', (ASN, registry, country, isp)) def onASNRecordReady(r): createTheSession(sid, peerIP, sensorId, int(r[0][0]), timestamp) def onSessionCreated(r): if self.versions.has_key(sid): self.simpleQuery( 'UPDATE `sessions` SET `client` = %s WHERE `id` = %s', (self.versions[sid], sid)) del self.versions[sid] else: self.versions[sid] = 1 def createTheSession(sid, peerIP, sensorId, asnid, timestamp): #Autor zmenil tvar timestamp, tu ho upravujem aby sedel s vasim timestamp_modified = datetime.datetime.fromtimestamp(timestamp).strftime('%Y-%m-%d %H:%M:%S') self.simpleQueryWithCallback(onSessionCreated, 'UPDATE `sessions` SET `starttime` = STR_TO_DATE(%s, %s), `sensor` = %s, `ip` = %s, `asnid` = %s' + \ ' WHERE `id` = %s', (timestamp_modified, '%Y-%m-%d %H:%i:%s', sensorId, peerIP, asnid, sid))#stary parsing: %Y-%m-%dT%H:%i:%s.%fZ try: querycmd1 = reverseIP(peerIP) + '.origin.asn.cymru.com' response1 = subprocess.Popen(['dig', '-t', 'TXT', querycmd1, '+short'], stdout=subprocess.PIPE).communicate()[0] response1List = response1.split('|') ASN = response1List[0].strip('" ') querycmd2 = 'AS' + ASN + '.asn.cymru.com' response2 = subprocess.Popen(['dig', '-t', 'TXT', querycmd2, '+short'], stdout=subprocess.PIPE).communicate()[0] except: ASN = "" response2 = "" log.msg("dig process error: " + str(sys.exc_info())) response2List = response2.split('|') if len(response2List) < 4: createTheSession(sid, peerIP, sensorId, '1', timestamp) else: isp = addslashes(response2List[4].replace('"', '')) network = addslashes(response1List[1].strip()) country = addslashes(response1List[2].strip()) registry = addslashes(response1List[3].strip()) isp = network + "-" + isp self.simpleQueryWithCallback(onASNRecordTest, 'SELECT `asnid` FROM `asinfo` WHERE `updated` = FALSE AND `asn` = %s AND `rir` = %s AND `country` = %s AND `asname` = %s ', (ASN, registry, country, isp)) def createSessionWhenever(self, sid, peerIP, hostIP, timestamp=None): def onSensorReady(r): id = int(r[0][0]) self.createASNForIP(sid, peerIP, id, timestamp) def onSensorInsert(r): self.simpleQueryWithCallback(onSensorReady, 'SELECT LAST_INSERT_ID()','') def onSensorSelect(r): if r: onSensorReady(r) else: self.simpleQueryWithCallback(onSensorInsert, 'INSERT INTO `sensors` (`ip`) VALUES (%s)', (hostIP,)) self.simpleQueryWithCallback(onSensorSelect, 'SELECT `id` FROM `sensors` WHERE `ip` = %s', (hostIP,)) def insert_wait(self, resource, url, scan_id, sha256): p = CONFIG.get('honeypot', 'log_path') + '/backlogs.sqlite' try: dbh = sqlite3.connect(p) cursor = dbh.cursor() dt = datetime.datetime.now() timestamp = dt.strftime("%Y-%m-%d %H:%M:%S") cursor.execute(""" INSERT INTO vtwait (scanid, hash, url, time, sha256) VALUES (?,?,?,?,?) """, (scan_id, resource, url, timestamp, sha256)) dbh.commit() cursor.close() except: log.msg("Unexpected error: " + str(sys.exc_info())) return True def check_wait(self): p = CONFIG.get('honeypot', 'log_path') + '/backlogs.sqlite' try: dbh = sqlite3.connect(p) cursor = dbh.cursor() r = cursor.execute(""" SELECT scanid, hash, url, time, sha256 FROM vtwait""") for record in r: scanid = format(record[0]) hash = format(record[1]) url = format(record[2]) sha256 = format(record[4]) j, jsonString = self.get_vt_report(scanid) if (not j is None) and (j["response_code"] == 1): if "scans" in j.keys(): args = {'shasum': hash, 'url': url, 'permalink': j["permalink"], 'positives': j['positives'], 'total': j['total'], 'sha256' : sha256} args_scan = {'shasum': hash, 'sha256' : sha256, 'permalink': j['permalink'], 'json': jsonString} self.handleVirustotal(args, args_scan) cursor.execute(""" DELETE FROM vtwait WHERE scanid = ?""", (scanid,) ) dbh.commit() cursor.close() except: log.msg("Unexpected error: " + str(sys.exc_info())) return True def get_vt_report(self, resource): url = "https://www.virustotal.com/vtapi/v2/file/report" parameters = {"resource": resource, "apikey": self.apiKey} data = urllib.urlencode(parameters) req = urllib2.Request(url, data) response = urllib2.urlopen(req) jsonString = response.read() try: j = json.loads(jsonString ) except: j = None return j, jsonString def post_file(self, aFileName, aUrl=None): file_to_send = open(aFileName, "rb").read() h = hashlib.sha1() h.update(file_to_send) h256 = hashlib.sha256() h256.update(file_to_send) j, jsonString = self.get_vt_report(h.hexdigest()) if j is None: response = -2 else: response = j["response_code"] if response == 1: # file known log.msg("post_file(): file known") if "scans" in j.keys(): args = {'shasum': h.hexdigest(), 'sha256' : h256.hexdigest(), 'url': aUrl, 'permalink': j['permalink'], 'positives' : j['positives'], 'total' : j['total']} args_scan = {'shasum': h.hexdigest(), 'sha256' : h256, 'permalink': j['permalink'], 'json': jsonString} self.handleVirustotal(args, args_scan) else: response = 2 elif response == 0: # file not known log.msg("post_file(): sending the file to VT...") register_openers() datagen, headers = multipart_encode({"file": open(aFileName, "rb")}) request = urllib2.Request("https://www.virustotal.com/vtapi/v2/file/scan?apikey=" + self.apiKey, datagen, headers) jsonString = urllib2.urlopen(request).read() log.msg("post_file(): response is " + jsonString) j = json.loads(jsonString) self.insert_wait(h.hexdigest(), aUrl, j["scan_id"], h256.hexdigest()) return response def make_comment(resource): apikey = CONFIG.get('virustotal', 'apikey') url = "https://www.virustotal.com/vtapi/v2/comments/put" parameters = {"resource": resource, "comment": "captured by ssh honeypot", "apikey": apikey} data = urllib.urlencode(parameters) req = urllib2.Request(url, data) response = urllib2.urlopen(req) json = response.read() def handleVirustotal(self, args, args2): def insert_done(r): self.handleVirustotalScan(args2) def select_done(r): if r: id = r[0][0] else: d = self.db.runQuery('INSERT INTO `virustotals`' + \ ' (`shasum`, `sha256`, `url`, `timestamp`, `permalink`, `positives`, `count`)' + \ ' VALUES (%s, %s, %s, FROM_UNIXTIME(%s), %s, %s, %s)', (args['shasum'], args['sha256'], args['url'], self.nowUnix(), args['permalink'], args['positives'], args['total'],)) d.addCallbacks(insert_done, self.sqlerror) d = self.db.runQuery('SELECT `id` FROM `virustotals` WHERE `permalink` = %s', (args['permalink'],)) d.addCallbacks(select_done, self.sqlerror) def handleVirustotalScan(self, args): def insert_results(r): scan_id = r[0][0] jsonData = json.loads(args['json']) scans = jsonData['scans'] for av, val in scans.items(): res = val['result'] # not detected = '' -> NULL if res == '': res = None self.simpleQuery('INSERT INTO `virustotalscans`' + \ ' (`scan_id`, `scanner`, `result`)' + \ ' VALUES (%s, %s, %s)', (scan_id, av, res, )) d = self.db.runQuery('SELECT `id` FROM `virustotals` WHERE `permalink` = %s', (args['permalink'],)) d.addCallbacks(insert_results, self.sqlerror) ############################ def message_to_new(message): return_string = "{\"eventid\": \"cowrie.client.version\", " session_position = message.find("\"session\"") timestamp_position = message.find("\"timestamp\"") session = message[session_position + 11:timestamp_position - 2] version_position = message.find("\"version\"") time_position = message.find("\"time\"") version = message[version_position + 11:time_position - 2] version = "\"" + version[2:version.rfind('_')] + '",' + version[version.rfind('_') + 2:-3] return_string = return_string + "\"session\": " + session + ", \"version\": " + version + "}" return_string = return_string.replace("\\", "") print("JSON") print(return_string) return return_string @defer.inlineCallbacks def write(self, entry): if entry["eventid"] == 'cowrie.session.connect': self.simpleQuery('INSERT INTO `sessions` (`id`, `starttime`, `sensor`, `ip`)' + \ ' VALUES (%s, STR_TO_DATE(%s, %s), %s, %s)', (entry['session'], '1991-1-1 1:1:1', '%Y-%m-%d %H:%i:%s', '1', entry['src_ip']))#stary parsing: %Y-%m-%dT%H:%i:%s.%fZ elif entry["eventid"] == 'cowrie.login.success': self.simpleQuery('INSERT INTO `auth` (`session`, `success`, `username`, `password`, `timestamp`) ' 'VALUES (%s, %s, %s, %s, FROM_UNIXTIME(%s))', (entry["session"], 1, entry['username'], entry['password'], entry["time"])) elif entry["eventid"] == 'cowrie.login.failed': self.simpleQuery('INSERT INTO `auth` (`session`, `success`, `username`, `password`, `timestamp`) ' 'VALUES (%s, %s, %s, %s, FROM_UNIXTIME(%s))', (entry["session"], 0, entry['username'], entry['password'], entry["time"])) elif entry["eventid"] == 'cowrie.session.params': self.simpleQuery('INSERT INTO `params` (`session`, `arch`) ' 'VALUES (%s, %s)', (entry["session"], entry["arch"])) elif entry["eventid"] == 'cowrie.command.input': self.simpleQuery('INSERT INTO `input` (`session`, `timestamp`, `success`, `input`) ' 'VALUES (%s, FROM_UNIXTIME(%s), %s , %s)', (entry["session"], entry["time"], 1, entry["input"])) elif entry["eventid"] == 'cowrie.command.failed': self.simpleQuery('INSERT INTO `input` (`session`, `timestamp`, `success`, `input`) ' 'VALUES (%s, FROM_UNIXTIME(%s), %s , %s)', (entry["session"], entry["time"], 0, entry["input"])) elif entry["eventid"] == 'cowrie.session.file_download': self.simpleQuery('INSERT INTO `downloads` (`session`, `timestamp`, `url`, `outfile`, `shasum`) ' 'VALUES (%s, FROM_UNIXTIME(%s), %s, %s, %s)', (entry["session"], entry["time"], entry['url'], entry['outfile'], entry['shasum'])) self.post_file(entry["outfile"], entry["url"]) elif entry["eventid"] == 'cowrie.session.file_download.failed': self.simpleQuery('INSERT INTO `downloads` (`session`, `timestamp`, `url`, `outfile`, `shasum`) ' 'VALUES (%s, FROM_UNIXTIME(%s), %s, %s, %s)', (entry["session"], entry["time"], entry['url'], 'NULL', 'NULL')) elif entry["eventid"] == 'cowrie.session.file_upload': self.simpleQuery('INSERT INTO `downloads` (`session`, `timestamp`, `url`, `outfile`, `shasum`) ' 'VALUES (%s, FROM_UNIXTIME(%s), %s, %s, %s)', (entry["session"], entry["time"], '', entry['outfile'], entry['shasum'])) self.post_file(entry["outfile"]) elif entry["eventid"] == 'cowrie.session.input': self.simpleQuery('INSERT INTO `input` (`session`, `timestamp`, `realm`, `input`) ' 'VALUES (%s, FROM_UNIXTIME(%s), %s , %s)', (entry["session"], entry["time"], entry["realm"], entry["input"])) elif entry["eventid"] == 'cowrie.client.version': try: version_string = entry["version"] hostport = json.loads(version_string[version_string.rfind('_') + 1:-1])["hostport"] entry['src_ip'] = hostport[:hostport.rfind(':')]; entry['version'] = version_string[1:version_string.rfind('_')] extraPresent = True except: extraPresent = False self.createSessionWhenever(entry['session'], entry['src_ip'], self.sensor, entry['time']) #yield self.db.runQuery( # 'UPDATE `sessions` SET `ip` = %s WHERE `id` = %s', # (hostport[:hostport.rfind(':')], entry['session'],)) r = yield self.db.runQuery( 'SELECT `id` FROM `clients` ' 'WHERE `version` = %s', (entry['version'],)) if r: id = int(r[0][0]) else: yield self.db.runQuery( 'INSERT INTO `clients` (`version`) ' 'VALUES (%s)', (entry['version'],)) r = yield self.db.runQuery('SELECT LAST_INSERT_ID()') id = int(r[0][0]) if not self.versions.has_key(entry['session']): self.versions[entry['session']] = id else: del self.versions[entry['session']] self.simpleQuery( 'UPDATE `sessions` SET `client` = %s WHERE `id` = %s', (id, entry["session"])) elif entry["eventid"] == 'cowrie.client.size': self.simpleQuery( 'UPDATE `sessions` ' 'SET `termsize` = %s ' 'WHERE `id` = %s', ('%sx%s' % (entry['width'], entry['height']), entry["session"])) elif entry["eventid"] == 'cowrie.session.closed': self.simpleQuery( 'UPDATE `sessions` ' 'SET `endtime` = FROM_UNIXTIME(%s) ' 'WHERE `id` = %s', (entry["time"], entry["session"])) elif entry["eventid"] == 'cowrie.log.closed': self.simpleQuery( 'INSERT INTO `ttylog` (`session`, `ttylog`, `size`) ' 'VALUES (%s, %s, %s)', (entry["session"], entry["ttylog"], entry["size"])) elif entry["eventid"] == 'cowrie.client.fingerprint': self.simpleQuery( 'INSERT INTO `keyfingerprints` (`session`, `username`, `fingerprint`) ' 'VALUES (%s, %s, %s)', (entry["session"], entry["username"], entry["fingerprint"]))
class _HamlibProxy(ExportedState): # pylint: disable=no-member """ Abstract class for objects which export state proxied to a hamlib daemon. """ implements(IComponent, IProxy) def __init__(self, protocol): # info from hamlib self.__cache = {} self.__caps = {} self.__levels = [] # invert command table # TODO: we only need to do this once per class, really self._how_to_command = {key: command for command, keys in self._commands.iteritems() for key in keys} # keys are same as __cache, values are functions to call with new values from rig self._cell_updaters = {} self.__communication_error = False self.__last_error = (-1e9, '', 0) self.__protocol = protocol self.__disconnect_deferred = defer.Deferred() protocol._set_proxy(self) # TODO: If hamlib backend supports "transceive mode", use it in lieu of polling self.__poller_slow = LoopingCall(self.__poll_slow) self.__poller_fast = LoopingCall(self.__poll_fast) self.__poller_slow.start(2.0) self.__poller_fast.start(0.2) self._ready_deferred = protocol.rc_send('dump_caps') def sync(self): # TODO: Replace 'sync' with more specifically meaningful operations d = self.__protocol.rc_send(self._dummy_command) d.addCallback(lambda _: None) # ignore result return d def close(self): """implements IComponent""" self.__protocol.transport.loseConnection() return self.when_closed() # used for tests, not part of IComponent def when_closed(self): return fork_deferred(self.__disconnect_deferred) def _ehs_get(self, name_in_cmd): if name_in_cmd in self.__cache: return self.__cache[name_in_cmd] else: return 0.0 def _clientReceived(self, command, key, value): self.__communication_error = False if command == 'dump_caps': def write(key): self.__caps[key] = value if key == 'Get level': # add to polling info for info in value.strip().split(' '): match = re.match(r'^(\w+)\([^()]+\)$', info) # part in parens is probably min/max/step info, but we don't have any working examples to test against (they are all 0) if match: self.__levels.append(match.group(1)) else: log.err('Unrecognized level description from %s: %r' % (self._server_name, info)) # remove irregularity keymatch = re.match(r'(Can [gs]et )([\w\s,/-]+)', key) if keymatch and keymatch.group(2) in _cap_remap: for mapped in _cap_remap[keymatch.group(2)]: write(keymatch.group(1) + mapped) else: write(key) else: self.__update_cache_and_cells(key, value) def _clientReceivedLevel(self, level_name, value_str): self.__update_cache_and_cells(level_name + ' level', value_str) def _clientError(self, cmd, error_number): if cmd.startswith('get_'): # these getter failures are boring, probably us polling something not implemented if error_number == RIG_ENIMPL or error_number == RIG_ENTARGET or error_number == RIG_BUSERROR: return elif error_number == RIG_ETIMEOUT: self.__communication_error = True return self.__last_error = (time.time(), cmd, error_number) self.state_changed('errors') def __update_cache_and_cells(self, key, value): self.__cache[key] = value if key in self._cell_updaters: self._cell_updaters[key](value) def _clientConnectionLost(self, reason): self.__poller_slow.stop() self.__poller_fast.stop() self.__disconnect_deferred.callback(None) def _ehs_set(self, name_full, value): if not isinstance(value, str): raise TypeError() name_in_cmd = self._how_to_command[name_full] # raises if cannot set if value != self.__cache[name_full]: self.__cache[name_full] = value self.__protocol.rc_send( 'set_' + name_in_cmd, ' '.join(self.__cache[arg_name] for arg_name in self._commands[name_in_cmd])) def state_def(self, callback): super(_HamlibProxy, self).state_def(callback) for name in self._info: can_get = self.__caps.get('Can get ' + name) if can_get is None: log.msg('No can-get information for ' + name) if can_get != 'Y': # TODO: Handle 'E' condition continue writable = name in self._how_to_command and self.__caps.get('Can set ' + name) == 'Y' _install_cell(self, name, False, writable, callback, self.__caps) for level_name in self.__levels: # TODO support writable levels _install_cell(self, level_name + ' level', True, False, callback, self.__caps) def __poll_fast(self): # TODO: Stop if we're getting behind p = self.__protocol self.poll_fast(p.rc_send) for level_name in self.__levels: p.rc_send('get_level', level_name) def __poll_slow(self): # TODO: Stop if we're getting behind p = self.__protocol self.poll_slow(p.rc_send) @exported_value(type=Notice(always_visible=False), changes='explicit') def get_errors(self): if self.__communication_error: return 'Rig not responding.' else: (error_time, cmd, error_number) = self.__last_error if error_time > time.time() - 10: return u'%s: %s' % (cmd, error_number) else: return u'' def poll_fast(self, send): raise NotImplementedError() def poll_slow(self, send): raise NotImplementedError()
class Auth(JSONComponent): """ The component that implements authentification into the JSON interface. """ def __init__(self): super(Auth, self).__init__("Auth") self.worker = LoopingCall(self._clean_sessions) self.worker.start(5) def _clean_sessions(self): config = component.get("DelugeWeb").config session_ids = config["sessions"].keys() now = time.gmtime() for session_id in session_ids: session = config["sessions"][session_id] if "expires" not in session: del config["sessions"][session_id] continue if time.gmtime(session["expires"]) < now: del config["sessions"][session_id] continue def _create_session(self, request, login='******'): """ Creates a new session. :keyword login: the username of the user logging in, currently \ only for future use currently. :type login: string """ m = hashlib.md5() m.update(login) m.update(str(time.time())) m.update(str(random.getrandbits(40))) m.update(m.hexdigest()) session_id = m.hexdigest() config = component.get("DelugeWeb").config expires, expires_str = make_expires(config["session_timeout"]) checksum = str(make_checksum(session_id)) request.addCookie('_session_id', session_id + checksum, path=request.base + "json", expires=expires_str) log.debug("Creating session for %s", login) config = component.get("DelugeWeb").config if type(config["sessions"]) is list: config.config["sessions"] = {} config["sessions"][session_id] = { "login": login, "level": AUTH_LEVEL_ADMIN, "expires": expires } return True def check_password(self, password): config = component.get("DelugeWeb").config if "pwd_md5" in config.config: # We are using the 1.2-dev auth method log.debug("Received a password via the 1.2-dev auth method") m = hashlib.md5() m.update(config["pwd_salt"]) m.update(password) if m.hexdigest() == config['pwd_md5']: # We want to move the password over to sha1 and remove # the old passwords from the config file. self._change_password(password) del config.config["pwd_md5"] # Remove the older password if there is now. if "old_pwd_md5" in config.config: del config.config["old_pwd_salt"] del config.config["old_pwd_md5"] return True elif "old_pwd_md5" in config.config: # We are using the 1.1 webui auth method log.debug("Received a password via the 1.1 auth method") from base64 import decodestring m = hashlib.md5() m.update(decodestring(config["old_pwd_salt"])) m.update(password) if m.digest() == decodestring(config["old_pwd_md5"]): # We want to move the password over to sha1 and remove # the old passwords from the config file. self._change_password(password) del config.config["old_pwd_salt"] del config.config["old_pwd_md5"] return True elif "pwd_sha1" in config.config: # We are using the 1.2 auth method log.debug("Received a password via the 1.2 auth method") s = hashlib.sha1() s.update(config["pwd_salt"]) s.update(password) if s.hexdigest() == config["pwd_sha1"]: return True else: # Can't detect which method we should be using so just deny # access. log.debug("Failed to detect the login method") return False def check_request(self, request, method=None, level=None): """ Check to ensure that a request is authorised to call the specified method of authentication level. :param request: The HTTP request in question :type request: twisted.web.http.Request :keyword method: Check the specified method :type method: function :keyword level: Check the specified auth level :type level: integer :raises: Exception """ config = component.get("DelugeWeb").config session_id = get_session_id(request.getCookie("_session_id")) if session_id not in config["sessions"]: auth_level = AUTH_LEVEL_NONE session_id = None else: session = config["sessions"][session_id] auth_level = session["level"] expires, expires_str = make_expires(config["session_timeout"]) session["expires"] = expires _session_id = request.getCookie("_session_id") request.addCookie('_session_id', _session_id, path=request.base + "json", expires=expires_str) if method: if not hasattr(method, "_json_export"): raise Exception("Not an exported method") method_level = getattr(method, "_json_auth_level") if method_level is None: raise Exception("Method has no auth level") level = method_level if level is None: raise Exception("No level specified to check against") request.auth_level = auth_level request.session_id = session_id if auth_level < level: raise AuthError("Not authenticated") def _change_password(self, new_password): """ Change the password. This is to allow the UI to change/reset a password. :param new_password: the password to change to :type new_password: string """ log.debug("Changing password") salt = hashlib.sha1(str(random.getrandbits(40))).hexdigest() s = hashlib.sha1(salt) s.update(new_password) config = component.get("DelugeWeb").config config["pwd_salt"] = salt config["pwd_sha1"] = s.hexdigest() return True @export def change_password(self, old_password, new_password): """ Change the password. :param old_password: the current password :type old_password: string :param new_password: the password to change to :type new_password: string """ if not self.check_password(old_password): return False return self._change_password(new_password) @export(AUTH_LEVEL_NONE) def check_session(self, session_id=None): """ Check a session to see if it's still valid. :returns: True if the session is valid, False if not. :rtype: booleon """ return __request__.session_id is not None @export def delete_session(self): """ Removes a session. :param session_id: the id for the session to remove :type session_id: string """ d = Deferred() config = component.get("DelugeWeb").config del config["sessions"][__request__.session_id] return True @export(AUTH_LEVEL_NONE) def login(self, password): """ Test a password to see if it's valid. :param password: the password to test :type password: string :returns: a session id or False :rtype: string or False """ if self.check_password(password): return self._create_session(__request__) else: log.error('Login failed (ClientIP %s)', __request__.getClientIP()) return False
class DriverManager(object): """ This class acts as factory and manager For plug & play managment: the whole "check and or set device id" procedure should not take place during the normal connect etc process but in a completely seperate set of phases(something like a "setup" and used to check for correct device on port/device id) the hardware manager's connect method needs to be modified: - if the device was successfully associated with a port in the p&p detection phase, use that port info - if the device was not identified , has no id, etc then use the current "use the first available port" method perhaps we should use a method similar to the way drivers are installed on mswindows type systems: - ask for removal of cable if already connected - ask for re plug of cable - do a diff on the previous/new list of com/tty devices - do id generation/association """ def __init__(self, hardware_poll_frequency=2, *args, **kwargs): self.hardware_poll_requency = hardware_poll_frequency self._drivers = {} self._port_to_driver_bindings = PortDriverBindings() self._hardware_interfaces = {} self._hardware_poller = LoopingCall(self.update_device_list) def setup(self): """setup the driver manager""" log.msg("Driver Manager setup succesfully", system="Driver Manager", logLevel=logging.CRITICAL) self._hardware_poller.start(self.hardware_poll_requency, now=True) #reactor.callLater(self.hardware_poll_requency, self.update_device_list) @defer.inlineCallbacks def teardown(self): if self._hardware_poller.running: self._hardware_poller.stop() yield self.clear_drivers() log.msg("Shutting down, goodbye!", system="DriverManager", logLevel=logging.CRITICAL) """ ########################################################################### The following are the driver "CRUD" (Create, read, update,delete) methods """ def add_driver(self, driver_class, *args, **kwargs): """add a driver to the list of manager drivers driver_class : the class of the driver params : the parameters to pass to the constructor of the driver """ driver = driver_class(*args, **kwargs) if not driver.is_configured: driver.connection_mode = 0 self._register_driver(driver) return driver def update_driver(self, driver_id=None, *args, **kwargs): driver = self.get_driver(driver_id) driver.update(*args, **kwargs) def get_driver(self, driver_id=None): """get a driver, based on its id""" if driver_id is None: raise UnknownDriver() driver = self._drivers.get(driver_id) if driver is None: raise UnknownDriver() return driver def get_drivers(self, filters=None): """ Returns the list of drivers, filtered by the filters param the filters is a dictionary of list, with each key beeing an attribute to check, and the values in the list , values of that param to check against """ deferred = defer.Deferred() def filters_check(driver, filters): """driver: a driver instance filters: see above """ for key in filters.keys(): if not getattr(driver, key) in filters[key]: return False return True def _get_drivers(filters, driver_list): """filters: see above driver_list: a list of driver instances """ if filters: return [ driver for driver in driver_list if filters_check(driver, filters) ] else: return driver_list deferred.addCallback(_get_drivers, self._drivers.values()) reactor.callLater(0.5, deferred.callback, filters) return deferred def delete_driver(self, driver_id=None): """ Remove a driver Params: driver_id: the driver_id of the driver """ deferred = defer.Deferred() def remove(driver_id): driver = self._drivers.get(driver_id) del self._drivers[driver_id] self._unregister_driver(driver) if driver.is_connected: driver.disconnect() log.msg("Removed driver %s" % str(driver), system="DriverManager", logLevel=logging.CRITICAL) deferred.addCallback(remove) reactor.callLater(0, deferred.callback, driver_id) return deferred @defer.inlineCallbacks def clear_drivers(self): """ Removes & deletes ALL the drivers (disconnecting them first) This should be used with care, as well as checks on client side """ for driver_id in self._drivers.keys(): yield self.delete_driver(driver_id=driver_id) """ ########################################################################### The following are helper methods """ def get_unbound_ports(self, hardware_interface): hardware_interface_info = self._hardware_interfaces.get( hardware_interface) unbound_ports = hardware_interface_info.get_unbound_ports() return unbound_ports """ ########################################################################### The following are driver related methods """ @defer.inlineCallbacks def set_remote_id(self, driver, port=None): """this method is usually used the first time a device is connected, to set its correct device id""" log.msg("Setting remote id", driver, system="DriverManager", logLevel=logging.DEBUG) if not port: unbound_ports = self._port_to_driver_bindings.get_unbound_ports() if unbound_ports > 0: port = unbound_ports[0] driver.connection_mode = 0 yield driver.bind(port).addCallbacks(callback=self._driver_binding_succeeded, \ callbackKeywords={"driver": driver, "port": port}, errback=self._driver_binding_failed) else: log.msg("Impossible to do device binding, no ports available", system="DriverManager") driver.connection_mode = 1 """ ########################################################################### The following are the plug&play and registration methods for drivers """ @defer.inlineCallbacks def connect_to_hardware(self, driver_id=None, port=None, connection_mode=1): """driver_id : the id of the driver to connect connection_mode: the mode in which to connect the driver first search if driver is already bound, if it is use that data :param connection_mode: mode 2 is special case for forced connection """ driver = self.get_driver(driver_id=driver_id) # if port is None: # unbound_ports = self.get_unbound_ports(driver.hardware_interface_class) # if len(unbound_ports) == 0: # raise Exception("No port specified and no port available") # port = unbound_ports[0] if connection_mode == 2: self._port_to_driver_bindings.bind(driver, port) elif connection_mode == 0: pass # if driver.hardware_id is None: # raise Exception("No HardwareId specified Cannot configure hardware to port permabind ") log.msg("Connecting to hardware in mode:", connection_mode, "to port", port, system="DriverManager", logLevel=logging.CRITICAL) yield driver.connect(port=port, connection_mode=connection_mode) def upload_firmware(self, firmware): """upload a specific firmware to a given device perhaps use this kind of field as helper: target_hardware = "Generic_arduino" """ pass def _register_driver(self, driver=None): if driver is None: raise UnknownDriver() """register a driver in the system""" log.msg("Registering driver", driver, system="DriverManager", logLevel=logging.DEBUG) drv_inteface_class = driver.hardware_interface_class if not drv_inteface_class in self._hardware_interfaces: self._hardware_interfaces[ drv_inteface_class] = HardwareInterfaceInfo(drv_inteface_class) self._hardware_interfaces[drv_inteface_class].add_drivers([driver]) self._drivers[driver.cid] = driver def _unregister_driver(self, driver): """for driver removal from the list of registered drivers""" log.msg("Unregistering driver", driver, system="DriverManager", logLevel=logging.DEBUG) self._hardware_interfaces[ driver.hardware_interface_class].remove_drivers([driver]) @defer.inlineCallbacks def _setup_drivers(self, hardware_interface): """ method to iterate over drivers and try to bind them to the correct available port The only "blocking/waiting" element is with drivers with the same connection type: ie : serial devices need to be done one by one, same with webcams, etc BUT !! you can have these at the same time: -bind serial devices -bind webcams """ unbound_drivers = self._hardware_interfaces[ hardware_interface].get_unbound_drivers() if len(unbound_drivers) > 0: log.msg("Setting up drivers", system="DriverManager", logLevel=logging.INFO) for driver in unbound_drivers: yield self._attempt_binding( driver, self._hardware_interfaces[hardware_interface].bindings) @defer.inlineCallbacks def _attempt_binding(self, driver, binding): """this methods tries to bind a given driver to a correct port should it fail, it will try to do the binding with the next available port,until either: * the port binding was sucessfull * all driver/port combos were tried binding can only be done with drivers that have authentification enabled, are not bound, and are not currently busy """ for port in binding.get_driver_untested_ports(driver): binding_ok = False if not driver.is_bound and driver.do_authentification and not driver.is_busy: try: yield driver.connect(port=port, connection_mode=1) binding_ok = True except Exception as inst: pass if binding_ok: self._hardware_interfaces[ driver.hardware_interface_class].bind(driver, port) log.msg("Driver", str(driver), "plugged in to port", port, system="DriverManager", logLevel=logging.DEBUG) driver.is_bound = True if not driver.auto_connect: driver.disconnect() else: self._hardware_interfaces[ driver.hardware_interface_class].add_to_tested( driver, port) driver.disconnect() log.msg("failed to plug ", driver, "to port", port, system="DriverManager", logLevel=logging.DEBUG) def check_for_port_changes(self, old_list, new_list): """ returns a list of added and removed ports based on an old list and a new list of ports we don't do a preliminary "if len(old_list) != len(new_list):" since even with the same amount of detected devices, the actual devices in the list could be different """ set_1 = set(old_list) set_2 = set(new_list) added_ports = set_2 - set_1 removed_ports = set_1 - set_2 if len(added_ports) == 0: added_ports = None if len(removed_ports) == 0: removed_ports = None return (added_ports, removed_ports) @defer.inlineCallbacks def update_device_list(self): """ Method that gets called regularly, scans for newly plugged in/out devices and either * tries to start the binding process if new devices were found * does all the necessary to remove a binding/do the cleanup, if a device was disconnected """ for hardware_interface, connection_info in self._hardware_interfaces.items( ): old_ports = connection_info.get_ports() new_ports = (yield connection_info.list_ports()) added_ports, removed_ports = self.check_for_port_changes( old_ports, new_ports) if added_ports: log.msg("Ports added:", added_ports, " to connection type", hardware_interface, system="DriverManager", logLevel=logging.DEBUG) connection_info.add_ports(list(added_ports)) if added_ports or (len(connection_info.get_unbound_drivers()) > 0 and len(connection_info.get_unbound_ports()) > 0): #log.msg("New ports/drivers detected: These ports were added", added_ports, logLevel=logging.DEBUG) yield self._setup_drivers(hardware_interface) if removed_ports: log.msg("Ports removed:", removed_ports, logLevel=logging.DEBUG) oldBoundDrivers = connection_info.get_bound_drivers() connection_info.remove_ports(list(removed_ports)) newBoundDrivers = connection_info.get_bound_drivers() for driver in set(oldBoundDrivers) - set(newBoundDrivers): port = driver.hardwareHandler.port log.msg("Driver", driver, "plugged out of port", port, " in connection type", hardware_interface, system="DriverManager", logLevel=logging.DEBUG) driver.is_bound = False
def start(self): lc = LoopingCall(self.scan_feed) lc.start(self.interval) lc2 = LoopingCall(self.scan_downloading) lc2.start(self.interval * 2)
def __init__(self): self.output('TWS init') self.channel = 'tws' self.config = Config(self.channel) self.username = self.config.get('USERNAME') self.password = self.config.get('PASSWORD') self.http_port = int(self.config.get('HTTP_PORT')) self.tcp_port = int(self.config.get('TCP_PORT')) self.callback_timeout = int(self.config.get('CALLBACK_TIMEOUT')) if not self.callback_timeout: self.callback_timeout = DEFAULT_TWS_CALLBACK_TIMEOUT self.output('callback_timeout=%d' % self.callback_timeout) self.enable_ticker = bool(int(self.config.get('ENABLE_TICKER'))) self.log_api_messages = bool(int(self.config.get('LOG_API_MESSAGES'))) self.output_second_ticks = bool( int(self.config.get('ENABLE_SECONDS_TICK'))) self.suppress_error_codes = [ int(c) for c in self.config.get('SUPPRESS_ERROR_CODES').split(',') ] self.label = 'TWS Gateway' self.current_account = '' self.clients = set([]) self.orders = {} self.pending_orders = {} self.openorder_callbacks = [] self.accounts = [] self.account_data = {} self.pending_account_data_requests = set([]) self.positions = {} self.position_callbacks = [] self.executions = {} self.execution_callbacks = [] self.bardata_callbacks = [] self.cancel_callbacks = [] self.order_callbacks = [] self.addsymbol_callbacks = [] self.accountdata_callbacks = [] self.last_connection_status = '' self.connection_status = 'Initializing' self.LastError = -1 self.next_order_id = -1 self.last_minute = -1 self.handlers = { 'error': self.error_handler, 'tickSize': self.handle_tick_size, 'tickPrice': self.handle_tick_price, 'tickString': self.handle_tick_string, 'nextValidId': self.handle_next_valid_id, 'currentTime': self.handle_time, 'managedAccounts': self.handle_accounts, 'orderStatus': self.handle_order_status, 'openOrder': self.handle_open_order, 'openOrderEnd': self.handle_open_order_end, 'execDetails': self.handle_exec_details, 'execDetailsEnd': self.handle_exec_details_end, 'position': self.handle_position, 'positionEnd': self.handle_position_end, 'historicalData': self.handle_historical_data, 'updateAccountValue': self.handle_account_value, 'accountDownloadEnd': self.handle_account_download_end, } self.ticker_ids = {} self.symbols = {} self.symbols_by_id = {} self.primary_exchange_map = {} self.tws_conn = None repeater = LoopingCall(self.EverySecond) # start repeater on even RTC second t = time.time() t = 1 - (t - int(t)) time.sleep(t) repeater.start(1)
class APITestsMixin(APIAssertionsMixin): """ Helpers for writing tests for the Docker Volume Plugin API. """ NODE_A = uuid4() NODE_B = uuid4() def initialize(self): """ Create initial objects for the ``VolumePlugin``. """ self.volume_plugin_reactor = Clock() self.flocker_client = SimpleCountingProxy(FakeFlockerClient()) # The conditional_create operation used by the plugin relies on # the passage of time... so make sure time passes! We still use a # fake clock since some tests want to skip ahead. self.looping = LoopingCall( lambda: self.volume_plugin_reactor.advance(0.001)) self.looping.start(0.001) self.addCleanup(self.looping.stop) def test_pluginactivate(self): """ ``/Plugins.Activate`` indicates the plugin is a volume driver. """ # Docker 1.8, at least, sends "null" as the body. Our test # infrastructure has the opposite bug so just going to send some # other garbage as the body (12345) to demonstrate that it's # ignored as per the spec which declares no body. return self.assertResult(b"POST", b"/Plugin.Activate", 12345, OK, {u"Implements": [u"VolumeDriver"]}) def test_remove(self): """ ``/VolumeDriver.Remove`` returns a successful result. """ return self.assertResult(b"POST", b"/VolumeDriver.Remove", {u"Name": u"vol"}, OK, {u"Err": u""}) def test_unmount(self): """ ``/VolumeDriver.Unmount`` returns a successful result. """ return self.assertResult(b"POST", b"/VolumeDriver.Unmount", {u"Name": u"vol"}, OK, {u"Err": u""}) def test_create_with_profile(self): """ Calling the ``/VolumerDriver.Create`` API with an ``Opts`` value of "profile=[gold,silver,bronze] in the request body JSON create a volume with a given name with [gold,silver,bronze] profile. """ profile = sampled_from(["gold", "silver", "bronze"]).example() name = random_name(self) d = self.assertResult(b"POST", b"/VolumeDriver.Create", { u"Name": name, 'Opts': { u"profile": profile } }, OK, {u"Err": u""}) d.addCallback( lambda _: self.flocker_client.list_datasets_configuration()) d.addCallback(list) d.addCallback( lambda result: self.assertItemsEqual(result, [ Dataset(dataset_id=result[0].dataset_id, primary=self.NODE_A, maximum_size=int(DEFAULT_SIZE.to_Byte()), metadata={ NAME_FIELD: name, u"clusterhq:flocker:profile": unicode(profile) }) ])) return d def test_create_with_size(self): """ Calling the ``/VolumerDriver.Create`` API with an ``Opts`` value of "size=<somesize> in the request body JSON create a volume with a given name and random size between 1-100G """ name = random_name(self) size = integers(min_value=1, max_value=75).example() expression = volume_expression.example() size_opt = "".join(str(size)) + expression d = self.assertResult(b"POST", b"/VolumeDriver.Create", { u"Name": name, 'Opts': { u"size": size_opt } }, OK, {u"Err": u""}) real_size = int(parse_num(size_opt).to_Byte()) d.addCallback( lambda _: self.flocker_client.list_datasets_configuration()) d.addCallback(list) d.addCallback( lambda result: self.assertItemsEqual(result, [ Dataset(dataset_id=result[0].dataset_id, primary=self.NODE_A, maximum_size=real_size, metadata={ NAME_FIELD: name, u"maximum_size": unicode(real_size) }) ])) return d @given(expr=volume_expression, size=integers(min_value=75, max_value=100)) def test_parsenum_size(self, expr, size): """ Send different forms of size expressions to ``parse_num``, we expect G(Gigabyte) size results. :param expr str: A string representing the size expression :param size int: A string representing the volume size """ expected_size = int(GiB(size).to_Byte()) return self.assertEqual(expected_size, int(parse_num(str(size) + expr).to_Byte())) @given(expr=sampled_from(["KB", "MB", "GB", "TB", ""]), size=integers(min_value=1, max_value=100)) def test_parsenum_all_sizes(self, expr, size): """ Send standard size expressions to ``parse_num`` in many sizes, we expect to get correct size results. :param expr str: A string representing the size expression :param size int: A string representing the volume size """ if expr is "KB": expected_size = int(KiB(size).to_Byte()) elif expr is "MB": expected_size = int(MiB(size).to_Byte()) elif expr is "GB": expected_size = int(GiB(size).to_Byte()) elif expr is "TB": expected_size = int(TiB(size).to_Byte()) else: expected_size = int(Byte(size).to_Byte()) return self.assertEqual(expected_size, int(parse_num(str(size) + expr).to_Byte())) @given(size=sampled_from( [u"foo10Gb", u"10bar10", "10foogib", "10Gfoo", "GIB", "bar10foo"])) def test_parsenum_bad_size(self, size): """ Send unacceptable size expressions, upon error users should expect to receive Flocker's ``DEFAULT_SIZE`` :param size str: A string representing the bad volume size """ return self.assertEqual(int(DEFAULT_SIZE.to_Byte()), int(parse_num(size).to_Byte())) def create(self, name): """ Call the ``/VolumeDriver.Create`` API to create a volume with the given name. :param unicode name: The name of the volume to create. :return: ``Deferred`` that fires when the volume that was created. """ return self.assertResult(b"POST", b"/VolumeDriver.Create", {u"Name": name}, OK, {u"Err": u""}) def test_create_creates(self): """ ``/VolumeDriver.Create`` creates a new dataset in the configuration. """ name = u"myvol" d = self.create(name) d.addCallback( lambda _: self.flocker_client.list_datasets_configuration()) d.addCallback(list) d.addCallback( lambda result: self.assertItemsEqual(result, [ Dataset(dataset_id=result[0].dataset_id, primary=self.NODE_A, maximum_size=int(DEFAULT_SIZE.to_Byte()), metadata={NAME_FIELD: name}) ])) return d def test_create_duplicate_name(self): """ If a dataset with the given name already exists, ``/VolumeDriver.Create`` succeeds without create a new volume. """ name = u"thename" # Create a dataset out-of-band with matching name but non-matching # dataset ID: d = self.flocker_client.create_dataset(self.NODE_A, int(DEFAULT_SIZE.to_Byte()), metadata={NAME_FIELD: name}) d.addCallback(lambda _: self.create(name)) d.addCallback( lambda _: self.flocker_client.list_datasets_configuration()) d.addCallback(lambda results: self.assertEqual(len(list(results)), 1)) return d def test_create_duplicate_name_race_condition(self): """ If a dataset with the given name is created while the ``/VolumeDriver.Create`` call is in flight, the call does not result in an error. """ name = u"thename" # Create a dataset out-of-band with matching dataset ID and name # which the docker plugin won't be able to see. def create_after_list(): # Clean up the patched version: del self.flocker_client.list_datasets_configuration # But first time we're called, we create dataset and lie about # its existence: d = self.flocker_client.create_dataset(self.NODE_A, int(DEFAULT_SIZE.to_Byte()), metadata={NAME_FIELD: name}) d.addCallback( lambda _: DatasetsConfiguration(tag=u"1234", datasets={})) return d self.flocker_client.list_datasets_configuration = create_after_list return self.create(name) def _flush_volume_plugin_reactor_on_endpoint_render(self): """ This method patches ``self.app`` so that after any endpoint is rendered, the reactor used by the volume plugin is advanced repeatedly until there are no more ``delayedCalls`` pending on the reactor. """ real_execute_endpoint = self.app.execute_endpoint def patched_execute_endpoint(*args, **kwargs): val = real_execute_endpoint(*args, **kwargs) while self.volume_plugin_reactor.getDelayedCalls(): pending_calls = self.volume_plugin_reactor.getDelayedCalls() next_expiration = min(t.getTime() for t in pending_calls) now = self.volume_plugin_reactor.seconds() self.volume_plugin_reactor.advance( max(0.0, next_expiration - now)) return val self.patch(self.app, 'execute_endpoint', patched_execute_endpoint) def test_mount(self): """ ``/VolumeDriver.Mount`` sets the primary of the dataset with matching name to the current node and then waits for the dataset to actually arrive. """ name = u"myvol" dataset_id = uuid4() # Create dataset on a different node: d = self.flocker_client.create_dataset(self.NODE_B, int(DEFAULT_SIZE.to_Byte()), metadata={NAME_FIELD: name}, dataset_id=dataset_id) self._flush_volume_plugin_reactor_on_endpoint_render() # Pretend that it takes 5 seconds for the dataset to get established on # Node A. self.volume_plugin_reactor.callLater( 5.0, self.flocker_client.synchronize_state) d.addCallback(lambda _: self.assertResult( b"POST", b"/VolumeDriver.Mount", {u"Name": name}, OK, { u"Err": u"", u"Mountpoint": u"/flocker/{}".format(dataset_id) })) d.addCallback(lambda _: self.flocker_client.list_datasets_state()) def final_assertions(datasets): self.assertEqual( [self.NODE_A], [d.primary for d in datasets if d.dataset_id == dataset_id]) # There should be less than 20 calls to list_datasets_state over # the course of 5 seconds. self.assertLess( self.flocker_client.num_calls('list_datasets_state'), 20) d.addCallback(final_assertions) return d def test_mount_timeout(self): """ ``/VolumeDriver.Mount`` sets the primary of the dataset with matching name to the current node and then waits for the dataset to actually arrive. If it does not arrive within 120 seconds, then it returns an error up to docker. """ name = u"myvol" dataset_id = uuid4() # Create dataset on a different node: d = self.flocker_client.create_dataset(self.NODE_B, int(DEFAULT_SIZE.to_Byte()), metadata={NAME_FIELD: name}, dataset_id=dataset_id) self._flush_volume_plugin_reactor_on_endpoint_render() # Pretend that it takes 500 seconds for the dataset to get established # on Node A. This should be longer than the timeout. self.volume_plugin_reactor.callLater( 500.0, self.flocker_client.synchronize_state) d.addCallback(lambda _: self.assertResult( b"POST", b"/VolumeDriver.Mount", {u"Name": name}, OK, { u"Err": u"Timed out waiting for dataset to mount.", u"Mountpoint": u"" })) return d def test_mount_already_exists(self): """ ``/VolumeDriver.Mount`` sets the primary of the dataset with matching name to the current node and then waits for the dataset to actually arrive when used by the volumes that already exist and don't have a special dataset ID. """ name = u"myvol" d = self.flocker_client.create_dataset(self.NODE_A, int(DEFAULT_SIZE.to_Byte()), metadata={NAME_FIELD: name}) def created(dataset): self.flocker_client.synchronize_state() result = self.assertResult( b"POST", b"/VolumeDriver.Mount", {u"Name": name}, OK, { u"Err": u"", u"Mountpoint": u"/flocker/{}".format(dataset.dataset_id) }) result.addCallback( lambda _: self.flocker_client.list_datasets_state()) result.addCallback(lambda ds: self.assertEqual([self.NODE_A], [ d.primary for d in ds if d.dataset_id == dataset.dataset_id ])) return result d.addCallback(created) return d def test_unknown_mount(self): """ ``/VolumeDriver.Mount`` returns an error when asked to mount a non-existent volume. """ name = u"myvol" return self.assertResult( b"POST", b"/VolumeDriver.Mount", {u"Name": name}, OK, {u"Err": u"Could not find volume with given name."}) def test_path(self): """ ``/VolumeDriver.Path`` returns the mount path of the given volume if it is currently known. """ name = u"myvol" d = self.create(name) # The dataset arrives as state: d.addCallback(lambda _: self.flocker_client.synchronize_state()) d.addCallback(lambda _: self.assertResponseCode( b"POST", b"/VolumeDriver.Mount", {u"Name": name}, OK)) d.addCallback( lambda _: self.flocker_client.list_datasets_configuration()) d.addCallback(lambda datasets_config: self.assertResult( b"POST", b"/VolumeDriver.Path", {u"Name": name}, OK, { u"Err": u"", u"Mountpoint": u"/flocker/{}".format(datasets_config.datasets.keys()[0]) })) return d def test_path_existing(self): """ ``/VolumeDriver.Path`` returns the mount path of the given volume if it is currently known, including for a dataset that was created not by the plugin. """ name = u"myvol" d = self.flocker_client.create_dataset(self.NODE_A, int(DEFAULT_SIZE.to_Byte()), metadata={NAME_FIELD: name}) def created(dataset): self.flocker_client.synchronize_state() return self.assertResult( b"POST", b"/VolumeDriver.Path", {u"Name": name}, OK, { u"Err": u"", u"Mountpoint": u"/flocker/{}".format(dataset.dataset_id) }) d.addCallback(created) return d def test_unknown_path(self): """ ``/VolumeDriver.Path`` returns an error when asked for the mount path of a non-existent volume. """ name = u"myvol" return self.assertResult( b"POST", b"/VolumeDriver.Path", {u"Name": name}, OK, {u"Err": u"Could not find volume with given name."}) def test_non_local_path(self): """ ``/VolumeDriver.Path`` returns an error when asked for the mount path of a volume that is not mounted locally. This can happen as a result of ``docker inspect`` on a container that has been created but is still waiting for its volume to arrive from another node. It seems like Docker may also call this after ``/VolumeDriver.Create``, so again while waiting for a volume to arrive. """ name = u"myvol" dataset_id = uuid4() # Create dataset on node B: d = self.flocker_client.create_dataset(self.NODE_B, int(DEFAULT_SIZE.to_Byte()), metadata={NAME_FIELD: name}, dataset_id=dataset_id) d.addCallback(lambda _: self.flocker_client.synchronize_state()) # Ask for path on node A: d.addCallback(lambda _: self.assertResult( b"POST", b"/VolumeDriver.Path", {u"Name": name}, OK, { u"Err": "Volume not available.", u"Mountpoint": u"" })) return d @capture_logging(lambda self, logger: self.assertEqual( len(logger.flushTracebacks(CustomException)), 1)) def test_unexpected_error_reporting(self, logger): """ If an unexpected error occurs Docker gets back a useful error message. """ def error(): raise CustomException("I've made a terrible mistake") self.patch(self.flocker_client, "list_datasets_configuration", error) return self.assertResult( b"POST", b"/VolumeDriver.Path", {u"Name": u"whatever"}, OK, {u"Err": "CustomException: I've made a terrible mistake"}) @capture_logging(None) def test_bad_request(self, logger): """ If a ``BadRequest`` exception is raised it is converted to appropriate JSON. """ def error(): raise make_bad_request(code=423, Err=u"no good") self.patch(self.flocker_client, "list_datasets_configuration", error) return self.assertResult(b"POST", b"/VolumeDriver.Path", {u"Name": u"whatever"}, 423, {u"Err": "no good"}) def test_unsupported_method(self): """ If an unsupported method is requested the 405 Not Allowed response code is returned. """ return self.assertResponseCode(b"BAD_METHOD", b"/VolumeDriver.Path", None, NOT_ALLOWED) def test_unknown_uri(self): """ If an unknown URI path is requested the 404 Not Found response code is returned. """ return self.assertResponseCode(b"BAD_METHOD", b"/xxxnotthere", None, NOT_FOUND) def test_empty_host(self): """ If an empty host header is sent to the Docker plugin it does not blow up, instead operating normally. E.g. for ``Plugin.Activate`` call returns the ``Implements`` response. """ return self.assertResult(b"POST", b"/Plugin.Activate", 12345, OK, {u"Implements": [u"VolumeDriver"]}, additional_headers={b"Host": [""]}) def test_get(self): """ ``/VolumeDriver.Get`` returns the mount path of the given volume if it is currently known. """ name = u"myvol" d = self.create(name) # The dataset arrives as state: d.addCallback(lambda _: self.flocker_client.synchronize_state()) d.addCallback(lambda _: self.assertResponseCode( b"POST", b"/VolumeDriver.Mount", {u"Name": name}, OK)) d.addCallback( lambda _: self.flocker_client.list_datasets_configuration()) d.addCallback(lambda datasets_config: self.assertResult( b"POST", b"/VolumeDriver.Get", {u"Name": name}, OK, { u"Err": u"", u"Volume": { u"Name": name, u"Mountpoint": u"/flocker/{}".format(datasets_config.datasets.keys()[0]) } })) return d def test_get_existing(self): """ ``/VolumeDriver.Get`` returns the mount path of the given volume if it is currently known, including for a dataset that was created not by the plugin. """ name = u"myvol" d = self.flocker_client.create_dataset(self.NODE_A, int(DEFAULT_SIZE.to_Byte()), metadata={NAME_FIELD: name}) def created(dataset): self.flocker_client.synchronize_state() return self.assertResult( b"POST", b"/VolumeDriver.Get", {u"Name": name}, OK, { u"Err": u"", u"Volume": { u"Name": name, u"Mountpoint": u"/flocker/{}".format( dataset.dataset_id) } }) d.addCallback(created) return d def test_unknown_get(self): """ ``/VolumeDriver.Get`` returns an error when asked for the mount path of a non-existent volume. """ name = u"myvol" return self.assertResult( b"POST", b"/VolumeDriver.Get", {u"Name": name}, OK, {u"Err": u"Could not find volume with given name."}) def test_non_local_get(self): """ ``/VolumeDriver.Get`` returns an empty mount point when asked about a volume that is not mounted locally. """ name = u"myvol" dataset_id = uuid4() # Create dataset on node B: d = self.flocker_client.create_dataset(self.NODE_B, int(DEFAULT_SIZE.to_Byte()), metadata={NAME_FIELD: name}, dataset_id=dataset_id) d.addCallback(lambda _: self.flocker_client.synchronize_state()) # Ask for path on node A: d.addCallback(lambda _: self.assertResult( b"POST", b"/VolumeDriver.Get", {u"Name": name}, OK, { u"Err": u"", u"Volume": { u"Name": name, u"Mountpoint": u"" } })) return d def test_list(self): """ ``/VolumeDriver.List`` returns the mount path of the given volume if it is currently known and an empty mount point for non-local volumes. """ name = u"myvol" remote_name = u"myvol3" d = gatherResults([ self.flocker_client.create_dataset(self.NODE_A, int(DEFAULT_SIZE.to_Byte()), metadata={NAME_FIELD: name}), self.flocker_client.create_dataset( self.NODE_B, int(DEFAULT_SIZE.to_Byte()), metadata={NAME_FIELD: remote_name}) ]) # The datasets arrive as state: d.addCallback(lambda _: self.flocker_client.synchronize_state()) d.addCallback( lambda _: self.flocker_client.list_datasets_configuration()) d.addCallback(lambda datasets_config: self.assertResult( b"POST", b"/VolumeDriver.List", {}, OK, { u"Err": u"", u"Volumes": sorted([ { u"Name": name, u"Mountpoint": u"/flocker/{}".format([ key for (key, value) in datasets_config.datasets. items() if value.metadata["name"] == name ][0]) }, { u"Name": remote_name, u"Mountpoint": u"" }, ]) })) return d def test_list_no_metadata_name(self): """ ``/VolumeDriver.List`` omits volumes that don't have a metadata field for their name. """ d = self.flocker_client.create_dataset(self.NODE_A, int(DEFAULT_SIZE.to_Byte()), metadata={}) d.addCallback(lambda _: self.assertResult( b"POST", b"/VolumeDriver.List", {}, OK, { u"Err": u"", u"Volumes": [] })) return d
self.gamespace_p2.puck.speedx = pickle.loads(data[2]) self.gamespace_p2.puck.speedy = pickle.loads(data[3]) def quit(self): self.transport.loseConnection() def write(self, data): self.transport.write(data) class ClientConnFactory(ClientFactory): def __init__(self, gs): self.gs = gs self.client_connection = ClientConn(self.gs) def buildProtocol(self, addr): self.gs.quit = self.client_connection.quit self.gs.write = self.client_connection.write return self.client_connection if __name__ == '__main__': gs = GameSpace() # 3) Start the game loop lc = LoopingCall(gs.game_loop) lc.start(1 / 60) reactor.connectTCP(CLIENT_HOST, CLIENT_PORT, ClientConnFactory(gs)) reactor.run() gs.game_loop() lc.stop()
class WindowManager(object): WINDOW_KEY = 'windows' FLIGHT_KEY = 'inflight' FLIGHT_STATS_KEY = 'flightstats' MAP_KEY = 'keymap' def __init__(self, redis, window_size=100, flight_lifetime=None, gc_interval=10): self.window_size = window_size self.flight_lifetime = flight_lifetime or (gc_interval * window_size) self.redis = redis self.clock = self.get_clock() self.gc = LoopingCall(self.clear_expired_flight_keys) self.gc.clock = self.clock self.gc.start(gc_interval) self._monitor = None def noop(self, *args, **kwargs): pass def stop(self): if self._monitor and self._monitor.running: self._monitor.stop() if self.gc.running: self.gc.stop() def get_windows(self): return self.redis.zrange(self.window_key(), 0, -1) @inlineCallbacks def window_exists(self, window_id): score = yield self.redis.zscore(self.window_key(), window_id) if score is not None: returnValue(True) returnValue(False) def window_key(self, *keys): return ':'.join([self.WINDOW_KEY] + map(unicode, keys)) def flight_key(self, *keys): return self.window_key(self.FLIGHT_KEY, *keys) def stats_key(self, *keys): return self.window_key(self.FLIGHT_STATS_KEY, *keys) def map_key(self, *keys): return self.window_key(self.MAP_KEY, *keys) def get_clock(self): return reactor def get_clocktime(self): return self.clock.seconds() @inlineCallbacks def create_window(self, window_id, strict=False): if strict and (yield self.window_exists(window_id)): raise WindowException('Window already exists: %s' % (window_id,)) clock_time = self.get_clocktime() yield self.redis.zadd(self.WINDOW_KEY, **{ window_id: clock_time, }) returnValue(clock_time) @inlineCallbacks def remove_window(self, window_id): waiting_list = yield self.count_waiting(window_id) if waiting_list: raise WindowException('Window not empty') yield self.redis.zrem(self.WINDOW_KEY, window_id) @inlineCallbacks def add(self, window_id, data, key=None): key = key or uuid.uuid4().get_hex() # The redis.set() has to complete before redis.lpush(), # otherwise the key can be popped from the window before the # data is available. yield self.redis.set(self.window_key(window_id, key), json.dumps(data)) yield self.redis.lpush(self.window_key(window_id), key) returnValue(key) @inlineCallbacks def get_next_key(self, window_id): window_key = self.window_key(window_id) inflight_key = self.flight_key(window_id) waiting_list = yield self.count_waiting(window_id) if waiting_list == 0: return flight_size = yield self.count_in_flight(window_id) room_available = self.window_size - flight_size if room_available > 0: log.debug('Window %s has space for %s' % (window_key, room_available)) next_key = yield self.redis.rpoplpush(window_key, inflight_key) if next_key: yield self._set_timestamp(window_id, next_key) returnValue(next_key) def _set_timestamp(self, window_id, flight_key): return self.redis.zadd(self.stats_key(window_id), **{ flight_key: self.get_clocktime(), }) def _clear_timestamp(self, window_id, flight_key): return self.redis.zrem(self.stats_key(window_id), flight_key) def count_waiting(self, window_id): window_key = self.window_key(window_id) return self.redis.llen(window_key) def count_in_flight(self, window_id): flight_key = self.flight_key(window_id) return self.redis.llen(flight_key) def get_expired_flight_keys(self, window_id): return self.redis.zrangebyscore(self.stats_key(window_id), '-inf', self.get_clocktime() - self.flight_lifetime) @inlineCallbacks def clear_expired_flight_keys(self): windows = yield self.get_windows() for window_id in windows: expired_keys = yield self.get_expired_flight_keys(window_id) for key in expired_keys: yield self.redis.lrem(self.flight_key(window_id), key, 1) @inlineCallbacks def get_data(self, window_id, key): json_data = yield self.redis.get(self.window_key(window_id, key)) returnValue(json.loads(json_data)) @inlineCallbacks def remove_key(self, window_id, key): yield self.redis.lrem(self.flight_key(window_id), key, 1) yield self.redis.delete(self.window_key(window_id, key)) yield self.redis.delete(self.stats_key(window_id, key)) yield self.clear_external_id(window_id, key) yield self._clear_timestamp(window_id, key) @inlineCallbacks def set_external_id(self, window_id, flight_key, external_id): yield self.redis.set(self.map_key(window_id, 'internal', external_id), flight_key) yield self.redis.set(self.map_key(window_id, 'external', flight_key), external_id) def get_internal_id(self, window_id, external_id): return self.redis.get(self.map_key(window_id, 'internal', external_id)) def get_external_id(self, window_id, flight_key): return self.redis.get(self.map_key(window_id, 'external', flight_key)) @inlineCallbacks def clear_external_id(self, window_id, flight_key): external_id = yield self.get_external_id(window_id, flight_key) if external_id: yield self.redis.delete(self.map_key(window_id, 'external', flight_key)) yield self.redis.delete(self.map_key(window_id, 'internal', external_id)) def monitor(self, key_callback, interval=10, cleanup=True, cleanup_callback=None): if self._monitor is not None: raise WindowException('Monitor already started') self._monitor = LoopingCall(lambda: self._monitor_windows( key_callback, cleanup, cleanup_callback)) self._monitor.clock = self.get_clock() self._monitor.start(interval) @inlineCallbacks def _monitor_windows(self, key_callback, cleanup=True, cleanup_callback=None): windows = yield self.get_windows() for window_id in windows: key = (yield self.get_next_key(window_id)) while key: yield key_callback(window_id, key) key = (yield self.get_next_key(window_id)) # Remove empty windows if required if cleanup and not ((yield self.count_waiting(window_id)) or (yield self.count_in_flight(window_id))): if cleanup_callback: cleanup_callback(window_id) yield self.remove_window(window_id)
class Core(CorePluginBase): totals = {} #class var to catch only updating this once per session in enable. def enable(self): log.debug("Stats plugin enabled") self.core = component.get("Core") self.stats ={} self.count = {} self.intervals = [1, 5, 30, 300] self.last_update = {} t = time.time() for i in self.intervals: self.stats[i] = {} self.last_update[i] = t self.count[i] = 0 self.config = configmanager.ConfigManager("stats.conf", DEFAULT_PREFS) self.saved_stats = configmanager.ConfigManager("stats.totals", DEFAULT_TOTALS) if self.totals == {}: self.totals.update(self.saved_stats.config) self.length = self.config["length"] #self.stats = get_key(self.saved_stats, "stats") or {} self.stats_keys = [] self.add_stats( 'upload_rate', 'download_rate', 'num_connections', 'dht_nodes', 'dht_cache_nodes', 'dht_torrents', 'num_peers', ) self.update_stats() self.update_timer = LoopingCall(self.update_stats) self.update_timer.start(self.config["update_interval"]) self.save_timer = LoopingCall(self.save_stats) self.save_timer.start(60) def disable(self): self.save_stats() try: self.update_timer.stop() self.save_timer.stop() except: pass def add_stats(self, *stats): for stat in stats: if stat not in self.stats_keys: self.stats_keys.append(stat) for i in self.intervals: if stat not in self.stats[i]: self.stats[i][stat] = [] def update_stats(self): try: #Get all possible stats! stats = {} for key in self.stats_keys: #try all keys we have, very inefficient but saves having to #work out where a key comes from... try: stats.update(self.core.get_session_status([key])) except AttributeError: pass stats["num_connections"] = self.core.get_num_connections() stats.update(self.core.get_config_values(["max_download", "max_upload", "max_num_connections"])) # status = self.core.session.status() # for stat in dir(status): # if not stat.startswith('_') and stat not in stats: # stats[stat] = getattr(status, stat, None) update_time = time.time() self.last_update[1] = update_time #extract the ones we are interested in #adding them to the 1s array for stat, stat_list in self.stats[1].iteritems(): if stat in stats: stat_list.insert(0, int(stats[stat])) else: stat_list.insert(0, 0) if len(stat_list) > self.length: stat_list.pop() def update_interval(interval, base, multiplier): self.count[interval] = self.count[interval] + 1 if self.count[interval] >= interval: self.last_update[interval] = update_time self.count[interval] = 0 current_stats = self.stats[interval] for stat, stat_list in self.stats[base].iteritems(): try: avg = mean(stat_list[0:multiplier]) except ValueError: avg = 0 current_stats[stat].insert(0, avg) if len(current_stats[stat]) > self.length: current_stats[stat].pop() update_interval(5, 1, 5) update_interval(30, 5, 6) update_interval(300, 30, 10) except Exception, e: log.error("Stats update error %s" % e) return True
class NormalPMTFlow(LabradServer): name = 'NormalPMTFlow' onNewCount = Signal(SIGNALID, 'signal: new count', 'v') onNewSetting = Signal(SIGNALID + 1, 'signal: new setting', '(ss)') ready = Signal(SIGNALID + 2, 'signal: ready', 'b') @inlineCallbacks def initServer(self): self.saveFolder = ['', 'PMT Counts'] self.dataSetName = 'PMT Counts' self.collectTimes = {'Normal': 0.100, 'Differential': 0.100} self.lastDifferential = {'ON': 0, 'OFF': 0} self.currentMode = 'Normal' self.dv = None self.pulser = None self.collectTimeRange = None self.dataSet = None self.requestList = [] self.listeners = set() self.recording = LoopingCall(self._record) yield self.connect_data_vault() yield self.connect_pulser() @inlineCallbacks def connect_data_vault(self): try: self.dv = yield self.client.data_vault yield self.dv.cd(self.saveFolder, True) print 'Connected: Data Vault' except AttributeError: self.dv = None print 'Not Connected: Data Vault' @inlineCallbacks def disconnect_data_vault(self): print 'Not Connected: Data Vault' self.dv = None @inlineCallbacks def connect_pulser(self): try: self.pulser = yield self.client.pulser self.collectTimeRange = yield self.pulser.get_collection_time() print 'Connected: Pulser' except AttributeError: self.pulser = None self.collectTimeRange = None print 'Not Connected: Pulser' @inlineCallbacks def disconnect_pulser(self): print 'Not Connected: Pulser' self.pulser = None self.dataSet = None def initContext(self, c): """Initialize a new context object.""" self.listeners.add(c.ID) def expireContext(self, c): self.listeners.remove(c.ID) def getOtherListeners(self, c): notified = self.listeners.copy() notified.remove(c.ID) return notified @inlineCallbacks def makeNewDataSet(self): folder = self.saveFolder name = self.dataSetName yield self.dv.cd(folder, True) self.dataSet = yield self.dv.new( name, [('t', 'num')], [('KiloCounts/sec', '866 ON', 'num'), ('KiloCounts/sec', '866 OFF', 'num'), ('KiloCounts/sec', 'Differential Signal', 'num')]) self.startTime = time.time() yield self.addParameters() returnValue(self.dataSet) @inlineCallbacks def addParameters(self): yield self.dv.add_parameter('plotLive', True) yield self.dv.add_parameter('startTime', self.startTime) @setting(0, 'Set Save Folder', folder='*s', returns='') def setSaveFolder(self, c, folder): try: yield self.dv.cd(folder, True) self.saveFolder = folder except AttributeError: print 'failing' from labrad.types import Error raise Error("NotReady") @setting(1, 'Start New Dataset', setName='s', returns='s') def setNewDataSet(self, c, setName=None): """Starts new dataset, if name not provided, it will be the same""" if setName is not None: self.dataSetName = setName dataset = yield self.makeNewDataSet() name = dataset[1] otherListeners = self.getOtherListeners(c) self.onNewSetting(('dataset', name), otherListeners) returnValue(name) @setting(2, "Set Mode", mode='s', returns='') def setMode(self, c, mode): """ Start recording Time Resolved Counts into Data Vault """ if mode not in self.collectTimes.keys(): raise Exception('Incorrect Mode') if not self.recording.running: self.currentMode = mode yield self.pulser.set_mode(mode) else: yield self.dostopRecording() self.currentMode = mode yield self.pulser.set_mode(mode) yield self.dorecordData() otherListeners = self.getOtherListeners(c) self.onNewSetting(('mode', mode), otherListeners) @setting(3, 'getCurrentMode', returns='s') def getCurrentMode(self, c): """ Returns the currently running mode """ return self.currentMode @setting(4, 'Record Data', returns='') def recordData(self, c): """ Starts recording data of the current PMT mode into datavault """ setname = yield self.dorecordData() otherListeners = self.getOtherListeners(c) if setname is not None: setname = setname[1] self.onNewSetting(('dataset', setname), otherListeners) self.onNewSetting(('state', 'on'), otherListeners) @inlineCallbacks def dorecordData(self): newSet = None self.keepRunning = True yield self.pulser.set_collection_time( self.collectTimes[self.currentMode], self.currentMode) yield self.pulser.set_mode(self.currentMode) if self.currentMode == 'Differential': yield self._programPulserDiff() if self.dataSet is None: newSet = yield self.makeNewDataSet() self.recording.start(self.collectTimes[self.currentMode] / 2.0) returnValue(newSet) @setting(5, returns='') def stopRecording(self, c): """ Stop recording counts into Data Vault """ yield self.dostopRecording() otherListeners = self.getOtherListeners(c) self.onNewSetting(('state', 'off'), otherListeners) @inlineCallbacks def dostopRecording(self): yield self.recording.stop() if self.currentMode == 'Differential': yield self._stopPulserDiff() @setting(6, returns='b') def isRunning(self, c): """ Returns whether or not currently recording """ return self.recording.running @setting(7, returns='s') def currentDataSet(self, c): if self.dataSet is None: return '' name = self.dataSet[1] return name @setting(8, 'Set Time Length', timelength='v') def setTimeLength(self, c, timelength): """Sets the time length for the current mode""" mode = self.currentMode if mode not in self.collectTimes.keys(): raise Exception('Incorrect Mode') if not self.collectTimeRange[0] <= timelength <= self.collectTimeRange[ 1]: raise Exception('Incorrect Recording Time') self.collectTimes[mode] = timelength initrunning = self.recording.running #if recording when the call is made, need to stop and restart if initrunning: yield self.recording.stop() yield self.pulser.set_collection_time(timelength, mode) if initrunning: if mode == 'Differential': yield self._stopPulserDiff() yield self._programPulserDiff() self.recording.start(timelength / 2.0) otherListeners = self.getOtherListeners(c) self.onNewSetting(('timelength', str(timelength)), otherListeners) @setting(9, 'Get Next Counts', type='s', number='w', average='b', returns=['*v', 'v']) def getNextCounts(self, c, type, number, average=False): """ Acquires next number of counts, where type can be 'ON' or 'OFF' or 'DIFF' Average is optionally True if the counts should be averaged Note in differential mode, Diff counts get updates every time, but ON and OFF get updated every 2 times. """ if type not in ['ON', 'OFF', 'DIFF']: raise Exception('Incorrect type') if type in ['OFF', 'DIFF'] and self.currentMode == 'Normal': raise Exception('in the wrong mode to process this request') if not 0 < number < 1000: raise Exception('Incorrect Number') if not self.recording.running: raise Exception('Not currently recording') d = Deferred() self.requestList.append(self.readingRequest(d, type, number)) data = yield d if average: data = sum(data) / len(data) returnValue(data) @setting(10, 'Get Time Length', returns='v') def getMode(self, c): """ Returns the current timelength of in the current mode """ return self.collectTimes[self.currentMode] @setting(11, 'Get Time Length Range', returns='(vv)') def get_time_length_range(self, c): return self.collectTimeRange @inlineCallbacks def _programPulserDiff(self): yield self.pulser.new_sequence() countRate = self.collectTimes['Differential'] yield self.pulser.add_ttl_pulse('DiffCountTrigger', 0.0, 10.0e-6) yield self.pulser.add_ttl_pulse('DiffCountTrigger', countRate, 10.0e-6) yield self.pulser.add_ttl_pulse('866DP', 0.0, countRate) yield self.pulser.extend_sequence_length(2 * countRate) yield self.pulser.program_sequence() yield self.pulser.start_infinite() @inlineCallbacks def _stopPulserDiff(self): yield self.pulser.complete_infinite_iteration() yield self.pulser.wait_sequence_done() yield self.pulser.stop_sequence() class readingRequest(): def __init__(self, d, type, count): self.d = d self.count = count self.type = type self.data = [] def is_fulfilled(self): return len(self.data) == self.count def processRequests(self, data): if not len(self.requestList): return for dataPoint in data: for item, req in enumerate(self.requestList): if dataPoint[1] != 0 and req.type == 'ON': req.data.append(dataPoint[1]) if dataPoint[2] != 0 and req.type == 'OFF': req.data.append(dataPoint[1]) if dataPoint[3] != 0 and req.type == 'DIFF': req.data.append(dataPoint[1]) if req.is_fulfilled(): req.d.callback(req.data) del (self.requestList[item]) @inlineCallbacks def _record(self): rawdata = yield self.pulser.get_pmt_counts() print rawdata if len(rawdata) != 0: if self.currentMode == 'Normal': toDataVault = [ [elem[2] - self.startTime, elem[0], 0, 0] for elem in rawdata ] # converting to format [time, normal count, 0 , 0] elif self.currentMode == 'Differential': toDataVault = self.convertDifferential(rawdata) self.processRequests( toDataVault) #if we have any requests, process them self.processSignals(toDataVault) yield self.dv.add(toDataVault) def processSignals(self, data): lastPt = data[-1] NormalCount = lastPt[1] self.onNewCount(NormalCount) def convertDifferential(self, rawdata): totalData = [] for dataPoint in rawdata: t = str(dataPoint[1]) self.lastDifferential[t] = float(dataPoint[0]) diff = self.lastDifferential['ON'] - self.lastDifferential['OFF'] totalData.append([ dataPoint[2] - self.startTime, self.lastDifferential['ON'], self.lastDifferential['OFF'], diff ]) return totalData
class MetricLogObserver(util_service.ReconfigurableServiceMixin, service.MultiService): _reactor = reactor def __init__(self): service.MultiService.__init__(self) self.setName('metrics') self.enabled = False self.periodic_task = None self.periodic_interval = None self.log_task = None self.log_interval = None # Mapping of metric type to handlers for that type self.handlers = {} # Register our default handlers self.registerHandler(MetricCountEvent, MetricCountHandler(self)) self.registerHandler(MetricTimeEvent, MetricTimeHandler(self)) self.registerHandler(MetricAlarmEvent, MetricAlarmHandler(self)) self.getHandler(MetricCountEvent).addWatcher( AttachedWorkersWatcher(self)) def reconfigServiceWithBuildbotConfig(self, new_config): # first, enable or disable if new_config.metrics is None: self.disable() else: self.enable() metrics_config = new_config.metrics # Start up periodic logging log_interval = metrics_config.get('log_interval', 60) if log_interval != self.log_interval: if self.log_task: self.log_task.stop() self.log_task = None if log_interval: self.log_task = LoopingCall(self.report) self.log_task.clock = self._reactor self.log_task.start(log_interval) # same for the periodic task periodic_interval = metrics_config.get('periodic_interval', 10) if periodic_interval != self.periodic_interval: if self.periodic_task: self.periodic_task.stop() self.periodic_task = None if periodic_interval: self.periodic_task = LoopingCall(periodicCheck, self._reactor) self.periodic_task.clock = self._reactor self.periodic_task.start(periodic_interval) # upcall return util_service.ReconfigurableServiceMixin.reconfigServiceWithBuildbotConfig( self, new_config) def stopService(self): self.disable() service.MultiService.stopService(self) def enable(self): if self.enabled: return log.addObserver(self.emit) self.enabled = True def disable(self): if not self.enabled: return if self.periodic_task: self.periodic_task.stop() self.periodic_task = None if self.log_task: self.log_task.stop() self.log_task = None log.removeObserver(self.emit) self.enabled = False def registerHandler(self, interface, handler): old = self.getHandler(interface) self.handlers[interface] = handler return old def getHandler(self, interface): return self.handlers.get(interface) def emit(self, eventDict): # Ignore non-statistic events metric = eventDict.get('metric') if not metric or not isinstance(metric, MetricEvent): return if metric.__class__ not in self.handlers: return h = self.handlers[metric.__class__] h.handle(eventDict, metric) for w in h.watchers: w.run() def asDict(self): retval = {} for interface, handler in self.handlers.items(): retval.update(handler.asDict()) return retval def report(self): try: for interface, handler in self.handlers.items(): report = handler.report() if not report: continue for line in report.split("\n"): log.msg(line) except Exception: log.err(None, "generating metric report")
def test_tls_auth_denied(self): """ A MQTT client offering the wrong certificate won't be authenticated. """ reactor, router, server_factory, session_factory = build_mqtt_server() real_reactor = selectreactor.SelectReactor() logger = make_logger() session, pump = connect_application_session( server_factory, ObservingSession, component_config=ComponentConfig(realm="mqtt")) endpoint = create_listening_endpoint_from_config( { "type": "tcp", "port": 1099, "interface": "0.0.0.0", "tls": { "certificate": "server.crt", "key": "server.key", "dhparam": "dhparam", "ca_certificates": ["ca.cert.pem", "intermediate.cert.pem"] }, }, FilePath(__file__).sibling('certs').path, real_reactor, logger) client_endpoint = create_connecting_endpoint_from_config( { "type": "tcp", "host": "127.0.0.1", "port": 1099, "tls": { # BAD key: trusted by the CA, but wrong ID "certificate": "client_1.crt", "hostname": "localhost", "key": "client_1.key", "ca_certificates": ["ca.cert.pem", "intermediate.cert.pem"] }, }, FilePath(__file__).sibling('certs').path, real_reactor, logger) p = [] l = endpoint.listen(server_factory) class TestProtocol(Protocol): data = b"" expected = (ConnACK(session_present=False, return_code=1).serialise()) def dataReceived(self_, data): self_.data = self_.data + data if len(self_.data) == len(self_.expected): self.assertEqual(self_.data, self_.expected) real_reactor.stop() @l.addCallback def _listening(factory): d = client_endpoint.connect(Factory.forProtocol(TestProtocol)) @d.addCallback def _(proto): p.append(proto) proto.transport.write( Connect( client_id="test123", flags=ConnectFlags(clean_session=False)).serialise()) proto.transport.write( Publish(duplicate=False, qos_level=1, retain=False, topic_name="test", payload=b"{}", packet_identifier=1).serialise()) lc = LoopingCall(pump.flush) lc.clock = real_reactor lc.start(0.01) def timeout(): print("Timing out :(") real_reactor.stop() print(self.logs.log_text.getvalue()) # Timeout, just in case real_reactor.callLater(10, timeout) real_reactor.run() client_protocol = p[0] # We get a CONNECT self.assertEqual( client_protocol.data, ConnACK(session_present=False, return_code=1).serialise()) client_protocol.data = b"" pump.flush() # No events! self.assertEqual(len(session.events), 0)
class AudioFromFiles(baseaudio.AudioDevice): _infp = _outfp = None LC = None def __init__(self): try: from __main__ import app except: log.msg("couldn't find app preferences, no file audio", system="audio") raise ValueError('no app for filename preferences?') device = app.getPref('audio_device') if not device: log.msg( "need to provide audio_device with files" " as infile,outfile", system="audio") raise ValueError('no audio_device specified') files = device.split(',', 2) if len(files) != 2: raise ValueError("device spec should be infile,outfile") self.infile, self.outfile = files baseaudio.AudioDevice.__init__(self) self.close() def _push_up_some_data(self): if not self.encoder or self._infp is None: return data = self._infp.read(320) #log.msg("pushing %d bytes from file"%(len(data)), system="audio") if self.encoder and data: self.encoder.handle_audio(data) def write(self, bytes): if self._outfp is not None: return self._outfp.write(bytes) def _getFiles(self): if self.infile: self._infp = open(self.infile, 'rb') else: self._infp = None if self.outfile: self._outfp = open(self.outfile, 'wb') else: self._outfp = None def _close(self): #print "close called", self._closed, self.LC if self._infp is not None: self._infp.close() if self._outfp is not None: self._outfp.close() self._closed = True self._infp = self._outfp = None self.LC.stop() self.LC = None def openDev(self): from shtoom.util import stack self._getFiles() #print "openDev called!", self._closed, self.LC, stack() if self.LC is not None: return if self._infp is None and self._outfp is None: self._getFiles() self.LC = LoopingCall(self._push_up_some_data) #print self, "creating LoopingCall", self.LC, stack() self.LC.start(0.020)
class JobPerspective(object): def __init__(self, jobId, jobSpec): self.jobId = jobId self.jobSpec = jobSpec self.mapping = {} self.health = JobHealth.OK self.task = LoopingCall(self.state) self._started = False self._finished = False def addSlave(self, slave, remoteJobId): self.mapping[slave] = remoteJobId def removeSlave(self, slave): self.mapping.pop(slave) @inlineCallbacks def handleSlaveError(self, slave, error): # XXX should handle different errors differently # for now we'll just assume the slave gets killed off log.error("Job health compromised: lost a slave server") self.health = JobHealth.ERROR log.warn("Removing slave %s://%s:%d/%s" % (slave.slaveSpec.scheme, slave.slaveSpec.host, slave.slaveSpec.port, slave.slaveSpec.path)) self.removeSlave(slave) SlaveAllocator.degrade(slave) # we should probably mark the job state as unknown and cancel # the whole thing, since it couldn't be completed as asked yield self._jobOp("stopJob", ignoreHealth=True) returnValue(True) def _jobIsStarted(self): if self._started == False: log.debug("Starting job %d" % self.jobId) for slave in self.mapping.iterkeys(): SlaveAllocator.markAsRunning(slave) self.task.start(120) self._started = True def _jobIsFinished(self): if self._finished == False: log.debug("Finishing job %d" % self.jobId) for slave in self.mapping.iterkeys(): SlaveAllocator.markAsFinished(slave) self.task.stop() self._finished = True # this is just a pass-through to the DeferredList callback def _jobOpSlaveCallback(self, value): return value # though here we want to intercept errors def _jobOpSlaveErrback(self, error, slave): self.handleSlaveError(slave, error) return error @inlineCallbacks def _jobOp(self, operation, *args, **kwargs): try: if kwargs["ignoreHealth"] == True: pass else: raise except: if self.health == JobHealth.ERROR: returnValue(False) requests = [] for slave, remoteId in self.mapping.iteritems(): request = getattr(slave, "%s" % operation)(remoteId, *args) request.addCallback(self._jobOpSlaveCallback) request.addErrback(self._jobOpSlaveErrback, slave) requests.append(request) deferredList = DeferredList(requests, consumeErrors=True) yield deferredList returnValue(deferredList.result) @inlineCallbacks def start(self): request = self._jobOp("startJob") yield request self._jobIsStarted() returnValue(request.result) def pause(self): return self._jobOp("pauseJob") def resume(self): return self._jobOp("resumeJob") def stop(self): return self._jobOp("stopJob") @inlineCallbacks def state(self): # this line is repeated twice: once at the beginning for # new calls to state(), and once after checking the DeferredList # in case the health has suffered during the call if self.health == JobHealth.ERROR: returnValue(JobState.ERROR) request = self._jobOp("jobState") yield request # see above if self.health == JobHealth.ERROR: returnValue(JobState.ERROR) states = [] for (result, state) in request.result: if result == True: states.append(int(json.loads(state))) jobState = AggregateJobResults._aggregateState(states) if jobState == JobState.COMPLETE: self._jobIsFinished() returnValue(jobState) @inlineCallbacks def results(self, shortResults): request = self._jobOp("jobResults", shortResults) yield request if self.health == JobHealth.ERROR: returnValue(False) results = request.result aggregateResults = AggregateJobResults() # decode all json. for speed's sake, try once as a list comprehension, but if one is # false then the list comprehension will fail -- fall back to a for loop, # rejecting failed responses decodedResults = [] try: decodedResults = \ [(status, JobResults(json.loads(result))) for (status, result) in results] except: log.debug("Could not decode all results. results: %s" % results) for (status, result) in results: if status == True: decodedResults.append( (status, JobResults(json.loads(result)))) # set the job ID to the master's job ID aggregateResults.job_id = self.jobId # combine and add results from all the slave servers. this # aggregates things like bytes transferred, requests completed, etc. aggregateResults.aggregate( [result for (status, result) in decodedResults], self.jobSpec.statsInterval, shortResults) # if we're doing no stats, cut out results_byTime complete if shortResults == True: try: del (aggregateResults.results_byTime) except AttributeError: pass # if the job is unhealthy, stop the whole thing, and overwrite whatever the # merged state was if self.health == JobHealth.ERROR: aggregateResults.job_state = JobState.ERROR # if the job is done, cancel the timer if aggregateResults.job_state == JobState.COMPLETE: self._jobIsFinished() returnValue(aggregateResults)
current[u'sensors'].append({ u'id': sensor, u'value': value, u'unit': unit, u'status': status }) self._last_value = current returnValue(self._last_value) if __name__ == '__main__': from pprint import pprint from twisted.internet import reactor from twisted.internet.task import LoopingCall monitor = HWMonitor() @inlineCallbacks def loop(): res = yield monitor.poll() pprint(res) lc = LoopingCall(loop) lc.start(10.) reactor.run()
class Graph_PyQtGraph(QtGui.QWidget): def __init__(self, config, reactor, cxn=None, parent=None): super(Graph_PyQtGraph, self).__init__(parent) from labrad.units import WithUnit as U self.U = U self.cxn = cxn self.pv = self.cxn.parametervault self.reactor = reactor self.artists = {} self.should_stop = False self.name = config.name self.vline_name = config.vline self.vline_param = config.vline_param self.hline_name = config.hline self.hline_param = config.hline_param self.show_points = config.show_points self.grid_on = config.grid_on self.scatter_plot = config.scatter_plot self.dataset_queue = Queue.Queue(config.max_datasets) self.live_update_loop = LoopingCall(self.update_figure) self.live_update_loop.start(0) colors = ['r', 'g', 'y', 'c', 'm', 'w'] self.colorChooser = itertools.cycle(colors) self.initUI() @inlineCallbacks def initUI(self): self.tracelist = TraceList(self) self.pw = pg.PlotWidget() if self.vline_name: self.inf = pg.InfiniteLine(movable=True, angle=90, label=self.vline_name + '{value:0.0f}', labelOpts={ 'position': 0.9, 'color': (200, 200, 100), 'fill': (200, 200, 200, 50), 'movable': True }) init_value = yield self.get_init_vline() self.inf.setValue(init_value) self.inf.setPen(width=5.0) if self.hline_name: self.inf = pg.InfiniteLine(movable=True, angle=0, label=self.hline_name + '{value:0.0f}', labelOpts={ 'position': 0.9, 'color': (200, 200, 100), 'fill': (200, 200, 200, 50), 'movable': True }) init_value = yield self.get_init_hline() self.inf.setValue(init_value) self.inf.setPen(width=5.0) self.coords = QtGui.QLabel('') self.title = QtGui.QLabel(self.name) frame = QtGui.QFrame() splitter = QtGui.QSplitter() splitter.addWidget(self.tracelist) hbox = QtGui.QHBoxLayout() vbox = QtGui.QVBoxLayout() vbox.addWidget(self.title) vbox.addWidget(self.pw) vbox.addWidget(self.coords) frame.setLayout(vbox) splitter.addWidget(frame) hbox.addWidget(splitter) self.setLayout(hbox) #self.legend = self.pw.addLegend() self.tracelist.itemChanged.connect(self.checkboxChanged) self.pw.plot([], []) vb = self.pw.plotItem.vb self.img = pg.ImageItem() vb.addItem(self.img) if self.vline_name: vb.addItem(self.inf) self.inf.sigPositionChangeFinished.connect(self.vline_changed) if self.hline_name: vb.addItem(self.inf) self.inf.sigPositionChangeFinished.connect(self.hline_changed) self.pw.scene().sigMouseMoved.connect(self.mouseMoved) self.pw.sigRangeChanged.connect(self.rangeChanged) def getItemColor(self, color): color_dict = { "r": QtGui.QColor(QtCore.Qt.red).lighter(130), "g": QtGui.QColor(QtCore.Qt.green), "y": QtGui.QColor(QtCore.Qt.yellow), "c": QtGui.QColor(QtCore.Qt.cyan), "m": QtGui.QColor(QtCore.Qt.magenta).lighter(120), "w": QtGui.QColor(QtCore.Qt.white) } return color_dict[color] def update_figure(self): for ident, params in self.artists.iteritems(): if params.shown: try: ds = params.dataset index = params.index current_update = ds.updateCounter if params.last_update < current_update: x = ds.data[:, 0] y = ds.data[:, index + 1] params.last_update = current_update params.artist.setData(x, y) except: pass def add_artist(self, ident, dataset, index, no_points=False): ''' no_points is an override parameter to the global show_points setting. It is to allow data fits to be plotted without points ''' new_color = self.colorChooser.next() if self.show_points and not no_points: line = self.pw.plot([], [], symbol='o', symbolBrush=self.getItemColor(new_color), name=ident, pen=self.getItemColor(new_color), connect=self.scatter_plot) else: line = self.pw.plot([], [], pen=self.getItemColor(new_color), name=ident) if self.grid_on: self.pw.showGrid(x=True, y=True) self.artists[ident] = artistParameters(line, dataset, index, True) self.tracelist.addTrace(ident, new_color) def remove_artist(self, ident): try: artist = self.artists[ident].artist self.pw.removeItem(artist) #self.legend.removeItem(ident) self.tracelist.removeTrace(ident) self.artists[ident].shown = False try: del self.artists[ident] except KeyError: pass except: print "remove failed" def display(self, ident, shown): try: artist = self.artists[ident].artist if shown: self.pw.addItem(artist) self.artists[ident].shown = True else: self.pw.removeItem(artist) #self.legend.removeItem(ident) self.artists[ident].shown = False except KeyError: raise Exception('404 Artist not found') def checkboxChanged(self): for ident, item in self.tracelist.trace_dict.iteritems(): try: if item.checkState() and not self.artists[ident].shown: self.display(ident, True) if not item.checkState() and self.artists[ident].shown: self.display(ident, False) except KeyError: # this means the artist has been deleted. pass def rangeChanged(self): lims = self.pw.viewRange() self.pointsToKeep = lims[0][1] - lims[0][0] self.current_limits = [lims[0][0], lims[0][1]] @inlineCallbacks def add_dataset(self, dataset): try: self.dataset_queue.put(dataset, block=False) except Queue.Full: remove_ds = self.dataset_queue.get() self.remove_dataset(remove_ds) self.dataset_queue.put(dataset, block=False) labels = yield dataset.getLabels() for i, label in enumerate(labels): self.add_artist(label, dataset, i) @inlineCallbacks def remove_dataset(self, dataset): labels = yield dataset.getLabels() for label in labels: self.remove_artist(label) def set_xlimits(self, limits): self.pw.setXRange(limits[0], limits[1]) self.current_limits = limits def set_ylimits(self, limits): self.pw.setYRange(limits[0], limits[1]) def mouseMoved(self, pos): #print "Image position:", self.img.mapFromScene(pos) pnt = self.img.mapFromScene(pos) string = '(' + str(pnt.x()) + ' , ' + str(pnt.y()) + ')' self.coords.setText(string) @inlineCallbacks def get_init_vline(self): init_vline = yield self.pv.get_parameter(self.vline_param[0], self.vline_param[1]) returnValue(init_vline) @inlineCallbacks def get_init_hline(self): init_hline = yield self.pv.get_parameter(self.hline_param[0], self.hline_param[1]) returnValue(init_hline) @inlineCallbacks def vline_changed(self, sig): val = self.inf.value() param = yield self.pv.get_parameter(self.vline_param[0], self.vline_param[1]) units = param.units val = self.U(val, units) yield self.pv.set_parameter(self.vline_param[0], self.vline_param[1], val) @inlineCallbacks def hline_changed(self, sig): val = self.inf.value() param = yield self.pv.get_parameter(self.hline_param[0], self.hline_param[1]) units = param.units val = self.U(val, units) yield self.pv.set_parameter(self.hline_param[0], self.hline_param[1], val)
"""
def makeService(self, options): try: logger.info('Initializing configuration') try: self._init_config(options) except Exception as e: logger.error( "Aborting service startup due to configuration error: {}". format(e)) raise e logger.info('Initializing logging') self._init_logging() self._check_enabled() #logger.enable_bootstrap_logging(self.tapname) assert (issubclass(self.service_cls, ApiService)) self.anchore_service = self.service_cls(options=options) self.anchore_service.initialize(self.global_configuration) self.resource_nodes[ b'version'].versions = self.anchore_service.versions # application object application = service.Application( "Service-" + '-'.join(self.anchore_service.name)) self.twistd_service = service.MultiService() self.twistd_service.setServiceParent(application) if self.anchore_service.task_handlers_enabled: logger.info('Starting monitor thread') lc = self._get_api_monitor(self.anchore_service) lc.start(1) else: logger.warn( 'Skipped start of monitor threads due to task_handlers_enabled=false in config, or found ANCHORE_ENGINE_DISABLE_MONITORS in env' ) thread_stats_interval = int( self.service_config.get('debug_thread_stats_dump_interval', 0)) if thread_stats_interval > 0: logger.info( 'Based on service config, starting the thread stats dumper' ) monitor = LoopingCall(dump_stats) monitor.start(thread_stats_interval) logger.info('Building api handlers') s = self._build_api_service() s.setServiceParent(self.twistd_service) if enable_dangerous_debug_cli: logger.warn( 'Loading *dangerous* debug/telnet service as specified by debug config' ) self.makeDebugCLIService({ 'protocolFactory': ColoredManhole, 'protocolArgs': (None, ), 'telnet': 6023, }).setServiceParent(self.twistd_service) return self.twistd_service except Exception as err: logger.exception("cannot create/init/register service: " + self.service_cls.__service_name__ + " - exception: " + str(err)) raise Exception("cannot start service (see above for information)") finally: pass
def start(self): partitions = [] try: while not partitions: yield self._client.load_metadata_for_topics(self.topic) #self._client.load_metadata_for_topics(self.topic) e = self._client.metadata_error_for_topic(self.topic) if e: log.warning('no-metadata-for-topic', error=e, topic=self.topic) else: partitions = self._client.topic_partitions[self.topic] break time.sleep(20) except KafkaUnavailableError: log.error("unable-to-communicate-with-Kafka-brokers") self.stop() def _note_consumer_stopped(result, consumer): log.info('consumer-stopped', consumer=consumer, result=result) for partition in partitions: c = Consumer(self._client, self.topic, partition, self.msg_processor) self._consumer_list.append(c) log.info('consumer-started', topic=self.topic, partition=partition) d = c.start(OFFSET_LATEST) d.addBoth(_note_consumer_stopped, c) self._consumer_d_list.append(d) # Now read the list of existing dashboards from Grafana and create the # dictionary of dashboard timers. If we've crashed there will be # dashboards there. Just add them and if they're no longer valid # they'll be deleted. If they are valid then they'll persist. #print("Starting main loop") try: retrys = 10 while True: r = requests.get(self.grafana_url + "/datasources") if r.status_code == requests.codes.ok: break else: retrys -= 1 if retrys == 0: log.error("unable-to-communicate-with-grafana") self.stop() time.sleep(10) j = r.json() data_source = False for i in j: if i["name"] == "Voltha Stats": data_source = True break if not data_source: r = requests.post(self.grafana_url + "/datasources", data={ "name": "Voltha Stats", "type": "graphite", "access": "proxy", "url": "http://localhost:81" }) log.info('data-source-added', status=r.status_code, text=r.text) retrys = 10 while True: r = requests.get(self.grafana_url + "/search?") if r.status_code == requests.codes.ok: break else: retrys -= 1 if retrys == 0: log.error("unable-to-communicate-with-grafana") self.stop() time.sleep(10) j = r.json() for i in j: # Look for dashboards that have a title of *olt.[[:hexidgit:]]. # These will be the ones of interest. Others should just be left # alone. #print(i['title']) match = re.search(r'(.*olt)\.([0-9a-zA-Z]+)', i['title']) if match and match.lastindex > 0: #print(match.group(1), match.group(2)) self.dash_meta[match.group(2)] = {} self.dash_meta[match.group( 2)]['timer'] = self.timer_duration # 10 min self.dash_meta[match.group(2)]['device'] = match.group(1) self.dash_meta[match.group(2)]['created'] = False self.dash_meta[match.group(2)]['ports'] = {} # TODO: We should really capture all of the chart data # including the rows, panels, and data points being logged. # This is good enough for now though to determine if # there's already a dashboard for a given device. def countdown_processor(): # Called every X (timer_resolution) seconds to count down each of the # dash timers. If a timer reaches 0 the corresponding # dashboard is removed. #log.info("Counting down.") try: for dashboard in self.dash_meta.keys(): #print("Counting down %s." %dashboard) # Issue a log if the counter decrement is somewhat relevant if(self.dash_meta[dashboard]['timer'] % 100 == 0 and \ self.dash_meta[dashboard]['timer'] != self.timer_duration): log.info("counting-down", dashboard=dashboard, timer=self.dash_meta[dashboard]['timer']) self.dash_meta[dashboard][ 'timer'] -= self.timer_resolution if self.dash_meta[dashboard]['timer'] <= 0: # Delete the dashboard here log.info("FIXME:-Should-delete-the-dashboard-here", dashboard=dashboard) pass except: e = sys.exc_info() log.error("error", error=e) # Start the dashboard countdown processor log.info("starting-countdown-processor") lc = LoopingCall(countdown_processor) lc.start(self.timer_resolution) @inlineCallbacks def template_checker(): try: # Called every so often (timer_resolution seconds because it's # convenient) to check if a template dashboard has been defined # in Grafana. If it has been, replace the built in template # with the one provided r = requests.get(self.grafana_url + "/search?query=template") db = r.json() if len(db) == 1: # Apply the template yield self.dash_template.apply_template(db[0]) elif len(db) != 0: # This is an error, log it. log.warning("More-than-one-template-provided-ignoring") except: e = sys.exc_info() log.error("error", error=e) log.info("starting-template-checker") lc = LoopingCall(template_checker) lc.start(self.timer_resolution) except: e = sys.exc_info() log.error("error", error=e)
class ReadRequestLoadScenario(object): """ A scenario that places load on the cluster by performing read requests at a specified rate. :ivar reactor: Reactor to use. :ivar cluster: `BenchmarkCluster` containing the control service. :ivar request_rate: The target number of requests per second. :ivar sample_size: The number of samples to collect when measuring the rate. :ivar timeout: Maximum time in seconds to wait for the requested rate to be reached. """ def __init__(self, reactor, cluster, request_rate=10, sample_size=DEFAULT_SAMPLE_SIZE, timeout=45): self._maintained = Deferred() self.reactor = reactor self.control_service = cluster.get_control_service(reactor) self.request_rate = request_rate self.timeout = timeout self.rate_measurer = RateMeasurer(sample_size) self.max_outstanding = 10 * request_rate # Send requests per second self.loop = LoopingCall.withCount(self._request_and_measure) self.loop.clock = self.reactor self.monitor_loop = LoopingCall(self.check_rate) self.monitor_loop.clock = self.reactor def _request_and_measure(self, count): """ Update the rate with the current value and send `request_rate` number of new requests. :param count: The number of seconds passed since the last time `_request_and_measure` was called. """ for i in range(count): self.rate_measurer.update_rate() def handle_request_error(result): self.rate_measurer.request_failed(result) write_failure(result) for i in range(self.request_rate): d = self.control_service.list_nodes() self.rate_measurer.request_sent() d.addCallbacks(self.rate_measurer.response_received, errback=handle_request_error) def _fail(self, exception): """ Fail the scenario. Stop the monitor loop and throw the error. """ self.monitor_loop.stop() self._maintained.errback(exception) def check_rate(self): """ Verify that the rate hasn't decreased and that the scenario is not overloaded. A scenario is overloaded if there are too many outstanding requests. :raise: `RequestRateTooLow` if the rate has dropped. :raise: `RequestOverload` if the scenario is overloaded. """ rate = self.rate_measurer.rate() if rate < self.request_rate: self._fail(RequestRateTooLow(rate)) if self.rate_measurer.outstanding() > self.max_outstanding: self._fail(RequestOverload()) def start(self): """ :return: A Deferred that fires when the desired scenario is established (e.g. that a certain load is being applied). """ self.loop.start(interval=1) def reached_target_rate(): return self.rate_measurer.rate() >= self.request_rate def handle_timeout(failure): failure.trap(CancelledError) raise RequestRateNotReached waiting_for_target_rate = loop_until(self.reactor, reached_target_rate, repeat(1)) timeout(self.reactor, waiting_for_target_rate, self.timeout) waiting_for_target_rate.addErrback(handle_timeout) # Start monitoring the scenario as soon as the target rate is reached. def monitor_scenario_status(result): self.monitor_loop.start(interval=1) waiting_for_target_rate.addCallback(monitor_scenario_status) return waiting_for_target_rate def maintained(self): """ :return: A Deferred that fires with an errback if the desired scenario fails to hold between being established and being stopped. This Deferred never fires with a callback. """ return self._maintained def stop(self): """ Stop the scenario from being maintained by stopping all the loops that may be executing. :return: A Deferred that fires when the scenario has stopped. """ if self.monitor_loop.running: self.monitor_loop.stop() if self.loop.running: self.loop.stop() outstanding_requests = self.rate_measurer.outstanding() if outstanding_requests > 0: msg = ( "There are {num_requests} outstanding requests. " "Waiting {num_seconds} seconds for them to complete.").format( num_requests=outstanding_requests, num_seconds=self.timeout) Message.log(key='outstanding_requests', value=msg) with start_action(action_type=u'flocker:benchmark:scenario:stop', scenario='read_request_load'): def handle_timeout(failure): failure.trap(CancelledError) msg = ("Force stopping the scenario. " "There are {num_requests} outstanding requests").format( num_requests=outstanding_requests) Message.log(key='force_stop_request', value=msg) def no_outstanding_requests(): return self.rate_measurer.outstanding() == 0 scenario_stopped = loop_until(self.reactor, no_outstanding_requests, repeat(1)) timeout(self.reactor, scenario_stopped, self.timeout) scenario_stopped.addErrback(handle_timeout) scenario = DeferredContext(scenario_stopped) scenario.addActionFinish() return scenario.result
class ServerEventManager(pb.Root): def __init__(self): self.clients = {} #organized {client_id: client_obj} self.total_clients = 0 self.world_lists = WordManager( "free") #a dict with the words preloaded self.lobbys = {} self.total_game_requests = 0 self.looping_call = LoopingCall(self.remote_post, e.CopyableEvent()) self.looping_call.start(5.0) def dead_client(self, client): """ removes client from dict of clients (self.clients) """ del self.clients[client.client_id] def remote_get_word_list(self, word_list_name): return self.world_lists[word_list_name] def remote_register_client(self, root_obj, client_nickname): """ returns True if is accepted, returns False returns emtpy string (False) if client passed emtpy string (figured it might help later. Either way it evaluates to false) """ if client_nickname == "": return "" client_id = "Client" + str(self.total_clients) self.total_clients = self.total_clients + 1 client = Client(root_obj, client_id, client_nickname) client.observers.append(self) #add self as observer self.clients[client_id] = client print client_id + " : ///nickname: " + client_nickname return client_id def remote_get_unique_game_id(self): """ not <i>guaranteed</i> to be unique. But in practice it will be. Worst case scenario user gets a "game name in use" popup """ game_name = "Game" + str(self.total_game_requests) if game_name in self.lobbys: for i in xrange(26): letter_name = game_name + chr(97 + i) if letter_name not in self.lobbys: return letter_name self.total_game_requests = self.total_game_requests + 1 return game_name #sorry bud, out of luck def remote_make_game_lobby(self, lobby_id, word_list, score_system_mode, num_teams, round_time, leeway_time): """ returns false if lobby_id is in use. Assumes word_list_id will be correct. Don't f**k me over me. """ if lobby_id in self.lobbys: return (False, "lobby id in use") elif not word_list: return (False, "word list is empty") else: if isinstance(word_list, str): #if it wasn't a local copy. word_list = self.world_lists[word_list] self.lobbys[lobby_id] = Lobby(self, word_list, lobby_id, score_system_mode, num_teams, round_time, leeway_time) return (True, "success") def remote_join_lobby(self, client_id, lobby_id): """ returns the lobby (a root_obj) if success, else returns None """ #note: game is set to None before game starts => check game to see #if game has started if lobby_id not in self.lobbys or self.lobbys[lobby_id].game: return None lobby = self.lobbys[lobby_id] lobby.new_player(self.clients[client_id]) return lobby def remote_join_lobby_score(self, client_id, lobby_id): if lobby_id not in self.lobbys or self.lobbys[lobby_id].game: return None lobby = self.lobbys[lobby_id] lobby.new_player_score(self.clients[client_id]) return lobby def remote_get_word_list_options(self): """ returns name of all the word lists """ return self.world_lists.keys() #untested def remote_post(self, event, client_id=""): """ attempts to notify all users of event. removes them if they are dead """ #event.originator = client_id #print "Recieved ", event.name, " from ", client_id for client in self.clients.values( ): #don't itervalues; we're possibly changing dictionary try: client.root_obj.callRemote('notify', event) except pb.DeadReferenceError: print "calling client.remove_client() on: ", client.client_id, client.nickname client.remove_client()
class BlobAvailabilityTracker: """ Class to track peer counts for known blobs, and to discover new popular blobs Attributes: availability (dict): dictionary of peers for known blobs """ def __init__(self, blob_manager, peer_finder, dht_node): self.availability = {} self._last_mean_availability = Decimal(0.0) self._blob_manager = blob_manager self._peer_finder = peer_finder self._dht_node = dht_node self._check_mine = LoopingCall(self._update_mine) def start(self): log.info("Starting blob availability tracker.") self._check_mine.start(600) def stop(self): log.info("Stopping blob availability tracker.") if self._check_mine.running: self._check_mine.stop() def get_blob_availability(self, blob, timeout=None): def _get_peer_count(peers): have_blob = sum(1 for peer in peers if peer.is_available()) return {blob: have_blob} d = self._peer_finder.find_peers_for_blob(blob, timeout) d.addCallback(_get_peer_count) return d def get_availability_for_blobs(self, blobs, timeout=None): dl = [ self.get_blob_availability(blob, timeout) for blob in blobs if blob ] d = defer.DeferredList(dl) d.addCallback( lambda results: [val for success, val in results if success]) return d @property def last_mean_availability(self): return max(Decimal(0.01), self._last_mean_availability) def _update_peers_for_blob(self, blob): def _save_peer_info(blob_hash, peers): v = {blob_hash: peers} self.availability.update(v) return v d = self._peer_finder.find_peers_for_blob(blob) d.addCallback( lambda r: [[c.host, c.port, c.is_available()] for c in r]) d.addCallback(lambda peers: _save_peer_info(blob, peers)) return d def _update_mine(self): def _get_peers(blobs): dl = [] for hash in blobs: dl.append(self._update_peers_for_blob(hash)) return defer.DeferredList(dl) def sample(blobs): return random.sample(blobs, min(len(blobs), 10)) start = time.time() log.debug('==> Updating the peers for my blobs') d = self._blob_manager.get_all_verified_blobs() # as far as I can tell, this only is used to set _last_mean_availability # which... seems like a very expensive operation for such little payoff. # so taking a sample should get about the same effect as querying the entire # list of blobs d.addCallback(sample) d.addCallback(_get_peers) d.addCallback(lambda _: self._set_mean_peers()) d.addCallback(lambda _: log.debug( '<== Done updating peers for my blobs. Took %s seconds', time.time() - start)) # although unused, need to return or else the looping call # could overrun on a previous call return d def _set_mean_peers(self): num_peers = [ len(self.availability[blob]) for blob in self.availability ] mean = Decimal(sum(num_peers)) / Decimal(max(1, len(num_peers))) self._last_mean_availability = mean
class SyncFolder(PatternMatchingEventHandler): def __init__(self, local_dir, remote_dircap, tahoe=None, ignore_patterns=None): _ignore_patterns = [ '*.gridsync-versions*', '*.part*', '*(conflicted copy *-*-* *-*-*)*' ] if ignore_patterns: _ignore_patterns += ignore_patterns super(SyncFolder, self).__init__(ignore_patterns=_ignore_patterns) if not tahoe: from gridsync.tahoe import Tahoe tahoe = Tahoe() self.tahoe = tahoe self.local_dir = os.path.expanduser(local_dir) self.remote_dircap = remote_dircap self.versions_dir = os.path.join(self.local_dir, '.gridsync-versions') self.local_snapshot = 0 self.filesystem_modified = False self.do_backup = False self.do_sync = False self.sync_state = 0 self.sync_log = [] self.keep_versions = 1 self.local_checker = LoopingCall(self.check_for_changes) self.remote_checker = LoopingCall(reactor.callInThread, self.sync) logging.debug("{} initialized; " "{} <-> {}".format(self, self.local_dir, self.remote_dircap)) def on_modified(self, event): self.filesystem_modified = True #try: # reactor.callFromThread(self.local_checker.start, 1) #except AssertionError: # return reactor.callFromThread(self._start_local_checker) def _start_local_checker(self): # XXX: For some (qt5reactor-related?) reason, the AssertionError # raised by trying to start the (already-running) local_checker timer # above won't catch if called from reactor.callFromThread. Why is this? try: self.local_checker.start(1) except AssertionError: return def check_for_changes(self): if self.filesystem_modified: self.filesystem_modified = False else: reactor.callFromThread(self.local_checker.stop) reactor.callInThread(self.sync, force_backup=True) def start(self): try: self.remote_dircap_alias = self.tahoe.aliasify(self.remote_dircap) except ValueError: # TODO: Alert user alias is garbled? pass except LookupError: # TODO: Alert user alias is missing? pass logging.info("Starting Observer in {}...".format(self.local_dir)) try: self.observer = Observer() self.observer.schedule(self, self.local_dir, recursive=True) self.observer.start() except Exception as error: logging.error(error) reactor.callFromThread(self.remote_checker.start, 30) def _create_conflicted_copy(self, filepath): base, extension = os.path.splitext(filepath) mtime = int(os.path.getmtime(filepath)) t = datetime.datetime.fromtimestamp(mtime) tag = t.strftime('.(conflicted copy %Y-%m-%d %H-%M-%S)') tagged_filepath = base + tag + extension logging.debug("Creating conflicted copy of {} {}".format( filepath, tagged_filepath)) os.rename(filepath, tagged_filepath) os.utime(tagged_filepath, (-1, mtime)) def _create_versioned_copy(self, filename, mtime): local_filepath = os.path.join(self.local_dir, filename) base, extension = os.path.splitext(filename) t = datetime.datetime.fromtimestamp(mtime) tag = t.strftime('.(%Y-%m-%d %H-%M-%S)') newname = base + tag + extension versioned_filepath = newname.replace(self.local_dir, self.versions_dir) if not os.path.isdir(os.path.dirname(versioned_filepath)): os.makedirs(os.path.dirname(versioned_filepath)) logging.info("Creating {}".format(versioned_filepath)) shutil.copy2(local_filepath, versioned_filepath) def get_local_metadata(self, basedir=None): metadata = {} if not basedir: basedir = self.local_dir for root, dirs, files in os.walk(basedir, followlinks=True): for name in dirs: path = os.path.join(root, name) if not path.startswith(self.versions_dir): metadata[path] = {} for name in files: path = os.path.join(root, name) if not path.startswith(self.versions_dir): metadata[path] = { 'mtime': int(os.path.getmtime(path)), 'size': os.path.getsize(path) } return metadata def get_remote_metadata(self, dircap, basedir=''): metadata = {} jobs = [] logging.debug("Getting remote metadata from {}...".format(dircap)) url = '{}uri/{}/?t=json'.format(self.tahoe.node_url, dircap) received_data = requests.get(url).json() for filename, data in received_data[1]['children'].items(): path = '/'.join([basedir, filename]).strip('/') metadata[path] = { 'uri': data[1]['ro_uri'], 'mtime': int(data[1]['metadata']['mtime']) } if data[0] == 'dirnode': jobs.append( deferToThread(self.get_remote_metadata, '/'.join([dircap, filename]), path)) results = blockingCallFromThread(reactor, gatherResults, jobs) for result in results: metadata.update(result) return metadata def sync(self, snapshot=None, force_backup=False): if self.sync_state: logging.debug("Sync already in progress; queueing to end...") self.do_sync = True return if not snapshot: try: ls = self.tahoe.ls(self.remote_dircap) if not ls: logging.debug("No /Archives found; " "performing (first?) backup...") self.sync_state += 1 self.backup(self.local_dir, self.remote_dircap_alias) self.sync_complete(ls) return except Exception as error: logging.error(error) return # XXX: It might be preferable to just check the dircap of /Latest/ pre_sync_archives = self.tahoe.ls(self.remote_dircap + "/Archives") available_snapshot = pre_sync_archives[-1] if self.local_snapshot == available_snapshot: if force_backup: self.sync_state += 1 self.backup(self.local_dir, self.remote_dircap_alias) self.sync_complete(pre_sync_archives) return else: snapshot = available_snapshot remote_path = self.remote_dircap + '/Archives/' + snapshot logging.info("Syncing {} with {}...".format(self.local_dir, snapshot)) self.sync_state += 1 self.local_metadata = self.get_local_metadata(self.local_dir) self.remote_metadata = self.get_remote_metadata(remote_path) # TODO: If tahoe.get_metadata() fails or doesn't contain a # valid snapshot, jump to backup? jobs = [] for file, metadata in self.remote_metadata.items(): if metadata['uri'].startswith('URI:DIR'): dirpath = os.path.join(self.local_dir, file) if not os.path.isdir(dirpath): logging.info("Creating directory: {}...".format(dirpath)) os.makedirs(dirpath) for file, metadata in self.remote_metadata.items(): if not metadata['uri'].startswith('URI:DIR'): filepath = os.path.join(self.local_dir, file) remote_mtime = metadata['mtime'] if filepath in self.local_metadata: local_filesize = self.local_metadata[filepath]['size'] local_mtime = self.local_metadata[filepath]['mtime'] if local_mtime < remote_mtime: logging.debug("[<] {} is older than remote version; " "downloading {}...".format(file, file)) if self.keep_versions: self._create_versioned_copy(filepath, local_mtime) jobs.append( deferToThread(self.download, remote_path + '/' + file, filepath, remote_mtime)) elif local_mtime > remote_mtime: logging.debug("[>] {} is newer than remote version; " "backup scheduled".format(file)) self.do_backup = True else: logging.debug("[.] {} is up to date.".format(file)) else: logging.debug("[?] {} is missing; " "downloading {}...".format(file, file)) jobs.append( deferToThread(self.download, remote_path + '/' + file, filepath, remote_mtime)) for file, metadata in self.local_metadata.items(): fn = file.split(self.local_dir + os.path.sep)[1] if fn not in self.remote_metadata: if metadata: recovery_uri = self.tahoe.stored(file, metadata['size'], metadata['mtime']) if recovery_uri: logging.debug("[x] {} removed from latest snapshot; " "deleting local file...".format(file)) if self.keep_versions: self._create_versioned_copy(file, local_mtime) try: os.remove(file) except Exception as error: logging.error(error) else: logging.debug("[!] {} isn't stored; " "backup scheduled".format(fn)) self.do_backup = True blockingCallFromThread(reactor, gatherResults, jobs) if self.do_backup: self.backup(self.local_dir, self.remote_dircap_alias) self.do_backup = False if self.do_sync: self.sync() self.sync_complete(pre_sync_archives) def sync_complete(self, pre_sync_archives): post_sync_archives = self.tahoe.ls(self.remote_dircap + "/Archives") if len(post_sync_archives) - len(pre_sync_archives) <= 1: self.local_snapshot = post_sync_archives[-1] logging.info("Synchronized {} with {}".format( self.local_dir, self.local_snapshot)) else: logging.warn("Remote state changed during sync") # TODO: Re-sync/merge overlooked snapshot self.sync_state -= 1 def download(self, remote_uri, local_filepath, mtime=None): url = self.tahoe.node_url + 'uri/' + remote_uri download_path = local_filepath + '.part' #if os.path.exists(download_path): # XXX: Resuming may not be a good idea, as the existent (local) # parts may no longer be present in the latest (remote) version of # the file. Perhaps an integrity/filecap check should be required? # size = os.path.getsize(download_path) # logging.debug("Partial download of {} found; resuming byte {}..."\ # .format(local_filepath, size)) # request.headers['Range'] = 'bytes={}-'.format(size) # TODO: Handle exceptions.. if os.path.isfile(download_path) or os.path.isdir(download_path): raise OSError("File exists: '{}'".format(download_path)) r = requests.get(url, stream=True) r.raise_for_status() recv = 0 with open(download_path, 'wb') as f: for chunk in r.iter_content(4096): f.write(chunk) recv += len(chunk) if os.path.isfile(local_filepath): local_filesize = os.path.getsize(local_filepath) if not self.tahoe.stored(file, local_filesize, mtime): self._create_conflicted_copy(local_filepath) os.rename(download_path, local_filepath) if mtime: os.utime(local_filepath, (-1, mtime)) self.sync_log.append("Downloaded {}".format( local_filepath.lstrip(self.local_dir))) return recv def backup(self, local_dir, remote_dircap): excludes = ['--exclude=' + x for x in self.ignore_patterns] output = self.tahoe.command(['backup', '-v'] + excludes + [local_dir, remote_dircap]) for line in output.split('\n'): if line.startswith('uploading'): filename = line[11:][:-3].lstrip(self.local_dir) self.sync_log.append("Uploaded {}".format(filename)) def stop(self): logging.info("Stopping Observer in {}...".format(self.local_dir)) try: self.observer.stop() self.observer.join() except Exception as error: logging.error(error) self.remote_checker.stop()