class USBClient(Protocol):
def __init__(self, network):
self.network = network
self.usb_list = []
self.lc = LoopingCall(self.timerEvent)
self.serialBuffer = bytes("")
def timerEvent(self):
self.lc.stop()
for cli in client_list:
cli.transport.write(self.serialBuffer)
self.serialBuffer = bytes("")
def connectionFailed(self):
log.msg ("Connection Failed:", self)
reactor.stop()
def connectionMade(self):
usb_list.append(self)
log.msg("Connected to %s device" % device)
def dataReceived(self, data):
#log.msg("data resrvr")
self.serialBuffer += data
if self.lc.running:
self.lc.stop()
self.lc.start(0.2, False)
def sendLine(self, cmd):
print cmd
self.transport.write(cmd + "\r\n")
def outReceived(self, data):
self.data = self.data + data
class Rain(object):
"""
Make it rain.
Rain only occurs during spring.
"""
implements(IAutomaton)
blocks = tuple()
def __init__(self):
self.season_loop = LoopingCall(self.check_season)
def scan(self, chunk):
pass
def feed(self, coords):
pass
def start(self):
self.season_loop.start(5 * 60)
def stop(self):
self.season_loop.stop()
def check_season(self):
if factory.world.season.name == "spring":
factory.vane.weather = "rainy"
reactor.callLater(1 * 60, setattr, factory.vane, "weather",
"sunny")
name = "rain"
def modbus_master(module, properties):
log.debug('Modbus master module : ' + str(module))
# Modbus Master
#--------------------------------------------------------------------------#
# initialize your data store
#--------------------------------------------------------------------------#
store = ModbusSlaveContext(
co = ModbusSequentialDataBlock(0, [0]*100),
hr = ModbusSequentialDataBlock(0, [0]*100))
context = ModbusServerContext(slaves=store, single=True)
#--------------------------------------------------------------------------#
# initialize the server information
#--------------------------------------------------------------------------#
identity = ModbusDeviceIdentification()
identity.VendorName = 'ASO+AKO'
identity.ProductCode = 'DYODE'
identity.VendorUrl = 'yoloswag'
identity.ProductName = 'DYODE'
identity.ModelName = 'BSides LV release'
identity.MajorMinorRevision = '0.9'
#--------------------------------------------------------------------------#
# run the server you want
#--------------------------------------------------------------------------#
time = 1 # 5 seconds delay
loop = LoopingCall(f=modbus_master_update, a=(module, properties, context))
loop.start(time, now=False) # initially delay by time
StartTcpServer(context, identity=identity, address=("0.0.0.0", \
properties['port_out']))
def build_reactor(options, **kwargs):
web_socket_instance = FilteredWebSocketFactory(**kwargs)
subscriber = kwargs.pop("subscriber", None)
if options.key and options.cert:
with open(options.key) as keyFile:
with open(options.cert) as certFile:
cert = ssl.PrivateCertificate.loadPEM(keyFile.read() + certFile.read())
reactor.listenSSL(options.port, web_socket_instance, cert.options())
else:
reactor.listenTCP(
options.port,
web_socket_instance
)
if subscriber is not None:
reactor.callInThread(
subscriber.listener,
web_socket_instance
)
reactor.addSystemEventTrigger(
"before",
"shutdown",
subscriber.kill
)
# Start the consumer loop
consumer_loop = LoopingCall(
web_socket_instance.consumer
)
consumer_loop.start(0.001, now=False)
return web_socket_instance
class PollingDataStream(DataStream):
""" A self-polling data stream.
This class represents a data stream that wakes up at a given frequency,
and calls the :meth:`poll` method.
"""
frequency = None # Either a timedelta object, or the number of seconds
now = False
def __init__(self):
super(PollingDataStream, self).__init__()
self.timer = LoopingCall(self.poll)
if isinstance(self.frequency, timedelta):
seconds = (
self.frequency.seconds
+ (self.frequency.days * 24 * 60 * 60)
+ (self.frequency.microseconds / 1000000.0)
)
else:
seconds = self.frequency
log.debug("Setting a %s second timer" % seconds)
self.timer.start(seconds, now=self.now)
def poll(self):
raise NotImplementedError
def stop(self):
super(PollingDataStream, self).stop()
try:
if hasattr(self, "timer"):
self.timer.stop()
except Exception, e:
self.log.warn(e)
class Pinger(object):
"""
An periodic AMP ping helper.
"""
def __init__(self, reactor):
"""
:param IReactorTime reactor: The reactor to use to schedule the pings.
"""
self.reactor = reactor
def start(self, protocol, interval):
"""
Start sending some pings.
:param AMP protocol: The protocol over which to send the pings.
:param timedelta interval: The interval at which to send the pings.
"""
def ping():
protocol.callRemote(NoOp)
self._pinging = LoopingCall(ping)
self._pinging.clock = self.reactor
self._pinging.start(interval.total_seconds(), now=False)
def stop(self):
"""
Stop sending the pings.
"""
self._pinging.stop()
class LeaseService(Service):
"""
Manage leases.
In particular, clear out expired leases once a second.
:ivar _reactor: A ``IReactorTime`` provider.
:ivar _persistence_service: The persistence service to act with.
:ivar _lc: A ``twisted.internet.task.LoopingCall`` run every second
to update the configured leases by releasing leases that have
expired.
"""
def __init__(self, reactor, persistence_service):
self._reactor = reactor
self._persistence_service = persistence_service
def startService(self):
self._lc = LoopingCall(self._expire)
self._lc.clock = self._reactor
self._lc.start(1)
def stopService(self):
self._lc.stop()
def _expire(self):
now = datetime.fromtimestamp(self._reactor.seconds(), tz=UTC)
def expire(leases):
updated_leases = leases.expire(now)
for dataset_id in set(leases) - set(updated_leases):
_LOG_EXPIRE(dataset_id=dataset_id,
node_id=leases[dataset_id].node_id).write()
return updated_leases
return update_leases(expire, self._persistence_service)
class BandwidthEstimator:
bufsize = 20
totalBytes = 0
def __init__(self, message, length):
self.length = length
self.message = message
self.estim = []
self.bytes = 0
self.call = LoopingCall(self.estimateBandwidth)
self.call.start(1)
def estimateBandwidth(self):
bytes = self.bytes
self.totalBytes += bytes
self.estim.append(bytes)
self.message("%0.2f k/s (%0.2d%%)"
% ((sum(self.estim) / len(self.estim)) / 1024.,
(float(self.totalBytes) / self.length) * 100))
if len(self.estim) > self.bufsize:
self.estim.pop(0)
self.bytes = 0
def stop(self):
self.call.stop()
self.estimateBandwidth()
self.message("Finished receiving: %d bytes (%d%%)" % (
self.totalBytes, (float(self.totalBytes) / self.length) * 100))
def openShell(self, transport):
"""
Write 60 lines of data to the transport, then exit.
"""
proto = protocol.Protocol()
proto.makeConnection(transport)
transport.makeConnection(wrapProtocol(proto))
# Send enough bytes to the connection so that a rekey is triggered in
# the client.
def write(counter):
i = counter()
if i == 60:
call.stop()
transport.session.conn.sendRequest(
transport.session, 'exit-status', '\x00\x00\x00\x00')
transport.loseConnection()
else:
transport.write("line #%02d\n" % (i,))
# The timing for this loop is an educated guess (and/or the result of
# experimentation) to exercise the case where a packet is generated
# mid-rekey. Since the other side of the connection is (so far) the
# OpenSSH command line client, there's no easy way to determine when the
# rekey has been initiated. If there were, then generating a packet
# immediately at that time would be a better way to test the
# functionality being tested here.
call = LoopingCall(write, count().next)
call.start(0.01)
def __init__(self):
self.output("TWS init")
self.username = environ["TXTRADER_USERNAME"]
self.password = environ["TXTRADER_PASSWORD"]
self.xmlrpc_port = int(environ["TXTRADER_XMLRPC_PORT"])
self.tcp_port = int(environ["TXTRADER_TCP_PORT"])
self.callback_timeout = int(environ["TXTRADER_TWS_CALLBACK_TIMEOUT"])
if not self.callback_timeout:
self.callback_timeout = DEFAULT_TWS_CALLBACK_TIMEOUT
self.output("callback_timeout=%d" % self.callback_timeout)
self.enable_ticker = bool(int(environ["TXTRADER_ENABLE_TICKER"]))
self.label = "TWS Gateway"
self.channel = "tws"
self.current_account = ""
self.clients = set([])
self.orders = {}
self.pending_orders = {}
self.openorder_callbacks = []
self.accounts = []
self.account_data = {}
self.positions = {}
self.position_callbacks = []
self.executions = {}
self.execution_callbacks = []
self.bardata_callbacks = []
self.cancel_callbacks = []
self.order_callbacks = []
self.addsymbol_callbacks = []
self.accountdata_callbacks = []
self.last_connection_status = ""
self.connection_status = "Initializing"
self.LastError = -1
self.next_order_id = -1
self.last_minute = -1
self.handlers = {
"error": self.error_handler,
"tickSize": self.handle_tick_size,
"tickPrice": self.handle_tick_price,
"tickString": self.handle_tick_string,
"nextValidId": self.handle_next_valid_id,
"currentTime": self.handle_time,
"managedAccounts": self.handle_accounts,
"orderStatus": self.handle_order_status,
"openOrder": self.handle_open_order,
"openOrderEnd": self.handle_open_order_end,
"execDetails": self.handle_exec_details,
"execDetailsEnd": self.handle_exec_details_end,
"position": self.handle_position,
"positionEnd": self.handle_position_end,
"historicalData": self.handle_historical_data,
"updateAccountValue": self.handle_account_value,
"accountDownloadEnd": self.handle_account_download_end,
}
self.ticker_ids = {}
self.symbols = {}
self.symbols_by_id = {}
self.primary_exchange_map = {}
self.tws_conn = None
repeater = LoopingCall(self.EverySecond)
repeater.start(1)
class _ExchangeRate(object):
"""Download an exchange rate from Yahoo Finance using Twisted."""
def __init__(self, name):
self._value = None
self._name = name
# External API:
def latest_value(self):
"""Return the latest exchange rate value.
May be None if no value is available.
"""
return self._value
def start(self):
"""Start the background process."""
self._lc = LoopingCall(self._download)
# Run immediately, and then every 30 seconds:
self._lc.start(30, now=True)
def _download(self):
"""Download the page."""
print("Downloading!")
def parse(result):
print("Got %r back from Yahoo." % (result,))
values = result.strip().split(",")
self._value = float(values[1])
d = getPage(
"http://download.finance.yahoo.com/d/quotes.csv?e=.csv&f=c4l1&s=%s=X"
% (self._name,))
d.addCallback(parse)
d.addErrback(log.err)
return d
class SendsManyFileDescriptors(ConnectableProtocol):
paused = False
def connectionMade(self):
self.socket = socket()
self.transport.registerProducer(self, True)
def sender():
self.transport.sendFileDescriptor(self.socket.fileno())
self.transport.write(b"x")
self.task = LoopingCall(sender)
self.task.clock = self.transport.reactor
self.task.start(0).addErrback(err, "Send loop failure")
def stopProducing(self):
self._disconnect()
def resumeProducing(self):
self._disconnect()
def pauseProducing(self):
self.paused = True
self.transport.unregisterProducer()
self._disconnect()
def _disconnect(self):
self.task.stop()
self.transport.abortConnection()
self.other.transport.abortConnection()
def make_lc(self, reactor, func):
if DEBUG:
self.stdout_length = 0
self.stderr_length = 0
def _(lc, reactor):
if DEBUG:
stdout = self.stdout.getvalue()
stderr = self.stderr.getvalue()
if self.stdout.getvalue()[self.stdout_length:]:
print(self.stdout.getvalue()[self.stdout_length:],
file=sys.__stdout__)
if self.stderr.getvalue()[self.stderr_length:]:
print(self.stderr.getvalue()[self.stderr_length:],
file=sys.__stderr__)
self.stdout_length = len(stdout)
self.stderr_length = len(stderr)
return func(lc, reactor)
lc = LoopingCall(_)
lc.a = (lc, reactor)
lc.clock = reactor
lc.start(0.1)
return lc
class ICUMonitorFactory(protocol.ClientFactory):
def __init__(self, gui_msg_callback, gui_ip, gui_port):
self.ip = gui_ip
self.port = gui_port
self.msg_callback = gui_msg_callback
self.objmonit = None
def buildProtocol(self, addr):
return ICUMonitor(self.msg_callback)
def clientConnectionLost(self, connector, reason):
pass
def clientConnectionFailed(self, connector, reason):
pass
def sendmsg(self, gui_reactor):
gui_reactor.connectTCP(self.ip, self.port, self)
def startsend(self, gui_reactor, timerep):
self.loop = LoopingCall(self.sendmsg, gui_reactor)
self.loop.start(timerep)
def stopsend(self):
self.loop.stop()
class ProvisionerQueryService(ServiceProcess):
"""Provisioner querying service
"""
declare = ServiceProcess.service_declare(name='provisioner_query',
version='0.1.0',
dependencies=[])
def slc_init(self):
interval = float(self.spawn_args.get("interval_seconds",
DEFAULT_QUERY_INTERVAL))
self.client = ProvisionerClient(self)
log.debug('Starting provisioner query loop - %s second interval',
interval)
self.loop = LoopingCall(self.query)
self.loop.start(interval)
def slc_terminate(self):
if self.loop:
self.loop.stop()
@defer.inlineCallbacks
def query(self):
try:
yield self._do_query()
except Exception,e:
log.error("Error sending provisioner query request: %s", e,
exc_info=True)
class CacheService(Service):
def __init__(self, *args, **kwargs):
self.call = None
self.node_updater = NodeCacheUpdater()
self.cluster_updater = ClusterCacheUpdater()
self.vm_updater = VirtualMachineCacheUpdater()
self.job_updater = JobCacheUpdater()
def update_cache(self):
""" a single run of all update classes """
return DeferredList(
[
self.vm_updater.update(),
self.job_updater.update(),
self.node_updater.update(),
self.cluster_updater.update(),
]
)
def startService(self):
self.call = LoopingCall(self.update_cache)
self.call.start(settings.PERIODIC_CACHE_REFRESH)
def stopService(self):
if self.call is not None:
self.call.stop()
class EventGenerator(Node):
""" (TEST) A timer-based event generator """
# --- Initialization
def __init__(self):
self.count = 0
# --- Interfaces
def configure(self):
self.events = (
'event1',
'event2',
)
def setup(self):
self.status.set_GREEN()
self.loop1 = LoopingCall(self.loop_cb1)
self.loop1.start(3, False)
self.loop2 = LoopingCall(self.loop_cb2)
self.loop2.start(4, False)
# --- Workers
def loop_cb1(self):
self.count += 1
self.sendEvent('event1', self.count)
def loop_cb2(self):
self.count += 1
self.sendEvent('event2', self.count)
def onJoin(self, details):
log.msg("PiMonitor connected")
extra = self.config.extra
self._id = extra['id']
self.publish_temperature = True
self.threshold = 0
self._tick = 0
self._cpu_temp_celsius = None
def scanTemperature():
self._cpu_temp_celsius = float(open("/sys/class/thermal/thermal_zone0/temp").read()) / 1000.
if self.publish_temperature:
self.publish(u'io.crossbar.examples.pi.{}.tempmon.on_temperature'.format(self._id), self._tick, self._cpu_temp_celsius)
self._tick += 1
if self.threshold > 0 and self._cpu_temp_celsius > self.threshold:
self.publish(u'io.crossbar.examples.pi.{}.tempmon.on_threshold_exceeded'.format(self._id), self._tick, self._cpu_temp_celsius)
scan = LoopingCall(scanTemperature)
scan.start(1)
for proc in [self.get_temperature, self.impose_stress, self.toggle_publish, self.set_threshold]:
uri = u'io.crossbar.examples.pi.{}.tempmon.{}'.format(self._id, proc.__name__)
yield self.register(proc, uri)
log.msg("Registered {}".format(uri))
log.msg("PiMonitor ready.")
def __init__(self):
"""
Initialize the model. This doesn't add any documents yet.
"""
silenceGensim()
self.dictionaries = dict()
self.preprocessor = TokenizingPorter2Stemmer()
#this dict keeps a model for every source type
# (since e.g. RSS feeds should be treated separately from twitter feeds)
self.models = dict()
#this dict keeps a dictionary for every source type
self.dictionaries = dict()
self.queue = Queue()
self.modelQueue = Queue()
self.nodeCommunicator = NodeCommunicator(self, LISTEN_PORT)
self.nodeCommunicator.registerWithNode(CORE_IP, REGISTER_PORT) # register this node with the core
ln.info("LSA Initialized")
self.updating = False
loop = LoopingCall(self.update)
loop.start(5)
reactor.run()
def delayed_setup(self):
self.feeds = []
self.failures.clear()
self.feed_times.clear()
self.targets.clear()
self.tasks.clear()
for name, target in self.config["targets"].items():
proto = target["protocol"]
if proto in self.factory_manager.factories:
self.targets[name] = target
else:
self.logger.warn("Unknown protocol '%s' in target '%s'"
% (proto, name))
for feed in self.config["feeds"]:
append = True
for target in feed["targets"]:
if target["name"] not in self.targets:
self.logger.warn("Unknown target '%s' for feed '%s'"
% (target["name"], feed["name"]))
append = False
break
if append:
self.feeds.append(feed)
for feed in self.feeds:
task = LoopingCall(self.check_feed, feed)
self.tasks[feed["name"]] = task
task.start(feed["frequency"])
class Hoster: def __init__(self, screen, port): self.screen = screen self.port = port def start(self): # Set up the connection between the state and the network; # using queues gives a degree of seperation between the # communication and game logic, and making them be deferred # keeps it all asynchronous. inqueue = DeferredQueue() outqueue = DeferredQueue() #Initialize GameState gs = GameState(self,self.screen,1,inqueue,outqueue) #Create Looping Call self.lc = LoopingCall(gs.gameLoop) self.lc.start(.0166666666) #Begin Listening connfactory = CommandConnFactory(inqueue,outqueue) self.listening = reactor.listenTCP(self.port,connfactory) def stop(self, nextscreen=None): # Stop the GameState logic, and let go of the port on which we're listening self.lc.stop() self.listening.stopListening() # Start up the next screen, if there is one if nextscreen: nextscreen.start() else: reactor.stop()
class List(set):
def __init__(self, filename, url='', mode='MERGE', refreshPeriod=0):
set.__init__(self)
self.filename = filename
self.url = url
self.mode = mode
self.load()
if url != '' and refreshPeriod != 0:
self.lc = LoopingCall(self.update)
self.lc.start(refreshPeriod)
def load(self):
"""
Load the list from the specified file.
"""
self.clear()
#simple touch to create non existent files
try:
open(self.filename, 'a').close()
with open(self.filename, 'r') as fh:
for l in fh.readlines():
self.add(re.split("#", l)[0].rstrip("[ , \n,\t]"))
except:
pass
def dump(self):
"""
Dump the list to the specified file.
"""
try:
with open(self.filename, 'w') as fh:
for l in self:
fh.write(l + "\n")
except:
pass
def handleData(self, data):
if self.mode == 'REPLACE':
self.clear()
for elem in data.split('\n'):
if elem != '':
self.add(elem)
def processData(self, data):
try:
if len(data) != 0:
self.handleData(data)
self.dump()
except Exception:
pass
def update(self):
pageFetchedDeferred = getPageCached(self.url)
pageFetchedDeferred.addCallback(self.processData)
return pageFetchedDeferred
class UdpSender(DatagramProtocol):
"""
Class that will send UDP packets to UDP Listener
"""
def __init__(self, host, count, size, duration):
# LoopingCall does not know whether UDP socket is actually writable
self.looper = None
self.host = host
self.count = count
self.duration = duration
self.start = time.time()
self.sent = 0
self.data = array.array('c', 'X' * size)
def startProtocol(self):
self.looper = LoopingCall(self.sendData)
period = self.duration / float(self.count)
self.looper.start(period, now=False)
def stopProtocol(self):
if (self.looper is not None):
self.looper.stop()
self.looper = None
def datagramReceived(self, data, (host, port)):
pass
def begin_logging(self):
def logging_callback(*args, **kwargs):
print("number of players: " + str(len(self.clients)))
print("number of lobbies: " + str(len(self.lobbys)))
print("Total number of connects: " + str(self.total_clients))
looping_call = LoopingCall(logging_callback)
looping_call.start(60.0*60.0) #every hour
def authenticate(self, twistedStanza):
"""authentication callback"""
self._send = twistedStanza.send
# add observers for incoming message / presence requests
# twistedStanza.addObserver('/iq', self.iqHandler)
# twistedStanza.addObserver('/presence', self.dialogHandler.presenceHandler)
twistedStanza.addObserver("/presence", self.presenceHandler)
twistedStanza.addObserver("//event/stream/error", self.streamErrorHandler)
# twistedStanza.addObserver('//event/stream/error', self.streamErrorHandler)
# twistedStanza.addObserver('/message', self.dialogHandler.messageHandler)
twistedStanza.addObserver("/message", self.messageHandler)
self.requestRoster()
# let'em know we are online
presence = domish.Element((JABBER_CLIENT_NS, "presence"))
presence.addElement("status").addContent("Online")
self.send(presence)
keepalive = LoopingCall(self.loopEntry)
keepalive.start(60)
self.joinChatrooms(self.chatrooms)
def addSlave(self, slaveSpec):
# try to catch a slave reconnection; else assign a new slave ID
try:
(connectedSlave, connectedSlaveStatus, connectedSlaveTask) = self._getSlaveBySlaveSpec(slaveSpec, asTuple=True)
slaveId = self._getSlaveIdByObject(connectedSlave)
log.warn("Slave %d (%s://%s:%d/%s) reconnecting. Resetting slave state!" % (slaveId, connectedSlave.slaveSpec.scheme, connectedSlave.slaveSpec.host, connectedSlave.slaveSpec.port, connectedSlave.slaveSpec.path))
except SlaveNotFound:
slaveNo = self._getSlaveNo()
else:
slaveNo = slaveId
slave = SlavePerspective(slaveSpec)
status = SlaveState()
status.state = SlaveState.CONNECTED
# check that the slave is up and running
request = self.checkHealth(slave)
yield request
if request.result != True:
raise SlaveConnectionError
else:
status.state = SlaveState.IDLE
# set up a heartbeat call
task = LoopingCall(self.checkHealth, slave)
task.start(60, now=False)
self.slaves[slaveNo] = (slave, status, task)
returnValue(slaveNo)
class LiveStream(WebSocketClientProtocol): # pragma: no cover
""" Internal class used to call the websocket callbacks. """
sendqueue = []
sending = False
def _sender(self):
if len(self.sendqueue):
while self.sendqueue:
data = self.sendqueue.pop()
self.sendMessage(data)
def queueMessage(self, message):
self.sendqueue.append(message)
if not self.sending:
self.sending = True
self._loopingcall = LoopingCall(self._sender)
self._loopingcall.start(0.1)
def onOpen(self):
self.factory.datasift['send_message'] = self.queueMessage
self.factory.datasift['on_open']()
def onClose(self, wasClean, code, reason):
self.factory.datasift['on_close'](wasClean, code, reason)
def onMessage(self, msg, binary):
self.factory.datasift['on_message'](msg, binary)
self.factory.resetDelay()
def onPing(self, payload):
self.factory.resetDelay()
class Blinker(object):
def __init__(self, port, interval=0.5):
self.port = port
self.interval = interval
self.state = None
self.transition = None
def set(self, *args, **kwargs):
self.blink()
def unset(self):
self.transition_stop()
self.port.unset()
def blink(self):
self.transition_stop()
self.transition = LoopingCall(self.blink_task)
self.transition.start(self.interval)
def transition_stop(self):
if self.transition and self.transition.running:
self.transition.stop()
def blink_task(self):
if self.state:
self.state = False
self.port.set()
else:
self.state = True
self.port.unset()
class NetworkSpeedDaemon(VMCDaemon):
"""
I enque fake SIG_SPEED UnsolicitedNotifications
"""
def __init__(self, frequency, device, notification_manager):
super(NetworkSpeedDaemon, self).__init__(frequency, device, notification_manager)
self.netspeed = NetworkSpeed()
def start(self):
log.msg("DAEMONS: DAEMON %s started..." % self.__class__.__name__)
self.netspeed.start()
self.loop = LoopingCall(self.function)
self.loop.start(self.frequency)
def stop(self):
log.msg("DAEMONS: DAEMON %s stopped..." % self.__class__.__name__)
self.loop.stop()
self.netspeed.stop()
def function(self):
up, down = self.netspeed["up"], self.netspeed["down"]
n = N.UnsolicitedNotification(N.SIG_SPEED, bps_to_human(up, down))
self.manager.on_notification_received(n)
def add(self, request, save_time = 0):
self.data[request] = time.time() + save_time * 60
d = request.notifyFinish()
d.addCallback(self.notifyFinished, request)
d.addErrback(self.notifyFinished, request)
call = LoopingCall(self.prune)
call.start(60)
class TelnetProtocol(Telnet, StatefulTelnetProtocol, Session):
"""
Each player connecting over telnet (ie using most traditional mud
clients) gets a telnet protocol instance assigned to them. All
communication between game and player goes through here.
"""
def __init__(self, *args, **kwargs):
self.protocol_name = "telnet"
super(TelnetProtocol, self).__init__(*args, **kwargs)
def connectionMade(self):
"""
This is called when the connection is first established.
"""
# initialize the session
self.line_buffer = ""
client_address = self.transport.client
client_address = client_address[0] if client_address else None
# this number is counted down for every handshake that completes.
# when it reaches 0 the portal/server syncs their data
self.handshakes = 7 # naws, ttype, mccp, mssp, msdp, gmcp, mxp
self.init_session(self.protocol_name, client_address,
self.factory.sessionhandler)
# negotiate client size
self.naws = naws.Naws(self)
# negotiate ttype (client info)
# Obs: mudlet ttype does not seem to work if we start mccp before ttype. /Griatch
self.ttype = ttype.Ttype(self)
# negotiate mccp (data compression) - turn this off for wireshark analysis
self.mccp = Mccp(self)
# negotiate mssp (crawler communication)
self.mssp = mssp.Mssp(self)
# oob communication (MSDP, GMCP) - two handshake calls!
self.oob = telnet_oob.TelnetOOB(self)
# mxp support
self.mxp = Mxp(self)
# add this new connection to sessionhandler so
# the Server becomes aware of it.
self.sessionhandler.connect(self)
# timeout the handshakes in case the client doesn't reply at all
from evennia.utils.utils import delay
delay(2, callback=self.handshake_done, force=True)
# TCP/IP keepalive watches for dead links
self.transport.setTcpKeepAlive(1)
# The TCP/IP keepalive is not enough for some networks;
# we have to complement it with a NOP keep-alive.
self.protocol_flags["NOPKEEPALIVE"] = True
self.nop_keep_alive = None
self.toggle_nop_keepalive()
def _send_nop_keepalive(self):
"Send NOP keepalive unless flag is set"
if self.protocol_flags.get("NOPKEEPALIVE"):
self._write(IAC + NOP)
def toggle_nop_keepalive(self):
"""
Allow to toggle the NOP keepalive for those sad clients that
can't even handle a NOP instruction. This is turned off by the
protocol_flag NOPKEEPALIVE (settable e.g. by the default
`@option` command).
"""
if self.nop_keep_alive and self.nop_keep_alive.running:
self.nop_keep_alive.stop()
else:
self.nop_keep_alive = LoopingCall(self._send_nop_keepalive)
self.nop_keep_alive.start(30, now=False)
def handshake_done(self, force=False):
"""
This is called by all telnet extensions once they are finished.
When all have reported, a sync with the server is performed.
The system will force-call this sync after a small time to handle
clients that don't reply to handshakes at all.
"""
if self.handshakes > 0:
if force:
self.sessionhandler.sync(self)
return
self.handshakes -= 1
if self.handshakes <= 0:
# do the sync
self.sessionhandler.sync(self)
def enableRemote(self, option):
"""
This sets up the remote-activated options we allow for this protocol.
Args:
option (char): The telnet option to enable.
Returns:
enable (bool): If this option should be enabled.
"""
return (option == LINEMODE or option == ttype.TTYPE
or option == naws.NAWS or option == MCCP
or option == mssp.MSSP)
def enableLocal(self, option):
"""
Call to allow the activation of options for this protocol
Args:
option (char): The telnet option to enable locally.
Returns:
enable (bool): If this option should be enabled.
"""
return (option == MCCP or option == ECHO)
def disableLocal(self, option):
"""
Disable a given option locally.
Args:
option (char): The telnet option to disable locally.
"""
if option == ECHO:
return True
if option == MCCP:
self.mccp.no_mccp(option)
return True
else:
return super(TelnetProtocol, self).disableLocal(option)
def connectionLost(self, reason):
"""
this is executed when the connection is lost for whatever
reason. it can also be called directly, from the disconnect
method
Args:
reason (str): Motivation for losing connection.
"""
self.sessionhandler.disconnect(self)
self.transport.loseConnection()
def applicationDataReceived(self, data):
"""
Telnet method called when non-telnet-command data is coming in
over the telnet connection. We pass it on to the game engine
directly.
Args:
string (str): Incoming data.
"""
if not data:
data = [data]
elif data.strip() == NULL:
# this is an ancient type of keepalive used by some
# legacy clients. There should never be a reason to send a
# lone NULL character so this seems to be a safe thing to
# support for backwards compatibility. It also stops the
# NULL from continously popping up as an unknown command.
data = [_IDLE_COMMAND]
else:
data = _RE_LINEBREAK.split(data)
if self.line_buffer and len(data) > 1:
# buffer exists, it is terminated by the first line feed
data[0] = self.line_buffer + data[0]
self.line_buffer = ""
# if the last data split is empty, it means all splits have
# line breaks, if not, it is unterminated and must be
# buffered.
self.line_buffer += data.pop()
# send all data chunks
for dat in data:
self.data_in(text=dat + "\n")
def _write(self, data):
"hook overloading the one used in plain telnet"
data = data.replace('\n', '\r\n').replace('\r\r\n', '\r\n')
super(TelnetProtocol, self)._write(mccp_compress(self, data))
def sendLine(self, line):
"""
Hook overloading the one used by linereceiver.
Args:
line (str): Line to send.
"""
#escape IAC in line mode, and correctly add \r\n
line += self.delimiter
line = line.replace(IAC, IAC + IAC).replace('\n', '\r\n')
return self.transport.write(mccp_compress(self, line))
# Session hooks
def disconnect(self, reason=None):
"""
generic hook for the engine to call in order to
disconnect this protocol.
Args:
reason (str): Reason for disconnecting.
"""
self.data_out(text=((reason or "", ), {}))
self.connectionLost(reason)
def data_in(self, **kwargs):
"""
Data User -> Evennia
Kwargs:
kwargs (any): Options from the protocol.
"""
#from evennia.server.profiling.timetrace import timetrace
#text = timetrace(text, "telnet.data_in")
self.sessionhandler.data_in(self, **kwargs)
def data_out(self, **kwargs):
"""
Data Evennia -> User
Kwargs:
kwargs (any): Options to the protocol
"""
self.sessionhandler.data_out(self, **kwargs)
# send_* methods
def send_text(self, *args, **kwargs):
"""
Send text data. This is an in-band telnet operation.
Args:
text (str): The first argument is always the text string to send. No other arguments
are considered.
Kwargs:
options (dict): Send-option flags
- mxp: Enforce MXP link support.
- ansi: Enforce no ANSI colors.
- xterm256: Enforce xterm256 colors, regardless of TTYPE.
- noxterm256: Enforce no xterm256 color support, regardless of TTYPE.
- nomarkup: Strip all ANSI markup. This is the same as noxterm256,noansi
- raw: Pass string through without any ansi processing
(i.e. include Evennia ansi markers but do not
convert them into ansi tokens)
- echo: Turn on/off line echo on the client. Turn
off line echo for client, for example for password.
Note that it must be actively turned back on again!
"""
text = args[0] if args else ""
if text is None:
return
text = to_str(text, force_string=True)
# handle arguments
options = kwargs.get("options", {})
flags = self.protocol_flags
xterm256 = options.get(
"xterm256",
flags.get('XTERM256', False) if flags["TTYPE"] else True)
useansi = options.get(
"ansi",
flags.get('ANSI', False) if flags["TTYPE"] else True)
raw = options.get("raw", flags.get("RAW", False))
nomarkup = options.get(
"nomarkup", flags.get("NOMARKUP", not (xterm256 or useansi)))
echo = options.get("echo", None)
mxp = options.get("mxp", flags.get("MXP", False))
screenreader = options.get("screenreader",
flags.get("SCREENREADER", False))
if screenreader:
# screenreader mode cleans up output
text = ansi.parse_ansi(text,
strip_ansi=True,
xterm256=False,
mxp=False)
text = _RE_SCREENREADER_REGEX.sub("", text)
if options.get("send_prompt"):
# send a prompt instead.
if not raw:
# processing
prompt = ansi.parse_ansi(_RE_N.sub("", text) + "{n",
strip_ansi=nomarkup,
xterm256=xterm256)
if mxp:
prompt = mxp_parse(prompt)
prompt = prompt.replace(IAC, IAC + IAC).replace('\n', '\r\n')
prompt += IAC + GA
self.transport.write(mccp_compress(self, prompt))
else:
if echo is not None:
# turn on/off echo. Note that this is a bit turned around since we use
# echo as if we are "turning off the client's echo" when telnet really
# handles it the other way around.
if echo:
# by telling the client that WE WON'T echo, the client knows
# that IT should echo. This is the expected behavior from
# our perspective.
self.transport.write(mccp_compress(self,
IAC + WONT + ECHO))
else:
# by telling the client that WE WILL echo, the client can
# safely turn OFF its OWN echo.
self.transport.write(mccp_compress(self,
IAC + WILL + ECHO))
if raw:
# no processing
self.sendLine(text)
return
else:
# we need to make sure to kill the color at the end in order
# to match the webclient output.
linetosend = ansi.parse_ansi(_RE_N.sub("", text) + "{n",
strip_ansi=nomarkup,
xterm256=xterm256,
mxp=mxp)
if mxp:
linetosend = mxp_parse(linetosend)
self.sendLine(linetosend)
def send_prompt(self, *args, **kwargs):
"""
Send a prompt - a text without a line end. See send_text for argument options.
"""
kwargs["options"].update({"send_prompt": True})
self.send_text(*args, **kwargs)
def send_default(self, cmdname, *args, **kwargs):
"""
Send other oob data
"""
if not cmdname == "options":
self.oob.data_out(cmdname, *args, **kwargs)
class AdapterPmMetrics:
class Metrics:
def __init__(self, config, value):
self.config = config
self.value = value
def __init__(self,device):
self.pm_names = {'tx_64_pkts','tx_65_127_pkts', 'tx_128_255_pkts',
'tx_256_511_pkts', 'tx_512_1023_pkts',
'tx_1024_1518_pkts', 'tx_1519_9k_pkts', 'rx_64_pkts',
'rx_65_127_pkts', 'rx_128_255_pkts', 'rx_256_511_pkts',
'rx_512_1023_pkts', 'rx_1024_1518_pkts',
'rx_1519_9k_pkts', 'tx_pkts', 'rx_pkts',
'tx_bytes', 'rx_bytes'}
# This is just to generate more realistic looking values. This would
# not be implemented in a normal adapter.
self.rand_ranges = dict (
tx_64_pkts=[50, 55],
tx_65_127_pkts=[55,60],
tx_128_255_pkts=[60,65],
tx_256_511_pkts=[85,90],
tx_512_1023_pkts=[90,95],
tx_1024_1518_pkts=[60,65],
tx_1519_9k_pkts=[50,55],
rx_64_pkts=[50, 55],
rx_65_127_pkts=[55,60],
rx_128_255_pkts=[60,65],
rx_256_511_pkts=[85,90],
rx_512_1023_pkts=[90,95],
rx_1024_1518_pkts=[60,65],
rx_1519_9k_pkts=[50,55],
tx_pkts=[90,100],
rx_pkts=[90,100],
rx_bytes=[90000,100000],
tx_bytes=[90000,100000]
)
self.device = device
self.id = device.id
self.default_freq = 150
self.grouped = False
self.freq_override = False
self.pon_metrics = dict()
self.nni_metrics = dict()
self.lc = None
for m in self.pm_names:
self.pon_metrics[m] = \
self.Metrics(config = PmConfig(name=m,
type=PmConfig.COUNTER,
enabled=True), value = 0)
self.nni_metrics[m] = \
self.Metrics(config = PmConfig(name=m,
type=PmConfig.COUNTER,
enabled=True), value = 0)
def update(self, pm_config):
if self.default_freq != pm_config.default_freq:
# Update the callback to the new frequency.
self.default_freq = pm_config.default_freq
self.lc.stop()
self.lc.start(interval=self.default_freq/10)
for m in pm_config.metrics:
self.pon_metrics[m.name].config.enabled = m.enabled
self.nni_metrics[m.name].config.enabled = m.enabled
def make_proto(self):
pm_config = PmConfigs(
id=self.id,
default_freq=self.default_freq,
grouped = False,
freq_override = False)
for m in sorted(self.pon_metrics):
pm=self.pon_metrics[m]
pm_config.metrics.extend([PmConfig(name=pm.config.name,
type=pm.config.type,
enabled=pm.config.enabled)])
return pm_config
def collect_pon_metrics(self):
import random
rtrn_pon_metrics = dict()
for m in self.pm_names:
if self.pon_metrics[m].config.enabled:
self.pon_metrics[m].value += \
random.randint(self.rand_ranges[m][0], self.rand_ranges[m][1])
rtrn_pon_metrics[m] = self.pon_metrics[m].value
return rtrn_pon_metrics
def collect_nni_metrics(self):
import random
rtrn_nni_metrics = dict()
for m in self.pm_names:
if self.nni_metrics[m].config.enabled:
self.nni_metrics[m].value += \
random.randint(self.rand_ranges[m][0], self.rand_ranges[m][1])
rtrn_nni_metrics[m] = self.nni_metrics[m].value
return rtrn_nni_metrics
def start_collector(self, device_name, device_id, callback):
prefix = 'voltha.{}.{}'.format(device_name, device_id)
self.lc = LoopingCall(callback, device_id, prefix)
self.lc.start(interval=self.default_freq/10)
class ConsumerNexus(Nexus):
def __init__(self, below_nexus, layer):
super().__init__(below_nexus, layer)
self.onping = None
self.consumer_finished_cb = None
self.handshake_finished = False
self.ping_interval = None
self.ping_start_time = None
###########################################################################
def is_layer_message(self, msg):
if msg['message_class'] != "NOTIFICATION":
return False
return msg['notification_name'] in {
"NOTIFY_PROVIDER", "NOTIFY_PROVIDER_NOT_READY", "NOTIFY_PING"
}
def on_message(self, below_nexus, msg):
logging.info("consumer nexus got msg")
if not self.is_layer_message(msg):
super().on_message(below_nexus, msg)
return
if msg['notification_name'] == "NOTIFY_PROVIDER":
if not self.handshake_finished:
self.handshake_finished = True
# nexus is announced to the transact layer
# TODO, this message might be better to handle in the transact
# layer instead. Refactor might be needed.
self.consumer_finished_cb(self)
# pass the message above for the transact layer to process
super().on_message(below_nexus, msg)
elif msg['notification_name'] == "NOTIFY_PROVIDER_NOT_READY":
logging.info("provider not ready, waiting")
elif msg['notification_name'] == "NOTIFY_PONG":
if not self.ping_start_time:
return
msecs = (time.time() - self.ping_start_time) * 1000
if self.onping:
self.onping(self, round(msecs))
self.ping_start_time = None
def on_bin_message(self, below_nexus, msg_bytes):
logging.info("provider nexus got raw msg")
super().on_bin_message(below_nexus, msg_bytes)
###########################################################################
def start_handshake(self, consumer_finished_cb):
self.consumer_finished_cb = consumer_finished_cb
self.send(RequestProvider())
def send_ping(self):
self.ping_start_time = time.time()
self.send(RequestPing())
def start_pinging(self):
logging.info("START PING")
self.ping_interval = LoopingCall(self.send_ping)
self.ping_interval.start(3.0, now=True)
def stop_pinging(self):
logging.info("STOP PING")
if not self.ping_interval:
return
self.ping_interval.stop()
self.ping_interval = None
class FabricNode(node.Node):
"""
Crossbar.io FX node personality.
"""
DEFAULT_CONFIG_PATH = 'edge/node/config/pairme.json'
NODE_CONTROLLER = FabricNodeControllerSession
def __init__(self, personality, cbdir=None, reactor=None, native_workers=None, options=None):
node.Node.__init__(self, personality, cbdir, reactor, native_workers, options)
# looping call that runs the node controller watchdog
self._watchdog_looper = None
# the node controller realm (where eg worker controller live). we explicitly use the
# same realm as Crossbar.io OSS
self._realm = 'crossbar'
# enable docker daemon integration
self._enable_docker = None
# when running in managed mode, this will hold the bridge session
# attached to the local management router
self._bridge_session = None
# if this node has a proper management uplink configured to connect to
self._has_management_config = False
# if this node was connected to its configured management uplink successfully at least once
# during run-time (since last reboot) of this node
self._was_management_connected = False
# when we periodically check for a node activation file, the looping call for doing
# the check - and automatically shutdown when an activation file was found (after boot)
self._check_for_activation_file = None
# when self._was_management_connected, the URL we've been connected to
self._management_url = None
# when running in managed mode, this will hold the management uplink session to
# the crossbar master node
self._manager = None
self._manager_runner = None
# the node's management realm when running in managed mode (this comes from CFC!)
self._management_realm = None
# the node's ID when running in managed mode (this comes from CFC!)
self._node_id = None
# node extra when running in managed mode (this comes from CFC!)
self._node_extra = None
# when the node starts up, it will connect to CFC, and then apply the
# local node configuration, and this flag will be set. when the CFC connection
# is lost, and then reestablished, the local node configuration should NOT
# be applied a second time though - hence this flag
self._local_config_applied = False
# We really only need to see this once (?)
self._displayed_pairing_message = False
# for automatic ID assignment of "makers" within workers of type "xbrmm"
self._maker_no = 1
def load_config(self, configfile=None):
"""
Check and load the node configuration from:
* from ``.crossbar/config.json`` or
* from built-in (empty) default configuration
This is the _second_ function being called after the Node has been instantiated.
IMPORTANT: this function is run _before_ start of Twisted reactor!
"""
config_source = None
config_path = None
# if the node hasn't been configured from XBR network, fallback to loading config from local config file
if not self._config:
default_filename = pkg_resources.resource_filename('crossbar', self.DEFAULT_CONFIG_PATH)
with open(default_filename) as f:
default_config = json.load(f)
config_source, config_path = node.Node.load_config(self, configfile, default_config)
self.log.info('Node configuration loaded from {config_source} ({config_path})',
config_source=hlid(config_source),
config_path=hlid(config_path))
# Docker host integration
if _HAS_DOCKER and self._config and 'controller' in self._config:
self._enable_docker = self._config['controller'].get('enable_docker', False)
return config_source, config_path
def _watchdog(self):
# on Linux, check that we start with sufficient system entropy
entropy_avail = None
if sys.platform.startswith('linux'):
try:
with open('/proc/sys/kernel/random/entropy_avail', 'r') as ent:
entropy_avail = int(ent.read())
# FIXME: my machine never has more than ~ 300 units available, 1000 seems a little optomistic!
if entropy_avail < 64:
self.log.warn('WARNING: insufficient entropy ({} bytes) available - try installing rng-tools'.format(entropy_avail))
except PermissionError:
# this happens when packaged as a snap: the code prevented from reading a location
# # that is not allowed to a confined snap package
entropy_avail = -1
# check for at least 100MB free memory
mem_avail = psutil.virtual_memory().available // 2 ** 20
if mem_avail < 100:
self.log.warn('WARNING: available memory dropped to {mem_avail} MB', mem_avail=mem_avail)
self.log.trace('WATCHDOG: entropy_avail {entropy_avail} bytes, mem_avail {mem_avail} MB',
entropy_avail=entropy_avail, mem_avail=mem_avail)
@inlineCallbacks
def start(self, node_id=None):
self.log.info('{note} [{method}]',
note=hl('Starting node (initialize edge-node personality) ..', color='green', bold=True),
method=hltype(FabricNode.start))
# run watchdog at 5Hz
self._watchdog_looper = LoopingCall(self._watchdog)
self._watchdog_looper.start(.2)
res = yield node.Node.start(self, node_id or self._node_id)
return res
@inlineCallbacks
def boot(self, use_activation_file=True, use_default_fabric=False):
self.log.info('Booting node {method}', method=hltype(FabricNode.boot))
def reboot():
self.stop(restart=True)
# determine the transport configuration of the management uplink for this node
fabric_transport_config = compute_mgmt_uplink_config(self.log, self._cbdir, self._config, reboot,
use_activation_file=use_activation_file,
use_default_fabric=use_default_fabric)
# now finally, if we do have a transport configuration for the management uplink at this point,
# then start the management uplink ..
if fabric_transport_config:
self._has_management_config = True
url = fabric_transport_config['url']
hostname = None
if 'tls' in fabric_transport_config.get('endpoint', {}):
hostname = fabric_transport_config['endpoint']['tls']['hostname']
self._manager_runner = ApplicationRunner(
url=url,
realm=None,
extra=None,
ssl=optionsForClientTLS(hostname) if hostname else None,
)
def make(config):
# extra info forwarded to CFC client session
extra = {
'node': self,
'on_ready': Deferred(),
'on_exit': Deferred(),
'node_key': self._node_key,
}
@inlineCallbacks
def on_ready_success(res):
try:
self._manager, self._management_realm, self._management_session_id, self._node_id, self._node_extra = res
if self._bridge_session:
try:
yield self._bridge_session.attach_manager(
self._manager, self._management_realm, self._node_id)
except:
self.log.failure()
else:
while True:
try:
# we actually test the management uplink by calling a procedure on the master
yield self._manager.call('crossbarfabriccenter.mrealm.get_status')
except ApplicationError as e:
if e.error == 'wamp.error.no_such_procedure':
self.log.info('Could not get master status ("wamp.error.no_such_procedure") - retrying in 5 secs ..')
else:
self.log.failure()
yield sleep(5)
except:
self.log.failure()
self.log.info('Could not get master status - retrying in 5 secs ..')
yield sleep(5)
else:
self.log.info(
click.style(
'Connected to Crossbar.io FX Master at management realm "{realm}", set node ID "{node_id}" (extra={node_extra}, session_id={session_id})',
fg='green',
bold=True),
realm=self._management_realm,
node_id=self._node_id,
node_extra=self._node_extra,
session_id=self._manager._session_id)
# if the management uplink was successfully established and tested once, mark it so
if not self._was_management_connected:
self._was_management_connected = True
self._management_url = url
try:
worker_ids = yield self._bridge_session.call(
'crossbar.get_workers')
for worker_id in worker_ids:
yield self._bridge_session.call(
'crossbar.worker.{}.set_node_id'.format(worker_id),
self._node_id)
except:
self.log.warn(
'INTERNAL ERROR: could not set node_id "{node_id}" after CFC connection was established',
node_id=self._node_id)
self.log.failure()
break
else:
self.log.warn(
'Uplink Crossbar.io FX Master session established, but no bridge session setup!'
)
except Exception as e:
self.log.warn('error in on_ready: {}'.format(e))
# ok, we are connected to CFC and normally will be configurated programmatically from there.
# however, it is still possible to apply any local node configuration by setting
#
# node_extra:
# {
# "on_start_apply_config", true
# }
#
# node_extra comes from CFC and has to be configured there (when the node is paired)
#
if self._node_extra:
# by default, apply local config (from a node configuration file, if there is one)
on_start_apply_config = self._node_extra.get('on_start_apply_config', True)
if on_start_apply_config:
if not self._local_config_applied:
self.log.info('Applying local node configuration (on_start_apply_config is enabled)')
yield self.boot_from_config(self._config)
self._local_config_applied = True
else:
self.log.info('Local node configuration was already applied - skipping')
else:
self.log.info('Skipping any local node configuration (no local config or on_start_apply_config is "off")')
def on_ready_error(err):
if isinstance(
err.value,
ApplicationError) and err.value.error in ['fabric.auth-failed.node-unpaired', 'fabric.auth-failed.node-already-connected']:
if not self._displayed_pairing_message:
self._displayed_pairing_message = True
self.log.error(
click.style(err.value.error_message().upper(), fg='red', bold=True))
self.stop()
else:
self.log.error(click.style(
'Could not connect to CFC: {error}', fg='red', bold=True), error=err.value
)
@inlineCallbacks
def on_exit_success(reason):
if self._bridge_session:
try:
yield self._bridge_session.detach_manager()
except:
self.log.failure()
else:
self.log.debug(
'Disconnected from Crossbar.io FX Master for management realm "{realm}"',
realm=self._management_realm)
else:
self.log.warn(
'Uplink Crossbar.io FX Master session lost, but no bridge session setup!')
self._manager, self._management_realm, self._management_session_id, self._node_id, self._node_extra = None, None, None, None, None
def on_exit_error(err):
print(err)
extra['on_ready'].addCallbacks(on_ready_success, on_ready_error)
extra['on_exit'].addCallbacks(on_exit_success, on_exit_error)
config = ComponentConfig(extra=extra)
session = NodeManagementSession(self._manager_runner, config)
return session
self.log.info('Connecting to Crossbar.io FX Master at {url} ..', url=url)
yield self._manager_runner.run(make, start_reactor=False, auto_reconnect=True)
else:
# here, we don't have a management uplink :(
self.log.info(
hl('No management uplink configured (running unmanaged/single-node)',
color='red',
bold=True))
self._has_management_config = False
# nevertheless, now boot from local node config!
yield self.boot_from_config(self._config)
self._local_config_applied = True
def _add_extra_controller_components(self, controller_config):
extra = {
'node_key': self._node_key,
'controller_config': controller_config,
}
cfg = ComponentConfig(self._realm, extra=extra)
self._bridge_session = NodeManagementBridgeSession(cfg)
router = self._router_factory.get(self._realm)
self._router_session_factory.add(self._bridge_session, router, authrole='trusted')
def _set_shutdown_triggers(self, controller_options):
if 'shutdown' in controller_options:
self._node_shutdown_triggers = controller_options['shutdown']
self.log.info("Using node shutdown triggers {triggers} from configuration", triggers=self._node_shutdown_triggers)
else:
# NODE_SHUTDOWN_ON_SHUTDOWN_REQUESTED
# NODE_SHUTDOWN_ON_WORKER_EXIT
# NODE_SHUTDOWN_ON_WORKER_EXIT_WITH_ERROR
# NODE_SHUTDOWN_ON_LAST_WORKER_EXIT
# in managed mode, a node - by default - only shuts down when explicitly asked to,
# or upon a fatal error in the node controller
self._node_shutdown_triggers = [checkconfig.NODE_SHUTDOWN_ON_SHUTDOWN_REQUESTED]
self.log.info("Using default node shutdown triggers {triggers}", triggers=self._node_shutdown_triggers)
def _add_worker_role(self, worker_auth_role, options):
worker_role_config = {
# each (native) worker is authenticated under a worker-specific authrole
"name":
worker_auth_role,
"permissions": [
# the worker requires these permissions to work:
{
# management API provided by the worker. note that the worker management API is provided under
# the URI prefix "crossbar.worker.<worker_id>". note that the worker is also authenticated
# under authrole <worker_auth_role> on realm "crossbar"
"uri": worker_auth_role,
"match": "prefix",
"allow": {
"call": True,
"register": True,
"publish": True,
"subscribe": True
},
"disclose": {
"caller": True,
"publisher": True
},
"cache": True
},
{
# controller procedure called by the worker (to check for controller status)
"uri": "crossbar.get_status",
"match": "exact",
"allow": {
"call": True,
"register": False,
"publish": False,
"subscribe": False
},
"disclose": {
"caller": True,
"publisher": True
},
"cache": True
}
]
}
# if configured to expose the controller connection within the worker (to make it available
# in user code such as dynamic authenticators and router/container components), also add
# permissions to actually use the (local) node management API
if options.get('expose_controller', True):
vendor_permissions = {
"uri": "crossbar.",
"match": "prefix",
"allow": {
"call": True,
"register": False,
"publish": False,
"subscribe": True
},
"disclose": {
"caller": True,
"publisher": True
},
"cache": True
}
worker_role_config["permissions"].append(vendor_permissions)
vendor_permissions = {
"uri": "crossbarfabriccenter.",
"match": "prefix",
"allow": {
"call": True,
"register": True,
"publish": True,
"subscribe": True
},
"disclose": {
"caller": True,
"publisher": True
},
"cache": True
}
worker_role_config["permissions"].append(vendor_permissions)
self._router_factory.add_role(self._realm, worker_role_config)
self.log.info(
'worker-specific role "{authrole}" added on node management router realm "{realm}" {func}',
func=hltype(self._add_worker_role),
authrole=hlid(worker_role_config['name']),
realm=hlid(self._realm))
def _extend_worker_args(self, args, options):
if 'expose_shared' in options and options['expose_shared']:
args.extend(['--expose_shared=true'])
if 'expose_controller' in options and options['expose_controller']:
args.extend(['--expose_controller=true'])
@inlineCallbacks
def _configure_native_worker_connections(self, worker_logname, worker_id, worker):
# start connections (such as PostgreSQL database connection pools)
# to run embedded in the router
for connection in worker.get('connections', []):
if 'id' in connection:
connection_id = connection.pop('id')
else:
connection_id = 'connection{:03d}'.format(self._connection_no)
self._connection_no += 1
yield self._controller.call('crossbar.worker.{}.start_connection'.format(worker_id), connection_id, connection, options=CallOptions())
self.log.info(
"{logname}: connection '{connection}' started",
logname=worker_logname,
connection=connection_id,
)
@inlineCallbacks
def _configure_native_worker_router(self, worker_logname, worker_id, worker):
# setup db connection pool
yield self._configure_native_worker_connections(worker_logname, worker_id, worker)
# in this case, call the base class method _after_ above - because we want db connections
# to be available when router/container components might start ..
yield node.Node._configure_native_worker_router(self, worker_logname, worker_id, worker)
@inlineCallbacks
def _configure_native_worker_container(self, worker_logname, worker_id, worker):
# setup db connection pool
yield self._configure_native_worker_connections(worker_logname, worker_id, worker)
# in this case, call the base class method _after_ above - because we want db connections
# to be available when router/container components might start ..
yield node.Node._configure_native_worker_container(self, worker_logname, worker_id, worker)
@inlineCallbacks
def _configure_native_worker_hostmonitor(self, worker_logname, worker_id, worker):
# after the native worker has started, and the HostMonitor controller session
# has attached to the local node router, we need to do the following, if we
# want it to work _also_ from a local node config.json. driving from CFC
# at run-time always works (also without the bits here)
# 1. the native workers' common options are configured by calling into the worker
yield self._configure_native_worker_common(worker_logname, worker_id, worker)
# 2. the host monitor specific actions need to be done, namely, starting the monitoring
monitoring_config = worker.get('monitor', None)
yield self._controller.call('crossbar.worker.{}.start_monitoring'.format(worker_id), monitoring_config, options=CallOptions())
@inlineCallbacks
def _configure_native_worker_xbrmm(self, worker_logname, worker_id, worker):
# 1. configure native workers' common options
yield self._configure_native_worker_common(worker_logname, worker_id, worker)
# 2. configure market makers (defined within the xbrmm worker)
for maker in worker.get('makers', []):
if 'id' in maker:
maker_id = maker.pop('id')
else:
maker_id = 'maker{:03d}'.format(self._maker_no)
self._maker_no += 1
maker['id'] = maker_id
yield self._controller.call('crossbar.worker.{}.start_market_maker'.format(worker_id), maker_id, maker, options=CallOptions())
def compute_mgmt_uplink_config(log, cbdir, config, fn_reboot=None, use_activation_file=True, use_default_fabric=False):
"""
Determine the transport configuration of the management uplink for this node
in the following order (using the first one that succeeds):
* node activation file
* management URL environment variable
* node configuration file
* built-in default ("master.xbr.network")
:param cbdir:
:param config:
:param fn_reboot:
:param use_activation_file:
:param use_default_fabric:
:return:
"""
fabric_transport_config = None
# [1] pick up auto-activation files dropped by a master node (`${CBDIR}/key.activate`)
if not fabric_transport_config and use_activation_file:
def do_check_for_activation_file(activation_file, reboot_on_discover):
if os.path.isfile(activation_file):
tags = _parse_activation_file(activation_file)
is_secure, hostname, port, _, _, _ = parse_url(tags['management-url'])
config = {
'type': 'websocket',
'url': tags['management-url'],
'endpoint': {
'type': 'tcp',
'host': hostname,
'port': port,
'timeout': 5
}
}
if is_secure:
config['endpoint']['tls'] = {
'hostname': hostname
}
_msg = 'Found auto-activation file "{}", using management URL "{}" - [1]'.format(activation_file,
config['url'])
log.info(click.style(_msg, fg='red', bold=True))
if reboot_on_discover and fn_reboot:
# stop the node and enforce complete reboot - which will then pick up the new configuration
fn_reboot()
# return the management uplink transport configuration, as derived from the activation file
return config
# an activation file must be placed next to the node key pair (key.pub, key.priv)
activation_file = os.path.join(cbdir, 'key.activate')
# check and maybe load config from activation file
fabric_transport_config = do_check_for_activation_file(activation_file, reboot_on_discover=False)
# if there wasn't an activation file, periodically check for ..
if not fabric_transport_config:
lc = LoopingCall(do_check_for_activation_file, activation_file, reboot_on_discover=True)
lc.start(1)
log.info('Looping call to check for node activation file started! - [1]')
# [2] management uplink configured via env var
if not fabric_transport_config:
url = os.environ['CROSSBAR_FABRIC_URL'].strip() if 'CROSSBAR_FABRIC_URL' in os.environ else ''
if url != '':
secure, host, tcp_port, _, _, _ = parse_url(url)
fabric_transport_config = {
'type': 'websocket',
'url': url,
'endpoint': {
'type': 'tcp',
'host': host,
'port': tcp_port,
'timeout': 5
}
}
if secure:
fabric_transport_config['endpoint']['tls'] = {
'hostname': host
}
log.info(
click.style('Using management uplink at "{}" (from envvar) - [2]'.format(url),
fg='red', bold=True))
# [3] user has configured a custom management uplink in the node configuration
if not fabric_transport_config:
if 'controller' in config and 'fabric' in config['controller'] and config['controller']['fabric']:
fabric_config = config['controller']['fabric']
# allow to deactivate the management uplink connecting transport by setting "transport" to null
fabric_transport_config = fabric_config.get('transport', None)
if fabric_transport_config:
url = fabric_transport_config.get('url', None)
log.info(
click.style('Using management uplink at "{}" (from node configuration) - [3]'.format(url),
fg='red', bold=True))
else:
log.info(
click.style('Management uplink deactivated - [3]',
fg='red', bold=True))
# [4] use hard-coded default management uplink
if not fabric_transport_config and use_default_fabric:
# default CFC (= our cloud hosted CFC service)
fabric_transport_config = {
'type': 'websocket',
'url': 'wss://master.xbr.network/ws',
'endpoint': {
'type': 'tcp',
'host': 'master.xbr.network',
'port': 443,
'timeout': 5,
'tls': {
'hostname': 'master.xbr.network'
}
}
}
log.info(
click.style(
'Using default fabric controller at URL "{}" (from envvar) - [4]'.format(fabric_transport_config['url']),
fg='red', bold=True))
return fabric_transport_config
_FLUSH_CACHE(_IDMAPPER_CACHE_MAXSIZE)
if _MAINTENANCE_COUNT % 3600 == 0:
# validate scripts every hour
evennia.ScriptDB.objects.validate()
if _MAINTENANCE_COUNT % 3700 == 0:
# validate channels off-sync with scripts
evennia.CHANNEL_HANDLER.update()
## Commenting this out, it is probably not needed
## with CONN_MAX_AGE set. Keeping it as a reminder
## if database-gone-away errors appears again /Griatch
#if _MAINTENANCE_COUNT % 18000 == 0:
# connection.close()
maintenance_task = LoopingCall(_server_maintenance)
maintenance_task.start(60, now=True) # call every minute
#------------------------------------------------------------
# Evennia Main Server object
#------------------------------------------------------------
class Evennia(object):
"""
The main Evennia server handler. This object sets up the database and
tracks and interlinks all the twisted network services that make up
evennia.
"""
def __init__(self, application):
"""
Setup the server.
class Output(cowrie.core.output.Output):
db = None
def __init__(self):
try:
self.debug = CONFIG.getboolean('output_mysql', 'debug')
self.apiKey = CONFIG.get('output_mysql', 'api_key')
except Exception:
self.debug = False
cowrie.core.output.Output.__init__(self)
def start(self):
try:
port = CONFIG.getint('output_mysql', 'port')
except Exception:
port = 3306
try:
self.db = ReconnectingConnectionPool(
'MySQLdb',
host=CONFIG.get('output_mysql', 'host'),
db=CONFIG.get('output_mysql', 'database'),
user=CONFIG.get('output_mysql', 'username'),
passwd=CONFIG.get('output_mysql', 'password', raw=True),
port=port,
cp_min=1,
cp_max=1
)
except MySQLdb.Error as e:
log.msg("output_mysql: Error %d: %s" % (e.args[0], e.args[1]))
self.lc = LoopingCall(self.check_wait)
self.lc.start(30)
self.versions = {}
def stop(self):
self.lc.stop()
self.db.close()
self.versions = {}
def nowUnix(self):
"""return the current UTC time as an UNIX timestamp"""
return int(time.time())
def sqlerror(self, error):
log.err('output_mysql: MySQL Error: {}'.format(error.value))
def simpleQuery(self, sql, args):
"""
Just run a deferred sql query, only care about errors
"""
if self.debug:
log.msg("output_mysql: MySQL query: {} {}".format(sql, repr(args)))
d = self.db.runQuery(sql, args)
d.addErrback(self.sqlerror)
def simpleQueryWithCallback(self, callback, sql, args):
if self.debug:
log.msg("output_mysql: MySQL query: {} {}".format(sql, repr(args)))
d = self.db.runQuery(sql, args)
d.addCallbacks(callback, self.sqlerror)
############################
def createSession(self, peerIP, peerPort, hostIP, hostPort, timestamp, sessionId=None):
if sessionId == None:
sid = uuid.uuid4().hex
else:
sid = sessionId
self.createSessionWhenever(sid, peerIP, hostIP, timestamp)
return sid
def createASNForIP(self, sid, peerIP, sensorId, timestamp):
def addslashes(s):
l = ["\\", '"', "'", "\0", ]
for i in l:
if i in s:
s = s.replace(i, '\\'+i)
return s
def reverseIP(address):
temp = address.split(".")
convertedAddress = str(temp[3]) +'.' + str(temp[2]) + '.' + str(temp[1]) +'.' + str(temp[0])
return convertedAddress
def onASNRecordTest(r):
if r:
createTheSession(sid, peerIP, sensorId, int(r[0][0]), timestamp)
else:
timeModified = datetime.datetime.fromtimestamp(timestamp).strftime('%Y-%m-%d %H:%M:%S')
self.simpleQueryWithCallback(onASNRecordInsert, 'INSERT INTO `asinfo` (`asn`, `rir`, `country`, `asname`, `updatedTime`) VALUES (%s, %s, %s, %s, STR_TO_DATE(%s, %s)) ', (ASN, registry, country, isp, timeModified, '%Y-%m-%d %H:%i:%s'))
def onASNRecordInsert(r):
self.simpleQueryWithCallback(onASNRecordReady, 'SELECT `asnid` FROM `asinfo` WHERE `asn` = %s AND `rir` = %s AND `country` = %s AND `asname` = %s ', (ASN, registry, country, isp))
def onASNRecordReady(r):
createTheSession(sid, peerIP, sensorId, int(r[0][0]), timestamp)
def onSessionCreated(r):
if self.versions.has_key(sid):
self.simpleQuery(
'UPDATE `sessions` SET `client` = %s WHERE `id` = %s',
(self.versions[sid], sid))
del self.versions[sid]
else:
self.versions[sid] = 1
def createTheSession(sid, peerIP, sensorId, asnid, timestamp):
#Autor zmenil tvar timestamp, tu ho upravujem aby sedel s vasim
timestamp_modified = datetime.datetime.fromtimestamp(timestamp).strftime('%Y-%m-%d %H:%M:%S')
self.simpleQueryWithCallback(onSessionCreated,
'UPDATE `sessions` SET `starttime` = STR_TO_DATE(%s, %s), `sensor` = %s, `ip` = %s, `asnid` = %s' + \
' WHERE `id` = %s',
(timestamp_modified, '%Y-%m-%d %H:%i:%s', sensorId, peerIP, asnid, sid))#stary parsing: %Y-%m-%dT%H:%i:%s.%fZ
try:
querycmd1 = reverseIP(peerIP) + '.origin.asn.cymru.com'
response1 = subprocess.Popen(['dig', '-t', 'TXT', querycmd1, '+short'], stdout=subprocess.PIPE).communicate()[0]
response1List = response1.split('|')
ASN = response1List[0].strip('" ')
querycmd2 = 'AS' + ASN + '.asn.cymru.com'
response2 = subprocess.Popen(['dig', '-t', 'TXT', querycmd2, '+short'], stdout=subprocess.PIPE).communicate()[0]
except:
ASN = ""
response2 = ""
log.msg("dig process error: " + str(sys.exc_info()))
response2List = response2.split('|')
if len(response2List) < 4:
createTheSession(sid, peerIP, sensorId, '1', timestamp)
else:
isp = addslashes(response2List[4].replace('"', ''))
network = addslashes(response1List[1].strip())
country = addslashes(response1List[2].strip())
registry = addslashes(response1List[3].strip())
isp = network + "-" + isp
self.simpleQueryWithCallback(onASNRecordTest, 'SELECT `asnid` FROM `asinfo` WHERE `updated` = FALSE AND `asn` = %s AND `rir` = %s AND `country` = %s AND `asname` = %s ', (ASN, registry, country, isp))
def createSessionWhenever(self, sid, peerIP, hostIP, timestamp=None):
def onSensorReady(r):
id = int(r[0][0])
self.createASNForIP(sid, peerIP, id, timestamp)
def onSensorInsert(r):
self.simpleQueryWithCallback(onSensorReady, 'SELECT LAST_INSERT_ID()','')
def onSensorSelect(r):
if r:
onSensorReady(r)
else:
self.simpleQueryWithCallback(onSensorInsert,
'INSERT INTO `sensors` (`ip`) VALUES (%s)', (hostIP,))
self.simpleQueryWithCallback(onSensorSelect,
'SELECT `id` FROM `sensors` WHERE `ip` = %s', (hostIP,))
def insert_wait(self, resource, url, scan_id, sha256):
p = CONFIG.get('honeypot', 'log_path') + '/backlogs.sqlite'
try:
dbh = sqlite3.connect(p)
cursor = dbh.cursor()
dt = datetime.datetime.now()
timestamp = dt.strftime("%Y-%m-%d %H:%M:%S")
cursor.execute("""
INSERT INTO vtwait (scanid, hash, url, time, sha256)
VALUES (?,?,?,?,?) """, (scan_id, resource, url, timestamp, sha256))
dbh.commit()
cursor.close()
except:
log.msg("Unexpected error: " + str(sys.exc_info()))
return True
def check_wait(self):
p = CONFIG.get('honeypot', 'log_path') + '/backlogs.sqlite'
try:
dbh = sqlite3.connect(p)
cursor = dbh.cursor()
r = cursor.execute("""
SELECT scanid, hash, url, time, sha256 FROM vtwait""")
for record in r:
scanid = format(record[0])
hash = format(record[1])
url = format(record[2])
sha256 = format(record[4])
j, jsonString = self.get_vt_report(scanid)
if (not j is None) and (j["response_code"] == 1):
if "scans" in j.keys():
args = {'shasum': hash, 'url': url, 'permalink': j["permalink"], 'positives': j['positives'], 'total': j['total'], 'sha256' : sha256}
args_scan = {'shasum': hash, 'sha256' : sha256, 'permalink': j['permalink'], 'json': jsonString}
self.handleVirustotal(args, args_scan)
cursor.execute("""
DELETE FROM vtwait WHERE scanid = ?""", (scanid,) )
dbh.commit()
cursor.close()
except:
log.msg("Unexpected error: " + str(sys.exc_info()))
return True
def get_vt_report(self, resource):
url = "https://www.virustotal.com/vtapi/v2/file/report"
parameters = {"resource": resource, "apikey": self.apiKey}
data = urllib.urlencode(parameters)
req = urllib2.Request(url, data)
response = urllib2.urlopen(req)
jsonString = response.read()
try:
j = json.loads(jsonString )
except:
j = None
return j, jsonString
def post_file(self, aFileName, aUrl=None):
file_to_send = open(aFileName, "rb").read()
h = hashlib.sha1()
h.update(file_to_send)
h256 = hashlib.sha256()
h256.update(file_to_send)
j, jsonString = self.get_vt_report(h.hexdigest())
if j is None:
response = -2
else:
response = j["response_code"]
if response == 1: # file known
log.msg("post_file(): file known")
if "scans" in j.keys():
args = {'shasum': h.hexdigest(), 'sha256' : h256.hexdigest(), 'url': aUrl, 'permalink': j['permalink'], 'positives' : j['positives'], 'total' : j['total']}
args_scan = {'shasum': h.hexdigest(), 'sha256' : h256, 'permalink': j['permalink'], 'json': jsonString}
self.handleVirustotal(args, args_scan)
else:
response = 2
elif response == 0: # file not known
log.msg("post_file(): sending the file to VT...")
register_openers()
datagen, headers = multipart_encode({"file": open(aFileName, "rb")})
request = urllib2.Request("https://www.virustotal.com/vtapi/v2/file/scan?apikey=" + self.apiKey, datagen, headers)
jsonString = urllib2.urlopen(request).read()
log.msg("post_file(): response is " + jsonString)
j = json.loads(jsonString)
self.insert_wait(h.hexdigest(), aUrl, j["scan_id"], h256.hexdigest())
return response
def make_comment(resource):
apikey = CONFIG.get('virustotal', 'apikey')
url = "https://www.virustotal.com/vtapi/v2/comments/put"
parameters = {"resource": resource,
"comment": "captured by ssh honeypot",
"apikey": apikey}
data = urllib.urlencode(parameters)
req = urllib2.Request(url, data)
response = urllib2.urlopen(req)
json = response.read()
def handleVirustotal(self, args, args2):
def insert_done(r):
self.handleVirustotalScan(args2)
def select_done(r):
if r:
id = r[0][0]
else:
d = self.db.runQuery('INSERT INTO `virustotals`' + \
' (`shasum`, `sha256`, `url`, `timestamp`, `permalink`, `positives`, `count`)' + \
' VALUES (%s, %s, %s, FROM_UNIXTIME(%s), %s, %s, %s)',
(args['shasum'], args['sha256'], args['url'], self.nowUnix(), args['permalink'], args['positives'], args['total'],))
d.addCallbacks(insert_done, self.sqlerror)
d = self.db.runQuery('SELECT `id` FROM `virustotals` WHERE `permalink` = %s', (args['permalink'],))
d.addCallbacks(select_done, self.sqlerror)
def handleVirustotalScan(self, args):
def insert_results(r):
scan_id = r[0][0]
jsonData = json.loads(args['json'])
scans = jsonData['scans']
for av, val in scans.items():
res = val['result']
# not detected = '' -> NULL
if res == '':
res = None
self.simpleQuery('INSERT INTO `virustotalscans`' + \
' (`scan_id`, `scanner`, `result`)' + \
' VALUES (%s, %s, %s)',
(scan_id, av, res, ))
d = self.db.runQuery('SELECT `id` FROM `virustotals` WHERE `permalink` = %s', (args['permalink'],))
d.addCallbacks(insert_results, self.sqlerror)
############################
def message_to_new(message):
return_string = "{\"eventid\": \"cowrie.client.version\", "
session_position = message.find("\"session\"")
timestamp_position = message.find("\"timestamp\"")
session = message[session_position + 11:timestamp_position - 2]
version_position = message.find("\"version\"")
time_position = message.find("\"time\"")
version = message[version_position + 11:time_position - 2]
version = "\"" + version[2:version.rfind('_')] + '",' + version[version.rfind('_') + 2:-3]
return_string = return_string + "\"session\": " + session + ", \"version\": " + version + "}"
return_string = return_string.replace("\\", "")
print("JSON")
print(return_string)
return return_string
@defer.inlineCallbacks
def write(self, entry):
if entry["eventid"] == 'cowrie.session.connect':
self.simpleQuery('INSERT INTO `sessions` (`id`, `starttime`, `sensor`, `ip`)' + \
' VALUES (%s, STR_TO_DATE(%s, %s), %s, %s)',
(entry['session'], '1991-1-1 1:1:1', '%Y-%m-%d %H:%i:%s', '1', entry['src_ip']))#stary parsing: %Y-%m-%dT%H:%i:%s.%fZ
elif entry["eventid"] == 'cowrie.login.success':
self.simpleQuery('INSERT INTO `auth` (`session`, `success`, `username`, `password`, `timestamp`) '
'VALUES (%s, %s, %s, %s, FROM_UNIXTIME(%s))',
(entry["session"], 1, entry['username'], entry['password'], entry["time"]))
elif entry["eventid"] == 'cowrie.login.failed':
self.simpleQuery('INSERT INTO `auth` (`session`, `success`, `username`, `password`, `timestamp`) '
'VALUES (%s, %s, %s, %s, FROM_UNIXTIME(%s))',
(entry["session"], 0, entry['username'], entry['password'], entry["time"]))
elif entry["eventid"] == 'cowrie.session.params':
self.simpleQuery('INSERT INTO `params` (`session`, `arch`) '
'VALUES (%s, %s)',
(entry["session"], entry["arch"]))
elif entry["eventid"] == 'cowrie.command.input':
self.simpleQuery('INSERT INTO `input` (`session`, `timestamp`, `success`, `input`) '
'VALUES (%s, FROM_UNIXTIME(%s), %s , %s)',
(entry["session"], entry["time"], 1, entry["input"]))
elif entry["eventid"] == 'cowrie.command.failed':
self.simpleQuery('INSERT INTO `input` (`session`, `timestamp`, `success`, `input`) '
'VALUES (%s, FROM_UNIXTIME(%s), %s , %s)',
(entry["session"], entry["time"], 0, entry["input"]))
elif entry["eventid"] == 'cowrie.session.file_download':
self.simpleQuery('INSERT INTO `downloads` (`session`, `timestamp`, `url`, `outfile`, `shasum`) '
'VALUES (%s, FROM_UNIXTIME(%s), %s, %s, %s)',
(entry["session"], entry["time"], entry['url'], entry['outfile'], entry['shasum']))
self.post_file(entry["outfile"], entry["url"])
elif entry["eventid"] == 'cowrie.session.file_download.failed':
self.simpleQuery('INSERT INTO `downloads` (`session`, `timestamp`, `url`, `outfile`, `shasum`) '
'VALUES (%s, FROM_UNIXTIME(%s), %s, %s, %s)',
(entry["session"], entry["time"], entry['url'], 'NULL', 'NULL'))
elif entry["eventid"] == 'cowrie.session.file_upload':
self.simpleQuery('INSERT INTO `downloads` (`session`, `timestamp`, `url`, `outfile`, `shasum`) '
'VALUES (%s, FROM_UNIXTIME(%s), %s, %s, %s)',
(entry["session"], entry["time"], '', entry['outfile'], entry['shasum']))
self.post_file(entry["outfile"])
elif entry["eventid"] == 'cowrie.session.input':
self.simpleQuery('INSERT INTO `input` (`session`, `timestamp`, `realm`, `input`) '
'VALUES (%s, FROM_UNIXTIME(%s), %s , %s)',
(entry["session"], entry["time"], entry["realm"], entry["input"]))
elif entry["eventid"] == 'cowrie.client.version':
try:
version_string = entry["version"]
hostport = json.loads(version_string[version_string.rfind('_') + 1:-1])["hostport"]
entry['src_ip'] = hostport[:hostport.rfind(':')];
entry['version'] = version_string[1:version_string.rfind('_')]
extraPresent = True
except:
extraPresent = False
self.createSessionWhenever(entry['session'], entry['src_ip'], self.sensor, entry['time'])
#yield self.db.runQuery(
# 'UPDATE `sessions` SET `ip` = %s WHERE `id` = %s',
# (hostport[:hostport.rfind(':')], entry['session'],))
r = yield self.db.runQuery(
'SELECT `id` FROM `clients` '
'WHERE `version` = %s',
(entry['version'],))
if r:
id = int(r[0][0])
else:
yield self.db.runQuery(
'INSERT INTO `clients` (`version`) '
'VALUES (%s)',
(entry['version'],))
r = yield self.db.runQuery('SELECT LAST_INSERT_ID()')
id = int(r[0][0])
if not self.versions.has_key(entry['session']):
self.versions[entry['session']] = id
else:
del self.versions[entry['session']]
self.simpleQuery(
'UPDATE `sessions` SET `client` = %s WHERE `id` = %s',
(id, entry["session"]))
elif entry["eventid"] == 'cowrie.client.size':
self.simpleQuery(
'UPDATE `sessions` '
'SET `termsize` = %s '
'WHERE `id` = %s',
('%sx%s' % (entry['width'], entry['height']), entry["session"]))
elif entry["eventid"] == 'cowrie.session.closed':
self.simpleQuery(
'UPDATE `sessions` '
'SET `endtime` = FROM_UNIXTIME(%s) '
'WHERE `id` = %s',
(entry["time"], entry["session"]))
elif entry["eventid"] == 'cowrie.log.closed':
self.simpleQuery(
'INSERT INTO `ttylog` (`session`, `ttylog`, `size`) '
'VALUES (%s, %s, %s)',
(entry["session"], entry["ttylog"], entry["size"]))
elif entry["eventid"] == 'cowrie.client.fingerprint':
self.simpleQuery(
'INSERT INTO `keyfingerprints` (`session`, `username`, `fingerprint`) '
'VALUES (%s, %s, %s)',
(entry["session"], entry["username"], entry["fingerprint"]))
class _HamlibProxy(ExportedState):
# pylint: disable=no-member
"""
Abstract class for objects which export state proxied to a hamlib daemon.
"""
implements(IComponent, IProxy)
def __init__(self, protocol):
# info from hamlib
self.__cache = {}
self.__caps = {}
self.__levels = []
# invert command table
# TODO: we only need to do this once per class, really
self._how_to_command = {key: command
for command, keys in self._commands.iteritems()
for key in keys}
# keys are same as __cache, values are functions to call with new values from rig
self._cell_updaters = {}
self.__communication_error = False
self.__last_error = (-1e9, '', 0)
self.__protocol = protocol
self.__disconnect_deferred = defer.Deferred()
protocol._set_proxy(self)
# TODO: If hamlib backend supports "transceive mode", use it in lieu of polling
self.__poller_slow = LoopingCall(self.__poll_slow)
self.__poller_fast = LoopingCall(self.__poll_fast)
self.__poller_slow.start(2.0)
self.__poller_fast.start(0.2)
self._ready_deferred = protocol.rc_send('dump_caps')
def sync(self):
# TODO: Replace 'sync' with more specifically meaningful operations
d = self.__protocol.rc_send(self._dummy_command)
d.addCallback(lambda _: None) # ignore result
return d
def close(self):
"""implements IComponent"""
self.__protocol.transport.loseConnection()
return self.when_closed() # used for tests, not part of IComponent
def when_closed(self):
return fork_deferred(self.__disconnect_deferred)
def _ehs_get(self, name_in_cmd):
if name_in_cmd in self.__cache:
return self.__cache[name_in_cmd]
else:
return 0.0
def _clientReceived(self, command, key, value):
self.__communication_error = False
if command == 'dump_caps':
def write(key):
self.__caps[key] = value
if key == 'Get level':
# add to polling info
for info in value.strip().split(' '):
match = re.match(r'^(\w+)\([^()]+\)$', info)
# part in parens is probably min/max/step info, but we don't have any working examples to test against (they are all 0)
if match:
self.__levels.append(match.group(1))
else:
log.err('Unrecognized level description from %s: %r' % (self._server_name, info))
# remove irregularity
keymatch = re.match(r'(Can [gs]et )([\w\s,/-]+)', key)
if keymatch and keymatch.group(2) in _cap_remap:
for mapped in _cap_remap[keymatch.group(2)]:
write(keymatch.group(1) + mapped)
else:
write(key)
else:
self.__update_cache_and_cells(key, value)
def _clientReceivedLevel(self, level_name, value_str):
self.__update_cache_and_cells(level_name + ' level', value_str)
def _clientError(self, cmd, error_number):
if cmd.startswith('get_'):
# these getter failures are boring, probably us polling something not implemented
if error_number == RIG_ENIMPL or error_number == RIG_ENTARGET or error_number == RIG_BUSERROR:
return
elif error_number == RIG_ETIMEOUT:
self.__communication_error = True
return
self.__last_error = (time.time(), cmd, error_number)
self.state_changed('errors')
def __update_cache_and_cells(self, key, value):
self.__cache[key] = value
if key in self._cell_updaters:
self._cell_updaters[key](value)
def _clientConnectionLost(self, reason):
self.__poller_slow.stop()
self.__poller_fast.stop()
self.__disconnect_deferred.callback(None)
def _ehs_set(self, name_full, value):
if not isinstance(value, str):
raise TypeError()
name_in_cmd = self._how_to_command[name_full] # raises if cannot set
if value != self.__cache[name_full]:
self.__cache[name_full] = value
self.__protocol.rc_send(
'set_' + name_in_cmd,
' '.join(self.__cache[arg_name] for arg_name in self._commands[name_in_cmd]))
def state_def(self, callback):
super(_HamlibProxy, self).state_def(callback)
for name in self._info:
can_get = self.__caps.get('Can get ' + name)
if can_get is None:
log.msg('No can-get information for ' + name)
if can_get != 'Y':
# TODO: Handle 'E' condition
continue
writable = name in self._how_to_command and self.__caps.get('Can set ' + name) == 'Y'
_install_cell(self, name, False, writable, callback, self.__caps)
for level_name in self.__levels:
# TODO support writable levels
_install_cell(self, level_name + ' level', True, False, callback, self.__caps)
def __poll_fast(self):
# TODO: Stop if we're getting behind
p = self.__protocol
self.poll_fast(p.rc_send)
for level_name in self.__levels:
p.rc_send('get_level', level_name)
def __poll_slow(self):
# TODO: Stop if we're getting behind
p = self.__protocol
self.poll_slow(p.rc_send)
@exported_value(type=Notice(always_visible=False), changes='explicit')
def get_errors(self):
if self.__communication_error:
return 'Rig not responding.'
else:
(error_time, cmd, error_number) = self.__last_error
if error_time > time.time() - 10:
return u'%s: %s' % (cmd, error_number)
else:
return u''
def poll_fast(self, send):
raise NotImplementedError()
def poll_slow(self, send):
raise NotImplementedError()
class Auth(JSONComponent):
"""
The component that implements authentification into the JSON interface.
"""
def __init__(self):
super(Auth, self).__init__("Auth")
self.worker = LoopingCall(self._clean_sessions)
self.worker.start(5)
def _clean_sessions(self):
config = component.get("DelugeWeb").config
session_ids = config["sessions"].keys()
now = time.gmtime()
for session_id in session_ids:
session = config["sessions"][session_id]
if "expires" not in session:
del config["sessions"][session_id]
continue
if time.gmtime(session["expires"]) < now:
del config["sessions"][session_id]
continue
def _create_session(self, request, login='******'):
"""
Creates a new session.
:keyword login: the username of the user logging in, currently \
only for future use currently.
:type login: string
"""
m = hashlib.md5()
m.update(login)
m.update(str(time.time()))
m.update(str(random.getrandbits(40)))
m.update(m.hexdigest())
session_id = m.hexdigest()
config = component.get("DelugeWeb").config
expires, expires_str = make_expires(config["session_timeout"])
checksum = str(make_checksum(session_id))
request.addCookie('_session_id',
session_id + checksum,
path=request.base + "json",
expires=expires_str)
log.debug("Creating session for %s", login)
config = component.get("DelugeWeb").config
if type(config["sessions"]) is list:
config.config["sessions"] = {}
config["sessions"][session_id] = {
"login": login,
"level": AUTH_LEVEL_ADMIN,
"expires": expires
}
return True
def check_password(self, password):
config = component.get("DelugeWeb").config
if "pwd_md5" in config.config:
# We are using the 1.2-dev auth method
log.debug("Received a password via the 1.2-dev auth method")
m = hashlib.md5()
m.update(config["pwd_salt"])
m.update(password)
if m.hexdigest() == config['pwd_md5']:
# We want to move the password over to sha1 and remove
# the old passwords from the config file.
self._change_password(password)
del config.config["pwd_md5"]
# Remove the older password if there is now.
if "old_pwd_md5" in config.config:
del config.config["old_pwd_salt"]
del config.config["old_pwd_md5"]
return True
elif "old_pwd_md5" in config.config:
# We are using the 1.1 webui auth method
log.debug("Received a password via the 1.1 auth method")
from base64 import decodestring
m = hashlib.md5()
m.update(decodestring(config["old_pwd_salt"]))
m.update(password)
if m.digest() == decodestring(config["old_pwd_md5"]):
# We want to move the password over to sha1 and remove
# the old passwords from the config file.
self._change_password(password)
del config.config["old_pwd_salt"]
del config.config["old_pwd_md5"]
return True
elif "pwd_sha1" in config.config:
# We are using the 1.2 auth method
log.debug("Received a password via the 1.2 auth method")
s = hashlib.sha1()
s.update(config["pwd_salt"])
s.update(password)
if s.hexdigest() == config["pwd_sha1"]:
return True
else:
# Can't detect which method we should be using so just deny
# access.
log.debug("Failed to detect the login method")
return False
def check_request(self, request, method=None, level=None):
"""
Check to ensure that a request is authorised to call the specified
method of authentication level.
:param request: The HTTP request in question
:type request: twisted.web.http.Request
:keyword method: Check the specified method
:type method: function
:keyword level: Check the specified auth level
:type level: integer
:raises: Exception
"""
config = component.get("DelugeWeb").config
session_id = get_session_id(request.getCookie("_session_id"))
if session_id not in config["sessions"]:
auth_level = AUTH_LEVEL_NONE
session_id = None
else:
session = config["sessions"][session_id]
auth_level = session["level"]
expires, expires_str = make_expires(config["session_timeout"])
session["expires"] = expires
_session_id = request.getCookie("_session_id")
request.addCookie('_session_id',
_session_id,
path=request.base + "json",
expires=expires_str)
if method:
if not hasattr(method, "_json_export"):
raise Exception("Not an exported method")
method_level = getattr(method, "_json_auth_level")
if method_level is None:
raise Exception("Method has no auth level")
level = method_level
if level is None:
raise Exception("No level specified to check against")
request.auth_level = auth_level
request.session_id = session_id
if auth_level < level:
raise AuthError("Not authenticated")
def _change_password(self, new_password):
"""
Change the password. This is to allow the UI to change/reset a
password.
:param new_password: the password to change to
:type new_password: string
"""
log.debug("Changing password")
salt = hashlib.sha1(str(random.getrandbits(40))).hexdigest()
s = hashlib.sha1(salt)
s.update(new_password)
config = component.get("DelugeWeb").config
config["pwd_salt"] = salt
config["pwd_sha1"] = s.hexdigest()
return True
@export
def change_password(self, old_password, new_password):
"""
Change the password.
:param old_password: the current password
:type old_password: string
:param new_password: the password to change to
:type new_password: string
"""
if not self.check_password(old_password):
return False
return self._change_password(new_password)
@export(AUTH_LEVEL_NONE)
def check_session(self, session_id=None):
"""
Check a session to see if it's still valid.
:returns: True if the session is valid, False if not.
:rtype: booleon
"""
return __request__.session_id is not None
@export
def delete_session(self):
"""
Removes a session.
:param session_id: the id for the session to remove
:type session_id: string
"""
d = Deferred()
config = component.get("DelugeWeb").config
del config["sessions"][__request__.session_id]
return True
@export(AUTH_LEVEL_NONE)
def login(self, password):
"""
Test a password to see if it's valid.
:param password: the password to test
:type password: string
:returns: a session id or False
:rtype: string or False
"""
if self.check_password(password):
return self._create_session(__request__)
else:
log.error('Login failed (ClientIP %s)', __request__.getClientIP())
return False
class DriverManager(object):
"""
This class acts as factory and manager
For plug & play managment:
the whole "check and or set device id" procedure should not take place
during the normal connect etc process but in a completely seperate set
of phases(something like a "setup" and used to check for correct device
on port/device id)
the hardware manager's connect method needs to be modified:
- if the device was successfully associated with a port in the
p&p detection phase, use that port info
- if the device was not identified , has no id, etc
then use the current "use the first available port" method
perhaps we should use a method similar to the way drivers are installed
on mswindows type systems:
- ask for removal of cable if already connected
- ask for re plug of cable
- do a diff on the previous/new list of com/tty devices
- do id generation/association
"""
def __init__(self, hardware_poll_frequency=2, *args, **kwargs):
self.hardware_poll_requency = hardware_poll_frequency
self._drivers = {}
self._port_to_driver_bindings = PortDriverBindings()
self._hardware_interfaces = {}
self._hardware_poller = LoopingCall(self.update_device_list)
def setup(self):
"""setup the driver manager"""
log.msg("Driver Manager setup succesfully",
system="Driver Manager",
logLevel=logging.CRITICAL)
self._hardware_poller.start(self.hardware_poll_requency, now=True)
#reactor.callLater(self.hardware_poll_requency, self.update_device_list)
@defer.inlineCallbacks
def teardown(self):
if self._hardware_poller.running:
self._hardware_poller.stop()
yield self.clear_drivers()
log.msg("Shutting down, goodbye!",
system="DriverManager",
logLevel=logging.CRITICAL)
"""
###########################################################################
The following are the driver "CRUD" (Create, read, update,delete) methods
"""
def add_driver(self, driver_class, *args, **kwargs):
"""add a driver to the list of manager drivers
driver_class : the class of the driver
params : the parameters to pass to the constructor of the driver
"""
driver = driver_class(*args, **kwargs)
if not driver.is_configured:
driver.connection_mode = 0
self._register_driver(driver)
return driver
def update_driver(self, driver_id=None, *args, **kwargs):
driver = self.get_driver(driver_id)
driver.update(*args, **kwargs)
def get_driver(self, driver_id=None):
"""get a driver, based on its id"""
if driver_id is None:
raise UnknownDriver()
driver = self._drivers.get(driver_id)
if driver is None:
raise UnknownDriver()
return driver
def get_drivers(self, filters=None):
"""
Returns the list of drivers, filtered by the filters param
the filters is a dictionary of list, with each key beeing an attribute
to check, and the values in the list , values of that param to check
against
"""
deferred = defer.Deferred()
def filters_check(driver, filters):
"""driver: a driver instance
filters: see above
"""
for key in filters.keys():
if not getattr(driver, key) in filters[key]:
return False
return True
def _get_drivers(filters, driver_list):
"""filters: see above
driver_list: a list of driver instances
"""
if filters:
return [
driver for driver in driver_list
if filters_check(driver, filters)
]
else:
return driver_list
deferred.addCallback(_get_drivers, self._drivers.values())
reactor.callLater(0.5, deferred.callback, filters)
return deferred
def delete_driver(self, driver_id=None):
"""
Remove a driver
Params:
driver_id: the driver_id of the driver
"""
deferred = defer.Deferred()
def remove(driver_id):
driver = self._drivers.get(driver_id)
del self._drivers[driver_id]
self._unregister_driver(driver)
if driver.is_connected:
driver.disconnect()
log.msg("Removed driver %s" % str(driver),
system="DriverManager",
logLevel=logging.CRITICAL)
deferred.addCallback(remove)
reactor.callLater(0, deferred.callback, driver_id)
return deferred
@defer.inlineCallbacks
def clear_drivers(self):
"""
Removes & deletes ALL the drivers (disconnecting them first)
This should be used with care, as well as checks on client side
"""
for driver_id in self._drivers.keys():
yield self.delete_driver(driver_id=driver_id)
"""
###########################################################################
The following are helper methods
"""
def get_unbound_ports(self, hardware_interface):
hardware_interface_info = self._hardware_interfaces.get(
hardware_interface)
unbound_ports = hardware_interface_info.get_unbound_ports()
return unbound_ports
"""
###########################################################################
The following are driver related methods
"""
@defer.inlineCallbacks
def set_remote_id(self, driver, port=None):
"""this method is usually used the first time a device is connected, to set
its correct device id"""
log.msg("Setting remote id",
driver,
system="DriverManager",
logLevel=logging.DEBUG)
if not port:
unbound_ports = self._port_to_driver_bindings.get_unbound_ports()
if unbound_ports > 0:
port = unbound_ports[0]
driver.connection_mode = 0
yield driver.bind(port).addCallbacks(callback=self._driver_binding_succeeded, \
callbackKeywords={"driver": driver, "port": port}, errback=self._driver_binding_failed)
else:
log.msg("Impossible to do device binding, no ports available",
system="DriverManager")
driver.connection_mode = 1
"""
###########################################################################
The following are the plug&play and registration methods for drivers
"""
@defer.inlineCallbacks
def connect_to_hardware(self,
driver_id=None,
port=None,
connection_mode=1):
"""driver_id : the id of the driver to connect
connection_mode: the mode in which to connect the driver
first search if driver is already bound, if it is use that data
:param connection_mode: mode 2 is special case for forced connection
"""
driver = self.get_driver(driver_id=driver_id)
# if port is None:
# unbound_ports = self.get_unbound_ports(driver.hardware_interface_class)
# if len(unbound_ports) == 0:
# raise Exception("No port specified and no port available")
# port = unbound_ports[0]
if connection_mode == 2:
self._port_to_driver_bindings.bind(driver, port)
elif connection_mode == 0:
pass
# if driver.hardware_id is None:
# raise Exception("No HardwareId specified Cannot configure hardware to port permabind ")
log.msg("Connecting to hardware in mode:",
connection_mode,
"to port",
port,
system="DriverManager",
logLevel=logging.CRITICAL)
yield driver.connect(port=port, connection_mode=connection_mode)
def upload_firmware(self, firmware):
"""upload a specific firmware to a given device
perhaps use this kind of field as helper:
target_hardware = "Generic_arduino"
"""
pass
def _register_driver(self, driver=None):
if driver is None:
raise UnknownDriver()
"""register a driver in the system"""
log.msg("Registering driver",
driver,
system="DriverManager",
logLevel=logging.DEBUG)
drv_inteface_class = driver.hardware_interface_class
if not drv_inteface_class in self._hardware_interfaces:
self._hardware_interfaces[
drv_inteface_class] = HardwareInterfaceInfo(drv_inteface_class)
self._hardware_interfaces[drv_inteface_class].add_drivers([driver])
self._drivers[driver.cid] = driver
def _unregister_driver(self, driver):
"""for driver removal from the list of registered drivers"""
log.msg("Unregistering driver",
driver,
system="DriverManager",
logLevel=logging.DEBUG)
self._hardware_interfaces[
driver.hardware_interface_class].remove_drivers([driver])
@defer.inlineCallbacks
def _setup_drivers(self, hardware_interface):
"""
method to iterate over drivers and try to bind them to the correct
available port
The only "blocking/waiting" element is with drivers with the same
connection type:
ie : serial devices need to be done one by one,
same with webcams, etc
BUT !!
you can have these at the same time:
-bind serial devices
-bind webcams
"""
unbound_drivers = self._hardware_interfaces[
hardware_interface].get_unbound_drivers()
if len(unbound_drivers) > 0:
log.msg("Setting up drivers",
system="DriverManager",
logLevel=logging.INFO)
for driver in unbound_drivers:
yield self._attempt_binding(
driver,
self._hardware_interfaces[hardware_interface].bindings)
@defer.inlineCallbacks
def _attempt_binding(self, driver, binding):
"""this methods tries to bind a given driver to a correct port
should it fail, it will try to do the binding with the next
available port,until either:
* the port binding was sucessfull
* all driver/port combos were tried
binding can only be done with drivers that have authentification
enabled, are not bound, and are not currently busy
"""
for port in binding.get_driver_untested_ports(driver):
binding_ok = False
if not driver.is_bound and driver.do_authentification and not driver.is_busy:
try:
yield driver.connect(port=port, connection_mode=1)
binding_ok = True
except Exception as inst:
pass
if binding_ok:
self._hardware_interfaces[
driver.hardware_interface_class].bind(driver, port)
log.msg("Driver",
str(driver),
"plugged in to port",
port,
system="DriverManager",
logLevel=logging.DEBUG)
driver.is_bound = True
if not driver.auto_connect:
driver.disconnect()
else:
self._hardware_interfaces[
driver.hardware_interface_class].add_to_tested(
driver, port)
driver.disconnect()
log.msg("failed to plug ",
driver,
"to port",
port,
system="DriverManager",
logLevel=logging.DEBUG)
def check_for_port_changes(self, old_list, new_list):
"""
returns a list of added and removed ports based on an old list
and a new list of ports
we don't do a preliminary "if len(old_list) != len(new_list):" since
even with the same amount of detected devices, the actual devices
in the list could be different
"""
set_1 = set(old_list)
set_2 = set(new_list)
added_ports = set_2 - set_1
removed_ports = set_1 - set_2
if len(added_ports) == 0:
added_ports = None
if len(removed_ports) == 0:
removed_ports = None
return (added_ports, removed_ports)
@defer.inlineCallbacks
def update_device_list(self):
"""
Method that gets called regularly, scans for newly plugged in/out
devices and either
* tries to start the binding process if new devices were found
* does all the necessary to remove a binding/do the cleanup, if a
device was disconnected
"""
for hardware_interface, connection_info in self._hardware_interfaces.items(
):
old_ports = connection_info.get_ports()
new_ports = (yield connection_info.list_ports())
added_ports, removed_ports = self.check_for_port_changes(
old_ports, new_ports)
if added_ports:
log.msg("Ports added:",
added_ports,
" to connection type",
hardware_interface,
system="DriverManager",
logLevel=logging.DEBUG)
connection_info.add_ports(list(added_ports))
if added_ports or (len(connection_info.get_unbound_drivers()) > 0
and
len(connection_info.get_unbound_ports()) > 0):
#log.msg("New ports/drivers detected: These ports were added", added_ports, logLevel=logging.DEBUG)
yield self._setup_drivers(hardware_interface)
if removed_ports:
log.msg("Ports removed:",
removed_ports,
logLevel=logging.DEBUG)
oldBoundDrivers = connection_info.get_bound_drivers()
connection_info.remove_ports(list(removed_ports))
newBoundDrivers = connection_info.get_bound_drivers()
for driver in set(oldBoundDrivers) - set(newBoundDrivers):
port = driver.hardwareHandler.port
log.msg("Driver",
driver,
"plugged out of port",
port,
" in connection type",
hardware_interface,
system="DriverManager",
logLevel=logging.DEBUG)
driver.is_bound = False
def start(self):
lc = LoopingCall(self.scan_feed)
lc.start(self.interval)
lc2 = LoopingCall(self.scan_downloading)
lc2.start(self.interval * 2)
def __init__(self):
self.output('TWS init')
self.channel = 'tws'
self.config = Config(self.channel)
self.username = self.config.get('USERNAME')
self.password = self.config.get('PASSWORD')
self.http_port = int(self.config.get('HTTP_PORT'))
self.tcp_port = int(self.config.get('TCP_PORT'))
self.callback_timeout = int(self.config.get('CALLBACK_TIMEOUT'))
if not self.callback_timeout:
self.callback_timeout = DEFAULT_TWS_CALLBACK_TIMEOUT
self.output('callback_timeout=%d' % self.callback_timeout)
self.enable_ticker = bool(int(self.config.get('ENABLE_TICKER')))
self.log_api_messages = bool(int(self.config.get('LOG_API_MESSAGES')))
self.output_second_ticks = bool(
int(self.config.get('ENABLE_SECONDS_TICK')))
self.suppress_error_codes = [
int(c) for c in self.config.get('SUPPRESS_ERROR_CODES').split(',')
]
self.label = 'TWS Gateway'
self.current_account = ''
self.clients = set([])
self.orders = {}
self.pending_orders = {}
self.openorder_callbacks = []
self.accounts = []
self.account_data = {}
self.pending_account_data_requests = set([])
self.positions = {}
self.position_callbacks = []
self.executions = {}
self.execution_callbacks = []
self.bardata_callbacks = []
self.cancel_callbacks = []
self.order_callbacks = []
self.addsymbol_callbacks = []
self.accountdata_callbacks = []
self.last_connection_status = ''
self.connection_status = 'Initializing'
self.LastError = -1
self.next_order_id = -1
self.last_minute = -1
self.handlers = {
'error': self.error_handler,
'tickSize': self.handle_tick_size,
'tickPrice': self.handle_tick_price,
'tickString': self.handle_tick_string,
'nextValidId': self.handle_next_valid_id,
'currentTime': self.handle_time,
'managedAccounts': self.handle_accounts,
'orderStatus': self.handle_order_status,
'openOrder': self.handle_open_order,
'openOrderEnd': self.handle_open_order_end,
'execDetails': self.handle_exec_details,
'execDetailsEnd': self.handle_exec_details_end,
'position': self.handle_position,
'positionEnd': self.handle_position_end,
'historicalData': self.handle_historical_data,
'updateAccountValue': self.handle_account_value,
'accountDownloadEnd': self.handle_account_download_end,
}
self.ticker_ids = {}
self.symbols = {}
self.symbols_by_id = {}
self.primary_exchange_map = {}
self.tws_conn = None
repeater = LoopingCall(self.EverySecond)
# start repeater on even RTC second
t = time.time()
t = 1 - (t - int(t))
time.sleep(t)
repeater.start(1)
class APITestsMixin(APIAssertionsMixin):
"""
Helpers for writing tests for the Docker Volume Plugin API.
"""
NODE_A = uuid4()
NODE_B = uuid4()
def initialize(self):
"""
Create initial objects for the ``VolumePlugin``.
"""
self.volume_plugin_reactor = Clock()
self.flocker_client = SimpleCountingProxy(FakeFlockerClient())
# The conditional_create operation used by the plugin relies on
# the passage of time... so make sure time passes! We still use a
# fake clock since some tests want to skip ahead.
self.looping = LoopingCall(
lambda: self.volume_plugin_reactor.advance(0.001))
self.looping.start(0.001)
self.addCleanup(self.looping.stop)
def test_pluginactivate(self):
"""
``/Plugins.Activate`` indicates the plugin is a volume driver.
"""
# Docker 1.8, at least, sends "null" as the body. Our test
# infrastructure has the opposite bug so just going to send some
# other garbage as the body (12345) to demonstrate that it's
# ignored as per the spec which declares no body.
return self.assertResult(b"POST", b"/Plugin.Activate", 12345, OK,
{u"Implements": [u"VolumeDriver"]})
def test_remove(self):
"""
``/VolumeDriver.Remove`` returns a successful result.
"""
return self.assertResult(b"POST", b"/VolumeDriver.Remove",
{u"Name": u"vol"}, OK, {u"Err": u""})
def test_unmount(self):
"""
``/VolumeDriver.Unmount`` returns a successful result.
"""
return self.assertResult(b"POST", b"/VolumeDriver.Unmount",
{u"Name": u"vol"}, OK, {u"Err": u""})
def test_create_with_profile(self):
"""
Calling the ``/VolumerDriver.Create`` API with an ``Opts`` value
of "profile=[gold,silver,bronze] in the request body JSON create a
volume with a given name with [gold,silver,bronze] profile.
"""
profile = sampled_from(["gold", "silver", "bronze"]).example()
name = random_name(self)
d = self.assertResult(b"POST", b"/VolumeDriver.Create", {
u"Name": name,
'Opts': {
u"profile": profile
}
}, OK, {u"Err": u""})
d.addCallback(
lambda _: self.flocker_client.list_datasets_configuration())
d.addCallback(list)
d.addCallback(
lambda result: self.assertItemsEqual(result, [
Dataset(dataset_id=result[0].dataset_id,
primary=self.NODE_A,
maximum_size=int(DEFAULT_SIZE.to_Byte()),
metadata={
NAME_FIELD: name,
u"clusterhq:flocker:profile": unicode(profile)
})
]))
return d
def test_create_with_size(self):
"""
Calling the ``/VolumerDriver.Create`` API with an ``Opts`` value
of "size=<somesize> in the request body JSON create a volume
with a given name and random size between 1-100G
"""
name = random_name(self)
size = integers(min_value=1, max_value=75).example()
expression = volume_expression.example()
size_opt = "".join(str(size)) + expression
d = self.assertResult(b"POST", b"/VolumeDriver.Create", {
u"Name": name,
'Opts': {
u"size": size_opt
}
}, OK, {u"Err": u""})
real_size = int(parse_num(size_opt).to_Byte())
d.addCallback(
lambda _: self.flocker_client.list_datasets_configuration())
d.addCallback(list)
d.addCallback(
lambda result: self.assertItemsEqual(result, [
Dataset(dataset_id=result[0].dataset_id,
primary=self.NODE_A,
maximum_size=real_size,
metadata={
NAME_FIELD: name,
u"maximum_size": unicode(real_size)
})
]))
return d
@given(expr=volume_expression, size=integers(min_value=75, max_value=100))
def test_parsenum_size(self, expr, size):
"""
Send different forms of size expressions
to ``parse_num``, we expect G(Gigabyte) size results.
:param expr str: A string representing the size expression
:param size int: A string representing the volume size
"""
expected_size = int(GiB(size).to_Byte())
return self.assertEqual(expected_size,
int(parse_num(str(size) + expr).to_Byte()))
@given(expr=sampled_from(["KB", "MB", "GB", "TB", ""]),
size=integers(min_value=1, max_value=100))
def test_parsenum_all_sizes(self, expr, size):
"""
Send standard size expressions to ``parse_num`` in
many sizes, we expect to get correct size results.
:param expr str: A string representing the size expression
:param size int: A string representing the volume size
"""
if expr is "KB":
expected_size = int(KiB(size).to_Byte())
elif expr is "MB":
expected_size = int(MiB(size).to_Byte())
elif expr is "GB":
expected_size = int(GiB(size).to_Byte())
elif expr is "TB":
expected_size = int(TiB(size).to_Byte())
else:
expected_size = int(Byte(size).to_Byte())
return self.assertEqual(expected_size,
int(parse_num(str(size) + expr).to_Byte()))
@given(size=sampled_from(
[u"foo10Gb", u"10bar10", "10foogib", "10Gfoo", "GIB", "bar10foo"]))
def test_parsenum_bad_size(self, size):
"""
Send unacceptable size expressions, upon error
users should expect to receive Flocker's ``DEFAULT_SIZE``
:param size str: A string representing the bad volume size
"""
return self.assertEqual(int(DEFAULT_SIZE.to_Byte()),
int(parse_num(size).to_Byte()))
def create(self, name):
"""
Call the ``/VolumeDriver.Create`` API to create a volume with the
given name.
:param unicode name: The name of the volume to create.
:return: ``Deferred`` that fires when the volume that was created.
"""
return self.assertResult(b"POST", b"/VolumeDriver.Create",
{u"Name": name}, OK, {u"Err": u""})
def test_create_creates(self):
"""
``/VolumeDriver.Create`` creates a new dataset in the configuration.
"""
name = u"myvol"
d = self.create(name)
d.addCallback(
lambda _: self.flocker_client.list_datasets_configuration())
d.addCallback(list)
d.addCallback(
lambda result: self.assertItemsEqual(result, [
Dataset(dataset_id=result[0].dataset_id,
primary=self.NODE_A,
maximum_size=int(DEFAULT_SIZE.to_Byte()),
metadata={NAME_FIELD: name})
]))
return d
def test_create_duplicate_name(self):
"""
If a dataset with the given name already exists,
``/VolumeDriver.Create`` succeeds without create a new volume.
"""
name = u"thename"
# Create a dataset out-of-band with matching name but non-matching
# dataset ID:
d = self.flocker_client.create_dataset(self.NODE_A,
int(DEFAULT_SIZE.to_Byte()),
metadata={NAME_FIELD: name})
d.addCallback(lambda _: self.create(name))
d.addCallback(
lambda _: self.flocker_client.list_datasets_configuration())
d.addCallback(lambda results: self.assertEqual(len(list(results)), 1))
return d
def test_create_duplicate_name_race_condition(self):
"""
If a dataset with the given name is created while the
``/VolumeDriver.Create`` call is in flight, the call does not
result in an error.
"""
name = u"thename"
# Create a dataset out-of-band with matching dataset ID and name
# which the docker plugin won't be able to see.
def create_after_list():
# Clean up the patched version:
del self.flocker_client.list_datasets_configuration
# But first time we're called, we create dataset and lie about
# its existence:
d = self.flocker_client.create_dataset(self.NODE_A,
int(DEFAULT_SIZE.to_Byte()),
metadata={NAME_FIELD: name})
d.addCallback(
lambda _: DatasetsConfiguration(tag=u"1234", datasets={}))
return d
self.flocker_client.list_datasets_configuration = create_after_list
return self.create(name)
def _flush_volume_plugin_reactor_on_endpoint_render(self):
"""
This method patches ``self.app`` so that after any endpoint is
rendered, the reactor used by the volume plugin is advanced repeatedly
until there are no more ``delayedCalls`` pending on the reactor.
"""
real_execute_endpoint = self.app.execute_endpoint
def patched_execute_endpoint(*args, **kwargs):
val = real_execute_endpoint(*args, **kwargs)
while self.volume_plugin_reactor.getDelayedCalls():
pending_calls = self.volume_plugin_reactor.getDelayedCalls()
next_expiration = min(t.getTime() for t in pending_calls)
now = self.volume_plugin_reactor.seconds()
self.volume_plugin_reactor.advance(
max(0.0, next_expiration - now))
return val
self.patch(self.app, 'execute_endpoint', patched_execute_endpoint)
def test_mount(self):
"""
``/VolumeDriver.Mount`` sets the primary of the dataset with matching
name to the current node and then waits for the dataset to
actually arrive.
"""
name = u"myvol"
dataset_id = uuid4()
# Create dataset on a different node:
d = self.flocker_client.create_dataset(self.NODE_B,
int(DEFAULT_SIZE.to_Byte()),
metadata={NAME_FIELD: name},
dataset_id=dataset_id)
self._flush_volume_plugin_reactor_on_endpoint_render()
# Pretend that it takes 5 seconds for the dataset to get established on
# Node A.
self.volume_plugin_reactor.callLater(
5.0, self.flocker_client.synchronize_state)
d.addCallback(lambda _: self.assertResult(
b"POST", b"/VolumeDriver.Mount", {u"Name": name}, OK, {
u"Err": u"",
u"Mountpoint": u"/flocker/{}".format(dataset_id)
}))
d.addCallback(lambda _: self.flocker_client.list_datasets_state())
def final_assertions(datasets):
self.assertEqual(
[self.NODE_A],
[d.primary for d in datasets if d.dataset_id == dataset_id])
# There should be less than 20 calls to list_datasets_state over
# the course of 5 seconds.
self.assertLess(
self.flocker_client.num_calls('list_datasets_state'), 20)
d.addCallback(final_assertions)
return d
def test_mount_timeout(self):
"""
``/VolumeDriver.Mount`` sets the primary of the dataset with matching
name to the current node and then waits for the dataset to
actually arrive. If it does not arrive within 120 seconds, then it
returns an error up to docker.
"""
name = u"myvol"
dataset_id = uuid4()
# Create dataset on a different node:
d = self.flocker_client.create_dataset(self.NODE_B,
int(DEFAULT_SIZE.to_Byte()),
metadata={NAME_FIELD: name},
dataset_id=dataset_id)
self._flush_volume_plugin_reactor_on_endpoint_render()
# Pretend that it takes 500 seconds for the dataset to get established
# on Node A. This should be longer than the timeout.
self.volume_plugin_reactor.callLater(
500.0, self.flocker_client.synchronize_state)
d.addCallback(lambda _: self.assertResult(
b"POST", b"/VolumeDriver.Mount", {u"Name": name}, OK, {
u"Err": u"Timed out waiting for dataset to mount.",
u"Mountpoint": u""
}))
return d
def test_mount_already_exists(self):
"""
``/VolumeDriver.Mount`` sets the primary of the dataset with matching
name to the current node and then waits for the dataset to
actually arrive when used by the volumes that already exist and
don't have a special dataset ID.
"""
name = u"myvol"
d = self.flocker_client.create_dataset(self.NODE_A,
int(DEFAULT_SIZE.to_Byte()),
metadata={NAME_FIELD: name})
def created(dataset):
self.flocker_client.synchronize_state()
result = self.assertResult(
b"POST", b"/VolumeDriver.Mount", {u"Name": name}, OK, {
u"Err": u"",
u"Mountpoint": u"/flocker/{}".format(dataset.dataset_id)
})
result.addCallback(
lambda _: self.flocker_client.list_datasets_state())
result.addCallback(lambda ds: self.assertEqual([self.NODE_A], [
d.primary for d in ds if d.dataset_id == dataset.dataset_id
]))
return result
d.addCallback(created)
return d
def test_unknown_mount(self):
"""
``/VolumeDriver.Mount`` returns an error when asked to mount a
non-existent volume.
"""
name = u"myvol"
return self.assertResult(
b"POST", b"/VolumeDriver.Mount", {u"Name": name}, OK,
{u"Err": u"Could not find volume with given name."})
def test_path(self):
"""
``/VolumeDriver.Path`` returns the mount path of the given volume if
it is currently known.
"""
name = u"myvol"
d = self.create(name)
# The dataset arrives as state:
d.addCallback(lambda _: self.flocker_client.synchronize_state())
d.addCallback(lambda _: self.assertResponseCode(
b"POST", b"/VolumeDriver.Mount", {u"Name": name}, OK))
d.addCallback(
lambda _: self.flocker_client.list_datasets_configuration())
d.addCallback(lambda datasets_config: self.assertResult(
b"POST", b"/VolumeDriver.Path", {u"Name": name}, OK, {
u"Err":
u"",
u"Mountpoint":
u"/flocker/{}".format(datasets_config.datasets.keys()[0])
}))
return d
def test_path_existing(self):
"""
``/VolumeDriver.Path`` returns the mount path of the given volume if
it is currently known, including for a dataset that was created
not by the plugin.
"""
name = u"myvol"
d = self.flocker_client.create_dataset(self.NODE_A,
int(DEFAULT_SIZE.to_Byte()),
metadata={NAME_FIELD: name})
def created(dataset):
self.flocker_client.synchronize_state()
return self.assertResult(
b"POST", b"/VolumeDriver.Path", {u"Name": name}, OK, {
u"Err": u"",
u"Mountpoint": u"/flocker/{}".format(dataset.dataset_id)
})
d.addCallback(created)
return d
def test_unknown_path(self):
"""
``/VolumeDriver.Path`` returns an error when asked for the mount path
of a non-existent volume.
"""
name = u"myvol"
return self.assertResult(
b"POST", b"/VolumeDriver.Path", {u"Name": name}, OK,
{u"Err": u"Could not find volume with given name."})
def test_non_local_path(self):
"""
``/VolumeDriver.Path`` returns an error when asked for the mount path
of a volume that is not mounted locally.
This can happen as a result of ``docker inspect`` on a container
that has been created but is still waiting for its volume to
arrive from another node. It seems like Docker may also call this
after ``/VolumeDriver.Create``, so again while waiting for a
volume to arrive.
"""
name = u"myvol"
dataset_id = uuid4()
# Create dataset on node B:
d = self.flocker_client.create_dataset(self.NODE_B,
int(DEFAULT_SIZE.to_Byte()),
metadata={NAME_FIELD: name},
dataset_id=dataset_id)
d.addCallback(lambda _: self.flocker_client.synchronize_state())
# Ask for path on node A:
d.addCallback(lambda _: self.assertResult(
b"POST", b"/VolumeDriver.Path", {u"Name": name}, OK, {
u"Err": "Volume not available.",
u"Mountpoint": u""
}))
return d
@capture_logging(lambda self, logger: self.assertEqual(
len(logger.flushTracebacks(CustomException)), 1))
def test_unexpected_error_reporting(self, logger):
"""
If an unexpected error occurs Docker gets back a useful error message.
"""
def error():
raise CustomException("I've made a terrible mistake")
self.patch(self.flocker_client, "list_datasets_configuration", error)
return self.assertResult(
b"POST", b"/VolumeDriver.Path", {u"Name": u"whatever"}, OK,
{u"Err": "CustomException: I've made a terrible mistake"})
@capture_logging(None)
def test_bad_request(self, logger):
"""
If a ``BadRequest`` exception is raised it is converted to appropriate
JSON.
"""
def error():
raise make_bad_request(code=423, Err=u"no good")
self.patch(self.flocker_client, "list_datasets_configuration", error)
return self.assertResult(b"POST", b"/VolumeDriver.Path",
{u"Name": u"whatever"}, 423,
{u"Err": "no good"})
def test_unsupported_method(self):
"""
If an unsupported method is requested the 405 Not Allowed response
code is returned.
"""
return self.assertResponseCode(b"BAD_METHOD", b"/VolumeDriver.Path",
None, NOT_ALLOWED)
def test_unknown_uri(self):
"""
If an unknown URI path is requested the 404 Not Found response code is
returned.
"""
return self.assertResponseCode(b"BAD_METHOD", b"/xxxnotthere", None,
NOT_FOUND)
def test_empty_host(self):
"""
If an empty host header is sent to the Docker plugin it does not blow
up, instead operating normally. E.g. for ``Plugin.Activate`` call
returns the ``Implements`` response.
"""
return self.assertResult(b"POST",
b"/Plugin.Activate",
12345,
OK, {u"Implements": [u"VolumeDriver"]},
additional_headers={b"Host": [""]})
def test_get(self):
"""
``/VolumeDriver.Get`` returns the mount path of the given volume if
it is currently known.
"""
name = u"myvol"
d = self.create(name)
# The dataset arrives as state:
d.addCallback(lambda _: self.flocker_client.synchronize_state())
d.addCallback(lambda _: self.assertResponseCode(
b"POST", b"/VolumeDriver.Mount", {u"Name": name}, OK))
d.addCallback(
lambda _: self.flocker_client.list_datasets_configuration())
d.addCallback(lambda datasets_config: self.assertResult(
b"POST", b"/VolumeDriver.Get", {u"Name": name}, OK, {
u"Err": u"",
u"Volume": {
u"Name":
name,
u"Mountpoint":
u"/flocker/{}".format(datasets_config.datasets.keys()[0])
}
}))
return d
def test_get_existing(self):
"""
``/VolumeDriver.Get`` returns the mount path of the given volume if
it is currently known, including for a dataset that was created
not by the plugin.
"""
name = u"myvol"
d = self.flocker_client.create_dataset(self.NODE_A,
int(DEFAULT_SIZE.to_Byte()),
metadata={NAME_FIELD: name})
def created(dataset):
self.flocker_client.synchronize_state()
return self.assertResult(
b"POST", b"/VolumeDriver.Get", {u"Name": name}, OK, {
u"Err": u"",
u"Volume": {
u"Name": name,
u"Mountpoint": u"/flocker/{}".format(
dataset.dataset_id)
}
})
d.addCallback(created)
return d
def test_unknown_get(self):
"""
``/VolumeDriver.Get`` returns an error when asked for the mount path
of a non-existent volume.
"""
name = u"myvol"
return self.assertResult(
b"POST", b"/VolumeDriver.Get", {u"Name": name}, OK,
{u"Err": u"Could not find volume with given name."})
def test_non_local_get(self):
"""
``/VolumeDriver.Get`` returns an empty mount point when asked about a
volume that is not mounted locally.
"""
name = u"myvol"
dataset_id = uuid4()
# Create dataset on node B:
d = self.flocker_client.create_dataset(self.NODE_B,
int(DEFAULT_SIZE.to_Byte()),
metadata={NAME_FIELD: name},
dataset_id=dataset_id)
d.addCallback(lambda _: self.flocker_client.synchronize_state())
# Ask for path on node A:
d.addCallback(lambda _: self.assertResult(
b"POST", b"/VolumeDriver.Get", {u"Name": name}, OK, {
u"Err": u"",
u"Volume": {
u"Name": name,
u"Mountpoint": u""
}
}))
return d
def test_list(self):
"""
``/VolumeDriver.List`` returns the mount path of the given volume if
it is currently known and an empty mount point for non-local
volumes.
"""
name = u"myvol"
remote_name = u"myvol3"
d = gatherResults([
self.flocker_client.create_dataset(self.NODE_A,
int(DEFAULT_SIZE.to_Byte()),
metadata={NAME_FIELD: name}),
self.flocker_client.create_dataset(
self.NODE_B,
int(DEFAULT_SIZE.to_Byte()),
metadata={NAME_FIELD: remote_name})
])
# The datasets arrive as state:
d.addCallback(lambda _: self.flocker_client.synchronize_state())
d.addCallback(
lambda _: self.flocker_client.list_datasets_configuration())
d.addCallback(lambda datasets_config: self.assertResult(
b"POST", b"/VolumeDriver.List", {}, OK, {
u"Err":
u"",
u"Volumes":
sorted([
{
u"Name":
name,
u"Mountpoint":
u"/flocker/{}".format([
key for (key, value) in datasets_config.datasets.
items() if value.metadata["name"] == name
][0])
},
{
u"Name": remote_name,
u"Mountpoint": u""
},
])
}))
return d
def test_list_no_metadata_name(self):
"""
``/VolumeDriver.List`` omits volumes that don't have a metadata field
for their name.
"""
d = self.flocker_client.create_dataset(self.NODE_A,
int(DEFAULT_SIZE.to_Byte()),
metadata={})
d.addCallback(lambda _: self.assertResult(
b"POST", b"/VolumeDriver.List", {}, OK, {
u"Err": u"",
u"Volumes": []
}))
return d
self.gamespace_p2.puck.speedx = pickle.loads(data[2])
self.gamespace_p2.puck.speedy = pickle.loads(data[3])
def quit(self):
self.transport.loseConnection()
def write(self, data):
self.transport.write(data)
class ClientConnFactory(ClientFactory):
def __init__(self, gs):
self.gs = gs
self.client_connection = ClientConn(self.gs)
def buildProtocol(self, addr):
self.gs.quit = self.client_connection.quit
self.gs.write = self.client_connection.write
return self.client_connection
if __name__ == '__main__':
gs = GameSpace()
# 3) Start the game loop
lc = LoopingCall(gs.game_loop)
lc.start(1 / 60)
reactor.connectTCP(CLIENT_HOST, CLIENT_PORT, ClientConnFactory(gs))
reactor.run()
gs.game_loop()
lc.stop()
class WindowManager(object):
WINDOW_KEY = 'windows'
FLIGHT_KEY = 'inflight'
FLIGHT_STATS_KEY = 'flightstats'
MAP_KEY = 'keymap'
def __init__(self, redis, window_size=100, flight_lifetime=None,
gc_interval=10):
self.window_size = window_size
self.flight_lifetime = flight_lifetime or (gc_interval * window_size)
self.redis = redis
self.clock = self.get_clock()
self.gc = LoopingCall(self.clear_expired_flight_keys)
self.gc.clock = self.clock
self.gc.start(gc_interval)
self._monitor = None
def noop(self, *args, **kwargs):
pass
def stop(self):
if self._monitor and self._monitor.running:
self._monitor.stop()
if self.gc.running:
self.gc.stop()
def get_windows(self):
return self.redis.zrange(self.window_key(), 0, -1)
@inlineCallbacks
def window_exists(self, window_id):
score = yield self.redis.zscore(self.window_key(), window_id)
if score is not None:
returnValue(True)
returnValue(False)
def window_key(self, *keys):
return ':'.join([self.WINDOW_KEY] + map(unicode, keys))
def flight_key(self, *keys):
return self.window_key(self.FLIGHT_KEY, *keys)
def stats_key(self, *keys):
return self.window_key(self.FLIGHT_STATS_KEY, *keys)
def map_key(self, *keys):
return self.window_key(self.MAP_KEY, *keys)
def get_clock(self):
return reactor
def get_clocktime(self):
return self.clock.seconds()
@inlineCallbacks
def create_window(self, window_id, strict=False):
if strict and (yield self.window_exists(window_id)):
raise WindowException('Window already exists: %s' % (window_id,))
clock_time = self.get_clocktime()
yield self.redis.zadd(self.WINDOW_KEY, **{
window_id: clock_time,
})
returnValue(clock_time)
@inlineCallbacks
def remove_window(self, window_id):
waiting_list = yield self.count_waiting(window_id)
if waiting_list:
raise WindowException('Window not empty')
yield self.redis.zrem(self.WINDOW_KEY, window_id)
@inlineCallbacks
def add(self, window_id, data, key=None):
key = key or uuid.uuid4().get_hex()
# The redis.set() has to complete before redis.lpush(),
# otherwise the key can be popped from the window before the
# data is available.
yield self.redis.set(self.window_key(window_id, key),
json.dumps(data))
yield self.redis.lpush(self.window_key(window_id), key)
returnValue(key)
@inlineCallbacks
def get_next_key(self, window_id):
window_key = self.window_key(window_id)
inflight_key = self.flight_key(window_id)
waiting_list = yield self.count_waiting(window_id)
if waiting_list == 0:
return
flight_size = yield self.count_in_flight(window_id)
room_available = self.window_size - flight_size
if room_available > 0:
log.debug('Window %s has space for %s' % (window_key,
room_available))
next_key = yield self.redis.rpoplpush(window_key, inflight_key)
if next_key:
yield self._set_timestamp(window_id, next_key)
returnValue(next_key)
def _set_timestamp(self, window_id, flight_key):
return self.redis.zadd(self.stats_key(window_id), **{
flight_key: self.get_clocktime(),
})
def _clear_timestamp(self, window_id, flight_key):
return self.redis.zrem(self.stats_key(window_id), flight_key)
def count_waiting(self, window_id):
window_key = self.window_key(window_id)
return self.redis.llen(window_key)
def count_in_flight(self, window_id):
flight_key = self.flight_key(window_id)
return self.redis.llen(flight_key)
def get_expired_flight_keys(self, window_id):
return self.redis.zrangebyscore(self.stats_key(window_id),
'-inf', self.get_clocktime() - self.flight_lifetime)
@inlineCallbacks
def clear_expired_flight_keys(self):
windows = yield self.get_windows()
for window_id in windows:
expired_keys = yield self.get_expired_flight_keys(window_id)
for key in expired_keys:
yield self.redis.lrem(self.flight_key(window_id), key, 1)
@inlineCallbacks
def get_data(self, window_id, key):
json_data = yield self.redis.get(self.window_key(window_id, key))
returnValue(json.loads(json_data))
@inlineCallbacks
def remove_key(self, window_id, key):
yield self.redis.lrem(self.flight_key(window_id), key, 1)
yield self.redis.delete(self.window_key(window_id, key))
yield self.redis.delete(self.stats_key(window_id, key))
yield self.clear_external_id(window_id, key)
yield self._clear_timestamp(window_id, key)
@inlineCallbacks
def set_external_id(self, window_id, flight_key, external_id):
yield self.redis.set(self.map_key(window_id, 'internal', external_id),
flight_key)
yield self.redis.set(self.map_key(window_id, 'external', flight_key),
external_id)
def get_internal_id(self, window_id, external_id):
return self.redis.get(self.map_key(window_id, 'internal', external_id))
def get_external_id(self, window_id, flight_key):
return self.redis.get(self.map_key(window_id, 'external', flight_key))
@inlineCallbacks
def clear_external_id(self, window_id, flight_key):
external_id = yield self.get_external_id(window_id, flight_key)
if external_id:
yield self.redis.delete(self.map_key(window_id, 'external',
flight_key))
yield self.redis.delete(self.map_key(window_id, 'internal',
external_id))
def monitor(self, key_callback, interval=10, cleanup=True,
cleanup_callback=None):
if self._monitor is not None:
raise WindowException('Monitor already started')
self._monitor = LoopingCall(lambda: self._monitor_windows(
key_callback, cleanup, cleanup_callback))
self._monitor.clock = self.get_clock()
self._monitor.start(interval)
@inlineCallbacks
def _monitor_windows(self, key_callback, cleanup=True,
cleanup_callback=None):
windows = yield self.get_windows()
for window_id in windows:
key = (yield self.get_next_key(window_id))
while key:
yield key_callback(window_id, key)
key = (yield self.get_next_key(window_id))
# Remove empty windows if required
if cleanup and not ((yield self.count_waiting(window_id)) or
(yield self.count_in_flight(window_id))):
if cleanup_callback:
cleanup_callback(window_id)
yield self.remove_window(window_id)
class Core(CorePluginBase):
totals = {} #class var to catch only updating this once per session in enable.
def enable(self):
log.debug("Stats plugin enabled")
self.core = component.get("Core")
self.stats ={}
self.count = {}
self.intervals = [1, 5, 30, 300]
self.last_update = {}
t = time.time()
for i in self.intervals:
self.stats[i] = {}
self.last_update[i] = t
self.count[i] = 0
self.config = configmanager.ConfigManager("stats.conf", DEFAULT_PREFS)
self.saved_stats = configmanager.ConfigManager("stats.totals", DEFAULT_TOTALS)
if self.totals == {}:
self.totals.update(self.saved_stats.config)
self.length = self.config["length"]
#self.stats = get_key(self.saved_stats, "stats") or {}
self.stats_keys = []
self.add_stats(
'upload_rate',
'download_rate',
'num_connections',
'dht_nodes',
'dht_cache_nodes',
'dht_torrents',
'num_peers',
)
self.update_stats()
self.update_timer = LoopingCall(self.update_stats)
self.update_timer.start(self.config["update_interval"])
self.save_timer = LoopingCall(self.save_stats)
self.save_timer.start(60)
def disable(self):
self.save_stats()
try:
self.update_timer.stop()
self.save_timer.stop()
except:
pass
def add_stats(self, *stats):
for stat in stats:
if stat not in self.stats_keys:
self.stats_keys.append(stat)
for i in self.intervals:
if stat not in self.stats[i]:
self.stats[i][stat] = []
def update_stats(self):
try:
#Get all possible stats!
stats = {}
for key in self.stats_keys:
#try all keys we have, very inefficient but saves having to
#work out where a key comes from...
try:
stats.update(self.core.get_session_status([key]))
except AttributeError:
pass
stats["num_connections"] = self.core.get_num_connections()
stats.update(self.core.get_config_values(["max_download",
"max_upload",
"max_num_connections"]))
# status = self.core.session.status()
# for stat in dir(status):
# if not stat.startswith('_') and stat not in stats:
# stats[stat] = getattr(status, stat, None)
update_time = time.time()
self.last_update[1] = update_time
#extract the ones we are interested in
#adding them to the 1s array
for stat, stat_list in self.stats[1].iteritems():
if stat in stats:
stat_list.insert(0, int(stats[stat]))
else:
stat_list.insert(0, 0)
if len(stat_list) > self.length:
stat_list.pop()
def update_interval(interval, base, multiplier):
self.count[interval] = self.count[interval] + 1
if self.count[interval] >= interval:
self.last_update[interval] = update_time
self.count[interval] = 0
current_stats = self.stats[interval]
for stat, stat_list in self.stats[base].iteritems():
try:
avg = mean(stat_list[0:multiplier])
except ValueError:
avg = 0
current_stats[stat].insert(0, avg)
if len(current_stats[stat]) > self.length:
current_stats[stat].pop()
update_interval(5, 1, 5)
update_interval(30, 5, 6)
update_interval(300, 30, 10)
except Exception, e:
log.error("Stats update error %s" % e)
return True
class NormalPMTFlow(LabradServer):
name = 'NormalPMTFlow'
onNewCount = Signal(SIGNALID, 'signal: new count', 'v')
onNewSetting = Signal(SIGNALID + 1, 'signal: new setting', '(ss)')
ready = Signal(SIGNALID + 2, 'signal: ready', 'b')
@inlineCallbacks
def initServer(self):
self.saveFolder = ['', 'PMT Counts']
self.dataSetName = 'PMT Counts'
self.collectTimes = {'Normal': 0.100, 'Differential': 0.100}
self.lastDifferential = {'ON': 0, 'OFF': 0}
self.currentMode = 'Normal'
self.dv = None
self.pulser = None
self.collectTimeRange = None
self.dataSet = None
self.requestList = []
self.listeners = set()
self.recording = LoopingCall(self._record)
yield self.connect_data_vault()
yield self.connect_pulser()
@inlineCallbacks
def connect_data_vault(self):
try:
self.dv = yield self.client.data_vault
yield self.dv.cd(self.saveFolder, True)
print 'Connected: Data Vault'
except AttributeError:
self.dv = None
print 'Not Connected: Data Vault'
@inlineCallbacks
def disconnect_data_vault(self):
print 'Not Connected: Data Vault'
self.dv = None
@inlineCallbacks
def connect_pulser(self):
try:
self.pulser = yield self.client.pulser
self.collectTimeRange = yield self.pulser.get_collection_time()
print 'Connected: Pulser'
except AttributeError:
self.pulser = None
self.collectTimeRange = None
print 'Not Connected: Pulser'
@inlineCallbacks
def disconnect_pulser(self):
print 'Not Connected: Pulser'
self.pulser = None
self.dataSet = None
def initContext(self, c):
"""Initialize a new context object."""
self.listeners.add(c.ID)
def expireContext(self, c):
self.listeners.remove(c.ID)
def getOtherListeners(self, c):
notified = self.listeners.copy()
notified.remove(c.ID)
return notified
@inlineCallbacks
def makeNewDataSet(self):
folder = self.saveFolder
name = self.dataSetName
yield self.dv.cd(folder, True)
self.dataSet = yield self.dv.new(
name, [('t', 'num')],
[('KiloCounts/sec', '866 ON', 'num'),
('KiloCounts/sec', '866 OFF', 'num'),
('KiloCounts/sec', 'Differential Signal', 'num')])
self.startTime = time.time()
yield self.addParameters()
returnValue(self.dataSet)
@inlineCallbacks
def addParameters(self):
yield self.dv.add_parameter('plotLive', True)
yield self.dv.add_parameter('startTime', self.startTime)
@setting(0, 'Set Save Folder', folder='*s', returns='')
def setSaveFolder(self, c, folder):
try:
yield self.dv.cd(folder, True)
self.saveFolder = folder
except AttributeError:
print 'failing'
from labrad.types import Error
raise Error("NotReady")
@setting(1, 'Start New Dataset', setName='s', returns='s')
def setNewDataSet(self, c, setName=None):
"""Starts new dataset, if name not provided, it will be the same"""
if setName is not None: self.dataSetName = setName
dataset = yield self.makeNewDataSet()
name = dataset[1]
otherListeners = self.getOtherListeners(c)
self.onNewSetting(('dataset', name), otherListeners)
returnValue(name)
@setting(2, "Set Mode", mode='s', returns='')
def setMode(self, c, mode):
"""
Start recording Time Resolved Counts into Data Vault
"""
if mode not in self.collectTimes.keys():
raise Exception('Incorrect Mode')
if not self.recording.running:
self.currentMode = mode
yield self.pulser.set_mode(mode)
else:
yield self.dostopRecording()
self.currentMode = mode
yield self.pulser.set_mode(mode)
yield self.dorecordData()
otherListeners = self.getOtherListeners(c)
self.onNewSetting(('mode', mode), otherListeners)
@setting(3, 'getCurrentMode', returns='s')
def getCurrentMode(self, c):
"""
Returns the currently running mode
"""
return self.currentMode
@setting(4, 'Record Data', returns='')
def recordData(self, c):
"""
Starts recording data of the current PMT mode into datavault
"""
setname = yield self.dorecordData()
otherListeners = self.getOtherListeners(c)
if setname is not None:
setname = setname[1]
self.onNewSetting(('dataset', setname), otherListeners)
self.onNewSetting(('state', 'on'), otherListeners)
@inlineCallbacks
def dorecordData(self):
newSet = None
self.keepRunning = True
yield self.pulser.set_collection_time(
self.collectTimes[self.currentMode], self.currentMode)
yield self.pulser.set_mode(self.currentMode)
if self.currentMode == 'Differential':
yield self._programPulserDiff()
if self.dataSet is None:
newSet = yield self.makeNewDataSet()
self.recording.start(self.collectTimes[self.currentMode] / 2.0)
returnValue(newSet)
@setting(5, returns='')
def stopRecording(self, c):
"""
Stop recording counts into Data Vault
"""
yield self.dostopRecording()
otherListeners = self.getOtherListeners(c)
self.onNewSetting(('state', 'off'), otherListeners)
@inlineCallbacks
def dostopRecording(self):
yield self.recording.stop()
if self.currentMode == 'Differential':
yield self._stopPulserDiff()
@setting(6, returns='b')
def isRunning(self, c):
"""
Returns whether or not currently recording
"""
return self.recording.running
@setting(7, returns='s')
def currentDataSet(self, c):
if self.dataSet is None: return ''
name = self.dataSet[1]
return name
@setting(8, 'Set Time Length', timelength='v')
def setTimeLength(self, c, timelength):
"""Sets the time length for the current mode"""
mode = self.currentMode
if mode not in self.collectTimes.keys():
raise Exception('Incorrect Mode')
if not self.collectTimeRange[0] <= timelength <= self.collectTimeRange[
1]:
raise Exception('Incorrect Recording Time')
self.collectTimes[mode] = timelength
initrunning = self.recording.running #if recording when the call is made, need to stop and restart
if initrunning:
yield self.recording.stop()
yield self.pulser.set_collection_time(timelength, mode)
if initrunning:
if mode == 'Differential':
yield self._stopPulserDiff()
yield self._programPulserDiff()
self.recording.start(timelength / 2.0)
otherListeners = self.getOtherListeners(c)
self.onNewSetting(('timelength', str(timelength)), otherListeners)
@setting(9,
'Get Next Counts',
type='s',
number='w',
average='b',
returns=['*v', 'v'])
def getNextCounts(self, c, type, number, average=False):
"""
Acquires next number of counts, where type can be 'ON' or 'OFF' or 'DIFF'
Average is optionally True if the counts should be averaged
Note in differential mode, Diff counts get updates every time, but ON and OFF
get updated every 2 times.
"""
if type not in ['ON', 'OFF', 'DIFF']: raise Exception('Incorrect type')
if type in ['OFF', 'DIFF'] and self.currentMode == 'Normal':
raise Exception('in the wrong mode to process this request')
if not 0 < number < 1000: raise Exception('Incorrect Number')
if not self.recording.running:
raise Exception('Not currently recording')
d = Deferred()
self.requestList.append(self.readingRequest(d, type, number))
data = yield d
if average:
data = sum(data) / len(data)
returnValue(data)
@setting(10, 'Get Time Length', returns='v')
def getMode(self, c):
"""
Returns the current timelength of in the current mode
"""
return self.collectTimes[self.currentMode]
@setting(11, 'Get Time Length Range', returns='(vv)')
def get_time_length_range(self, c):
return self.collectTimeRange
@inlineCallbacks
def _programPulserDiff(self):
yield self.pulser.new_sequence()
countRate = self.collectTimes['Differential']
yield self.pulser.add_ttl_pulse('DiffCountTrigger', 0.0, 10.0e-6)
yield self.pulser.add_ttl_pulse('DiffCountTrigger', countRate, 10.0e-6)
yield self.pulser.add_ttl_pulse('866DP', 0.0, countRate)
yield self.pulser.extend_sequence_length(2 * countRate)
yield self.pulser.program_sequence()
yield self.pulser.start_infinite()
@inlineCallbacks
def _stopPulserDiff(self):
yield self.pulser.complete_infinite_iteration()
yield self.pulser.wait_sequence_done()
yield self.pulser.stop_sequence()
class readingRequest():
def __init__(self, d, type, count):
self.d = d
self.count = count
self.type = type
self.data = []
def is_fulfilled(self):
return len(self.data) == self.count
def processRequests(self, data):
if not len(self.requestList): return
for dataPoint in data:
for item, req in enumerate(self.requestList):
if dataPoint[1] != 0 and req.type == 'ON':
req.data.append(dataPoint[1])
if dataPoint[2] != 0 and req.type == 'OFF':
req.data.append(dataPoint[1])
if dataPoint[3] != 0 and req.type == 'DIFF':
req.data.append(dataPoint[1])
if req.is_fulfilled():
req.d.callback(req.data)
del (self.requestList[item])
@inlineCallbacks
def _record(self):
rawdata = yield self.pulser.get_pmt_counts()
print rawdata
if len(rawdata) != 0:
if self.currentMode == 'Normal':
toDataVault = [
[elem[2] - self.startTime, elem[0], 0, 0]
for elem in rawdata
] # converting to format [time, normal count, 0 , 0]
elif self.currentMode == 'Differential':
toDataVault = self.convertDifferential(rawdata)
self.processRequests(
toDataVault) #if we have any requests, process them
self.processSignals(toDataVault)
yield self.dv.add(toDataVault)
def processSignals(self, data):
lastPt = data[-1]
NormalCount = lastPt[1]
self.onNewCount(NormalCount)
def convertDifferential(self, rawdata):
totalData = []
for dataPoint in rawdata:
t = str(dataPoint[1])
self.lastDifferential[t] = float(dataPoint[0])
diff = self.lastDifferential['ON'] - self.lastDifferential['OFF']
totalData.append([
dataPoint[2] - self.startTime, self.lastDifferential['ON'],
self.lastDifferential['OFF'], diff
])
return totalData
class MetricLogObserver(util_service.ReconfigurableServiceMixin,
service.MultiService):
_reactor = reactor
def __init__(self):
service.MultiService.__init__(self)
self.setName('metrics')
self.enabled = False
self.periodic_task = None
self.periodic_interval = None
self.log_task = None
self.log_interval = None
# Mapping of metric type to handlers for that type
self.handlers = {}
# Register our default handlers
self.registerHandler(MetricCountEvent, MetricCountHandler(self))
self.registerHandler(MetricTimeEvent, MetricTimeHandler(self))
self.registerHandler(MetricAlarmEvent, MetricAlarmHandler(self))
self.getHandler(MetricCountEvent).addWatcher(
AttachedWorkersWatcher(self))
def reconfigServiceWithBuildbotConfig(self, new_config):
# first, enable or disable
if new_config.metrics is None:
self.disable()
else:
self.enable()
metrics_config = new_config.metrics
# Start up periodic logging
log_interval = metrics_config.get('log_interval', 60)
if log_interval != self.log_interval:
if self.log_task:
self.log_task.stop()
self.log_task = None
if log_interval:
self.log_task = LoopingCall(self.report)
self.log_task.clock = self._reactor
self.log_task.start(log_interval)
# same for the periodic task
periodic_interval = metrics_config.get('periodic_interval', 10)
if periodic_interval != self.periodic_interval:
if self.periodic_task:
self.periodic_task.stop()
self.periodic_task = None
if periodic_interval:
self.periodic_task = LoopingCall(periodicCheck,
self._reactor)
self.periodic_task.clock = self._reactor
self.periodic_task.start(periodic_interval)
# upcall
return util_service.ReconfigurableServiceMixin.reconfigServiceWithBuildbotConfig(
self, new_config)
def stopService(self):
self.disable()
service.MultiService.stopService(self)
def enable(self):
if self.enabled:
return
log.addObserver(self.emit)
self.enabled = True
def disable(self):
if not self.enabled:
return
if self.periodic_task:
self.periodic_task.stop()
self.periodic_task = None
if self.log_task:
self.log_task.stop()
self.log_task = None
log.removeObserver(self.emit)
self.enabled = False
def registerHandler(self, interface, handler):
old = self.getHandler(interface)
self.handlers[interface] = handler
return old
def getHandler(self, interface):
return self.handlers.get(interface)
def emit(self, eventDict):
# Ignore non-statistic events
metric = eventDict.get('metric')
if not metric or not isinstance(metric, MetricEvent):
return
if metric.__class__ not in self.handlers:
return
h = self.handlers[metric.__class__]
h.handle(eventDict, metric)
for w in h.watchers:
w.run()
def asDict(self):
retval = {}
for interface, handler in self.handlers.items():
retval.update(handler.asDict())
return retval
def report(self):
try:
for interface, handler in self.handlers.items():
report = handler.report()
if not report:
continue
for line in report.split("\n"):
log.msg(line)
except Exception:
log.err(None, "generating metric report")
def test_tls_auth_denied(self):
"""
A MQTT client offering the wrong certificate won't be authenticated.
"""
reactor, router, server_factory, session_factory = build_mqtt_server()
real_reactor = selectreactor.SelectReactor()
logger = make_logger()
session, pump = connect_application_session(
server_factory,
ObservingSession,
component_config=ComponentConfig(realm="mqtt"))
endpoint = create_listening_endpoint_from_config(
{
"type": "tcp",
"port": 1099,
"interface": "0.0.0.0",
"tls": {
"certificate": "server.crt",
"key": "server.key",
"dhparam": "dhparam",
"ca_certificates":
["ca.cert.pem", "intermediate.cert.pem"]
},
},
FilePath(__file__).sibling('certs').path, real_reactor, logger)
client_endpoint = create_connecting_endpoint_from_config(
{
"type": "tcp",
"host": "127.0.0.1",
"port": 1099,
"tls": {
# BAD key: trusted by the CA, but wrong ID
"certificate": "client_1.crt",
"hostname": "localhost",
"key": "client_1.key",
"ca_certificates":
["ca.cert.pem", "intermediate.cert.pem"]
},
},
FilePath(__file__).sibling('certs').path,
real_reactor,
logger)
p = []
l = endpoint.listen(server_factory)
class TestProtocol(Protocol):
data = b""
expected = (ConnACK(session_present=False,
return_code=1).serialise())
def dataReceived(self_, data):
self_.data = self_.data + data
if len(self_.data) == len(self_.expected):
self.assertEqual(self_.data, self_.expected)
real_reactor.stop()
@l.addCallback
def _listening(factory):
d = client_endpoint.connect(Factory.forProtocol(TestProtocol))
@d.addCallback
def _(proto):
p.append(proto)
proto.transport.write(
Connect(
client_id="test123",
flags=ConnectFlags(clean_session=False)).serialise())
proto.transport.write(
Publish(duplicate=False,
qos_level=1,
retain=False,
topic_name="test",
payload=b"{}",
packet_identifier=1).serialise())
lc = LoopingCall(pump.flush)
lc.clock = real_reactor
lc.start(0.01)
def timeout():
print("Timing out :(")
real_reactor.stop()
print(self.logs.log_text.getvalue())
# Timeout, just in case
real_reactor.callLater(10, timeout)
real_reactor.run()
client_protocol = p[0]
# We get a CONNECT
self.assertEqual(
client_protocol.data,
ConnACK(session_present=False, return_code=1).serialise())
client_protocol.data = b""
pump.flush()
# No events!
self.assertEqual(len(session.events), 0)
class AudioFromFiles(baseaudio.AudioDevice):
_infp = _outfp = None
LC = None
def __init__(self):
try:
from __main__ import app
except:
log.msg("couldn't find app preferences, no file audio",
system="audio")
raise ValueError('no app for filename preferences?')
device = app.getPref('audio_device')
if not device:
log.msg(
"need to provide audio_device with files"
" as infile,outfile",
system="audio")
raise ValueError('no audio_device specified')
files = device.split(',', 2)
if len(files) != 2:
raise ValueError("device spec should be infile,outfile")
self.infile, self.outfile = files
baseaudio.AudioDevice.__init__(self)
self.close()
def _push_up_some_data(self):
if not self.encoder or self._infp is None:
return
data = self._infp.read(320)
#log.msg("pushing %d bytes from file"%(len(data)), system="audio")
if self.encoder and data:
self.encoder.handle_audio(data)
def write(self, bytes):
if self._outfp is not None:
return self._outfp.write(bytes)
def _getFiles(self):
if self.infile:
self._infp = open(self.infile, 'rb')
else:
self._infp = None
if self.outfile:
self._outfp = open(self.outfile, 'wb')
else:
self._outfp = None
def _close(self):
#print "close called", self._closed, self.LC
if self._infp is not None:
self._infp.close()
if self._outfp is not None:
self._outfp.close()
self._closed = True
self._infp = self._outfp = None
self.LC.stop()
self.LC = None
def openDev(self):
from shtoom.util import stack
self._getFiles()
#print "openDev called!", self._closed, self.LC, stack()
if self.LC is not None:
return
if self._infp is None and self._outfp is None:
self._getFiles()
self.LC = LoopingCall(self._push_up_some_data)
#print self, "creating LoopingCall", self.LC, stack()
self.LC.start(0.020)
class JobPerspective(object):
def __init__(self, jobId, jobSpec):
self.jobId = jobId
self.jobSpec = jobSpec
self.mapping = {}
self.health = JobHealth.OK
self.task = LoopingCall(self.state)
self._started = False
self._finished = False
def addSlave(self, slave, remoteJobId):
self.mapping[slave] = remoteJobId
def removeSlave(self, slave):
self.mapping.pop(slave)
@inlineCallbacks
def handleSlaveError(self, slave, error):
# XXX should handle different errors differently
# for now we'll just assume the slave gets killed off
log.error("Job health compromised: lost a slave server")
self.health = JobHealth.ERROR
log.warn("Removing slave %s://%s:%d/%s" %
(slave.slaveSpec.scheme, slave.slaveSpec.host,
slave.slaveSpec.port, slave.slaveSpec.path))
self.removeSlave(slave)
SlaveAllocator.degrade(slave)
# we should probably mark the job state as unknown and cancel
# the whole thing, since it couldn't be completed as asked
yield self._jobOp("stopJob", ignoreHealth=True)
returnValue(True)
def _jobIsStarted(self):
if self._started == False:
log.debug("Starting job %d" % self.jobId)
for slave in self.mapping.iterkeys():
SlaveAllocator.markAsRunning(slave)
self.task.start(120)
self._started = True
def _jobIsFinished(self):
if self._finished == False:
log.debug("Finishing job %d" % self.jobId)
for slave in self.mapping.iterkeys():
SlaveAllocator.markAsFinished(slave)
self.task.stop()
self._finished = True
# this is just a pass-through to the DeferredList callback
def _jobOpSlaveCallback(self, value):
return value
# though here we want to intercept errors
def _jobOpSlaveErrback(self, error, slave):
self.handleSlaveError(slave, error)
return error
@inlineCallbacks
def _jobOp(self, operation, *args, **kwargs):
try:
if kwargs["ignoreHealth"] == True:
pass
else:
raise
except:
if self.health == JobHealth.ERROR:
returnValue(False)
requests = []
for slave, remoteId in self.mapping.iteritems():
request = getattr(slave, "%s" % operation)(remoteId, *args)
request.addCallback(self._jobOpSlaveCallback)
request.addErrback(self._jobOpSlaveErrback, slave)
requests.append(request)
deferredList = DeferredList(requests, consumeErrors=True)
yield deferredList
returnValue(deferredList.result)
@inlineCallbacks
def start(self):
request = self._jobOp("startJob")
yield request
self._jobIsStarted()
returnValue(request.result)
def pause(self):
return self._jobOp("pauseJob")
def resume(self):
return self._jobOp("resumeJob")
def stop(self):
return self._jobOp("stopJob")
@inlineCallbacks
def state(self):
# this line is repeated twice: once at the beginning for
# new calls to state(), and once after checking the DeferredList
# in case the health has suffered during the call
if self.health == JobHealth.ERROR:
returnValue(JobState.ERROR)
request = self._jobOp("jobState")
yield request
# see above
if self.health == JobHealth.ERROR:
returnValue(JobState.ERROR)
states = []
for (result, state) in request.result:
if result == True:
states.append(int(json.loads(state)))
jobState = AggregateJobResults._aggregateState(states)
if jobState == JobState.COMPLETE:
self._jobIsFinished()
returnValue(jobState)
@inlineCallbacks
def results(self, shortResults):
request = self._jobOp("jobResults", shortResults)
yield request
if self.health == JobHealth.ERROR:
returnValue(False)
results = request.result
aggregateResults = AggregateJobResults()
# decode all json. for speed's sake, try once as a list comprehension, but if one is
# false then the list comprehension will fail -- fall back to a for loop,
# rejecting failed responses
decodedResults = []
try:
decodedResults = \
[(status, JobResults(json.loads(result))) for (status, result) in results]
except:
log.debug("Could not decode all results. results: %s" % results)
for (status, result) in results:
if status == True:
decodedResults.append(
(status, JobResults(json.loads(result))))
# set the job ID to the master's job ID
aggregateResults.job_id = self.jobId
# combine and add results from all the slave servers. this
# aggregates things like bytes transferred, requests completed, etc.
aggregateResults.aggregate(
[result for (status, result) in decodedResults],
self.jobSpec.statsInterval, shortResults)
# if we're doing no stats, cut out results_byTime complete
if shortResults == True:
try:
del (aggregateResults.results_byTime)
except AttributeError:
pass
# if the job is unhealthy, stop the whole thing, and overwrite whatever the
# merged state was
if self.health == JobHealth.ERROR:
aggregateResults.job_state = JobState.ERROR
# if the job is done, cancel the timer
if aggregateResults.job_state == JobState.COMPLETE:
self._jobIsFinished()
returnValue(aggregateResults)
current[u'sensors'].append({
u'id': sensor,
u'value': value,
u'unit': unit,
u'status': status
})
self._last_value = current
returnValue(self._last_value)
if __name__ == '__main__':
from pprint import pprint
from twisted.internet import reactor
from twisted.internet.task import LoopingCall
monitor = HWMonitor()
@inlineCallbacks
def loop():
res = yield monitor.poll()
pprint(res)
lc = LoopingCall(loop)
lc.start(10.)
reactor.run()
class Graph_PyQtGraph(QtGui.QWidget):
def __init__(self, config, reactor, cxn=None, parent=None):
super(Graph_PyQtGraph, self).__init__(parent)
from labrad.units import WithUnit as U
self.U = U
self.cxn = cxn
self.pv = self.cxn.parametervault
self.reactor = reactor
self.artists = {}
self.should_stop = False
self.name = config.name
self.vline_name = config.vline
self.vline_param = config.vline_param
self.hline_name = config.hline
self.hline_param = config.hline_param
self.show_points = config.show_points
self.grid_on = config.grid_on
self.scatter_plot = config.scatter_plot
self.dataset_queue = Queue.Queue(config.max_datasets)
self.live_update_loop = LoopingCall(self.update_figure)
self.live_update_loop.start(0)
colors = ['r', 'g', 'y', 'c', 'm', 'w']
self.colorChooser = itertools.cycle(colors)
self.initUI()
@inlineCallbacks
def initUI(self):
self.tracelist = TraceList(self)
self.pw = pg.PlotWidget()
if self.vline_name:
self.inf = pg.InfiniteLine(movable=True,
angle=90,
label=self.vline_name + '{value:0.0f}',
labelOpts={
'position': 0.9,
'color': (200, 200, 100),
'fill': (200, 200, 200, 50),
'movable': True
})
init_value = yield self.get_init_vline()
self.inf.setValue(init_value)
self.inf.setPen(width=5.0)
if self.hline_name:
self.inf = pg.InfiniteLine(movable=True,
angle=0,
label=self.hline_name + '{value:0.0f}',
labelOpts={
'position': 0.9,
'color': (200, 200, 100),
'fill': (200, 200, 200, 50),
'movable': True
})
init_value = yield self.get_init_hline()
self.inf.setValue(init_value)
self.inf.setPen(width=5.0)
self.coords = QtGui.QLabel('')
self.title = QtGui.QLabel(self.name)
frame = QtGui.QFrame()
splitter = QtGui.QSplitter()
splitter.addWidget(self.tracelist)
hbox = QtGui.QHBoxLayout()
vbox = QtGui.QVBoxLayout()
vbox.addWidget(self.title)
vbox.addWidget(self.pw)
vbox.addWidget(self.coords)
frame.setLayout(vbox)
splitter.addWidget(frame)
hbox.addWidget(splitter)
self.setLayout(hbox)
#self.legend = self.pw.addLegend()
self.tracelist.itemChanged.connect(self.checkboxChanged)
self.pw.plot([], [])
vb = self.pw.plotItem.vb
self.img = pg.ImageItem()
vb.addItem(self.img)
if self.vline_name:
vb.addItem(self.inf)
self.inf.sigPositionChangeFinished.connect(self.vline_changed)
if self.hline_name:
vb.addItem(self.inf)
self.inf.sigPositionChangeFinished.connect(self.hline_changed)
self.pw.scene().sigMouseMoved.connect(self.mouseMoved)
self.pw.sigRangeChanged.connect(self.rangeChanged)
def getItemColor(self, color):
color_dict = {
"r": QtGui.QColor(QtCore.Qt.red).lighter(130),
"g": QtGui.QColor(QtCore.Qt.green),
"y": QtGui.QColor(QtCore.Qt.yellow),
"c": QtGui.QColor(QtCore.Qt.cyan),
"m": QtGui.QColor(QtCore.Qt.magenta).lighter(120),
"w": QtGui.QColor(QtCore.Qt.white)
}
return color_dict[color]
def update_figure(self):
for ident, params in self.artists.iteritems():
if params.shown:
try:
ds = params.dataset
index = params.index
current_update = ds.updateCounter
if params.last_update < current_update:
x = ds.data[:, 0]
y = ds.data[:, index + 1]
params.last_update = current_update
params.artist.setData(x, y)
except:
pass
def add_artist(self, ident, dataset, index, no_points=False):
'''
no_points is an override parameter to the global show_points setting.
It is to allow data fits to be plotted without points
'''
new_color = self.colorChooser.next()
if self.show_points and not no_points:
line = self.pw.plot([], [],
symbol='o',
symbolBrush=self.getItemColor(new_color),
name=ident,
pen=self.getItemColor(new_color),
connect=self.scatter_plot)
else:
line = self.pw.plot([], [],
pen=self.getItemColor(new_color),
name=ident)
if self.grid_on:
self.pw.showGrid(x=True, y=True)
self.artists[ident] = artistParameters(line, dataset, index, True)
self.tracelist.addTrace(ident, new_color)
def remove_artist(self, ident):
try:
artist = self.artists[ident].artist
self.pw.removeItem(artist)
#self.legend.removeItem(ident)
self.tracelist.removeTrace(ident)
self.artists[ident].shown = False
try:
del self.artists[ident]
except KeyError:
pass
except:
print "remove failed"
def display(self, ident, shown):
try:
artist = self.artists[ident].artist
if shown:
self.pw.addItem(artist)
self.artists[ident].shown = True
else:
self.pw.removeItem(artist)
#self.legend.removeItem(ident)
self.artists[ident].shown = False
except KeyError:
raise Exception('404 Artist not found')
def checkboxChanged(self):
for ident, item in self.tracelist.trace_dict.iteritems():
try:
if item.checkState() and not self.artists[ident].shown:
self.display(ident, True)
if not item.checkState() and self.artists[ident].shown:
self.display(ident, False)
except KeyError: # this means the artist has been deleted.
pass
def rangeChanged(self):
lims = self.pw.viewRange()
self.pointsToKeep = lims[0][1] - lims[0][0]
self.current_limits = [lims[0][0], lims[0][1]]
@inlineCallbacks
def add_dataset(self, dataset):
try:
self.dataset_queue.put(dataset, block=False)
except Queue.Full:
remove_ds = self.dataset_queue.get()
self.remove_dataset(remove_ds)
self.dataset_queue.put(dataset, block=False)
labels = yield dataset.getLabels()
for i, label in enumerate(labels):
self.add_artist(label, dataset, i)
@inlineCallbacks
def remove_dataset(self, dataset):
labels = yield dataset.getLabels()
for label in labels:
self.remove_artist(label)
def set_xlimits(self, limits):
self.pw.setXRange(limits[0], limits[1])
self.current_limits = limits
def set_ylimits(self, limits):
self.pw.setYRange(limits[0], limits[1])
def mouseMoved(self, pos):
#print "Image position:", self.img.mapFromScene(pos)
pnt = self.img.mapFromScene(pos)
string = '(' + str(pnt.x()) + ' , ' + str(pnt.y()) + ')'
self.coords.setText(string)
@inlineCallbacks
def get_init_vline(self):
init_vline = yield self.pv.get_parameter(self.vline_param[0],
self.vline_param[1])
returnValue(init_vline)
@inlineCallbacks
def get_init_hline(self):
init_hline = yield self.pv.get_parameter(self.hline_param[0],
self.hline_param[1])
returnValue(init_hline)
@inlineCallbacks
def vline_changed(self, sig):
val = self.inf.value()
param = yield self.pv.get_parameter(self.vline_param[0],
self.vline_param[1])
units = param.units
val = self.U(val, units)
yield self.pv.set_parameter(self.vline_param[0], self.vline_param[1],
val)
@inlineCallbacks
def hline_changed(self, sig):
val = self.inf.value()
param = yield self.pv.get_parameter(self.hline_param[0],
self.hline_param[1])
units = param.units
val = self.U(val, units)
yield self.pv.set_parameter(self.hline_param[0], self.hline_param[1],
val)
"""
def makeService(self, options):
try:
logger.info('Initializing configuration')
try:
self._init_config(options)
except Exception as e:
logger.error(
"Aborting service startup due to configuration error: {}".
format(e))
raise e
logger.info('Initializing logging')
self._init_logging()
self._check_enabled()
#logger.enable_bootstrap_logging(self.tapname)
assert (issubclass(self.service_cls, ApiService))
self.anchore_service = self.service_cls(options=options)
self.anchore_service.initialize(self.global_configuration)
self.resource_nodes[
b'version'].versions = self.anchore_service.versions
# application object
application = service.Application(
"Service-" + '-'.join(self.anchore_service.name))
self.twistd_service = service.MultiService()
self.twistd_service.setServiceParent(application)
if self.anchore_service.task_handlers_enabled:
logger.info('Starting monitor thread')
lc = self._get_api_monitor(self.anchore_service)
lc.start(1)
else:
logger.warn(
'Skipped start of monitor threads due to task_handlers_enabled=false in config, or found ANCHORE_ENGINE_DISABLE_MONITORS in env'
)
thread_stats_interval = int(
self.service_config.get('debug_thread_stats_dump_interval', 0))
if thread_stats_interval > 0:
logger.info(
'Based on service config, starting the thread stats dumper'
)
monitor = LoopingCall(dump_stats)
monitor.start(thread_stats_interval)
logger.info('Building api handlers')
s = self._build_api_service()
s.setServiceParent(self.twistd_service)
if enable_dangerous_debug_cli:
logger.warn(
'Loading *dangerous* debug/telnet service as specified by debug config'
)
self.makeDebugCLIService({
'protocolFactory': ColoredManhole,
'protocolArgs': (None, ),
'telnet': 6023,
}).setServiceParent(self.twistd_service)
return self.twistd_service
except Exception as err:
logger.exception("cannot create/init/register service: " +
self.service_cls.__service_name__ +
" - exception: " + str(err))
raise Exception("cannot start service (see above for information)")
finally:
pass
def start(self):
partitions = []
try:
while not partitions:
yield self._client.load_metadata_for_topics(self.topic)
#self._client.load_metadata_for_topics(self.topic)
e = self._client.metadata_error_for_topic(self.topic)
if e:
log.warning('no-metadata-for-topic',
error=e,
topic=self.topic)
else:
partitions = self._client.topic_partitions[self.topic]
break
time.sleep(20)
except KafkaUnavailableError:
log.error("unable-to-communicate-with-Kafka-brokers")
self.stop()
def _note_consumer_stopped(result, consumer):
log.info('consumer-stopped', consumer=consumer, result=result)
for partition in partitions:
c = Consumer(self._client, self.topic, partition,
self.msg_processor)
self._consumer_list.append(c)
log.info('consumer-started', topic=self.topic, partition=partition)
d = c.start(OFFSET_LATEST)
d.addBoth(_note_consumer_stopped, c)
self._consumer_d_list.append(d)
# Now read the list of existing dashboards from Grafana and create the
# dictionary of dashboard timers. If we've crashed there will be
# dashboards there. Just add them and if they're no longer valid
# they'll be deleted. If they are valid then they'll persist.
#print("Starting main loop")
try:
retrys = 10
while True:
r = requests.get(self.grafana_url + "/datasources")
if r.status_code == requests.codes.ok:
break
else:
retrys -= 1
if retrys == 0:
log.error("unable-to-communicate-with-grafana")
self.stop()
time.sleep(10)
j = r.json()
data_source = False
for i in j:
if i["name"] == "Voltha Stats":
data_source = True
break
if not data_source:
r = requests.post(self.grafana_url + "/datasources",
data={
"name": "Voltha Stats",
"type": "graphite",
"access": "proxy",
"url": "http://localhost:81"
})
log.info('data-source-added',
status=r.status_code,
text=r.text)
retrys = 10
while True:
r = requests.get(self.grafana_url + "/search?")
if r.status_code == requests.codes.ok:
break
else:
retrys -= 1
if retrys == 0:
log.error("unable-to-communicate-with-grafana")
self.stop()
time.sleep(10)
j = r.json()
for i in j:
# Look for dashboards that have a title of *olt.[[:hexidgit:]].
# These will be the ones of interest. Others should just be left
# alone.
#print(i['title'])
match = re.search(r'(.*olt)\.([0-9a-zA-Z]+)', i['title'])
if match and match.lastindex > 0:
#print(match.group(1), match.group(2))
self.dash_meta[match.group(2)] = {}
self.dash_meta[match.group(
2)]['timer'] = self.timer_duration # 10 min
self.dash_meta[match.group(2)]['device'] = match.group(1)
self.dash_meta[match.group(2)]['created'] = False
self.dash_meta[match.group(2)]['ports'] = {}
# TODO: We should really capture all of the chart data
# including the rows, panels, and data points being logged.
# This is good enough for now though to determine if
# there's already a dashboard for a given device.
def countdown_processor():
# Called every X (timer_resolution) seconds to count down each of the
# dash timers. If a timer reaches 0 the corresponding
# dashboard is removed.
#log.info("Counting down.")
try:
for dashboard in self.dash_meta.keys():
#print("Counting down %s." %dashboard)
# Issue a log if the counter decrement is somewhat relevant
if(self.dash_meta[dashboard]['timer'] % 100 == 0 and \
self.dash_meta[dashboard]['timer'] != self.timer_duration):
log.info("counting-down",
dashboard=dashboard,
timer=self.dash_meta[dashboard]['timer'])
self.dash_meta[dashboard][
'timer'] -= self.timer_resolution
if self.dash_meta[dashboard]['timer'] <= 0:
# Delete the dashboard here
log.info("FIXME:-Should-delete-the-dashboard-here",
dashboard=dashboard)
pass
except:
e = sys.exc_info()
log.error("error", error=e)
# Start the dashboard countdown processor
log.info("starting-countdown-processor")
lc = LoopingCall(countdown_processor)
lc.start(self.timer_resolution)
@inlineCallbacks
def template_checker():
try:
# Called every so often (timer_resolution seconds because it's
# convenient) to check if a template dashboard has been defined
# in Grafana. If it has been, replace the built in template
# with the one provided
r = requests.get(self.grafana_url +
"/search?query=template")
db = r.json()
if len(db) == 1:
# Apply the template
yield self.dash_template.apply_template(db[0])
elif len(db) != 0:
# This is an error, log it.
log.warning("More-than-one-template-provided-ignoring")
except:
e = sys.exc_info()
log.error("error", error=e)
log.info("starting-template-checker")
lc = LoopingCall(template_checker)
lc.start(self.timer_resolution)
except:
e = sys.exc_info()
log.error("error", error=e)
class ReadRequestLoadScenario(object):
"""
A scenario that places load on the cluster by performing read
requests at a specified rate.
:ivar reactor: Reactor to use.
:ivar cluster: `BenchmarkCluster` containing the control service.
:ivar request_rate: The target number of requests per second.
:ivar sample_size: The number of samples to collect when measuring
the rate.
:ivar timeout: Maximum time in seconds to wait for the requested
rate to be reached.
"""
def __init__(self,
reactor,
cluster,
request_rate=10,
sample_size=DEFAULT_SAMPLE_SIZE,
timeout=45):
self._maintained = Deferred()
self.reactor = reactor
self.control_service = cluster.get_control_service(reactor)
self.request_rate = request_rate
self.timeout = timeout
self.rate_measurer = RateMeasurer(sample_size)
self.max_outstanding = 10 * request_rate
# Send requests per second
self.loop = LoopingCall.withCount(self._request_and_measure)
self.loop.clock = self.reactor
self.monitor_loop = LoopingCall(self.check_rate)
self.monitor_loop.clock = self.reactor
def _request_and_measure(self, count):
"""
Update the rate with the current value and send `request_rate`
number of new requests.
:param count: The number of seconds passed since the last time
`_request_and_measure` was called.
"""
for i in range(count):
self.rate_measurer.update_rate()
def handle_request_error(result):
self.rate_measurer.request_failed(result)
write_failure(result)
for i in range(self.request_rate):
d = self.control_service.list_nodes()
self.rate_measurer.request_sent()
d.addCallbacks(self.rate_measurer.response_received,
errback=handle_request_error)
def _fail(self, exception):
"""
Fail the scenario. Stop the monitor loop and throw the
error.
"""
self.monitor_loop.stop()
self._maintained.errback(exception)
def check_rate(self):
"""
Verify that the rate hasn't decreased and that the scenario is
not overloaded. A scenario is overloaded if there are too many
outstanding requests.
:raise: `RequestRateTooLow` if the rate has dropped.
:raise: `RequestOverload` if the scenario is overloaded.
"""
rate = self.rate_measurer.rate()
if rate < self.request_rate:
self._fail(RequestRateTooLow(rate))
if self.rate_measurer.outstanding() > self.max_outstanding:
self._fail(RequestOverload())
def start(self):
"""
:return: A Deferred that fires when the desired scenario is
established (e.g. that a certain load is being applied).
"""
self.loop.start(interval=1)
def reached_target_rate():
return self.rate_measurer.rate() >= self.request_rate
def handle_timeout(failure):
failure.trap(CancelledError)
raise RequestRateNotReached
waiting_for_target_rate = loop_until(self.reactor, reached_target_rate,
repeat(1))
timeout(self.reactor, waiting_for_target_rate, self.timeout)
waiting_for_target_rate.addErrback(handle_timeout)
# Start monitoring the scenario as soon as the target rate is reached.
def monitor_scenario_status(result):
self.monitor_loop.start(interval=1)
waiting_for_target_rate.addCallback(monitor_scenario_status)
return waiting_for_target_rate
def maintained(self):
"""
:return: A Deferred that fires with an errback if the desired
scenario fails to hold between being established and being
stopped. This Deferred never fires with a callback.
"""
return self._maintained
def stop(self):
"""
Stop the scenario from being maintained by stopping all the
loops that may be executing.
:return: A Deferred that fires when the scenario has stopped.
"""
if self.monitor_loop.running:
self.monitor_loop.stop()
if self.loop.running:
self.loop.stop()
outstanding_requests = self.rate_measurer.outstanding()
if outstanding_requests > 0:
msg = (
"There are {num_requests} outstanding requests. "
"Waiting {num_seconds} seconds for them to complete.").format(
num_requests=outstanding_requests,
num_seconds=self.timeout)
Message.log(key='outstanding_requests', value=msg)
with start_action(action_type=u'flocker:benchmark:scenario:stop',
scenario='read_request_load'):
def handle_timeout(failure):
failure.trap(CancelledError)
msg = ("Force stopping the scenario. "
"There are {num_requests} outstanding requests").format(
num_requests=outstanding_requests)
Message.log(key='force_stop_request', value=msg)
def no_outstanding_requests():
return self.rate_measurer.outstanding() == 0
scenario_stopped = loop_until(self.reactor,
no_outstanding_requests, repeat(1))
timeout(self.reactor, scenario_stopped, self.timeout)
scenario_stopped.addErrback(handle_timeout)
scenario = DeferredContext(scenario_stopped)
scenario.addActionFinish()
return scenario.result
class ServerEventManager(pb.Root):
def __init__(self):
self.clients = {} #organized {client_id: client_obj}
self.total_clients = 0
self.world_lists = WordManager(
"free") #a dict with the words preloaded
self.lobbys = {}
self.total_game_requests = 0
self.looping_call = LoopingCall(self.remote_post, e.CopyableEvent())
self.looping_call.start(5.0)
def dead_client(self, client):
"""
removes client from dict of clients (self.clients)
"""
del self.clients[client.client_id]
def remote_get_word_list(self, word_list_name):
return self.world_lists[word_list_name]
def remote_register_client(self, root_obj, client_nickname):
"""
returns True if is accepted, returns False
returns emtpy string (False) if client passed emtpy string
(figured it might help later. Either way it evaluates to false)
"""
if client_nickname == "":
return ""
client_id = "Client" + str(self.total_clients)
self.total_clients = self.total_clients + 1
client = Client(root_obj, client_id, client_nickname)
client.observers.append(self) #add self as observer
self.clients[client_id] = client
print client_id + " : ///nickname: " + client_nickname
return client_id
def remote_get_unique_game_id(self):
"""
not <i>guaranteed</i> to be unique. But in practice
it will be. Worst case scenario user gets a "game name in
use" popup
"""
game_name = "Game" + str(self.total_game_requests)
if game_name in self.lobbys:
for i in xrange(26):
letter_name = game_name + chr(97 + i)
if letter_name not in self.lobbys:
return letter_name
self.total_game_requests = self.total_game_requests + 1
return game_name #sorry bud, out of luck
def remote_make_game_lobby(self, lobby_id, word_list, score_system_mode,
num_teams, round_time, leeway_time):
"""
returns false if lobby_id is in use. Assumes word_list_id
will be correct. Don't f**k me over me.
"""
if lobby_id in self.lobbys:
return (False, "lobby id in use")
elif not word_list:
return (False, "word list is empty")
else:
if isinstance(word_list, str): #if it wasn't a local copy.
word_list = self.world_lists[word_list]
self.lobbys[lobby_id] = Lobby(self, word_list, lobby_id,
score_system_mode, num_teams,
round_time, leeway_time)
return (True, "success")
def remote_join_lobby(self, client_id, lobby_id):
"""
returns the lobby (a root_obj) if success, else returns None
"""
#note: game is set to None before game starts => check game to see
#if game has started
if lobby_id not in self.lobbys or self.lobbys[lobby_id].game:
return None
lobby = self.lobbys[lobby_id]
lobby.new_player(self.clients[client_id])
return lobby
def remote_join_lobby_score(self, client_id, lobby_id):
if lobby_id not in self.lobbys or self.lobbys[lobby_id].game:
return None
lobby = self.lobbys[lobby_id]
lobby.new_player_score(self.clients[client_id])
return lobby
def remote_get_word_list_options(self):
"""
returns name of all the word lists
"""
return self.world_lists.keys()
#untested
def remote_post(self, event, client_id=""):
"""
attempts to notify all users of event. removes them if they
are dead
"""
#event.originator = client_id
#print "Recieved ", event.name, " from ", client_id
for client in self.clients.values(
): #don't itervalues; we're possibly changing dictionary
try:
client.root_obj.callRemote('notify', event)
except pb.DeadReferenceError:
print "calling client.remove_client() on: ", client.client_id, client.nickname
client.remove_client()
class BlobAvailabilityTracker:
"""
Class to track peer counts for known blobs, and to discover new popular blobs
Attributes:
availability (dict): dictionary of peers for known blobs
"""
def __init__(self, blob_manager, peer_finder, dht_node):
self.availability = {}
self._last_mean_availability = Decimal(0.0)
self._blob_manager = blob_manager
self._peer_finder = peer_finder
self._dht_node = dht_node
self._check_mine = LoopingCall(self._update_mine)
def start(self):
log.info("Starting blob availability tracker.")
self._check_mine.start(600)
def stop(self):
log.info("Stopping blob availability tracker.")
if self._check_mine.running:
self._check_mine.stop()
def get_blob_availability(self, blob, timeout=None):
def _get_peer_count(peers):
have_blob = sum(1 for peer in peers if peer.is_available())
return {blob: have_blob}
d = self._peer_finder.find_peers_for_blob(blob, timeout)
d.addCallback(_get_peer_count)
return d
def get_availability_for_blobs(self, blobs, timeout=None):
dl = [
self.get_blob_availability(blob, timeout) for blob in blobs if blob
]
d = defer.DeferredList(dl)
d.addCallback(
lambda results: [val for success, val in results if success])
return d
@property
def last_mean_availability(self):
return max(Decimal(0.01), self._last_mean_availability)
def _update_peers_for_blob(self, blob):
def _save_peer_info(blob_hash, peers):
v = {blob_hash: peers}
self.availability.update(v)
return v
d = self._peer_finder.find_peers_for_blob(blob)
d.addCallback(
lambda r: [[c.host, c.port, c.is_available()] for c in r])
d.addCallback(lambda peers: _save_peer_info(blob, peers))
return d
def _update_mine(self):
def _get_peers(blobs):
dl = []
for hash in blobs:
dl.append(self._update_peers_for_blob(hash))
return defer.DeferredList(dl)
def sample(blobs):
return random.sample(blobs, min(len(blobs), 10))
start = time.time()
log.debug('==> Updating the peers for my blobs')
d = self._blob_manager.get_all_verified_blobs()
# as far as I can tell, this only is used to set _last_mean_availability
# which... seems like a very expensive operation for such little payoff.
# so taking a sample should get about the same effect as querying the entire
# list of blobs
d.addCallback(sample)
d.addCallback(_get_peers)
d.addCallback(lambda _: self._set_mean_peers())
d.addCallback(lambda _: log.debug(
'<== Done updating peers for my blobs. Took %s seconds',
time.time() - start))
# although unused, need to return or else the looping call
# could overrun on a previous call
return d
def _set_mean_peers(self):
num_peers = [
len(self.availability[blob]) for blob in self.availability
]
mean = Decimal(sum(num_peers)) / Decimal(max(1, len(num_peers)))
self._last_mean_availability = mean
class SyncFolder(PatternMatchingEventHandler):
def __init__(self,
local_dir,
remote_dircap,
tahoe=None,
ignore_patterns=None):
_ignore_patterns = [
'*.gridsync-versions*', '*.part*',
'*(conflicted copy *-*-* *-*-*)*'
]
if ignore_patterns:
_ignore_patterns += ignore_patterns
super(SyncFolder, self).__init__(ignore_patterns=_ignore_patterns)
if not tahoe:
from gridsync.tahoe import Tahoe
tahoe = Tahoe()
self.tahoe = tahoe
self.local_dir = os.path.expanduser(local_dir)
self.remote_dircap = remote_dircap
self.versions_dir = os.path.join(self.local_dir, '.gridsync-versions')
self.local_snapshot = 0
self.filesystem_modified = False
self.do_backup = False
self.do_sync = False
self.sync_state = 0
self.sync_log = []
self.keep_versions = 1
self.local_checker = LoopingCall(self.check_for_changes)
self.remote_checker = LoopingCall(reactor.callInThread, self.sync)
logging.debug("{} initialized; "
"{} <-> {}".format(self, self.local_dir,
self.remote_dircap))
def on_modified(self, event):
self.filesystem_modified = True
#try:
# reactor.callFromThread(self.local_checker.start, 1)
#except AssertionError:
# return
reactor.callFromThread(self._start_local_checker)
def _start_local_checker(self):
# XXX: For some (qt5reactor-related?) reason, the AssertionError
# raised by trying to start the (already-running) local_checker timer
# above won't catch if called from reactor.callFromThread. Why is this?
try:
self.local_checker.start(1)
except AssertionError:
return
def check_for_changes(self):
if self.filesystem_modified:
self.filesystem_modified = False
else:
reactor.callFromThread(self.local_checker.stop)
reactor.callInThread(self.sync, force_backup=True)
def start(self):
try:
self.remote_dircap_alias = self.tahoe.aliasify(self.remote_dircap)
except ValueError:
# TODO: Alert user alias is garbled?
pass
except LookupError:
# TODO: Alert user alias is missing?
pass
logging.info("Starting Observer in {}...".format(self.local_dir))
try:
self.observer = Observer()
self.observer.schedule(self, self.local_dir, recursive=True)
self.observer.start()
except Exception as error:
logging.error(error)
reactor.callFromThread(self.remote_checker.start, 30)
def _create_conflicted_copy(self, filepath):
base, extension = os.path.splitext(filepath)
mtime = int(os.path.getmtime(filepath))
t = datetime.datetime.fromtimestamp(mtime)
tag = t.strftime('.(conflicted copy %Y-%m-%d %H-%M-%S)')
tagged_filepath = base + tag + extension
logging.debug("Creating conflicted copy of {} {}".format(
filepath, tagged_filepath))
os.rename(filepath, tagged_filepath)
os.utime(tagged_filepath, (-1, mtime))
def _create_versioned_copy(self, filename, mtime):
local_filepath = os.path.join(self.local_dir, filename)
base, extension = os.path.splitext(filename)
t = datetime.datetime.fromtimestamp(mtime)
tag = t.strftime('.(%Y-%m-%d %H-%M-%S)')
newname = base + tag + extension
versioned_filepath = newname.replace(self.local_dir, self.versions_dir)
if not os.path.isdir(os.path.dirname(versioned_filepath)):
os.makedirs(os.path.dirname(versioned_filepath))
logging.info("Creating {}".format(versioned_filepath))
shutil.copy2(local_filepath, versioned_filepath)
def get_local_metadata(self, basedir=None):
metadata = {}
if not basedir:
basedir = self.local_dir
for root, dirs, files in os.walk(basedir, followlinks=True):
for name in dirs:
path = os.path.join(root, name)
if not path.startswith(self.versions_dir):
metadata[path] = {}
for name in files:
path = os.path.join(root, name)
if not path.startswith(self.versions_dir):
metadata[path] = {
'mtime': int(os.path.getmtime(path)),
'size': os.path.getsize(path)
}
return metadata
def get_remote_metadata(self, dircap, basedir=''):
metadata = {}
jobs = []
logging.debug("Getting remote metadata from {}...".format(dircap))
url = '{}uri/{}/?t=json'.format(self.tahoe.node_url, dircap)
received_data = requests.get(url).json()
for filename, data in received_data[1]['children'].items():
path = '/'.join([basedir, filename]).strip('/')
metadata[path] = {
'uri': data[1]['ro_uri'],
'mtime': int(data[1]['metadata']['mtime'])
}
if data[0] == 'dirnode':
jobs.append(
deferToThread(self.get_remote_metadata,
'/'.join([dircap, filename]), path))
results = blockingCallFromThread(reactor, gatherResults, jobs)
for result in results:
metadata.update(result)
return metadata
def sync(self, snapshot=None, force_backup=False):
if self.sync_state:
logging.debug("Sync already in progress; queueing to end...")
self.do_sync = True
return
if not snapshot:
try:
ls = self.tahoe.ls(self.remote_dircap)
if not ls:
logging.debug("No /Archives found; "
"performing (first?) backup...")
self.sync_state += 1
self.backup(self.local_dir, self.remote_dircap_alias)
self.sync_complete(ls)
return
except Exception as error:
logging.error(error)
return
# XXX: It might be preferable to just check the dircap of /Latest/
pre_sync_archives = self.tahoe.ls(self.remote_dircap + "/Archives")
available_snapshot = pre_sync_archives[-1]
if self.local_snapshot == available_snapshot:
if force_backup:
self.sync_state += 1
self.backup(self.local_dir, self.remote_dircap_alias)
self.sync_complete(pre_sync_archives)
return
else:
snapshot = available_snapshot
remote_path = self.remote_dircap + '/Archives/' + snapshot
logging.info("Syncing {} with {}...".format(self.local_dir, snapshot))
self.sync_state += 1
self.local_metadata = self.get_local_metadata(self.local_dir)
self.remote_metadata = self.get_remote_metadata(remote_path)
# TODO: If tahoe.get_metadata() fails or doesn't contain a
# valid snapshot, jump to backup?
jobs = []
for file, metadata in self.remote_metadata.items():
if metadata['uri'].startswith('URI:DIR'):
dirpath = os.path.join(self.local_dir, file)
if not os.path.isdir(dirpath):
logging.info("Creating directory: {}...".format(dirpath))
os.makedirs(dirpath)
for file, metadata in self.remote_metadata.items():
if not metadata['uri'].startswith('URI:DIR'):
filepath = os.path.join(self.local_dir, file)
remote_mtime = metadata['mtime']
if filepath in self.local_metadata:
local_filesize = self.local_metadata[filepath]['size']
local_mtime = self.local_metadata[filepath]['mtime']
if local_mtime < remote_mtime:
logging.debug("[<] {} is older than remote version; "
"downloading {}...".format(file, file))
if self.keep_versions:
self._create_versioned_copy(filepath, local_mtime)
jobs.append(
deferToThread(self.download,
remote_path + '/' + file, filepath,
remote_mtime))
elif local_mtime > remote_mtime:
logging.debug("[>] {} is newer than remote version; "
"backup scheduled".format(file))
self.do_backup = True
else:
logging.debug("[.] {} is up to date.".format(file))
else:
logging.debug("[?] {} is missing; "
"downloading {}...".format(file, file))
jobs.append(
deferToThread(self.download, remote_path + '/' + file,
filepath, remote_mtime))
for file, metadata in self.local_metadata.items():
fn = file.split(self.local_dir + os.path.sep)[1]
if fn not in self.remote_metadata:
if metadata:
recovery_uri = self.tahoe.stored(file, metadata['size'],
metadata['mtime'])
if recovery_uri:
logging.debug("[x] {} removed from latest snapshot; "
"deleting local file...".format(file))
if self.keep_versions:
self._create_versioned_copy(file, local_mtime)
try:
os.remove(file)
except Exception as error:
logging.error(error)
else:
logging.debug("[!] {} isn't stored; "
"backup scheduled".format(fn))
self.do_backup = True
blockingCallFromThread(reactor, gatherResults, jobs)
if self.do_backup:
self.backup(self.local_dir, self.remote_dircap_alias)
self.do_backup = False
if self.do_sync:
self.sync()
self.sync_complete(pre_sync_archives)
def sync_complete(self, pre_sync_archives):
post_sync_archives = self.tahoe.ls(self.remote_dircap + "/Archives")
if len(post_sync_archives) - len(pre_sync_archives) <= 1:
self.local_snapshot = post_sync_archives[-1]
logging.info("Synchronized {} with {}".format(
self.local_dir, self.local_snapshot))
else:
logging.warn("Remote state changed during sync")
# TODO: Re-sync/merge overlooked snapshot
self.sync_state -= 1
def download(self, remote_uri, local_filepath, mtime=None):
url = self.tahoe.node_url + 'uri/' + remote_uri
download_path = local_filepath + '.part'
#if os.path.exists(download_path):
# XXX: Resuming may not be a good idea, as the existent (local)
# parts may no longer be present in the latest (remote) version of
# the file. Perhaps an integrity/filecap check should be required?
# size = os.path.getsize(download_path)
# logging.debug("Partial download of {} found; resuming byte {}..."\
# .format(local_filepath, size))
# request.headers['Range'] = 'bytes={}-'.format(size)
# TODO: Handle exceptions..
if os.path.isfile(download_path) or os.path.isdir(download_path):
raise OSError("File exists: '{}'".format(download_path))
r = requests.get(url, stream=True)
r.raise_for_status()
recv = 0
with open(download_path, 'wb') as f:
for chunk in r.iter_content(4096):
f.write(chunk)
recv += len(chunk)
if os.path.isfile(local_filepath):
local_filesize = os.path.getsize(local_filepath)
if not self.tahoe.stored(file, local_filesize, mtime):
self._create_conflicted_copy(local_filepath)
os.rename(download_path, local_filepath)
if mtime:
os.utime(local_filepath, (-1, mtime))
self.sync_log.append("Downloaded {}".format(
local_filepath.lstrip(self.local_dir)))
return recv
def backup(self, local_dir, remote_dircap):
excludes = ['--exclude=' + x for x in self.ignore_patterns]
output = self.tahoe.command(['backup', '-v'] + excludes +
[local_dir, remote_dircap])
for line in output.split('\n'):
if line.startswith('uploading'):
filename = line[11:][:-3].lstrip(self.local_dir)
self.sync_log.append("Uploaded {}".format(filename))
def stop(self):
logging.info("Stopping Observer in {}...".format(self.local_dir))
try:
self.observer.stop()
self.observer.join()
except Exception as error:
logging.error(error)
self.remote_checker.stop()
