class NetworkSpeedDaemon(VMCDaemon):
"""
I enque fake SIG_SPEED UnsolicitedNotifications
"""
def __init__(self, frequency, device, notification_manager):
super(NetworkSpeedDaemon, self).__init__(frequency, device, notification_manager)
self.netspeed = NetworkSpeed()
def start(self):
log.msg("DAEMONS: DAEMON %s started..." % self.__class__.__name__)
self.netspeed.start()
self.loop = LoopingCall(self.function)
self.loop.start(self.frequency)
def stop(self):
log.msg("DAEMONS: DAEMON %s stopped..." % self.__class__.__name__)
self.loop.stop()
self.netspeed.stop()
def function(self):
up, down = self.netspeed["up"], self.netspeed["down"]
n = N.UnsolicitedNotification(N.SIG_SPEED, bps_to_human(up, down))
self.manager.on_notification_received(n)
class LeaseService(Service):
"""
Manage leases.
In particular, clear out expired leases once a second.
:ivar _reactor: A ``twisted.internet.reactor`` implementation.
:ivar _persistence_service: The persistence service to act with.
:ivar _lc: A ``twisted.internet.task.LoopingCall`` run every second
to update the configured leases by releasing leases that have
expired.
"""
def __init__(self, reactor, persistence_service):
if reactor is None:
reactor = default_reactor
self._reactor = reactor
self._persistence_service = persistence_service
def startService(self):
self._lc = LoopingCall(self._expire)
self._lc.clock = self._reactor
self._lc.start(1)
def stopService(self):
self._lc.stop()
def _expire(self):
now = datetime.now(tz=UTC)
return update_leases(lambda leases: leases.expire(now),
self._persistence_service)
class USBClient(Protocol):
def __init__(self, network):
self.network = network
self.usb_list = []
self.lc = LoopingCall(self.timerEvent)
self.serialBuffer = bytes("")
def timerEvent(self):
self.lc.stop()
for cli in client_list:
cli.transport.write(self.serialBuffer)
self.serialBuffer = bytes("")
def connectionFailed(self):
log.msg ("Connection Failed:", self)
reactor.stop()
def connectionMade(self):
usb_list.append(self)
log.msg("Connected to %s device" % device)
def dataReceived(self, data):
#log.msg("data resrvr")
self.serialBuffer += data
if self.lc.running:
self.lc.stop()
self.lc.start(0.2, False)
def sendLine(self, cmd):
print cmd
self.transport.write(cmd + "\r\n")
def outReceived(self, data):
self.data = self.data + data
class IconAnimator:
lc = None
def __init__(self, box, icons):
self.box = box
self.icons = icons
for icon in icons:
icon.show()
self.stop()
def tick(self):
self.position += 1
self.position %= len(self.icons)
c = self.box.get_children()
if c:
self.box.remove(c[0])
self.box.add(self.icons[self.position])
def start(self):
self.animating = True
self.tick()
self.lc = LoopingCall(self.tick)
self.lc.start(1.0)
def stop(self, index=0):
self.animating = False
self.position = index - 1
self.tick()
if self.lc is not None:
self.lc.stop()
self.lc = None
class Joiner: def __init__(self, screen, address, port): self.screen = screen self.address = address self.port = port def start(self): # Set up the connection between the state and the network; # using queues gives a degree of seperation between the # communication and game logic, and making them be deferred # keeps it all asynchronous. inqueue = DeferredQueue() outqueue = DeferredQueue() #Initialize GameState gs = GameState(self,self.screen,2,inqueue,outqueue) #Create Looping Call self.lc = LoopingCall(gs.gameLoop) self.lc.start(.0166666666) #Begin Connecting connfactory = CommandConnFactory(inqueue,outqueue) reactor.connectTCP(self.address,self.port,connfactory) def stop(self, nextscreen=None): # Stop the GameState logic self.lc.stop() # Start up the next screen, if there is one if nextscreen: nextscreen.start() else: reactor.stop()
class EnvisalinkClientFactory(ReconnectingClientFactory):
# we will only ever have one connection to envisalink at a time
envisalinkClient = None
_currentLoopingCall = None
def __init__(self, config):
self._config = config
def buildProtocol(self, addr):
logging.debug("%s connection estblished to %s:%s", addr.type, addr.host, addr.port)
logging.debug("resetting connection delay")
self.resetDelay()
self.envisalinkClient = EnvisalinkClient(self._config)
# check on the state of the envisalink connection repeatedly
self._currentLoopingCall = LoopingCall(self.envisalinkClient.check_alive)
self._currentLoopingCall.start(1)
return self.envisalinkClient
def startedConnecting(self, connector):
logging.debug("Started to connect to Envisalink...")
def clientConnectionLost(self, connector, reason):
logging.debug('Lost connection to Envisalink. Reason: %s', str(reason))
self._currentLoopingCall.stop()
ReconnectingClientFactory.clientConnectionLost(self, connector, reason)
def clientConnectionFailed(self, connector, reason):
logging.debug('Connection failed to Envisalink. Reason: %s', str(reason))
self._currentLoopingCall.stop()
ReconnectingClientFactory.clientConnectionFailed(self, connector,
reason)
class VOEventBroadcasterFactory(ServerFactory):
IAMALIVE_INTERVAL = 60 # Sent iamalive every IAMALIVE_INTERVAL seconds
protocol = VOEventBroadcaster
def __init__(self, local_ivo, test_interval):
# test_interval is the time in seconds between sending test events to
# the network. 0 to disable.
self.local_ivo = local_ivo
self.test_interval = test_interval
self.broadcasters = []
self.alive_loop = LoopingCall(self.sendIAmAlive)
self.test_loop = LoopingCall(self.sendTestEvent)
def startFactory(self):
self.alive_loop.start(self.IAMALIVE_INTERVAL)
if self.test_interval:
self.test_loop.start(self.test_interval)
return ServerFactory.startFactory(self)
def stopFactory(self):
self.alive_loop.stop()
if self.test_loop.running:
self.test_loop.stop()
return ServerFactory.stopFactory(self)
def sendIAmAlive(self):
log.debug("Broadcasting iamalive")
for broadcaster in self.broadcasters:
broadcaster.sendIAmAlive()
def sendTestEvent(self):
log.debug("Broadcasting test event")
test_event = broker_test_message(self.local_ivo)
for broadcaster in self.broadcasters:
broadcaster.send_event(test_event)
class Autofetcher(object):
"""
Download name claims as they occur
"""
def __init__(self, api):
self._api = api
self._checker = LoopingCall(self._check_for_new_claims)
self.best_block = None
def start(self):
reactor.addSystemEventTrigger('before', 'shutdown', self.stop)
self._checker.start(5)
def stop(self):
log.info("Stopping autofetcher")
self._checker.stop()
def _check_for_new_claims(self):
block = self._api.get_best_blockhash()
if block != self.best_block:
log.info("Checking new block for name claims, block hash: %s" % block)
self.best_block = block
transactions = self._api.get_block({'blockhash': block})['tx']
for t in transactions:
c = self._api.get_claims_for_tx({'txid': t})
if len(c):
for i in c:
log.info("Downloading stream for claim txid: %s" % t)
self._api.get({'name': t, 'stream_info': json.loads(i['value'])})
class FileConsumer(object):
_looper = None
_producer = None
def __init__(self, reactor=None, interval=0.01):
self._reactor = reactor
self._interval = interval
self._io = StringIO()
def registerProducer(self, producer, isPush):
if not isPush and not self._reactor:
raise ValueError("can't read from a pull producer with no reactor")
self._producer = producer
if not isPush:
self._looper = LoopingCall(self._pull)
self._looper.clock = self._reactor
self._looper.start(self._interval, now=False)
def unregisterProducer(self):
self._producer = None
if self._looper:
self._looper.stop()
def _pull(self):
self._producer.resumeProducing()
def write(self, data):
self._io.write(data)
def value(self):
return self._io.getvalue()
class TimerService(_VolatileDataService):
"""Service to periodically call a function
Every C{step} seconds call the given function with the given arguments.
The service starts the calls when it starts, and cancels them
when it stops.
"""
volatile = ["_loop"]
def __init__(self, step, callable, *args, **kwargs):
self.step = step
self.call = (callable, args, kwargs)
def startService(self):
service.Service.startService(self)
from twisted.internet.task import LoopingCall
callable, args, kwargs = self.call
self._loop = LoopingCall(callable, *args, **kwargs)
self._loop.start(self.step, now=True).addErrback(self._failed)
def _failed(self, why):
self._loop.running = False
log.err(why)
def stopService(self):
if self._loop.running:
self._loop.stop()
return service.Service.stopService(self)
class TimeOnlineLabel(gtk.Label):
"""A display of how long the current session has been online for."""
def __init__(self):
gtk.Label.__init__(self)
self.looping_call = LoopingCall(self.update_time)
self.start_time = None
def start_counting(self):
"""Start ticking."""
self.start_time = datetime.now()
#don't leave our display blank
self.update_time()
self.looping_call.start(0.5) #update it twice per second
def stop_counting(self):
"""We only count time online; stop counting."""
self.looping_call.stop()
def update_time(self):
"""Should tick once a second. Displays the current running count of
how long's been spent online.
"""
delta = datetime.now() - self.start_time
#chop off microseconds, for neatness
delta = timedelta(delta.days, delta.seconds)
self.set_text('Time online: %s' % delta)
class RelayProtocol(RTMProtocol):
clock = reactor
def __init__(self, session_data):
RTMProtocol.__init__(self)
self.session_data = session_data
self.bot_user_id = session_data['self']['id']
self.lc = None
self.relay = None
def onOpen(self):
self.relay = self.factory.relay
self.lc = LoopingCall(self.send_ping)
self.lc.clock = self.clock
self.lc.start(3, now=True)
def onClose(self, wasClean, code, reason):
if self.relay is not None:
self.relay.remove_protocol(self.bot_user_id)
if self.lc is not None:
self.lc.stop()
def onMessage(self, payload, isBinary):
data = json.loads(payload)
self.relay.relay(self.bot_user_id, data)
def send_ping(self):
return self.send_message({
"type": "ping",
})
def send_message(self, data):
return self.sendMessage(json.dumps(data))
class UdpSender(DatagramProtocol):
"""
Class that will send UDP packets to UDP Listener
"""
def __init__(self, host, count, size, duration):
# LoopingCall does not know whether UDP socket is actually writable
self.looper = None
self.host = host
self.count = count
self.duration = duration
self.start = time.time()
self.sent = 0
self.data = array.array('c', 'X' * size)
def startProtocol(self):
self.looper = LoopingCall(self.sendData)
period = self.duration / float(self.count)
self.looper.start(period, now=False)
def stopProtocol(self):
if (self.looper is not None):
self.looper.stop()
self.looper = None
def datagramReceived(self, data, (host, port)):
pass
class Rain(object):
"""
Make it rain.
Rain only occurs during spring.
"""
implements(IAutomaton)
blocks = tuple()
def __init__(self):
self.season_loop = LoopingCall(self.check_season)
def scan(self, chunk):
pass
def feed(self, coords):
pass
def start(self):
self.season_loop.start(5 * 60)
def stop(self):
self.season_loop.stop()
def check_season(self):
if factory.world.season.name == "spring":
factory.vane.weather = "rainy"
reactor.callLater(1 * 60, setattr, factory.vane, "weather",
"sunny")
name = "rain"
class CopyEncoder(object):
"""
This encoder just reads from a file. It ignores the requirement to limit
the playback speed so clients will drift around depending on when they
start and how large of a buffer they have. The UI updates will also drift
from the audio playback. But it has no external requirments, and it simple.
"""
zope.interface.classProvides(IEncoder)
def __init__(self, song, data_callback):
self.song = song
self.data_callback = data_callback
self.done = defer.Deferred()
def encode(self):
self.file = urllib2.urlopen(self.song.uri)
self.lc = LoopingCall(self.process_file)
self.lc.start(.25) # TODO: bit rate?
return self.done
def process_file(self):
data = self.file.read(1024 * 8)
if not data:
self.lc.stop()
self.file.close()
self.done.callback(None)
return
self.data_callback(data)
class PeriodicTasksService(Service):
def __init__(self, clock, taskFunction, interval=60):
self.clock = clock
self.taskFunction = taskFunction
self.interval = interval
self._delayedCall = None
self._looper = None
self._loopFinished = None
def startService(self):
Service.startService(self)
now = time.time()
startDelta = now // self.interval * self.interval + self.interval - now
self._delayedCall = self.clock.callLater(startDelta, self._startLooping)
def _startLooping(self):
self._delayedCall = None
self._looper = LoopingCall(self._call)
self._looper.clock = self.clock
self._loopFinished = self._looper.start(self.interval)
def _call(self):
d = deferToThread(self.taskFunction)
d.addErrback(log.err, "error calling periodic tasks")
return d
def stopService(self):
if self._delayedCall:
self._delayedCall.cancel()
self._delayedCall = None
return
self._looper.stop()
self._loopFinished.addCallback(lambda _: Service.stopService(self))
return self._loopFinished
class Pinger(object):
"""
An periodic AMP ping helper.
"""
def __init__(self, reactor):
"""
:param IReactorTime reactor: The reactor to use to schedule the pings.
"""
self.reactor = reactor
def start(self, protocol, interval):
"""
Start sending some pings.
:param AMP protocol: The protocol over which to send the pings.
:param timedelta interval: The interval at which to send the pings.
"""
def ping():
protocol.callRemote(NoOp)
self._pinging = LoopingCall(ping)
self._pinging.clock = self.reactor
self._pinging.start(interval.total_seconds(), now=False)
def stop(self):
"""
Stop sending the pings.
"""
self._pinging.stop()
class BandwidthEstimator:
bufsize = 20
totalBytes = 0
def __init__(self, message, length):
self.length = length
self.message = message
self.estim = []
self.bytes = 0
self.call = LoopingCall(self.estimateBandwidth)
self.call.start(1)
def estimateBandwidth(self):
bytes = self.bytes
self.totalBytes += bytes
self.estim.append(bytes)
self.message("%0.2f k/s (%0.2d%%)"
% ((sum(self.estim) / len(self.estim)) / 1024.,
(float(self.totalBytes) / self.length) * 100))
if len(self.estim) > self.bufsize:
self.estim.pop(0)
self.bytes = 0
def stop(self):
self.call.stop()
self.estimateBandwidth()
self.message("Finished receiving: %d bytes (%d%%)" % (
self.totalBytes, (float(self.totalBytes) / self.length) * 100))
class SendsManyFileDescriptors(ConnectableProtocol):
paused = False
def connectionMade(self):
self.socket = socket()
self.transport.registerProducer(self, True)
def sender():
self.transport.sendFileDescriptor(self.socket.fileno())
self.transport.write(b"x")
self.task = LoopingCall(sender)
self.task.clock = self.transport.reactor
self.task.start(0).addErrback(err, "Send loop failure")
def stopProducing(self):
self._disconnect()
def resumeProducing(self):
self._disconnect()
def pauseProducing(self):
self.paused = True
self.transport.unregisterProducer()
self._disconnect()
def _disconnect(self):
self.task.stop()
self.transport.abortConnection()
self.other.transport.abortConnection()
class PollingDataStream(DataStream):
""" A self-polling data stream.
This class represents a data stream that wakes up at a given frequency,
and calls the :meth:`poll` method.
"""
frequency = None # Either a timedelta object, or the number of seconds
now = False
def __init__(self):
super(PollingDataStream, self).__init__()
self.timer = LoopingCall(self.poll)
if isinstance(self.frequency, timedelta):
seconds = (
self.frequency.seconds
+ (self.frequency.days * 24 * 60 * 60)
+ (self.frequency.microseconds / 1000000.0)
)
else:
seconds = self.frequency
log.debug("Setting a %s second timer" % seconds)
self.timer.start(seconds, now=self.now)
def poll(self):
raise NotImplementedError
def stop(self):
super(PollingDataStream, self).stop()
try:
if hasattr(self, "timer"):
self.timer.stop()
except Exception, e:
self.log.warn(e)
class ICUMonitorFactory(protocol.ClientFactory):
def __init__(self, gui_msg_callback, gui_ip, gui_port):
self.ip = gui_ip
self.port = gui_port
self.msg_callback = gui_msg_callback
self.objmonit = None
def buildProtocol(self, addr):
return ICUMonitor(self.msg_callback)
def clientConnectionLost(self, connector, reason):
pass
def clientConnectionFailed(self, connector, reason):
pass
def sendmsg(self, gui_reactor):
gui_reactor.connectTCP(self.ip, self.port, self)
def startsend(self, gui_reactor, timerep):
self.loop = LoopingCall(self.sendmsg, gui_reactor)
self.loop.start(timerep)
def stopsend(self):
self.loop.stop()
class CommandExecutor(Runnable):
"""Executes a method every x seconds"""
__metaclass__ = ABCMeta
def __init__(self, pauseBetweenDuties=emu_config.botcommand_timeout, **kwargs):
""":param pauseBetweenDuties: How long to wait between invocations of performDuty() in seconds"""
super(CommandExecutor, self).__init__(**kwargs)
logging.debug("pause: %d, kwargs: %s"%(pauseBetweenDuties, kwargs))
self.pauseBetweenDuties = float(pauseBetweenDuties)
self.lc = LoopingCall(self.performDuty)
def start(self):
"""Starts the runnable, so that it regularly calls performDuty()"""
logging.debug("Started performing duties every %d seconds" % self.pauseBetweenDuties)
observer = log.PythonLoggingObserver()
observer.start()
lcDeferred = self.lc.start(self.pauseBetweenDuties)
lcDeferred.addErrback(self.errback)
@abstractmethod
def performDuty(self, *args, **kwargs):
"""Does the actual work. Is called regularly and should be implemented by subclasses."""
def stop(self):
"""Stops performDuty() from being called"""
logging.debug("Stopping %s" % self.name)
if self.lc.running:
self.lc.stop()
def errback(self, failure):
"""Given to defereds to report errors"""
logging.warning(
"%s in %s: %s %s" % (failure.type, self.name, failure.getErrorMessage(), failure.getBriefTraceback()))
class LeaseService(Service):
"""
Manage leases.
In particular, clear out expired leases once a second.
:ivar _reactor: A ``IReactorTime`` provider.
:ivar _persistence_service: The persistence service to act with.
:ivar _lc: A ``twisted.internet.task.LoopingCall`` run every second
to update the configured leases by releasing leases that have
expired.
"""
def __init__(self, reactor, persistence_service):
self._reactor = reactor
self._persistence_service = persistence_service
def startService(self):
self._lc = LoopingCall(self._expire)
self._lc.clock = self._reactor
self._lc.start(1)
def stopService(self):
self._lc.stop()
def _expire(self):
now = datetime.fromtimestamp(self._reactor.seconds(), tz=UTC)
def expire(leases):
updated_leases = leases.expire(now)
for dataset_id in set(leases) - set(updated_leases):
_LOG_EXPIRE(dataset_id=dataset_id,
node_id=leases[dataset_id].node_id).write()
return updated_leases
return update_leases(expire, self._persistence_service)
class ProvisionerQueryService(ServiceProcess):
"""Provisioner querying service
"""
declare = ServiceProcess.service_declare(name='provisioner_query',
version='0.1.0',
dependencies=[])
def slc_init(self):
interval = float(self.spawn_args.get("interval_seconds",
DEFAULT_QUERY_INTERVAL))
self.client = ProvisionerClient(self)
log.debug('Starting provisioner query loop - %s second interval',
interval)
self.loop = LoopingCall(self.query)
self.loop.start(interval)
def slc_terminate(self):
if self.loop:
self.loop.stop()
@defer.inlineCallbacks
def query(self):
try:
yield self._do_query()
except Exception,e:
log.error("Error sending provisioner query request: %s", e,
exc_info=True)
class CacheService(Service):
def __init__(self, *args, **kwargs):
self.call = None
self.node_updater = NodeCacheUpdater()
self.cluster_updater = ClusterCacheUpdater()
self.vm_updater = VirtualMachineCacheUpdater()
self.job_updater = JobCacheUpdater()
def update_cache(self):
""" a single run of all update classes """
return DeferredList(
[
self.vm_updater.update(),
self.job_updater.update(),
self.node_updater.update(),
self.cluster_updater.update(),
]
)
def startService(self):
self.call = LoopingCall(self.update_cache)
self.call.start(settings.PERIODIC_CACHE_REFRESH)
def stopService(self):
if self.call is not None:
self.call.stop()
class myApp(App):
'''a simple app'''
def __init__(self, reactor):
self.menu = [('&start', self.start),
('sto&p', self.stop),
('&quit', self.quit)]
App.__init__(self, reactor, 'Simple App', self.menu)
def start_clock(self):
self.__i__ = 0
self.__lc = LoopingCall(self.clock)
self.__lc.start(1.0)
def clock(self):
log.msg("clock", self.__i__)
self.widget('table').set_cells([(0, 0, self.__i__)])
self.widget('table').draw()
self.__i__ += 1
def start(self, key):
log.msg("menu: start", key)
def stop(self, key):
self.__lc.stop()
log.msg("menu: stop", key)
def list_box_active_item_changed(self, arg):
'''test callback,
here, we just add the callback to other listbox
'''
self.widget('table').set_cells([(1, 2, arg['active'])])
self.widget('table').draw()
class Blinker(object):
def __init__(self, port, interval=0.5):
self.port = port
self.interval = interval
self.state = None
self.transition = None
def set(self, *args, **kwargs):
self.blink()
def unset(self):
self.transition_stop()
self.port.unset()
def blink(self):
self.transition_stop()
self.transition = LoopingCall(self.blink_task)
self.transition.start(self.interval)
def transition_stop(self):
if self.transition and self.transition.running:
self.transition.stop()
def blink_task(self):
if self.state:
self.state = False
self.port.set()
else:
self.state = True
self.port.unset()
class Hoster: def __init__(self, screen, port): self.screen = screen self.port = port def start(self): # Set up the connection between the state and the network; # using queues gives a degree of seperation between the # communication and game logic, and making them be deferred # keeps it all asynchronous. inqueue = DeferredQueue() outqueue = DeferredQueue() #Initialize GameState gs = GameState(self,self.screen,1,inqueue,outqueue) #Create Looping Call self.lc = LoopingCall(gs.gameLoop) self.lc.start(.0166666666) #Begin Listening connfactory = CommandConnFactory(inqueue,outqueue) self.listening = reactor.listenTCP(self.port,connfactory) def stop(self, nextscreen=None): # Stop the GameState logic, and let go of the port on which we're listening self.lc.stop() self.listening.stopListening() # Start up the next screen, if there is one if nextscreen: nextscreen.start() else: reactor.stop()
class WebSocketConnection(WebSocketProtocol): def __init__(self, stateObject): self.state = stateObject self.opcode = TEXT WebSocketProtocol.__init__(self, WebSocketReceiver(self)) self.finished = Deferred() self.pingLoop = LoopingCall(self.doPing) def doPing(self): self.receiver.transport.sendFrame(PING, '') def write(self, data): self.receiver.transport.sendFrame(self.opcode, data) def sendFrame(self, opcode, data): self.receiver.transport.sendFrame(opcode, data) def writeSequence(self, data): for chunk in data: self.write(chunk) def connectionLost(self, reason): if self.pingLoop.running: self.pingLoop.stop() self.finished.callback(self)
class TrueAfricanUssdTransport(Transport):
CONFIG_CLASS = TrueAfricanUssdTransportConfig
TRANSPORT_TYPE = 'ussd'
SESSION_STATE_MAP = {
TransportUserMessage.SESSION_NONE: 'cont',
TransportUserMessage.SESSION_RESUME: 'cont',
TransportUserMessage.SESSION_CLOSE: 'end',
}
TIMEOUT_TASK_INTERVAL = 10
@inlineCallbacks
def setup_transport(self):
super(TrueAfricanUssdTransport, self).setup_transport()
config = self.get_static_config()
# Session handling
key_prefix = "trueafrican:%s" % self.transport_name
self.session_manager = yield SessionManager.from_redis_config(
config.redis_manager, key_prefix, config.session_timeout)
# XMLRPC Resource
self.web_resource = reactor.listenTCP(config.port,
server.Site(
XmlRpcResource(self)),
interface=config.interface)
# request tracking
self.clock = self.get_clock()
self._requests = {}
self.request_timeout = config.request_timeout
self.timeout_task = LoopingCall(self.request_timeout_cb)
self.timeout_task.clock = self.clock
self.timeout_task_d = self.timeout_task.start(
self.TIMEOUT_TASK_INTERVAL, now=False)
self.timeout_task_d.addErrback(log.err,
"Request timeout handler failed")
@inlineCallbacks
def teardown_transport(self):
yield self.web_resource.loseConnection()
if self.timeout_task.running:
self.timeout_task.stop()
yield self.timeout_task_d
yield self.session_manager.stop()
yield super(TrueAfricanUssdTransport, self).teardown_transport()
def get_clock(self):
"""
For easier stubbing in tests
"""
return reactor
def request_timeout_cb(self):
for request_id, request in self._requests.items():
timestamp = request.timestamp
if timestamp < self.clock.seconds() - self.request_timeout:
self.finish_expired_request(request_id, request)
def track_request(self, request_id, http_request, session):
d = Deferred()
self._requests[request_id] = Request(d, http_request, session,
self.clock.seconds())
return d
def _send_inbound(self, session_id, session, session_event, content):
transport_metadata = {'session_id': session_id}
request_id = self.generate_message_id()
self.publish_message(
message_id=request_id,
content=content,
to_addr=session['to_addr'],
from_addr=session['from_addr'],
session_event=session_event,
transport_name=self.transport_name,
transport_type=self.TRANSPORT_TYPE,
transport_metadata=transport_metadata,
)
return request_id
@inlineCallbacks
def handle_session_new(self, request, session_id, msisdn, to_addr):
session = yield self.session_manager.create_session(session_id,
from_addr=msisdn,
to_addr=to_addr)
session_event = TransportUserMessage.SESSION_NEW
request_id = self._send_inbound(session_id, session, session_event,
None)
r = yield self.track_request(request_id, request, session)
returnValue(r)
@inlineCallbacks
def handle_session_resume(self, request, session_id, content):
# This is an existing session.
session = yield self.session_manager.load_session(session_id)
if not session:
returnValue(self.response_for_error())
session_event = TransportUserMessage.SESSION_RESUME
request_id = self._send_inbound(session_id, session, session_event,
content)
r = yield self.track_request(request_id, request, session)
returnValue(r)
@inlineCallbacks
def handle_session_end(self, request, session_id):
session = yield self.session_manager.load_session(session_id)
if not session:
returnValue(self.response_for_error())
session_event = TransportUserMessage.SESSION_CLOSE
# send a response immediately, and don't (n)ack
# since this is not application-initiated
self._send_inbound(session_id, session, session_event, None)
response = {}
returnValue(response)
def handle_outbound_message(self, message):
in_reply_to = message['in_reply_to']
session_id = message['transport_metadata'].get('session_id')
content = message['content']
if not (in_reply_to and session_id and content):
return self.publish_nack(
user_message_id=message['message_id'],
sent_message_id=message['message_id'],
reason="Missing in_reply_to, content or session_id fields")
response = {
'session': session_id,
'type': self.SESSION_STATE_MAP[message['session_event']],
'message': content
}
log.msg("Sending outbound message %s: %s" %
(message['message_id'], response))
self.finish_request(in_reply_to, message['message_id'], response)
def response_for_error(self):
"""
Generic response for abnormal server side errors.
"""
response = {
'message': 'We encountered an error while processing your message',
'type': 'end'
}
return response
def finish_request(self, request_id, message_id, response):
request = self._requests.get(request_id)
if request is None:
# send a nack back, indicating that the original request had
# timed out before the outbound message reached us.
self.publish_nack(user_message_id=message_id,
sent_message_id=message_id,
reason='Exceeded request timeout')
else:
del self._requests[request_id]
# (n)ack publishing.
#
# Add a callback and errback, either of which will be invoked
# depending on whether the response was written to the client
# successfully or not
if request.http_request.content.closed:
request_done = fail(
Failure(TrueAfricanError("HTTP client closed connection")))
else:
request_done = request.http_request.notifyFinish()
request_done.addCallbacks(
lambda _: self._finish_success_cb(message_id),
lambda f: self._finish_failure_cb(f, message_id))
request.deferred.callback(response)
def finish_expired_request(self, request_id, request):
"""
Called on requests that timed out.
"""
del self._requests[request_id]
log.msg('Timing out on response for %s' % request.session['from_addr'])
request.deferred.callback(self.response_for_error())
def _finish_success_cb(self, message_id):
self.publish_ack(message_id, message_id)
def _finish_failure_cb(self, failure, message_id):
self.publish_nack(user_message_id=message_id,
sent_message_id=message_id,
reason=str(failure))
class MetricBuffer:
__slots__ = ('metric_path', 'interval_buffers', 'compute_task',
'configured', 'aggregation_frequency', 'aggregation_func')
def __init__(self, metric_path):
self.metric_path = metric_path
self.interval_buffers = {}
self.compute_task = None
self.configured = False
self.aggregation_frequency = None
self.aggregation_func = None
def input(self, datapoint):
(timestamp, value) = datapoint
interval = timestamp - (timestamp % self.aggregation_frequency)
if interval in self.interval_buffers:
buffer = self.interval_buffers[interval]
else:
buffer = self.interval_buffers[interval] = IntervalBuffer(interval)
buffer.input(datapoint)
def configure_aggregation(self, frequency, func):
self.aggregation_frequency = int(frequency)
self.aggregation_func = func
self.compute_task = LoopingCall(self.compute_value)
if settings['WRITE_BACK_FREQUENCY'] is not None:
compute_frequency = min(settings['WRITE_BACK_FREQUENCY'],
frequency)
else:
compute_frequency = frequency
self.compute_task.start(compute_frequency, now=False)
self.configured = True
def compute_value(self):
now = int(time.time())
current_interval = now - (now % self.aggregation_frequency)
age_threshold = current_interval - (
settings['MAX_AGGREGATION_INTERVALS'] * self.aggregation_frequency)
for buffer in list(self.interval_buffers.values()):
if buffer.active:
value = self.aggregation_func(buffer.values)
datapoint = (buffer.interval, value)
state.events.metricGenerated(self.metric_path, datapoint)
state.instrumentation.increment('aggregateDatapointsSent')
buffer.mark_inactive()
if buffer.interval < age_threshold:
del self.interval_buffers[buffer.interval]
if not self.interval_buffers:
self.close()
self.configured = False
del BufferManager.buffers[self.metric_path]
def close(self):
if self.compute_task and self.compute_task.running:
self.compute_task.stop()
@property
def size(self):
return sum([len(buf.values) for buf in self.interval_buffers.values()])
class InfiltrationProtocol(protocol):
game_mode = CTF_MODE
balanced_teams = None
defender = None
defender_score_loop = None
defender_return_call = None
attacker = None
attacker_dummy = None
attacker_dummy_calls = None
def on_map_change(self, map):
self.attacker = self.teams[ATTACKER_TEAM]
self.defender = self.attacker.other
self.defender_score_loop = LoopingCall(self.defender_score_cycle)
self.start_defender_score_loop()
protocol.on_map_change(self, map)
def on_map_leave(self):
if self.defender_score_loop and self.defender_score_loop.running:
self.defender_score_loop.stop()
self.defender_score_loop = None
self.end_attacker_dummy_calls()
protocol.on_map_leave(self)
def on_game_end(self):
self.end_attacker_dummy_calls()
self.start_defender_score_loop()
protocol.on_game_end(self)
def on_flag_spawn(self, x, y, z, flag, entity_id):
if flag.team is self.attacker:
return 0, 0, 63
return protocol.on_flag_spawn(self, x, y, z, flag, entity_id)
def start_defender_score_loop(self):
if self.defender_score_loop.running:
self.defender_score_loop.stop()
self.defender_score_loop.start(DEFENDER_SCORE_INTERVAL, now=False)
def defender_score_cycle(self):
dummy = DummyPlayer(self, self.defender)
dummy.score()
def attacker_dummy_score(self):
self.attacker_dummy.score()
if self.attacker_dummy is None:
# this could happen if the dummy capture caused the game to end
return
self.attacker_dummy_calls.pop(0)
if not self.attacker_dummy_calls:
self.end_attacker_dummy_calls()
def end_attacker_dummy_calls(self):
if self.attacker_dummy_calls:
for dummy_call in self.attacker_dummy_calls:
if dummy_call and dummy_call.active():
dummy_call.cancel()
self.attacker_dummy_calls = None
self.attacker_dummy = None
def cancel_defender_return_call(self):
if self.defender_return_call and self.defender_return_call.active(
):
self.defender_return_call.cancel()
self.defender_return_call = None
def fog_flash(self, color):
old_color = self.get_fog_color()
self.set_fog_color(color)
callLater(0.2, self.set_fog_color, old_color)
class Tunnel(object):
__single = None
def __init__(self,
settings,
crawl_keypair_filename=None,
dispersy_port=-1):
if Tunnel.__single:
raise RuntimeError("Tunnel is singleton")
Tunnel.__single = self
self.settings = settings
self.should_run = True
self.crawl_keypair_filename = crawl_keypair_filename
self.dispersy_port = dispersy_port
self.crawl_data = defaultdict(lambda: [])
self.crawl_message = {}
self.current_stats = [0, 0, 0]
self.history_stats = deque(maxlen=180)
self.start_tribler()
self.dispersy = self.session.lm.dispersy
self.community = None
self.clean_messages_lc = LoopingCall(self.clean_messages)
self.clean_messages_lc.start(1800)
self.build_history_lc = LoopingCall(self.build_history)
self.build_history_lc.start(60, now=True)
def get_instance(*args, **kw):
if Tunnel.__single is None:
Tunnel(*args, **kw)
return Tunnel.__single
get_instance = staticmethod(get_instance)
def clean_messages(self):
now = int(time.time())
for k in self.crawl_message.keys():
if now - 3600 > self.crawl_message[k]['time']:
self.crawl_message.pop(k)
clean_twisted_observers()
def build_history(self):
self.history_stats.append(self.get_stats())
def stats_handler(self, candidate, stats, message):
now = int(time.time())
logger.debug('@%d %r %r', now,
message.candidate.get_member().mid.encode('hex'),
json.dumps(stats))
candidate_mid = candidate.get_member().mid
stats = self.preprocess_stats(stats)
stats['time'] = now
stats_old = self.crawl_message.get(candidate_mid, None)
self.crawl_message[candidate_mid] = stats
if stats_old is None:
return
time_dif = float(stats['uptime'] - stats_old['uptime'])
if time_dif > 0:
for index, key in enumerate(
['bytes_orig', 'bytes_exit', 'bytes_relay']):
self.current_stats[index] = self.current_stats[index] * 0.875 + \
(((stats[key] - stats_old[key]) / time_dif) / 1024) * 0.125
def start_tribler(self):
config = SessionStartupConfig()
config.set_state_dir(
os.path.join(config.get_state_dir(), "tunnel-%d") %
self.settings.socks_listen_ports[0])
config.set_torrent_checking(False)
config.set_multicast_local_peer_discovery(False)
config.set_megacache(False)
config.set_dispersy(True)
config.set_mainline_dht(True)
config.set_torrent_collecting(False)
config.set_libtorrent(True)
config.set_dht_torrent_collecting(False)
config.set_enable_torrent_search(False)
config.set_videoserver_enabled(False)
config.set_dispersy_port(self.dispersy_port)
config.set_enable_torrent_search(False)
config.set_enable_channel_search(False)
config.set_enable_multichain(self.settings.enable_multichain)
# We do not want to load the TunnelCommunity in the session but instead our own community
config.set_tunnel_community_enabled(False)
self.session = Session(config)
self.session.start()
logger.info("Using Dispersy port %d" %
self.session.get_dispersy_port())
def start(self, introduce_port):
def start_community():
if self.crawl_keypair_filename:
keypair = read_keypair(self.crawl_keypair_filename)
member = self.dispersy.get_member(
private_key=self.dispersy.crypto.key_to_bin(keypair))
cls = TunnelCommunityCrawler
else:
if self.settings.enable_multichain:
from Tribler.community.multichain.community import MultiChainCommunity
member = self.dispersy.get_member(
private_key=self.session.multichain_keypair.key_to_bin(
))
self.dispersy.define_auto_load(MultiChainCommunity,
member,
load=True)
else:
member = self.dispersy.get_new_member(u"curve25519")
cls = HiddenTunnelCommunity
self.community = self.dispersy.define_auto_load(
cls, member, (self.session, self.settings), load=True)[0]
self.session.set_anon_proxy_settings(
2, ("127.0.0.1",
self.session.get_tunnel_community_socks5_listen_ports()))
if introduce_port:
self.community.add_discovered_candidate(
Candidate(('127.0.0.1', introduce_port), tunnel=False))
blockingCallFromThread(reactor, start_community)
self.session.set_download_states_callback(
self.download_states_callback, interval=4.0)
def download_states_callback(self, dslist):
try:
self.community.monitor_downloads(dslist)
except:
logger.error("Monitoring downloads failed")
return []
def stop(self):
if self.clean_messages_lc:
self.clean_messages_lc.stop()
self.clean_messages_lc = None
if self.build_history_lc:
self.build_history_lc.stop()
self.build_history_lc = None
if self.session:
logger.info("Going to shutdown session")
return self.session.shutdown()
def preprocess_stats(self, stats):
result = defaultdict(int)
result['uptime'] = stats['uptime']
keys_to_from = {
'bytes_orig': ('bytes_up', 'bytes_down'),
'bytes_exit': ('bytes_enter', 'bytes_exit'),
'bytes_relay': ('bytes_relay_up', 'bytes_relay_down')
}
for key_to, key_from in keys_to_from.iteritems():
result[key_to] = sum([stats.get(k, 0) for k in key_from])
return result
def get_stats(self):
return [round(f, 2) for f in self.current_stats]
class AdapterPmMetrics(object):
"""
Base class for Device Adapter PM Metrics Manager
Device specific (OLT, ONU, OpenOMCI, ...) will derive groups of PM information
and this base class is primarily used to provide a consistent interface to configure,
start, and stop statistics collection.
"""
DEFAULT_FREQUENCY_KEY = 'default-collection-frequency'
DEFAULT_COLLECTION_FREQUENCY = 15 * 10 # 1/10ths of a second
# If the collection object has a property of the following name, it will be used
# to retrieve the UTC Collection Timestamp (UTC seconds since epoch). If the collection
# object does not support this attribute, the current time will be used. If the attribute
# is supported, but returns None, this signals that no metrics are currently available
# for collection.
TIMESTAMP_ATTRIBUTE = 'timestamp'
def __init__(self, adapter_agent, device_id, logical_device_id,
grouped=False, freq_override=False, **kwargs):
"""
Initializer for shared Device Adapter PM metrics manager
:param adapter_agent: (AdapterAgent) Adapter agent for the device
:param device_id: (str) Device ID
:param logical_device_id: (str) VOLTHA Logical Device ID
:param grouped: (bool) Flag indicating if statistics are managed as a group
:param freq_override: (bool) Flag indicating if frequency collection can be specified
on a per group basis
:param kwargs: (dict) Device Adapter specific values
"""
self.log = structlog.get_logger(device_id=device_id)
self.device_id = device_id
self.adapter_agent = adapter_agent
self.name = adapter_agent.adapter_name
# Sanitize the vcore ID in the logical device ID
self.logical_device_id = '0000' + logical_device_id[4:]
device = self.adapter_agent.get_device(self.device_id)
self.serial_number = device.serial_number
self.default_freq = kwargs.get(AdapterPmMetrics.DEFAULT_FREQUENCY_KEY,
AdapterPmMetrics.DEFAULT_COLLECTION_FREQUENCY)
self.grouped = grouped
self.freq_override = grouped and freq_override
self.lc = None
self.pm_group_metrics = dict() # name -> PmGroupConfig
def update(self, pm_config):
# TODO: Move any common steps into base class
raise NotImplementedError('Your derived class should override this method')
def make_proto(self, pm_config=None):
raise NotImplementedError('Your derived class should override this method')
def start_collector(self, callback=None):
"""
Start the collection loop for an adapter if the frequency > 0
:param callback: (callable) Function to call to collect PM data
"""
self.log.info("starting-pm-collection", device_name=self.name)
if callback is None:
callback = self.collect_and_publish_metrics
if self.lc is None:
self.lc = LoopingCall(callback)
if self.default_freq > 0:
self.lc.start(interval=self.default_freq / 10)
def stop_collector(self):
""" Stop the collection loop"""
if self.lc is not None and self.default_freq > 0:
self.lc.stop()
def collect_group_metrics(self, group_name, group, names, config):
"""
Collect the metrics for a specific PM group.
This common collection method expects that the 'group object' provide as the second
parameter supports an attribute or property with the name of the value to
retrieve.
:param group_name: (str) The unique collection name. The name should not contain spaces.
:param group: (object) The object to query for the value of various attributes (PM names)
:param names: (set) A collection of PM names that, if implemented as a property in the object,
will return a value to store in the returned PM dictionary
:param config: (PMConfig) PM Configuration settings. The enabled flag is examined to determine
if the data associated with a PM Name will be collected.
:return: (MetricInformation) collected metrics
"""
assert ' ' not in group_name, 'Spaces are not allowed in metric titles, use an underscore'
if group is None:
return None
metrics = dict()
context = dict()
now = getattr(group, AdapterPmMetrics.TIMESTAMP_ATTRIBUTE) \
if hasattr(group, AdapterPmMetrics.TIMESTAMP_ATTRIBUTE) \
else arrow.utcnow().float_timestamp
if now is None:
return None # No metrics available at this time for collection
for (metric, t) in names:
if config[metric].type == PmConfig.CONTEXT and hasattr(group, metric):
context[metric] = str(getattr(group, metric))
elif config[metric].type in (PmConfig.COUNTER, PmConfig.GAUGE, PmConfig.STATE):
if config[metric].enabled and hasattr(group, metric):
metrics[metric] = getattr(group, metric)
# Check length of metric data. Will be zero if if/when individual group
# metrics can be disabled and all are (or or not supported by the
# underlying adapter)
if len(metrics) == 0:
return None
return MetricInformation(metadata=MetricMetaData(title=group_name,
ts=now,
logical_device_id=self.logical_device_id,
serial_no=self.serial_number,
device_id=self.device_id,
context=context),
metrics=metrics)
def collect_metrics(self, data=None):
"""
Collect metrics for this adapter.
The adapter type (OLT, ONU, ..) should provide a derived class where this
method iterates through all metrics and collects them up in a dictionary with
the group/metric name as the key, and the metric values as the contents.
The data collected (or passed in) is a list of pairs/tuples. Each
pair is composed of a MetricMetaData metadata-portion and list of MetricValuePairs
that contains a single individual metric or list of metrics if this is a
group metric.
This method is called for each adapter at a fixed frequency.
TODO: Currently all group metrics are collected on a single timer tick.
This needs to be fixed as independent group or instance collection is
desirable.
:param data: (list) Existing list of collected metrics (MetricInformation).
This is provided to allow derived classes to call into
further encapsulated classes.
:return: (list) metadata and metrics pairs - see description above
"""
raise NotImplementedError('Your derived class should override this method')
def collect_and_publish_metrics(self):
""" Request collection of all enabled metrics and publish them """
try:
data = self.collect_metrics()
self.publish_metrics(data)
except Exception as e:
self.log.exception('failed-to-collect-kpis', e=e)
def publish_metrics(self, data):
"""
Publish the metrics during a collection.
The data collected (or passed in) is a list of dictionary pairs/tuple. Each
pair is composed of a metadata-portion and a metrics-portion that contains
information for a specific instance of an individual metric or metric group.
:param data: (list) Existing list of collected metrics (MetricInformation)
to convert to a KPIEvent and publish
"""
self.log.debug('publish-metrics', data=data)
if len(data):
try:
# TODO: Existing adapters use the KpiEvent, if/when all existing
# adapters use the shared KPI library, we may want to
# deprecate the KPIEvent
kpi_event = KpiEvent2(
type=KpiEventType.slice,
ts=arrow.utcnow().float_timestamp,
slice_data=data
)
self.adapter_agent.submit_kpis(kpi_event)
except Exception as e:
self.log.exception('failed-to-submit-kpis', e=e)
class _ContinuousPolling(_PollLikeMixin, _DisconnectSelectableMixin):
"""
Schedule reads and writes based on the passage of time, rather than
notification.
This is useful for supporting polling filesystem files, which C{epoll(7)}
does not support.
The implementation uses L{_PollLikeMixin}, which is a bit hacky, but
re-implementing and testing the relevant code yet again is unappealing.
@ivar _reactor: The L{EPollReactor} that is using this instance.
@ivar _loop: A C{LoopingCall} that drives the polling, or L{None}.
@ivar _readers: A C{set} of C{FileDescriptor} objects that should be read
from.
@ivar _writers: A C{set} of C{FileDescriptor} objects that should be
written to.
"""
# Attributes for _PollLikeMixin
_POLL_DISCONNECTED = 1
_POLL_IN = 2
_POLL_OUT = 4
def __init__(self, reactor):
self._reactor = reactor
self._loop = None
self._readers = set()
self._writers = set()
def _checkLoop(self):
"""
Start or stop a C{LoopingCall} based on whether there are readers and
writers.
"""
if self._readers or self._writers:
if self._loop is None:
from twisted.internet.task import LoopingCall, _EPSILON
self._loop = LoopingCall(self.iterate)
self._loop.clock = self._reactor
# LoopingCall seems unhappy with timeout of 0, so use very
# small number:
self._loop.start(_EPSILON, now=False)
elif self._loop:
self._loop.stop()
self._loop = None
def iterate(self):
"""
Call C{doRead} and C{doWrite} on all readers and writers respectively.
"""
for reader in list(self._readers):
self._doReadOrWrite(reader, reader, self._POLL_IN)
for writer in list(self._writers):
self._doReadOrWrite(writer, writer, self._POLL_OUT)
def addReader(self, reader):
"""
Add a C{FileDescriptor} for notification of data available to read.
"""
self._readers.add(reader)
self._checkLoop()
def addWriter(self, writer):
"""
Add a C{FileDescriptor} for notification of data available to write.
"""
self._writers.add(writer)
self._checkLoop()
def removeReader(self, reader):
"""
Remove a C{FileDescriptor} from notification of data available to read.
"""
try:
self._readers.remove(reader)
except KeyError:
return
self._checkLoop()
def removeWriter(self, writer):
"""
Remove a C{FileDescriptor} from notification of data available to
write.
"""
try:
self._writers.remove(writer)
except KeyError:
return
self._checkLoop()
def removeAll(self):
"""
Remove all readers and writers.
"""
result = list(self._readers | self._writers)
# Don't reset to new value, since self.isWriting and .isReading refer
# to the existing instance:
self._readers.clear()
self._writers.clear()
return result
def getReaders(self):
"""
Return a list of the readers.
"""
return list(self._readers)
def getWriters(self):
"""
Return a list of the writers.
"""
return list(self._writers)
def isReading(self, fd):
"""
Checks if the file descriptor is currently being observed for read
readiness.
@param fd: The file descriptor being checked.
@type fd: L{twisted.internet.abstract.FileDescriptor}
@return: C{True} if the file descriptor is being observed for read
readiness, C{False} otherwise.
@rtype: C{bool}
"""
return fd in self._readers
def isWriting(self, fd):
"""
Checks if the file descriptor is currently being observed for write
readiness.
@param fd: The file descriptor being checked.
@type fd: L{twisted.internet.abstract.FileDescriptor}
@return: C{True} if the file descriptor is being observed for write
readiness, C{False} otherwise.
@rtype: C{bool}
"""
return fd in self._writers
class PreferencesManager(component.Component):
def __init__(self):
component.Component.__init__(self, 'PreferencesManager')
self.config = deluge.configmanager.ConfigManager(
'core.conf', DEFAULT_PREFS)
if 'proxies' in self.config:
log.warning(
'Updating config file for proxy, using "peer" values to fill new "proxy" setting'
)
self.config['proxy'].update(self.config['proxies']['peer'])
log.warning('New proxy config is: %s', self.config['proxy'])
del self.config['proxies']
if 'i2p_proxy' in self.config and self.config['i2p_proxy']['hostname']:
self.config['proxy'].update(self.config['i2p_proxy'])
self.config['proxy']['type'] = 6
del self.config['i2p_proxy']
if 'anonymous_mode' in self.config:
self.config['proxy']['anonymous_mode'] = self.config[
'anonymous_mode']
del self.config['anonymous_mode']
if 'proxy' in self.config:
for key in DEFAULT_PREFS['proxy']:
if key not in self.config['proxy']:
self.config['proxy'][key] = DEFAULT_PREFS['proxy'][key]
self.core = component.get('Core')
self.new_release_timer = None
def start(self):
# Set the initial preferences on start-up
for key in DEFAULT_PREFS:
self.do_config_set_func(key, self.config[key])
self.config.register_change_callback(self._on_config_value_change)
def stop(self):
if self.new_release_timer and self.new_release_timer.running:
self.new_release_timer.stop()
# Config set functions
def do_config_set_func(self, key, value):
on_set_func = getattr(self, '_on_set_' + key, None)
if on_set_func:
if log.isEnabledFor(logging.DEBUG):
log.debug('Config key: %s set to %s..', key, value)
on_set_func(key, value)
def _on_config_value_change(self, key, value):
if self.get_state() == 'Started':
self.do_config_set_func(key, value)
component.get('EventManager').emit(
ConfigValueChangedEvent(key, value))
def _on_set_torrentfiles_location(self, key, value):
if self.config['copy_torrent_file']:
try:
os.makedirs(value)
except OSError as ex:
log.debug('Unable to make directory: %s', ex)
def _on_set_listen_ports(self, key, value):
self.__set_listen_on()
def _on_set_listen_interface(self, key, value):
self.__set_listen_on()
def _on_set_outgoing_interface(self, key, value):
"""Set interface name or IP address for outgoing BitTorrent connections."""
value = value.strip() if value else ''
self.core.apply_session_settings({'outgoing_interfaces': value})
def _on_set_random_port(self, key, value):
self.__set_listen_on()
def __set_listen_on(self):
""" Set the ports and interface address to listen for incoming connections on."""
if self.config['random_port']:
if not self.config['listen_random_port']:
self.config['listen_random_port'] = random.randrange(
49152, 65525)
listen_ports = [self.config['listen_random_port']
] * 2 # use single port range
else:
self.config['listen_random_port'] = None
listen_ports = self.config['listen_ports']
if self.config['listen_interface']:
interface = self.config['listen_interface'].strip()
else:
interface = '0.0.0.0'
log.debug(
'Listen Interface: %s, Ports: %s with use_sys_port: %s',
interface,
listen_ports,
self.config['listen_use_sys_port'],
)
interfaces = [
'%s:%s' % (interface, port)
for port in range(listen_ports[0], listen_ports[1] + 1)
]
self.core.apply_session_settings({
'listen_system_port_fallback':
self.config['listen_use_sys_port'],
'listen_interfaces':
','.join(interfaces),
})
def _on_set_outgoing_ports(self, key, value):
self.__set_outgoing_ports()
def _on_set_random_outgoing_ports(self, key, value):
self.__set_outgoing_ports()
def __set_outgoing_ports(self):
port = (0 if self.config['random_outgoing_ports'] else
self.config['outgoing_ports'][0])
if port:
num_ports = (self.config['outgoing_ports'][1] -
self.config['outgoing_ports'][0])
num_ports = num_ports if num_ports > 1 else 5
else:
num_ports = 0
log.debug('Outgoing port set to %s with range: %s', port, num_ports)
self.core.apply_session_settings({
'outgoing_port': port,
'num_outgoing_ports': num_ports
})
def _on_set_peer_tos(self, key, value):
try:
self.core.apply_session_setting('peer_tos', int(value, 16))
except ValueError as ex:
log.error('Invalid tos byte: %s', ex)
def _on_set_dht(self, key, value):
lt_bootstraps = self.core.session.get_settings()['dht_bootstrap_nodes']
# Update list of lt bootstraps, using set to remove duplicates.
dht_bootstraps = set(
lt_bootstraps.split(',') + [
'router.bittorrent.com:6881',
'router.utorrent.com:6881',
'router.bitcomet.com:6881',
'dht.transmissionbt.com:6881',
'dht.aelitis.com:6881',
])
self.core.apply_session_settings({
'dht_bootstrap_nodes':
','.join(dht_bootstraps),
'enable_dht':
value
})
def _on_set_upnp(self, key, value):
self.core.apply_session_setting('enable_upnp', value)
def _on_set_natpmp(self, key, value):
self.core.apply_session_setting('enable_natpmp', value)
def _on_set_lsd(self, key, value):
self.core.apply_session_setting('enable_lsd', value)
def _on_set_utpex(self, key, value):
if value:
self.core.session.add_extension('ut_pex')
def _on_set_enc_in_policy(self, key, value):
self._on_set_encryption(key, value)
def _on_set_enc_out_policy(self, key, value):
self._on_set_encryption(key, value)
def _on_set_enc_level(self, key, value):
self._on_set_encryption(key, value)
def _on_set_encryption(self, key, value):
# Convert Deluge enc_level values to libtorrent enc_level values.
pe_enc_level = {
0: lt.enc_level.plaintext,
1: lt.enc_level.rc4,
2: lt.enc_level.both,
}
self.core.apply_session_settings({
'out_enc_policy':
lt.enc_policy(self.config['enc_out_policy']),
'in_enc_policy':
lt.enc_policy(self.config['enc_in_policy']),
'allowed_enc_level':
lt.enc_level(pe_enc_level[self.config['enc_level']]),
'prefer_rc4':
True,
})
def _on_set_max_connections_global(self, key, value):
self.core.apply_session_setting('connections_limit', value)
def _on_set_max_upload_speed(self, key, value):
# We need to convert Kb/s to B/s
value = -1 if value < 0 else int(value * 1024)
self.core.apply_session_setting('upload_rate_limit', value)
def _on_set_max_download_speed(self, key, value):
# We need to convert Kb/s to B/s
value = -1 if value < 0 else int(value * 1024)
self.core.apply_session_setting('download_rate_limit', value)
def _on_set_max_upload_slots_global(self, key, value):
self.core.apply_session_setting('unchoke_slots_limit', value)
def _on_set_max_half_open_connections(self, key, value):
self.core.apply_session_setting('half_open_limit', value)
def _on_set_max_connections_per_second(self, key, value):
self.core.apply_session_setting('connection_speed', value)
def _on_set_ignore_limits_on_local_network(self, key, value):
self.core.apply_session_setting('ignore_limits_on_local_network',
value)
def _on_set_share_ratio_limit(self, key, value):
# This value is a float percentage in deluge, but libtorrent needs int percentage.
self.core.apply_session_setting('share_ratio_limit', int(value * 100))
def _on_set_seed_time_ratio_limit(self, key, value):
# This value is a float percentage in deluge, but libtorrent needs int percentage.
self.core.apply_session_setting('seed_time_ratio_limit',
int(value * 100))
def _on_set_seed_time_limit(self, key, value):
# This value is stored in minutes in deluge, but libtorrent wants seconds
self.core.apply_session_setting('seed_time_limit', int(value * 60))
def _on_set_max_active_downloading(self, key, value):
self.core.apply_session_setting('active_downloads', value)
def _on_set_max_active_seeding(self, key, value):
self.core.apply_session_setting('active_seeds', value)
def _on_set_max_active_limit(self, key, value):
self.core.apply_session_setting('active_limit', value)
def _on_set_dont_count_slow_torrents(self, key, value):
self.core.apply_session_setting('dont_count_slow_torrents', value)
def _on_set_send_info(self, key, value):
"""sends anonymous stats home"""
log.debug('Sending anonymous stats..')
class SendInfoThread(threading.Thread):
def __init__(self, config):
self.config = config
threading.Thread.__init__(self)
def run(self):
import time
now = time.time()
# check if we've done this within the last week or never
if (now - self.config['info_sent']) >= (60 * 60 * 24 * 7):
try:
url = (
'http://deluge-torrent.org/stats_get.php?processor='
+ platform.machine() + '&python=' +
platform.python_version() + '&deluge=' +
deluge.common.get_version() + '&os=' +
platform.system() + '&plugins=' + quote_plus(
':'.join(self.config['enabled_plugins'])))
urlopen(url)
except IOError as ex:
log.debug(
'Network error while trying to send info: %s', ex)
else:
self.config['info_sent'] = now
if value:
SendInfoThread(self.config).start()
def _on_set_new_release_check(self, key, value):
if value:
log.debug('Checking for new release..')
threading.Thread(target=self.core.get_new_release).start()
if self.new_release_timer and self.new_release_timer.running:
self.new_release_timer.stop()
# Set a timer to check for a new release every 3 days
self.new_release_timer = LoopingCall(
self._on_set_new_release_check, 'new_release_check', True)
self.new_release_timer.start(72 * 60 * 60, False)
else:
if self.new_release_timer and self.new_release_timer.running:
self.new_release_timer.stop()
def _on_set_proxy(self, key, value):
# Initialise with type none and blank hostnames.
proxy_settings = {
'proxy_type': lt.proxy_type_t.none,
'i2p_hostname': '',
'proxy_hostname': '',
'proxy_hostnames': value['proxy_hostnames'],
'proxy_peer_connections': value['proxy_peer_connections'],
'proxy_tracker_connections': value['proxy_tracker_connections'],
'force_proxy': value['force_proxy'],
'anonymous_mode': value['anonymous_mode'],
}
if value['type'] == lt.proxy_type_t.i2p_proxy:
proxy_settings.update({
'proxy_type': lt.proxy_type_t.i2p_proxy,
'i2p_hostname': value['hostname'],
'i2p_port': value['port'],
})
elif value['type'] != lt.proxy_type_t.none:
proxy_settings.update({
'proxy_type': value['type'],
'proxy_hostname': value['hostname'],
'proxy_port': value['port'],
'proxy_username': value['username'],
'proxy_password': value['password'],
})
self.core.apply_session_settings(proxy_settings)
def _on_set_rate_limit_ip_overhead(self, key, value):
self.core.apply_session_setting('rate_limit_ip_overhead', value)
def _on_set_geoip_db_location(self, key, geoipdb_path):
# Load the GeoIP DB for country look-ups if available
if os.path.exists(geoipdb_path):
try:
self.core.geoip_instance = GeoIP.open(geoipdb_path,
GeoIP.GEOIP_STANDARD)
except AttributeError:
log.warning('GeoIP Unavailable')
else:
log.warning('Unable to find GeoIP database file: %s', geoipdb_path)
def _on_set_cache_size(self, key, value):
self.core.apply_session_setting('cache_size', value)
def _on_set_cache_expiry(self, key, value):
self.core.apply_session_setting('cache_expiry', value)
def _on_auto_manage_prefer_seeds(self, key, value):
self.core.apply_session_setting('auto_manage_prefer_seeds', value)
class MetricLogObserver(config.ReconfigurableServiceMixin,
service.MultiService):
_reactor = reactor
def __init__(self):
service.MultiService.__init__(self)
self.setName('metrics')
self.enabled = False
self.periodic_task = None
self.periodic_interval = None
self.log_task = None
self.log_interval = None
# Mapping of metric type to handlers for that type
self.handlers = {}
# Register our default handlers
self.registerHandler(MetricCountEvent, MetricCountHandler(self))
self.registerHandler(MetricTimeEvent, MetricTimeHandler(self))
self.registerHandler(MetricAlarmEvent, MetricAlarmHandler(self))
self.getHandler(MetricCountEvent).addWatcher(
AttachedSlavesWatcher(self))
def reconfigService(self, new_config):
# first, enable or disable
if new_config.metrics is None:
self.disable()
else:
self.enable()
metrics_config = new_config.metrics
# Start up periodic logging
log_interval = metrics_config.get('log_interval', 60)
if log_interval != self.log_interval:
if self.log_task:
self.log_task.stop()
self.log_task = None
if log_interval:
self.log_task = LoopingCall(self.report)
self.log_task.clock = self._reactor
self.log_task.start(log_interval)
# same for the periodic task
periodic_interval = metrics_config.get('periodic_interval', 10)
if periodic_interval != self.periodic_interval:
if self.periodic_task:
self.periodic_task.stop()
self.periodic_task = None
if periodic_interval:
self.periodic_task = LoopingCall(periodicCheck,
self._reactor)
self.periodic_task.clock = self._reactor
self.periodic_task.start(periodic_interval)
# upcall
return config.ReconfigurableServiceMixin.reconfigService(
self, new_config)
def stopService(self):
self.disable()
service.MultiService.stopService(self)
def enable(self):
if self.enabled:
return
log.addObserver(self.emit)
self.enabled = True
def disable(self):
if not self.enabled:
return
if self.periodic_task:
self.periodic_task.stop()
self.periodic_task = None
if self.log_task:
self.log_task.stop()
self.log_task = None
log.removeObserver(self.emit)
self.enabled = False
def registerHandler(self, interface, handler):
old = self.getHandler(interface)
self.handlers[interface] = handler
return old
def getHandler(self, interface):
return self.handlers.get(interface)
def emit(self, eventDict):
# Ignore non-statistic events
metric = eventDict.get('metric')
if not metric or not isinstance(metric, MetricEvent):
return
if metric.__class__ not in self.handlers:
return
h = self.handlers[metric.__class__]
h.handle(eventDict, metric)
for w in h.watchers:
w.run()
def asDict(self):
retval = {}
for interface, handler in self.handlers.iteritems():
retval.update(handler.asDict())
return retval
def report(self):
try:
for interface, handler in self.handlers.iteritems():
report = handler.report()
if not report:
continue
for line in report.split("\n"):
log.msg(line)
except Exception:
log.err(None, "generating metric report")
class IntervalTask(BaseTask):
"""
IntervalTask executes a callback at an interval.
Example: IntervalTask(myFunc, 60) -- calls myFunc() every 60 seconds.
"""
_transient_vars = ['_looper']
_callback = None
_interval = None
_immediate = False
_iterations = None
_args = []
_kwargs = {}
#: @type: LoopingCall
_looper = None
_last_run_time = None
_scheduled_time = None
run_count = 0
paused = False
_elapsed_at_pause = None
_paused_at_shutdown = False
def __init__(self, callback, interval, immediate=False, iterations=None,
args=None, kwargs=None):
"""
@param callback: The callable to call at intervals.
@param interval: The interval at which to execute the callback.
@type interval: int or float or None
@param immediate: If True, then the first iteration fires immediately.
@type immediate: bool
@param iterations: The maximum number of iterations this task should
run. If None, then there is no limit.
@type iterations: int
"""
super(IntervalTask, self).__init__()
self._callback = StoredCallback(callback)
self._interval = interval
self._immediate = immediate
self._iterations = iterations
if args is not None:
self._args = args
if kwargs is not None:
self._kwargs = kwargs
def __str__(self):
return "%s: %s" % (self.__class__.__name__, self._callback)
@property
def last_run_time(self):
return self._last_run_time if self.run_count else None
@property
def next_run_time(self):
if self.paused:
return None
last = max(self.last_run_time, self._scheduled_time)
return last + self._interval
@property
def active(self):
return (not self.paused) and (self._looper is not None)
def _schedule(self, interval=None):
interval = max(0, interval if interval is not None else self._interval)
self._scheduled_time = time.time()
now = self._immediate and not self.run_count
self._looper = LoopingCall(self._run)
d = self._looper.start(interval, now=now)
d.addErrback(self._errback)
def kill(self):
self._kill_looper()
super(IntervalTask, self).kill()
def _kill_looper(self):
if self._looper is not None and self._looper.running:
self._looper.stop()
def pause(self):
if not self.paused:
last = self._last_run_time or self._scheduled_time or time.time()
self._elapsed_at_pause = time.time() - last
self.paused = True
self._kill_looper()
return True
else:
return False
def unpause(self):
if self.paused:
del self.paused
elapsed = self._elapsed_at_pause or 0
self._schedule(self._interval - elapsed)
def _run(self):
if not self.alive:
self._kill_looper()
return
self.run_count += 1
self._last_run_time = time.time()
#noinspection PyBroadException
try:
self._callback(*self._args, **self._kwargs)
except:
logging.error("Error in %s:\n%s" % (self, traceback.format_exc()))
if self._iterations is not None and self.run_count >= self._iterations:
self.kill()
if self._elapsed_at_pause:
del self._elapsed_at_pause
self._kill_looper()
self._schedule()
def _errback(self, error):
"""
@type error: twisted.python.failure.Failure
"""
error.trap(Exception)
if error.type == CancelledError:
return
tb = ''.join(traceback.format_exception(error.type, error.value,
error.getTracebackObject()))
logging.error("Error in %s:\n%s" % (self, tb))
def server_startup(self):
if not self.alive:
return
if not self._paused_at_shutdown:
self.unpause()
try:
del self._paused_at_shutdown
except AttributeError:
pass
super(IntervalTask, self).server_startup()
def server_shutdown(self):
self._paused_at_shutdown = self.paused
self.pause()
super(IntervalTask, self).server_shutdown()
class BlobAvailabilityTracker(object):
"""
Class to track peer counts for known blobs, and to discover new popular blobs
Attributes:
availability (dict): dictionary of peers for known blobs
"""
def __init__(self, blob_manager, peer_finder, dht_node):
self.availability = {}
self._last_mean_availability = Decimal(0.0)
self._blob_manager = blob_manager
self._peer_finder = peer_finder
self._dht_node = dht_node
self._check_popular = LoopingCall(self._update_most_popular)
self._check_mine = LoopingCall(self._update_mine)
def start(self):
log.info("Starting %s", self)
self._check_popular.start(30)
self._check_mine.start(600)
def stop(self):
log.info("Stopping %s", self)
if self._check_popular.running:
self._check_popular.stop()
if self._check_mine.running:
self._check_mine.stop()
def get_blob_availability(self, blob):
def _get_peer_count(peers):
have_blob = sum(1 for peer in peers if peer.is_available())
return {blob: have_blob}
d = self._peer_finder.find_peers_for_blob(blob)
d.addCallback(_get_peer_count)
return d
def get_availability_for_blobs(self, blobs):
dl = [self.get_blob_availability(blob) for blob in blobs if blob]
d = defer.DeferredList(dl)
d.addCallback(
lambda results: [val for success, val in results if success])
return d
@property
def last_mean_availability(self):
return max(Decimal(0.01), self._last_mean_availability)
def _update_peers_for_blob(self, blob):
def _save_peer_info(blob_hash, peers):
v = {blob_hash: peers}
self.availability.update(v)
return v
d = self._peer_finder.find_peers_for_blob(blob)
d.addCallback(
lambda r: [[c.host, c.port, c.is_available()] for c in r])
d.addCallback(lambda peers: _save_peer_info(blob, peers))
return d
def _get_most_popular(self):
dl = []
for (hash, _) in self._dht_node.get_most_popular_hashes(100):
encoded = hash.encode('hex')
dl.append(self._update_peers_for_blob(encoded))
return defer.DeferredList(dl)
def _update_most_popular(self):
d = self._get_most_popular()
d.addCallback(lambda _: self._set_mean_peers())
def _update_mine(self):
def _get_peers(blobs):
dl = []
for hash in blobs:
dl.append(self._update_peers_for_blob(hash))
return defer.DeferredList(dl)
def sample(blobs):
return random.sample(blobs, min(len(blobs), 100))
start = time.time()
log.debug('==> Updating the peers for my blobs')
d = self._blob_manager.get_all_verified_blobs()
# as far as I can tell, this only is used to set _last_mean_availability
# which... seems like a very expensive operation for such little payoff.
# so taking a sample should get about the same effect as querying the entire
# list of blobs
d.addCallback(sample)
d.addCallback(_get_peers)
d.addCallback(lambda _: self._set_mean_peers())
d.addCallback(lambda _: log.debug(
'<== Done updating peers for my blobs. Took %s seconds',
time.time() - start))
# although unused, need to return or else the looping call
# could overrun on a previous call
return d
def _set_mean_peers(self):
num_peers = [
len(self.availability[blob]) for blob in self.availability
]
mean = Decimal(sum(num_peers)) / Decimal(max(1, len(num_peers)))
self._last_mean_availability = mean
class MonotoneSource(service.Service, util.ComparableMixin):
"""This source will poll a monotone server for changes and submit them to
the change master.
@param server_addr: monotone server specification (host:portno)
@param branch: monotone branch to watch
@param trusted_keys: list of keys whose code you trust
@param db_path: path to monotone database to pull into
@param pollinterval: interval in seconds between polls, defaults to 10 minutes
@param monotone_exec: path to monotone executable, defaults to "monotone"
"""
__implements__ = IChangeSource, service.Service.__implements__
compare_attrs = [
"server_addr", "trusted_keys", "db_path", "pollinterval", "branch",
"monotone_exec"
]
parent = None # filled in when we're added
done_revisions = []
last_revision = None
loop = None
d = None
tmpfile = None
monotone = None
volatile = ["loop", "d", "tmpfile", "monotone"]
def __init__(self,
server_addr,
branch,
trusted_keys,
db_path,
pollinterval=60 * 10,
monotone_exec="monotone"):
self.server_addr = server_addr
self.branch = branch
self.trusted_keys = trusted_keys
self.db_path = db_path
self.pollinterval = pollinterval
self.monotone_exec = monotone_exec
self.monotone = Monotone(self.monotone_exec, self.db_path)
def startService(self):
self.loop = LoopingCall(self.start_poll)
self.loop.start(self.pollinterval)
service.Service.startService(self)
def stopService(self):
self.loop.stop()
return service.Service.stopService(self)
def describe(self):
return "monotone_source %s %s" % (self.server_addr, self.branch)
def start_poll(self):
if self.d is not None:
log.msg("last poll still in progress, skipping next poll")
return
log.msg("starting poll")
self.d = self._maybe_init_db()
self.d.addCallback(self._do_netsync)
self.d.addCallback(self._get_changes)
self.d.addErrback(self._handle_error)
def _handle_error(self, failure):
log.err(failure)
self.d = None
def _maybe_init_db(self):
if not os.path.exists(self.db_path):
log.msg("init'ing db")
return self.monotone.db_init()
else:
log.msg("db already exists, migrating")
return self.monotone.db_migrate()
def _do_netsync(self, output):
return self.monotone.pull(self.server_addr, self.branch)
def _get_changes(self, output):
d = self._get_new_head()
d.addCallback(self._process_new_head)
return d
def _get_new_head(self):
# This function returns a deferred that resolves to a good pick of new
# head (or None if there is no good new head.)
# First need to get all new heads...
rcfile = """function get_revision_cert_trust(signers, id, name, val)
local trusted_signers = { %s }
local ts_table = {}
for k, v in pairs(trusted_signers) do ts_table[v] = 1 end
for k, v in pairs(signers) do
if ts_table[v] then
return true
end
end
return false
end
"""
trusted_list = ", ".join(
['"' + key + '"' for key in self.trusted_keys])
# mktemp is unsafe, but mkstemp is not 2.2 compatible.
tmpfile_name = tempfile.mktemp()
f = open(tmpfile_name, "w")
f.write(rcfile % trusted_list)
f.close()
d = self.monotone.get_heads(self.branch, tmpfile_name)
d.addCallback(self._find_new_head, tmpfile_name)
return d
def _find_new_head(self, new_heads, tmpfile_name):
os.unlink(tmpfile_name)
# Now get the old head's descendents...
if self.last_revision is not None:
d = self.monotone.descendents(self.last_revision)
else:
d = defer.succeed(new_heads)
d.addCallback(self._pick_new_head, new_heads)
return d
def _pick_new_head(self, old_head_descendents, new_heads):
for r in new_heads:
if r in old_head_descendents:
return r
return None
def _process_new_head(self, new_head):
if new_head is None:
log.msg("No new head")
self.d = None
return None
# Okay, we have a new head; we need to get all the revisions since
# then and create change objects for them.
# Step 1: simplify set of processed revisions.
d = self._simplify_revisions()
# Step 2: get the list of new revisions
d.addCallback(self._get_new_revisions, new_head)
# Step 3: add a change for each
d.addCallback(self._add_changes_for_revisions)
# Step 4: all done
d.addCallback(self._finish_changes, new_head)
return d
def _simplify_revisions(self):
d = self.monotone.erase_ancestors(self.done_revisions)
d.addCallback(self._reset_done_revisions)
return d
def _reset_done_revisions(self, new_done_revisions):
self.done_revisions = new_done_revisions
return None
def _get_new_revisions(self, blah, new_head):
if self.done_revisions:
return self.monotone.ancestry_difference(new_head,
self.done_revisions)
else:
# Don't force feed the builder with every change since the
# beginning of time when it's first started up.
return defer.succeed([new_head])
def _add_changes_for_revisions(self, revs):
d = defer.succeed(None)
for rid in revs:
d.addCallback(self._add_change_for_revision, rid)
return d
def _add_change_for_revision(self, blah, rid):
d = self.monotone.log(rid, 1)
d.addCallback(self._add_change_from_log, rid)
return d
def _add_change_from_log(self, log, rid):
d = self.monotone.get_revision(rid)
d.addCallback(self._add_change_from_log_and_revision, log, rid)
return d
def _add_change_from_log_and_revision(self, revision, log, rid):
# Stupid way to pull out everything inside quotes (which currently
# uniquely identifies filenames inside a changeset).
pieces = revision.split('"')
files = []
for i in range(len(pieces)):
if (i % 2) == 1:
files.append(pieces[i])
# Also pull out author key and date
author = "unknown author"
pieces = log.split('\n')
for p in pieces:
if p.startswith("Author:"):
author = p.split()[1]
self.parent.addChange(Change(author, files, log, revision=rid))
def _finish_changes(self, blah, new_head):
self.done_revisions.append(new_head)
self.last_revision = new_head
self.d = None
class AdapterPmMetrics:
def __init__(self, device):
self.pm_names = {
'tx_64_pkts', 'tx_65_127_pkts', 'tx_128_255_pkts',
'tx_256_511_pkts', 'tx_512_1023_pkts', 'tx_1024_1518_pkts',
'tx_1519_9k_pkts', 'rx_64_pkts', 'rx_65_127_pkts',
'rx_128_255_pkts', 'rx_256_511_pkts', 'rx_512_1023_pkts',
'rx_1024_1518_pkts', 'rx_1519_9k_pkts'
}
self.device = device
self.id = device.id
self.name = 'ponsim_onu'
self.default_freq = 150
self.grouped = False
self.freq_override = False
self.pm_metrics = None
self.pon_metrics_config = dict()
self.uni_metrics_config = dict()
self.lc = None
for m in self.pm_names:
self.pon_metrics_config[m] = PmConfig(name=m,
type=PmConfig.COUNTER,
enabled=True)
self.uni_metrics_config[m] = PmConfig(name=m,
type=PmConfig.COUNTER,
enabled=True)
def update(self, pm_config):
if self.default_freq != pm_config.default_freq:
# Update the callback to the new frequency.
self.default_freq = pm_config.default_freq
self.lc.stop()
self.lc.start(interval=self.default_freq / 10)
for m in pm_config.metrics:
self.pon_metrics_config[m.name].enabled = m.enabled
self.uni_metrics_config[m.name].enabled = m.enabled
def make_proto(self):
pm_config = PmConfigs(id=self.id,
default_freq=self.default_freq,
grouped=False,
freq_override=False)
for m in sorted(self.pon_metrics_config):
pm = self.pon_metrics_config[m] # Either will do they're the same
pm_config.metrics.extend(
[PmConfig(name=pm.name, type=pm.type, enabled=pm.enabled)])
return pm_config
def extract_metrics(self, stats):
rtrn_port_metrics = dict()
rtrn_port_metrics['pon'] = self.extract_pon_metrics(stats)
rtrn_port_metrics['uni'] = self.extract_uni_metrics(stats)
return rtrn_port_metrics
def extract_pon_metrics(self, stats):
rtrn_pon_metrics = dict()
for m in stats.metrics:
if m.port_name == "pon":
for p in m.packets:
if self.pon_metrics_config[p.name].enabled:
rtrn_pon_metrics[p.name] = p.value
return rtrn_pon_metrics
def extract_uni_metrics(self, stats):
rtrn_pon_metrics = dict()
for m in stats.metrics:
if m.port_name == "uni":
for p in m.packets:
if self.pon_metrics_config[p.name].enabled:
rtrn_pon_metrics[p.name] = p.value
return rtrn_pon_metrics
def start_collector(self, callback):
log.info("starting-pm-collection",
device_name=self.name,
device_id=self.device.id)
prefix = 'voltha.{}.{}'.format(self.name, self.device.id)
self.lc = LoopingCall(callback, self.device.id, prefix)
self.lc.start(interval=self.default_freq / 10)
def stop_collector(self):
log.info("stopping-pm-collection",
device_name=self.name,
device_id=self.device.id)
self.lc.stop()
class PahoMqttAdapter(BaseMqttAdapter, Service):
# If connection fails at first, retry connecting each X seconds
retry_interval = 5
def connect(self):
"""
Connect to MQTT broker.
"""
# TODO: Check if we can do asynchronous connection establishment.
# Currently, this is done synchronously which could harm
# other subsystems in timeout or otherwise blocking situations.
# Make MQTT client identifier even more unique by adding process id
pid = os.getpid()
client_id = '{}:{}'.format(self.name, str(pid))
# Connection establishment
self.client = mqtt.Client(client_id=client_id, clean_session=True)
# Optionally authenticate connection
if self.broker_username:
self.client.username_pw_set(self.broker_username,
self.broker_password)
# Set event handlers
self.client.on_connect = lambda *args: reactor.callFromThread(
self.on_connect, *args)
self.client.on_message = lambda *args: reactor.callFromThread(
self.on_message, *args)
self.client.on_log = lambda *args: reactor.callFromThread(
self.on_log, *args)
# Connect with retry
self.connect_loop = LoopingCall(self.connect_with_retry)
self.connect_loop.start(self.retry_interval, now=True)
def connect_with_retry(self):
try:
self.client.connect(self.broker_host,
port=self.broker_port,
keepalive=60)
self.connect_loop.stop()
except:
log.failure(
u'Error connecting to MQTT broker but retrying each {retry_interval} seconds',
retry_interval=self.retry_interval)
return
"""
This is part of the threaded client interface. Call this once to
start a new thread to process network traffic. This provides an
alternative to repeatedly calling loop() yourself.
"""
self.client.loop_start()
reactor.addSystemEventTrigger('before', 'shutdown',
self.client.loop_stop, True)
# The callback for when the client receives a CONNACK response from the server.
def on_connect(self, client, userdata, flags, rc):
"""
on_connect(client, userdata, flags, rc): called when the broker responds to our connection
request.
flags is a dict that contains response flags from the broker:
flags['session present'] - this flag is useful for clients that are
using clean session set to 0 only. If a client with clean
session=0, that reconnects to a broker that it has previously
connected to, this flag indicates whether the broker still has the
session information for the client. If 1, the session still exists.
The value of rc determines success or not:
0: Connection successful
1: Connection refused - incorrect protocol version
2: Connection refused - invalid client identifier
3: Connection refused - server unavailable
4: Connection refused - bad username or password
5: Connection refused - not authorised
6-255: Currently unused.
"""
log.debug(
"Connected to MQTT. userdata={userdata}, flags={flags}, rc={rc}",
userdata=userdata,
flags=flags,
rc=rc)
# Subscribing in on_connect() means that if we lose the connection and
# reconnect then subscriptions will be renewed.
#client.subscribe("$SYS/#")
self.subscribe()
# The callback for when a PUBLISH message is received from the server.
def on_message(self, client, userdata, message):
"""
on_message(client, userdata, message): called when a message has been received on a
topic that the client subscribes to. The message variable is a
MQTTMessage that describes all of the message parameters.
"""
# TODO: Do something with "client" object (paho.mqtt.client.Client)
topic = message.topic
payload = message.payload
# Make metadata dictionary to be passed as kwargs later
# Mungle topic and payload out of metadata
metadata = message.__dict__.copy()
del metadata['topic']
del metadata['payload']
# Mungle userdata into message
metadata['userdata'] = userdata
if not topic.endswith('error.json'):
log.debug(
'on_message: name={name}, topic={topic}, payload={payload}, kwargs={kwargs}',
name=self.name,
topic=topic,
payload=payload,
kwargs=metadata)
return self.callback(topic=topic, payload=payload, **metadata)
def publish(self, topic, payload):
log.debug(u'Publishing to topic={topic}, payload={payload}',
topic=topic,
payload=payload)
return self.client.publish(topic, payload)
def subscribe(self, *args):
#d = self.protocol.subscribe("foo/bar/baz", 0)
log.info(u"Subscribing to topics {subscriptions}. client={client}",
subscriptions=self.subscriptions,
client=self.client)
for topic in self.subscriptions:
log.info(u"Subscribing to topic '{topic}'", topic=topic)
# Topic name **must not** be unicode, so casting to string
e = self.client.subscribe(str(topic), qos=0)
def on_log(self, client, userdata, level, buf):
"""
on_log(client, userdata, level, buf): called when the client has log information. Define
to allow debugging. The level variable gives the severity of the message
and will be one of MQTT_LOG_INFO, MQTT_LOG_NOTICE, MQTT_LOG_WARNING,
MQTT_LOG_ERR, and MQTT_LOG_DEBUG. The message itself is in buf.
"""
log.debug(u'{message}. level={level_mqtt}, userdata={userdata}',
message=buf,
level_mqtt=level,
userdata=userdata)
class TinderboxPoller(base.ChangeSource):
"""This source will poll a tinderbox server for changes and submit
them to the change master."""
compare_attrs = ["tinderboxURL", "pollInterval", "tree", "branch"]
parent = None # filled in when we're added
loop = None
volatile = ['loop']
working = False
def __init__(self, tinderboxURL, branch, tree="Firefox", machines=[], pollInterval=30):
"""
@type tinderboxURL: string
@param tinderboxURL: The base URL of the Tinderbox server
(ie. http://tinderbox.mozilla.org)
@type tree: string
@param tree: The tree to look for changes in.
For example, Firefox trunk is 'Firefox'
@type branch: string
@param branch: The branch to look for changes in. This must
match the 'branch' option for the Scheduler.
@type machines: list
@param machines: A list of machine names to search for. Changes will
only register for machines that match individual
"machine" substrings
@type pollInterval: int
@param pollInterval: The time (in seconds) between queries for
changes
"""
self.tinderboxURL = tinderboxURL
self.tree = tree
self.branch = branch
self.machines = machines
self.pollInterval = pollInterval
self.previousChange = ''
self.lastPoll = time.time()
self.lastChange = time.time()
def startService(self):
self.loop = LoopingCall(self.poll)
base.ChangeSource.startService(self)
reactor.callLater(0, self.loop.start, self.pollInterval)
def stopService(self):
self.loop.stop()
return base.ChangeSource.stopService(self)
def describe(self):
str = ""
str += "Getting changes from the Tinderbox service running at %s " \
% self.tinderboxURL
str += "<br>Using tree: %s, branch %s, hostname %s" % (self.tree, self.branch, str(self.machines))
return str
def poll(self):
if self.working:
log.msg("Not polling Tinderbox because last poll is still working")
else:
self.working = True
d = self._get_changes()
d.addCallback(self._process_changes)
d.addBoth(self._finished)
return
def _finished(self, res):
assert self.working
self.working = False
# check for failure
if isinstance(res, failure.Failure):
log.msg("Tinderbox poll failed: %s" % res)
return res
def _make_url(self):
# build the tinderbox URL
url = self.tinderboxURL
url += "/" + self.tree
url += "/" + "quickparse.txt"
return url
def _get_changes(self):
url = self._make_url()
log.msg("Polling Tinderbox tree at %s" % url)
self.lastPoll = time.time()
# get the page, in pipe-delimited format
return defer.maybeDeferred(urlopen, url)
def _process_changes(self, query):
try:
tp = TinderboxParser(query)
buildList = tp.getData()
except InvalidResultError, e:
log.msg("Could not process Tinderbox query: " + e.value)
return
except EmptyResult:
return
if __name__ == '__main__':
if len(sys.argv) != 3:
print 'GAMESERVER: Usage: python game_server <number of players> <PORT>'
exit()
# grab the number of players in this game
player_count = int(sys.argv[1])
GAME_PORT = int(sys.argv[2])
player_range = range(player_count)
# initialize gamespace and generate the player objects within game
gs = GameSpace(player_range)
# Listen for players to connect
reactor.listenTCP(GAME_PORT, GameServerConnectionFactory(gs.players))
# initialize game loop
lc = LoopingCall(gs.game_loop_iterate)
lc.start(1.0 / 20)
# begin reactor event loop
reactor.run()
# after reactor stops, end game loop
lc.stop()
# clean exit
os._exit(0)
class TriggerBouncerCheck(Triggerable):
compare_attrs = Triggerable.compare_attrs + \
('minUptake', 'configRepo', 'checkMARs', 'username',
'password', 'pollInterval', 'pollTimeout')
working = False
loop = None
release_config = None
script_repo_revision = None
configRepo = None
def __init__(self, minUptake, configRepo, checkMARs=True,
username=None, password=None, pollInterval=5*60,
pollTimeout=12*60*60, **kwargs):
self.minUptake = minUptake
self.configRepo = configRepo
self.checkMARs = checkMARs
self.username = username
self.password = password
self.pollInterval = pollInterval
self.pollTimeout = pollTimeout
self.ss = None
self.set_props = None
Triggerable.__init__(self, **kwargs)
def trigger(self, ss, set_props=None):
self.ss = ss
self.set_props = set_props
props = Properties()
props.updateFromProperties(self.properties)
if set_props:
props.updateFromProperties(set_props)
self.script_repo_revision = props.getProperty('script_repo_revision')
assert self.script_repo_revision, 'script_repo_revision should be set'
self.release_config = props.getProperty('release_config')
assert self.release_config, 'release_config should be set'
def _run_loop(_):
self.loop = LoopingCall(self.poll)
reactor.callLater(0, self.loop.start, self.pollInterval)
reactor.callLater(self.pollTimeout, self.stopLoop,
'Timeout after %s' % self.pollTimeout)
d = self.getReleaseConfig()
d.addCallback(_run_loop)
def stopLoop(self, reason=None):
if reason:
log.msg('%s: Stopping uptake monitoring: %s' %
(self.__class__.__name__, reason))
if self.loop.running:
self.loop.stop()
else:
log.msg('%s: Loop has been alredy stopped' %
self.__class__.__name__)
def getReleaseConfig(self):
url = str('%s/raw-file/%s/%s' %
(self.configRepo, self.script_repo_revision, self.release_config))
d = getPage(url)
def setReleaseConfig(res):
c = {}
exec res in c
self.release_config = c.get('releaseConfig')
log.msg('%s: release_config loaded' % self.__class__.__name__)
d.addCallback(setReleaseConfig)
return d
def poll(self):
if self.working:
log.msg('%s: Not polling because last poll is still working'
% self.__class__.__name__)
return defer.succeed(None)
self.working = True
log.msg('%s: polling' % self.__class__.__name__)
bouncerProductName = self.release_config.get('bouncerProductName') or \
self.release_config.get('productName').capitalize()
d = get_release_uptake(
tuxedoServerUrl=self.release_config.get('tuxedoServerUrl'),
bouncerProductName=bouncerProductName,
version=self.release_config.get('version'),
platforms=self.release_config.get('enUSPlatforms'),
oldVersion=self.release_config.get('oldVersion'),
checkMARs=self.checkMARs,
username=self.username,
password=self.password)
d.addCallback(self.checkUptake)
d.addCallbacks(self.finished_ok, self.finished_failure)
return d
def checkUptake(self, uptake):
log.msg('%s: uptake is %s' % (self.__class__.__name__, uptake))
if uptake >= self.minUptake:
self.stopLoop('Reached required uptake: %s' % uptake)
Triggerable.trigger(self, self.ss, self.set_props)
def finished_ok(self, res):
log.msg('%s: polling finished' % (self.__class__.__name__))
assert self.working
self.working = False
return res
def finished_failure(self, f):
log.msg('%s failed:\n%s' % (self.__class__.__name__, f.getTraceback()))
assert self.working
self.working = False
return None # eat the failure
class triviabot(irc.IRCClient):
'''
This is the irc bot portion of the trivia bot.
It implements the whole program so is kinda big. The algorithm is
implemented by a series of callbacks, initiated by an admin on the
server.
'''
def __init__(self):
self._answer = Answer()
self._question = ''
self._scores = {}
self._clue_number = 0
self._admins = list(config.ADMINS)
self._game_channel = config.GAME_CHANNEL
self._current_points = 5
self._questions_dir = config.Q_DIR
self._lc = LoopingCall(self._play_game)
self._load_game()
self._votes = 0
self._voters = []
def _get_nickname(self):
return self.factory.nickname
nickname = property(_get_nickname)
def _get_lineRate(self):
return self.factory.lineRate
lineRate = property(_get_lineRate)
def _cmsg(self, dest, msg):
"""
Write a colorized message.
"""
self.msg(dest, "%s%s" % (config.COLOR_CODE, msg))
def _gmsg(self, msg):
"""
Write a message to the channel playing the trivia game.
"""
self._cmsg(self._game_channel, msg)
def _play_game(self):
'''
Implements the main loop of the game.
'''
points = {0: 5, 1: 3, 2: 2, 3: 1}
if self._clue_number == 0:
self._votes = 0
self._voters = []
self._get_new_question()
self._current_points = points[self._clue_number]
# Blank line.
self._gmsg("")
self._gmsg("Next question:")
self._gmsg(self._question)
self._gmsg("Clue: %s" % self._answer.current_clue())
self._clue_number += 1
# we must be somewhere in between
elif self._clue_number < 4:
self._current_points = points[self._clue_number]
self._gmsg("Question:")
self._gmsg(self._question)
self._gmsg('Clue: %s' % self._answer.give_clue())
self._clue_number += 1
# no one must have gotten it.
else:
self._gmsg('No one got it. The answer was: %s' %
self._answer.answer)
self._clue_number = 0
self._get_new_question()
#self._lc.reset()
def signedOn(self):
'''
Actions to perform on signon to the server.
'''
self.join(self._game_channel)
self.msg('NickServ', 'identify %s' % config.IDENT_STRING)
print("Signed on as %s." % (self.nickname, ))
if self.factory.running:
self._start(None, None, None)
else:
self._gmsg('Welcome to %s!' % self._game_channel)
self._gmsg("Have an admin start the game when you are ready.")
self._gmsg("For how to use this bot, just say ?help or")
self._gmsg("%s help." % self.nickname)
def joined(self, channel):
'''
Callback runs when the bot joins a channel
'''
print("Joined %s." % (channel, ))
def privmsg(self, user, channel, msg):
'''
Parses out each message and initiates doing the right thing
with it.
'''
user, temp = user.split('!')
print(user + " : " + channel + " : " + msg)
# need to strip off colors if present.
try:
while not msg[0].isalnum() and not msg[0] == '?':
msg = msg[1:]
except IndexError:
return
# parses each incoming line, and sees if it's a command for the bot.
try:
if (msg[0] == "?"):
command = msg.replace('?', '').split()[0]
args = msg.replace('?', '').split()[1:]
self.select_command(command, args, user, channel)
return
elif (msg.split()[0].find(self.nickname) == 0):
command = msg.split()[1]
args = msg.replace(self.nickname, '').split()[2:]
self.select_command(command, args, user, channel)
return
# if not, try to match the message to the answer.
else:
if msg.lower().strip() == self._answer.answer.lower():
self._winner(user, channel)
self._save_game()
except:
return
def _winner(self, user, channel):
'''
Congratulates the winner for guessing correctly and assigns
points appropriately, then signals that it was guessed.
'''
if channel != self._game_channel:
self.msg(channel,
"I'm sorry, answers must be given in the game channel.")
return
self._gmsg("%s GOT IT!" % user.upper())
try:
self._scores[user] += self._current_points
except:
self._scores[user] = self._current_points
if self._current_points == 1:
self._gmsg("%s point has been added to your score!" %
str(self._current_points))
else:
self._gmsg("%s points has been added to your score!" %
str(self._current_points))
self._clue_number = 0
self._get_new_question()
def ctcpQuery(self, user, channel, msg):
'''
Responds to ctcp requests.
Currently just reports them.
'''
print("CTCP recieved: " + user + ":" + channel + ": " + msg[0][0] +
" " + msg[0][1])
def _help(self, args, user, channel):
'''
Tells people how to use the bot.
Replies differently if you are an admin or a regular user.
Only responds to the user since there could be a game in
progress.
'''
try:
self._admins.index(user)
except:
self._cmsg(user, "I'm nameless's trivia bot.")
self._cmsg(
user, "Commands: score, standings, giveclue, help, "
"next, source")
return
self._cmsg(user, "I'm nameless's trivia bot.")
self._cmsg(
user, "Commands: score, standings, giveclue, help, next, "
"skip, source")
self._cmsg("Admin commands: die, set <user> <score>, start, stop, "
"save")
def _show_source(self, args, user, channel):
'''
Tells people how to use the bot.
Only responds to the user since there could be a game in
progress.
'''
self._cmsg(
user, 'My source can be found at: '
'https://github.com/rawsonj/triviabot')
def select_command(self, command, args, user, channel):
'''
Callback that responds to commands given to the bot.
Need to differentiate between priviledged users and regular
users.
'''
# set up command dicts.
unpriviledged_commands = {
'score': self._score,
'help': self._help,
'source': self._show_source,
'standings': self._standings,
'giveclue': self._give_clue,
'next': self._next_vote,
'skip': self._next_question
}
priviledged_commands = {
'die': self._die,
'set': self._set_user_score,
'start': self._start,
'stop': self._stop,
'save': self._save_game,
}
print command, args, user, channel
try:
self._admins.index(user)
is_admin = True
except:
is_admin = False
# the following takes care of sorting out functions and
# priviledges.
if not is_admin and priviledged_commands.has_key(command):
self.msg(channel, "%s: You don't tell me what to do." % user)
return
elif is_admin and priviledged_commands.has_key(command):
priviledged_commands[command](args, user, channel)
elif unpriviledged_commands.has_key(command):
unpriviledged_commands[command](args, user, channel)
else:
self.describe(channel,
'%slooks at %s oddly.' % (config.COLOR_CODE, user))
def _next_vote(self, args, user, channel):
'''Implements user voting for the next question.
Need to keep track of who voted, and how many votes.
'''
if not self._lc.running:
self._gmsg("We aren't playing right now.")
return
try:
self._voters.index(user)
self._gmsg("You already voted, %s, give someone else a chance to "
"hate this question" % user)
return
except:
if self._votes < 2:
self._votes += 1
self._voters.append(user)
print self._voters
self._gmsg("%s, you have voted. %s more votes needed to "
"skip." % (user, str(3 - self._votes)))
else:
self._votes = 0
self._voters = []
self._next_question(None, None, None)
def _start(self, args, user, channel):
'''
Starts the trivia game.
TODO: Load scores from last game, if any.
'''
if self._lc.running:
return
else:
self._lc.start(config.WAIT_INTERVAL)
self.factory.running = True
def _stop(self, *args):
'''
Stops the game and thanks people for playing,
then saves the scores.
'''
if not self._lc.running:
return
else:
self._lc.stop()
self._gmsg('Thanks for playing trivia!')
self._gmsg('Current rankings were:')
self._standings(None, self._game_channel, None)
self._gmsg('''Scores have been saved, and see you next game!''')
self._save_game()
self.factory.running = False
def _save_game(self, *args):
'''
Saves the game to the data directory.
'''
if not path.exists(config.SAVE_DIR):
makedirs(config.SAVE_DIR)
with open(config.SAVE_DIR + 'scores.json', 'w') as savefile:
json.dump(self._scores, savefile)
print "Scores have been saved."
def _load_game(self):
'''
Loads the running data from previous games.
'''
# ensure initialization
self._scores = {}
if not path.exists(config.SAVE_DIR):
print "Save directory doesn't exist."
return
try:
with open(config.SAVE_DIR + 'scores.json', 'r') as savefile:
temp_dict = json.load(savefile)
except:
print "Save file doesn't exist."
return
for name in temp_dict.keys():
self._scores[str(name)] = int(temp_dict[name])
print self._scores
print "Scores loaded."
def _set_user_score(self, args, user, channel):
'''
Administrative action taken to adjust scores, if needed.
'''
try:
self._scores[args[0]] = int(args[1])
except:
self._cmsg(user, args[0] + " not in scores database.")
return
self._cmsg(user, args[0] + " score set to " + args[1])
def _die(self, *args):
'''
Terminates execution of the bot.
Need to dig into twisted to figure out how this happens.
'''
global reactor
self.quit(message='This is triviabot, signing off.')
reactor.stop()
# figure out how to kill the bot
def _score(self, args, user, channel):
'''
Tells the user their score.
'''
try:
self._cmsg(user,
"Your current score is: %s" % str(self._scores[user]))
except:
self._cmsg(user, "You aren't in my database.")
def _next_question(self, args, user, channel):
'''
Administratively skips the current question.
'''
if not self._lc.running:
self._gmsg("We are not playing right now.")
return
self._gmsg("Question has been skipped. The answer was: %s" %
self._answer.answer)
self._clue_number = 0
self._lc.stop()
self._lc.start(config.WAIT_INTERVAL)
def _standings(self, args, user, channel):
'''
Tells the user the complete standings in the game.
TODO: order them.
'''
self._cmsg(user, "The current trivia standings are: ")
sorted_scores = sorted(self._scores.iteritems(),
key=lambda (k, v): (v, k),
reverse=True)
for rank, (player, score) in enumerate(sorted_scores, start=1):
formatted_score = "%s: %s: %s" % (rank, player, score)
self._cmsg(user, str(formatted_score))
def _give_clue(self, args, user, channel):
if not self._lc.running:
self._gmsg("we are not playing right now.")
return
self._cmsg(channel, "Question: ")
self._cmsg(channel, self._question)
self._cmsg(channel, "Clue: " + self._answer.current_clue())
def _get_new_question(self):
'''
Selects a new question from the questions directory and
sets it.
'''
damaged_question = True
while damaged_question:
#randomly select file
filename = choice(listdir(self._questions_dir))
fd = open(config.Q_DIR + filename)
lines = fd.read().splitlines()
myline = choice(lines)
fd.close()
try:
self._question, temp_answer = myline.split('`')
except ValueError:
print "Broken question:"
print myline
continue
self._answer.set_answer(temp_answer.strip())
damaged_question = False
class IRCUser(IRCBase):
def __init__(self, ircd, ip, uuid=None, host=None):
self.ircd = ircd
self.uuid = ircd.createUUID() if uuid is None else uuid
registrationTimeout = self.ircd.config.get("user_registration_timeout",
10)
self.nick = None
self.ident = None
if ip[0] == ":": # Normalize IPv6 address for IRC
ip = "0{}".format(ip)
if host is None:
self.realHost = ip
else:
self.realHost = host
self.ip = ip
self._hostStack = []
self._hostsByType = {}
self.gecos = None
self._metadata = CaseInsensitiveDictionary()
self.cache = {}
self.channels = []
self.modes = {}
self.connectedSince = now()
self.nickSince = now()
self.idleSince = now()
self._registerHolds = set(("connection", "dns", "NICK", "USER"))
self.disconnectedDeferred = Deferred()
self._messageBatches = {}
self._errorBatchName = None
self._errorBatch = []
self.ircd.users[self.uuid] = self
self.localOnly = False
self.secureConnection = False
self._pinger = LoopingCall(self._ping)
self._registrationTimeoutTimer = reactor.callLater(
registrationTimeout, self._timeoutRegistration)
self._connectHandlerTimer = None
self._startDNSResolving(registrationTimeout)
def _startDNSResolving(self, timeout):
ip = self.ip
if ipIsV4(ip):
addr = "{}.in-addr.arpa".format(".".join(reversed(ip.split("."))))
else:
addr = reversed(expandIPv6Address(ip).replace(":", ""))
addr = "{}.ip6.arpa".format(".".join(addr))
resolveDeferred = dnsClient.lookupPointer(addr, ((timeout / 2), ))
resolveDeferred.addCallbacks(callback=self._verifyDNSResolution,
callbackArgs=(timeout, ),
errback=self._cancelDNSResolution)
def _verifyDNSResolution(self, result, timeout):
name = result[0][0].payload.name.name
if len(name) > self.ircd.config.get("hostname_length", 64):
self._cancelDNSResolution()
return
if not isValidHost(name):
self._cancelDNSResolution()
return
resolveDeferred = dnsClient.getHostByName(name, ((timeout / 2), ))
resolveDeferred.addCallbacks(callback=self._completeDNSResolution,
errback=self._cancelDNSResolution,
callbackArgs=(name, ))
def _completeDNSResolution(self, result, name):
if result == self.ip:
self.realHost = name
self.register("dns")
def _cancelDNSResolution(self, error=None):
self.register("dns")
def connectionMade(self):
# We need to callLater the connect action call because the connection isn't fully set up yet,
# nor is it fully set up even with a delay of zero, which causes the message buffer not to be sent
# when the connection is closed.
# The "connection" register hold is used basically solely for the purposes of this to prevent potential
# race conditions with registration.
self._connectHandlerTimer = reactor.callLater(0.1,
self._callConnectAction)
if ISSLTransport.providedBy(self.transport):
self.secureConnection = True
def _callConnectAction(self):
self._connectHandlerTimer = None
if self.ircd.runActionUntilFalse("userconnect", self, users=[self]):
self.transport.loseConnection()
else:
self.register("connection")
def dataReceived(self, data):
self.ircd.runActionStandard("userrecvdata", self, data, users=[self])
try:
IRCBase.dataReceived(self, data)
except Exception:
self.ircd.log.failure(
"An error occurred while processing incoming data.")
if self.uuid in self.ircd.users:
self.disconnect("Error occurred")
def sendLine(self, line):
self.ircd.runActionStandard("usersenddata", self, line, users=[self])
IRCBase.sendLine(self, line)
def sendMessage(self, command, *args, **kw):
"""
Sends the given message to this user.
Accepts the following keyword arguments:
- prefix: The message prefix or None to suppress the default prefix
If not given, defaults to the server name.
- to: The destination of the message or None if the message has no
destination. The implicit destination is this user if this
argument isn't specified.
- tags: Dict of message tags to send.
- alwaysPrefixLastParam: For compatibility with some broken clients,
you might want some messages to always have the last parameter
prefixed with a colon. To do that, pass this as True.
"""
if "prefix" not in kw:
kw["prefix"] = self.ircd.name
if kw["prefix"] is None:
del kw["prefix"]
to = self.nick if self.nick else "*"
if "to" in kw:
to = kw["to"]
del kw["to"]
if to:
args = [to] + list(args)
self.ircd.runActionStandard("modifyoutgoingmessage", self, command,
args, kw)
IRCBase.sendMessage(self, command, *args, **kw)
def handleCommand(self, command, params, prefix, tags):
if self.uuid not in self.ircd.users:
return # we have been disconnected - ignore all further commands
if command in self.ircd.userCommands:
handlers = self.ircd.userCommands[command]
if not handlers:
return
data = None
spewRegWarning = True
affectedUsers = []
affectedChannels = []
for handler in handlers:
if handler[0].forRegistered is not None:
if (handler[0].forRegistered is True
and not self.isRegistered()) or (
handler[0].forRegistered is False
and self.isRegistered()):
continue
spewRegWarning = False
data = handler[0].parseParams(self, params, prefix, tags)
if data is not None:
affectedUsers = handler[0].affectedUsers(self, data)
affectedChannels = handler[0].affectedChannels(self, data)
if self not in affectedUsers:
affectedUsers.append(self)
break
if data is None:
if spewRegWarning:
if self.isRegistered():
self.sendMessage(irc.ERR_ALREADYREGISTERED,
"You may not reregister")
else:
self.sendMessage(irc.ERR_NOTREGISTERED, command,
"You have not registered")
elif self._hasBatchedErrors():
self._dispatchErrorBatch()
return
self._clearErrorBatch()
if self.ircd.runComboActionUntilValue(
(("commandpermission-{}".format(command), self, data),
("commandpermission", self, command, data)),
users=affectedUsers,
channels=affectedChannels) is False:
return
self.ircd.runComboActionStandard(
(("commandmodify-{}".format(command), self, data),
("commandmodify", self, command, data)),
users=affectedUsers,
channels=affectedChannels
) # This allows us to do processing without the "stop on empty" feature of runActionProcessing
for handler in handlers:
if handler[0].execute(self, data):
if handler[0].resetsIdleTime:
self.idleSince = now()
break # If the command executor returns True, it was handled
else:
return # Don't process commandextra if it wasn't handled
self.ircd.runComboActionStandard(
(("commandextra-{}".format(command), self, data),
("commandextra", self, command, data)),
users=affectedUsers,
channels=affectedChannels)
else:
if not self.ircd.runActionFlagTrue("commandunknown", self, command,
params, {}):
self.sendMessage(irc.ERR_UNKNOWNCOMMAND, command,
"Unknown command")
def createMessageBatch(self, batchName, batchType, batchParameters=None):
"""
Start a new message batch with the given batch name, type, and list of parameters.
If a batch with the given name already exists, that batch will be overwritten.
"""
self._messageBatches[batchName] = {
"type": batchType,
"parameters": batchParameters,
"messages": []
}
def sendMessageInBatch(self, batchName, command, *args, **kw):
"""
Adds a message to the batch with the given name.
"""
if batchName not in self._messageBatches:
return
self._messageBatches[batchName]["messages"].append((command, args, kw))
def sendBatch(self, batchName):
"""
Sends the messages in the given batch to the user.
"""
if batchName not in self._messageBatches:
return
batchType = self._messageBatches[batchName]["type"]
batchParameters = self._messageBatches[batchName]["parameters"]
self.ircd.runActionStandard("startbatchsend", self, batchName,
batchType, batchParameters)
for messageData in self._messageBatches[batchName]["messages"]:
self.sendMessage(messageData[0], *messageData[1], **messageData[2])
self.ircd.runActionStandard("endbatchsend", self, batchName, batchType,
batchParameters)
def startErrorBatch(self, batchName):
"""
Used to start an error batch when sending multiple error messages to a
user from a command's parseParams or from the commandpermission action.
"""
if not self._errorBatchName or not self._errorBatch: # Only the first batch should apply
self._errorBatchName = batchName
def sendBatchedError(self, batchName, command, *args, **kw):
"""
Adds an error to the current error batch if the specified error batch
is the current error batch.
"""
if batchName and self._errorBatchName == batchName:
self._errorBatch.append((command, args, kw))
def sendSingleError(self, batchName, command, *args, **kw):
"""
Creates a batch containing a single error and adds the specified error
to it.
"""
if not self._errorBatchName:
self._errorBatchName = batchName
self._errorBatch.append((command, args, kw))
def _hasBatchedErrors(self):
if self._errorBatch:
return True
return False
def _clearErrorBatch(self):
self._errorBatchName = None
self._errorBatch = []
def _dispatchErrorBatch(self):
for error in self._errorBatch:
self.sendMessage(error[0], *error[1], **error[2])
self._clearErrorBatch()
def filterConditionalTags(self, conditionalTags):
applyTags = {}
for tag, data in conditionalTags.iteritems():
value, check = data
if check(self):
applyTags[tag] = value
return applyTags
def connectionLost(self, reason):
if self.uuid in self.ircd.users:
self.disconnect("Connection reset")
self.disconnectedDeferred.callback(None)
def disconnect(self, reason):
"""
Disconnects the user from the server.
"""
self.ircd.log.debug(
"Disconnecting user {user.uuid} ({user.hostmask()}): {reason}",
user=self,
reason=reason)
# Sometimes, actions deferred from initial connection may cause registration to occur after disconnection if
# disconnection happens before registration completes. If the user is unregistered on disconnection, this prevents
# the user from completing registration.
self.addRegisterHold("QUIT")
if self._pinger:
if self._pinger.running:
self._pinger.stop()
self._pinger = None
if self._registrationTimeoutTimer:
if self._registrationTimeoutTimer.active():
self._registrationTimeoutTimer.cancel()
self._registrationTimeoutTimer = None
if self._connectHandlerTimer and self._connectHandlerTimer.active():
self._connectHandlerTimer.cancel()
self._connectHandlerTimer = None
self.ircd.recentlyQuitUsers[self.uuid] = now()
del self.ircd.users[self.uuid]
if self.isRegistered():
del self.ircd.userNicks[self.nick]
userSendList = [self]
while self.channels:
channel = self.channels[0]
userSendList.extend(channel.users.keys())
self._leaveChannel(channel)
userSendList = [
u for u in set(userSendList) if u.uuid[:3] == self.ircd.serverID
]
userSendList.remove(self)
self.ircd.runActionProcessing("quitmessage",
userSendList,
self,
reason,
users=[self] + userSendList)
self.ircd.runActionStandard("quit", self, reason, users=self)
self.transport.loseConnection()
def _timeoutRegistration(self):
if self.isRegistered():
self._pinger.start(self.ircd.config.get("user_ping_frequency", 60),
False)
return
self.disconnect("Registration timeout")
def _ping(self):
self.ircd.runActionStandard("pinguser", self)
def isRegistered(self):
"""
Returns True if this user session is fully registered.
"""
return not self._registerHolds
def register(self, holdName):
"""
Removes the specified hold on a user's registration. If this is the
last hold on a user, completes registration on the user.
"""
if holdName not in self._registerHolds:
return
self._registerHolds.remove(holdName)
if not self._registerHolds:
if not self.nick or self.nick in self.ircd.userNicks:
self._registerHolds.add("NICK")
if not self.ident or not self.gecos:
self._registerHolds.add("USER")
if self._registerHolds:
return
self._registerHolds.add(
"registercheck"
) # The user shouldn't be considered registered until we complete these final checks
if self.ircd.runActionUntilFalse("register", self, users=[self]):
self.transport.loseConnection()
return
self._registerHolds.remove("registercheck")
self.ircd.userNicks[self.nick] = self.uuid
self.ircd.log.debug(
"Registering user {user.uuid} ({user.hostmask()})", user=self)
versionWithName = "txircd-{}".format(version)
self.sendMessage(
irc.RPL_WELCOME,
"Welcome to the {} Internet Relay Chat Network {}".format(
self.ircd.config["network_name"], self.hostmask()))
self.sendMessage(
irc.RPL_YOURHOST, "Your host is {}, running version {}".format(
self.ircd.name, versionWithName))
self.sendMessage(
irc.RPL_CREATED, "This server was created {}".format(
self.ircd.startupTime.replace(microsecond=0)))
chanModes = "".join(
["".join(modes.keys()) for modes in self.ircd.channelModes])
chanModes += "".join(self.ircd.channelStatuses.keys())
self.sendMessage(
irc.RPL_MYINFO, self.ircd.name, versionWithName, "".join(
["".join(modes.keys()) for modes in self.ircd.userModes]),
chanModes)
self.sendISupport()
self.ircd.runActionStandard("welcome", self, users=[self])
def addRegisterHold(self, holdName):
"""
Adds a register hold to this user if the user is not yet registered.
"""
if not self._registerHolds:
return
self._registerHolds.add(holdName)
def sendISupport(self):
"""
Sends ISUPPORT to this user."""
isupportList = self.ircd.generateISupportList()
isupportMsgList = splitMessage(" ".join(isupportList), 350)
for line in isupportMsgList:
lineArgs = line.split(" ")
lineArgs.append("are supported by this server")
self.sendMessage(irc.RPL_ISUPPORT, *lineArgs)
def hostmask(self):
"""
Returns the user's hostmask.
"""
return "{}!{}@{}".format(self.nick, self.ident, self.host())
def hostmaskWithRealHost(self):
"""
Returns the user's hostmask using the user's real host rather than any
vhost that may have been applied.
"""
return "{}!{}@{}".format(self.nick, self.ident, self.realHost)
def hostmaskWithIP(self):
"""
Returns the user's hostmask using the user's IP address instead of the
host.
"""
return "{}!{}@{}".format(self.nick, self.ident, self.ip)
def changeNick(self, newNick, fromServer=None):
"""
Changes this user's nickname. If initiated by a remote server, that
server should be specified in the fromServer parameter.
"""
if newNick == self.nick:
return
if newNick in self.ircd.userNicks and self.ircd.userNicks[
newNick] != self.uuid:
return
oldNick = self.nick
if oldNick and oldNick in self.ircd.userNicks:
del self.ircd.userNicks[self.nick]
self.nick = newNick
self.nickSince = now()
if self.isRegistered():
self.ircd.userNicks[self.nick] = self.uuid
userSendList = [self]
for channel in self.channels:
userSendList.extend(channel.users.keys())
userSendList = [
u for u in set(userSendList)
if u.uuid[:3] == self.ircd.serverID
]
self.ircd.runActionProcessing("changenickmessage",
userSendList,
self,
oldNick,
users=userSendList)
self.ircd.runActionStandard("changenick",
self,
oldNick,
fromServer,
users=[self])
def changeIdent(self, newIdent, fromServer=None):
"""
Changes this user's ident. If initiated by a remote server, that server
should be specified in the fromServer parameter.
"""
if newIdent == self.ident:
return
if len(newIdent) > self.ircd.config.get("ident_length", 12):
return
oldIdent = self.ident
self.ident = newIdent
if self.isRegistered():
self.ircd.runActionStandard("changeident",
self,
oldIdent,
fromServer,
users=[self])
def host(self):
if not self._hostStack:
return self.realHost
return self._hostsByType[self._hostStack[-1]]
def changeHost(self, hostType, newHost, fromServer=None):
"""
Changes a user's host. If initiated by a remote server, that server
should be specified in the fromServer parameter.
"""
if hostType == "*":
return
if len(newHost) > self.ircd.config.get("hostname_length", 64):
return
if hostType in self._hostsByType and self._hostsByType[
hostType] == newHost:
return
oldHost = self.host()
self._hostsByType[hostType] = newHost
if hostType in self._hostStack:
self._hostStack.remove(hostType)
self._hostStack.append(hostType)
if self.isRegistered():
self.ircd.runComboActionStandard(
(("changehost", self, hostType, oldHost, fromServer),
("updatehost", self, hostType, oldHost, newHost, fromServer)),
users=[self])
def updateHost(self, hostType, newHost, fromServer=None):
"""
Updates the host of a given host type for the user. If initiated by
a remote server, that server should be specified in the fromServer
parameter.
"""
if hostType not in self._hostStack:
self.changeHost(hostType, newHost, fromServer)
return
if hostType == "*":
return
if len(newHost) > self.ircd.config.get("hostname_length", 64):
return
if hostType in self._hostsByType and self._hostsByType[
hostType] == newHost:
return
oldHost = self.host()
oldHostOfType = None
if hostType in self._hostsByType:
oldHostOfType = self._hostsByType[hostType]
self._hostsByType[hostType] = newHost
changedUserHost = (oldHost != self.host())
changedHostOfType = (oldHostOfType != newHost)
if self.isRegistered():
if changedUserHost and changedHostOfType:
self.ircd.runComboActionStandard(
(("changehost", self, hostType, oldHost, fromServer),
("updatehost", self, hostType, oldHost, newHost,
fromServer)),
users=[self])
elif changedHostOfType:
self.ircd.runActionStandard("updatehost",
self,
hostType,
oldHost,
newHost,
fromServer,
users=[self])
def resetHost(self, hostType, fromServer=None):
"""
Resets the user's host to the real host.
"""
if hostType not in self._hostsByType:
return
oldHost = self.host()
if hostType in self._hostStack:
self._hostStack.remove(hostType)
del self._hostsByType[hostType]
currentHost = self.host()
if currentHost != oldHost:
self.ircd.runComboActionStandard(
(("changehost", self, hostType, oldHost, fromServer),
("updatehost", self, hostType, oldHost, None, fromServer)),
users=[self])
else:
self.ircd.runActionStandard("updatehost",
self,
hostType,
oldHost,
None,
fromServer,
users=[self])
def currentHostType(self):
if self._hostStack:
return self._hostStack[-1]
return "*"
def changeGecos(self, newGecos, fromServer=None):
"""
Changes a user's real name. If initiated by a remote server, that
server should be specified in the fromServer parameter.
"""
if len(newGecos) > self.ircd.config.get("gecos_length", 128):
return
if newGecos == self.gecos:
return
oldGecos = self.gecos
self.gecos = newGecos
if self.isRegistered():
self.ircd.runActionStandard("changegecos",
self,
oldGecos,
fromServer,
users=[self])
def metadataKeyExists(self, key):
"""
Checks whether the specified key exists in the user's metadata.
"""
return key in self._metadata
def metadataKeyCase(self, key):
"""
Returns the specified key in the user's metadata in its original case.
Returns None if the given key is not in the user's metadata.
"""
if key not in self._metadata:
return None
return self._metadata[key][0]
def metadataValue(self, key):
"""
Returns the value of the given key in the user's metadata or None if
the given key is not in the user's metadata.
"""
if key not in self._metadata:
return None
return self._metadata[key][1]
def metadataVisibility(self, key):
"""
Returns the visibility value of the given key in the user's metadata or
None if the given key is not in the user's metadata.
"""
if key not in self._metadata:
return None
return self._metadata[key][2]
def metadataSetByUser(self, key):
"""
Returns whether the given key in the user's metadata was set by a user
or None if the given key is not in the user's metadata.
"""
if key not in self._metadata:
return None
return self._metadata[key][3]
def metadataList(self):
"""
Returns the list of metadata keys/values for the user as a list of
tuples in the format
[ (key, value, visibility, setByUser) ]
"""
return self._metadata.values()
def setMetadata(self, key, value, visibility, setByUser, fromServer=None):
"""
Sets metadata for the user. If initiated by a remote server, that
server should be specified in the fromServer parameter.
If the value is None, deletes the metadata at the provided key.
"""
if not isValidMetadataKey(key):
return False
oldData = None
if key in self._metadata:
oldData = self._metadata[key]
if setByUser and oldData and not oldData[3]:
return False
if setByUser and self.ircd.runActionUntilValue(
"usercansetmetadata", key, users=[self]) is False:
return False
if value is None:
if key in self._metadata:
del self._metadata[key]
elif not visibility:
return False
else:
self._metadata[key] = (key, value, visibility, setByUser)
oldValue = oldData[1] if oldData else None
self.ircd.runActionStandard("usermetadataupdate",
self,
key,
oldValue,
value,
visibility,
setByUser,
fromServer,
users=[self])
return True
def canSeeMetadataVisibility(self, visibility):
if visibility == "*":
return True
return self.ircd.runActionUntilValue("usercanseemetadata", self,
visibility) is not False
def joinChannel(self, channel, override=False):
"""
Joins the user to a channel. Specify the override parameter only if all
permission checks should be bypassed.
"""
if channel in self.channels:
return
if not override:
if self.ircd.runActionUntilValue("joinpermission",
channel,
self,
users=[self],
channels=[channel]) is False:
return
channel.users[self] = {"status": ""}
self.channels.append(channel)
newChannel = False
if channel.name not in self.ircd.channels:
newChannel = True
self.ircd.channels[channel.name] = channel
self.ircd.recentlyDestroyedChannels[channel.name] = False
# We need to send the JOIN message before doing other processing, as chancreate will do things like
# mode defaulting, which will send messages about the channel before the JOIN message, which is bad.
messageUsers = [
u for u in channel.users.iterkeys()
if u.uuid[:3] == self.ircd.serverID
]
self.ircd.runActionProcessing("joinmessage",
messageUsers,
channel,
self,
users=messageUsers,
channels=[channel])
if newChannel:
self.ircd.runActionStandard("channelcreate",
channel,
self,
channels=[channel])
self.ircd.runActionStandard("join",
channel,
self,
users=[self],
channels=[channel])
def leaveChannel(self,
channel,
partType="PART",
typeData={},
fromServer=None):
"""
Removes the user from a channel. The partType and typeData are used for
the leavemessage action to send the parting message. If the channel
leaving is initiated by a remote server, that server should be
specified in the fromServer parameter.
"""
if channel not in self.channels:
return
messageUsers = [
u for u in channel.users.iterkeys()
if u.uuid[:3] == self.ircd.serverID
]
self.ircd.runActionProcessing("leavemessage",
messageUsers,
channel,
self,
partType,
typeData,
fromServer,
users=[self],
channels=[channel])
self._leaveChannel(channel)
def _leaveChannel(self, channel):
self.ircd.runActionStandard("leave",
channel,
self,
users=[self],
channels=[channel])
self.channels.remove(channel)
del channel.users[self]
def setModes(self, modes, defaultSource):
"""
Sets modes on the user. Accepts modes as a list of tuples in the
format:
[ (adding, mode, param, setBy, setTime) ]
- adding: True if we're setting the mode; False if unsetting
- mode: The mode letter
- param: The mode's parameter; None if no parameter is needed for that
mode
- setBy: Optional, only used for list modes; a human-readable string
(typically server name or nick!user@host) for who/what set this
mode)
- setTime: Optional, only used for list modes; a datetime object
containing when the mode was set
The defaultSource is a valid user ID or server ID of someone who set
the modes. It is used as the source for announcements about the mode
change and as the default setter for any list modes who do not have the
setBy parameter specified.
The default time for list modes with no setTime specified is now().
"""
modeChanges = []
defaultSourceName = self._sourceName(defaultSource)
if defaultSourceName is None:
raise ValueError("Source must be a valid user or server ID.")
nowTime = now()
for modeData in modes:
mode = modeData[1]
if mode not in self.ircd.userModeTypes:
continue
setBy = defaultSourceName
setTime = nowTime
modeType = self.ircd.userModeTypes[mode]
adding = modeData[0]
if modeType in (ModeType.List, ModeType.ParamOnUnset,
ModeType.Param):
param = modeData[2]
else:
param = None
if modeType == ModeType.List:
dataCount = len(modeData)
if dataCount >= 4:
setBy = modeData[3]
if dataCount >= 5:
setTime = modeData[4]
if adding:
paramList = self.ircd.userModes[modeType][mode].checkSet(
self, param)
else:
paramList = self.ircd.userModes[modeType][mode].checkUnset(
self, param)
if paramList is None:
continue
for parameter in paramList:
if self._applyMode(adding, modeType, mode, parameter, setBy,
setTime):
modeChanges.append(
(adding, mode, parameter, setBy, setTime))
self._notifyModeChanges(modeChanges, defaultSource, defaultSourceName)
return modeChanges
def setModesByUser(self, user, modes, params, override=False):
"""
Parses a mode string specified by a user and sets those modes on the
user.
The user parameter should be the user who set the modes (usually, but
not always, this user).
The modes parameter is the actual modes string; parameters specified by
the user should be as a list of strings in params.
The override parameter should be used only when all permission checks
should be overridden.
"""
adding = True
changes = []
setBy = self._sourceName(user.uuid)
setTime = now()
for mode in modes:
if len(changes) >= self.ircd.config.get("modes_per_line", 20):
break
if mode == "+":
adding = True
continue
if mode == "-":
adding = False
continue
if mode not in self.ircd.userModeTypes:
user.sendMessage(irc.ERR_UNKNOWNMODE, mode,
"is unknown mode char to me")
continue
modeType = self.ircd.userModeTypes[mode]
param = None
if modeType in (ModeType.List, ModeType.ParamOnUnset) or (
adding and modeType == ModeType.Param):
try:
param = params.pop(0)
except IndexError:
if modeType == ModeType.List:
self.ircd.userModes[modeType][mode].showListParams(
user, self)
continue
if adding:
paramList = self.ircd.userModes[modeType][mode].checkSet(
self, param)
else:
paramList = self.ircd.userModes[modeType][mode].checkUnset(
self, param)
if paramList is None:
continue
for parameter in paramList:
if len(changes) >= self.ircd.config.get("modes_per_line", 20):
break
if not override and self.ircd.runActionUntilValue(
"modepermission-user-{}".format(mode),
self,
user,
adding,
parameter,
users=[self, user]) is False:
continue
if adding:
if modeType == ModeType.List:
if mode in self.modes and len(
self.modes[mode]) > self.ircd.config.get(
"user_listmode_limit", 128):
user.sendMessage(
irc.ERR_BANLISTFULL, self.name, parameter,
"Channel +{} list is full".format(mode))
continue
if self._applyMode(adding, modeType, mode, parameter, setBy,
setTime):
changes.append((adding, mode, parameter, setBy, setTime))
self._notifyModeChanges(changes, user.uuid, setBy)
return changes
def _applyMode(self, adding, modeType, mode, parameter, setBy, setTime):
if parameter:
if len(parameter) > 255:
return False
if " " in parameter:
return False
if adding:
if modeType == ModeType.List:
if mode in self.modes:
if len(self.modes[mode]) > self.ircd.config.get(
"user_listmode_limit", 128):
return False
for paramData in self.modes[mode]:
if parameter == paramData[0]:
return False
else:
self.modes[mode] = []
self.modes[mode].append((parameter, setBy, setTime))
return True
if mode in self.modes and self.modes[mode] == parameter:
return False
self.modes[mode] = parameter
return True
if modeType == ModeType.List:
if mode not in self.modes:
return False
for index, paramData in enumerate(self.modes[mode]):
if paramData[0] == parameter:
del self.modes[mode][index]
break
else:
return False
if not self.modes[mode]:
del self.modes[mode]
return True
if mode not in self.modes:
return False
if modeType == ModeType.ParamOnUnset and parameter != self.modes[mode]:
return False
del self.modes[mode]
return True
def _notifyModeChanges(self, modeChanges, source, sourceName):
if not modeChanges:
return
for change in modeChanges:
self.ircd.runActionStandard("modechange-user-{}".format(change[1]),
self,
change[3],
change[0],
change[2],
users=[self])
users = []
if source in self.ircd.users and source[:3] == self.ircd.serverID:
users.append(self.ircd.users[source])
if self.uuid[:3] == self.ircd.serverID:
users.append(self)
if users:
self.ircd.runActionProcessing("modemessage-user",
users,
self,
source,
sourceName,
modeChanges,
users=users)
self.ircd.runActionStandard("modechanges-user",
self,
source,
sourceName,
modeChanges,
users=[self])
def _sourceName(self, source):
if source in self.ircd.users:
return self.ircd.users[source].hostmask()
if source == self.ircd.serverID:
return self.ircd.name
if source in self.ircd.servers:
return self.ircd.servers[source].name
return None
def modeString(self, toUser):
"""
Get a user-reportable mode string for the modes set on the user.
"""
modeStr = ["+"]
params = []
for mode in self.modes:
modeType = self.ircd.userModeTypes[mode]
if modeType not in (ModeType.ParamOnUnset, ModeType.Param,
ModeType.NoParam):
continue
if modeType != ModeType.NoParam:
param = None
if toUser:
param = self.ircd.userModes[modeType][mode].showParam(
toUser, self)
if not param:
param = self.modes[mode]
else:
param = None
modeStr.append(mode)
if param:
params.append(param)
if params:
return "{} {}".format("".join(modeStr), " ".join(params))
return "".join(modeStr)
class MBDBChangeSource(base.ChangeSource):
debug = True
def __init__(self, pollInterval=30, branch='default'):
#base.ChangeSource.__init__(self)
self.pollInterval = pollInterval
self.latest = None
self.branch, created = \
Branch.objects.get_or_create(name=branch)
def startService(self):
self.loop = LoopingCall(self.poll)
base.ChangeSource.startService(self)
reactor.callLater(0, self.loop.start, self.pollInterval)
def stopService(self):
self.loop.stop()
return base.ChangeSource.stopService(self)
@transaction.commit_on_success
def poll(self):
'''Check for new pushes.
Hack around transactions on innodb, make this transaction
aware and transaction.commit() to get a new transaction
for our queries.
'''
transaction.commit()
if self.latest is None:
try:
self.latest = Push.objects.order_by('-pk')[0].id
except IndexError:
self.latest = 0
return
new_pushes = Push.objects.filter(pk__gt=self.latest).order_by('pk')
if self.debug:
log.msg('mbdb changesource found %d pushes after %d' %
(new_pushes.count(), self.latest))
push = None
for push in new_pushes:
self.submitChangesForPush(push)
if push is not None:
self.latest = push.id
def submitChangesForPush(self, push):
if self.debug:
log.msg('submitChangesForPush called')
repo = push.repository
if repo.forest is not None:
branch = repo.forest.name.encode('utf-8')
locale = repo.name[len(branch) + 1:].encode('utf-8')
else:
branch = repo.name.encode('utf-8')
for cs in push.changesets.filter(
branch=self.branch).order_by('pk'):
when = timegm(push.push_date.utctimetuple()) +\
push.push_date.microsecond/1000.0/1000
c = changes.Change(who=push.user.encode('utf-8'),
files=map(
lambda u: u.encode('utf-8'),
cs.files.values_list('path',
flat=True)),
revision=cs.revision.encode('utf-8'),
comments=cs.description.encode('utf-8'),
when=when,
branch=branch)
if repo.forest is not None:
# locale change
c.locale = locale
self.parent.addChange(c)
def replay(self,
builder,
startPush=None,
startTime=None,
endTime=None):
bm = self.parent.parent.botmaster
qd = {}
if startTime is not None:
qd['push_date__gte'] = startTime
if endTime is not None:
qd['push_date__lte'] = endTime
if startPush is not None:
qd['id__gte'] = startPush
q = Push.objects.filter(**qd).order_by('push_date')
i = q.iterator()
if self.debug:
log.msg('replay called for %d pushes' % q.count())
def next(_cb):
try:
p = i.next()
except StopIteration:
log.msg("done iterating")
return
self.submitChangesForPush(p)
def stumble():
bm.waitUntilBuilderIdle(builder).addCallback(_cb, _cb)
reactor.callLater(.5, stumble)
def cb(res, _cb):
reactor.callLater(.5, next, _cb)
next(cb)
def describe(self):
return str(self)
def __str__(self):
return "MBDBChangeSource"
def run_server(opt):
global gp10block
global gp40block
store = {}
if opt['gp10']['enable'] is True:
gp10block = Gp10ModbusSlave(opt['gp10']['deviceid'], **opt['gp10'])
store[opt['gp10']['deviceid']] = gp10block.store
if opt['gp40']['enable'] is True:
gp40block = Gp40ModbusSlave(opt['gp40']['deviceid'], **opt['gp40'])
store[opt['gp40']['deviceid']] = gp40block.store
if 0 == len(store):
print("ERR: enable GP10 or GP40.")
return
context = ModbusServerContext(slaves=store, single=False)
# ----------------------------------------------------------------------- #
# initialize the server information
# ----------------------------------------------------------------------- #
# If you don't set this or any fields, they are defaulted to empty strings.
# ----------------------------------------------------------------------- #
identity = ModbusDeviceIdentification()
identity.VendorName = 'RatocSystems, Inc.'
identity.ProductCode = 'RPi-Modbus'
identity.VendorUrl = 'https://github.com/ratocsystems'
identity.ProductName = 'RasPi Modbus Server'
identity.ModelName = 'RasPi Modbus Server'
identity.MajorMinorRevision = '1.0'
# ----------------------------------------------------------------------- #
# updater
# ----------------------------------------------------------------------- #
updater_500ms = LoopingCall(f=write_context_500ms, a=(context,))
updater_500ms.start(.5)
updater_1000ms = LoopingCall(f=write_context_1000ms, a=(context,))
updater_1000ms.start(1)
writer = LoopingCall(f=read_context, a=(context,))
writer.start(.1)
# ----------------------------------------------------------------------- #
# run the server you want
# ----------------------------------------------------------------------- #
if 'tcp' == opt['common']['protocol']:
# Tcp:
StartTcpServer(context, identity=identity, address=(opt['tcp']['host'], opt['tcp']['port']))
elif 'ascii' == opt['common']['protocol']:
# Ascii:
StartSerialServer(context, identity=identity, port=opt['serial']['device'], timeout=1, baudrate=opt['serial']['baudrate'])
elif 'rtu' == opt['common']['protocol']:
# RTU:
StartSerialServer(context, framer=ModbusRtuFramer, identity=identity, port=opt['serial']['device'], timeout=.005, baudrate=opt['serial']['baudrate'])
else:
print("ERR: select protocol tcp, rtu or ascii")
# ----------------------------------------------------------------------- #
# end proc
# ----------------------------------------------------------------------- #
updater_500ms.stop()
updater_1000ms.stop()
writer.stop()
if gp10block is not None:
del gp10block
if gp40block is not None:
del gp40block
class Core(CorePluginBase):
def enable(self):
self.config = deluge.configmanager.ConfigManager(
"centeragent.conf", DEFAULT_PREFS)
self.session = component.get("Core").session
self.alertmanager = component.get("AlertManager")
# connect to Center via HTTP
self.connect(self.config["IP"], self.config["Port"])
# polling timer every 5 min
self.polling_timer = LoopingCall(self.polling)
self.polling_timer.start(300)
def disable(self):
self.polling_timer.stop()
pass
def update(self):
pass
@export
def set_config(self, config):
"""Sets the config dictionary"""
for key in config.keys():
self.config[key] = config[key]
self.config.save()
@export
def get_config(self):
"""Returns the config dictionary"""
return self.config.config
def connect(self, ip, port):
log.info("Connecting...")
self.url = "http://" + ip + ":" + str(port)
response = urllib2.urlopen(self.url + "/register/")
r = json.loads(response.read())
if r['result'] == 'OK':
log.info("Connect to Center SUCCESS")
return True
return False
def fn(self, handle):
if not handle.is_valid():
return None
status = handle.status()
if status.is_finished:
return None
if not status.has_metadata:
return None
return handle
def report(self):
log.info("Report")
jobs = len(filter(self.fn, self.session.get_torrents()))
response = urllib2.urlopen(self.url + "/report/?" +
urllib.urlencode({'jobs': jobs}))
pass
def on_save_resume_data_alert_factory(self, _save_path):
save_path = _save_path
def on_save_resume_data_alert(alert):
# send to Center with save_path
# TODO
pass
return on_save_resume_data_alert
def add_torrent(self, torrent_resume_data, save_path):
pass
def migrate(self):
log.info("Send job")
# check if need to send a job to another
while True:
response = urllib2.urlopen(self.url + "/check/")
r = json.loads(response)
if r['result'] == "NO":
break
jobs = filter(self.fn, self.session.get_torrents())
if len(jobs) <= 0:
break
torrent = jobs[0]
torrent.pause()
torrent.flush_cache()
torrent.save_resume_data(3)
save_path = torrent.status().save_path
self.alertmanager.register_handler(
"save_resume_data_alert",
self.on_save_resume_data_alert_factory(save_path))
log.info("Recv job")
# check if need to recv job
response = urllib2.urlopen(self.url + "/recv/")
r = json.loads(response)
if r['result'] == "NO":
return
for a, b in r['jobs']:
self.add_torrent(a, b)
pass
def polling(self):
log.info("Polling")
if self.url:
self.report()
self.migrate()
class Tunnel(object):
def __init__(self, options, ipv8_port=-1, ipv8_address="0.0.0.0"):
self.options = options
self.should_run = True
self.ipv8_port = ipv8_port
self.ipv8_address = ipv8_address
self.session = None
self.community = None
self.clean_messages_lc = LoopingCall(self.clean_messages)
self.clean_messages_lc.start(1800)
self.clean_messages_lc = LoopingCall(self.periodic_bootstrap)
self.clean_messages_lc.start(30, now=False)
def clean_messages(self):
clean_twisted_observers()
def periodic_bootstrap(self):
self.session.lm.tunnel_community.bootstrap()
def tribler_started(self):
new_strategies = []
with self.session.lm.ipv8.overlay_lock:
for strategy, target_peers in self.session.lm.ipv8.strategies:
if strategy.overlay == self.session.lm.tunnel_community:
new_strategies.append((strategy, -1))
else:
new_strategies.append((strategy, target_peers))
self.session.lm.ipv8.strategies = new_strategies
def circuit_removed(self, _, __, ___, address):
self.session.lm.ipv8.network.remove_by_address(address)
self.session.lm.tunnel_community.bootstrap()
def start(self):
# Determine socks5 ports
socks5_port = self.options['socks5']
if "HELPER_INDEX" in os.environ and "HELPER_BASE" in os.environ:
base_port = int(os.environ["HELPER_BASE"])
socks5_port = base_port + int(os.environ["HELPER_INDEX"]) * 5
if socks5_port is not None:
socks_listen_ports = range(socks5_port, socks5_port + 5)
else:
socks_listen_ports = [
random.randint(1000, 65535) for _ in range(5)
]
config = TriblerConfig()
config.set_state_dir(
os.path.join(config.get_state_dir(), "tunnel-%d") %
socks_listen_ports[0])
config.set_tunnel_community_socks5_listen_ports(socks_listen_ports)
config.set_torrent_checking_enabled(False)
config.set_megacache_enabled(False)
config.set_dispersy_enabled(False)
config.set_ipv8_enabled(True)
config.set_mainline_dht_enabled(True)
config.set_torrent_collecting_enabled(False)
config.set_libtorrent_enabled(False)
config.set_video_server_enabled(False)
config.set_dispersy_port(self.ipv8_port)
config.set_ipv8_address(self.ipv8_address)
config.set_torrent_search_enabled(False)
config.set_channel_search_enabled(False)
config.set_trustchain_enabled(True)
config.set_credit_mining_enabled(False)
config.set_market_community_enabled(False)
config.set_mainline_dht_enabled(False)
config.set_tunnel_community_exitnode_enabled(bool(
self.options["exit"]))
config.set_trustchain_testnet(bool(self.options["testnet"]))
if self.options["restapi"] is not None:
config.set_http_api_enabled(True)
config.set_http_api_port(self.options["restapi"])
if "ipv8_bootstrap_override" in self.options:
config.set_ipv8_bootstrap_override(
self.options["ipv8_bootstrap_override"])
self.session = Session(config)
logger.info("Using IPv8 port %d" %
self.session.config.get_dispersy_port())
self.session.notifier.add_observer(self.circuit_removed, NTFY_TUNNEL,
[NTFY_REMOVE])
return self.session.start().addCallback(self.tribler_started)
def stop(self):
if self.clean_messages_lc:
self.clean_messages_lc.stop()
self.clean_messages_lc = None
if self.session:
logger.info("Going to shutdown session")
return self.session.shutdown()
class datehandler(AbstractHandler):
midnight = time(00, 00, 00)
def updatedate(self):
now = datetime.now().replace(microsecond=0)
self._tags['date'] = now.strftime("%d.%m.%Y %H:%M:%S")
self.checktags(now)
if now.time() == self.midnight:
self._tags['sunset'] = self.getsunset().strftime("%H:%M:%S")
self._tags['sunrise'] = self.getsunrise().strftime("%H:%M:%S")
if now.time().strftime("%H:%M:%S") == self._tags['sunset']:
self._settag('issunset', '1')
else:
self._settag('issunset', '0')
if now.time().strftime("%H:%M:%S") == self._tags['sunrise']:
self._settag('issunrise', '1')
else:
self._settag('issunrise', '0')
def loadtags(self):
self._tags['sunset'] = self.getsunset().strftime("%H:%M:%S")
self._tags['sunrise'] = self.getsunrise().strftime("%H:%M:%S")
self._tags['issunset'] = '0'
self._tags['issunrise'] = '0'
self.rrules = {}
for tag, params in self.config.items():
startdate = parse(params["start"])
enddate = parse(params["end"])
untildate = parse(params["until"])
freq = eval(params['freq'])
startrr = rrule(freq,
dtstart=startdate,
until=untildate,
cache=True)
endrr = rrule(freq, dtstart=enddate, until=untildate, cache=True)
self._tags[tag] = '0'
tagrrule = self.rrules.setdefault(tag, {})
tagrrule["startrr"] = startrr
tagrrule["endrr"] = endrr
def earthtool(self, param):
now = datetime.now()
xml = urllib2.urlopen("http://www.earthtools.org/sun/54.32/36.16/" +
str(now.day) + "/" + str(now.month) +
"/+4/0").read()
xmldoc = minidom.parseString(xml)
itemlist = xmldoc.getElementsByTagName(param)
return parse(itemlist[0].childNodes[0].nodeValue).time()
def getsunset(self):
sunset = time(hour=20, minute=00, second=0, microsecond=0)
try:
sunset = self.earthtool('sunset')
except:
self.logger.error("Cant get sunset", exc_info=1)
return sunset
def getsunrise(self):
sunrise = time(hour=8, minute=00, second=0, microsecond=0)
try:
sunrise = self.earthtool('sunrise')
except:
self.logger.error("Cant get sunrise", exc_info=1)
return sunrise
def checktag(self, tag, dt):
dt = dt.replace(second=0, microsecond=0)
if dt in self.rrules[tag]["startrr"]:
self._settag(tag, '1')
elif dt in self.rrules[tag]["endrr"]:
self._settag(tag, '0')
def checktags(self, dt):
for tag in self.rrules:
self.checktag(tag, dt)
def start(self):
AbstractHandler.start(self)
self.lc = LoopingCall(self.updatedate)
self.lc.start(1)
def stop(self):
AbstractHandler.stop(self)
if self.lc:
self.lc.stop()
class Fluid(object):
"""
Fluid simulator.
"""
implements(IAutomaton, IDigHook)
sponge = None
"""
Block that will soak up fluids and springs that are near it.
Defaults to None, which effectively disables this feature.
"""
def __init__(self, factory):
self.factory = factory
self.sponges = Block3DSpatialDict()
self.springs = Block2DSpatialDict()
self.tracked = set()
self.new = set()
self.loop = LoopingCall(self.process)
def start(self):
if not self.loop.running:
self.loop.start(self.step)
def stop(self):
if self.loop.running:
self.loop.stop()
def schedule(self):
if self.tracked:
self.start()
else:
self.stop()
@property
def blocks(self):
retval = [self.spring, self.fluid]
if self.sponge:
retval.append(self.sponge)
return retval
def feed(self, coordinates):
"""
Accept the coordinates and stash them for later processing.
"""
self.tracked.add(coordinates)
self.schedule()
scan = naive_scan
def update_fluid(self, w, coords, falling, level=0):
if not 0 <= coords[1] < 256:
return False
block = w.sync_get_block(coords)
if (block in self.whitespace and not
any(self.sponges.iteritemsnear(coords, 2))):
w.sync_set_block(coords, self.fluid)
if falling:
level |= FALLING
w.sync_set_metadata(coords, level)
self.new.add(coords)
return True
return False
def add_sponge(self, w, x, y, z):
# Track this sponge.
self.sponges[x, y, z] = True
# Destroy the water! Destroy!
for coords in itercube(x, y, z, 2):
try:
target = w.sync_get_block(coords)
if target == self.spring:
if (coords[0], coords[2]) in self.springs:
del self.springs[coords[0],
coords[2]]
w.sync_destroy(coords)
elif target == self.fluid:
w.sync_destroy(coords)
except ChunkNotLoaded:
pass
# And now mark our surroundings so that they can be
# updated appropriately.
for coords in itercube(x, y, z, 3):
if coords != (x, y, z):
self.new.add(coords)
def add_spring(self, w, x, y, z):
# Double-check that we weren't placed inside a sponge. That's just
# not going to work out.
if any(self.sponges.iteritemsnear((x, y, z), 2)):
w.sync_destroy((x, y, z))
return
# Track this spring.
self.springs[x, z] = y
# Neighbors on the xz-level.
neighbors = ((x - 1, y, z), (x + 1, y, z), (x, y, z - 1),
(x, y, z + 1))
# Spawn water from springs.
for coords in neighbors:
try:
self.update_fluid(w, coords, False)
except ChunkNotLoaded:
pass
# Is this water falling down to the next y-level? We don't really
# care, but we'll run the update nonetheless.
if y > 0:
# Our downstairs pal.
below = x, y - 1, z
self.update_fluid(w, below, True)
def add_fluid(self, w, x, y, z):
# Neighbors on the xz-level.
neighbors = ((x - 1, y, z), (x + 1, y, z), (x, y, z - 1),
(x, y, z + 1))
# Our downstairs pal.
below = (x, y - 1, z)
# Double-check that we weren't placed inside a sponge.
if any(self.sponges.iteritemsnear((x, y, z), 2)):
w.sync_destroy((x, y, z))
return
# First, figure out whether or not we should be spreading. Let's see
# if there are any springs nearby which are above us and thus able to
# fuel us.
if not any(springy >= y
for springy in
self.springs.itervaluesnear((x, z), self.levels + 1)):
# Oh noes, we're drying up! We should mark our neighbors and dry
# ourselves up.
self.new.update(neighbors)
if y:
self.new.add(below)
w.sync_destroy((x, y, z))
return
newmd = self.levels + 1
for coords in neighbors:
try:
jones = w.sync_get_block(coords)
if jones == self.spring:
newmd = 0
self.new.update(neighbors)
break
elif jones == self.fluid:
jonesmd = w.sync_get_metadata(coords) & ~FALLING
if jonesmd + 1 < newmd:
newmd = jonesmd + 1
except ChunkNotLoaded:
pass
current_md = w.sync_get_metadata((x,y,z))
if newmd > self.levels and current_md < FALLING:
# We should dry up.
self.new.update(neighbors)
if y:
self.new.add(below)
w.sync_destroy((x, y, z))
return
# Mark any neighbors which should adjust themselves. This will only
# mark lower water levels than ourselves, and only if they are
# definitely too low.
for coords in neighbors:
try:
neighbor = w.sync_get_metadata(coords)
if neighbor & ~FALLING > newmd + 1:
self.new.add(coords)
except ChunkNotLoaded:
pass
# Now, it's time to extend water. Remember, either the water flows
# downward to the next y-level, or it flows out across the xz-level,
# but *not* both.
# Fall down to the next y-level, if possible.
if y and self.update_fluid(w, below, True, newmd):
return
# Clamp our newmd and assign. Also, set ourselves again; we changed
# this time and we might change again.
if current_md < FALLING:
w.sync_set_metadata((x, y, z), newmd)
# If pending block is already above fluid, don't keep spreading.
if neighbor == self.fluid:
return
# Otherwise, just fill our neighbors with water, where applicable, and
# mark them.
if newmd < self.levels:
newmd += 1
for coords in neighbors:
try:
self.update_fluid(w, coords, False, newmd)
except ChunkNotLoaded:
pass
def remove_sponge(self, x, y, z):
# The evil sponge tyrant is gone. Flow, minions, flow!
for coords in itercube(x, y, z, 3):
if coords != (x, y, z):
self.new.add(coords)
def remove_spring(self, x, y, z):
# Neighbors on the xz-level.
neighbors = ((x - 1, y, z), (x + 1, y, z), (x, y, z - 1),
(x, y, z + 1))
# Destroyed spring. Add neighbors and below to blocks to update.
del self.springs[x, z]
self.new.update(neighbors)
if y:
# Our downstairs pal.
below = x, y - 1, z
self.new.add(below)
def process(self):
w = self.factory.world
for x, y, z in self.tracked:
# Try each block separately. If it can't be done, it'll be
# discarded from the set simply by not being added to the new set
# for the next iteration.
try:
block = w.sync_get_block((x, y, z))
if block == self.sponge:
self.add_sponge(w, x, y, z)
elif block == self.spring:
self.add_spring(w, x, y, z)
elif block == self.fluid:
self.add_fluid(w, x, y, z)
else:
# Hm, why would a pending block not be any of the things
# we care about? Maybe it used to be a spring or
# something?
if (x, z) in self.springs:
self.remove_spring(x, y, z)
elif (x, y, z) in self.sponges:
self.remove_sponge(x, y, z)
except ChunkNotLoaded:
pass
# Flush affected chunks.
to_flush = set()
for x, y, z in chain(self.tracked, self.new):
to_flush.add((x // 16, z // 16))
for x, z in to_flush:
d = self.factory.world.request_chunk(x, z)
d.addCallback(self.factory.flush_chunk)
self.tracked = self.new
self.new = set()
# Prune, and reschedule.
self.schedule()
@inlineCallbacks
def dig_hook(self, chunk, x, y, z, block):
"""
Check for neighboring water that might want to spread.
Also check to see whether we are, for example, dug ice that has turned
back into water.
"""
x += chunk.x * 16
z += chunk.z * 16
# Check for sponges first, since they will mark the entirety of the
# area.
if block == self.sponge:
for coords in itercube(x, y, z, 3):
self.tracked.add(coords)
else:
for coords in iterneighbors(x, y, z):
test_block = yield self.factory.world.get_block(coords)
if test_block in (self.spring, self.fluid):
self.tracked.add(coords)
self.schedule()
before = ("build",)
after = tuple()
class FamilyMember(automat.Automat):
"""
This class implements all the functionality of ``family_member()`` state machine.
"""
timers = {
'timer-10sec': (10.0, ['SUPPLIERS']),
}
def __init__(self,
customer_idurl,
debug_level=_DebugLevel,
log_events=_Debug,
log_transitions=_Debug,
publish_events=False,
**kwargs):
"""
Builds `family_member()` state machine.
"""
self.customer_idurl = customer_idurl
self.supplier_idurl = my_id.getLocalIDURL()
super(FamilyMember, self).__init__(name="family_member_%s_%s" % (
nameurl.GetName(self.customer_idurl),
nameurl.GetName(my_id.getLocalIDURL()),
),
state="AT_STARTUP",
debug_level=debug_level,
log_events=log_events,
log_transitions=log_transitions,
publish_events=publish_events,
**kwargs)
def state_changed(self, oldstate, newstate, event, *args, **kwargs):
"""
Method to catch the moment when `family_member()` state were changed.
"""
if event != 'instant' and newstate in [
'CONNECTED',
'DISCONNECTED',
]:
self.automat('instant')
def state_not_changed(self, curstate, event, *args, **kwargs):
"""
This method intended to catch the moment when some event was fired in the `family_member()`
but automat state was not changed.
"""
if event != 'instant' and curstate in [
'CONNECTED',
'DISCONNECTED',
]:
self.automat('instant')
def A(self, event, *args, **kwargs):
"""
The state machine code, generated using `visio2python <http://bitdust.io/visio2python/>`_ tool.
"""
#---AT_STARTUP---
if self.state == 'AT_STARTUP':
if event == 'init':
self.state = 'DISCONNECTED'
self.doInit(*args, **kwargs)
#---DISCONNECTED---
elif self.state == 'DISCONNECTED':
if event == 'shutdown':
self.state = 'CLOSED'
self.doDestroyMe(*args, **kwargs)
elif event == 'instant' and self.isAnyRequests(*args, **kwargs):
self.state = 'DHT_READ'
self.Attempts = 0
self.doPull(*args, **kwargs)
self.doDHTRead(*args, **kwargs)
elif event == 'family-refresh' or event == 'family-join' or event == 'family-leave':
self.doPush(event, *args, **kwargs)
elif event == 'contacts-received':
self.doCheckReply(*args, **kwargs)
#---DHT_READ---
elif self.state == 'DHT_READ':
if event == 'family-refresh' or event == 'family-join' or event == 'family-leave':
self.doPush(event, *args, **kwargs)
elif event == 'contacts-received':
self.doCheckReply(*args, **kwargs)
elif event == 'dht-read-fail':
self.state = 'DISCONNECTED'
self.Attempts = 0
self.doNotifyDisconnected(*args, **kwargs)
elif event == 'dht-value-exist' and self.isMyPositionOK(
*args, **kwargs) and not self.isLeaving(*args, **kwargs):
self.state = 'CONNECTED'
self.Attempts = 0
self.doNotifyConnected(*args, **kwargs)
elif (event == 'dht-value-exist'
and self.isMyPositionOK(*args, **kwargs)
and self.isLeaving(*args, **kwargs)) or event == 'shutdown':
self.state = 'CLOSED'
self.doDestroyMe(*args, **kwargs)
elif event == 'dht-value-not-exist' or (
event == 'dht-value-exist'
and not self.isMyPositionOK(*args, **kwargs)):
self.state = 'SUPPLIERS'
self.Attempts += 1
self.doRebuildFamily(*args, **kwargs)
self.doRequestSuppliersReview(*args, **kwargs)
#---SUPPLIERS---
elif self.state == 'SUPPLIERS':
if event == 'shutdown':
self.state = 'CLOSED'
self.doDestroyMe(*args, **kwargs)
elif event == 'family-refresh' or event == 'family-join' or event == 'family-leave':
self.doPush(event, *args, **kwargs)
elif event == 'contacts-received':
self.doCheckReply(*args, **kwargs)
elif (event == 'all-suppliers-agree'
or event == 'timer-10sec') and not self.isFamilyModified(
*args, **kwargs):
self.state = 'CONNECTED'
self.Attempts = 0
self.doNotifyConnected(*args, **kwargs)
elif (event == 'timer-10sec'
or event == 'all-suppliers-agree') and self.isFamilyModified(
*args, **kwargs):
self.state = 'DHT_WRITE'
self.doDHTWrite(*args, **kwargs)
elif event == 'one-supplier-not-agree':
self.doSolveConflict(*args, **kwargs)
self.doRequestSuppliersReview(*args, **kwargs)
#---DHT_WRITE---
elif self.state == 'DHT_WRITE':
if event == 'shutdown':
self.state = 'CLOSED'
self.doDestroyMe(*args, **kwargs)
elif event == 'family-refresh' or event == 'family-join' or event == 'family-leave':
self.doPush(event, *args, **kwargs)
elif event == 'contacts-received':
self.doCheckReply(*args, **kwargs)
elif event == 'dht-write-fail' and self.Attempts > 3:
self.state = 'DISCONNECTED'
self.Attempts = 0
self.doNotifyDisconnected(*args, **kwargs)
elif event == 'dht-write-fail' and self.Attempts <= 3:
self.state = 'DHT_READ'
self.doDHTRead(*args, **kwargs)
elif event == 'dht-write-ok':
self.state = 'CONNECTED'
self.Attempts = 0
self.doNotifyConnected(*args, **kwargs)
#---CLOSED---
elif self.state == 'CLOSED':
pass
#---CONNECTED---
elif self.state == 'CONNECTED':
if event == 'shutdown':
self.state = 'CLOSED'
self.doDestroyMe(*args, **kwargs)
elif event == 'disconnect':
self.state = 'DISCONNECTED'
self.doNotifyDisconnected(*args, **kwargs)
elif event == 'instant' and self.isAnyRequests(*args, **kwargs):
self.state = 'DHT_READ'
self.Attempts = 0
self.doPull(*args, **kwargs)
self.doDHTRead(*args, **kwargs)
elif event == 'family-refresh' or event == 'family-join' or event == 'family-leave':
self.doPush(event, *args, **kwargs)
elif event == 'contacts-received':
self.doCheckReply(*args, **kwargs)
return None
def isAnyRequests(self, *args, **kwargs):
"""
Condition method.
"""
return len(self.requests) > 0
def isLeaving(self, *args, **kwargs):
"""
Condition method.
"""
return self.current_request and self.current_request[
'command'] == 'family-leave'
def isMyPositionOK(self, *args, **kwargs):
"""
Condition method.
"""
dht_info_valid = self._do_validate_dht_info(args[0])
if not dht_info_valid:
return False
if self.current_request and self.current_request[
'command'] == 'family-leave':
if my_id.getLocalIDURL() not in dht_info_valid['suppliers']:
return True
my_info_valid = self._do_validate_my_info(self.my_info)
if not my_info_valid:
return False
latest_revision = self._do_detect_latest_revision(
dht_info_valid, my_info_valid)
if latest_revision == 0:
return False
try:
my_position = my_info_valid['suppliers'].index(
my_id.getLocalIDURL())
except:
my_position = -1
if my_position < 0:
return False
try:
existing_position = dht_info_valid['suppliers'].index(
my_id.getLocalIDURL())
except:
existing_position = -1
return existing_position > 0 and my_position > 0 and existing_position == my_position
def isFamilyModified(self, *args, **kwargs):
"""
Condition method.
"""
return self.transaction is not None
def doInit(self, *args, **kwargs):
"""
Action method.
"""
self.requests = []
self.current_request = None
self.dht_info = None
self.my_info = None
self.transaction = None
self.refresh_task = LoopingCall(self._on_family_refresh_task)
def doPush(self, event, *args, **kwargs):
"""
Action method.
"""
if event not in _ValidRequests:
raise Exception('Invalid request: %r' % args)
request = (args[0] if args else {}) or {}
request['command'] = event
self.requests.append(request)
def doPull(self, *args, **kwargs):
"""
Action method.
"""
self.current_request = self.requests.pop(0)
def doRebuildFamily(self, *args, **kwargs):
"""
Action method.
"""
dht_info_valid = self._do_validate_dht_info(args[0])
my_info_valid = self._do_validate_my_info(self.my_info)
latest_revision = self._do_detect_latest_revision(
dht_info_valid, my_info_valid)
merged_info = None
if latest_revision > 0:
merged_info = self._do_merge_revisions(dht_info_valid,
my_info_valid,
latest_revision)
if not merged_info:
merged_info = self._do_create_first_revision(self.current_request)
# if not merged_info:
# lg.err('failed to merge customer family info after reading from DHT, skip transaction')
# self.transaction = None
# return
possible_transaction = self._do_process_request(
merged_info, self.current_request)
if not possible_transaction:
lg.err(
'failed to process customer family change request, skip transaction'
)
return
self.transaction = self._do_increment_revision(possible_transaction)
if _Debug:
lg.out(
_DebugLevel,
'family_member._do_build_transaction : %r' % self.transaction)
self.refresh_period = DHT_RECORD_REFRESH_INTERVAL * settings.DefaultDesiredSuppliers(
)
if self.transaction:
known_ecc_map = self.transaction.get('ecc_map')
if known_ecc_map:
expected_suppliers_count = eccmap.GetEccMapSuppliersNumber(
known_ecc_map)
self.refresh_period = DHT_RECORD_REFRESH_INTERVAL * expected_suppliers_count
def doRequestSuppliersReview(self, *args, **kwargs):
"""
Action method.
"""
if not self.transaction:
self.automat('all-suppliers-agree')
return
self.suppliers_requests = []
for supplier_idurl in self.transaction['suppliers']:
if not supplier_idurl:
continue
if supplier_idurl == my_id.getLocalIDURL():
continue
outpacket = p2p_service.SendContacts(
remote_idurl=supplier_idurl,
json_payload={
'space': 'family_member',
'type': 'suppliers_list',
'customer_idurl': self.customer_idurl,
'customer_ecc_map': self.transaction['ecc_map'],
'transaction_revision': self.transaction['revision'],
'suppliers_list': self.transaction['suppliers'],
},
callbacks={
commands.Ack(): self._on_supplier_ack,
commands.Fail(): self._on_supplier_fail,
},
)
self.suppliers_requests.append(outpacket.PacketID)
if not self.suppliers_requests:
self.automat('all-suppliers-agree')
else:
if _Debug:
lg.out(
_DebugLevel,
'family_member.doRequestSuppliersReview sent to transaction for review to %d suppliers'
% len(self.suppliers_requests))
def doSolveConflict(self, *args, **kwargs):
"""
Action method.
"""
# TODO: take in account ecc_map while solving the conflict
# ecc_map = kwargs.get('ecc_map')
suppliers_list = kwargs.get('suppliers_list')
another_supplier_idurl = kwargs.get('supplier_idurl')
try:
another_supplier_position = suppliers_list.index(
another_supplier_idurl)
except:
another_supplier_position = -1
if another_supplier_position < 0:
# this must never happen actually... only if another supplier is really uncooperative
# this is dangerous because can lead to infinite loop between me and another supplier
lg.info(
'found uncooperative supplier %s who raised the conflict but replied with invalid response'
% another_supplier_idurl)
# TODO: solve later
self.transaction = None
else:
if len(self.transaction['suppliers']) <= another_supplier_position:
lg.warn(
'another supplier position larger than family size, failed to solve family conflict with supplier %s'
% another_supplier_idurl)
self.transaction = None
else:
if self.transaction['suppliers'][another_supplier_position]:
lg.warn(
'given position is not empty, failed to solve family conflict with supplier %s'
% another_supplier_idurl)
self.transaction = None
else:
self.transaction['suppliers'][
another_supplier_position] = another_supplier_idurl
lg.info(
'found desired position %d in the family and solved conflict with supplier %s'
% (
another_supplier_position,
another_supplier_idurl,
))
def doDHTRead(self, *args, **kwargs):
"""
Action method.
"""
d = dht_relations.read_customer_suppliers(self.customer_idurl)
d.addCallback(self._on_dht_read_success)
d.addErrback(self._on_dht_read_failed)
def doDHTWrite(self, *args, **kwargs):
"""
Action method.
"""
d = dht_relations.write_customer_suppliers(
customer_idurl=self.customer_idurl,
suppliers_list=self.transaction['suppliers'],
ecc_map=self.transaction['ecc_map'],
revision=self.transaction['revision'],
publisher_idurl=self.transaction['publisher_idurl'],
)
d.addCallback(self._on_dht_write_success)
d.addErrback(self._on_dht_write_failed)
def doNotifyConnected(self, *args, **kwargs):
"""
Action method.
"""
if _Debug:
lg.out(
_DebugLevel,
'family_memeber.doNotifyConnected\n my_info=%r\n dht_info=%r\n requests=%r'
% (
self.my_info,
self.dht_info,
self.requests,
))
to_be_closed = False
if self.current_request['command'] == 'family-leave':
to_be_closed = True
self.current_request = None
if self.refresh_task.running:
self.refresh_task.stop()
self.refresh_task.start(self.refresh_period, now=False)
if to_be_closed:
self.requests = []
self.automat('shutdown')
def doNotifyDisconnected(self, *args, **kwargs):
"""
Action method.
"""
if self.refresh_task.running:
self.refresh_task.stop()
self.current_request = None
def doCheckReply(self, *args, **kwargs):
"""
Action method.
"""
self._on_incoming_contacts_packet(args[0])
def doDestroyMe(self, *args, **kwargs):
"""
Remove all references to the state machine object to destroy it.
"""
self.requests = []
self.current_request = None
self.my_info = None
self.dht_info = None
self.transaction = None
self.refresh_task = None
delete_family(self.customer_idurl)
self.destroy()
#------------------------------------------------------------------------------
def _do_validate_dht_info(self, inp):
if not inp or not isinstance(inp, dict):
return None
out = inp.copy()
try:
dht_revision = int(out['revision'])
suppliers = out['suppliers']
ecc_map = out['ecc_map']
if dht_revision < 1:
raise Exception('invalid revision')
if not isinstance(suppliers, list) or len(suppliers) < 1:
raise Exception('must include some suppliers')
if ecc_map and ecc_map not in eccmap.EccMapNames():
raise Exception('invalid ecc_map name')
out['publisher_idurl']
# TODO: add publisher_signature and Validate method to check publisher signature
out['customer_idurl']
# TODO: add customer_signature and Validate method to check customer signature
except:
lg.exc()
lg.warn('skip invalid DHT info and assume DHT record is not exist')
return None
return out
def _do_validate_my_info(self, inp):
if not inp:
return None
if not inp or not isinstance(inp, dict):
return None # self._do_prepare_my_default_info()
out = inp.copy()
try:
my_revision = int(out['revision'])
if my_revision < 1:
raise Exception('invalid revision')
except:
lg.exc()
# out['revision'] = 0
return None
try:
suppliers = out['suppliers']
if not isinstance(suppliers, list) or len(suppliers) < 1:
raise Exception('must include some suppliers')
except:
lg.exc()
return None
try:
ecc_map = out['ecc_map']
if ecc_map and ecc_map not in eccmap.EccMapNames():
raise Exception('invalid ecc_map name')
except:
lg.exc()
return None
try:
out['publisher_idurl']
# TODO: if I am a publisher - revision number must be the same as my info
except:
return None
try:
customer_idurl = out['customer_idurl']
if customer_idurl != self.customer_idurl:
raise Exception('invalid customer_idurl')
except:
return None
return out
def _do_create_first_revision(self, request):
return {
'revision': 0,
'publisher_idurl': my_id.getLocalIDURL(
), # I will be a publisher of the first revision
'suppliers': request.get('family_snapshot'),
'ecc_map': request['ecc_map'],
'customer_idurl': self.customer_idurl,
}
def _do_create_possible_revision(self, latest_revision):
local_customer_meta_info = contactsdb.get_customer_meta_info(
self.customer_idurl)
possible_position = local_customer_meta_info.get('position', -1)
possible_suppliers = local_customer_meta_info.get('family_snapshot')
if possible_position > 0 and my_id.getLocalIDURL(
) not in possible_suppliers:
if len(possible_suppliers) > possible_position:
possible_suppliers[possible_position] = my_id.getLocalIDURL()
return {
'revision': latest_revision,
'publisher_idurl':
my_id.getLocalIDURL(), # I will be a publisher of that revision
'suppliers': possible_suppliers,
'ecc_map': local_customer_meta_info.get('ecc_map'),
'customer_idurl': self.customer_idurl,
}
def _do_create_revision_from_another_supplier(self, another_revision,
another_suppliers,
another_ecc_map):
local_customer_meta_info = contactsdb.get_customer_meta_info(
self.customer_idurl)
possible_position = local_customer_meta_info.get('position', -1)
if possible_position >= 0:
try:
another_suppliers[possible_position] = my_id.getLocalIDURL()
except:
lg.exc()
contactsdb.add_customer_meta_info(
self.customer_idurl, {
'ecc_map': another_ecc_map,
'position': possible_position,
'family_snapshot': another_suppliers,
})
return {
'revision': int(another_revision),
'publisher_idurl':
my_id.getLocalIDURL(), # I will be a publisher of that revision
'suppliers': another_suppliers,
'ecc_map': another_ecc_map,
'customer_idurl': self.customer_idurl,
}
def _do_detect_latest_revision(self, dht_info, my_info):
try:
my_revision = int(my_info['revision'])
except:
lg.warn(
'my own info is unknown or invalid, assume my revision is 0')
my_revision = 0
try:
dht_revision = int(dht_info['revision'])
except:
lg.warn('DHT info is unknown or invalid, assume DHT revision is 0')
dht_revision = 0
if my_revision == dht_revision:
return dht_revision
if my_revision > dht_revision:
# TODO: SECURITY need to find a solution to prevent cheating here
# another supplier could publish a record where he is only alone present and with a correct revision
# that means he actually brutally dropped all other suppliers from the family
lg.info(
'known DHT info for customer %s is more fresh, will rewrite DHT record'
% self.customer_idurl)
if my_revision > dht_revision + 1:
lg.warn(
'switching revision too far, normally always increase by one on every change'
)
return my_revision
return dht_revision
def _do_merge_revisions(self, dht_info, my_info, latest_revision):
if dht_info is None or not isinstance(dht_info, dict):
merged_info = my_info
else:
if latest_revision == int(dht_info['revision']):
if my_info is not None:
if latest_revision == int(my_info['revision']):
# I have same revision as info from DHT
merged_info = dht_info
else:
if int(my_info['revision']) > int(
dht_info['revision']):
# here my revision is higher, so I have some changes that needs to be published already
merged_info = my_info
else:
# here my revision is lower, I need to take info from DHT
merged_info = dht_info
else:
merged_info = dht_info
else:
# here my revision is higher, so I have some changes that needs to be published already
merged_info = my_info
if not merged_info:
return None
# make sure list of suppliers have correct length according to ecc_map
if not merged_info['ecc_map']:
known_ecc_map = contactsdb.get_customer_meta_info(
self.customer_idurl).get('ecc_map', None)
lg.warn('unknown ecc_map, will populate known value: %s' %
known_ecc_map)
merged_info['ecc_map'] = known_ecc_map
if merged_info['ecc_map']:
expected_suppliers_count = eccmap.GetEccMapSuppliersNumber(
merged_info['ecc_map'])
if len(merged_info['suppliers']) < expected_suppliers_count:
merged_info['suppliers'] += [
b'',
] * (expected_suppliers_count - len(merged_info['suppliers']))
elif len(merged_info['suppliers']) > expected_suppliers_count:
merged_info['suppliers'] = merged_info[
'suppliers'][:expected_suppliers_count]
if merged_info['revision'] != latest_revision:
lg.info('will switch known revision %d to the latest: %d' % (
merged_info['revision'],
latest_revision,
))
merged_info['revision'] = latest_revision
return merged_info
def _do_increment_revision(self, possible_transaction):
if self.dht_info:
if self.dht_info['suppliers'] == possible_transaction['suppliers']:
if self.dht_info['ecc_map'] == possible_transaction['ecc_map']:
if _Debug:
lg.out(
_DebugLevel,
'family_member._do_increment_revision did not found any changes, skip transaction'
)
return None
possible_transaction['revision'] += 1
possible_transaction['publisher_idurl'] = my_id.getLocalIDURL()
return possible_transaction
def _do_process_family_join_request(self, merged_info, current_request):
current_request_expected_suppliers_count = None
if current_request['ecc_map']:
current_request_expected_suppliers_count = eccmap.GetEccMapSuppliersNumber(
current_request['ecc_map'])
if current_request_expected_suppliers_count and current_request[
'position'] > current_request_expected_suppliers_count:
lg.warn(
'"family-join" request is not valid, supplier position %d greater than expected suppliers count %d for %s'
% (current_request['position'],
current_request_expected_suppliers_count,
current_request['ecc_map']))
return None
if merged_info['ecc_map'] and current_request[
'ecc_map'] and current_request['ecc_map'] != merged_info[
'ecc_map']:
lg.info(
'from "family-join" request, detected ecc_map change %s -> %s for customer %s'
% (merged_info['ecc_map'], current_request['ecc_map'],
self.customer_idurl))
merged_info['ecc_map'] = current_request['ecc_map']
if not merged_info['ecc_map'] and current_request['ecc_map']:
lg.info(
'from "family-join" request, detected ecc_map was set to %s for the first time for customer %s'
% (current_request['ecc_map'], self.customer_idurl))
merged_info['ecc_map'] = current_request['ecc_map']
if not merged_info['ecc_map']:
lg.warn(
'still did not found actual ecc_map from DHT or from the request'
)
return None
expected_suppliers_count = eccmap.GetEccMapSuppliersNumber(
merged_info['ecc_map'])
if not merged_info['suppliers']:
merged_info['suppliers'] = [
b'',
] * expected_suppliers_count
if len(merged_info['suppliers']) < expected_suppliers_count:
merged_info['suppliers'] += [
b'',
] * (expected_suppliers_count - len(merged_info['suppliers']))
else:
merged_info['suppliers'] = merged_info[
'suppliers'][:expected_suppliers_count]
try:
existing_position = merged_info['suppliers'].index(
current_request['supplier_idurl'])
except:
existing_position = -1
if current_request[
'position'] is not None and current_request['position'] >= 0:
if current_request['position'] >= expected_suppliers_count:
lg.warn(
'"family-join" request is not valid, supplier position greater than expected suppliers count'
)
return None
if existing_position >= 0 and existing_position != current_request[
'position']:
merged_info['suppliers'][existing_position] = b''
merged_info['suppliers'][current_request[
'position']] = current_request['supplier_idurl']
if _Debug:
lg.out(
_DebugLevel,
' found my IDURL on %d position and will move it on %d position in the family of customer %s'
% (existing_position, current_request['position'],
self.customer_idurl))
if merged_info['suppliers'][current_request[
'position']] != current_request['supplier_idurl']:
if merged_info['suppliers'][
current_request['position']] not in [b'', '', None]:
# TODO: SECURITY need to implement a signature verification and
# also build solution to validate that change was approved by customer
lg.warn(
'overwriting another supplier %s with my IDURL at position %d in family of customer %s'
% (
merged_info['suppliers'][
current_request['position']],
current_request['position'],
self.customer_idurl,
))
merged_info['suppliers'][current_request[
'position']] = current_request['supplier_idurl']
if _Debug:
lg.out(
_DebugLevel,
' placed supplier %s at known position %d in the family of customer %s'
% (current_request['supplier_idurl'],
current_request['position'], self.customer_idurl))
if current_request['supplier_idurl'] not in merged_info['suppliers']:
if b'' in merged_info['suppliers']:
first_empty_position = merged_info['suppliers'].index(b'')
merged_info['suppliers'][
first_empty_position] = current_request['supplier_idurl']
if _Debug:
lg.out(
_DebugLevel,
' placed supplier %s at first empty position %d in family of customer %s'
% (current_request['supplier_idurl'],
first_empty_position, self.customer_idurl))
else:
merged_info['suppliers'].append(
current_request['supplier_idurl'])
if _Debug:
lg.out(
_DebugLevel,
' added supplier %s to family of customer %s' %
(current_request['supplier_idurl'],
self.customer_idurl))
return merged_info
def _do_process_family_leave_request(self, merged_info, current_request):
try:
existing_position = merged_info['suppliers'].index(
current_request['supplier_idurl'])
except ValueError:
existing_position = -1
if existing_position < 0:
lg.warn(
'skip "family-leave" request, did not found supplier %r in customer family %r'
% (
current_request['supplier_idurl'],
self.customer_idurl,
))
return None
merged_info['suppliers'][existing_position] = b''
return merged_info
def _do_process_family_refresh_request(self, merged_info):
if not self.my_info:
self.my_info = self._do_create_possible_revision(
int(merged_info['revision']))
lg.warn(
'"family-refresh" request will use "possible" customer meta info: %r'
% self.my_info)
if int(self.my_info['revision']) > int(merged_info['revision']):
lg.info(
'"family-refresh" request will overwrite DHT record with my info because my revision is higher than record in DHT'
)
return self.my_info.copy()
try:
my_position = self.my_info['suppliers'].index(
my_id.getLocalIDURL())
except:
my_position = -1
if my_position < 0:
lg.warn(
'"family-refresh" request failed because my info not exist or not valid, my own position in the family is unknown'
)
return None
my_expected_suppliers_count = None
if self.my_info['ecc_map']:
my_expected_suppliers_count = eccmap.GetEccMapSuppliersNumber(
self.my_info['ecc_map'])
if my_expected_suppliers_count and my_position >= my_expected_suppliers_count:
lg.warn(
'"family-refresh" request failed because my info is not valid, supplier position greater than expected suppliers count'
)
return None
if len(merged_info['suppliers']) != my_expected_suppliers_count:
lg.warn(
'number of suppliers not expected during processing of "family-refresh" request'
)
if len(merged_info['suppliers']) < my_expected_suppliers_count:
merged_info['suppliers'] += [
b'',
] * (my_expected_suppliers_count -
len(merged_info['suppliers']))
else:
merged_info['suppliers'] = merged_info[
'suppliers'][:my_expected_suppliers_count]
try:
existing_position = merged_info['suppliers'].index(
my_id.getLocalIDURL())
except ValueError:
existing_position = -1
if existing_position < 0:
if merged_info['suppliers'][my_position] not in [b'', '', None]:
# TODO: SECURITY need to implement a signature verification and
# also build solution to validate that change was approved by customer
lg.warn(
'overwriting another supplier %s with my IDURL at position %d in family of customer %s'
% (
merged_info['suppliers'][my_position],
my_position,
self.customer_idurl,
))
merged_info['suppliers'][my_position] = my_id.getLocalIDURL()
if _Debug:
lg.out(
_DebugLevel,
' placed supplier %s at known position %d in the family of customer %s'
%
(my_id.getLocalIDURL(), my_position, self.customer_idurl))
existing_position = my_position
if existing_position != my_position:
merged_info['suppliers'][existing_position] = b''
merged_info['suppliers'][my_position] = my_id.getLocalIDURL()
if _Debug:
lg.out(
_DebugLevel,
' found my IDURL on %d position and will move it on %d position in the family of customer %s'
% (existing_position, my_position, self.customer_idurl))
return merged_info
def _do_process_request(self, merged_info, current_request):
if current_request['command'] == 'family-join':
return self._do_process_family_join_request(
merged_info, current_request)
if current_request['command'] == 'family-leave':
return self._do_process_family_leave_request(
merged_info, current_request)
if current_request['command'] == 'family-refresh':
return self._do_process_family_refresh_request(merged_info)
lg.err('invalid request command')
return None
def _on_family_refresh_task(self):
self.automat('family-refresh')
def _on_dht_read_success(self, dht_result):
if _Debug:
lg.out(
_DebugLevel,
'family_member._on_dht_read_success result: %r' % dht_result)
if dht_result:
self.dht_info = dht_result
self.automat('dht-value-exist', dht_result)
else:
self.dht_info = None
self.automat('dht-value-not-exist', None)
def _on_dht_read_failed(self, err):
self.dht_info = None
if _Debug:
lg.out(_DebugLevel, 'family_member._on_dht_read_failed : %r' % err)
self.automat('dht-read-fail')
def _on_dht_write_success(self, dht_result):
if _Debug:
lg.out(
_DebugLevel,
'family_member._on_dht_write_success result: %r' % dht_result)
self.my_info = self.transaction.copy()
self.dht_info = None
self.transaction = None
self.automat('dht-write-ok', dht_result)
def _on_dht_write_failed(self, err):
if _Debug:
lg.out(_DebugLevel, 'family_member._on_dht_write_failed')
self.transaction = None
self.dht_info = None
self.automat('dht-write-fail')
def _on_incoming_suppliers_list(self, inp):
# this packet came from another supplier who belongs to that family also
incoming_packet = inp['packet']
if _Debug:
lg.out(
_DebugLevel,
'family_member._on_incoming_suppliers_list with %s' %
incoming_packet)
if not self.my_info:
if _Debug:
lg.out(_DebugLevel,
' current DHT info is not yet known, skip')
return p2p_service.SendAck(incoming_packet)
try:
another_ecc_map = inp['customer_ecc_map']
another_suppliers_list = inp['suppliers_list']
another_revision = int(inp['transaction_revision'])
except:
lg.exc()
return p2p_service.SendFail(incoming_packet,
response=serialization.DictToBytes(
self.my_info))
if _Debug:
lg.out(
_DebugLevel,
' another_revision=%d another_ecc_map=%s another_suppliers_list=%r'
% (another_revision, another_ecc_map, another_suppliers_list))
if another_revision >= int(self.my_info['revision']):
self.my_info = self._do_create_revision_from_another_supplier(
another_revision, another_suppliers_list, another_ecc_map)
lg.info(
'another supplier have more fresh revision, update my info and raise "family-refresh" event'
)
self.automat('family-refresh')
return p2p_service.SendAck(incoming_packet)
if my_id.getLocalIDURL() not in another_suppliers_list:
lg.warn(
'another supplier is trying to remove my IDURL from the family of customer %s'
% self.customer_idurl)
return p2p_service.SendFail(incoming_packet,
response=serialization.DictToBytes(
self.my_info))
my_position_in_transaction = another_suppliers_list.index(
my_id.getLocalIDURL())
my_known_position = self.my_info['suppliers'].index(
my_id.getLocalIDURL())
if my_position_in_transaction != my_known_position:
lg.warn(
'another supplier is trying to put my IDURL on another position in the family of customer %s'
% self.customer_idurl)
return p2p_service.SendFail(incoming_packet,
response=serialization.DictToBytes(
self.my_info))
return p2p_service.SendAck(incoming_packet)
def _on_incoming_supplier_position(self, inp):
# this packet came from the customer, a godfather of the family ;)))
incoming_packet = inp['packet']
try:
ecc_map = inp['customer_ecc_map']
supplier_idurl = inp['supplier_idurl']
supplier_position = inp['supplier_position']
family_snapshot = inp.get('family_snapshot')
except:
lg.exc()
return None
if supplier_idurl != my_id.getLocalIDURL():
return p2p_service.SendFail(
incoming_packet,
'contacts packet with supplier position not addressed to me')
try:
_existing_position = self.my_info['suppliers'].index(
supplier_idurl)
except:
_existing_position = -1
contactsdb.add_customer_meta_info(
self.customer_idurl, {
'ecc_map': ecc_map,
'position': supplier_position,
'family_snapshot': family_snapshot,
})
if _Debug:
lg.out(
_DebugLevel,
'family_member._on_incoming_supplier_position stored new meta info for customer %s:\n'
% self.customer_idurl)
lg.out(
_DebugLevel,
' ecc_map=%s position=%s family_snapshot=%s' % (
ecc_map,
supplier_position,
family_snapshot,
))
return p2p_service.SendAck(incoming_packet)
def _on_incoming_contacts_packet(self, inp):
try:
contacts_type = inp['type']
incoming_packet = inp['packet']
except:
lg.exc()
return None
if contacts_type == 'suppliers_list':
return self._on_incoming_suppliers_list(inp)
elif contacts_type == 'supplier_position':
return self._on_incoming_supplier_position(inp)
return p2p_service.SendFail(incoming_packet, 'invalid contacts type')
def _on_supplier_ack(self, response, info):
if _Debug:
lg.out(_DebugLevel,
'family_member._on_supplier_ack with %r' % response)
if response.PacketID in self.suppliers_requests:
self.suppliers_requests.remove(response.PacketID)
if not self.suppliers_requests:
self.automat('all-suppliers-agree')
def _on_supplier_fail(self, response, info):
if _Debug:
lg.out(_DebugLevel,
'family_member._on_supplier_fail with %r' % response)
if response.PacketID in self.suppliers_requests:
self.suppliers_requests.remove(response.PacketID)
try:
json_payload = serialization.BytesToDict(response.Payload)
ecc_map = strng.to_text(json_payload['ecc_map'])
suppliers_list = list(map(strng.to_bin, json_payload['suppliers']))
except:
lg.exc()
if not self.suppliers_requests:
self.automat('all-suppliers-agree')
return None
self.automat('one-supplier-not-agree',
ecc_map=ecc_map,
suppliers_list=suppliers_list,
supplier_idurl=response.OwnerID)
class Character():
def __init__(self, name, address, udp, main):
self.name = name
self.address = address
self.udp = udp
self.heartBeatCount = 3
self.main = main
self.world = None
self.street = None
self.room = None
self.handlers = {
"disconnect": self.connectionLost,
"pos_upd": self.pos_upd_handler,
"mov": self.mov_handler,
"heartbeat": self.onHeartBeat
}
self.heartbeatloop = LoopingCall(self.sendHeartbeat)
self.heartbeatloop.start(5)
def stop(self):
self.heartbeatloop.stop()
self = None
def sendHeartbeat(self):
"""
Checks if this character instance is still connected.
Does three heartv beat retry
"""
if self.heartBeatCount == 0:
self.connectionLost({"id": self.name})
return
hb = json.dumps({'heartbeat': 1})
self.udp.reply(hb, self.address)
self.heartBeatCount -= 1
def onHeartBeat(self, val):
print "Hearbeat OK"
self.heartBeatCount = 3
def onRelay(self, data):
for handler, data in data.items():
if handler in self.handlers:
self.handlers[handler](data)
def reply(self, data):
self.udp.reply(data, self.address)
def connectionLost(self, data):
name = data['id']
print name, "has disconnected"
dc = json.dumps({'dc': {"id": name}})
self.main.send_to_others(name, dc)
self.main.removePlayer(name)
def pos_upd_handler(self, val):
#update receiver with the my current position
if val.has_key("pos") and val.has_key("id") and val.has_key("receiver")\
and len(val.keys())==2:
mes = {"pos_upd":{"id":val['receiver'],"pos":[val["pos"][0],\
val["pos"][1]]}}
self.send(val['receiver'], json.dumps(mes))
def mov_handler(self, val):
if val.has_key("dest") and val.has_key("id") and len(val.keys()) == 2:
mes = {"mov":{"id":val['id'],"dest":[val['dest'][0],\
val['dest'][1]]}}
self.main.send_to_others(self.name, json.dumps(mes))
def send(self, player, mes):
try:
self.main.players[player].reply(mes)
except Exception:
print self.main.players, player
print "Failed: Not in here anymore"
class _HamlibProxy(ExportedState):
# pylint: disable=no-member
"""
Abstract class for objects which export state proxied to a hamlib daemon.
"""
implements(IComponent, IProxy)
def __init__(self, protocol):
# info from hamlib
self.__cache = {}
self.__caps = {}
self.__levels = []
# invert command table
# TODO: we only need to do this once per class, really
self._how_to_command = {
key: command
for command, keys in self._commands.iteritems() for key in keys
}
# keys are same as __cache, values are functions to call with new values from rig
self._cell_updaters = {}
self.__communication_error = False
self.__last_error = (-1e9, '', 0)
self.__protocol = protocol
self.__disconnect_deferred = defer.Deferred()
protocol._set_proxy(self)
# TODO: If hamlib backend supports "transceive mode", use it in lieu of polling
self.__poller_slow = LoopingCall(self.__poll_slow)
self.__poller_fast = LoopingCall(self.__poll_fast)
self.__poller_slow.start(2.0)
self.__poller_fast.start(0.2)
self._ready_deferred = protocol.rc_send('dump_caps')
def sync(self):
# TODO: Replace 'sync' with more specifically meaningful operations
d = self.__protocol.rc_send(self._dummy_command)
d.addCallback(lambda _: None) # ignore result
return d
def close(self):
"""implements IComponent"""
self.__protocol.transport.loseConnection()
return self.when_closed() # used for tests, not part of IComponent
def when_closed(self):
return fork_deferred(self.__disconnect_deferred)
def _ehs_get(self, name_in_cmd):
if name_in_cmd in self.__cache:
return self.__cache[name_in_cmd]
else:
return 0.0
def _clientReceived(self, command, key, value):
self.__communication_error = False
if command == 'dump_caps':
def write(key):
self.__caps[key] = value
if key == 'Get level':
# add to polling info
for info in value.strip().split(' '):
match = re.match(r'^(\w+)\([^()]+\)$', info)
# part in parens is probably min/max/step info, but we don't have any working examples to test against (they are all 0)
if match:
self.__levels.append(match.group(1))
else:
log.err(
'Unrecognized level description from %s: %r' %
(self._server_name, info))
# remove irregularity
keymatch = re.match(r'(Can [gs]et )([\w\s,/-]+)', key)
if keymatch and keymatch.group(2) in _cap_remap:
for mapped in _cap_remap[keymatch.group(2)]:
write(keymatch.group(1) + mapped)
else:
write(key)
else:
self.__update_cache_and_cells(key, value)
def _clientReceivedLevel(self, level_name, value_str):
self.__update_cache_and_cells(level_name + ' level', value_str)
def _clientError(self, cmd, error_number):
if cmd.startswith('get_'):
# these getter failures are boring, probably us polling something not implemented
if error_number == RIG_ENIMPL or error_number == RIG_ENTARGET or error_number == RIG_BUSERROR:
return
elif error_number == RIG_ETIMEOUT:
self.__communication_error = True
return
self.__last_error = (time.time(), cmd, error_number)
def __update_cache_and_cells(self, key, value):
self.__cache[key] = value
if key in self._cell_updaters:
self._cell_updaters[key](value)
def _clientConnectionLost(self, reason):
self.__poller_slow.stop()
self.__poller_fast.stop()
self.__disconnect_deferred.callback(None)
def _ehs_set(self, name_full, value):
if not isinstance(value, str):
raise TypeError()
name_in_cmd = self._how_to_command[name_full] # raises if cannot set
if value != self.__cache[name_full]:
self.__cache[name_full] = value
self.__protocol.rc_send(
'set_' + name_in_cmd,
' '.join(self.__cache[arg_name]
for arg_name in self._commands[name_in_cmd]))
def state_def(self, callback):
super(_HamlibProxy, self).state_def(callback)
for name in self._info:
can_get = self.__caps.get('Can get ' + name)
if can_get is None:
log.msg('No can-get information for ' + name)
if can_get != 'Y':
# TODO: Handle 'E' condition
continue
writable = name in self._how_to_command and self.__caps.get(
'Can set ' + name) == 'Y'
_install_cell(self, name, False, writable, callback, self.__caps)
for level_name in self.__levels:
# TODO support writable levels
_install_cell(self, level_name + ' level', True, False, callback,
self.__caps)
def __poll_fast(self):
# TODO: Stop if we're getting behind
p = self.__protocol
self.poll_fast(p.rc_send)
for level_name in self.__levels:
p.rc_send('get_level', level_name)
def __poll_slow(self):
# TODO: Stop if we're getting behind
p = self.__protocol
self.poll_slow(p.rc_send)
@exported_value(type=Notice(always_visible=False))
def get_errors(self):
if self.__communication_error:
return 'Rig not responding.'
else:
(error_time, cmd, error_number) = self.__last_error
if error_time > time.time() - 10:
return u'%s: %s' % (cmd, error_number)
else:
return u''
def poll_fast(self, send):
raise NotImplementedError()
def poll_slow(self, send):
raise NotImplementedError()
class ResourceMonitor(object):
"""
This class is responsible for monitoring resources in Tribler.
Specifically, it fetches information from the Tribler core and writes it to a file.
"""
def __init__(self, interval):
self.interval = interval
self.request_manager = HTTPRequestManager()
self.monitor_memory_lc = LoopingCall(self.monitor_memory)
self.monitor_cpu_lc = LoopingCall(self.monitor_cpu)
self.start_time = time.time()
self.latest_memory_time = 0
self.latest_cpu_time = 0
# Create the output directory if it does not exist yet
output_dir = os.path.join(os.getcwd(), "output")
if not os.path.exists(output_dir):
os.makedirs(output_dir)
self.memory_stats_file_path = os.path.join(output_dir, 'memory_stats.csv')
with open(self.memory_stats_file_path, "w") as output_file:
output_file.write("time,memory_usage\n")
self.cpu_stats_file_path = os.path.join(output_dir, 'cpu_stats.csv')
with open(self.cpu_stats_file_path, "w") as output_file:
output_file.write("time,cpu_usage\n")
def start(self):
"""
Start the monitoring loop for the resources.
"""
self.monitor_memory_lc.start(self.interval)
self.monitor_cpu_lc.start(self.interval)
def stop(self):
"""
Stop the monitoring loop for the resources.
"""
if self.monitor_memory_lc and self.monitor_memory_lc.running:
self.monitor_memory_lc.stop()
self.monitor_memory_lc = None
if self.monitor_cpu_lc and self.monitor_cpu_lc.running:
self.monitor_cpu_lc.stop()
self.monitor_cpu_lc = None
def on_memory_history(self, response):
history = json.loads(response)
for history_item in history["memory_history"]:
if history_item["time"] > self.latest_memory_time:
self.latest_memory_time = history_item["time"]
time_diff = history_item["time"] - self.start_time
with open(self.memory_stats_file_path, "a") as output_file:
output_file.write("%s,%s\n" % (time_diff, history_item["mem"]))
def on_cpu_history(self, response):
history = json.loads(response)
for history_item in history["cpu_history"]:
if history_item["time"] > self.latest_cpu_time:
self.latest_cpu_time = history_item["time"]
time_diff = history_item["time"] - self.start_time
with open(self.cpu_stats_file_path, "a") as output_file:
output_file.write("%s,%s\n" % (time_diff, history_item["cpu"]))
def monitor_memory(self):
"""
Monitor the memory usage in Tribler.
"""
return self.request_manager.get_memory_history_core().addCallback(self.on_memory_history)
def monitor_cpu(self):
"""
Monitor the CPU usage in Tribler.
"""
return self.request_manager.get_cpu_history_core().addCallback(self.on_cpu_history)
class P2PConnector(automat.Automat):
"""
"""
fast = False
timers = {
'timer-20sec': (20.0, ['INCOMMING?']),
}
def init(self):
self.health_check_task = None
self.log_transitions = _Debug
def state_changed(self, oldstate, newstate, event, *args, **kwargs):
if newstate == 'INCOMMING?' and event != 'instant':
self.automat('instant')
if newstate == 'CONNECTED':
self.health_check_task = LoopingCall(
self._do_id_server_health_check)
self.health_check_task.start(config.conf().getInt(
'services/identity-propagate/health-check-interval-seconds'),
now=False)
else:
if self.health_check_task:
self.health_check_task.stop()
self.health_check_task = None
def A(self, event, *args, **kwargs):
#---AT_STARTUP---
if self.state == 'AT_STARTUP':
if event == 'init':
self.state = 'NETWORK?'
self.NeedPropagate = True
self.doInit(*args, **kwargs)
network_connector.A('reconnect')
#---NETWORK?---
elif self.state == 'NETWORK?':
if (event == 'network_connector.state' and args[0] == 'CONNECTED'):
self.state = 'MY_IDENTITY'
self.doUpdateMyIdentity(*args, **kwargs)
elif (event == 'network_connector.state'
and args[0] == 'DISCONNECTED'):
self.state = 'DISCONNECTED'
#---INCOMMING?---
elif self.state == 'INCOMMING?':
if event == 'inbox-packet' and not self.isUsingBestProto(
*args, **kwargs):
self.state = 'MY_IDENTITY'
self.doUpdateMyIdentity(*args, **kwargs)
self.doPopBestProto(*args, **kwargs)
elif event == 'timer-20sec' or (event == 'network_connector.state'
and args[0] == 'DISCONNECTED'):
self.state = 'DISCONNECTED'
self.doInitRatings(*args, **kwargs)
elif event == 'check-synchronize' or (event
== 'network_connector.state'
and args[0] == 'CONNECTED'):
self.state = 'MY_IDENTITY'
self.doUpdateMyIdentity(*args, **kwargs)
elif (event == 'instant'
and not self.isAnyPeersKnown(*args, **kwargs)) or (
event == 'inbox-packet'
and self.isUsingBestProto(*args, **kwargs)):
self.state = 'CONNECTED'
self.doInitRatings(*args, **kwargs)
self.doRestartCustomersRejector(*args, **kwargs)
#---CONNECTED---
elif self.state == 'CONNECTED':
if event == 'ping-contact':
self.doSendMyIdentity(*args, **kwargs)
elif (event == 'network_connector.state'
and args[0] == 'DISCONNECTED'):
self.state = 'DISCONNECTED'
elif event == 'check-synchronize' or (event
== 'network_connector.state'
and args[0] == 'CONNECTED'):
self.state = 'MY_IDENTITY'
self.doUpdateMyIdentity(*args, **kwargs)
elif (event == 'network_connector.state'
and args[0] not in ['CONNECTED', 'DISCONNECTED']):
self.state = 'NETWORK?'
#---DISCONNECTED---
elif self.state == 'DISCONNECTED':
if event == 'ping-contact':
self.doSendMyIdentity(*args, **kwargs)
elif event == 'inbox-packet' or event == 'check-synchronize' or (
(event == 'network_connector.state'
and args[0] == 'CONNECTED')):
self.state = 'MY_IDENTITY'
self.doUpdateMyIdentity(*args, **kwargs)
elif (event == 'network_connector.state' and args[0] not in [
'CONNECTED',
'DISCONNECTED',
]):
self.state = 'NETWORK?'
#---MY_IDENTITY---
elif self.state == 'MY_IDENTITY':
if event == 'my-id-updated' and self.isMyContactsChanged(
*args, **kwargs):
self.state = 'NETWORK?'
self.NeedPropagate = True
network_connector.A('check-reconnect')
elif event == 'my-id-updated' and not self.isMyContactsChanged(
*args, **kwargs) and (self.NeedPropagate
or self.isMyIdentityChanged(
*args, **kwargs)):
self.state = 'PROPAGATE'
self.doCheckRotatePropagateMyIdentity(*args, **kwargs)
elif event == 'my-id-updated' and not (
self.NeedPropagate or self.isMyIdentityChanged(
*args, **kwargs)) and (network_connector.A().state
is not 'CONNECTED'):
self.state = 'DISCONNECTED'
elif event == 'my-id-updated' and not (
self.NeedPropagate
or self.isMyIdentityChanged(*args, **kwargs)) and (
network_connector.A().state is 'CONNECTED'):
self.state = 'CONNECTED'
#---PROPAGATE---
elif self.state == 'PROPAGATE':
if event == 'my-id-propagated':
self.state = 'INCOMMING?'
self.NeedPropagate = False
self.doRestartFireHire(*args, **kwargs)
elif ((event == 'network_connector.state' and args[0]
== 'CONNECTED')) or event == 'check-synchronize':
self.state = 'MY_IDENTITY'
self.doUpdateMyIdentity(*args, **kwargs)
return None
def isUsingBestProto(self, *args, **kwargs):
"""
Condition method.
"""
return self._check_to_use_best_proto()
def isMyIdentityChanged(self, *args, **kwargs):
"""
Condition method.
"""
return args[0][1]
def isMyContactsChanged(self, *args, **kwargs):
"""
Condition method.
"""
return args[0][0]
def isAnyPeersKnown(self, *args, **kwargs):
"""
Condition method.
"""
return len(contactsdb.contacts_remote()) > 0
def doInit(self, *args, **kwargs):
version_number = bpio.ReadTextFile(
settings.VersionNumberFile()).strip()
if _Debug:
lg.out(
_DebugLevel - 6,
'p2p_connector.doInit RevisionNumber=%s' % str(version_number))
callback.append_inbox_callback(inbox)
def doSendMyIdentity(self, *args, **kwargs):
"""
Action method.
"""
propagate.single(args[0], wide=True)
def doUpdateMyIdentity(self, *args, **kwargs):
if _Debug:
lg.out(_DebugLevel - 6, 'p2p_connector.doUpdateMyIdentity')
self._update_my_identity()
def doCheckRotatePropagateMyIdentity(self, *args, **kwargs):
if _Debug:
lg.out(_DebugLevel - 6,
'p2p_connector.doCheckRotatePropagateMyIdentity')
self._check_rotate_propagate_my_identity()
def doPopBestProto(self, *args, **kwargs):
self._pop_active_proto()
def doInitRatings(self, *args, **kwargs):
ratings.init()
def doRestartCustomersRejector(self, *args, **kwargs):
"""
Action method.
"""
if driver.is_on('service_customer_patrol'):
# TODO: move this into a callback inside service_customer_patrol
from supplier import customers_rejector
customers_rejector.A('restart')
def doRestartFireHire(self, *args, **kwargs):
"""
Action method.
"""
if driver.is_on('service_employer'):
# TODO: use events instead
from customer import fire_hire
fire_hire.A('restart')
def _check_to_use_best_proto(self):
# if no incoming traffic - do nothing
if len(active_protos()) == 0:
return True
lid = my_id.getLocalIdentity()
order = lid.getProtoOrder()
# if no protocols in local identity - do nothing
if len(order) == 0:
return True
# when transport proxy is working we do not need to check our contacts at all
if settings.transportIsEnabled('proxy'):
if driver.is_on('service_proxy_transport'):
if settings.transportReceivingIsEnabled('proxy'):
try:
# TODO: change here to receive the value directly from service_proxy_transport object
router_idurl = driver.services(
)['service_proxy_transport'].transport.options[
'router_idurl']
except:
router_idurl = None
if router_idurl:
router_identity = identitycache.FromCache(router_idurl)
contacts_is_ok = True
router_protos = router_identity.getContactsByProto()
if lid.getContactsNumber() != len(router_protos):
contacts_is_ok = False
if contacts_is_ok:
for proto, contact in router_protos.items():
if lid.getProtoContact(proto) != contact:
contacts_is_ok = False
if contacts_is_ok:
if _Debug:
lg.out(
_DebugLevel - 6,
'p2p_connector._check_to_use_best_proto returning True : proxy_transport is OK'
)
return True
first = order[0]
# if first contact in local identity is not working yet
# but there is another working methods - switch first method
if first not in active_protos():
if _Debug:
lg.out(
_DebugLevel - 6,
'p2p_connector._check_to_use_best_proto first contact (%s) is not working! active_protos()=%s'
% (first, str(active_protos())))
return False
# #small hack to make udp as first method if all is fine
# if first != 'udp' and ('udp' in active_protos() and 'tcp' in active_protos()):
# lg.out(2, 'p2p_connector._check_to_use_best_proto first contact (%s) but UDP also works! active_protos()=%s' % (first, str(active_protos())))
# return False
# if tcp contact is on first place and it is working - we are VERY HAPPY! - no need to change anything - return False
if first == 'tcp' and 'tcp' in active_protos():
return True
# but if tcp method is not the first and it works - we want to TURN IT ON! - return True
if first != 'tcp' and 'tcp' in active_protos():
if _Debug:
lg.out(
_DebugLevel - 6,
'p2p_connector._check_to_use_best_proto tcp is not first but it works active_protos()=%s'
% str(active_protos()))
return False
# if we are using udp and it is working - this is fantastic!
if first == 'udp' and 'udp' in active_protos():
# but let's check if TCP is also working
# in that case we want to switch to TCP
if 'tcp' in active_protos():
return False
return True
# udp seems to be working and first contact is not working - so switch to udp
if first != 'udp' and 'udp' in active_protos():
if _Debug:
lg.out(
_DebugLevel - 6,
'p2p_connector._check_to_use_best_proto udp is not first but it works active_protos()=%s'
% str(active_protos()))
return False
# http seems to work and it is first - cool!
if first == 'http' and 'http' in active_protos():
return True
# but if http method is not the first and it works - we want to TURN IT ON! - return True
if first != 'http' and 'http' in active_protos():
if _Debug:
lg.out(
_DebugLevel - 6,
'p2p_connector._check_to_use_best_proto http is not first but it works active_protos()=%s'
% str(active_protos()))
return False
# if we are using proxy and it is working - that is fine - it must work always!
if first == 'proxy' and 'proxy' in active_protos():
return True
# proxy seems to be working and first contact is not working - so switch to proxy
if first != 'proxy' and 'proxy' in active_protos():
if _Debug:
lg.out(
_DebugLevel - 6,
'p2p_connector._check_to_use_best_proto proxy is not first but it works active_protos()=%s'
% str(active_protos()))
return False
# in other cases - do nothing
return True
def _pop_active_proto(self):
if len(active_protos()) == 0:
return
lid = my_id.getLocalIdentity()
order = lid.getProtoOrder()
first = order[0]
wantedproto = ''
# if first contact in local identity is not working yet
# but there is another working methods - switch first method
if first not in active_protos():
# take (but not remove) any item from the set
wantedproto = active_protos().pop()
active_protos().add(wantedproto)
# if proxy method is not the first but it works - switch to proxy
if first != 'proxy' and 'proxy' in active_protos():
wantedproto = 'proxy'
# if http method is not the first but it works - switch to http
if first != 'http' and 'http' in active_protos():
wantedproto = 'http'
# if udp method is not the first but it works - switch to udp
if first != 'udp' and 'udp' in active_protos():
wantedproto = 'udp'
# if tcp method is not the first but it works - switch to tcp
if first != 'tcp' and 'tcp' in active_protos():
wantedproto = 'tcp'
if _Debug:
lg.out(
_DebugLevel - 6,
'p2p_connector.PopWorkingProto will pop %s contact order=%s active_protos()=%s'
% (wantedproto, str(order), str(active_protos())))
# now move best proto on the top
# other users will use this method to send to us
lid.popProtoContact(wantedproto)
# save local id
# also need to propagate our identity
# other users must know our new contacts
my_id.setLocalIdentity(lid)
my_id.saveLocalIdentity()
def _update_my_identity(self):
contacts_changed = False
old_contacts = list(my_id.getLocalIdentity().getContacts())
identity_changed = my_id.rebuildLocalIdentity()
if not identity_changed and len(active_protos()) > 0:
self.automat('my-id-updated', (False, False))
return
new_contacts = my_id.getLocalIdentity().getContacts()
if len(old_contacts) != len(new_contacts):
contacts_changed = True
if not contacts_changed:
for pos in range(len(old_contacts)):
if old_contacts[pos] != new_contacts[pos]:
contacts_changed = True
break
if contacts_changed:
# erase all stats about received packets
# if some contacts in my identity has been changed
if _Debug:
lg.out(
4,
' my contacts were changed, erase active_protos() flags'
)
active_protos().clear()
if _Debug:
lg.out(
4, ' identity HAS %sBEEN CHANGED' %
('' if identity_changed else 'NOT '))
self.automat('my-id-updated', (contacts_changed, identity_changed))
def _check_rotate_propagate_my_identity(self):
# TODO: rebuild that method into another state machine
from p2p import id_rotator
def _do_propagate(*args):
if _Debug:
lg.out(_DebugLevel, 'p2p_connector._do_propagate')
if driver.is_on('service_entangled_dht'):
from dht import dht_service
dht_service.set_node_data('idurl',
my_id.getLocalID().to_text())
d = propagate.start(wide=True, refresh_cache=True)
d.addCallback(lambda contacts_list: self.automat(
'my-id-propagated', contacts_list))
if _Debug:
d.addErrback(
lg.errback,
debug=_Debug,
debug_level=_DebugLevel,
method='_check_rotate_propagate_my_identity._do_propagate')
def _on_propagate_failed(err):
lg.err('failed propagate my identity: %r' % err)
def _do_update(check_rotate_result):
if _Debug:
lg.out(
_DebugLevel,
'p2p_connector._do_update check_rotate_result=%r' %
check_rotate_result)
if not check_rotate_result:
lg.err(
'failed to rotate identity sources, skip propagating my identity'
)
self.automat('my-id-propagated', [])
return None
d = propagate.update()
d.addCallback(_do_propagate)
d.addErrback(_on_propagate_failed)
d.addErrback(lambda *args: self.automat('my-id-propagated', []))
def _do_rotate(check_result):
if _Debug:
lg.out(
_DebugLevel,
'p2p_connector._do_rotate check_result=%r' % check_result)
if check_result:
lg.info('identity sources are healthy, send my identity now')
_do_update(True)
return None
lg.err(
'identity sources are not healthy, will execute identity rotate flow now'
)
d = id_rotator.run()
d.addCallback(_do_update)
d.addErrback(lambda _: _do_update(False))
def _do_check(x):
if _Debug:
lg.out(_DebugLevel, 'p2p_connector._do_check')
d = id_rotator.check()
d.addCallback(_do_rotate)
d.addErrback(lambda _: _do_rotate(False))
_do_check(None)
def _do_id_server_health_check(self):
my_idurl = my_id.getLocalIdentity().getIDURL(as_original=True)
if _Debug:
lg.args(_DebugLevel, my_idurl=my_idurl)
def _verify(xmlsrc=None):
if not xmlsrc:
lg.err('my current identity server not healthy')
self.NeedPropagate = True
self.automat('check-synchronize')
return
remote_ident = identity.identity(xmlsrc=xmlsrc)
if not remote_ident.isCorrect() or not remote_ident.Valid():
lg.warn(
'my current identity server responded with bad identity file'
)
self.NeedPropagate = True
self.automat('check-synchronize')
return
if remote_ident.getIDURL(as_original=True) != my_idurl:
lg.warn(
'my current identity server responded with unknown identity'
)
self.NeedPropagate = True
self.automat('check-synchronize')
return
if _Debug:
lg.dbg(_DebugLevel, 'my current identity server is healthy')
last_time = identitycache.last_time_cached(my_idurl)
if last_time and time.time() - last_time < config.conf().getInt(
'services/identity-propagate/health-check-interval-seconds'):
if _Debug:
lg.dbg(
_DebugLevel,
'skip health check of my current identity server, last time cached %d seconds ago'
% (time.time() - last_time))
return
d = identitycache.immediatelyCaching(my_idurl, try_other_sources=False)
d.addCallback(_verify)
d.addErrback(lambda _: _verify())
class ALSAAudioDevice(baseaudio.AudioDevice):
writedev = None
readdev = None
def __repr__(self):
return "ALSAAudioDevice %s" % (self.isOpen() and "open" or "closed")
def openDev(self):
print "alsa OPEN", self._closed
try:
from __main__ import app
except:
app = None
if app is None:
device = DEFAULT_ALSA_DEVICE
else:
device = app.getPref('audio_device', DEFAULT_ALSA_DEVICE)
log.msg("alsaaudiodev opening device %s" % (device))
writedev = alsaaudio.PCM(alsaaudio.PCM_PLAYBACK,
alsaaudio.PCM_NONBLOCK, device)
self.writechannels = writedev.setchannels(1)
writedev.setrate(8000)
writedev.setformat(alsaaudio.PCM_FORMAT_S16_LE)
writedev.setperiodsize(160)
self.writedev = writedev
readdev = alsaaudio.PCM(alsaaudio.PCM_CAPTURE,
alsaaudio.PCM_NONBLOCK, device)
self.readchannels = readdev.setchannels(1)
readdev.setrate(8000)
readdev.setformat(alsaaudio.PCM_FORMAT_S16_LE)
readdev.setperiodsize(160)
self.readdev = readdev
self.LC = LoopingCall(self._push_up_some_data)
self.LC.start(0.010)
print "alsa OPENED", self._closed
def _push_up_some_data(self):
if self.readdev is None:
return
(l, data,) = self.readdev.read()
if self.readchannels == 2:
data = audioop.tomono(data, 2, 1, 1)
if self.encoder and data:
self.encoder.handle_audio(data)
def write(self, data):
if not hasattr(self, 'LC'):
return
assert self.isOpen(), "calling write() on closed %s"%(self,)
if self.writechannels == 2:
data = audioop.tostereo(data, 2, 1, 1)
wrote = self.writedev.write(data)
if not wrote: log.msg("ALSA overrun")
def _close(self):
print "alsa CLOSE", self._closed
if self.isOpen():
log.msg("alsaaudiodev closing")
try:
self.LC.stop()
except AttributeError:
# ? bug in Twisted? Not sure. This catch-and-ignore is a temporary workaround. --Zooko
pass
del self.LC
self.writedev = None
self.readdev = None
print "alsa CLOSED", self._closed
class ServerController(object):
def __init__(self, realm, view):
print('server cont!')
self.realm = realm
self.view = view
def _handleInput(self):
"""
Handle currently available pygame input events.
"""
for event in pygame.event.get():
if (event.type == pygame.QUIT) or ((event.type == pygame.KEYDOWN) and (event.key == QUIT)):
reactor.stop()
sys.exit()
if (event.type == pygame.KEYDOWN):
if (event.key == START_GAME):
self.realm.environment.startGame()
elif (event.key == RESET_GAME):
self.realm.environment.setPreGame()
def go(self):
self.previousTime = pygame.time.get_ticks()
self._inputCall = LoopingCall(self._handleInput)
d = self._inputCall.start(0.03)
return d
def stop(self):
self._inputCall.stop()
