class LoggingProcessProtocol(ProcessProtocol, object): """ A ProcessProtocol that logs all output to a file """ def __init__(self, commandname, maxbackups=3): log_name = commandname + ".log" log_dir = os.path.join(fs.adirs.user_log_dir, "processes") if not os.path.isdir(log_dir): os.makedirs(log_dir) log_name = os.path.join(log_dir, log_name) _backup_logs(log_name, maxbackups) self.log = Logger(observer=textFileLogObserver(io.open(log_name, "w")), namespace="") super(LoggingProcessProtocol, self).__init__() def connectionMade(self): self.finished = defer.Deferred() def outReceived(self, data): self.log.info("{data}", data=bytes_to_str(data.strip())) def errReceived(self, data): self.log.error("{data}", data=bytes_to_str(data.strip())) def processEnded(self, reason): if reason.check(ProcessDone): self.finished.callback(True) self.log.info("Process finished without error") else: self.finished.errback(reason) self.log.error("Process ended with error: {reason!r}", reason=reason)
def test_logger_namespace(self): """ A `twisted.logger.Logger` with a namespace gets that namespace as a prefix. """ fout = StringIO() log = Logger(namespace="ns", observer=FileLogObserver(fout, formatForSystemd)) log.info("info\n{more}", more="info") log.error("err") self.assertEqual(( "<6>[ns] info\n" "<6> info\n" "<3>[ns] err\n" ), fout.getvalue())
def render_POST(self, txrequest): logger = Logger(namespace='- UpdateDbSchedule -') args = native_stringify_dict(copy(txrequest.args), keys_only=False) args = dict((k, v[0]) for k, v in args.items()) id = args.pop('id') set_dic = args.copy() where_dic = {'id': id} lock.acquire() try: if database_type == 'mysql': database_connector.update_data(model=SpiderScheduleModel, set_dic=set_dic, where_dic=where_dic) else: database_connector.update(model_name='SpiderScheduleModel', update_dic=set_dic, filter_dic=where_dic) sta = 'ok' except Exception as E: logger.error('update fail: {}'.format(E)) sta = 'Update fail: {}'.format(E) lock.release() return {"node_name": self.root.nodename, 'update': sta}
def log_event(config: Config, token: Token, etype, ctx: dict): log = Logger() if type(token) is str: token = token.encode('utf-8') log.debug('token: ' + token.decode('utf-8')) t = Token.query.filter_by(token=token.decode("utf-8")).first() if t is None: log = Logger() log.error('Unable to find %s token' % token.decode("utf-8")) return e = Event(type=etype, token_id=t.id, token=t, timestamp=datetime.utcnow(), context=json.dumps(ctx)) db.add(e) db.commit() message = f""" DNS Checkin Date: {e.timestamp} UTC Token: {t.token} Tags: {t.context} Context: {json.dumps(ctx)} """ print(config.notifications) for n in config.notifications: if not n.isEnabled(): continue try: res = n.notify(message) print(res) print(res.request) except: log.error('error executing notify class %s' % n)
class MFTests(object): def __init__(self): self.log = Logger() def start_static(self): resource = File(os.getcwd() + '/tests/pages') factory = Site(resource) endpoint = endpoints.TcP4ServerEndpoint(reactor, 0) endpoint.listen(factory) # reactor.run() def send_request(self): pass def stop_callback(self, none): reactor.stop() def test_log_handler(self): handler = LogHandler() self.log.info('Test msg with {parameter} is OK', parameter="value") self.log.error('Test error with {parameter} is OK', parameter="value") self.log.error('Test error with {parameter} (isError={isError}) is OK', parameter="value", isError=False) self.log.error('Test error with {parameter} (isError={isError}) is OK', parameter="value", isError=True) d = defer.Deferred() reactor.callLater(0, d.callback, None) d.addCallback(self.stop_callback) d.addErrback( lambda err: print("callback error: %s\ncallback traceback: %s" % (err.getErrorMessage(), err.getTraceback()))) reactor.run() def test_server(self): d = defer.Deferred() reactor.callLater(3, d.callback, None) d.addCallback(self.stop_callback) #d.addCallback(self.send_request) d.addErrback( lambda err: print("callback error: %s\ncallback traceback: %s" % (err.getErrorMessage(), err.getTraceback()))) Server(port=1234, db_creds=None, snapshot_dir='~/tmp', user_agent='', debug=False).run()
class MFTests(object): def __init__(self): self.log = Logger() def start_static(self): resource = File(os.getcwd() + '/tests/pages') factory = Site(resource) endpoint = endpoints.TcP4ServerEndpoint(reactor, 0) endpoint.listen(factory) # reactor.run() def send_request(self): pass def stop_callback(self, none): reactor.stop() def test_log_handler(self): handler = LogHandler() self.log.info('Test msg with {parameter} is OK', parameter="value") self.log.error('Test error with {parameter} is OK', parameter="value") self.log.error('Test error with {parameter} (isError={isError}) is OK', parameter="value", isError=False) self.log.error('Test error with {parameter} (isError={isError}) is OK', parameter="value", isError=True) d = defer.Deferred() reactor.callLater(0, d.callback, None) d.addCallback(self.stop_callback) d.addErrback(lambda err: print("callback error: %s\ncallback traceback: %s" % (err.getErrorMessage(), err.getTraceback()))) reactor.run() def test_server(self): d = defer.Deferred() reactor.callLater(3, d.callback, None) d.addCallback(self.stop_callback) #d.addCallback(self.send_request) d.addErrback(lambda err: print("callback error: %s\ncallback traceback: %s" % (err.getErrorMessage(), err.getTraceback()))) Server(port=1234, db_creds=None, snapshot_dir='~/tmp', user_agent='', debug=False).run()
class Rest(object): def __init__(self, host='https://developer-api.nest.com', token=None, event_handler=None, net_type='lan'): self.log = Logger() self.host = host self.token = token self.event_handler = event_handler self.pool = HTTPConnectionPool(reactor, persistent=True) self.loc = None self.reconnect = False self.fail_count = 0 if event_handler: self.reconnect = True d = self.request( headers={ 'User-Agent': ['onDemand Rest Client'], 'Accept': ['text/event-stream'] }) d.addCallback(self.on_disconnect) def __getattr__(self, name): try: super(Rest, self).__getattr__(name) except AttributeError: return RestCall(self, name) def on_disconnect(self, reason): if not reason: reason = {'reason': 'no_message'} self.log.critical('disconnected: {reason}', reason=reason['reason']) if self.fail_count > 10: self.log.error('Max error count reached, aborting connection') def test_connectivity(count): if self.fail_count == count: self.fail_count = 0 self.fail_count += 1 c = self.fail_count reactor.callLater(10, test_connectivity, c) # @UndefinedVariable if self.reconnect: d = self.request( headers={ 'User-Agent': ['onDemand Rest Client'], 'Accept': ['text/event-stream'] }) d.addCallback(self.on_disconnect) def request( self, method='GET', path='', headers={ 'User-Agent': ['onDemand/1.0 (Rest_Client)'], 'Accept': ['application/json'] }, body=None): data = None if self.loc: host = '/'.join((self.loc, path)) else: host = '/'.join((self.host, path)) if self.token: host += '?auth=' + self.token if body: headers.update({'Content-Type': ['application/json']}) data = FileBodyProducer(StringIO(json.dumps(body))) agent = RedirectAgent(Agent(reactor, pool=self.pool)) d = agent.request(method, host, Headers(headers), data) def cbFail(fail): if hasattr(fail.value, 'response'): if hasattr(fail.value.response, 'code'): if fail.value.response.code == 307: loc = fail.value.response.headers.getRawHeaders( 'location') new = urlparse(loc[0]) newhost = '://'.join((new.scheme, new.netloc)) if newhost == self.host: self.loc = None else: self.loc = newhost self.log.debug('redirect: %s' % self.loc) data = FileBodyProducer(StringIO(json.dumps(body))) d = agent.request(method, loc[0], Headers(headers), data) d.addCallbacks(cbRequest, cbFail) return d elif fail.value.response.code == 404 and self.loc: self.loc = None host = '/'.join((self.host, path)) if self.token: host += '?auth=' + self.token d = self.request(method, host, Headers(headers), body) d.addCallbacks(cbRequest, cbFail) return d else: print(dir(fail.value)) print(fail.value.message) print(fail.value.args) self.log.error('unhandled failure: %s -- %s' % (fail.value.message, fail.value)) def cbRequest(response): # print 'Response version:', response.version # print 'Response code:', response.code # print 'Response phrase:', response.phrase # print 'Response headers:' # print pformat(list(response.headers.getAllRawHeaders())) finished = Deferred() response.deliverBody(RestHandle(finished, self.event_handler)) return finished d.addCallbacks(cbRequest, cbFail) return d
class PhotometerService(Service): BUFFER_SIZE = 1 def __init__(self, options, label): self.options = options self.label = label self.namespace = self.label.upper() setLogLevel(namespace=self.namespace, levelStr=options['log_messages']) setLogLevel(namespace=self.label, levelStr=options['log_level']) self.log = Logger(namespace=self.label) self.factory = self.buildFactory() self.protocol = None self.serport = None self.buffer = CircularBuffer(self.BUFFER_SIZE, self.log) self.counter = 0 # Handling of Asynchronous getInfo() self.info = None self.info_deferred = None if options['old_firmware']: self.info = { 'name' : self.options['name'], 'mac' : self.options['mac_address'], 'calib' : self.options['zp'], 'rev' : 2, } # Serial port Handling parts = chop(self.options['endpoint'], sep=':') if parts[0] != 'serial': self.log.critical("Incorrect endpoint type {ep}, should be 'serial'", ep=parts[0]) raise NotImplementedError def startService(self): ''' Starts the photometer service listens to a TESS Although it is technically a synchronous operation, it works well with inline callbacks ''' self.log.info("starting {name}", name=self.name) self.connect() def stopService(self): self.log.warn("stopping {name}", name=self.name) self.protocol.transport.loseConnection() self.protocol = None self.serport = None #self.parent.childStopped(self) return defer.succeed(None) #--------------------- # Extended Service API # -------------------- @inlineCallbacks def reloadService(self, new_options): ''' Reload configuration. Returns a Deferred ''' options = options[self.label] setLogLevel(namespace=self.label, levelStr=options['log_level']) setLogLevel(namespace=self.namespace, levelStr=options['log_messages']) self.options = options return defer.succeed(None) # ----------------------- # Specific photometer API # ----------------------- def handleInfo(self, reading): if self.info_deferred is not None: self.info = { 'name' : reading.get('name', None), 'calib' : reading.get('ZP', None), 'mac' : self.options['mac_address'], 'rev' : 2, } self.log.info("Photometer Info: {info}", info=self.info) self.info_deferred.callback(self.info) self.info_deferred = None def curate(self, reading): '''Readings ready for MQTT Tx according to our wire protocol''' reading['seq'] = self.counter self.counter += 1 self.last_tstamp = reading.pop('tstamp', None) if self.options['old_firmware']: reading['mag'] = round(self.options['zp'] - 2.5*math.log10(reading['freq']),2) reading['rev'] = 2 reading['name'] = self.options['name'] reading['alt'] = 0.0 reading['azi'] = 0.0 reading['wdBm'] = 0 reading.pop('zp', None) else: reading['mag'] = round(reading['ZP'] - 2.5*math.log10(reading['freq']),2) self.info = { 'name' : reading.get('name', None), 'calib' : reading.get('ZP', None), 'mac' : self.options['mac_address'], 'rev' : 2, } reading.pop('udp', None) reading.pop('ain', None) reading.pop('ZP', None) return reading def getInfo(self): '''Asynchronous operations''' if not self.options['old_firmware'] and self.info is None: deferred = defer.Deferred() deferred.addTimeout(60, reactor) self.info_deferred = deferred else: self.log.info("Photometer Info: {info}", info=self.info) deferred = defer.succeed(self.info) return deferred # -------------- # Helper methods # --------------- def connect(self): parts = chop(self.options['endpoint'], sep=':') endpoint = parts[1:] self.protocol = self.factory.buildProtocol(0) try: self.serport = SerialPort(self.protocol, endpoint[0], reactor, baudrate=endpoint[1]) except Exception as e: self.log.error("{excp}",excp=e) self.protocol = None else: self.gotProtocol(self.protocol) self.log.info("Using serial port {tty} @ {baud} bps", tty=endpoint[0], baud=endpoint[1]) def buildFactory(self): self.log.debug("Choosing a {model} factory", model=TESSW) import tessw.tessw factory = tessw.tessw.TESSProtocolFactory(self.namespace, self.options['old_firmware']) return factory def gotProtocol(self, protocol): self.log.debug("got protocol") self.buffer.registerProducer(protocol, True) self.protocol = protocol
class BlocClient(Service): """ Client to connect to bloc server """ def __init__(self, clock, server, interval, treq=treq, session_id=None): """ Create a BlocClient instance :param clock: An implementation of :obj:`IReactorTime`. Typically will be main twisted reactor. :param str server: server connection info in "server:port" form :param float interval: Frequency of heartbeat in seconds """ self.clock = clock self._server = server self._settled = False self._index = 0 self._total = 0 self._interval = interval self._loop = task.LoopingCall(self._heartbeat) self._loop.clock = self.clock if session_id is None: # pragma: no cover self._session_id = str(uuid.uuid1()) else: self._session_id = session_id self.log = Logger() self.treq = treq def startService(self): """ Start getting the index and effectively heartbeating """ super(BlocClient, self).startService() self._loop.start(self._interval, True) def _set_index(self, content): if content['status'] == 'SETTLED': self._settled = True self._index = content['index'] self._total = content['total'] else: self._settled = False def _url(self, segment): return 'http://{}/{}'.format(self._server, segment) def _get_index(self): d = self.treq.get(self._url("index"), headers={'Bloc-Session-ID': [self._session_id]}) d.addCallback(check_status, [200]) return d.addCallback(treq.json_content) def _error_allocating(self, f): self._settled = False self.log.error("Error getting index: {f}", f=f) def _heartbeat(self): d = self._get_index() d.addCallback(self._set_index) d.addTimeout(self._interval, self.clock) d.addErrback(self._error_allocating) return d def stopService(self): """ Delete session and stop heartbeating """ super(BlocClient, self).stopService() # Delete session before shutdown but do not worry about response if it not received # within 1 second because we don't want to block shutdown of twisted app and server will # anyway cancel the session without next heartbeat d = self.treq.delete(self._url("session"), headers={'Bloc-Session-ID': [self._session_id]}) d.addTimeout(1, self.clock) return d.addBoth(lambda r: self._loop.stop()) def get_index_total(self): """ Return (index, total) tuple if settled, None if settling. Here "index" is position of this node in the group and "total" is number of nodes in the group. For example, if there are two BlocClient instances talking to the server then one of them will get (1, 2) and other will get (2, 2). Note that this returns internal state last updated every "interval" seconds. """ if not self._settled: return None return (self._index, self._total)
class OurStreamProtocol(Protocol): """Protocol implementing ShinySDR's WebSocket service. This protocol's transport should be a txWS WebSocket transport. """ def __init__(self, caps, subscription_context): self.__log = Logger() self.__subscription_context = subscription_context self._caps = caps self._seenValues = {} self.inner = None def dataReceived(self, data): """Twisted Protocol implementation. Additionally, txWS takes no care with exceptions here, so we catch and log.""" # pylint: disable=broad-except try: if self.inner is None: # To work around txWS's lack of a notification when the URL is available, all clients send a dummy first message. self.__dispatch_url() else: self.inner.dataReceived(data) except Exception: self.__log.failure('Error processing incoming WebSocket message') def __dispatch_url(self): loc = self.transport.location self.__log.info('Stream connection to {url}', url=loc) path = [urllib.unquote(x) for x in loc.split('/')] assert path[0] == '' path[0:1] = [] cap_string = path[0].decode('utf-8') # TODO centralize url decoding if cap_string in self._caps: root_object = self._caps[cap_string] path[0:1] = [] else: raise Exception('Unknown cap') # TODO better error reporting if len(path) == 1 and path[0].startswith(b'audio?rate='): rate = int( json.loads(urllib.unquote(path[0][len(b'audio?rate='):]))) self.inner = AudioStreamInner(the_reactor, self.__send, root_object, rate) elif len(path) >= 1 and path[0] == CAP_OBJECT_PATH_ELEMENT: # note _lookup_block may throw. TODO: Better error reporting root_object = _lookup_block(root_object, path[1:]) self.inner = StateStreamInner( self.__send, root_object, loc, self.__subscription_context ) # note reuse of loc as HTTP path; probably will regret this else: raise Exception('Unknown path: %r' % (path, )) def connectionMade(self): """twisted Protocol implementation""" self.transport.setBinaryMode(True) # Unfortunately, txWS calls this too soon for transport.location to be available def connectionLost(self, reason): # pylint: disable=signature-differs """twisted Protocol implementation""" if self.inner is not None: self.inner.connectionLost(reason) def __send(self, message, safe_to_drop=False): if len(self.transport.transport.dataBuffer) > 1000000: # TODO: condition is horrible implementation-diving kludge # Don't accumulate indefinite buffer if we aren't successfully getting it onto the network. # TODO: There are no tests of this mechanism if safe_to_drop: self.__log.warn('Dropping data going to stream {url}', url=self.transport.location) else: self.__log.error( 'Dropping connection due to too much data on stream {url}', url=self.transport.location) self.transport.close(reason='Too much data buffered') else: self.transport.write(message)
class XmppEvent(object): def __init__(self, nodeId, parent, pubsub_addr): self.log = Logger() self.nodeId = nodeId self.parent = parent self.addr = pubsub_addr def publish(self, event): if len(self.parent.active_controllers) == 0: # self.log.debug('event cancelled') self.parent.registrations = [] return def success(res): # print('event sent') if res['type'] == 'error': self.log.error('Publish Event failed :%s' % res.toXml()) else: if 'Id' in res.children[0].children[0]['node']: self.log.debug('Event Published: %s' % res.toXml()) name, data = event if name == 'Seconds': return iq = IQ(self.parent.xmlstream, 'set') ps = domish.Element(('http://jabber.org/protocol/pubsub', 'pubsub')) publish = domish.Element((None, 'publish')) publish['node'] = '/'.join((self.nodeId, name)) item = domish.Element((None, 'item')) propertyset = domish.Element( ('urn:schemas-upnp-org:event-1-0', 'propertyset'), localPrefixes={'e': 'urn:schemas-upnp-org:event-1-0'}) prop = domish.Element((None, 'e:property')) evt = domish.Element((None, name)) if isinstance(data.value, dict): ev = domish.Element((data.namespace, 'Event')) inst = domish.Element((None, 'InstanceID')) inst['val'] = '0' for k, v in data.value.items: if 'namespace' in v: var = domish.Element((v['namespace'], k)) else: var = domish.Element((None, k)) if 'attrib' in v: attr = v['attrib'] else: attr = {} value = v['value'] if isinstance(value, bool): value = int(value) attr.update({'val': str(value).decode('utf-8')}) for attname, attval in attr: var[attname] = attval inst.addChild(var) ev.addChild(inst) evt.addChild(ev) else: # print(str(data.value).decode('utf-8')) if isinstance(data.value, bool): data.value = int(data.value) evt.addContent(str(data.value).decode('utf-8')) prop.addChild(evt) propertyset.addChild(prop) item.addChild(propertyset) publish.addChild(item) ps.addChild(publish) iq.addChild(ps) iq.addCallback(success) iq.send(to=self.addr)
logger.debug('Enter MySQL Password:'******'MySQL']['pass'] = raw_input() logger.debug( 'Enter old MySQL database name (the one from which you are going to port):' ) CONFIG['MySQL']['oldname'] = raw_input() logger.debug( 'Enter new MySQL database name (the one to which new data is transferred):' ) CONFIG['MySQL']['dbname'] = raw_input() logger.info('Checking DB Name conventions, and rules') if not CONFIG['MySQL']['dbname'].endswith('line'): logger.error( 'Database name should end with line (eg. Timeline), a strictly enforced nomenclature.' ) sys.exit(0) logger.info('DB naming rules check - clear') logger.info('Checking MySQL Credentials') try: MYSQL_CONN = mysql.connector.connect(user=CONFIG['MySQL']['user'], password=CONFIG['MySQL']['pass'], database=CONFIG['MySQL']['oldname']) except Exception, e: print 'Error MySQL', e os._exit(1)
class XmppEvent(object): def __init__(self, nodeId, parent, pubsub_addr): self.log = Logger() self.nodeId = nodeId self.parent = parent self.addr = pubsub_addr def publish(self, event): if len(self.parent.active_controllers) == 0: # self.log.debug('event cancelled') self.parent.registrations = [] return def success(res): # print('event sent') if res['type'] == 'error': self.log.error('Publish Event failed :%s' % res.toXml()) else: if 'Id' in res.children[0].children[0]['node']: self.log.debug('Event Published: %s' % res.toXml()) name, data = event if name == 'Seconds': return iq = IQ(self.parent.xmlstream, 'set') ps = domish.Element(('http://jabber.org/protocol/pubsub', 'pubsub')) publish = domish.Element((None, 'publish')) publish['node'] = '/'.join((self.nodeId, name)) item = domish.Element((None, 'item')) propertyset = domish.Element( ('urn:schemas-upnp-org:event-1-0', 'propertyset'), localPrefixes={'e': 'urn:schemas-upnp-org:event-1-0'}) prop = domish.Element((None, 'e:property')) evt = domish.Element((None, name)) if isinstance(data.value, dict): ev = domish.Element((data.namespace, 'Event')) inst = domish.Element((None, 'InstanceID')) inst['val'] = '0' for k, v in data.value.items: if 'namespace' in v: var = domish.Element((v['namespace'], k)) else: var = domish.Element((None, k)) if 'attrib' in v: attr = v['attrib'] else: attr = {} value = v['value'] if isinstance(value, bool): value = int(value) attr.update( {'val': str(value) .decode('utf-8')}) for attname, attval in attr: var[attname] = attval inst.addChild(var) ev.addChild(inst) evt.addChild(ev) else: # print(str(data.value).decode('utf-8')) if isinstance(data.value, bool): data.value = int(data.value) evt.addContent(str(data.value).decode('utf-8')) prop.addChild(evt) propertyset.addChild(prop) item.addChild(propertyset) publish.addChild(item) ps.addChild(publish) iq.addChild(ps) iq.addCallback(success) iq.send(to=self.addr)
signal.signal(signal.SIGUSR1, receive_signal) atexit.register(contacts.close) servers_file_path = '%s/.servers' % config['directory'] logger.info('loading server') try: servers = json.load(open(servers_file_path)) logger.info( 'read last read date from server neighbors from {servers_file_path}', servers_file_path=servers_file_path) except json.JSONDecodeError: logger.error( "Bad JSON in server file at {file_path} recovering automatically", file_path=servers_file_path) servers = {} except FileNotFoundError as err: servers = {} if config.get('servers'): for server in config.get('servers').split(','): if server not in servers: servers[server] = '1970-01-01T00:00Z' allowable_methods = [ '/status/scan:POST', '/status/send:POST', '/status/update:POST', '/sync:GET', '/admin/config:GET', '/admin/status:GET', '/status/result:POST', '/status/data_points:POST', '/init:POST' ]
class Discord(IRCClient): nickname = "discord" realname = "Discord" username = "******" versionName = "Discord" versionNum = "0.01" magicFile = "true.txt" def __init__(self, accessList): self.logger = Logger(observer=textFileLogObserver(sys.stdout)) self.accessList = [nick.lower() for nick in accessList] if not os.path.exists(self.magicFile): self.logger.info("Creating magic file") try: with open(self.magicFile, "a"): pass except Exception as ex: self.logger.error("Unable to create magic file! {0}".format(ex.message)) reactor.stop() self.markovGenerator = pymarkov.MarkovChainGenerator(self.magicFile) self.channels = [] self.channelPhrasers = {} self.logger.debug("Discord initialized") # Maybe add hook/plugin system here? self.commands = Commands.Commands(self) def removeChannel(self, channel): try: self.channels.remove(channel) self.channelPhrasers[channel].stop() del self.channelPhrasers[channel] except: self.logger.error("Error removing {channel} from collection", channel=channel) def insertPhrase(self, phrase): try: with open(self.magicFile, "a") as magicFile: magicFile.write("{0}\n".format(phrase)) try: file, ext = os.path.splitext(self.magicFile) os.remove("{0}-pickled{1}".format(file, ext)) # Simply re-populating the dictionary isn't enough for some reason self.markovGenerator = pymarkov.MarkovChainGenerator(self.magicFile, 2) except IOError as ex: self.logger.error("Unable to delete pickled file. {0}".format(ex.message)) except Exception as ex: self.logger.error("Unable to insert phrase into magic file! {0}".format(ex.message)) def kickedFrom(self, channel, kicker, message): self.removeChannel(channel) self.logger.info("Kicked from {channel} by {kicker}", channel=channel, kicker=kicker) def left(self, channel): self.removeChannel(channel) self.logger.info("Left {channel}", channel=channel) def handleMessage(self, user, channel, message): senderNickname = user.split("!")[0] if message.startswith("~reload") and senderNickname in self.accessList: self.logger.info("Reloading commands module") self.say(channel, "Reloading.") try: commandsModule = reload(Commands) self.commands = commandsModule.Commands(self) except Exception as ex: self.say(channel, "Failed to load commands module - {0}".format(ex.message)) elif message.startswith("~"): # Don't log commands to the brain commandMessage = message[1:] self.commands.handleCommand(user, channel, commandMessage) else: self.logger.info("Adding {message!r} to brain", message=message) # Avoid storing anything with the bot's name in it brainMessage = message.strip(self.nickname) self.insertPhrase(brainMessage) try: randomPhrase = self.generateSentence() if self.nickname in message and channel.startswith("#") and self.channelPhrasers[channel].running: phrase = "{0}, {1}".format(senderNickname, randomPhrase) self.say(channel, phrase) elif channel == self.nickname: self.logger.debug("Sending message to {nickname}", nickname=senderNickname) self.msg(senderNickname, randomPhrase) else: pass except IndexError as generationError: self.logger.error(generationError.message) def privmsg(self, user, channel, message): self.logger.info("Received message from {user} in {channel}", user=user, channel=channel) # deferToThread(self.handleMessage, user, channel, message) self.handleMessage(user, channel, message) def signedOn(self): self.logger.info("Signed on") self.join("#bots") def joined(self, channel): self.channels.append(channel) self.logger.info("Joined channel {channel!r}", channel=channel) channelPhraser = LoopingCall(self.sayRandomPhrase, channel) reactor.callLater(2, channelPhraser.start, 600) self.channelPhrasers[channel] = channelPhraser def generateSentence(self): try: sentence = self.markovGenerator.generate_sentence() sentence = sentence.strip("<{0}>".format(self.nickname)) sentence = sentence.strip(self.nickname) return sentence except (IndexError, ValueError) as ex: self.logger.error(ex.message) def sayRandomPhrase(self, channel): sentence = self.generateSentence() self.say(channel, sentence)
class TimeSchedule: def __init__(self, lock, host='127.0.0.1', port='6800'): config = Config() self.db = glv.get_value(key='sqlite_db') self.user_name = config.get('auth_username', '') self.user_password = config.get('auth_password', '') self.start_time = time.strftime("%Y %m %d %H %M %S", time.localtime()) self.server_port = 'http://{}:{}/'.format(host, port) self.schedule_post_url = 'http://{}:{}/schedule.json'.format( host, port) self.listproject_url = 'http://{}:{}/listprojects.json'.format( host, port) self.spider_task_dic = dict() self.projects = None self.db_lock = lock self.ts_lock = threading.Lock() self.CPU_THRESHOLD = 93 self.MEMORY_THRESHOLD = 96 self.schedule_logger = Logger(namespace='- Scheduler -') def run(self): time.sleep(3) self.projects = self.list_projects() self.schedule_logger.info('scheduler is running') count = 1 while True: schedule_sta = self.task_scheduler() if not schedule_sta and count == 1: self.schedule_logger.info('No Scheduled Spider in Database') count += 1 elif not schedule_sta and count != 1: count += 1 else: count = 1 time.sleep(1) def task_scheduler(self): self.ts_lock.acquire(blocking=True) self.db_lock.acquire() db_result = self.db.get( model_name='SpiderScheduleModel', key_list=['project', 'spider', 'schedule', 'args', 'status']) self.db_lock.release() self.ts_lock.release() schedule_list_raw = [{ 'project': x.project, 'spider': x.spider, 'schedule': x.schedule, 'args': x.args, 'status': x.status } for x in db_result if int(x.status) != 0] if db_result else [] schedule_sta = False if schedule_list_raw: for each_schedule in schedule_list_raw: project = each_schedule.get('project') if project in self.projects: schedule = each_schedule.get('schedule').replace('\\', '') try: schedule = json.loads(schedule) except: schedule = eval(schedule) try: next_time_sep = self.cal_time_sep(**schedule) next_time_sep = int(next_time_sep) + 1 if next_time_sep > 1: each_schedule['schedule'] = next_time_sep item = '{}-{}'.format(each_schedule['project'], each_schedule['spider']) self.ts_lock.acquire(blocking=True) if self.spider_task_dic.get(item) != 'waiting': self.spider_task_dic[item] = 'waiting' t = threading.Thread(target=self.poster, args=(each_schedule, )) try: t.start() except Exception as THError: self.schedule_logger.warn( 'start new job error [ {} ]: {}'. format(item, THError)) self.ts_lock.release() except ValueError: self.schedule_logger.error( 'spider runtime schedule error, please check the database' ) schedule_sta = True return schedule_sta def poster(self, dic): status = int(dic.pop('status')) project = dic.get('project') spider = dic.get('spider') job_str = " %s-%s " % (project, spider) args = dic.get('args').replace('\\', '') if args: args = eval(args) wait_time = dic.get('schedule') item = '{}-{}'.format(project, spider) if project and spider: data = { 'project': project, 'spider': spider, 'un': self.user_name, 'pwd': self.user_password } if args: data.update(args) self.schedule_logger.info( 'job {} is waiting, countdown {}s'.format(item, wait_time)) time.sleep(wait_time - 1) another_wait_time = 0 spider_runtime_avg = self.spiders_runtime(project=project, spider=spider) if status == 1: while not self.is_system_ok(): self.schedule_logger.warn( 'system is fully functioning, wait another 2 seconds to post schedule' ) time.sleep(2) another_wait_time += 3 if another_wait_time >= (wait_time - spider_runtime_avg): self.schedule_logger.warning( 'wait too long, cancel the job %s' % job_str) return None res = json.loads( requests.post(url=self.schedule_post_url, data=data).content) elif status == 2: res = json.loads( requests.post(url=self.schedule_post_url, data=data).content) elif status == 3: res = json.loads( requests.post(url=self.schedule_post_url, data=data).content) else: res = json.loads( requests.post(url=self.schedule_post_url, data=data).content) spider_status = res.get('status') if spider_status != 'ok': spider_status = 'error' else: self.schedule_logger.error( 'job project: {}, spider: {} post fail!'.format( project, spider)) spider_status = 'error' self.ts_lock.acquire(blocking=True) self.spider_task_dic[item] = spider_status self.ts_lock.release() def spiders_runtime(self, project, spider): self.db_lock.acquire() res = self.db.get(model_name='SpiderMonitor', key_list=['runtime'], filter_dic={ 'project': project, 'spider': spider }) self.db_lock.release() spider_list = [int(x.runtime) for x in res if x.runtime.isdigit()] if res else [0] return sum(spider_list) / len(spider_list) def list_projects(self): res = requests.get(url=self.listproject_url) projects = {} if res: projects_list = json.loads(res.content).get('projects') if projects_list: projects = set(projects_list) return projects def cal_time_sep( self, year='*', month='*', day='*', week='*', hour='*', minute='*', second='*', ): """ "%Y-%m-%d %H:%M:%S %w" """ y = int(time.strftime("%Y", time.localtime())) if year != '*' and '*' in year: y = int(year.split('/')[-1]) + y elif year.isdigit(): y = int(year) if week == '*': m = int(time.strftime("%m", time.localtime())) if month != '*' and '*' in month: m_raw = int(month.split('/')[-1]) if m_raw >= 12: raise ValueError( 'month value is too large, please set the year instead' ) m = m_raw + m if m > 12: y += m // 12 m = m % 12 elif month.isdigit(): m = int(month) days_in_this_month = self.how_many_days_in_this_month(y, m) d = int(time.strftime("%d", time.localtime())) if day != '*' and '*' in day: d_raw = int(day.split('/')[-1]) if d_raw > days_in_this_month: raise ValueError( 'day value is too large, please set the month or the year instead' ) d = d_raw + d if d > days_in_this_month: d = d - days_in_this_month m += 1 if m > 12: y += 1 m = m - 12 elif day.isdigit(): d = int(day) days_in_this_month = self.how_many_days_in_this_month(y, m) H = int(time.strftime("%H", time.localtime())) if hour != '*' and '*' in hour: H_raw = int(hour.split('/')[-1]) if H_raw > 24: raise ValueError( 'hour value is too large, please set the day instead') H = H_raw + H if H >= 24: H = H - 24 d += 1 if d > days_in_this_month: d = d - days_in_this_month m += 1 if m > 12: y += 1 m = m - 12 elif hour.isdigit(): H = int(hour) days_in_this_month = self.how_many_days_in_this_month(y, m) M = int(time.strftime("%M", time.localtime())) if minute != '*' and '*' in minute: M_raw = int(minute.split('/')[-1]) if M_raw > 60: raise ValueError( 'minute value is too large, please set the hour instead' ) M = M_raw + M if M >= 60: M = M - 60 H += 1 if H >= 24: H = H - 24 d += 1 if d > days_in_this_month: d = d - days_in_this_month m += 1 if m > 12: y += 1 m = m - 12 elif minute.isdigit(): M = int(minute) days_in_this_month = self.how_many_days_in_this_month(y, m) S = int(time.strftime("%S", time.localtime())) if second != '*' and '*' in second: S_raw = int(second.split('/')[-1]) if S_raw > 60: raise ValueError( 'second value is too large, please set the minute instead' ) S = S_raw + S if S >= 60: S = S - 60 M += 1 if M >= 60: M = M - 60 H += 1 if H >= 24: H = H - 24 d += 1 if d > days_in_this_month: d = d - days_in_this_month m += 1 if m > 12: y += 1 m = m - 12 elif second.isdigit(): S = int(second) time_sep = eval( "(datetime.datetime({},{},{}, {},{},{}) - datetime.datetime.now()).total_seconds()" .format(y, m, d, H, M, S)) else: week_in_this_year = int(time.strftime("%U", time.localtime())) w = int(time.strftime("%w", time.localtime())) if '*' in week: w_raw = int(week.split('/')[-1]) if w_raw >= 7: raise ValueError( 'week value is too large, please set the day or the month instead' ) if w_raw < w: week_in_this_year += 1 w = w_raw if week_in_this_year > 53: y += 1 week_in_this_year = week_in_this_year - 53 elif week.isdigit(): w = int(week) if int(week) < w: week_in_this_year += 1 H = int(time.strftime("%H", time.localtime())) if hour != '*' and '*' in hour: H_raw = int(hour.split('/')[-1]) if H_raw >= 24: raise ValueError( 'hour value is too large, please set the day instead') H = H_raw + H if H >= 24: H = H - 24 w += 1 if w >= 7: w = w - 7 week_in_this_year += 1 if week_in_this_year > 53: y += 1 week_in_this_year = week_in_this_year - 53 elif hour.isdigit(): H = int(hour) M = int(time.strftime("%M", time.localtime())) if minute != '*' and '*' in minute: M_raw = int(minute.split('/')[-1]) if M_raw >= 60: raise ValueError( 'minute value is too large, please set the hour instead' ) M = M_raw + M if M >= 60: M = M - 60 H += 1 if H >= 24: H = H - 24 w += 1 if w > 7: w = w - 7 week_in_this_year += 1 if week_in_this_year > 53: y += 1 week_in_this_year = week_in_this_year - 53 elif minute.isdigit(): M = int(minute) S = int(time.strftime("%S", time.localtime())) if second != '*' and '*' in second: S_raw = int(second.split('/')[-1]) if S_raw >= 60: raise ValueError( 'second value is too large, please set the minute instead' ) S = S_raw + S if S >= 60: S = S - 60 M += 1 if M >= 60: M = M - 60 H += 1 if H >= 24: H = H - 24 w += 1 if w > 7: w = w - 7 week_in_this_year += 1 if week_in_this_year > 53: y += 1 week_in_this_year = week_in_this_year - 53 elif second.isdigit(): S = int(second) if S >= 60: S = S - 60 M += 1 if M >= 60: M = M - 60 H += 1 if H >= 24: H = H - 24 w += 1 if w > 7: w = w - 7 week_in_this_year += 1 if week_in_this_year > 53: y += 1 week_in_this_year = week_in_this_year - 53 m, d = self.get_month_and_days_by_week( year=y, week_in_this_year=week_in_this_year, week=w) time_sep = eval( "(datetime.datetime({},{},{}, {},{},{}) - datetime.datetime.now()).total_seconds()" .format(y, m, d, H, M, S)) return time_sep def get_month_and_days_by_week(self, year, week_in_this_year, week): days = week_in_this_year * 7 + week if (year % 4 == 0 and year % 100 != 0) or (year % 400 == 0): Fe = 29 else: Fe = 28 month_lis = [31, Fe, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31] month_count = 1 days_count = 0 for month_days in month_lis: days = days - month_days if days > 0: month_count += 1 elif days == 0: days_count = 0 month_count += 1 break else: days_count = days + month_days break return [month_count, days_count] def how_many_days_in_this_month(self, y, m): if m in (1, 3, 5, 7, 8, 10, 12): days = 31 elif m in (4, 6, 9, 11): days = 30 else: if (y % 4 == 0 and y % 100 != 0) or (y % 400 == 0): days = 29 else: days = 28 return days def is_system_ok(self): is_pass = True cpu_list = psutil.cpu_percent(interval=1, percpu=True) memory_percent = psutil.virtual_memory().percent if cpu_list and memory_percent: is_cpu_ok = True if min(cpu_list) > self.CPU_THRESHOLD: is_cpu_ok = False is_memo_ok = True if memory_percent > self.MEMORY_THRESHOLD: is_memo_ok = False if not is_cpu_ok or not is_memo_ok: is_pass = False return is_pass
class MpdProtocol(LineReceiver): """ Twisted protocol to control remote mpd server """ def __init__(self): """ doc """ self.log = Logger() self.delimiter = "\n" self.deferreds = [] self.buff = {} self.idle = False self.list_index = 0 def connectionLost(self, reason): self.log.error("connection lost : {reason}", reason=reason) self._event({"changed": "disconnected"}) self.idle = False try: d = self.deferreds.pop(0) except: pass else: d.errback(reason) def connectionMade(self): self.log.debug("connected") def addCallback(self, d): self.deferreds.append(d) def noidle(self): d = defer.Deferred() d.addCallback(lambda ignored: ignored) self.deferreds.insert(0, d) self.sendLine("noidle") self.idle = False # print('noidle') def set_idle(self): self.sendLine("idle") self.idle = True # print('idle') def lineReceived(self, line): # print(line) if line.startswith("OK MPD"): self._event({"changed": "connected"}) elif line.startswith("OK"): # print('deferred length: %d' % len(self.deferreds)) self.list_index = 1 try: d = self.deferreds.pop(0) except: self.set_idle() self._event(self.buff) self.buff = {} return else: d.callback(self.buff) self.buff = {} elif line.startswith("ACK"): # print('deferred length: %d' % len(self.deferreds)) try: d = self.deferreds.pop(0) except: self.set_idle() self._event({"Error": line.split("}")[1]}) self.buff = {} return else: d.errback(Exception(line.split("}")[1])) self.buff = {} else: if len(line) > 0: k = line.split(":")[0] if isinstance(self.buff, list): if k in self.buff[self.list_index]: self.list_index += 1 self.buff.append({}) self.buff[self.list_index].update({k: " ".join(line.split()[1:])}) else: if k in self.buff: self.buff = [self.buff] self.list_index = 1 self.buff.append({k: " ".join(line.split()[1:])}) # if str(self.list_index) + k in self.buff: # self.list_index += 1 # self.buff.update( # {str(self.list_index) + line.split(':')[0]: # ' '.join(line.split()[1:])}) else: self.buff.update({k: " ".join(line.split()[1:])}) return if len(self.deferreds) == 0: self.set_idle()
class RainContent(GenericContent): def __init__(self, endpoint, factory): self.log = Logger(self.__class__.__name__) super().__init__(endpoint, factory) self.task = None def onBrokerConnected(self): self.task = task.LoopingCall(self.createForecast) self.task.start(self.config['RAIN_UPDATE_FREQ'], now=True) def _logFailure(self, failure): self.log.error("reported failure: {message}", message=failure.getErrorMessage()) return failure def _logSuccess(self, success, url): self.log.info("Success requesting {url}", url=url) return success def createForecast(self): url = self.config["RAIN_DATA_SOURCE"] self.log.debug("Grabbing rain forecast URL '%s'" % url) d = treq.get(url, timeout=5) d.addCallbacks(self.grab_http_response, self._logFailure) d.addCallbacks(self.parse_forecast_results, self._logFailure) d.addCallbacks(self.create_forcast, self._logFailure) d.addCallbacks(self.publish_forcast, self._logFailure) d.addCallback(self._logSuccess, url) def create_forcast(self, data): if data[0][0] == 0: # It's currently dry for rain, t in data: if rain > 0: return "Rain at %s" % t return None else: # It's raining. for rain, t in data: if rain == 0: return "Rain stop %s" % t return "Rain Rain Rain" def grab_http_response(self, response): if response.code != 200: raise RuntimeError("Status is not 200 but '%s'" % response.code) return readBody(response) def parse_forecast_results(self, content): raw_str = content.decode() raw_arr = [] for raw in raw_str.split("\r\n"): if not raw: continue rain_value, hour = raw.split("|") raw_arr.append([int(rain_value, 10), hour]) if len(raw_arr) == 0: raise RuntimeWarning( "API results where not in the expected format. They were: {}". format(raw_str)) return raw_arr def publish_forcast(self, forcast_string: str): def _logAll(*args): self.log.debug("all publishing complete args={args!r}", args=args) if forcast_string is None: # There's no Rain information to show. return msg = TextSingleLineLayout() msg.text = forcast_string msg.duration = self.config["RAIN_DISPLAY_DURATION"] msg.program = 'rain' msg.font_size = 15 d = self.publish(topic=LEDSLIE_TOPIC_TYPESETTER_1LINE, message=msg, qos=1) d.addCallbacks(_logAll, self._logFailure) return d
class XmppService(Service): timeout = True active_controllers = [] def __init__(self, device, user='******', secret='password', userlist=[], web_server=None): self.log = Logger() self.description = None self.reactor = reactor self.user = user self.services = {} self.nodes = [] self.registrations = [] self.active_controllers = [] self.webserver = web_server self.resource = web_server.resource device.location = user def _map_context(ctx): ctx.udc = UserDefinedContext(device.player) self._jid = _jid = ''.join( (user, '/', device.deviceType, ':uuid:', device.uuid)) self.device = device device.parent = self for service in device.services: # if appreg.get_application(service.tns, service.name): # name = service.name + '_' # else: # name = service.name # soap_service = type( # service.name, (ServiceBase,), service.soap_functions) # soap_service.tns = service.tns # app = Application( # [soap_service], # tns=soap_service.tns, # in_protocol=Soap11(xml_declaration=False), # out_protocol=Soap11(xml_declaration=False), # name=name) # app.event_manager.add_listener('method_call', _map_context) self.services.update({ str(service.serviceId): { 'app': TwistedXMPPApp(service.app), 'svc': service } }) print('name: %s, methods: %s' % (device.name, service.app.interface.service_method_map)) for var in service.stateVariables.values(): if var.sendEvents: self.nodes.append((var, service.serviceType, service)) self.users = {user: False} for user in userlist: self.users.update({user: False}) self.jid = jid = JID(_jid) self.factory = f = client.XMPPClientFactory(jid, secret) f.addBootstrap(xmlstream.STREAM_CONNECTED_EVENT, self.connected) f.addBootstrap(xmlstream.STREAM_END_EVENT, self.disconnected) f.addBootstrap(xmlstream.STREAM_AUTHD_EVENT, self.authenticated) f.addBootstrap(xmlstream.INIT_FAILED_EVENT, self.init_failed) # self.get_device_info(device) self.finished = defer.Deferred() # def get_device_info(self, device): # if len(device.icons) > 0: # icon = device.icons[0] # buf = read(open()) def startService(self): self.connector = SRVConnector(self.reactor, 'xmpp-client', self.jid.host, self.factory, defaultPort=5222) self.connector.connect() def stopService(self): for node in self.registrations: self.delete_ps_node(node) self.xmlstream.sendFooter() Service.stopService(self) def connected(self, xs): # print 'Connected.' # log.debug('!!!!!!!!!!!!!!!!!!!{app}', app=appreg._applications) self.xmlstream = xs # Log all traffic # xs.rawDataInFn = self.rawDataIn # xs.rawDataOutFn = self.rawDataOut def disconnected(self, xs): print 'Disconnected.' # self.finished.callback(None) def authenticated(self, xs): # print "Authenticated." xs.addObserver('/presence', self.on_presence) xs.addObserver('/iq', self.on_iq) presence = domish.Element((None, 'presence')) uc = domish.Element( ('urn:schemas-upnp-org:cloud-1-0', 'ConfigIdCloud')) uc['hash'] = 'uda' uc.addContent('1.0') # print(uc.toXml()) presence.addChild(uc) # print(presence.toXml()) xs.send(presence) disco = IQ(xs, 'get') disco.addElement(('http://jabber.org/protocol/disco#info', 'query')) disco.addCallback(self.check_server) disco.send(to='pubsub.xmpp.bertrandverdu.me') self.check_ps_nodes() self.reactor.callLater(30, self.ping) def ping(self): def pong(res): if res['type'] == 'result': # log.debug('pong !') self.timeout = False self.reactor.callLater(30, self.ping) else: self.log.error('ping error: %s' % res.toXml()) self.timeout = False # self.startService() def check_timeout(): if self.timeout: self.log.error('ping timeout !') # self.connector.connect() iq = IQ(self.xmlstream, 'get') iq.addElement('ping', 'urn:xmpp:ping') self.reactor.callLater(10, check_timeout) iq.addCallback(pong) self.timeout = True iq.send(to=self.jid.host) def rawDataIn(self, buf): print("Device RECV: %s" % unicode(buf, 'utf-8').encode('ascii', 'replace')) def rawDataOut(self, buf): print("Device SEND: %s" % unicode(buf, 'utf-8').encode('ascii', 'replace')) def check_server(self, result): def display_info(res): return e = fromstring(res.toXml()) dump(e) self.log.debug("server checked") display_info(result) # e = fromstring(result.toXml()) # dump(e) # iq = IQ(self.xmlstream, 'get') # ps = domish.Element( # ('http://jabber.org/protocol/pubsub#owner', 'pubsub')) # ps.addElement('default') # iq.addChild(ps) # iq.addCallback(display_info) # iq.send(to='pubsub.' + self.jid.host) # print(result.toXml()) def dump_description(self, dest): def sent(res): self.log.debug('description sent') pass self.log.debug('send description') # print(dest['id']) if self.description: self.log.debug('cached description') iq = IQ(self.xmlstream, 'result') iq.addRawXml(self.description) iq['id'] = dest['id'] else: self.log.debug('generate description') iq = IQ(self.xmlstream, 'result') query = domish.Element(('urn:schemas-upnp-org:cloud-1-0', 'query')) query['type'] = 'described' query['name'] = self.device.uuid d = tostring(self.device.dump(), encoding='unicode') query.addRawXml(d) # print(query.toXml()) for service in self.device.services: s = tostring(service.dump(), encoding='unicode') # print(s) query.addRawXml(s) # print(query.toXml()) self.description = query.toXml() iq.addChild(query) iq['id'] = dest['id'] # self.description = iq # print(iq.toXml()) iq.addCallback(sent) iq.send(to=dest['from']) def check_ps_nodes(self): def got_response(node, response): if response['type'] in ('error', 'cancel'): self.create_ps_node(node) elif response['type'] == 'result': node_name = '/'.join((self._jid, node[1])) if node_name in self.registrations: return self.registrations.append(node_name) self.log.debug('node {name} registered', name=node_name) event = XmppEvent(node_name, self, 'pubsub.' + self.jid.host) node[2].subscribe(event, 100) self.reactor.callLater(95, self.renew_subscription, *(node_name, node)) else: self.log.error('unknown response from server: %s' % response.toXml()) self.log.debug('check nodes: {nodes}', nodes=str(self.nodes)) IQ_ = IQ # Basic optimisation... element = domish.Element for node in self.nodes: iq = IQ_(self.xmlstream, 'get') query = element(('http://jabber.org/protocol/disco#info', 'query')) query['node'] = '/'.join((self._jid, node[1], node[0].name)) iq.addChild(query) iq.addCallback(got_response, node) iq.send(to='pubsub.' + self.jid.host) def create_ps_node(self, node): def registered(node, iq): if iq['type'] == 'result': node_name = '/'.join((self._jid, node[1])) if node_name in self.registrations: return event = XmppEvent(node_name, self, 'pubsub.' + self.jid.host) node[2].subscribe(event, 100) self.reactor.callLater(95, self.renew_subscription, *(node_name, node)) self.registrations.append(node_name) self.log.debug('node {node} registered', node=node_name) else: self.log.error('node creation {name} failed:{iq}', name=node, iq=iq.toXml()) iq = IQ(self.xmlstream, 'set') ps = domish.Element(('http://jabber.org/protocol/pubsub', 'pubsub')) create = domish.Element((None, 'create')) create['node'] = '/'.join((self._jid, node[1], node[0].name)) ps.addChild(create) configure = domish.Element((None, 'configure')) x = domish.Element(('jabber:x:data', 'x')) x['type'] = 'submit' field = domish.Element((None, 'field')) field['var'] = 'FORM_TYPE' field['type'] = 'hidden' field.addElement( 'value', content='http://jabber.org/protocol/pubsub#node_config') x.addChild(field) access = domish.Element((None, 'field')) access['var'] = 'pubsub#access_model' access.addElement('value', content='roster') x.addChild(access) # expire = domish.Element((None, 'field')) # expire['var'] = 'pubsub#item_expire' # expire.addElement('value', content='60') # x.addChild(expire) last = domish.Element((None, 'field')) last['var'] = 'pubsub#send_last_published_item' last.addElement('value', content='on_sub_and_presence') x.addChild(last) configure.addChild(x) ps.addChild(configure) iq.addChild(ps) iq.addCallback(registered, node) iq.send(to='pubsub.' + self.jid.host) def delete_ps_node(self, node): def deleted(res): if res['type'] == 'error': self.log.error('node deletion failed: %s' % res.toXml()) iq = IQ(self.xmlstream, 'set') ps = domish.Element( ('http://jabber.org/protocol/pubsub#owner', 'pubsub')) delete = domish.Element((None, 'delete')) delete['node'] = node ps.addChild(delete) iq.addChild(ps) iq.send(to='pubsub.' + self.jid.host) def renew_subscription(self, name, node): self.log.debug('renew %s : %s' % (name, (name in self.registrations))) if name in self.registrations: self.log.debug('renew: %s' % name) event = XmppEvent(name, self, 'pubsub.' + self.jid.host) node[2].subscribe(event, 100) self.reactor.callLater(95, self.renew_subscription, *(name, node)) def on_iq(self, iq): # print('received iq: %s' % iq.toXml().encode('utf-8')) user, host, res = parse(iq['from']) del (res) if not user: return jid = '@'.join((user, host)) self.log.debug('received request of type {typ} from {user}', typ=iq['type'], user=jid) if jid not in self.users and jid != self.user: self.log.info('rejected User: %s' % jid) return if iq['type'] == 'get': for child in iq.children: if child.name == 'query': if child['type'] == 'description': self.log.debug('description requested') self.dump_description(iq) elif iq['type'] == 'set': if iq.children[0].name == 'Envelope': self.log.debug('received rpc') root = iq.children[0] # print(root.toXml()) for child in root.children: if child.name == 'Header': res = self.services[ child.children[0]['serviceId']]['app'].handle_rpc( root.toXml(), child.children[0]['serviceId']) elif child.name == 'Body': decomposed = child.children[0].uri.split(':') guessed_id = ':'.join( (decomposed[0], decomposed[1], decomposed[2] + 'Id', decomposed[3])) res = self.services[str(guessed_id)]['app'].handle_rpc( root.toXml(), str(guessed_id)) else: self.log.warn('bad iq request: %s' % child.name) continue res.addCallback(self.respond_rpc, iq['from'], iq['id']) def respond_rpc(self, resp, to, queryID): # print('send: %s' % resp) # self.log.debug('respond rpc: %s' % resp[0][39:]) res = IQ(self.xmlstream, 'result') res['id'] = queryID if resp: for item in resp: res.addRawXml(item[39:].decode('utf-8')) # Skip the xml header res.send(to=to) def on_presence(self, presence): self.log.debug('received presence: %s' % presence.toXml().encode('utf-8')) if presence.hasAttribute('from'): user, host, res = parse(presence['from']) if presence['from'] in self.active_controllers: if presence.hasAttribute('type'): if presence['type'] == 'unavailable': self.active_controllers.remove(presence['from']) self.log.info('User {_from} disconnected', _from=presence['from']) return elif 'ControlPoint' in res: if presence.hasAttribute('type'): if presence['type'] == 'unavailable': return self.log.info('control point %s added' % presence['from']) if len(self.active_controllers) == 0: self.check_ps_nodes() self.active_controllers.append(presence['from']) del (res) jid = '@'.join((user, host)) if presence.hasAttribute('type'): if presence['type'] == 'subscribe': if jid in self.users: self.log.info('received subscription from %s' % jid) if self.users[jid] is False: iq = IQ(self.xmlstream, 'set') query = domish.Element( ('jabber:iq:roster', 'query')) item = domish.Element((None, 'item')) item['jid'] = jid item['name'] = jid item.addElement('group', content='UPnPCloud') query.addChild(item) iq.addChild(query) iq.addCallback(self.subscribed, jid) iq.send() else: self.log.error('subscription for user %s failed: %s' % (jid, 'Not in user list')) pres = domish.Element((None, 'presence')) pres['type'] = 'unsubscribed' pres['to'] = presence['from'] self.xmlstream.send(pres) def subscribed(self, jid, result): if result['type'] == 'result': self.log.info('user %s successfully suscribed' % jid) self.users.update({jid: True}) pres = domish.Element((None, 'presence')) pres['type'] = 'subscribed' pres['to'] = jid self.xmlstream.send(pres) else: self.log.error('subscription for user %s failed: %s' % (jid, result.toXml())) pres = domish.Element((None, 'presence')) pres['type'] = 'unsubscribed' pres['to'] = jid self.xmlstream.send(pres) def init_failed(self, failure): print "Initialization failed." print failure def register_art_url(self, url, cloud=False): if cloud: return None newurl = hashlib.md5(url).hexdigest() + url[-4:] self.resource.putChild(newurl, static.File(url)) return self.webserver.weburl % get_default_v4_address() + '/' + newurl
class RemoteClientBase( # pylint: disable=too-many-instance-attributes pb.Referenceable): PING_DELAY = 60 PING_MAX_FAILURES = 3 DEBUG = False def __init__(self): self.log_level = LogLevel.warn self.log = Logger(namespace="remote", observer=self._log_observer) self.id = app.get_host_id() self.name = app.get_host_name() self.join_options = {"corever": __version__} self.perspective = None self.agentpool = None self._ping_id = 0 self._ping_caller = None self._ping_counter = 0 self._reactor_stopped = False self._exit_code = 0 @provider(ILogObserver) def _log_observer(self, event): if not self.DEBUG and (event["log_namespace"] != self.log.namespace or self.log_level > event["log_level"]): return msg = formatEvent(event) click.echo("%s [%s] %s" % ( datetime.fromtimestamp( event["log_time"]).strftime("%Y-%m-%d %H:%M:%S"), event["log_level"].name, msg, )) def connect(self): self.log.info("Name: {name}", name=self.name) self.log.info("Connecting to PlatformIO Remote Development Cloud") # pylint: disable=protected-access proto, options = endpoints._parse(__pioremote_endpoint__) proto = proto[0] factory = RemoteClientFactory() factory.remote_client = self factory.sslContextFactory = None if proto == "ssl": factory.sslContextFactory = SSLContextFactory(options["host"]) reactor.connectSSL( options["host"], int(options["port"]), factory, factory.sslContextFactory, ) elif proto == "tcp": reactor.connectTCP(options["host"], int(options["port"]), factory) else: raise exception.PlatformioException( "Unknown PIO Remote Cloud protocol") reactor.run() if self._exit_code != 0: raise exception.ReturnErrorCode(self._exit_code) def cb_client_authorization_failed(self, err): msg = "Bad account credentials" if err.check(pb.Error): msg = err.getErrorMessage() self.log.error(msg) self.disconnect(exit_code=1) def cb_client_authorization_made(self, perspective): self.log.info("Successfully authorized") self.perspective = perspective d = perspective.callRemote("join", self.id, self.name, self.join_options) d.addCallback(self._cb_client_join_made) d.addErrback(self.cb_global_error) def _cb_client_join_made(self, result): code = result[0] if code == 1: self.agentpool = result[1] self.agent_pool_ready() self.restart_ping() elif code == 2: self.remote_service(*result[1:]) def remote_service(self, command, options): if command == "disconnect": self.log.error("PIO Remote Cloud disconnected: {msg}", msg=options.get("message")) self.disconnect() def restart_ping(self, reset_counter=True): # stop previous ping callers self.stop_ping(reset_counter) self._ping_caller = reactor.callLater(self.PING_DELAY, self._do_ping) def _do_ping(self): self._ping_counter += 1 self._ping_id = int(time()) d = self.perspective.callRemote("service", "ping", {"id": self._ping_id}) d.addCallback(self._cb_pong) d.addErrback(self._cb_pong) def stop_ping(self, reset_counter=True): if reset_counter: self._ping_counter = 0 if not self._ping_caller or not self._ping_caller.active(): return self._ping_caller.cancel() self._ping_caller = None def _cb_pong(self, result): if not isinstance(result, failure.Failure) and self._ping_id == result: self.restart_ping() return if self._ping_counter >= self.PING_MAX_FAILURES: self.stop_ping() self.perspective.broker.transport.loseConnection() else: self.restart_ping(reset_counter=False) def agent_pool_ready(self): raise NotImplementedError def disconnect(self, exit_code=None): self.stop_ping() if exit_code is not None: self._exit_code = exit_code if reactor.running and not self._reactor_stopped: self._reactor_stopped = True reactor.stop() def cb_disconnected(self, _): self.stop_ping() self.perspective = None self.agentpool = None def cb_global_error(self, err): if err.check(pb.PBConnectionLost, defer.CancelledError): return msg = err.getErrorMessage() if err.check(pb.DeadReferenceError): msg = "Remote Client has been terminated" elif "PioAgentNotStartedError" in str(err.type): msg = ("Could not find active agents. Please start it before on " "a remote machine using `pio remote agent start` command.\n" "See http://docs.platformio.org/page/plus/pio-remote.html") else: maintenance.on_platformio_exception(Exception(err.type)) click.secho(msg, fg="red", err=True) self.disconnect(exit_code=1)
class Controller(service.MultiService): targets = {} services = [] binary_light_list = [] hvac_list = [] media_player_list = [] messager = None server_list = [] shutter_list = [] camera_list = [] multiscreen_list = [] dimmable_light_list = [] ambi_light_list = [] event_catcher = None cloud_event_catcher = None subscriptions = {} subscriptions_cloud = {} searchables = {} ready_to_close = False current_device = None cloud = False lan = False agent = None def __init__( self, parent=None, searchables=None, xmldir=None, network='lan', cloud_user=None, cloud_servers=[], logger=None, uid=None, messager=None): self.connected = False self.messager = messager self.app_paused = False self.fail_count = 0 if not logger: self.log = Logger() else: self.log = logger self.log.debug('UPnP controller starts') self.xmldir = xmldir self.devices = {} self._services = {} self.parent = parent # self.amp = ControllerAmp(self) if uid: self.uuid = uid else: self.uuid = str( uuid.uuid5( uuid.NAMESPACE_DNS, socket.gethostname() + 'onDemand_Controller')) if searchables: for typ in searchables: self.searchables.update({typ[0]: typ[1]}) # print(self.searchables) else: self.searchables = {'upnp:rootdevice': self.log.debug} if network in ('lan', 'both'): self.log.debug('UPnP classic enabled') self.lan = True self.listener = ssdp.SSDP_Listener(self) self.mcast = internet.MulticastServer( # @UndefinedVariable SSDP_PORT, self.listener, listenMultiple=True, interface=SSDP_ADDR_V4) self.mcast.setServiceParent(self) self.ssdp_cli = ssdp.SSDP_Client( self, get_default_v4_address(), device=False) self.ucast = internet.UDPServer( # @UndefinedVariable 0, self.ssdp_cli, self.ssdp_cli.interface) self.ucast.setServiceParent(self) # self.agent = Agent(reactor) if network in ('cloud', 'both'): if cloud_user: self.log.debug('UPnP Cloud enabled') self.cloud = True self._jid, secret = cloud_user self.users = {self._jid: {'state': True}} for user in cloud_servers: self.users.update({user: {'state': False}}) self.hosts = {} self.resourcepart = ''.join(( 'urn:schemas-upnp-org:cloud-1-0:ControlPoint:1:uuid:', self.uuid)) full_jid = ''.join( (self._jid, '/', self.resourcepart)) self.jid = jid = JID(full_jid) self.reactor = reactor f = client.XMPPClientFactory(jid, secret) f.addBootstrap( xmlstream.STREAM_CONNECTED_EVENT, self.cloud_connected) f.addBootstrap( xmlstream.STREAM_END_EVENT, self.cloud_disconnected) f.addBootstrap( xmlstream.STREAM_AUTHD_EVENT, self.authenticated) f.addBootstrap( xmlstream.INIT_FAILED_EVENT, self.cloud_failed) self.connector = endpoints.HostnameEndpoint( reactor, jid.host, 5222) self.factory = f # factory = Factory() # factory.protocol = ControllerAmp(self) # amp_service = internet.TCPServer( # @UndefinedVariable # 4343, factory) # amp_service.setServiceParent(self) # self.connector = SRVConnector( # reactor, 'xmpp-client', jid.host, f, defaultPort=5222) # log.startLogging(sys.stdout) def startService(self): ''' ''' service.MultiService.startService(self) if self.cloud: self.connector.connect(self.factory) self.log.debug('Cloud Service started') if self.lan: t = task.LoopingCall(self.search_devices) t.start(15) self.log.debug('SSDP Service started') def resume(self): self.app_paused = False if not self.connected: if self.cloud: self.connector.connect(self.factory) self.log.debug('Cloud Service started') if self.lan: t = task.LoopingCall(self.search_devices) t.start(15) self.log.debug('SSDP Service started') def stopService(self): self.log.debug('Stopping controller service...') self.clean() # d.addCallback(lambda ignored: service.MultiService.stopService(self)) service.MultiService.stopService(self) # reactor.callLater(10, reactor.stop) # @UndefinedVariable def cloud_disconnected(self, reason): if not reason: reason = 'Unknown' self.log.warn('Cloud Server disconnected: %s' % reason) self.connected = False if not self.app_paused and self.fail_count < 10: self.fail_count += 1 self.resume() def cloud_failed(self, failure): self.log.error('Cloud Login failed: %s' % str(failure)) # self.xmlstream.sendFooter() def clean(self): return reactor.callInThread( # @UndefinedVariable threads.blockingCallFromThread, *(reactor, self.cleanfunc)) def cleanfunc(self): def cleaned(res): self.log.debug('cleaned') if self.cloud: self.xmlstream.sendFooter() dl = [] if self.lan: for name in self.subscriptions.keys(): dl.append(self.unsubscribe(name)) if self.cloud: for name in self.subscriptions_cloud.keys(): dl.append(self.unsubscribe(name)) d = defer.DeferredList(dl) d.addCallback(cleaned) return d def cloud_connected(self, xs): self.log.debug('Cloud Connected') self.fail_count = 0 self.connected = True self._services = {} self.subscriptions = {} self.xmlstream = xs # xs.rawDataInFn = self.rawDataIn def authenticated(self, xs): self.log.debug('Cloud Authenticated') presence = domish.Element((None, 'presence')) xs.send(presence) xs.addObserver('/presence', self.on_presence) xs.addObserver('/iq', self.on_iq) xs.addObserver('/message', self.on_event) disco = IQ(xs, 'get') disco.addElement(('http://jabber.org/protocol/disco#items', 'query')) disco.addCallback(self.cloud_discovered) disco.send() # self.reactor.callLater(120, xs.sendFooter) self.reactor.callLater(5, self.check_users) def check_users(self): for user, value in self.users.items(): if value['state'] is False: iq = IQ(self.xmlstream, 'set') query = domish.Element(('jabber:iq:roster', 'query')) item = domish.Element((None, 'item')) item['name'] = user item['jid'] = user item.addElement('group', content='hosts') query.addChild(item) iq.addChild(query) iq.addCallback(self.cloud_subscribe, user) # print('send IQ: %s' % (iq.toXml().encode('utf-8'))) iq.send() def cloud_subscribe(self, jid, result): self.log.debug('Subscribe callback from %s' % jid) presence = domish.Element((None, 'presence')) presence['type'] = 'subscribe' presence['to'] = jid self.xmlstream.send(presence) def on_event(self, message): if not self.cloud_event_catcher: reactor.callLater(1, self.on_event, message) # @UndefinedVariable return if message.name == 'iq': if message['type'] == 'result': try: last = '' for child in message.children[0].children[0].children: last = child.children[0] except KeyError: return # print(message.toXml()) # print(last.toXml()) self.cloud_event_catcher.receive(last.toXml().encode('utf-8')) elif message.children[0].name == 'event': evt = message.children[0] items = evt.children[0] node_name = str(items['node']) if node_name in self.subscriptions_cloud: for item in items.children: propertyset = item.children[0] self.cloud_event_catcher.receive( (node_name, propertyset.toXml().encode('utf-8'),)) def rawDataIn(self, buf): print( "Device RECV: %s" % unicode(buf, 'utf-8').encode('ascii', 'replace')) def on_presence(self, resp): self.log.debug('got presence: %s' % resp.toXml().encode('utf-8')) # print('from :%s' % resp['from']) user, host, res = parse(resp['from']) jid = '@'.join((user, host)) if resp.hasAttribute('type'): if resp['type'] == 'subscribed': if jid in self.users: self.users[jid].update({'state': True}) if 'services' in self.users[jid]: self.users[jid]['services'].append(res) else: self.users[jid].update({'services': [res]}) presence = domish.Element((None, 'presence')) presence['type'] = 'subscribe' presence['to'] = resp['from'] self.xmlstream.send(presence) else: presence = domish.Element((None, 'presence')) presence['type'] = 'denying' presence['to'] = resp['from'] self.xmlstream.send(presence) elif resp['type'] == 'unsubscribed': if jid in self.users: self.log.warn('subscription failed: %s' % resp['from']) return for child in resp.elements(): if child.name == 'ConfigIdCloud': self.log.debug('Found UPnP Cloud device : %s type is: %s' % ( jid, res)) info = IQ(self.xmlstream, 'get') # info['to'] = resp['from'] query = domish.Element( ('urn:schemas-upnp-org:cloud-1-0', 'query')) query['type'] = 'description' query['name'] = ':'.join(res.split(':')[-2:]) info.addChild(query) info.addCallback(self.on_description, res) # info.send() info.send(to=resp['from']) def on_description(self, resource, iq): location = iq['from'] clbk = self.searchables[ self.searchables.keys()[0]] if iq['type'] == 'result': if iq.children[0].name == 'query'\ and iq.children[0]['type'] == 'described': self.update_devices( resource, location, clbk, xml=iq.children[0].children[0].toXml()) def cloud_discovered(self, iq): self.log.debug('Discovered item: %s' % iq.toXml().encode('utf-8')) if iq['type'] == 'result': for child in iq.children: if child.name == 'query': for grandchild in child.children: if grandchild['jid'].encode('utf-8') == self.full_jid: continue if grandchild['name'].encode('utf-8')\ in self.hosts: self.hosts[ grandchild['name'].encode('utf-8')].append( grandchild['jid'].encode('utf-8')) else: self.hosts.update( {grandchild['name'].encode('utf-8'): [grandchild['jid'].encode('utf-8')]}) # print(self.hosts) def on_iq(self, iq): pass # print('got iq: %s' % iq.toXml()) # try: # print('from :%s' % iq['from']) # except KeyError: # print('From I don\'t know: %s' % iq.toXml()) # print('type: %s' % iq['type']) def search_devices(self): for search in self.searchables: self.ssdp_cli.send_MSEARCH(search, uuid=self.uuid) def update_hosts(self, host, unicast=False): if 'location' in host: if 'usn' in host: if host['usn'] in self.devices: return device = host['usn'].split('::') if len(device) > 1: uid = device[0].split(':')[1] if uid in self.devices: return typ = device[1] if typ in self.searchables: self.update_devices( uid, host['location'], self.searchables[typ]) # self.devices.append(uid) def send_message(self, message_type, name, id_, value): if self.messager: if isinstance(value, dict): self.messager.callRemote(message_type, name=name, id_=id_, value=json.dumps(value)) # for v in value.iteritems(): # if not v or isinstance(v, dict): # print('zap') # continue # print(v) # self.messager.callRemote(message_type, # name=name, # id_=id_, # value=':'.join((k, v))) else: self.messager.callRemote(message_type, name=name, id_=id_, value=value) def update_devices(self, uid, location, callback_fct, xml=None): def device_parsed(dic): self.devices.update(dic) if callable(callback_fct): callback_fct(dic) else: self.send_message(Event, callback_fct, uid, dic[uid]) if self.messager: self.messager.parent.notify( 'New Device detected:', dic[uid]['name']) uid = bytes(uid) self.log.debug('new device %s: %s' % (uid, location)) if '@' in location: if xml: device_parsed(self.parse_host(xml, location, uid)) return else: if not self.agent: self.agent = Agent(reactor) d = self.agent.request('GET', location) d.addCallback(readBody) d.addCallback(self.parse_host, *(location, uid)) d.addCallback(device_parsed) def parse_host(self, xml, location, uid): typ = 'upnp' loc = None if '@' in location: url_prefix = ''.join(('xmpp://', location)) net = 'cloud' else: url_prefix = urlparse(location).netloc net = 'lan' try: root = et.fromstring(xml) except: self.log.error('bad xml: %s' % xml) return {} host = {} icon = None for children in root: if children.tag.split('}')[-1] == 'device': for att in children: if att.tag.split('}')[-1] == 'friendlyName': fname = att.text if att.tag.split('}')[-1] == 'deviceType': devtype = att.text if 'Source' in att.text: typ = 'oh' if att.tag.split('}')[-1] == 'iconList': for ico in att: # log.debug(ico) for info in ico: if info.tag.split('}')[-1] == 'width': if int(info.text) <= 96: if ico[4].text.startswith('/'): icon = 'http://'\ + url_prefix\ + ico[4].text else: icon = ico[4].text if att.tag.split('}')[-1] == 'serviceList': svc = {} for serv in att: d = {} for info in serv: if 'URL' in info.tag.split('}')[-1]: if net == 'lan': d.update({info.tag.split('}')[-1]: 'http://' + url_prefix + info.text}) else: d.update( {info.tag.split('}')[-1]: url_prefix + info.text}) else: d.update( {info.tag.split('}')[-1]: info.text}) svc.update({d['serviceType']: d}) if att.tag.split('}')[-1] == 'X_location': loc = att.text host.update( {uid: { 'name': fname, 'devtype': devtype, 'icon': icon, 'services': svc, 'type': typ, 'network': net, 'location': location, 'loc': loc}}) # log.debug(host) return host def subscribe(self, *args, **kwargs): if args[0][args[0].keys()[0]]['network'] == 'lan': return self.subscribe_classic(*args, **kwargs) else: return self.subscribe_cloud(*args, **kwargs) def subscribe_classic( self, device, svc, var, callback_fct=None, callback_args=()): if not callback_fct: callback_fct = self.log.debug name = device.keys()[0] dev = device[name] def subscribe_failed(err, name): self.parent.remove_device(name.split('_')[0]) def subscribed(req, raddr, host, name): try: uuid = req.headers.getRawHeaders('sid')[0] print('subscription uuid = %s' % uuid) if name in self.subscriptions: if host in self.subscriptions[name]: self.subscriptions[name][host].update({uuid: raddr}) else: self.subscriptions[name].update({host: {uuid: raddr}}) else: self.subscriptions.update({name: {host: {uuid: raddr}}}) reactor.callLater( # @UndefinedVariable 20, self.renew_subscription, uuid) return name except TypeError: return subscribe_failed(None, name) if self.event_catcher is None: self.event_catcher = EventServer() self.event_catcher.setServiceParent(self) subscription_id = '_'.join((name, svc.split(':')[-2])) childpath = '_'.join((subscription_id, 'event',)) # log.error(childpath) if childpath in self.event_catcher.catcher.childs: self.event_catcher.catcher.childs[childpath].update( {var: (callback_fct, callback_args,)}) else: self.event_catcher.catcher.childs.update( {childpath: {var: (callback_fct, callback_args,)}}) # log.error(self.event_catcher.catcher.childs) if subscription_id in self.subscriptions: for k, value in self.event_catcher.catcher.unfiltered.items(): if k == var: if value == 'False': value = False elif value == 'True': value = True if isinstance(callback_args, str)\ or isinstance(callback_args, bool): callback_fct(value, callback_args) else: callback_fct(value, *callback_args) del self.event_catcher.catcher.unfiltered[k] return defer.succeed(None) else: self.subscriptions.update({subscription_id: {}}) clbk = '<' + 'http://' + get_default_v4_address() + ':' +\ str(self.event_catcher.getPort()) + '/' + childpath + '>' # print(clbk) headers = {'HOST': [get_default_v4_address() + ':' + str(self.event_catcher.getPort())], 'CALLBACK': [clbk], 'NT': ['upnp:event'], 'TIMEOUT': ['Second-25']} if svc in dev['services']: self.log.error(svc) addr = dev['services'][svc]['eventSubURL'] self.log.error(addr) d = self.agent.request( 'SUBSCRIBE', addr, Headers(headers)) d.addCallbacks( subscribed, subscribe_failed, callbackArgs=(addr, headers['HOST'][0], subscription_id), errbackArgs=(subscription_id,)) return d # log.error(dev['services']) return defer.fail(Exception('Service unknow')) def renew_subscription(self, sid): def renewed(res): # print('subscription %s successfully renewed' % sid) reactor.callLater( # @UndefinedVariable 20, self.renew_subscription, sid) def failed(res): for name in self.subscriptions: for host in self.subscriptions[name]: if sid in self.subscriptions[name][host]: del self.subscriptions[name][host][sid] self.parent.remove_device(name.split('_')[0]) for name in self.subscriptions: for host in self.subscriptions[name]: if sid in self.subscriptions[name][host]: headers = {'HOST': [host], 'SID': [sid], 'TIMEOUT': ['Second-25']} d = self.agent.request( 'SUBSCRIBE', self.subscriptions[name][host][sid], Headers(headers)) d.addCallbacks(renewed, failed) return d def unsubscribe(self, name): print('unsuscribe: %s' % name) deferreds = [] if name in self.subscriptions: for host in self.subscriptions[name]: for sid in self.subscriptions[name][host]: deferreds.append(self.unsubscribe_host( sid, host, self.subscriptions[name][host][sid], name)) if name in self.subscriptions_cloud: return self.unsubscribe_cloud(name) if len(deferreds) > 0: # print(deferreds) d = defer.DeferredList(deferreds) else: d = defer.succeed('nothing to do') return d def unsubscribe_cloud(self, name): def unsubscribed(name, d, res): if res['type'] == 'result': # print('unsubscribed: %s' % name) del self.subscriptions_cloud[name] print('ok') d.callback(None) else: d.errback(Exception(res.toXml())) d = defer.Deferred() iq = IQ(self.xmlstream, 'set') ps = domish.Element(('http://jabber.org/protocol/pubsub', 'pubsub')) unsubscribe = domish.Element((None, 'unsubscribe')) unsubscribe['node'] = name unsubscribe['jid'] = self.jid.full() ps.addChild(unsubscribe) iq.addChild(ps) iq.addCallback(unsubscribed, name, d) iq.send(to='pubsub.' + self.jid.host) return d def unsubscribe_host(self, sid, host, addr, name=None): # log.debug( # 'unsubscribe uuid host addr: %s %s %s' % (sid, host, addr)) def unsubscribed(res): # print('subscription %s successfully cancelled' % sid) if name: if len(self.subscriptions[name][host]) == 1: del self.subscriptions[name] else: del self.subscriptions[name][host][sid] return res headers = {'HOST': [host], 'SID': [sid]} d = self.agent.request( 'UNSUBSCRIBE', addr, Headers(headers)) d.addCallback(unsubscribed) return d def subscribe_cloud( self, device, svc, var, callback_fct=None, callback_args=()): # print('suscribe to %s' % var) name = device.keys()[0] dev = device[name] if not callback_fct: callback_fct = self.log.debug d = defer.Deferred() def subscribe_failed(err, name): self.parent.remove_device(name.split('_')[0]) def subscribed(node_name, deferred, iq): if iq['type'] == 'result': self.subscriptions_cloud[str(node_name)] = True # print('%s suscribed !' % str(node_name)) # iq = IQ(self.xmlstream, 'get') # ps = domish.Element( # ('http://jabber.org/protocol/pubsub', 'pubsub')) # items = domish.Element((None, 'items')) # items['node'] = node_name # items['max_items'] = '1' # ps.addChild(items) # iq.addChild(ps) # iq.addCallback(self.on_event) # iq.send(to='pubsub.' + self.jid.host) # print(iq.toXml()) deferred.callback(str(node_name)) else: deferred.errback(Exception('subscription to %s failed: %s' % (node_name, iq.toXml()))) if svc in dev['services']: # print('service %s ok' % svc) # print('subscriptions :%s' % self.subscriptions_cloud) if not self.cloud_event_catcher: self.cloud_event_catcher = CloudEventCatcher( {}, {}, logger=self.log) subscription_name = '/'.join((dev['location'], svc, var)) # subscription_service = svc if subscription_name in self.cloud_event_catcher.callbacks: self.cloud_event_catcher.callbacks[subscription_name].update( {var: (callback_fct, callback_args,)}) else: self.cloud_event_catcher.callbacks.update( {subscription_name: {var: (callback_fct, callback_args,)}}) # if var in self.cloud_event_catcher.callbacks: # self.cloud_event_catcher.callbacks[var].update( # {var: (callback_fct, callback_args,)}) # else: # self.cloud_event_catcher.callbacks.update( # {var: {var: (callback_fct, callback_args,)}}) # log.error(self.event_catcher.catcher.childs) if subscription_name in self.subscriptions_cloud: if self.subscriptions_cloud[subscription_name]: # print('already subscribed: %s' % subscription_name) for k, value in\ self.cloud_event_catcher.unfiltered_dict.items(): # print('is %s == %s ?' % (k, var)) if k == var: if value == 'False': value = False elif value == 'True': value = True if isinstance(callback_args, str)\ or isinstance(callback_args, bool): callback_fct(value, callback_args) else: callback_fct(value, *callback_args) del self.cloud_event_catcher.unfiltered_dict[k] return defer.succeed(None) self.subscriptions_cloud.update({str(subscription_name): False}) # print(subscription_name) # print(subscription_service) iq = IQ(self.xmlstream, 'set') ps = domish.Element( ('http://jabber.org/protocol/pubsub', 'pubsub')) subscribe = domish.Element((None, 'subscribe')) subscribe['node'] = subscription_name subscribe['jid'] = self.jid.full() ps.addChild(subscribe) iq.addChild(ps) iq.addCallback(subscribed, subscription_name, d) iq.send(to='pubsub.' + self.jid.host) return d return defer.fail(Exception('Service unknow')) def get_client(self, device, service): if self.xmldir is not None: client = None else: import importlib module_name = service.split(':')[-2] app = getattr(importlib.import_module( 'upnpy_spyne.services.templates.' + module_name.lower()), module_name) if device['network'] == 'lan': client = Client( device['services'][service]['controlURL'], Application([app], app.tns, in_protocol=Soap11(), out_protocol=Soap11())) client.set_options( out_header={'Content-Type': ['text/xml;charset="utf-8"'], 'Soapaction': [app.tns]}) else: url = (self.xmlstream, device['location'],) client = Client( url, Application([app], app.tns, in_protocol=Soap11(xml_declaration=False), out_protocol=Soap11(xml_declaration=False)), cloud=True) # print('**********%s' % service) # print(device['services'][service]) return client def call(self, device, service, func, params=()): if isinstance(device, dict): devname = device.keys()[0] dev = device[devname] else: devname = device dev = self.devices[device] if devname not in self._services: client = self.get_client(dev, service) self._services.update({devname: {service: client.service}}) elif service not in self._services[devname]: client = self.get_client(dev, service) self._services[devname].update({service: client.service}) try: f = getattr( self._services[devname][service], func) except AttributeError: self.log.error( 'function %s not found for service %s' % (func, service)) return defer.fail(Exception( 'function %s not found for service %s' % (func, service))) try: if len(params) > 0: if isinstance(params, str): d = f(params) else: d = f(*params) else: d = f() except TypeError: # boolean has no len d = f(params) d.addErrback( lambda failure, fname: self.log.error( '%s call failed : %s' % (fname, failure.getErrorMessage())), func) return d
class MpdProtocol(LineReceiver): ''' Twisted protocol to control remote mpd server ''' def __init__(self): ''' doc ''' self.log = Logger() self.delimiter = "\n" self.deferreds = [] self.buff = {} self.idle = False self.list_index = 0 def connectionLost(self, reason): self.log.error('connection lost : {reason}', reason=reason) self._event({'changed': 'disconnected'}) self.idle = False try: d = self.deferreds.pop(0) except: pass else: d.errback(reason) def connectionMade(self): self.log.debug('connected') def addCallback(self, d): self.deferreds.append(d) def noidle(self): d = defer.Deferred() d.addCallback(lambda ignored: ignored) self.deferreds.insert(0, d) self.sendLine('noidle') self.idle = False # print('noidle') def set_idle(self): self.sendLine('idle') self.idle = True # print('idle') def lineReceived(self, line): # print(line) if line.startswith('OK MPD'): self._event({'changed': 'connected'}) elif line.startswith('OK'): # print('deferred length: %d' % len(self.deferreds)) self.list_index = 1 try: d = self.deferreds.pop(0) except: self.set_idle() self._event(self.buff) self.buff = {} return else: d.callback(self.buff) self.buff = {} elif line.startswith('ACK'): # print('deferred length: %d' % len(self.deferreds)) try: d = self.deferreds.pop(0) except: self.set_idle() self._event({'Error': line.split('}')[1]}) self.buff = {} return else: d.errback(Exception(line.split('}')[1])) self.buff = {} else: if len(line) > 0: k = line.split(':')[0] if isinstance(self.buff, list): if k in self.buff[self.list_index]: self.list_index += 1 self.buff.append({}) self.buff[self.list_index].update( {k: ' '.join(line.split()[1:])}) else: if k in self.buff: self.buff = [self.buff] self.list_index = 1 self.buff.append({k: ' '.join(line.split()[1:])}) # if str(self.list_index) + k in self.buff: # self.list_index += 1 # self.buff.update( # {str(self.list_index) + line.split(':')[0]: # ' '.join(line.split()[1:])}) else: self.buff.update({k: ' '.join(line.split()[1:])}) return if len(self.deferreds) == 0: self.set_idle()
class TimeSchedule: def __init__(self, lock, host='127.0.0.1', port='6800'): config = Config() self.db = glv.get_value(key='sqlite_db') self.user_name = config.get('auth_username', '') self.user_password = config.get('auth_password', '') self.start_time = time.strftime("%Y %m %d %H %M %S", time.localtime()) self.server_port = 'http://{}:{}/'.format(host, port) self.schedule_post_url = 'http://{}:{}/schedule.json'.format(host, port) self.listproject_url = 'http://{}:{}/listprojects.json'.format(host, port) self.spider_task_dic = dict() self.projects = None self.db_lock = lock self.ts_lock = threading.Lock() self._keys_set = { "year", "month", "day", "week", "hour", "minute", "second", "y", "m", "d", "w", "H", "M", "S", } self._keys_dic = { "y": "year", "m": "month", "d": "day", "w": "week", "H": "hour", "M": "minute", "S": "second", } self._keys_set_lis = [[y for y in x] for x in self._keys_set] self.CPU_THRESHOLD = 93 self.MEMORY_THRESHOLD = 96 self.schedule_logger = Logger(namespace='- Scheduler -') def run(self): time.sleep(3) self.projects = self.list_projects() self.schedule_logger.info('scheduler is running') count = 1 while True: schedule_sta = self.task_scheduler() if not schedule_sta and count == 1: self.schedule_logger.info('No Scheduled Spider in Database') count += 1 elif not schedule_sta and count != 1: count += 1 else: count = 1 time.sleep(1) def task_scheduler(self): self.ts_lock.acquire(blocking=True) self.db_lock.acquire() db_result = self.db.get(model_name='SpiderScheduleModel', key_list=['hash_str', 'project', 'spider', 'schedule', 'args', 'runtime', 'status']) self.db_lock.release() self.ts_lock.release() schedule_list_raw = [ {'hash_str': x.hash_str, 'project': x.project, 'spider': x.spider, 'schedule': x.schedule, 'args': x.args, 'runtime': x.runtime, 'status': x.status} for x in db_result if int(x.status) != 0 ] if db_result else [] schedule_sta = False if schedule_list_raw: for each_schedule in schedule_list_raw: project = each_schedule.get('project') runtime = int(each_schedule.get('runtime')) if project in self.projects and runtime > 0: schedule = each_schedule.get('schedule') if any([x in schedule for x in self._keys_set]): try: schedule = json.loads(schedule) except: schedule = eval(schedule) try: if isinstance(schedule, dict): for key in schedule.keys(): if key not in self._keys_set: mean_key = self._check_key(key) raise ValueError( 'found "{}" in your schedule dict, maybe you mean "{}"'.format(key, mean_key)) if key in self._keys_dic: val = schedule.pop(key) schedule[self._keys_dic[key]] = val next_time_sep = self.cal_time_sep(**schedule) else: next_time_sep = self.cal_time_sep(schedule_str=schedule, is_str=True) next_time_sep = int(next_time_sep) + 1 if next_time_sep > 1: each_schedule['schedule'] = next_time_sep item = '{}-{}'.format(each_schedule['project'], each_schedule['spider']) self.ts_lock.acquire(blocking=True) if self.spider_task_dic.get(item) != 'waiting': self.spider_task_dic[item] = 'waiting' t = threading.Thread(target=self.poster, args=(each_schedule,)) try: t.start() except Exception as THError: self.schedule_logger.warn('start new job error [ {} ]: {}'.format(item, THError)) self.ts_lock.release() except ValueError as V: self.schedule_logger.error('spider runtime schedule error, please check the database: {}'.format(V)) schedule_sta = True return schedule_sta def poster(self, dic): hash_str = dic.get('hash_str') status = int(dic.pop('status')) project = dic.get('project') spider = dic.get('spider') job_str = " %s-%s " % (project, spider) args = dic.get('args') try: args = json.loads(args) except: args = eval(args) wait_time = dic.get('schedule') item = '{}-{}'.format(project, spider) if project and spider: data = {'project': project, 'spider': spider, 'un': self.user_name, 'pwd': self.user_password} if args: args = self._spider_args_method(args, hash_str) data.update(args) self.schedule_logger.info('job {} is waiting, countdown {}s'.format(item, wait_time)) time.sleep(wait_time - 1) another_wait_time = 0 spider_runtime_avg = self.spiders_runtime(project=project, spider=spider) if status == 1: while not self.is_system_ok(): self.schedule_logger.warn('system is fully functioning, wait another 2 seconds to post schedule') time.sleep(2) another_wait_time += 3 if another_wait_time >= (wait_time - spider_runtime_avg): self.schedule_logger.warning('wait too long, cancel the job %s' % job_str) return None res = json.loads(requests.post(url=self.schedule_post_url, data=data).content) elif status == 2: res = json.loads(requests.post(url=self.schedule_post_url, data=data).content) elif status == 3: res = json.loads(requests.post(url=self.schedule_post_url, data=data).content) else: res = json.loads(requests.post(url=self.schedule_post_url, data=data).content) spider_status = res.get('status') if spider_status != 'ok': spider_status = 'error' else: self.schedule_logger.error('job project: {}, spider: {} post fail!'.format(project, spider)) spider_status = 'error' self.ts_lock.acquire(blocking=True) if spider_status == 'ok': self._run_countdown(project=project, spider=spider) self.spider_task_dic[item] = spider_status self.ts_lock.release() def _spider_args_method(self, args, hash_str): args_raw = args.copy() if args: method = args.pop('method', 'normal') if method == 'auto_increment': next_args = {k: str(int(v)+1) if isinstance(v, int) or (isinstance(v, str) and v.isdigit()) else v for k, v in args.items()} elif isinstance(method, dict): ex_md = method.get('expression') fc_md = method.get('function') if ex_md: next_args = eval(ex_md) if fc_md: exec(fc_md) else: next_args = args next_args.update({'method': method}) self.db.update('SpiderScheduleModel', update_dic={'args': next_args}, filter_dic={"hash_str": hash_str}) return args return args_raw def spiders_runtime(self, project, spider): self.db_lock.acquire() res = self.db.get(model_name='SpiderMonitor', key_list=['runtime'], filter_dic={'project': project, 'spider': spider}) self.db_lock.release() spider_list = [int(x.runtime) for x in res if x.runtime.isdigit()] if res else [0] return sum(spider_list) / len(spider_list) def list_projects(self): res = requests.get(url=self.listproject_url) projects = {} if res: projects_list = json.loads(res.content).get('projects') if projects_list: projects = set(projects_list) return projects def cal_time_sep(self, year='*', month='*', day='*', week='*', hour='*', minute='*', second='*', schedule_str=None, is_str=False ): """ "%Y-%m-%d %H:%M:%S %w" """ if is_str: s = [int(x.strip()) for x in schedule_str.split(',')] time_sep = (datetime.datetime(s[0], s[1], s[2], s[3], s[4], s[5]) - datetime.datetime.now()).total_seconds() return time_sep y = int(time.strftime("%Y", time.localtime())) if year != '*' and '*' in year: y = int(year.split('/')[-1]) + y elif year.isdigit(): y = int(year) if week == '*': m = int(time.strftime("%m", time.localtime())) if month != '*' and '*' in month: m_raw = int(month.split('/')[-1]) if m_raw >= 12: raise ValueError('month value is too large, please set the year instead') m = m_raw + m if m > 12: y += m // 12 m = m % 12 elif month.isdigit(): m = int(month) days_in_this_month = self.how_many_days_in_this_month(y, m) d = int(time.strftime("%d", time.localtime())) if day != '*' and '*' in day: d_raw = int(day.split('/')[-1]) if d_raw > days_in_this_month: raise ValueError('day value is too large, please set the month or the year instead') d = d_raw + d if d > days_in_this_month: d = d - days_in_this_month m += 1 if m > 12: y += 1 m = m - 12 elif day.isdigit(): d = int(day) days_in_this_month = self.how_many_days_in_this_month(y, m) H = int(time.strftime("%H", time.localtime())) if hour != '*' and '*' in hour: H_raw = int(hour.split('/')[-1]) if H_raw > 24: raise ValueError('hour value is too large, please set the day instead') H = H_raw + H if H >= 24: H = H - 24 d += 1 if d > days_in_this_month: d = d - days_in_this_month m += 1 if m > 12: y += 1 m = m - 12 elif hour.isdigit(): H = int(hour) days_in_this_month = self.how_many_days_in_this_month(y, m) M = int(time.strftime("%M", time.localtime())) if minute != '*' and '*' in minute: M_raw = int(minute.split('/')[-1]) if M_raw > 60: raise ValueError('minute value is too large, please set the hour instead') M = M_raw + M if M >= 60: M = M - 60 H += 1 if H >= 24: H = H - 24 d += 1 if d > days_in_this_month: d = d - days_in_this_month m += 1 if m > 12: y += 1 m = m - 12 elif minute.isdigit(): M = int(minute) days_in_this_month = self.how_many_days_in_this_month(y, m) S = int(time.strftime("%S", time.localtime())) if second != '*' and '*' in second: S_raw = int(second.split('/')[-1]) if S_raw > 60: raise ValueError('second value is too large, please set the minute instead') S = S_raw + S if S >= 60: S = S - 60 M += 1 if M >= 60: M = M - 60 H += 1 if H >= 24: H = H - 24 d += 1 if d > days_in_this_month: d = d - days_in_this_month m += 1 if m > 12: y += 1 m = m - 12 elif second.isdigit(): S = int(second) time_sep = eval( "(datetime.datetime({},{},{}, {},{},{}) - datetime.datetime.now()).total_seconds()".format(y, m, d, H, M, S)) else: week_in_this_year = int(time.strftime("%U", time.localtime())) w = int(time.strftime("%w", time.localtime())) if '*' in week: w_raw = int(week.split('/')[-1]) if w_raw >= 7: raise ValueError('week value is too large, please set the day or the month instead') if w_raw < w: week_in_this_year += 1 w = w_raw if week_in_this_year > 53: y += 1 week_in_this_year = week_in_this_year - 53 elif week.isdigit(): w = int(week) if int(week) < w: week_in_this_year += 1 H = int(time.strftime("%H", time.localtime())) if hour != '*' and '*' in hour: H_raw = int(hour.split('/')[-1]) if H_raw >= 24: raise ValueError('hour value is too large, please set the day instead') H = H_raw + H if H >= 24: H = H - 24 w += 1 if w >= 7: w = w - 7 week_in_this_year += 1 if week_in_this_year > 53: y += 1 week_in_this_year = week_in_this_year - 53 elif hour.isdigit(): H = int(hour) M = int(time.strftime("%M", time.localtime())) if minute != '*' and '*' in minute: M_raw = int(minute.split('/')[-1]) if M_raw >= 60: raise ValueError('minute value is too large, please set the hour instead') M = M_raw + M if M >= 60: M = M - 60 H += 1 if H >= 24: H = H - 24 w += 1 if w > 7: w = w - 7 week_in_this_year += 1 if week_in_this_year > 53: y += 1 week_in_this_year = week_in_this_year - 53 elif minute.isdigit(): M = int(minute) S = int(time.strftime("%S", time.localtime())) if second != '*' and '*' in second: S_raw = int(second.split('/')[-1]) if S_raw >= 60: raise ValueError('second value is too large, please set the minute instead') S = S_raw + S if S >= 60: S = S - 60 M += 1 if M >= 60: M = M - 60 H += 1 if H >= 24: H = H - 24 w += 1 if w > 7: w = w - 7 week_in_this_year += 1 if week_in_this_year > 53: y += 1 week_in_this_year = week_in_this_year - 53 elif second.isdigit(): S = int(second) if S >= 60: S = S - 60 M += 1 if M >= 60: M = M - 60 H += 1 if H >= 24: H = H - 24 w += 1 if w > 7: w = w - 7 week_in_this_year += 1 if week_in_this_year > 53: y += 1 week_in_this_year = week_in_this_year - 53 m, d = self.get_month_and_days_by_week(year=y, week_in_this_year=week_in_this_year, week=w) time_sep = eval( "(datetime.datetime({},{},{}, {},{},{}) - datetime.datetime.now()).total_seconds()".format(y, m, d, H, M, S)) return time_sep def get_month_and_days_by_week(self, year, week_in_this_year, week): days = week_in_this_year * 7 + week if (year % 4 == 0 and year % 100 != 0) or (year % 400 == 0): Fe = 29 else: Fe = 28 month_lis = [31, Fe, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31] month_count = 1 days_count = 0 for month_days in month_lis: days = days - month_days if days > 0: month_count += 1 elif days == 0: days_count = 0 month_count += 1 break else: days_count = days + month_days break return [month_count, days_count] def how_many_days_in_this_month(self, y, m): if m in (1, 3, 5, 7, 8, 10, 12): days = 31 elif m in (4, 6, 9, 11): days = 30 else: if (y % 4 == 0 and y % 100 != 0) or (y % 400 == 0): days = 29 else: days = 28 return days def is_system_ok(self): is_pass = True cpu_list = psutil.cpu_percent(interval=1, percpu=True) memory_percent = psutil.virtual_memory().percent if cpu_list and memory_percent: is_cpu_ok = True if min(cpu_list) > self.CPU_THRESHOLD: is_cpu_ok = False is_memo_ok = True if memory_percent > self.MEMORY_THRESHOLD: is_memo_ok = False if not is_cpu_ok or not is_memo_ok: is_pass = False return is_pass def _check_key(self, key): key_lis = [x for x in key] count_dic = dict() for ksl in self._keys_set_lis: o_key = ''.join(ksl) score = 0 for k in key_lis: if k in ksl: score += 1 count_dic[o_key] = score best_math = sorted(count_dic, key=count_dic.__getitem__, reverse=True)[0] return best_math def _run_countdown(self, project, spider): db_schedule = self.db.get(model_name='SpiderScheduleModel', key_list=['id', 'runtime'], filter_dic={'project': project, 'spider': spider}) run_time_in_db = [x.runtime for x in db_schedule][0] if db_schedule else 0 the_id = [x.id for x in db_schedule][0] if db_schedule else None if run_time_in_db > 0 and the_id is not None: rt = int(run_time_in_db) - 1 self.db.update(model_name='SpiderScheduleModel', update_dic={"runtime": rt}, filter_dic={"id": the_id})
class JWTClientIPAuthResource(resource.Resource): """ Validates JWT token passed in via a HTTP query parameter. Sets a session cookie in order to authenticate subsequent requests. """ isLeaf = True def __init__(self, param: bytes, cookie: bytes, header: bytes, keyfile: str, sessttl: int): self.log = Logger() self.key = jwt.JWK() self.param = param self.cookie = cookie self.header = header # Very naive session store. Extract and improve if necessary. self.sessttl = sessttl self.sessions = set() with open(keyfile, 'rb') as stream: self.key.import_from_pem(stream.read()) def render(self, request: server.Request) -> bytes: # Deny by default. request.setResponseCode(401) # Get session cookie value if any. sessionid = request.getCookie(self.cookie) if sessionid is not None: if sessionid in self.sessions: request.setResponseCode(200) self.log.info("Session: Validation succeeded") return b"" else: self.log.info("Session: Invalid session id") # Token is passed as a query parameter in the original URL. origurl = http.urlparse(request.getHeader(self.header)) query = http.parse_qs(origurl.query) args = query.get(self.param, []) if len(args) != 1: self.log.error("Request: Token {param} missing", param=self.param) return b"" try: token = jwt.JWT(key=self.key, jwt=args[0].decode()) except (jwt.JWTExpired, jwt.JWTNotYetValid, jwt.JWTMissingClaim, jwt.JWTInvalidClaimValue, jwt.JWTInvalidClaimFormat, jwt.JWTMissingKeyID, jwt.JWTMissingKey) as error: self.log.error("JWT token: {error}", error=error) return b"" except Exception: self.log.failure("JWT token: Unknown exception") return b"" try: claims = json.loads(token.claims) except json.JSONDecodeError as error: self.log.failure("JWT token: Claims {error}", error=error) return b"" # Collect session parameters from claims. sessparams = claims.get("session", {}) kwargs = { "expires": sessparams.get("expires", None), "domain": sessparams.get("domain", None), "path": sessparams.get("path", None), "secure": sessparams.get("secure", None), "httpOnly": sessparams.get("httpOnly", None), "sameSite": sessparams.get("sameSite", None), } # Use maxAge for session ttl if it is present, convert it into a str # type as required by the addCookie call. if "maxAge" in sessparams: kwargs["max_age"] = str(sessparams["maxAge"]) sessttl = int(sessparams["maxAge"]) else: sessttl = self.sessttl # Generate a new session id and remember it. Also clean it up after # ttl seconds. sessionid = secrets.token_urlsafe(nbytes=16).encode() self.sessions.add(sessionid) reactor.callLater(sessttl, self._session_remove, sessionid) self.log.info("Session: Created, num sessions: {sessions}", sessions=len(self.sessions)) # Set cookie in the browser. request.addCookie(self.cookie, sessionid, **kwargs) request.setResponseCode(200) self.log.info("JWT token: Validation succeeded") return b"" def _session_remove(self, sessionid: bytes): self.sessions.remove(sessionid) self.log.info("Session: Removed, num sessions: {sessions}", sessions=len(self.sessions))
class FCMv1Router(FCMRouter): """FCM v1 HTTP Router Implementation Note: FCM v1 is a newer version of the FCM HTTP API. """ def __init__(self, conf, router_conf, metrics): """Create a new FCM router and connect to FCM""" self.conf = conf self.router_conf = router_conf self.metrics = metrics self.min_ttl = router_conf.get("ttl", 60) self.dryRun = router_conf.get("dryrun", False) self.collapseKey = router_conf.get("collapseKey", "webpush") self.version = router_conf["version"] self.log = Logger() self.clients = {} try: for (sid, creds) in router_conf["creds"].items(): self.clients[sid] = FCMv1(project_id=creds["projectid"], service_cred_path=creds["auth"], logger=self.log, metrics=self.metrics) except Exception as e: self.log.error("Could not instantiate FCMv1: missing credentials,", ex=e) raise IOError("FCMv1 Bridge not initiated in main") self._base_tags = ["platform:fcmv1"] self.log.debug("Starting FCMv1 router...") def amend_endpoint_response(self, response, router_data): # type: (JSONDict, JSONDict) -> None response["senderid"] = router_data["app_id"] def register(self, uaid, router_data, app_id, *args, **kwargs): # type: (str, JSONDict, str, *Any, **Any) -> None """Validate that the FCM Instance Token is in the ``router_data``""" # "token" is the FCM token generated by the client. if "token" not in router_data: raise self._error("connect info missing FCM Instance 'token'", status=401, uri=kwargs.get('uri'), senderid=repr(app_id)) if app_id not in self.clients: raise self._error("Invalid SenderID", status=410, errno=105) router_data["app_id"] = app_id def route_notification(self, notification, uaid_data): """Start the FCM notification routing, returns a deferred""" router_data = uaid_data["router_data"] return self._route(notification, router_data) def _route(self, notification, router_data): """Blocking FCM call to route the notification""" # THIS MUST MATCH THE CHANNELID GENERATED BY THE REGISTRATION SERVICE # Currently this value is in hex form. data = {"chid": notification.channel_id.hex} if not router_data.get("token"): raise self._error( "No registration token found. " "Rejecting message.", 410, errno=106, log_exception=False) # Payload data is optional. The endpoint handler validates that the # correct encryption headers are included with the data. if notification.data: mdata = self.router_conf.get('max_data', 4096) if notification.data_length > mdata: raise self._error("This message is intended for a " + "constrained device and is limited " + "to 3070 bytes. Converted buffer too " + "long by %d bytes" % (notification.data_length - mdata), 413, errno=104, log_exception=False) data['body'] = notification.data data['con'] = notification.headers['encoding'] if 'encryption' in notification.headers: data['enc'] = notification.headers['encryption'] if 'crypto_key' in notification.headers: data['cryptokey'] = notification.headers['crypto_key'] elif 'encryption_key' in notification.headers: data['enckey'] = notification.headers['encryption_key'] # registration_ids are the FCM instance tokens (specified during # registration. router_ttl = min(self.MAX_TTL, max(self.min_ttl, notification.ttl or 0)) try: d = self.clients[router_data["app_id"]].send( token=router_data.get("token"), payload={ "collapse_key": self.collapseKey, "data_message": data, "dry_run": self.dryRun or ('dryrun' in router_data), "ttl": router_ttl }) except KeyError: raise self._error("Invalid Application ID specified", 404, errno=106, log_exception=False) d.addCallback(self._process_reply, notification, router_data, router_ttl) d.addErrback(self._process_error) return d def _process_error(self, failure): err = failure.value if isinstance(err, FCMAuthenticationError): self.log.error("FCM Authentication Error: {}".format(err)) raise RouterException("Server error", status_code=500, errno=901) if isinstance(err, TimeoutError): self.log.warn("FCM Timeout: %s" % err) self.metrics.increment("notification.bridge.error", tags=make_tags(self._base_tags, reason="timeout")) raise RouterException("Server error", status_code=502, errno=903, log_exception=False) if isinstance(err, ConnectError): self.log.warn("FCM Unavailable: %s" % err) self.metrics.increment("notification.bridge.error", tags=make_tags( self._base_tags, reason="connection_unavailable")) raise RouterException("Server error", status_code=502, errno=902, log_exception=False) if isinstance(err, FCMNotFoundError): self.log.debug("FCM Recipient not found: %s" % err) self.metrics.increment("notification.bridge.error", tags=make_tags(self._base_tags, reason="recpient_gone")) raise RouterException("FCM Recipient no longer available", status_code=404, errno=106, log_exception=False) if isinstance(err, RouterException): self.log.warn("FCM Error: {}".format(err)) self.metrics.increment("notification.bridge.error", tags=make_tags(self._base_tags, reason="server_error")) return failure def _error(self, err, status, **kwargs): """Error handler that raises the RouterException""" self.log.debug(err, **kwargs) return RouterException(err, status_code=status, response_body=err, **kwargs) def _process_reply(self, reply, notification, router_data, ttl): """Process FCM send reply""" # Failures are returned as non-200 messages (404, 410, etc.) self.metrics.increment("notification.bridge.sent", tags=self._base_tags) self.metrics.increment("notification.message_data", notification.data_length, tags=make_tags(self._base_tags, destination="Direct")) location = "%s/m/%s" % (self.conf.endpoint_url, notification.version) return RouterResponse(status_code=201, response_body="", headers={ "TTL": ttl, "Location": location }, logged_status=200)
def cli(): log = Logger() parser = argparse.ArgumentParser(prog=__version__.package) parser.add_argument('--version', action='version', version=__version__.public()) parser.add_argument( '--secrets', help= 'Directory that contains files named "username" and "password". These files should contain your Nest username and password.' ) parser.add_argument('--username', help='Your Nest username.') parser.add_argument('--password', help='Your Nest password.') parser.add_argument( '--endpoint', default=default_endpoint, help= 'Twisted endpoint declaration for internal web service. Default is "{}".' .format(default_endpoint)) options = parser.parse_args() output = textFileLogObserver(sys.stderr) globalLogBeginner.beginLoggingTo([output]) username = None password = None if options.secrets is not None: if not os.path.isdir(options.secrets): log.critical('{secrets:} is not a directory', secrets=options.secrets) sys.exit(1) username_file = os.path.join(options.secrets, 'username') password_file = os.path.join(options.secrets, 'password') try: with open(username_file, mode='r', encoding='utf-8') as f: username = f.read().strip() except FileNotFoundError: log.error( 'Secrets path specified but username file {username_file:} not found!', username_file=username_file) except PermissionError: log.error( 'Unable to open username file {username_file:} for reading!', username_file=username_file) try: with open(password_file, mode='r', encoding='utf-8') as f: password = f.read().strip() except FileNotFoundError: log.error( 'Secrets path specified but password file {password_file:} not found!', password_file=password_file) except PermissionError: log.error( 'Unable to open password file {password_file:} for reading!', password_file=password_file) if options.username is not None: username = options.username if options.password is not None: password = options.password if username is None: log.critical('Username must be specified!') sys.exit(1) if password is None: log.critical('Password must be specified!') sys.exit(1) m = Main(reactor, username, password, options.endpoint) reactor.run()
class BaseProtocol(serialBytesProtocol): def __init__(self, shorthand=True, callback=None, escaped=False, error_callback=None): serialBytesProtocol.__init__(self) if callback: self.callbacks = [callback] else: self.callbacks = [] self.setRawMode() self.shorthand = shorthand self._escaped = escaped self.log = Logger() self.requests = {} self.command_id = 0 self.buffer = None # self.reading = False def get_id(self): try: self.command_id += 1 return intToByte(self.command_id) except ValueError: self.command_id = 1 return intToByte(1) def connect(self, f): if f.callback: self.callbacks.append(f.callback) f.proto = self def rawDataReceived(self, data): for byte in data: if self.buffer: self.buffer.fill(byte) if self.buffer.remaining_bytes() == 0: try: # Try to parse and return result self.buffer.parse() # Ignore empty frames if len(self.buffer.data) == 0: self.buffer = None except ValueError: # Bad frame, so restart self.log.warn('Bad frame: %r' % self.buffer.raw_data) else: self.read_frame(self.buffer.data) self.buffer = None # self.reading = False else: if byte == Frame.START_BYTE: # self.reading == True self.buffer = Frame(escaped=self._escaped) def read_frame(self, frame): """ read_frame: binary data -> {'id':str, 'param':binary data, ...} read_frame takes a data packet received from an XBee device and converts it into a dictionary. This dictionary provides names for each segment of binary data as specified in the api_responses spec. """ # Fetch the first byte, identify the packet # If the spec doesn't exist, raise exception packet_id = frame[0:1] try: name = self.api_responses[packet_id] except AttributeError: raise NotImplementedError( "API response specifications could not be found; " + "use a derived class which defines 'api_responses'.") except KeyError: # Check to see if this ID can be found among transmittible packets for cmd_name, cmd in list(self.api_frames.items()): if cmd['id']['default'] == packet_id: msg = "Incoming frame with id {packet_id} looks like a " +\ "command frame of type '{cmd_name}' (these should " +\ " not be received). Are you sure your devices " +\ "are in API mode?" self.log.error( msg, packet_id=bytes(frame), cmd_name=cmd_name) return self.log.error("Unrecognized response packet with id byte {f}", f=frame[0]) return # Current byte index in the data stream packet = self.api_frames[name] index = 0 callback = False # Result info info = {'id': name} # packet_spec = packet['structure'] # Parse the packet in the order specified if 'frame_id' in packet: callback = True # if packet['len'] == 'null_terminated': # field_data = b'' # while frame[index:index + 1] != b'\x00': # field_data += frame[index:index + 1] # index += 1 # index += 1 # info[name] for field, dic in packet.items(): if dic['len'] == 'null_terminated': field_data = b'' while frame[index:index] != b'\x00': field_data += frame[index:index] index += 1 index += 1 info[field] = field_data elif dic['len'] is not None: # Store the number of bytes specified # Are we trying to read beyond the last data element? if index + dic['len'] > len(frame): raise ValueError( "Response packet was shorter than expected") field_data = frame[index:index + dic['len']] info[field] = field_data index += dic['len'] # If the data field has no length specified, store any # leftover bytes and quit else: field_data = frame[index:-1] # Were there any remaining bytes? if field_data: # If so, store them info[field] = field_data index += len(field_data) + 1 break # If there are more bytes than expected, raise an exception if index + 1 < len(frame): raise ValueError( "Response packet was longer than expected; " + "expected: %d, got: %d bytes" % (index, len(frame))) # Apply parsing rules if any exist if 'parsing' in packet: for parse_rule in packet['parsing']: # Only apply a rule if it is relevant (raw data is available) if parse_rule[0] in info: # Apply the parse function to the indicated field and # replace the raw data with the result info[parse_rule[0]] = parse_rule[1](self, info) if callback: if info['frame_id'] in self.requests: self.requests[info['frame_id']].callback(info) del self.requests[info['frame_id']] else: self.log.warn('Response without request: %r' % info) elif self.callbacks: for callback in self.callbacks: callback(info) else: self.log.debug(info) def _build_command(self, cmd, **kwargs): """ _build_command: string (binary data) ... -> binary data _build_command will construct a command packet according to the specified command's specification in api_commands. It will expect named arguments for all fields other than those with a default value or a length of 'None'. Each field will be written out in the order they are defined in the command definition. """ try: cmd_spec = self.api_frames[cmd] except AttributeError: raise NotImplementedError( "API command specifications could not be found; " + "use a derived class which defines 'api_commands'.") packet = b'' if 'frame_id' in kwargs: fid = kwargs['frame_id'] elif cmd in ['source_route']: fid = b'\x00' else: fid = self.get_id() for name, dic in cmd_spec.items(): if name == 'frame_id': data = fid elif name in kwargs: data = kwargs[name] else: if dic['len']: if dic['default']: data = dic['default'] else: raise KeyError( "The expected field %s of length %d was " + "not provided" % (name, dic['len'])) else: data = None if dic['len'] and len(data) != dic['len']: raise ValueError( "The data provided for '%s' was not %d bytes long" % (name, dic['len'])) if data: packet += data return packet, fid def send(self, cmd, **kwargs): """ send: string param=binary data ... -> None When send is called with the proper arguments, an API command will be written to the serial port for this XBee device containing the proper instructions and data. This method must be called with named arguments in accordance with the api_command specification. Arguments matching all field names other than those in reserved_names (like 'id' and 'order') should be given, unless they are of variable length (of 'None' in the specification. Those are optional). """ # Pass through the keyword arguments # if self.reading: # return task.deferLater(.5, self.send, cmd, **kwargs) packet, fid = self._build_command(cmd, **kwargs) d = defer.Deferred() self.requests.update({fid: d}) f = Frame(packet).output() self.transport.write(f) return d def _parse_samples_header(self, io_bytes): """ _parse_samples_header: binary data in XBee IO data format -> (int, [int ...], [int ...], int, int) _parse_samples_header will read the first three bytes of the binary data given and will return the number of samples which follow, a list of enabled digital inputs, a list of enabled analog inputs, the dio_mask, and the size of the header in bytes """ header_size = 4 # number of samples (always 1?) is the first byte sample_count = byteToInt(io_bytes[0]) # part of byte 1 and byte 2 are the DIO mask ( 9 bits ) dio_mask = ( byteToInt(io_bytes[1]) << 8 | byteToInt(io_bytes[2])) & 0x01FF # upper 7 bits of byte 1 is the AIO mask aio_mask = byteToInt(io_bytes[3]) & 0xFE >> 1 # print(byteToInt(io_bytes[3]) & 0xFE >> 1) # print(aio_mask) # sorted lists of enabled channels; value is position of bit in mask dio_chans = [] aio_chans = [] for i in range(0, 9): if dio_mask & (1 << i): dio_chans.append(i) dio_chans.sort() for i in range(0, 7): if aio_mask & (1 << i): aio_chans.append(i) aio_chans.sort() return (sample_count, dio_chans, aio_chans, dio_mask, header_size) def _parse_samples(self, io_bytes): """ _parse_samples: binary data in XBee IO data format -> [ {"dio-0":True, "dio-1":False, "adc-0":100"}, ...] _parse_samples reads binary data from an XBee device in the IO data format specified by the API. It will then return a dictionary indicating the status of each enabled IO port. """ sample_count, dio_chans, aio_chans, dio_mask, header_size = \ self._parse_samples_header(io_bytes) samples = [] # split the sample data into a list, so it can be pop()'d # self.log.debug('%r' % io_bytes) sample_bytes = [byteToInt(c) for c in io_bytes[header_size:]] # self.log.debug('%r' % sample_bytes) # self.log.debug('%r' % aio_chans) # repeat for every sample provided for sample_ind in range(0, sample_count): # @UnusedVariable tmp_samples = {} if dio_chans: # we have digital data digital_data_set = ( sample_bytes.pop(0) << 8 | sample_bytes.pop(0)) digital_values = dio_mask & digital_data_set for i in dio_chans: tmp_samples['dio-{0}'.format(i)] = True if ( digital_values >> i) & 1 else False for i in aio_chans: analog_sample = ( sample_bytes.pop(0) << 8 | sample_bytes.pop(0)) tmp_samples['adc-{0}'.format(i)] = int( (analog_sample * 1200.0) / 1023.0) samples.append(tmp_samples) return samples def _parse_sensor_data(self, io_bytes): # TODO return [{'data': io_bytes}] def __getattr__(self, name): """ If a method by the name of a valid api command is called, the arguments will be automatically sent to an appropriate send() call """ # If api_commands is not defined, raise NotImplementedError\ # If its not defined, _getattr__ will be called with its name if name == 'api_frames': raise NotImplementedError( "API command specifications could not be found; use a " + "derived class which defines 'api_commands'.") # Is shorthand enabled, and is the called name a command? if self.shorthand and name in self.api_frames: # If so, simply return a function which passes its arguments # to an appropriate send() call return lambda **kwargs: self.send(name, **kwargs) else: raise AttributeError("XBee has no attribute '%s'" % name)
class PhotometerService(ClientService): def __init__(self, options, reference): self.options = options self.namespace = 'ref.' if reference else 'test' self.label = self.namespace.upper() setLogLevel(namespace=self.label, levelStr=options['log_messages']) setLogLevel(namespace=self.namespace, levelStr=options['log_level']) self.log = Logger(namespace=self.namespace) self.reference = reference # Flag, is this instance for the reference photometer self.factory = self.buildFactory() self.protocol = None self.serport = None self.info = None # Photometer info self.buffer = CircularBuffer(options['size'], self.log) parts = chop(self.options['endpoint'], sep=':') if parts[0] == 'tcp': endpoint = clientFromString(reactor, self.options['endpoint']) ClientService.__init__(self, endpoint, self.factory, retryPolicy=backoffPolicy(initialDelay=0.5, factor=3.0)) @inlineCallbacks def startService(self): ''' Starts the photometer service listens to a TESS Although it is technically a synchronous operation, it works well with inline callbacks ''' self.log.info("starting {name} service", name=self.name) yield self.connect() self.info = yield self.getInfo() if self.reference: returnValue(None) # Now this is for the test photometer only if self.options['dry_run']: self.log.info('Dry run. Will stop here ...') yield self.stopService() elif self.info is None: yield self.stopService() elif self.options['zero_point'] is not None: try: result = yield self.protocol.writeZeroPoint( self.options['zero_point']) except Exception as e: self.log.error("Timeout when updating Zero Point") self.log.failure("{excp}", excp=e) else: self.log.info("[{label}] Writen ZP : {zp:0.2f}", label=self.label, zp=result['zp']) finally: yield self.stopService() def stopService(self): self.log.info("stopping {name} service", name=self.name) try: reactor.callLater(0, reactor.stop) except Exception as e: log.error("could not stop the reactor") return defer.succeed(None) # -------------- # Photometer API # -------------- def writeZeroPoint(self, zero_point): '''Writes Zero Point to the device. Returns a Deferred''' return self.protocol.writeZeroPoint(zero_point) def getPhotometerInfo(self): if self.protocol is None: self.log.warn("Requested photometer info but no protocol yet!") return defer.fail() if self.info is None: return self.getInfo() else: return defer.succeed(self.info) # -------------- # Helper methods # --------------- @inlineCallbacks def connect(self): parts = chop(self.options['endpoint'], sep=':') if parts[0] == 'serial': endpoint = parts[1:] self.protocol = self.factory.buildProtocol(0) try: self.serport = SerialPort(self.protocol, endpoint[0], reactor, baudrate=endpoint[1]) except Exception as e: self.log.error("{excp}", excp=e) yield self.stopService() else: self.gotProtocol(self.protocol) self.log.info("Using serial port {tty} at {baud} bps", tty=endpoint[0], baud=endpoint[1]) else: ClientService.startService(self) try: protocol = yield self.whenConnected(failAfterFailures=1) except Exception as e: self.log.error("{excp}", excp=e) yield self.stopService() else: self.gotProtocol(protocol) self.log.info("Using TCP endpoint {endpoint}", endpoint=self.options['endpoint']) @inlineCallbacks def getInfo(self): try: info = yield self.protocol.readPhotometerInfo() except Exception as e: self.log.error("Timeout when reading photometer info") info = self.fixIt() returnValue(info) # May be None else: info['model'] = self.options['model'] info['label'] = self.label self.log.info("[{label}] Model : {value}", label=self.label, value=info['model']) self.log.info("[{label}] Name : {value}", label=self.label, value=info['name']) self.log.info("[{label}] MAC : {value}", label=self.label, value=info['mac']) self.log.info("[{label}] Zero Point: {value:.02f} (old)", label=self.label, value=info['zp']) self.log.info("[{label}] Firmware : {value}", label=self.label, value=info['firmware']) returnValue(info) def fixIt(self): parts = chop(self.options['endpoint'], sep=':') if self.reference and (self.options['model'] == TESSW) and parts[0] == 'serial': info = { 'model': TESSW, 'label': self.label, 'name': self.options['name'], 'mac': self.options['mac'], 'zp': 20.50, 'firmware': "", } self.log.error("Fixed photometer info with defaults {info}", info=info) return info else: return None def limitedStart(self): '''Detects the case where only the Test photometer service is started''' if self.reference: return False return (self.options['dry_run'] or self.options['zero_point'] is not None) def buildFactory(self): if self.options['model'] == TESSW: self.log.debug("Choosing a {model} factory", model=TESSW) import zptess.tessw factory = zptess.tessw.TESSProtocolFactory(self.label) elif self.options['model'] == TESSP: self.log.debug("Choosing a {model} factory", model=TESSP) import zptess.tessp factory = zptess.tessp.TESSProtocolFactory(self.label) else: self.log.debug("Choosing a {model} factory", model=TAS) import zptess.tas factory = zptess.tas.TESSProtocolFactory(self.label) return factory def gotProtocol(self, protocol): self.log.debug("got protocol") protocol.setContext(self.options['endpoint']) self.buffer.registerProducer(protocol, True) if self.limitedStart(): protocol.stopProducing( ) # We don need to feed messages to the buffer self.protocol = protocol
class SerialClientProtocol(LineReceiver): def connectionMade(self): self.log = Logger(namespace="serial") self.log.debug("serial port connected") def lineReceived(self, line): from . import ws_factory self.log.debug("serial rx: {line}", line=line) try: data = line.strip() if data == b"r": if not ws_factory.client: self.log.error("websocket client not connected") else: self.log.debug("serial tx: k") self.sendLine(b"k") data_json = json.dumps([{"data_type": "counter"}]) self.log.debug("sending data to ws: {data}", data=data_json) ws_factory.client.sendMessage(data_json.encode("utf8")) else: commands = data.split(b"#") data_array = [] for command in commands: values = command.split(b",") cmd = values[0].decode() data_dict = {"data_type": cmd} if values[0] in [b"vel", b"pos", b"lap"]: if len(values) > 1: data_dict[cmd + "_1"] = int.from_bytes( values[1], "big") if values[1] else 0 else: data_dict[cmd + "_1"] = 0 if len(values) > 2: data_dict[cmd + "_2"] = int.from_bytes( values[2], "big") else: data_dict[cmd + "_2"] = 0 elif values[0] == b"goal": data_dict["goal_lap"] = int.from_bytes( values[1], "big") elif values[0] == b"play": if len(values) > 1: data_dict["status"] = True if int.from_bytes( values[1], "big") == 1 else False else: data_dict["status"] = False elif values[0] == b"win": data_dict["winner"] = int.from_bytes(values[1], "big") elif values[0] == b"time": if len(values) > 1: data_dict["seconds"] = int.from_bytes( values[1], "big") / 1000 else: data_dict["seconds"] = 0.0 else: self.log.error("command not supported: {cmd}", cmd=values[0]) data_dict = None if data_dict: data_array.append(data_dict) if data_array: if not ws_factory.client: self.log.error("websocket client not connected") else: data_json = json.dumps(data_array) self.log.debug("sending data to ws: {data}", data=data_json) ws_factory.client.sendMessage(data_json.encode("utf8")) pass except Exception as e: self.log.error("error: {e}", e=e) def sendMessage(self, message: str): self.log.debug("serial tx: {message}", message=message) self.sendLine(message.encode("utf8"))
class IRCd(Service): def __init__(self, configFileName): self.config = Config(self, configFileName) self.boundPorts = {} self.loadedModules = {} self._loadedModuleData = {} self._unloadingModules = {} self.commonModules = set() self.userCommands = {} self.serverCommands = {} self.channelModes = ({}, {}, {}, {}) self.channelStatuses = {} self.channelStatusSymbols = {} self.channelStatusOrder = [] self.channelModeTypes = {} self.userModes = ({}, {}, {}, {}) self.userModeTypes = {} self.actions = {} self.storage = None self.storageSyncer = None self.dataCache = {} self.functionCache = {} self.serverID = None self.name = None self.isupport_tokens = { "CASEMAPPING": "strict-rfc1459", "CHANTYPES": "#", } self._uid = self._genUID() self.users = {} self.userNicks = CaseInsensitiveDictionary() self.channels = CaseInsensitiveDictionary(WeakValueDictionary) self.servers = {} self.serverNames = CaseInsensitiveDictionary() self.recentlyQuitUsers = {} self.recentlyQuitServers = {} self.recentlyDestroyedChannels = CaseInsensitiveDictionary() self.pruneRecentlyQuit = None self.pruneRecentChannels = None self._logFilter = LogLevelFilterPredicate() filterObserver = FilteringLogObserver(globalLogPublisher, (self._logFilter,)) self.log = Logger("txircd", observer=filterObserver) self.startupTime = None def startService(self): self.log.info("Starting up...") self.startupTime = now() self.log.info("Loading configuration...") self.config.reload() self.name = self.config["server_name"] self.serverID = self.config["server_id"] self.log.info("Loading storage...") self.storage = shelve.open(self.config["datastore_path"], writeback=True) self.storageSyncer = LoopingCall(self.storage.sync) self.storageSyncer.start(self.config.get("storage_sync_interval", 5), now=False) self.log.info("Starting processes...") self.pruneRecentlyQuit = LoopingCall(self.pruneQuit) self.pruneRecentlyQuit.start(10, now=False) self.pruneRecentChannels = LoopingCall(self.pruneChannels) self.pruneRecentChannels.start(15, now=False) self.log.info("Loading modules...") self._loadModules() self.log.info("Binding ports...") self._bindPorts() self.log.info("txircd started!") try: self._logFilter.setLogLevelForNamespace("txircd", LogLevel.levelWithName(self.config["log_level"])) except (KeyError, InvalidLogLevelError): self._logFilter.setLogLevelForNamespace("txircd", LogLevel.warn) self.runActionStandard("startup") def stopService(self): stopDeferreds = [] self.log.info("Disconnecting servers...") serverList = self.servers.values() # Take the list of server objects self.servers = {} # And then destroy the server dict to inhibit server objects generating lots of noise for server in serverList: if server.nextClosest == self.serverID: stopDeferreds.append(server.disconnectedDeferred) allUsers = self.users.keys() for user in allUsers: if user[:3] == server.serverID: del self.users[user] server.transport.loseConnection() self.log.info("Disconnecting users...") userList = self.users.values() # Basically do the same thing I just did with the servers self.users = {} for user in userList: if user.transport: stopDeferreds.append(user.disconnectedDeferred) user.transport.loseConnection() self.log.info("Unloading modules...") moduleList = self.loadedModules.keys() for module in moduleList: self._unloadModule(module, False) # Incomplete unload is done to save time and because side effects are destroyed anyway self.log.info("Stopping processes...") if self.pruneRecentlyQuit.running: self.pruneRecentlyQuit.stop() if self.pruneRecentChannels.running: self.pruneRecentChannels.stop() self.log.info("Closing data storage...") if self.storageSyncer.running: self.storageSyncer.stop() self.storage.close() # a close() will sync() also self.log.info("Releasing ports...") stopDeferreds.extend(self._unbindPorts()) return DeferredList(stopDeferreds) def _loadModules(self): for module in getPlugins(IModuleData, txircd.modules): if module.name in self.loadedModules: continue if module.core or module.name in self.config["modules"]: self._loadModuleData(module) for moduleName in self.config["modules"]: if moduleName not in self.loadedModules: self.log.warn("The module {module} failed to load.", module=moduleName) def loadModule(self, moduleName): """ Loads a module of the specified name. Raises ModuleLoadError if the module cannot be loaded. If the specified module is currently being unloaded, returns the DeferredList specified by the module when it was unloading with a callback to try to load the module again once it succeeds. """ if moduleName in self._unloadingModules: deferList = self._unloadingModules[moduleName] deferList.addCallback(self._tryLoadAgain, moduleName) return deferList for module in getPlugins(IModuleData, txircd.modules): if module.name == moduleName: rebuild(importlib.import_module(module.__module__)) # getPlugins doesn't recompile modules, so let's do that ourselves. self._loadModuleData(module) self.log.info("Loaded module {module}.", module=moduleName) break def _tryLoadAgain(self, _, moduleName): self.loadModule(moduleName) def _loadModuleData(self, module): if not IModuleData.providedBy(module): raise ModuleLoadError ("???", "Module does not implement module interface") if not module.name: raise ModuleLoadError ("???", "Module did not provide a name") if module.name in self.loadedModules: self.log.debug("Not loading {module.name} because it's already loaded", module=module) return self.log.debug("Beginning to load {module.name}...", module=module) module.hookIRCd(self) try: module.verifyConfig(self.config) except ConfigError as e: raise ModuleLoadError(module.name, e) self.log.debug("Loading hooks from {module.name}...", module=module) moduleData = { "channelmodes": module.channelModes(), "usermodes": module.userModes(), "actions": module.actions(), "usercommands": module.userCommands(), "servercommands": module.serverCommands() } newChannelModes = ({}, {}, {}, {}) newChannelStatuses = {} newUserModes = ({}, {}, {}, {}) newActions = {} newUserCommands = {} newServerCommands = {} common = False self.log.debug("Processing hook data from {module.name}...", module=module) for mode in moduleData["channelmodes"]: if mode[0] in self.channelModeTypes: raise ModuleLoadError (module.name, "Tries to implement channel mode +{} when that mode is already implemented.".format(mode[0])) if not IMode.providedBy(mode[2]): raise ModuleLoadError (module.name, "Returns a channel mode object (+{}) that doesn't implement IMode.".format(mode[0])) if mode[1] == ModeType.Status: if mode[4] in self.channelStatusSymbols: raise ModuleLoadError (module.name, "Tries to create a channel rank with symbol {} when that symbol is already in use.".format(mode[4])) try: newChannelStatuses[mode[0]] = (mode[4], mode[3], mode[2]) except IndexError: raise ModuleLoadError (module.name, "Specifies channel status mode {} without a rank or symbol".format(mode[0])) else: newChannelModes[mode[1]][mode[0]] = mode[2] common = True for mode in moduleData["usermodes"]: if mode[0] in self.userModeTypes: raise ModuleLoadError (module.name, "Tries to implement user mode +{} when that mode is already implemented.".format(mode[0])) if not IMode.providedBy(mode[2]): raise ModuleLoadError (module.name, "Returns a user mode object (+{}) that doesn't implement IMode.".format(mode[0])) newUserModes[mode[1]][mode[0]] = mode[2] common = True for action in moduleData["actions"]: if action[0] not in newActions: newActions[action[0]] = [(action[2], action[1])] else: newActions[action[0]].append((action[2], action[1])) for command in moduleData["usercommands"]: if not ICommand.providedBy(command[2]): raise ModuleLoadError (module.name, "Returns a user command object ({}) that doesn't implement ICommand.".format(command[0])) if command[0] not in newUserCommands: newUserCommands[command[0]] = [] newUserCommands[command[0]].append((command[2], command[1])) for command in moduleData["servercommands"]: if not ICommand.providedBy(command[2]): raise ModuleLoadError (module.name, "Returns a server command object ({}) that doesnt implement ICommand.".format(command[0])) if command[0] not in newServerCommands: newServerCommands[command[0]] = [] newServerCommands[command[0]].append((command[2], command[1])) common = True if not common: common = module.requiredOnAllServers self.log.debug("Loaded data from {module.name}; committing data and calling hooks...", module=module) module.load() self.loadedModules[module.name] = module self._loadedModuleData[module.name] = moduleData if common: self.commonModules.add(module.name) self.runActionStandard("moduleload", module.name) for modeType, typeSet in enumerate(newChannelModes): for mode, implementation in typeSet.iteritems(): self.channelModeTypes[mode] = modeType self.channelModes[modeType][mode] = implementation for mode, data in newChannelStatuses.iteritems(): self.channelModeTypes[mode] = ModeType.Status self.channelStatuses[mode] = data self.channelStatusSymbols[data[0]] = mode for index, status in enumerate(self.channelStatusOrder): if self.channelStatuses[status][1] < data[1]: self.channelStatusOrder.insert(index, mode) break else: self.channelStatusOrder.append(mode) for modeType, typeSet in enumerate(newUserModes): for mode, implementation in typeSet.iteritems(): self.userModeTypes[mode] = modeType self.userModes[modeType][mode] = implementation for action, actionList in newActions.iteritems(): if action not in self.actions: self.actions[action] = [] for actionData in actionList: for index, handlerData in enumerate(self.actions[action]): if handlerData[1] < actionData[1]: self.actions[action].insert(index, actionData) break else: self.actions[action].append(actionData) for command, dataList in newUserCommands.iteritems(): if command not in self.userCommands: self.userCommands[command] = [] for data in dataList: for index, cmd in enumerate(self.userCommands[command]): if cmd[1] < data[1]: self.userCommands[command].insert(index, data) break else: self.userCommands[command].append(data) for command, dataList in newServerCommands.iteritems(): if command not in self.serverCommands: self.serverCommands[command] = [] for data in dataList: for index, cmd in enumerate(self.serverCommands[command]): if cmd[1] < data[1]: self.serverCommands[command].insert(index, data) break else: self.serverCommands[command].append(data) self.log.debug("Module {module.name} is now fully loaded.", module=module) def unloadModule(self, moduleName): """ Unloads the loaded module with the given name. Raises ValueError if the module cannot be unloaded because it's a core module. """ self._unloadModule(moduleName, True) self.log.info("Unloaded module {module}.", module=moduleName) def _unloadModule(self, moduleName, fullUnload): unloadDeferreds = [] if moduleName not in self.loadedModules: return module = self.loadedModules[moduleName] if fullUnload and module.core: raise ValueError ("The module you're trying to unload is a core module.") moduleData = self._loadedModuleData[moduleName] d = module.unload() if d is not None: unloadDeferreds.append(d) if fullUnload: d = module.fullUnload() if d is not None: unloadDeferreds.append(d) for modeData in moduleData["channelmodes"]: if fullUnload: # Unset modes on full unload if modeData[1] == ModeType.Status: for channel in self.channels.itervalues(): removeFromChannel = [] for user, userData in channel.user.iteritems(): if modeData[0] in userData["status"]: removeFromChannel.append((False, modeData[0], user.uuid)) channel.setModes(removeFromChannel, self.serverID) elif modeData[1] == ModeType.List: for channel in self.channels.itervalues(): if modeData[0] in channel.modes: removeFromChannel = [] for paramData in channel.modes[modeData[0]]: removeFromChannel.append((False, modeData[0], paramData[0])) channel.setModes(removeFromChannel, self.serverID) else: for channel in self.channels.itervalues(): if modeData[0] in channel.modes: channel.setModes([(False, modeData[0], channel.modes[modeData[0]])], self.serverID) if modeData[1] == ModeType.Status: del self.channelStatuses[modeData[0]] del self.channelStatusSymbols[modeData[4]] self.channelStatusOrder.remove(modeData[0]) else: del self.channelModes[modeData[1]][modeData[0]] del self.channelModeTypes[modeData[0]] for modeData in moduleData["usermodes"]: if fullUnload: # Unset modes on full unload if modeData[1] == ModeType.List: for user in self.users.itervalues(): if modeData[0] in user.modes: removeFromUser = [] for paramData in user.modes[modeData[0]]: removeFromUser.append((False, modeData[0], paramData[0])) user.setModes(removeFromUser, self.serverID) else: for user in self.users.itervalues(): if modeData[0] in user.modes: user.setModes([(False, modeData[0], user.modes[modeData[0]])], self.serverID) del self.userModes[modeData[1]][modeData[0]] del self.userModeTypes[modeData[0]] for actionData in moduleData["actions"]: self.actions[actionData[0]].remove((actionData[2], actionData[1])) if not self.actions[actionData[0]]: del self.actions[actionData[0]] for commandData in moduleData["usercommands"]: self.userCommands[commandData[0]].remove((commandData[2], commandData[1])) if not self.userCommands[commandData[0]]: del self.userCommands[commandData[0]] for commandData in moduleData["servercommands"]: self.serverCommands[commandData[0]].remove((commandData[2], commandData[1])) if not self.serverCommands[commandData[0]]: del self.serverCommands[commandData[0]] del self.loadedModules[moduleName] del self._loadedModuleData[moduleName] if fullUnload: self.runActionStandard("moduleunload", module.name) if unloadDeferreds: deferList = DeferredList(unloadDeferreds) self._unloadingModules[moduleName] = deferList deferList.addCallback(self._removeFromUnloadingList, moduleName) return deferList def _removeFromUnloadingList(self, _, moduleName): del self._unloadingModules[moduleName] def reloadModule(self, moduleName): """ Reloads the module with the given name. Returns a DeferredList if the module unloads with one or more Deferreds. May raise ModuleLoadError if the module cannot be loaded. """ deferList = self._unloadModule(moduleName, False) if deferList is None: deferList = self.loadModule(moduleName) else: deferList.addCallback(lambda result: self.loadModule(moduleName)) return deferList def verifyConfig(self, config): # IRCd if "server_name" not in config: raise ConfigValidationError("server_name", "required item not found in configuration file.") if not isinstance(config["server_name"], basestring): raise ConfigValidationError("server_name", "value must be a string") if len(config["server_name"]) > 64: config["server_name"] = config["server_name"][:64] self.logConfigValidationWarning("server_name", "value is too long and has been truncated", config["server_name"]) if not re.match(r"^[a-zA-Z0-9.-]+\.[a-zA-Z0-9.-]+$", config["server_name"]): raise ConfigValidationError("server_name", "server name must look like a valid hostname.") if "server_id" in config: if not isinstance(config["server_id"], basestring): raise ConfigValidationError("server_id", "value must be a string") else: config["server_id"] = config["server_id"].upper() else: randFromName = random.Random(config["server_name"]) serverID = randFromName.choice(string.digits) + randFromName.choice(string.digits + string.ascii_uppercase) + randFromName.choice(string.digits + string.ascii_uppercase) config["server_id"] = serverID if len(config["server_id"]) != 3 or not config["server_id"].isalnum() or not config["server_id"][0].isdigit(): raise ConfigValidationError("server_id", "value must be a 3-character alphanumeric string starting with a number.") if "server_description" not in config: raise ConfigValidationError("server_description", "required item not found in configuration file.") if not isinstance(config["server_description"], basestring): raise ConfigValidationError("server_description", "value must be a string") if not config["server_description"]: raise ConfigValidationError("server_description", "value must not be an empty string") if len(config["server_description"]) > 255: config["server_description"] = config["server_description"][:255] self.logConfigValidationWarning("server_description", "value is too long and has been truncated", config["server_description"]) if "network_name" not in config: raise ConfigValidationError("network_name", "required item not found in configuration file.") if not isinstance(config["network_name"], basestring): raise ConfigValidationError("network_name", "value must be a string") if not config["network_name"]: raise ConfigValidationError("network_name", "value must not be an empty string") if " " in config["network_name"]: raise ConfigValidationError("network_name", "value cannot have spaces") if len(config["network_name"]) > 32: config["network_name"] = config["network_name"][:32] self.logConfigValidationWarning("network_name", "value is too long", config["network_name"]) if "bind_client" not in config: config["bind_client"] = [ "tcp:6667:interface={::}" ] self.logConfigValidationWarning("bind_client", "no default client binding specified", "[ \"tcp:6667:interface={::}\" ]") if not isinstance(config["bind_client"], list): raise ConfigValidationError("bind_client", "value must be a list") for bindDesc in config["bind_client"]: if not isinstance(bindDesc, basestring): raise ConfigValidationError("bind_client", "every entry must be a string") if "bind_server" not in config: config["bind_server"] = [] if not isinstance(config["bind_server"], list): raise ConfigValidationError("bind_server", "value must be a list") for bindDesc in config["bind_server"]: if not isinstance(bindDesc, basestring): raise ConfigValidationError("bind_server", "every entry must be a string") if "modules" not in config: config["modules"] = [] if not isinstance(config["modules"], list): raise ConfigValidationError("modules", "value must be a list") for module in config["modules"]: if not isinstance(module, basestring): raise ConfigValidationError("modules", "every entry must be a string") if "links" in config: if not isinstance(config["links"], dict): raise ConfigValidationError("links", "value must be a dictionary") for desc, server in config["links"].iteritems(): if not isinstance(desc, basestring): raise ConfigValidationError("links", "\"{}\" is an invalid server description".format(desc)) if not isinstance(server, dict): raise ConfigValidationError("links", "values for \"{}\" must be a dictionary".format(desc)) if "connect_descriptor" not in server: raise ConfigValidationError("links", "server \"{}\" must contain a \"connect_descriptor\" value".format(desc)) if "in_password" in server: if not isinstance(server["in_password"], basestring): config["links"][desc]["in_password"] = str(server["in_password"]) if "out_password" in server: if not isinstance(server["out_password"], basestring): config["links"][desc]["out_password"] = str(server["out_password"]) if "datastore_path" not in config: config["datastore_path"] = "data.db" if "storage_sync_interval" in config and not isinstance(config["storage_sync_interval"], int): raise ConfigValidationError(config["storage_sync_interval"], "invalid number") # Channels if "channel_name_length" in config: if not isinstance(config["channel_name_length"], int) or config["channel_name_length"] < 0: raise ConfigValidationError("channel_name_length", "invalid number") elif config["channel_name_length"] > 64: config["channel_name_length"] = 64 self.logConfigValidationWarning("channel_name_length", "value is too large", 64) if "modes_per_line" in config: if not isinstance(config["modes_per_line"], int) or config["modes_per_line"] < 0: raise ConfigValidationError("modes_per_line", "invalid number") elif config["modes_per_line"] > 20: config["modes_per_line"] = 20 self.logConfigValidationWarning("modes_per_line", "value is too large", 20) if "channel_listmode_limit" in config: if not isinstance(config["channel_listmode_limit"], int) or config["channel_listmode_limit"] < 0: raise ConfigValidationError("channel_listmode_limit", "invalid number") if config["channel_listmode_limit"] > 256: config["channel_listmode_limit"] = 256 self.logConfigValidationWarning("channel_listmode_limit", "value is too large", 256) # Users if "user_registration_timeout" in config: if not isinstance(config["user_registration_timeout"], int) or config["user_registration_timeout"] < 0: raise ConfigValidationError("user_registration_timeout", "invalid number") elif config["user_registration_timeout"] < 10: config["user_registration_timeout"] = 10 self.logConfigValidationWarning("user_registration_timeout", "timeout could be too short for clients to register in time", 10) if "user_ping_frequency" in config and (not isinstance(config["user_ping_frequency"], int) or config["user_ping_frequency"] < 0): raise ConfigValidationError("user_ping_frequency", "invalid number") if "hostname_length" in config: if not isinstance(config["hostname_length"], int) or config["hostname_length"] < 0: raise ConfigValidationError("hostname_length", "invalid number") elif config["hostname_length"] > 64: config["hostname_length"] = 64 self.logConfigValidationWarning("hostname_length", "value is too large", 64) elif config["hostname_length"] < 4: config["hostname_length"] = 4 self.logConfigValidationWarning("hostname_length", "value is too small", 4) if "ident_length" in config: if not isinstance(config["ident_length"], int) or config["ident_length"] < 0: raise ConfigValidationError("ident_length", "invalid number") elif config["ident_length"] > 12: config["ident_length"] = 12 self.logConfigValidationWarning("ident_length", "value is too large", 12) elif config["ident_length"] < 1: config["ident_length"] = 1 self.logConfigValidationWarning("ident_length", "value is too small", 1) if "gecos_length" in config: if not isinstance(config["gecos_length"], int) or config["gecos_length"] < 0: raise ConfigValidationError("gecos_length", "invalid number") elif config["gecos_length"] > 128: config["gecos_length"] = 128 self.logConfigValidationWarning("gecos_length", "value is too large", 128) elif config["gecos_length"] < 1: config["gecos_length"] = 1 self.logConfigValidationWarning("gecos_length", "value is too small", 1) if "user_listmode_limit" in config: if not isinstance(config["user_listmode_limit"], int) or config["user_listmode_limit"] < 0: raise ConfigValidationError("user_listmode_limit", "invalid number") if config["user_listmode_limit"] > 256: config["user_listmode_limit"] = 256 self.logConfigValidationWarning("user_listmode_limit", "value is too large", 256) # Servers if "server_registration_timeout" in config: if not isinstance(config["server_registration_timeout"], int) or config["server_registration_timeout"] < 0: raise ConfigValidationError("server_registration_timeout", "invalid number") elif config["server_registration_timeout"] < 10: config["server_registration_timeout"] = 10 self.logConfigValidationWarning("server_registration_timeout", "timeout could be too short for servers to register in time", 10) if "server_ping_frequency" in config and (not isinstance(config["server_ping_frequency"], int) or config["server_ping_frequency"] < 0): raise ConfigValidationError("server_ping_frequency", "invalid number") for module in self.loadedModules.itervalues(): module.verifyConfig(config) def logConfigValidationWarning(self, key, message, default): self.log.warn("Config value \"{configKey}\" is invalid ({message}); the value has been set to a default of \"{default}\".", configKey=key, message=message, default=default) def rehash(self): """ Reloads the configuration file and applies changes. """ self.log.info("Rehashing...") self.config.reload() d = self._unbindPorts() # Unbind the ports that are bound if d: # And then bind the new ones DeferredList(d).addCallback(lambda result: self._bindPorts()) else: self._bindPorts() try: self._logFilter.setLogLevelForNamespace("txircd", LogLevel.levelWithName(self.config["log_level"])) except (KeyError, InvalidLogLevelError): pass # If we can't set a new log level, we'll keep the old one for module in self.loadedModules.itervalues(): module.rehash() def _bindPorts(self): for bindDesc in self.config["bind_client"]: try: endpoint = serverFromString(reactor, unescapeEndpointDescription(bindDesc)) except ValueError as e: self.log.error(e) continue listenDeferred = endpoint.listen(UserFactory(self)) listenDeferred.addCallback(self._savePort, bindDesc, "client") listenDeferred.addErrback(self._logNotBound, bindDesc) for bindDesc in self.config["bind_server"]: try: endpoint = serverFromString(reactor, unescapeEndpointDescription(bindDesc)) except ValueError as e: self.log.error(e) continue listenDeferred = endpoint.listen(ServerListenFactory(self)) listenDeferred.addCallback(self._savePort, bindDesc, "server") listenDeferred.addErrback(self._logNotBound, bindDesc) def _unbindPorts(self): deferreds = [] for port in self.boundPorts.itervalues(): d = port.stopListening() if d: deferreds.append(d) return deferreds def _savePort(self, port, desc, portType): self.boundPorts[desc] = port self.log.debug("Bound endpoint '{endpointDescription}' for {portType} connections.", endpointDescription=desc, portType=portType) def _logNotBound(self, err, desc): self.log.error("Could not bind '{endpointDescription}': {errorMsg}", endpointDescription=desc, errorMsg=err) def createUUID(self): """ Gets the next UUID for a new client. """ newUUID = self.serverID + self._uid.next() while newUUID in self.users: # It'll take over 1.5 billion connections to loop around, but we still newUUID = self.serverID + self._uid.next() # want to be extra safe and avoid collisions self.log.debug("Generated new UUID {uuid}", uuid=newUUID) return newUUID def _genUID(self): uid = "AAAAAA" while True: yield uid uid = self._incrementUID(uid) def _incrementUID(self, uid): if uid == "Z": # The first character must be a letter return "A" # So wrap that around if uid[-1] == "9": return self._incrementUID(uid[:-1]) + "A" if uid[-1] == "Z": return uid[:-1] + "0" return uid[:-1] + chr(ord(uid[-1]) + 1) def pruneQuit(self): compareTime = now() - timedelta(seconds=10) remove = [] for uuid, timeQuit in self.recentlyQuitUsers.iteritems(): if timeQuit < compareTime: remove.append(uuid) for uuid in remove: del self.recentlyQuitUsers[uuid] remove = [] for serverID, timeQuit in self.recentlyQuitServers.iteritems(): if timeQuit < compareTime: remove.append(serverID) for serverID in remove: del self.recentlyQuitServers[serverID] def pruneChannels(self): removeChannels = [] for channel, remove in self.recentlyDestroyedChannels.iteritems(): if remove: removeChannels.append(channel) elif channel not in self.channels: self.recentlyDestroyedChannels[channel] = True for channel in removeChannels: del self.recentlyDestroyedChannels[channel] def generateISupportList(self): isupport = self.isupport_tokens.copy() statusSymbolOrder = "".join([self.channelStatuses[status][0] for status in self.channelStatusOrder]) isupport["CHANMODES"] = ",".join(["".join(modes) for modes in self.channelModes]) isupport["CHANNELLEN"] = self.config.get("channel_name_length", 64) isupport["NETWORK"] = self.config["network_name"] isupport["PREFIX"] = "({}){}".format("".join(self.channelStatusOrder), statusSymbolOrder) isupport["STATUSMSG"] = statusSymbolOrder isupport["USERMODES"] = ",".join(["".join(modes) for modes in self.userModes]) self.runActionStandard("buildisupport", isupport) isupportList = [] for key, val in isupport.iteritems(): if val is None: isupportList.append(key) else: isupportList.append("{}={}".format(key, val)) return isupportList def connectServer(self, name): """ Connect a server with the given name in the configuration. Returns a Deferred for the connection when we can successfully connect or None if the server is already connected or if we're unable to find information for that server in the configuration. """ if name in self.serverNames: return None if name not in self.config.get("links", {}): return None serverConfig = self.config["links"][name] endpoint = clientFromString(reactor, unescapeEndpointDescription(serverConfig["connect_descriptor"])) d = endpoint.connect(ServerConnectFactory(self)) d.addCallback(self._completeServerConnection, name) return d def _completeServerConnection(self, result, name): self.log.info("Connected to server {serverName}", serverName=name) self.runActionStandard("initiateserverconnection", result) def broadcastToServers(self, fromServer, command, *params, **kw): """ Broadcasts a message to all connected servers. The fromServer parameter should be the server from which the message came; if this server is the originating server, specify None for fromServer. """ for server in self.servers.itervalues(): if server.nextClosest == self.serverID and server != fromServer: server.sendMessage(command, *params, **kw) def _getActionModes(self, actionName, *params, **kw): users = [] channels = [] if "users" in kw: users = kw["users"] if "channels" in kw: channels = kw["channels"] functionList = [] if users: genericUserActionName = "modeactioncheck-user-{}".format(actionName) genericUserActionNameWithChannel = "modeactioncheck-user-withchannel-{}".format(actionName) for modeType in self.userModes: for mode, modeObj in modeType.iteritems(): if actionName not in modeObj.affectedActions: continue priority = modeObj.affectedActions[actionName] actionList = [] # Because Python doesn't properly capture variables in lambdas, we have to force static capture # by wrapping lambdas in more lambdas. # I wish Python wasn't this gross. for action in self.actions.get("modeactioncheck-user", []): actionList.append(((lambda action, actionName, mode: lambda user, *params: action[0](actionName, mode, user, *params))(action, actionName, mode), action[1])) for action in self.actions.get("modeactioncheck-user-withchannel", []): for channel in channels: actionList.append(((lambda action, actionName, mode, channel: lambda user, *params: action[0](actionName, mode, user, channel, *params))(action, actionName, mode, channel), action[1])) for action in self.actions.get(genericUserActionName, []): actionList.append(((lambda action, mode: lambda user, *params: action[0](mode, user, *params))(action, mode), action[1])) for action in self.actions.get(genericUserActionNameWithChannel, []): for channel in channels: actionList.append(((lambda action, mode, channel: lambda user, *params: action[0](mode, user, channel, *params))(action, mode, channel), action[1])) modeUserActionName = "modeactioncheck-user-{}-{}".format(mode, actionName) modeUserActionNameWithChannel = "modeactioncheck-user-withchannel-{}-{}".format(mode, actionName) for action in self.actions.get(modeUserActionNameWithChannel, []): for channel in channels: actionList.append(((lambda action, channel: lambda user, *params: action[0](user, channel, *params))(action, channel), action[1])) actionList = sorted(self.actions.get(modeUserActionName, []) + actionList, key=lambda action: action[1], reverse=True) applyUsers = [] for user in users: for action in actionList: param = action[0](user, *params) if param is not None: if param is not False: applyUsers.append((user, param)) break for user, param in applyUsers: functionList.append(((lambda modeObj, actionName, user, param: lambda *params: modeObj.apply(actionName, user, param, *params))(modeObj, actionName, user, param), priority)) if channels: genericChannelActionName = "modeactioncheck-channel-{}".format(actionName) genericChannelActionNameWithUser = "******".format(actionName) for modeType in self.channelModes: for mode, modeObj in modeType.iteritems(): if actionName not in modeObj.affectedActions: continue priority = modeObj.affectedActions[actionName] actionList = [] for action in self.actions.get("modeactioncheck-channel", []): actionList.append(((lambda action, actionName, mode: lambda channel, *params: action[0](actionName, mode, channel, *params))(action, actionName, mode), action[1])) for action in self.actions.get("modeactioncheck-channel-withuser", []): for user in users: actionList.append(((lambda action, actionName, mode, user: lambda channel, *params: action[0](actionName, mode, channel, user, *params))(action, actionName, mode, user), action[1])) for action in self.actions.get(genericChannelActionName, []): actionList.append(((lambda action, mode: lambda channel, *params: action[0](mode, channel, *params))(action, mode), action[1])) for action in self.actions.get(genericChannelActionNameWithUser, []): for user in users: actionList.append(((lambda action, mode, user: lambda channel, *params: action[0](mode, channel, user, *params))(action, mode, user), action[1])) modeChannelActionName = "modeactioncheck-channel-{}-{}".format(mode, actionName) modeChannelActionNameWithUser = "******".format(mode, actionName) for action in self.actions.get(modeChannelActionNameWithUser, []): for user in users: actionList.append(((lambda action, user: lambda channel, *params: action[0](channel, user, *params))(action, user), action[1])) actionList = sorted(self.actions.get(modeChannelActionName, []) + actionList, key=lambda action: action[1], reverse=True) applyChannels = [] for channel in channels: for action in actionList: param = action[0](channel, *params) if param is not None: if param is not False: applyChannels.append((channel, param)) break for channel, param in applyChannels: functionList.append(((lambda modeObj, actionName, channel, param: lambda *params: modeObj.apply(actionName, channel, param, *params))(modeObj, actionName, channel, param), priority)) return functionList def _getActionFunctionList(self, actionName, *params, **kw): functionList = self.actions.get(actionName, []) functionList = functionList + self._getActionModes(actionName, *params, **kw) return sorted(functionList, key=lambda action: action[1], reverse=True) def _combineActionFunctionLists(self, actionLists): """ Combines multiple lists of action functions into one. Assumes all lists are sorted. Takes a dict mapping action names to their action function lists. Returns a list in priority order (highest to lowest) of (actionName, function) tuples. """ fullActionList = [] for actionName, actionList in actionLists.iteritems(): insertPos = 0 for action in actionList: try: while fullActionList[insertPos][1] > action[1]: insertPos += 1 fullActionList.insert(insertPos, (actionName, action[0])) except IndexError: fullActionList.append((actionName, action[0])) insertPos += 1 return fullActionList def runActionStandard(self, actionName, *params, **kw): """ Calls all functions for a given action with the given parameters in priority order. Accepts the 'users' and 'channels' keyword arguments to determine which mode handlers should be included. """ actionList = self._getActionFunctionList(actionName, *params, **kw) for action in actionList: action[0](*params) def runActionUntilTrue(self, actionName, *params, **kw): """ Calls functions for a given action with the given parameters in priority order until one of them returns a true value. Returns True when one of the functions returned True. Accepts the 'users' and 'channels' keyword arguments to determine which mode handlers should be included. """ actionList = self._getActionFunctionList(actionName, *params, **kw) for action in actionList: if action[0](*params): return True return False def runActionUntilFalse(self, actionName, *params, **kw): """ Calls functions for a given action with the given parameters in priority order until one of them returns a false value. Returns True when one of the functions returned False. Accepts the 'users' and 'channels' keyword arguments to determine which mode handlers should be included. """ actionList = self._getActionFunctionList(actionName, *params, **kw) for action in actionList: if not action[0](*params): return True return False def runActionUntilValue(self, actionName, *params, **kw): """ Calls functions for a given action with the given parameters in priority order until one of them returns a non-None value. Returns the value returned by the function that returned a non-None value. Accepts the 'users' and 'channels' keyword arguments to determine which mode handlers should be included. """ actionList = self._getActionFunctionList(actionName, *params, **kw) for action in actionList: value = action[0](*params) if value is not None: return value return None def runActionFlagTrue(self, actionName, *params, **kw): """ Calls all functions for a given action with the given parameters in priority order. Returns True when one of the functions returns a true value. Accepts the 'users' and 'channels' keyword arguments to determine which mode handlers should be included. """ oneIsTrue = False actionList = self._getActionFunctionList(actionName, *params, **kw) for action in actionList: if action[0](*params): oneIsTrue = True return oneIsTrue def runActionFlagFalse(self, actionName, *params, **kw): """ Calls all functions for a given action with the given parameters in priority order. Returns True when one of the functions returns a false value. Accepts the 'users' and 'channels' keyword arguments to determine which mode handlers should be included. """ oneIsFalse = False actionList = self._getActionFunctionList(actionName, *params, **kw) for action in actionList: if action[0](*params): oneIsFalse = True return oneIsFalse def runActionProcessing(self, actionName, data, *params, **kw): """ Calls functions for a given action with the given parameters in priority order until the provided data is all processed (the data parameter becomes empty). Accepts 'users' and 'channels' keyword arguments to determine which mode handlers should be included. """ actionList = self._getActionFunctionList(actionName, data, *params, **kw) for action in actionList: action[0](data, *params) if not data: return def runActionProcessingMultiple(self, actionName, dataList, *params, **kw): """ Calls functions for a given action with the given parameters in priority order until the provided data is all processed (all of the data structures in the dataList parameter become empty). Accepts 'users' and 'channels' keyword arguments to determine which mode handlers should be included. """ paramList = dataList + params actionList = self._getActionFunctionList(actionName, *paramList, **kw) for action in actionList: action[0](*paramList) for data in dataList: if data: break else: return def runComboActionStandard(self, actionList, **kw): """ Calls all functions for the given actions with the given parameters in priority order. Actions are specifed as a list of tuples: [ ("action1", param1, param2, ...), ("action2", param1, param2, ...) ] Accepts 'users' and 'channels' keyword arguments to determine which mode handlers should be included. """ actionFuncLists = {} actionParameters = {} for action in actionList: parameters = action[1:] actionParameters[action[0]] = parameters actionFuncLists[action[0]] = self._getActionFunctionList(action[0], *parameters, **kw) funcList = self._combineActionFunctionLists(actionFuncLists) for actionName, actionFunc in funcList: actionFunc(*actionParameters[actionName]) def runComboActionUntilTrue(self, actionList, **kw): """ Calls functions for the given actions with the given parameters in priority order until one of the functions returns a true value. Actions are specified as a list of tuples: [ ("action1", param1, param2, ...), ("action2", param1, param2, ...) ] Returns True if one of the functions returned a true value. Accepts 'users' and 'channels' keyword arguments to determine which mode handlers should be included. """ actionFuncLists = {} actionParameters = {} for action in actionList: parameters = action[1:] actionParameters[action[0]] = parameters actionFuncLists[action[0]] = self._getActionFunctionList(action[0], *parameters, **kw) funcList = self._combineActionFunctionLists(actionFuncLists) for actionName, actionFunc in funcList: if actionFunc(*actionParameters[actionName]): return True return False def runComboActionUntilFalse(self, actionList, **kw): """ Calls functions for the given actions with the given parameters in priority order until one of the functions returns a false value. Actions are specified as a list of tuples: [ ("action1", param1, param2, ...), ("action2", param1, param2, ...) ] Returns True if one of the functions returned a false value. Accepts 'users' and 'channels' keyword arguments to determine which mode handlers should be included. """ actionFuncLists = {} actionParameters = {} for action in actionList: parameters = action[1:] actionParameters[action[0]] = parameters actionFuncLists[action[0]] = self._getActionFunctionList(action[0], *parameters, **kw) funcList = self._combineActionFunctionLists(actionFuncLists) for actionName, actionFunc in funcList: if not actionFunc(*actionParameters[actionName]): return True return False def runComboActionUntilValue(self, actionList, **kw): """ Calls functions for the given actions with the given parameters in priority order until one of the functions returns a non-None value. Actions are specified as a list of tuples: [ ("action1", param1, param2, ...), ("action2", param1, param2, ...) ] Returns the value returned by the function that returned a non-None value. Accepts 'users' and 'channels' keyword arguments to determine which mode handlers should be included. """ actionFuncLists = {} actionParameters = {} for action in actionList: parameters = action[1:] actionParameters[action[0]] = parameters actionFuncLists[action[0]] = self._getActionFunctionList(action[0], *parameters, **kw) funcList = self._combineActionFunctionLists(actionFuncLists) for actionName, actionFunc in funcList: value = actionFunc(*actionParameters[actionName]) if value is not None: return value return None def runComboActionFlagTrue(self, actionList, **kw): """ Calls all functions for the given actions with the given parameters in priority order. Actions are specified as a list of tuples: [ ("action1", param1, param2, ...), ("action2", param1, param2, ...) ] Returns True if any of the functions called returned a true value. Accepts 'users' and 'channels' keyword arguments to determine which mode handlers should be included. """ actionFuncLists = {} actionParameters = {} for action in actionList: parameters = action[1:] actionParameters[action[0]] = parameters actionFuncLists[action[0]] = self._getActionFunctionList(action[0], *parameters, **kw) funcList = self._combineActionFunctionLists(actionFuncLists) oneIsTrue = False for actionName, actionFunc in funcList: if actionFunc(*actionParameters[actionName]): oneIsTrue = True return oneIsTrue def runComboActionFlagFalse(self, actionList, **kw): """ Calls all functions for the given actions with the given parameters in priority order. Actions are specified as a list of tuples: [ ("action1", param1, param2, ...), ("action2", param1, param2, ...) ] Returns True if any of the functions called returned a false value. Accepts 'users' and 'channels' keyword arguments to determine which mode handlers should be included. """ actionFuncLists = {} actionParameters = {} for action in actionList: parameters = action[1:] actionParameters[action[0]] = parameters actionFuncLists[action[0]] = self._getActionFunctionList(action[0], *parameters, **kw) funcList = self._combineActionFunctionLists(actionFuncLists) oneIsFalse = False for actionName, actionFunc in funcList: if not actionFunc(*actionParameters[actionName]): oneIsFalse = True return oneIsFalse def runComboActionProcessing(self, data, actionList, **kw): """ Calls functions for the given actions with the given parameters in priority order until the data given has been processed (the data parameter becomes empty). Actions are specified as a list of tuples: [ ("action1", param1, param2, ...), ("action2", param1, param2, ...) ] Accepts 'users' and 'channels' keyword arguments to determine which mode handlers should be included. """ actionFuncLists = {} actionParameters = {} for action in actionList: parameters = [data] + action[1:] actionParameters[action[0]] = parameters actionFuncLists[action[0]] = self._getActionFunctionList(action[0], *parameters, **kw) funcList = self._combineActionFunctionLists(actionFuncLists) for actionName, actionFunc in funcList: actionFunc(*actionParameters[actionName]) if not data: break def runComboActionProcessingMultiple(self, dataList, actionList, **kw): """ Calls functions for the given actions with the given parameters in priority order until the data given has been processed (all the data items in the dataList parameter become empty). Actions are specified as a list of tuples: [ ("action1", param1, param2, ...), ("action2", param1, param2, ...) ] Accepts 'users' and 'channels' keyword arguments to determine which mode handlers should be included. """ actionFuncLists = {} actionParameters = {} for action in actionList: parameters = dataList + action[1:] actionParameters[action[0]] = parameters actionFuncLists[action[0]] = self._getActionFunctionList(action[0], *parameters, **kw) funcList = self._combineActionFunctionLists(actionFuncLists) for actionName, actionFunc in funcList: actionFunc(*actionParameters[actionName]) for data in dataList: if data: break else: return
class FSProtocol(basic.LineReceiver): class State(Enum): IDLE = 0 POLLING = 1 POLLED = 2 RAW = 3 def __init__(self): self.log = Logger() self.setIdleState() def setIdleState(self): self.my_state = self.State.IDLE self.wait_for_list = None self.file_transfer_peer = None self.setLineMode() def connectionMade(self): self.log.info("New connection established: {peer!r}, {pid!s}", peer=self.transport.getPeer(), pid=id(self)) self.factory.protocols.append(self) self.transport.write(b'Hello form server\r\n') def send(self, command, argument): if command != "": command += " " self.transport.write(command.upper().encode() + argument.encode() + b'\r\n') # self.log.info("Sending to {peer!r}: {command!s} {argument!s}", # command=command, argument=argument) def doReject(self, line): self.log.error("We had to say no to {peer!r}: {line}", peer=self.transport.getPeer(), line=line) self.transport.write(b'REJECT ' + line.encode() + b'\r\n') # def doRequest(self, regex): # self.log.debug("server asks for files: {regex!s}", regex=regex) # self.transport.write(b'REQUEST ' + regex.encode() + b'\r\n') def requestFiles(self, regex): ''' Called by factory, if one of users requested a search ''' if self.my_state != self.State.IDLE or self.wait_for_list is not None: return defer.fail(RuntimeError("Connection is busy")) # Go to POLLED state, so no one will interrupt us self.my_state = self.State.POLLED self.send("REQUEST", regex) def debuglogc(data): self.log.debug("wait_for_list:requestFiles:callback: {data!r}", data=data) return data def debugloge(err): self.log.debug("wait_for_list:requestFiles:errback: {err!r}", err=err) return err self.wait_for_list = defer.Deferred().addCallbacks( debuglogc, debugloge) # We create Deferred, that will be fired as the response comes from user return self.wait_for_list def trySetStateIdle(self): # Some operation has finished # Check if there is file request # if len(self.outer_req) > 0: # # Set state # self.state = self.State.POLLED # # Ask peer # regex,d = self.outer_req[0] # self.transport.write("TODO some message" + regex + b'\r\n') pass def handleFind(self, regex): ''' Hanles the FIND request from user ''' self.my_state = self.State.POLLING # The factory will ask all the others about our regexp # The result here is the string d = self.factory.getFiles(self, regex) def response(ans): self.log.info("FIND command returns next {filelist!s}", filelist=ans) self.send('RESPONSE', ans) def error(e): self.doReject(e.getErrorMessage()) def idle(ignore): self.my_state = self.State.IDLE d.addCallbacks(response, error) d.addBoth(idle) def lineReceived(self, line): self.log.info("lineReceived: {peer!r} got message: {line!s}", peer=self.transport.getPeer(), line=line) line = line.decode() data = cleanInput(line) if len(data) == 0 or data == '': self.log.info("Oh, never mind, it's gone") self.doReject("Seems like a bunch of tabs") return command = data[0].upper() argument = data[1] if len(data) > 1 else None # Analize the command if command == 'FIND': # Find files from other users if self.my_state != self.State.IDLE: self.doReject("Another operation is in run") return if argument is None: self.doReject("Regex expected") return self.handleFind(argument) elif command == 'FILES': # Polled user returns list of files here if self.my_state != self.State.POLLED: self.doReject( "Got list of files, but there is no request for them") return # self.log.debug("Got list of files from {peer!r}", peer=self.transport.getPeer()) if argument is None: self.wait_for_list.errback( RuntimeError("This peer has no such files")) else: # Return self id and list of files self.wait_for_list.callback(':'.join([str(id(self)), argument])) # Cleanup self.wait_for_list = None self.my_state = self.State.IDLE elif command == 'GET': # User asks for file ''' This is a routine of connection, whitch needs file ''' # Check if we ready if self.my_state != self.State.IDLE: self.doReject("Can not give you file at the moment") return # Get arguments if argument is None: self.doReject("Expected filename for GET operation") return try: pid, filename = argument.split(':', 1) except (IndexError): self.doReject("not valid filename") return # Find the peer peer = self.factory.findConnById(pid) # Check if we are ready if peer is None: self.doReject("Sorry, the peer seems afk") return if peer.my_state != peer.State.IDLE: self.doReject("Sorry, the peer seems busy") return # Ready to begin, don't interrupt us please peer.my_state = peer.State.RAW self.my_state = self.State.RAW peer.file_transfer_peer = self self.file_transfer_peer = peer peer.send("OBTAIN", filename) self.send("RAW", filename) self.setRawMode() peer.setRawMode() elif command == 'ALICE': # User remembers how this project started self.send('', "nice girl") else: self.doReject("not a protocol command: %s" % command) def rawDataReceived(self, data): ''' Should be called, if user sends data to peer in response ''' if self.file_transfer_peer is None: self.log.warning("RECIEVING RAW DATA WITHOUT PEER TO RESEND") return peer = self.file_transfer_peer self.log.info("RAW: resending data: %d KB" % len(data)) peer.transport.write(data) if not data.endswith(b'\r\n'): # Transmission not done yet return self.log.info("RAW: Transmission complete") peer.setIdleState() self.setIdleState() def connectionLost(self, reason): self.log.info("Connection closed: {peer!r}, {reason!r}", peer=self.transport.getPeer(), reason=reason) self.factory.protocols.remove(self)
class WebSocketDispatcherProtocol(Protocol): """Protocol implementing ShinySDR's WebSocket service. This protocol's transport should be a txWS WebSocket transport. """ def __init__(self, caps, subscription_context): self.__log = Logger() self.__subscription_context = subscription_context self._caps = caps self._seenValues = {} self.inner = None def dataReceived(self, data): """Twisted Protocol implementation. Additionally, txWS takes no care with exceptions here, so we catch and log.""" # pylint: disable=broad-except try: if self.inner is None: # To work around txWS's lack of a notification when the URL is available, all clients send a dummy first message. self.__dispatch_url() else: self.inner.dataReceived(data) except Exception: self.__log.failure('Error processing incoming WebSocket message') def __dispatch_url(self): self.__log.info('Stream connection to {url}', url=self.transport.location) _scheme, _netloc, path_bytes, _params, query_bytes, _fragment = urlparse( bytes_or_ascii(self.transport.location)) # py2/3: unquote returns str in either version but we want Unicode path = [ six.text_type(urllib.parse.unquote(x)) for x in path_bytes.split(b'/') ] assert path[0] == '' path[0:1] = [] cap_string = path[0] if cap_string in self._caps: root_object = self._caps[cap_string] path[0:1] = [] else: raise Exception('Unknown cap') # TODO better error reporting if path == [AUDIO_STREAM_PATH_ELEMENT]: options = parse_audio_stream_options(parse_qs(query_bytes, 1)) self.inner = AudioStreamInner(the_reactor, self.__send, root_object, options.sample_rate) elif len(path) >= 1 and path[0] == CAP_OBJECT_PATH_ELEMENT: # note _lookup_block may throw. TODO: Better error reporting root_object = _lookup_block(root_object, path[1:]) self.inner = StateStreamInner( self.__send, root_object, path_bytes.decode('utf-8'), self.__subscription_context ) # note reuse of WS path as HTTP path; probably will regret this else: raise Exception('Unknown path: %r' % (path, )) def connectionMade(self): """twisted Protocol implementation""" self.transport.setBinaryMode(True) # Unfortunately, txWS calls this too soon for transport.location to be available def connectionLost(self, reason): # pylint: disable=signature-differs """twisted Protocol implementation""" if self.inner is not None: self.inner.connectionLost(reason) def __send(self, message, safe_to_drop=False): if len(self.transport.transport.dataBuffer) > 1000000: # TODO: condition is horrible implementation-diving kludge # Don't accumulate indefinite buffer if we aren't successfully getting it onto the network. # TODO: There are no tests of this mechanism if safe_to_drop: self.__log.warn('Dropping data going to stream {url}', url=self.transport.location) else: self.__log.error( 'Dropping connection due to too much data on stream {url}', url=self.transport.location) self.transport.close(reason='Too much data buffered') else: self.transport.write(message)
class Rest(object): def __init__( self, host='https://developer-api.nest.com', token=None, event_handler=None, net_type='lan'): self.log = Logger() self.host = host self.token = token self.event_handler = event_handler self.pool = HTTPConnectionPool(reactor, persistent=True) self.loc = None self.reconnect = False self.fail_count = 0 if event_handler: self.reconnect = True d = self.request(headers={'User-Agent': ['onDemand Rest Client'], 'Accept': ['text/event-stream']}) d.addCallback(self.on_disconnect) def __getattr__(self, name): try: super(Rest, self).__getattr__(name) except AttributeError: return RestCall(self, name) def on_disconnect(self, reason): if not reason: reason = {'reason': 'no_message'} self.log.critical( 'disconnected: {reason}', reason=reason['reason']) if self.fail_count > 10: self.log.error('Max error count reached, aborting connection') def test_connectivity(count): if self.fail_count == count: self.fail_count = 0 self.fail_count += 1 c = self.fail_count reactor.callLater(10, test_connectivity, c) # @UndefinedVariable if self.reconnect: d = self.request(headers={'User-Agent': ['onDemand Rest Client'], 'Accept': ['text/event-stream']}) d.addCallback(self.on_disconnect) def request(self, method='GET', path='', headers={'User-Agent': ['onDemand/1.0 (Rest_Client)'], 'Accept': ['application/json']}, body=None): data = None if self.loc: host = '/'.join((self.loc, path)) else: host = '/'.join((self.host, path)) if self.token: host += '?auth=' + self.token if body: headers.update({'Content-Type': ['application/json']}) data = FileBodyProducer(StringIO(json.dumps(body))) agent = RedirectAgent(Agent(reactor, pool=self.pool)) d = agent.request(method, host, Headers(headers), data) def cbFail(fail): if hasattr(fail.value, 'response'): if hasattr(fail.value.response, 'code'): if fail.value.response.code == 307: loc = fail.value.response.headers.getRawHeaders( 'location') new = urlparse(loc[0]) newhost = '://'.join((new.scheme, new.netloc)) if newhost == self.host: self.loc = None else: self.loc = newhost self.log.debug('redirect: %s' % self.loc) data = FileBodyProducer(StringIO(json.dumps(body))) d = agent.request( method, loc[0], Headers(headers), data) d.addCallbacks(cbRequest, cbFail) return d elif fail.value.response.code == 404 and self.loc: self.loc = None host = '/'.join((self.host, path)) if self.token: host += '?auth=' + self.token d = self.request(method, host, Headers(headers), body) d.addCallbacks(cbRequest, cbFail) return d else: print(dir(fail.value)) print(fail.value.message) print(fail.value.args) self.log.error('unhandled failure: %s -- %s' % ( fail.value.message, fail.value)) def cbRequest(response): # print 'Response version:', response.version # print 'Response code:', response.code # print 'Response phrase:', response.phrase # print 'Response headers:' # print pformat(list(response.headers.getAllRawHeaders())) finished = Deferred() response.deliverBody(RestHandle(finished, self.event_handler)) return finished d.addCallbacks(cbRequest, cbFail) return d
class PostgresListenerService(Service, object): """Listens for NOTIFY messages from postgres. A new connection is made to postgres with the isolation level of autocommit. This connection is only used for listening for notifications. Any query that needs to take place because of a notification should use its own connection. This class runs inside of the reactor. Any long running action that occurrs based on a notification should defer its action to a thread to not block the reactor. :ivar connection: A database connection within one of Django's wrapper. :ivar connectionFileno: The fileno of the underlying database connection. :ivar connecting: a :class:`Deferred` while connecting, `None` at all other times. :ivar disconnecting: a :class:`Deferred` while disconnecting, `None` at all other times. """ # Seconds to wait to handle new notifications. When the notifications set # is empty it will wait this amount of time to check again for new # notifications. HANDLE_NOTIFY_DELAY = 0.5 CHANNEL_REGISTRAR_DELAY = 0.5 def __init__(self, alias="default"): self.alias = alias self.listeners = defaultdict(list) self.autoReconnect = False self.connection = None self.connectionFileno = None self.notifications = set() self.notifier = task.LoopingCall(self.handleNotifies) self.notifierDone = None self.connecting = None self.disconnecting = None self.registeredChannels = set() self.channelRegistrar = task.LoopingCall( lambda: ensureDeferred(self.registerChannels())) self.channelRegistrarDone = None self.log = Logger(__name__, self) self.events = EventGroup("connected", "disconnected") def startService(self): """Start the listener.""" super(PostgresListenerService, self).startService() self.autoReconnect = True return self.tryConnection() def stopService(self): """Stop the listener.""" super(PostgresListenerService, self).stopService() self.autoReconnect = False return self.loseConnection() def connected(self): """Return True if connected.""" if self.connection is None: return False if self.connection.connection is None: return False return self.connection.connection.closed == 0 def logPrefix(self): """Return nice name for twisted logging. This is required to satisfy `IReadDescriptor`, which inherits from `ILoggingContext`. """ return self.log.namespace def isSystemChannel(self, channel): """Return True if channel is a system channel.""" return channel.startswith("sys_") def doRead(self): """Poll the connection and process any notifications.""" try: self.connection.connection.poll() except Exception: # If the connection goes down then `OperationalError` is raised. # It contains no pgcode or pgerror to identify the reason so no # special consideration can be made for it. Hence all errors are # treated the same, and we assume that the connection is broken. # # We do NOT return a failure, which would signal to the reactor # that the connection is broken in some way, because the reactor # will end up removing this instance from its list of selectables # but not from its list of readable fds, or something like that. # The point is that the reactor's accounting gets muddled. Things # work correctly if we manage the disconnection ourselves. # self.loseConnection(Failure(error.ConnectionLost())) else: # Add each notify to to the notifications set. This removes # duplicate notifications when one entity in the database is # updated multiple times in a short interval. Accumulating # notifications and allowing the listener to pick them up in # batches is imperfect but good enough, and simple. notifies = self.connection.connection.notifies if len(notifies) != 0: for notify in notifies: if self.isSystemChannel(notify.channel): # System level message; pass it to the registered # handler immediately. if notify.channel in self.listeners: # Be defensive in that if a handler does not exist # for this channel then the channel should be # unregisted and removed from listeners. if len(self.listeners[notify.channel]) > 0: handler = self.listeners[notify.channel][0] handler(notify.channel, notify.payload) else: self.unregisterChannel(notify.channel) del self.listeners[notify.channel] else: # Unregister the channel since no listener is # registered for this channel. self.unregisterChannel(notify.channel) else: # Place non-system messages into the queue to be # processed. self.notifications.add( (notify.channel, notify.payload)) # Delete the contents of the connection's notifies list so # that we don't process them a second time. del notifies[:] def fileno(self): """Return the fileno of the connection.""" return self.connectionFileno def startReading(self): """Add this listener to the reactor.""" self.connectionFileno = self.connection.connection.fileno() reactor.addReader(self) def stopReading(self): """Remove this listener from the reactor.""" try: reactor.removeReader(self) except IOError as error: # ENOENT here means that the fd has already been unregistered # from the underlying poller. It is as yet unclear how we get # into this state, so for now we ignore it. See epoll_ctl(2). if error.errno != ENOENT: raise finally: self.connectionFileno = None def register(self, channel, handler): """Register listening for notifications from a channel. When a notification is received for that `channel` the `handler` will be called with the action and object id. """ handlers = self.listeners[channel] if self.isSystemChannel(channel) and len(handlers) > 0: # A system can only be registered once. This is because the # message is passed directly to the handler and the `doRead` # method does not wait for it to finish if its a defer. This is # different from normal handlers where we will call each and wait # for all to resolve before continuing to the next event. raise PostgresListenerRegistrationError( "System channel '%s' has already been registered." % channel) else: handlers.append(handler) self.runChannelRegistrar() def unregister(self, channel, handler): """Unregister listening for notifications from a channel. `handler` needs to be same handler that was registered. """ if channel not in self.listeners: raise PostgresListenerUnregistrationError( "Channel '%s' is not registered with the listener." % channel) handlers = self.listeners[channel] if handler in handlers: handlers.remove(handler) else: raise PostgresListenerUnregistrationError( "Handler is not registered on that channel '%s'." % channel) if len(handlers) == 0: # Channels have already been registered. Unregister the channel. del self.listeners[channel] self.runChannelRegistrar() @synchronous def createConnection(self): """Create new database connection.""" db = connections.databases[self.alias] backend = load_backend(db["ENGINE"]) return backend.DatabaseWrapper(db, self.alias, allow_thread_sharing=True) @synchronous def startConnection(self): """Start the database connection.""" self.connection = self.createConnection() self.connection.connect() self.connection.set_autocommit(True) @synchronous def stopConnection(self): """Stop database connection.""" # The connection is often in an unexpected state here -- for # unexplained reasons -- so be careful when unpealing layers. connection_wrapper, self.connection = self.connection, None if connection_wrapper is not None: connection = connection_wrapper.connection if connection is not None and not connection.closed: connection_wrapper.commit() connection_wrapper.close() def tryConnection(self): """Keep retrying to make the connection.""" if self.connecting is None: if self.disconnecting is not None: raise RuntimeError( "Cannot attempt to make new connection before " "pending disconnection has finished.") def cb_connect(_): self.log.info("Listening for database notifications.") def eb_connect(failure): self.log.error( "Unable to connect to database: {error}", error=failure.getErrorMessage(), ) if failure.check(CancelledError): return failure elif self.autoReconnect: return deferLater(reactor, 3, connect) else: return failure def connect(interval=self.HANDLE_NOTIFY_DELAY): d = deferToThread(self.startConnection) d.addCallback(callOut, self.runChannelRegistrar) d.addCallback(lambda result: self.channelRegistrarDone) d.addCallback(callOut, self.events.connected.fire) d.addCallback(callOut, self.startReading) d.addCallback(callOut, self.runHandleNotify, interval) # On failure ensure that the database connection is stopped. d.addErrback(callOut, deferToThread, self.stopConnection) d.addCallbacks(cb_connect, eb_connect) return d def done(): self.connecting = None self.connecting = connect().addBoth(callOut, done) return self.connecting def loseConnection(self, reason=Failure(error.ConnectionDone())): """Request that the connection be dropped.""" if self.disconnecting is None: self.registeredChannels.clear() d = self.disconnecting = Deferred() d.addBoth(callOut, self.stopReading) d.addBoth(callOut, self.cancelChannelRegistrar) d.addBoth(callOut, self.cancelHandleNotify) d.addBoth(callOut, deferToThread, self.stopConnection) d.addBoth(callOut, self.connectionLost, reason) def done(): self.disconnecting = None d.addBoth(callOut, done) if self.connecting is None: # Already/never connected: begin shutdown now. self.disconnecting.callback(None) else: # Still connecting: cancel before disconnect. self.connecting.addErrback(suppress, CancelledError) self.connecting.chainDeferred(self.disconnecting) self.connecting.cancel() return self.disconnecting def connectionLost(self, reason): """Reconnect when the connection is lost.""" self.connection = None if reason.check(error.ConnectionDone): self.log.debug("Connection closed.") elif reason.check(error.ConnectionLost): self.log.debug("Connection lost.") else: self.log.failure("Connection lost.", reason) if self.autoReconnect: reactor.callLater(3, self.tryConnection) self.events.disconnected.fire(reason) def registerChannel(self, channel): """Register the channel.""" with closing(self.connection.cursor()) as cursor: if self.isSystemChannel(channel): # This is a system channel so listen only called once. cursor.execute("LISTEN %s;" % channel) else: # Not a system channel so listen called once for each action. for action in sorted(map_enum(ACTIONS).values()): cursor.execute("LISTEN %s_%s;" % (channel, action)) def unregisterChannel(self, channel): """Unregister the channel.""" with closing(self.connection.cursor()) as cursor: if self.isSystemChannel(channel): # This is a system channel so unlisten only called once. cursor.execute("UNLISTEN %s;" % channel) else: # Not a system channel so unlisten called once for each action. for action in sorted(map_enum(ACTIONS).values()): cursor.execute("UNLISTEN %s_%s;" % (channel, action)) async def registerChannels(self): """Listen/unlisten to channels that were registered/unregistered. When a call to register() or unregister() is made, the listeners dict is updated, and the keys of that dict represents all the channels that we should listen to. The service keeps a list of channels that it already listens to in the registeredChannels dict. We issue a call to postgres to listen to all channels that are in listeners but not in registeredChannels, and a call to unlisten for all channels that are in registeredChannels but not in listeners. """ to_register = set(self.listeners.keys()).difference( self.registeredChannels) to_unregister = self.registeredChannels.difference( set(self.listeners.keys())) # If there's nothing to do, we can stop the loop. If there is # any work to be done, we do the work, and then check # whether we should stop at the beginning of the next loop # iteration. The reason is that every time we yield, another # deferred might call register() or unregister(). if not to_register and not to_unregister: self.channelRegistrar.stop() else: for channel in to_register: await deferToThread(self.registerChannel, channel) self.registeredChannels.add(channel) for channel in to_unregister: await deferToThread(self.unregisterChannel, channel) self.registeredChannels.remove(channel) def convertChannel(self, channel): """Convert the postgres channel to a registered channel and action. :raise PostgresListenerNotifyError: When {channel} is not registered or {action} is not in `ACTIONS`. """ channel, action = channel.split("_", 1) if channel not in self.listeners: raise PostgresListenerNotifyError( "%s is not a registered channel." % channel) if action not in map_enum(ACTIONS).values(): raise PostgresListenerNotifyError("%s action is not supported." % action) return channel, action def runChannelRegistrar(self): """Start the loop for listening to channels in postgres. It will only start if the service is connected to postgres. """ if self.connection is not None and not self.channelRegistrar.running: self.channelRegistrarDone = self.channelRegistrar.start( self.CHANNEL_REGISTRAR_DELAY, now=True) def cancelChannelRegistrar(self): """Stop the loop for listening to channels in postgres.""" if self.channelRegistrar.running: self.channelRegistrar.stop() return self.channelRegistrarDone else: return succeed(None) def runHandleNotify(self, delay=0, clock=reactor): """Defer later the `handleNotify`.""" if not self.notifier.running: self.notifierDone = self.notifier.start(delay, now=False) def cancelHandleNotify(self): """Cancel the deferred `handleNotify` call.""" if self.notifier.running: self.notifier.stop() return self.notifierDone else: return succeed(None) def handleNotifies(self, clock=reactor): """Process all notify message in the notifications set.""" def gen_notifications(notifications): while len(notifications) != 0: yield notifications.pop() return task.coiterate( self.handleNotify(notification, clock=clock) for notification in gen_notifications(self.notifications)) def handleNotify(self, notification, clock=reactor): """Process a notify message in the notifications set.""" channel, payload = notification try: channel, action = self.convertChannel(channel) except PostgresListenerNotifyError: # Log the error and continue processing the remaining # notifications. self.log.failure("Failed to convert channel {channel!r}.", channel=channel) else: defers = [] handlers = self.listeners[channel] # XXX: There could be an arbitrary number of listeners. Should we # limit concurrency here? Perhaps even do one at a time. for handler in handlers: d = defer.maybeDeferred(handler, action, payload) d.addErrback(lambda failure: self.log.failure( "Failure while handling notification to {channel!r}: " "{payload!r}", failure, channel=channel, payload=payload, )) defers.append(d) return defer.DeferredList(defers)
class Discord(IRCClient): nickname = "discord" realname = "Discord" username = "******" versionName = "Discord" versionNum = "0.01" magicFile = "true.txt" def __init__(self, accessList): self.logger = Logger(observer=textFileLogObserver(sys.stdout)) self.accessList = [nick.lower() for nick in accessList] if not os.path.exists(self.magicFile): self.logger.info("Creating magic file") try: with open(self.magicFile, "a"): pass except Exception as ex: self.logger.error("Unable to create magic file! {0}".format( ex.message)) reactor.stop() self.markovGenerator = pymarkov.MarkovChainGenerator(self.magicFile) self.channels = [] self.channelPhrasers = {} self.logger.debug("Discord initialized") # Maybe add hook/plugin system here? self.commands = Commands.Commands(self) def removeChannel(self, channel): try: self.channels.remove(channel) self.channelPhrasers[channel].stop() del self.channelPhrasers[channel] except: self.logger.error("Error removing {channel} from collection", channel=channel) def insertPhrase(self, phrase): try: with open(self.magicFile, "a") as magicFile: magicFile.write("{0}\n".format(phrase)) try: file, ext = os.path.splitext(self.magicFile) os.remove("{0}-pickled{1}".format(file, ext)) # Simply re-populating the dictionary isn't enough for some reason self.markovGenerator = pymarkov.MarkovChainGenerator( self.magicFile, 2) except IOError as ex: self.logger.error("Unable to delete pickled file. {0}".format( ex.message)) except Exception as ex: self.logger.error( "Unable to insert phrase into magic file! {0}".format( ex.message)) def kickedFrom(self, channel, kicker, message): self.removeChannel(channel) self.logger.info("Kicked from {channel} by {kicker}", channel=channel, kicker=kicker) def left(self, channel): self.removeChannel(channel) self.logger.info("Left {channel}", channel=channel) def handleMessage(self, user, channel, message): senderNickname = user.split("!")[0] if message.startswith("~reload") and senderNickname in self.accessList: self.logger.info("Reloading commands module") self.say(channel, "Reloading.") try: commandsModule = reload(Commands) self.commands = commandsModule.Commands(self) except Exception as ex: self.say( channel, "Failed to load commands module - {0}".format(ex.message)) elif message.startswith("~"): # Don't log commands to the brain commandMessage = message[1:] self.commands.handleCommand(user, channel, commandMessage) else: self.logger.info("Adding {message!r} to brain", message=message) # Avoid storing anything with the bot's name in it brainMessage = message.strip(self.nickname) self.insertPhrase(brainMessage) try: randomPhrase = self.generateSentence() if self.nickname in message and channel.startswith( "#") and self.channelPhrasers[channel].running: phrase = "{0}, {1}".format(senderNickname, randomPhrase) self.say(channel, phrase) elif channel == self.nickname: self.logger.debug("Sending message to {nickname}", nickname=senderNickname) self.msg(senderNickname, randomPhrase) else: pass except IndexError as generationError: self.logger.error(generationError.message) def privmsg(self, user, channel, message): self.logger.info("Received message from {user} in {channel}", user=user, channel=channel) # deferToThread(self.handleMessage, user, channel, message) self.handleMessage(user, channel, message) def signedOn(self): self.logger.info("Signed on") self.join("#bots") def joined(self, channel): self.channels.append(channel) self.logger.info("Joined channel {channel!r}", channel=channel) channelPhraser = LoopingCall(self.sayRandomPhrase, channel) reactor.callLater(2, channelPhraser.start, 600) self.channelPhrasers[channel] = channelPhraser def generateSentence(self): try: sentence = self.markovGenerator.generate_sentence() sentence = sentence.strip("<{0}>".format(self.nickname)) sentence = sentence.strip(self.nickname) return sentence except (IndexError, ValueError) as ex: self.logger.error(ex.message) def sayRandomPhrase(self, channel): sentence = self.generateSentence() self.say(channel, sentence)