class LoggingProcessProtocol(ProcessProtocol, object):
"""
A ProcessProtocol that logs all output to a file
"""
def __init__(self, commandname, maxbackups=3):
log_name = commandname + ".log"
log_dir = os.path.join(fs.adirs.user_log_dir, "processes")
if not os.path.isdir(log_dir):
os.makedirs(log_dir)
log_name = os.path.join(log_dir, log_name)
_backup_logs(log_name, maxbackups)
self.log = Logger(observer=textFileLogObserver(io.open(log_name, "w")),
namespace="")
super(LoggingProcessProtocol, self).__init__()
def connectionMade(self):
self.finished = defer.Deferred()
def outReceived(self, data):
self.log.info("{data}", data=bytes_to_str(data.strip()))
def errReceived(self, data):
self.log.error("{data}", data=bytes_to_str(data.strip()))
def processEnded(self, reason):
if reason.check(ProcessDone):
self.finished.callback(True)
self.log.info("Process finished without error")
else:
self.finished.errback(reason)
self.log.error("Process ended with error: {reason!r}",
reason=reason)
def test_logger_namespace(self):
"""
A `twisted.logger.Logger` with a namespace gets that namespace as a prefix.
"""
fout = StringIO()
log = Logger(namespace="ns", observer=FileLogObserver(fout, formatForSystemd))
log.info("info\n{more}", more="info")
log.error("err")
self.assertEqual((
"<6>[ns] info\n"
"<6> info\n"
"<3>[ns] err\n"
), fout.getvalue())
def render_POST(self, txrequest):
logger = Logger(namespace='- UpdateDbSchedule -')
args = native_stringify_dict(copy(txrequest.args), keys_only=False)
args = dict((k, v[0]) for k, v in args.items())
id = args.pop('id')
set_dic = args.copy()
where_dic = {'id': id}
lock.acquire()
try:
if database_type == 'mysql':
database_connector.update_data(model=SpiderScheduleModel,
set_dic=set_dic,
where_dic=where_dic)
else:
database_connector.update(model_name='SpiderScheduleModel',
update_dic=set_dic,
filter_dic=where_dic)
sta = 'ok'
except Exception as E:
logger.error('update fail: {}'.format(E))
sta = 'Update fail: {}'.format(E)
lock.release()
return {"node_name": self.root.nodename, 'update': sta}
def log_event(config: Config, token: Token, etype, ctx: dict):
log = Logger()
if type(token) is str:
token = token.encode('utf-8')
log.debug('token: ' + token.decode('utf-8'))
t = Token.query.filter_by(token=token.decode("utf-8")).first()
if t is None:
log = Logger()
log.error('Unable to find %s token' % token.decode("utf-8"))
return
e = Event(type=etype,
token_id=t.id,
token=t,
timestamp=datetime.utcnow(),
context=json.dumps(ctx))
db.add(e)
db.commit()
message = f"""
DNS Checkin
Date: {e.timestamp} UTC
Token: {t.token}
Tags: {t.context}
Context: {json.dumps(ctx)}
"""
print(config.notifications)
for n in config.notifications:
if not n.isEnabled():
continue
try:
res = n.notify(message)
print(res)
print(res.request)
except:
log.error('error executing notify class %s' % n)
class MFTests(object):
def __init__(self):
self.log = Logger()
def start_static(self):
resource = File(os.getcwd() + '/tests/pages')
factory = Site(resource)
endpoint = endpoints.TcP4ServerEndpoint(reactor, 0)
endpoint.listen(factory)
# reactor.run()
def send_request(self):
pass
def stop_callback(self, none):
reactor.stop()
def test_log_handler(self):
handler = LogHandler()
self.log.info('Test msg with {parameter} is OK', parameter="value")
self.log.error('Test error with {parameter} is OK', parameter="value")
self.log.error('Test error with {parameter} (isError={isError}) is OK',
parameter="value",
isError=False)
self.log.error('Test error with {parameter} (isError={isError}) is OK',
parameter="value",
isError=True)
d = defer.Deferred()
reactor.callLater(0, d.callback, None)
d.addCallback(self.stop_callback)
d.addErrback(
lambda err: print("callback error: %s\ncallback traceback: %s" %
(err.getErrorMessage(), err.getTraceback())))
reactor.run()
def test_server(self):
d = defer.Deferred()
reactor.callLater(3, d.callback, None)
d.addCallback(self.stop_callback)
#d.addCallback(self.send_request)
d.addErrback(
lambda err: print("callback error: %s\ncallback traceback: %s" %
(err.getErrorMessage(), err.getTraceback())))
Server(port=1234,
db_creds=None,
snapshot_dir='~/tmp',
user_agent='',
debug=False).run()
class MFTests(object):
def __init__(self):
self.log = Logger()
def start_static(self):
resource = File(os.getcwd() + '/tests/pages')
factory = Site(resource)
endpoint = endpoints.TcP4ServerEndpoint(reactor, 0)
endpoint.listen(factory)
# reactor.run()
def send_request(self):
pass
def stop_callback(self, none):
reactor.stop()
def test_log_handler(self):
handler = LogHandler()
self.log.info('Test msg with {parameter} is OK', parameter="value")
self.log.error('Test error with {parameter} is OK', parameter="value")
self.log.error('Test error with {parameter} (isError={isError}) is OK', parameter="value", isError=False)
self.log.error('Test error with {parameter} (isError={isError}) is OK', parameter="value", isError=True)
d = defer.Deferred()
reactor.callLater(0, d.callback, None)
d.addCallback(self.stop_callback)
d.addErrback(lambda err: print("callback error: %s\ncallback traceback: %s" % (err.getErrorMessage(), err.getTraceback())))
reactor.run()
def test_server(self):
d = defer.Deferred()
reactor.callLater(3, d.callback, None)
d.addCallback(self.stop_callback)
#d.addCallback(self.send_request)
d.addErrback(lambda err: print("callback error: %s\ncallback traceback: %s" % (err.getErrorMessage(), err.getTraceback())))
Server(port=1234, db_creds=None, snapshot_dir='~/tmp', user_agent='', debug=False).run()
class Rest(object):
def __init__(self,
host='https://developer-api.nest.com',
token=None,
event_handler=None,
net_type='lan'):
self.log = Logger()
self.host = host
self.token = token
self.event_handler = event_handler
self.pool = HTTPConnectionPool(reactor, persistent=True)
self.loc = None
self.reconnect = False
self.fail_count = 0
if event_handler:
self.reconnect = True
d = self.request(
headers={
'User-Agent': ['onDemand Rest Client'],
'Accept': ['text/event-stream']
})
d.addCallback(self.on_disconnect)
def __getattr__(self, name):
try:
super(Rest, self).__getattr__(name)
except AttributeError:
return RestCall(self, name)
def on_disconnect(self, reason):
if not reason:
reason = {'reason': 'no_message'}
self.log.critical('disconnected: {reason}', reason=reason['reason'])
if self.fail_count > 10:
self.log.error('Max error count reached, aborting connection')
def test_connectivity(count):
if self.fail_count == count:
self.fail_count = 0
self.fail_count += 1
c = self.fail_count
reactor.callLater(10, test_connectivity, c) # @UndefinedVariable
if self.reconnect:
d = self.request(
headers={
'User-Agent': ['onDemand Rest Client'],
'Accept': ['text/event-stream']
})
d.addCallback(self.on_disconnect)
def request(
self,
method='GET',
path='',
headers={
'User-Agent': ['onDemand/1.0 (Rest_Client)'],
'Accept': ['application/json']
},
body=None):
data = None
if self.loc:
host = '/'.join((self.loc, path))
else:
host = '/'.join((self.host, path))
if self.token:
host += '?auth=' + self.token
if body:
headers.update({'Content-Type': ['application/json']})
data = FileBodyProducer(StringIO(json.dumps(body)))
agent = RedirectAgent(Agent(reactor, pool=self.pool))
d = agent.request(method, host, Headers(headers), data)
def cbFail(fail):
if hasattr(fail.value, 'response'):
if hasattr(fail.value.response, 'code'):
if fail.value.response.code == 307:
loc = fail.value.response.headers.getRawHeaders(
'location')
new = urlparse(loc[0])
newhost = '://'.join((new.scheme, new.netloc))
if newhost == self.host:
self.loc = None
else:
self.loc = newhost
self.log.debug('redirect: %s' % self.loc)
data = FileBodyProducer(StringIO(json.dumps(body)))
d = agent.request(method, loc[0], Headers(headers),
data)
d.addCallbacks(cbRequest, cbFail)
return d
elif fail.value.response.code == 404 and self.loc:
self.loc = None
host = '/'.join((self.host, path))
if self.token:
host += '?auth=' + self.token
d = self.request(method, host, Headers(headers), body)
d.addCallbacks(cbRequest, cbFail)
return d
else:
print(dir(fail.value))
print(fail.value.message)
print(fail.value.args)
self.log.error('unhandled failure: %s -- %s' %
(fail.value.message, fail.value))
def cbRequest(response):
# print 'Response version:', response.version
# print 'Response code:', response.code
# print 'Response phrase:', response.phrase
# print 'Response headers:'
# print pformat(list(response.headers.getAllRawHeaders()))
finished = Deferred()
response.deliverBody(RestHandle(finished, self.event_handler))
return finished
d.addCallbacks(cbRequest, cbFail)
return d
class PhotometerService(Service):
BUFFER_SIZE = 1
def __init__(self, options, label):
self.options = options
self.label = label
self.namespace = self.label.upper()
setLogLevel(namespace=self.namespace, levelStr=options['log_messages'])
setLogLevel(namespace=self.label, levelStr=options['log_level'])
self.log = Logger(namespace=self.label)
self.factory = self.buildFactory()
self.protocol = None
self.serport = None
self.buffer = CircularBuffer(self.BUFFER_SIZE, self.log)
self.counter = 0
# Handling of Asynchronous getInfo()
self.info = None
self.info_deferred = None
if options['old_firmware']:
self.info = {
'name' : self.options['name'],
'mac' : self.options['mac_address'],
'calib' : self.options['zp'],
'rev' : 2,
}
# Serial port Handling
parts = chop(self.options['endpoint'], sep=':')
if parts[0] != 'serial':
self.log.critical("Incorrect endpoint type {ep}, should be 'serial'", ep=parts[0])
raise NotImplementedError
def startService(self):
'''
Starts the photometer service listens to a TESS
Although it is technically a synchronous operation, it works well
with inline callbacks
'''
self.log.info("starting {name}", name=self.name)
self.connect()
def stopService(self):
self.log.warn("stopping {name}", name=self.name)
self.protocol.transport.loseConnection()
self.protocol = None
self.serport = None
#self.parent.childStopped(self)
return defer.succeed(None)
#---------------------
# Extended Service API
# --------------------
@inlineCallbacks
def reloadService(self, new_options):
'''
Reload configuration.
Returns a Deferred
'''
options = options[self.label]
setLogLevel(namespace=self.label, levelStr=options['log_level'])
setLogLevel(namespace=self.namespace, levelStr=options['log_messages'])
self.options = options
return defer.succeed(None)
# -----------------------
# Specific photometer API
# -----------------------
def handleInfo(self, reading):
if self.info_deferred is not None:
self.info = {
'name' : reading.get('name', None),
'calib' : reading.get('ZP', None),
'mac' : self.options['mac_address'],
'rev' : 2,
}
self.log.info("Photometer Info: {info}", info=self.info)
self.info_deferred.callback(self.info)
self.info_deferred = None
def curate(self, reading):
'''Readings ready for MQTT Tx according to our wire protocol'''
reading['seq'] = self.counter
self.counter += 1
self.last_tstamp = reading.pop('tstamp', None)
if self.options['old_firmware']:
reading['mag'] = round(self.options['zp'] - 2.5*math.log10(reading['freq']),2)
reading['rev'] = 2
reading['name'] = self.options['name']
reading['alt'] = 0.0
reading['azi'] = 0.0
reading['wdBm'] = 0
reading.pop('zp', None)
else:
reading['mag'] = round(reading['ZP'] - 2.5*math.log10(reading['freq']),2)
self.info = {
'name' : reading.get('name', None),
'calib' : reading.get('ZP', None),
'mac' : self.options['mac_address'],
'rev' : 2,
}
reading.pop('udp', None)
reading.pop('ain', None)
reading.pop('ZP', None)
return reading
def getInfo(self):
'''Asynchronous operations'''
if not self.options['old_firmware'] and self.info is None:
deferred = defer.Deferred()
deferred.addTimeout(60, reactor)
self.info_deferred = deferred
else:
self.log.info("Photometer Info: {info}", info=self.info)
deferred = defer.succeed(self.info)
return deferred
# --------------
# Helper methods
# ---------------
def connect(self):
parts = chop(self.options['endpoint'], sep=':')
endpoint = parts[1:]
self.protocol = self.factory.buildProtocol(0)
try:
self.serport = SerialPort(self.protocol, endpoint[0], reactor, baudrate=endpoint[1])
except Exception as e:
self.log.error("{excp}",excp=e)
self.protocol = None
else:
self.gotProtocol(self.protocol)
self.log.info("Using serial port {tty} @ {baud} bps", tty=endpoint[0], baud=endpoint[1])
def buildFactory(self):
self.log.debug("Choosing a {model} factory", model=TESSW)
import tessw.tessw
factory = tessw.tessw.TESSProtocolFactory(self.namespace, self.options['old_firmware'])
return factory
def gotProtocol(self, protocol):
self.log.debug("got protocol")
self.buffer.registerProducer(protocol, True)
self.protocol = protocol
class BlocClient(Service):
"""
Client to connect to bloc server
"""
def __init__(self, clock, server, interval, treq=treq, session_id=None):
"""
Create a BlocClient instance
:param clock: An implementation of :obj:`IReactorTime`. Typically will be main twisted
reactor.
:param str server: server connection info in "server:port" form
:param float interval: Frequency of heartbeat in seconds
"""
self.clock = clock
self._server = server
self._settled = False
self._index = 0
self._total = 0
self._interval = interval
self._loop = task.LoopingCall(self._heartbeat)
self._loop.clock = self.clock
if session_id is None: # pragma: no cover
self._session_id = str(uuid.uuid1())
else:
self._session_id = session_id
self.log = Logger()
self.treq = treq
def startService(self):
"""
Start getting the index and effectively heartbeating
"""
super(BlocClient, self).startService()
self._loop.start(self._interval, True)
def _set_index(self, content):
if content['status'] == 'SETTLED':
self._settled = True
self._index = content['index']
self._total = content['total']
else:
self._settled = False
def _url(self, segment):
return 'http://{}/{}'.format(self._server, segment)
def _get_index(self):
d = self.treq.get(self._url("index"),
headers={'Bloc-Session-ID': [self._session_id]})
d.addCallback(check_status, [200])
return d.addCallback(treq.json_content)
def _error_allocating(self, f):
self._settled = False
self.log.error("Error getting index: {f}", f=f)
def _heartbeat(self):
d = self._get_index()
d.addCallback(self._set_index)
d.addTimeout(self._interval, self.clock)
d.addErrback(self._error_allocating)
return d
def stopService(self):
"""
Delete session and stop heartbeating
"""
super(BlocClient, self).stopService()
# Delete session before shutdown but do not worry about response if it not received
# within 1 second because we don't want to block shutdown of twisted app and server will
# anyway cancel the session without next heartbeat
d = self.treq.delete(self._url("session"),
headers={'Bloc-Session-ID': [self._session_id]})
d.addTimeout(1, self.clock)
return d.addBoth(lambda r: self._loop.stop())
def get_index_total(self):
"""
Return (index, total) tuple if settled, None if settling. Here "index" is position of this
node in the group and "total" is number of nodes in the group. For example, if there are
two BlocClient instances talking to the server then one of them will get (1, 2) and other
will get (2, 2). Note that this returns internal state last updated every "interval"
seconds.
"""
if not self._settled:
return None
return (self._index, self._total)
class OurStreamProtocol(Protocol):
"""Protocol implementing ShinySDR's WebSocket service.
This protocol's transport should be a txWS WebSocket transport.
"""
def __init__(self, caps, subscription_context):
self.__log = Logger()
self.__subscription_context = subscription_context
self._caps = caps
self._seenValues = {}
self.inner = None
def dataReceived(self, data):
"""Twisted Protocol implementation.
Additionally, txWS takes no care with exceptions here, so we catch and log."""
# pylint: disable=broad-except
try:
if self.inner is None:
# To work around txWS's lack of a notification when the URL is available, all clients send a dummy first message.
self.__dispatch_url()
else:
self.inner.dataReceived(data)
except Exception:
self.__log.failure('Error processing incoming WebSocket message')
def __dispatch_url(self):
loc = self.transport.location
self.__log.info('Stream connection to {url}', url=loc)
path = [urllib.unquote(x) for x in loc.split('/')]
assert path[0] == ''
path[0:1] = []
cap_string = path[0].decode('utf-8') # TODO centralize url decoding
if cap_string in self._caps:
root_object = self._caps[cap_string]
path[0:1] = []
else:
raise Exception('Unknown cap') # TODO better error reporting
if len(path) == 1 and path[0].startswith(b'audio?rate='):
rate = int(
json.loads(urllib.unquote(path[0][len(b'audio?rate='):])))
self.inner = AudioStreamInner(the_reactor, self.__send,
root_object, rate)
elif len(path) >= 1 and path[0] == CAP_OBJECT_PATH_ELEMENT:
# note _lookup_block may throw. TODO: Better error reporting
root_object = _lookup_block(root_object, path[1:])
self.inner = StateStreamInner(
self.__send, root_object, loc, self.__subscription_context
) # note reuse of loc as HTTP path; probably will regret this
else:
raise Exception('Unknown path: %r' % (path, ))
def connectionMade(self):
"""twisted Protocol implementation"""
self.transport.setBinaryMode(True)
# Unfortunately, txWS calls this too soon for transport.location to be available
def connectionLost(self, reason):
# pylint: disable=signature-differs
"""twisted Protocol implementation"""
if self.inner is not None:
self.inner.connectionLost(reason)
def __send(self, message, safe_to_drop=False):
if len(self.transport.transport.dataBuffer) > 1000000:
# TODO: condition is horrible implementation-diving kludge
# Don't accumulate indefinite buffer if we aren't successfully getting it onto the network.
# TODO: There are no tests of this mechanism
if safe_to_drop:
self.__log.warn('Dropping data going to stream {url}',
url=self.transport.location)
else:
self.__log.error(
'Dropping connection due to too much data on stream {url}',
url=self.transport.location)
self.transport.close(reason='Too much data buffered')
else:
self.transport.write(message)
class XmppEvent(object):
def __init__(self, nodeId, parent, pubsub_addr):
self.log = Logger()
self.nodeId = nodeId
self.parent = parent
self.addr = pubsub_addr
def publish(self, event):
if len(self.parent.active_controllers) == 0:
# self.log.debug('event cancelled')
self.parent.registrations = []
return
def success(res):
# print('event sent')
if res['type'] == 'error':
self.log.error('Publish Event failed :%s' % res.toXml())
else:
if 'Id' in res.children[0].children[0]['node']:
self.log.debug('Event Published: %s' % res.toXml())
name, data = event
if name == 'Seconds':
return
iq = IQ(self.parent.xmlstream, 'set')
ps = domish.Element(('http://jabber.org/protocol/pubsub', 'pubsub'))
publish = domish.Element((None, 'publish'))
publish['node'] = '/'.join((self.nodeId, name))
item = domish.Element((None, 'item'))
propertyset = domish.Element(
('urn:schemas-upnp-org:event-1-0', 'propertyset'),
localPrefixes={'e': 'urn:schemas-upnp-org:event-1-0'})
prop = domish.Element((None, 'e:property'))
evt = domish.Element((None, name))
if isinstance(data.value, dict):
ev = domish.Element((data.namespace, 'Event'))
inst = domish.Element((None, 'InstanceID'))
inst['val'] = '0'
for k, v in data.value.items:
if 'namespace' in v:
var = domish.Element((v['namespace'], k))
else:
var = domish.Element((None, k))
if 'attrib' in v:
attr = v['attrib']
else:
attr = {}
value = v['value']
if isinstance(value, bool):
value = int(value)
attr.update({'val': str(value).decode('utf-8')})
for attname, attval in attr:
var[attname] = attval
inst.addChild(var)
ev.addChild(inst)
evt.addChild(ev)
else:
# print(str(data.value).decode('utf-8'))
if isinstance(data.value, bool):
data.value = int(data.value)
evt.addContent(str(data.value).decode('utf-8'))
prop.addChild(evt)
propertyset.addChild(prop)
item.addChild(propertyset)
publish.addChild(item)
ps.addChild(publish)
iq.addChild(ps)
iq.addCallback(success)
iq.send(to=self.addr)
logger.debug('Enter MySQL Password:'******'MySQL']['pass'] = raw_input()
logger.debug(
'Enter old MySQL database name (the one from which you are going to port):'
)
CONFIG['MySQL']['oldname'] = raw_input()
logger.debug(
'Enter new MySQL database name (the one to which new data is transferred):'
)
CONFIG['MySQL']['dbname'] = raw_input()
logger.info('Checking DB Name conventions, and rules')
if not CONFIG['MySQL']['dbname'].endswith('line'):
logger.error(
'Database name should end with line (eg. Timeline), a strictly enforced nomenclature.'
)
sys.exit(0)
logger.info('DB naming rules check - clear')
logger.info('Checking MySQL Credentials')
try:
MYSQL_CONN = mysql.connector.connect(user=CONFIG['MySQL']['user'],
password=CONFIG['MySQL']['pass'],
database=CONFIG['MySQL']['oldname'])
except Exception, e:
print 'Error MySQL', e
os._exit(1)
class XmppEvent(object):
def __init__(self, nodeId, parent, pubsub_addr):
self.log = Logger()
self.nodeId = nodeId
self.parent = parent
self.addr = pubsub_addr
def publish(self, event):
if len(self.parent.active_controllers) == 0:
# self.log.debug('event cancelled')
self.parent.registrations = []
return
def success(res):
# print('event sent')
if res['type'] == 'error':
self.log.error('Publish Event failed :%s' % res.toXml())
else:
if 'Id' in res.children[0].children[0]['node']:
self.log.debug('Event Published: %s' % res.toXml())
name, data = event
if name == 'Seconds':
return
iq = IQ(self.parent.xmlstream, 'set')
ps = domish.Element(('http://jabber.org/protocol/pubsub', 'pubsub'))
publish = domish.Element((None, 'publish'))
publish['node'] = '/'.join((self.nodeId, name))
item = domish.Element((None, 'item'))
propertyset = domish.Element(
('urn:schemas-upnp-org:event-1-0', 'propertyset'),
localPrefixes={'e': 'urn:schemas-upnp-org:event-1-0'})
prop = domish.Element((None, 'e:property'))
evt = domish.Element((None, name))
if isinstance(data.value, dict):
ev = domish.Element((data.namespace, 'Event'))
inst = domish.Element((None, 'InstanceID'))
inst['val'] = '0'
for k, v in data.value.items:
if 'namespace' in v:
var = domish.Element((v['namespace'], k))
else:
var = domish.Element((None, k))
if 'attrib' in v:
attr = v['attrib']
else:
attr = {}
value = v['value']
if isinstance(value, bool):
value = int(value)
attr.update(
{'val': str(value)
.decode('utf-8')})
for attname, attval in attr:
var[attname] = attval
inst.addChild(var)
ev.addChild(inst)
evt.addChild(ev)
else:
# print(str(data.value).decode('utf-8'))
if isinstance(data.value, bool):
data.value = int(data.value)
evt.addContent(str(data.value).decode('utf-8'))
prop.addChild(evt)
propertyset.addChild(prop)
item.addChild(propertyset)
publish.addChild(item)
ps.addChild(publish)
iq.addChild(ps)
iq.addCallback(success)
iq.send(to=self.addr)
signal.signal(signal.SIGUSR1, receive_signal)
atexit.register(contacts.close)
servers_file_path = '%s/.servers' % config['directory']
logger.info('loading server')
try:
servers = json.load(open(servers_file_path))
logger.info(
'read last read date from server neighbors from {servers_file_path}',
servers_file_path=servers_file_path)
except json.JSONDecodeError:
logger.error(
"Bad JSON in server file at {file_path} recovering automatically",
file_path=servers_file_path)
servers = {}
except FileNotFoundError as err:
servers = {}
if config.get('servers'):
for server in config.get('servers').split(','):
if server not in servers:
servers[server] = '1970-01-01T00:00Z'
allowable_methods = [
'/status/scan:POST', '/status/send:POST', '/status/update:POST',
'/sync:GET', '/admin/config:GET', '/admin/status:GET',
'/status/result:POST', '/status/data_points:POST', '/init:POST'
]
class Discord(IRCClient):
nickname = "discord"
realname = "Discord"
username = "******"
versionName = "Discord"
versionNum = "0.01"
magicFile = "true.txt"
def __init__(self, accessList):
self.logger = Logger(observer=textFileLogObserver(sys.stdout))
self.accessList = [nick.lower() for nick in accessList]
if not os.path.exists(self.magicFile):
self.logger.info("Creating magic file")
try:
with open(self.magicFile, "a"):
pass
except Exception as ex:
self.logger.error("Unable to create magic file! {0}".format(ex.message))
reactor.stop()
self.markovGenerator = pymarkov.MarkovChainGenerator(self.magicFile)
self.channels = []
self.channelPhrasers = {}
self.logger.debug("Discord initialized")
# Maybe add hook/plugin system here?
self.commands = Commands.Commands(self)
def removeChannel(self, channel):
try:
self.channels.remove(channel)
self.channelPhrasers[channel].stop()
del self.channelPhrasers[channel]
except:
self.logger.error("Error removing {channel} from collection", channel=channel)
def insertPhrase(self, phrase):
try:
with open(self.magicFile, "a") as magicFile:
magicFile.write("{0}\n".format(phrase))
try:
file, ext = os.path.splitext(self.magicFile)
os.remove("{0}-pickled{1}".format(file, ext))
# Simply re-populating the dictionary isn't enough for some reason
self.markovGenerator = pymarkov.MarkovChainGenerator(self.magicFile, 2)
except IOError as ex:
self.logger.error("Unable to delete pickled file. {0}".format(ex.message))
except Exception as ex:
self.logger.error("Unable to insert phrase into magic file! {0}".format(ex.message))
def kickedFrom(self, channel, kicker, message):
self.removeChannel(channel)
self.logger.info("Kicked from {channel} by {kicker}", channel=channel, kicker=kicker)
def left(self, channel):
self.removeChannel(channel)
self.logger.info("Left {channel}", channel=channel)
def handleMessage(self, user, channel, message):
senderNickname = user.split("!")[0]
if message.startswith("~reload") and senderNickname in self.accessList:
self.logger.info("Reloading commands module")
self.say(channel, "Reloading.")
try:
commandsModule = reload(Commands)
self.commands = commandsModule.Commands(self)
except Exception as ex:
self.say(channel, "Failed to load commands module - {0}".format(ex.message))
elif message.startswith("~"):
# Don't log commands to the brain
commandMessage = message[1:]
self.commands.handleCommand(user, channel, commandMessage)
else:
self.logger.info("Adding {message!r} to brain", message=message)
# Avoid storing anything with the bot's name in it
brainMessage = message.strip(self.nickname)
self.insertPhrase(brainMessage)
try:
randomPhrase = self.generateSentence()
if self.nickname in message and channel.startswith("#") and self.channelPhrasers[channel].running:
phrase = "{0}, {1}".format(senderNickname, randomPhrase)
self.say(channel, phrase)
elif channel == self.nickname:
self.logger.debug("Sending message to {nickname}", nickname=senderNickname)
self.msg(senderNickname, randomPhrase)
else:
pass
except IndexError as generationError:
self.logger.error(generationError.message)
def privmsg(self, user, channel, message):
self.logger.info("Received message from {user} in {channel}", user=user, channel=channel)
# deferToThread(self.handleMessage, user, channel, message)
self.handleMessage(user, channel, message)
def signedOn(self):
self.logger.info("Signed on")
self.join("#bots")
def joined(self, channel):
self.channels.append(channel)
self.logger.info("Joined channel {channel!r}", channel=channel)
channelPhraser = LoopingCall(self.sayRandomPhrase, channel)
reactor.callLater(2, channelPhraser.start, 600)
self.channelPhrasers[channel] = channelPhraser
def generateSentence(self):
try:
sentence = self.markovGenerator.generate_sentence()
sentence = sentence.strip("<{0}>".format(self.nickname))
sentence = sentence.strip(self.nickname)
return sentence
except (IndexError, ValueError) as ex:
self.logger.error(ex.message)
def sayRandomPhrase(self, channel):
sentence = self.generateSentence()
self.say(channel, sentence)
class TimeSchedule:
def __init__(self, lock, host='127.0.0.1', port='6800'):
config = Config()
self.db = glv.get_value(key='sqlite_db')
self.user_name = config.get('auth_username', '')
self.user_password = config.get('auth_password', '')
self.start_time = time.strftime("%Y %m %d %H %M %S", time.localtime())
self.server_port = 'http://{}:{}/'.format(host, port)
self.schedule_post_url = 'http://{}:{}/schedule.json'.format(
host, port)
self.listproject_url = 'http://{}:{}/listprojects.json'.format(
host, port)
self.spider_task_dic = dict()
self.projects = None
self.db_lock = lock
self.ts_lock = threading.Lock()
self.CPU_THRESHOLD = 93
self.MEMORY_THRESHOLD = 96
self.schedule_logger = Logger(namespace='- Scheduler -')
def run(self):
time.sleep(3)
self.projects = self.list_projects()
self.schedule_logger.info('scheduler is running')
count = 1
while True:
schedule_sta = self.task_scheduler()
if not schedule_sta and count == 1:
self.schedule_logger.info('No Scheduled Spider in Database')
count += 1
elif not schedule_sta and count != 1:
count += 1
else:
count = 1
time.sleep(1)
def task_scheduler(self):
self.ts_lock.acquire(blocking=True)
self.db_lock.acquire()
db_result = self.db.get(
model_name='SpiderScheduleModel',
key_list=['project', 'spider', 'schedule', 'args', 'status'])
self.db_lock.release()
self.ts_lock.release()
schedule_list_raw = [{
'project': x.project,
'spider': x.spider,
'schedule': x.schedule,
'args': x.args,
'status': x.status
} for x in db_result if int(x.status) != 0] if db_result else []
schedule_sta = False
if schedule_list_raw:
for each_schedule in schedule_list_raw:
project = each_schedule.get('project')
if project in self.projects:
schedule = each_schedule.get('schedule').replace('\\', '')
try:
schedule = json.loads(schedule)
except:
schedule = eval(schedule)
try:
next_time_sep = self.cal_time_sep(**schedule)
next_time_sep = int(next_time_sep) + 1
if next_time_sep > 1:
each_schedule['schedule'] = next_time_sep
item = '{}-{}'.format(each_schedule['project'],
each_schedule['spider'])
self.ts_lock.acquire(blocking=True)
if self.spider_task_dic.get(item) != 'waiting':
self.spider_task_dic[item] = 'waiting'
t = threading.Thread(target=self.poster,
args=(each_schedule, ))
try:
t.start()
except Exception as THError:
self.schedule_logger.warn(
'start new job error [ {} ]: {}'.
format(item, THError))
self.ts_lock.release()
except ValueError:
self.schedule_logger.error(
'spider runtime schedule error, please check the database'
)
schedule_sta = True
return schedule_sta
def poster(self, dic):
status = int(dic.pop('status'))
project = dic.get('project')
spider = dic.get('spider')
job_str = " %s-%s " % (project, spider)
args = dic.get('args').replace('\\', '')
if args:
args = eval(args)
wait_time = dic.get('schedule')
item = '{}-{}'.format(project, spider)
if project and spider:
data = {
'project': project,
'spider': spider,
'un': self.user_name,
'pwd': self.user_password
}
if args:
data.update(args)
self.schedule_logger.info(
'job {} is waiting, countdown {}s'.format(item, wait_time))
time.sleep(wait_time - 1)
another_wait_time = 0
spider_runtime_avg = self.spiders_runtime(project=project,
spider=spider)
if status == 1:
while not self.is_system_ok():
self.schedule_logger.warn(
'system is fully functioning, wait another 2 seconds to post schedule'
)
time.sleep(2)
another_wait_time += 3
if another_wait_time >= (wait_time - spider_runtime_avg):
self.schedule_logger.warning(
'wait too long, cancel the job %s' % job_str)
return None
res = json.loads(
requests.post(url=self.schedule_post_url,
data=data).content)
elif status == 2:
res = json.loads(
requests.post(url=self.schedule_post_url,
data=data).content)
elif status == 3:
res = json.loads(
requests.post(url=self.schedule_post_url,
data=data).content)
else:
res = json.loads(
requests.post(url=self.schedule_post_url,
data=data).content)
spider_status = res.get('status')
if spider_status != 'ok':
spider_status = 'error'
else:
self.schedule_logger.error(
'job project: {}, spider: {} post fail!'.format(
project, spider))
spider_status = 'error'
self.ts_lock.acquire(blocking=True)
self.spider_task_dic[item] = spider_status
self.ts_lock.release()
def spiders_runtime(self, project, spider):
self.db_lock.acquire()
res = self.db.get(model_name='SpiderMonitor',
key_list=['runtime'],
filter_dic={
'project': project,
'spider': spider
})
self.db_lock.release()
spider_list = [int(x.runtime) for x in res
if x.runtime.isdigit()] if res else [0]
return sum(spider_list) / len(spider_list)
def list_projects(self):
res = requests.get(url=self.listproject_url)
projects = {}
if res:
projects_list = json.loads(res.content).get('projects')
if projects_list:
projects = set(projects_list)
return projects
def cal_time_sep(
self,
year='*',
month='*',
day='*',
week='*',
hour='*',
minute='*',
second='*',
):
"""
"%Y-%m-%d %H:%M:%S %w"
"""
y = int(time.strftime("%Y", time.localtime()))
if year != '*' and '*' in year:
y = int(year.split('/')[-1]) + y
elif year.isdigit():
y = int(year)
if week == '*':
m = int(time.strftime("%m", time.localtime()))
if month != '*' and '*' in month:
m_raw = int(month.split('/')[-1])
if m_raw >= 12:
raise ValueError(
'month value is too large, please set the year instead'
)
m = m_raw + m
if m > 12:
y += m // 12
m = m % 12
elif month.isdigit():
m = int(month)
days_in_this_month = self.how_many_days_in_this_month(y, m)
d = int(time.strftime("%d", time.localtime()))
if day != '*' and '*' in day:
d_raw = int(day.split('/')[-1])
if d_raw > days_in_this_month:
raise ValueError(
'day value is too large, please set the month or the year instead'
)
d = d_raw + d
if d > days_in_this_month:
d = d - days_in_this_month
m += 1
if m > 12:
y += 1
m = m - 12
elif day.isdigit():
d = int(day)
days_in_this_month = self.how_many_days_in_this_month(y, m)
H = int(time.strftime("%H", time.localtime()))
if hour != '*' and '*' in hour:
H_raw = int(hour.split('/')[-1])
if H_raw > 24:
raise ValueError(
'hour value is too large, please set the day instead')
H = H_raw + H
if H >= 24:
H = H - 24
d += 1
if d > days_in_this_month:
d = d - days_in_this_month
m += 1
if m > 12:
y += 1
m = m - 12
elif hour.isdigit():
H = int(hour)
days_in_this_month = self.how_many_days_in_this_month(y, m)
M = int(time.strftime("%M", time.localtime()))
if minute != '*' and '*' in minute:
M_raw = int(minute.split('/')[-1])
if M_raw > 60:
raise ValueError(
'minute value is too large, please set the hour instead'
)
M = M_raw + M
if M >= 60:
M = M - 60
H += 1
if H >= 24:
H = H - 24
d += 1
if d > days_in_this_month:
d = d - days_in_this_month
m += 1
if m > 12:
y += 1
m = m - 12
elif minute.isdigit():
M = int(minute)
days_in_this_month = self.how_many_days_in_this_month(y, m)
S = int(time.strftime("%S", time.localtime()))
if second != '*' and '*' in second:
S_raw = int(second.split('/')[-1])
if S_raw > 60:
raise ValueError(
'second value is too large, please set the minute instead'
)
S = S_raw + S
if S >= 60:
S = S - 60
M += 1
if M >= 60:
M = M - 60
H += 1
if H >= 24:
H = H - 24
d += 1
if d > days_in_this_month:
d = d - days_in_this_month
m += 1
if m > 12:
y += 1
m = m - 12
elif second.isdigit():
S = int(second)
time_sep = eval(
"(datetime.datetime({},{},{}, {},{},{}) - datetime.datetime.now()).total_seconds()"
.format(y, m, d, H, M, S))
else:
week_in_this_year = int(time.strftime("%U", time.localtime()))
w = int(time.strftime("%w", time.localtime()))
if '*' in week:
w_raw = int(week.split('/')[-1])
if w_raw >= 7:
raise ValueError(
'week value is too large, please set the day or the month instead'
)
if w_raw < w:
week_in_this_year += 1
w = w_raw
if week_in_this_year > 53:
y += 1
week_in_this_year = week_in_this_year - 53
elif week.isdigit():
w = int(week)
if int(week) < w:
week_in_this_year += 1
H = int(time.strftime("%H", time.localtime()))
if hour != '*' and '*' in hour:
H_raw = int(hour.split('/')[-1])
if H_raw >= 24:
raise ValueError(
'hour value is too large, please set the day instead')
H = H_raw + H
if H >= 24:
H = H - 24
w += 1
if w >= 7:
w = w - 7
week_in_this_year += 1
if week_in_this_year > 53:
y += 1
week_in_this_year = week_in_this_year - 53
elif hour.isdigit():
H = int(hour)
M = int(time.strftime("%M", time.localtime()))
if minute != '*' and '*' in minute:
M_raw = int(minute.split('/')[-1])
if M_raw >= 60:
raise ValueError(
'minute value is too large, please set the hour instead'
)
M = M_raw + M
if M >= 60:
M = M - 60
H += 1
if H >= 24:
H = H - 24
w += 1
if w > 7:
w = w - 7
week_in_this_year += 1
if week_in_this_year > 53:
y += 1
week_in_this_year = week_in_this_year - 53
elif minute.isdigit():
M = int(minute)
S = int(time.strftime("%S", time.localtime()))
if second != '*' and '*' in second:
S_raw = int(second.split('/')[-1])
if S_raw >= 60:
raise ValueError(
'second value is too large, please set the minute instead'
)
S = S_raw + S
if S >= 60:
S = S - 60
M += 1
if M >= 60:
M = M - 60
H += 1
if H >= 24:
H = H - 24
w += 1
if w > 7:
w = w - 7
week_in_this_year += 1
if week_in_this_year > 53:
y += 1
week_in_this_year = week_in_this_year - 53
elif second.isdigit():
S = int(second)
if S >= 60:
S = S - 60
M += 1
if M >= 60:
M = M - 60
H += 1
if H >= 24:
H = H - 24
w += 1
if w > 7:
w = w - 7
week_in_this_year += 1
if week_in_this_year > 53:
y += 1
week_in_this_year = week_in_this_year - 53
m, d = self.get_month_and_days_by_week(
year=y, week_in_this_year=week_in_this_year, week=w)
time_sep = eval(
"(datetime.datetime({},{},{}, {},{},{}) - datetime.datetime.now()).total_seconds()"
.format(y, m, d, H, M, S))
return time_sep
def get_month_and_days_by_week(self, year, week_in_this_year, week):
days = week_in_this_year * 7 + week
if (year % 4 == 0 and year % 100 != 0) or (year % 400 == 0):
Fe = 29
else:
Fe = 28
month_lis = [31, Fe, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]
month_count = 1
days_count = 0
for month_days in month_lis:
days = days - month_days
if days > 0:
month_count += 1
elif days == 0:
days_count = 0
month_count += 1
break
else:
days_count = days + month_days
break
return [month_count, days_count]
def how_many_days_in_this_month(self, y, m):
if m in (1, 3, 5, 7, 8, 10, 12):
days = 31
elif m in (4, 6, 9, 11):
days = 30
else:
if (y % 4 == 0 and y % 100 != 0) or (y % 400 == 0):
days = 29
else:
days = 28
return days
def is_system_ok(self):
is_pass = True
cpu_list = psutil.cpu_percent(interval=1, percpu=True)
memory_percent = psutil.virtual_memory().percent
if cpu_list and memory_percent:
is_cpu_ok = True
if min(cpu_list) > self.CPU_THRESHOLD:
is_cpu_ok = False
is_memo_ok = True
if memory_percent > self.MEMORY_THRESHOLD:
is_memo_ok = False
if not is_cpu_ok or not is_memo_ok:
is_pass = False
return is_pass
class MpdProtocol(LineReceiver):
"""
Twisted protocol to control remote mpd server
"""
def __init__(self):
"""
doc
"""
self.log = Logger()
self.delimiter = "\n"
self.deferreds = []
self.buff = {}
self.idle = False
self.list_index = 0
def connectionLost(self, reason):
self.log.error("connection lost : {reason}", reason=reason)
self._event({"changed": "disconnected"})
self.idle = False
try:
d = self.deferreds.pop(0)
except:
pass
else:
d.errback(reason)
def connectionMade(self):
self.log.debug("connected")
def addCallback(self, d):
self.deferreds.append(d)
def noidle(self):
d = defer.Deferred()
d.addCallback(lambda ignored: ignored)
self.deferreds.insert(0, d)
self.sendLine("noidle")
self.idle = False
# print('noidle')
def set_idle(self):
self.sendLine("idle")
self.idle = True
# print('idle')
def lineReceived(self, line):
# print(line)
if line.startswith("OK MPD"):
self._event({"changed": "connected"})
elif line.startswith("OK"):
# print('deferred length: %d' % len(self.deferreds))
self.list_index = 1
try:
d = self.deferreds.pop(0)
except:
self.set_idle()
self._event(self.buff)
self.buff = {}
return
else:
d.callback(self.buff)
self.buff = {}
elif line.startswith("ACK"):
# print('deferred length: %d' % len(self.deferreds))
try:
d = self.deferreds.pop(0)
except:
self.set_idle()
self._event({"Error": line.split("}")[1]})
self.buff = {}
return
else:
d.errback(Exception(line.split("}")[1]))
self.buff = {}
else:
if len(line) > 0:
k = line.split(":")[0]
if isinstance(self.buff, list):
if k in self.buff[self.list_index]:
self.list_index += 1
self.buff.append({})
self.buff[self.list_index].update({k: " ".join(line.split()[1:])})
else:
if k in self.buff:
self.buff = [self.buff]
self.list_index = 1
self.buff.append({k: " ".join(line.split()[1:])})
# if str(self.list_index) + k in self.buff:
# self.list_index += 1
# self.buff.update(
# {str(self.list_index) + line.split(':')[0]:
# ' '.join(line.split()[1:])})
else:
self.buff.update({k: " ".join(line.split()[1:])})
return
if len(self.deferreds) == 0:
self.set_idle()
class RainContent(GenericContent):
def __init__(self, endpoint, factory):
self.log = Logger(self.__class__.__name__)
super().__init__(endpoint, factory)
self.task = None
def onBrokerConnected(self):
self.task = task.LoopingCall(self.createForecast)
self.task.start(self.config['RAIN_UPDATE_FREQ'], now=True)
def _logFailure(self, failure):
self.log.error("reported failure: {message}",
message=failure.getErrorMessage())
return failure
def _logSuccess(self, success, url):
self.log.info("Success requesting {url}", url=url)
return success
def createForecast(self):
url = self.config["RAIN_DATA_SOURCE"]
self.log.debug("Grabbing rain forecast URL '%s'" % url)
d = treq.get(url, timeout=5)
d.addCallbacks(self.grab_http_response, self._logFailure)
d.addCallbacks(self.parse_forecast_results, self._logFailure)
d.addCallbacks(self.create_forcast, self._logFailure)
d.addCallbacks(self.publish_forcast, self._logFailure)
d.addCallback(self._logSuccess, url)
def create_forcast(self, data):
if data[0][0] == 0: # It's currently dry
for rain, t in data:
if rain > 0:
return "Rain at %s" % t
return None
else: # It's raining.
for rain, t in data:
if rain == 0:
return "Rain stop %s" % t
return "Rain Rain Rain"
def grab_http_response(self, response):
if response.code != 200:
raise RuntimeError("Status is not 200 but '%s'" % response.code)
return readBody(response)
def parse_forecast_results(self, content):
raw_str = content.decode()
raw_arr = []
for raw in raw_str.split("\r\n"):
if not raw:
continue
rain_value, hour = raw.split("|")
raw_arr.append([int(rain_value, 10), hour])
if len(raw_arr) == 0:
raise RuntimeWarning(
"API results where not in the expected format. They were: {}".
format(raw_str))
return raw_arr
def publish_forcast(self, forcast_string: str):
def _logAll(*args):
self.log.debug("all publishing complete args={args!r}", args=args)
if forcast_string is None: # There's no Rain information to show.
return
msg = TextSingleLineLayout()
msg.text = forcast_string
msg.duration = self.config["RAIN_DISPLAY_DURATION"]
msg.program = 'rain'
msg.font_size = 15
d = self.publish(topic=LEDSLIE_TOPIC_TYPESETTER_1LINE,
message=msg,
qos=1)
d.addCallbacks(_logAll, self._logFailure)
return d
class XmppService(Service):
timeout = True
active_controllers = []
def __init__(self,
device,
user='******',
secret='password',
userlist=[],
web_server=None):
self.log = Logger()
self.description = None
self.reactor = reactor
self.user = user
self.services = {}
self.nodes = []
self.registrations = []
self.active_controllers = []
self.webserver = web_server
self.resource = web_server.resource
device.location = user
def _map_context(ctx):
ctx.udc = UserDefinedContext(device.player)
self._jid = _jid = ''.join(
(user, '/', device.deviceType, ':uuid:', device.uuid))
self.device = device
device.parent = self
for service in device.services:
# if appreg.get_application(service.tns, service.name):
# name = service.name + '_'
# else:
# name = service.name
# soap_service = type(
# service.name, (ServiceBase,), service.soap_functions)
# soap_service.tns = service.tns
# app = Application(
# [soap_service],
# tns=soap_service.tns,
# in_protocol=Soap11(xml_declaration=False),
# out_protocol=Soap11(xml_declaration=False),
# name=name)
# app.event_manager.add_listener('method_call', _map_context)
self.services.update({
str(service.serviceId): {
'app': TwistedXMPPApp(service.app),
'svc': service
}
})
print('name: %s, methods: %s' %
(device.name, service.app.interface.service_method_map))
for var in service.stateVariables.values():
if var.sendEvents:
self.nodes.append((var, service.serviceType, service))
self.users = {user: False}
for user in userlist:
self.users.update({user: False})
self.jid = jid = JID(_jid)
self.factory = f = client.XMPPClientFactory(jid, secret)
f.addBootstrap(xmlstream.STREAM_CONNECTED_EVENT, self.connected)
f.addBootstrap(xmlstream.STREAM_END_EVENT, self.disconnected)
f.addBootstrap(xmlstream.STREAM_AUTHD_EVENT, self.authenticated)
f.addBootstrap(xmlstream.INIT_FAILED_EVENT, self.init_failed)
# self.get_device_info(device)
self.finished = defer.Deferred()
# def get_device_info(self, device):
# if len(device.icons) > 0:
# icon = device.icons[0]
# buf = read(open())
def startService(self):
self.connector = SRVConnector(self.reactor,
'xmpp-client',
self.jid.host,
self.factory,
defaultPort=5222)
self.connector.connect()
def stopService(self):
for node in self.registrations:
self.delete_ps_node(node)
self.xmlstream.sendFooter()
Service.stopService(self)
def connected(self, xs):
# print 'Connected.'
# log.debug('!!!!!!!!!!!!!!!!!!!{app}', app=appreg._applications)
self.xmlstream = xs
# Log all traffic
# xs.rawDataInFn = self.rawDataIn
# xs.rawDataOutFn = self.rawDataOut
def disconnected(self, xs):
print 'Disconnected.'
# self.finished.callback(None)
def authenticated(self, xs):
# print "Authenticated."
xs.addObserver('/presence', self.on_presence)
xs.addObserver('/iq', self.on_iq)
presence = domish.Element((None, 'presence'))
uc = domish.Element(
('urn:schemas-upnp-org:cloud-1-0', 'ConfigIdCloud'))
uc['hash'] = 'uda'
uc.addContent('1.0')
# print(uc.toXml())
presence.addChild(uc)
# print(presence.toXml())
xs.send(presence)
disco = IQ(xs, 'get')
disco.addElement(('http://jabber.org/protocol/disco#info', 'query'))
disco.addCallback(self.check_server)
disco.send(to='pubsub.xmpp.bertrandverdu.me')
self.check_ps_nodes()
self.reactor.callLater(30, self.ping)
def ping(self):
def pong(res):
if res['type'] == 'result':
# log.debug('pong !')
self.timeout = False
self.reactor.callLater(30, self.ping)
else:
self.log.error('ping error: %s' % res.toXml())
self.timeout = False
# self.startService()
def check_timeout():
if self.timeout:
self.log.error('ping timeout !')
# self.connector.connect()
iq = IQ(self.xmlstream, 'get')
iq.addElement('ping', 'urn:xmpp:ping')
self.reactor.callLater(10, check_timeout)
iq.addCallback(pong)
self.timeout = True
iq.send(to=self.jid.host)
def rawDataIn(self, buf):
print("Device RECV: %s" %
unicode(buf, 'utf-8').encode('ascii', 'replace'))
def rawDataOut(self, buf):
print("Device SEND: %s" %
unicode(buf, 'utf-8').encode('ascii', 'replace'))
def check_server(self, result):
def display_info(res):
return
e = fromstring(res.toXml())
dump(e)
self.log.debug("server checked")
display_info(result)
# e = fromstring(result.toXml())
# dump(e)
# iq = IQ(self.xmlstream, 'get')
# ps = domish.Element(
# ('http://jabber.org/protocol/pubsub#owner', 'pubsub'))
# ps.addElement('default')
# iq.addChild(ps)
# iq.addCallback(display_info)
# iq.send(to='pubsub.' + self.jid.host)
# print(result.toXml())
def dump_description(self, dest):
def sent(res):
self.log.debug('description sent')
pass
self.log.debug('send description')
# print(dest['id'])
if self.description:
self.log.debug('cached description')
iq = IQ(self.xmlstream, 'result')
iq.addRawXml(self.description)
iq['id'] = dest['id']
else:
self.log.debug('generate description')
iq = IQ(self.xmlstream, 'result')
query = domish.Element(('urn:schemas-upnp-org:cloud-1-0', 'query'))
query['type'] = 'described'
query['name'] = self.device.uuid
d = tostring(self.device.dump(), encoding='unicode')
query.addRawXml(d)
# print(query.toXml())
for service in self.device.services:
s = tostring(service.dump(), encoding='unicode')
# print(s)
query.addRawXml(s)
# print(query.toXml())
self.description = query.toXml()
iq.addChild(query)
iq['id'] = dest['id']
# self.description = iq
# print(iq.toXml())
iq.addCallback(sent)
iq.send(to=dest['from'])
def check_ps_nodes(self):
def got_response(node, response):
if response['type'] in ('error', 'cancel'):
self.create_ps_node(node)
elif response['type'] == 'result':
node_name = '/'.join((self._jid, node[1]))
if node_name in self.registrations:
return
self.registrations.append(node_name)
self.log.debug('node {name} registered', name=node_name)
event = XmppEvent(node_name, self, 'pubsub.' + self.jid.host)
node[2].subscribe(event, 100)
self.reactor.callLater(95, self.renew_subscription,
*(node_name, node))
else:
self.log.error('unknown response from server: %s' %
response.toXml())
self.log.debug('check nodes: {nodes}', nodes=str(self.nodes))
IQ_ = IQ # Basic optimisation...
element = domish.Element
for node in self.nodes:
iq = IQ_(self.xmlstream, 'get')
query = element(('http://jabber.org/protocol/disco#info', 'query'))
query['node'] = '/'.join((self._jid, node[1], node[0].name))
iq.addChild(query)
iq.addCallback(got_response, node)
iq.send(to='pubsub.' + self.jid.host)
def create_ps_node(self, node):
def registered(node, iq):
if iq['type'] == 'result':
node_name = '/'.join((self._jid, node[1]))
if node_name in self.registrations:
return
event = XmppEvent(node_name, self, 'pubsub.' + self.jid.host)
node[2].subscribe(event, 100)
self.reactor.callLater(95, self.renew_subscription,
*(node_name, node))
self.registrations.append(node_name)
self.log.debug('node {node} registered', node=node_name)
else:
self.log.error('node creation {name} failed:{iq}',
name=node,
iq=iq.toXml())
iq = IQ(self.xmlstream, 'set')
ps = domish.Element(('http://jabber.org/protocol/pubsub', 'pubsub'))
create = domish.Element((None, 'create'))
create['node'] = '/'.join((self._jid, node[1], node[0].name))
ps.addChild(create)
configure = domish.Element((None, 'configure'))
x = domish.Element(('jabber:x:data', 'x'))
x['type'] = 'submit'
field = domish.Element((None, 'field'))
field['var'] = 'FORM_TYPE'
field['type'] = 'hidden'
field.addElement(
'value', content='http://jabber.org/protocol/pubsub#node_config')
x.addChild(field)
access = domish.Element((None, 'field'))
access['var'] = 'pubsub#access_model'
access.addElement('value', content='roster')
x.addChild(access)
# expire = domish.Element((None, 'field'))
# expire['var'] = 'pubsub#item_expire'
# expire.addElement('value', content='60')
# x.addChild(expire)
last = domish.Element((None, 'field'))
last['var'] = 'pubsub#send_last_published_item'
last.addElement('value', content='on_sub_and_presence')
x.addChild(last)
configure.addChild(x)
ps.addChild(configure)
iq.addChild(ps)
iq.addCallback(registered, node)
iq.send(to='pubsub.' + self.jid.host)
def delete_ps_node(self, node):
def deleted(res):
if res['type'] == 'error':
self.log.error('node deletion failed: %s' % res.toXml())
iq = IQ(self.xmlstream, 'set')
ps = domish.Element(
('http://jabber.org/protocol/pubsub#owner', 'pubsub'))
delete = domish.Element((None, 'delete'))
delete['node'] = node
ps.addChild(delete)
iq.addChild(ps)
iq.send(to='pubsub.' + self.jid.host)
def renew_subscription(self, name, node):
self.log.debug('renew %s : %s' % (name, (name in self.registrations)))
if name in self.registrations:
self.log.debug('renew: %s' % name)
event = XmppEvent(name, self, 'pubsub.' + self.jid.host)
node[2].subscribe(event, 100)
self.reactor.callLater(95, self.renew_subscription, *(name, node))
def on_iq(self, iq):
# print('received iq: %s' % iq.toXml().encode('utf-8'))
user, host, res = parse(iq['from'])
del (res)
if not user:
return
jid = '@'.join((user, host))
self.log.debug('received request of type {typ} from {user}',
typ=iq['type'],
user=jid)
if jid not in self.users and jid != self.user:
self.log.info('rejected User: %s' % jid)
return
if iq['type'] == 'get':
for child in iq.children:
if child.name == 'query':
if child['type'] == 'description':
self.log.debug('description requested')
self.dump_description(iq)
elif iq['type'] == 'set':
if iq.children[0].name == 'Envelope':
self.log.debug('received rpc')
root = iq.children[0]
# print(root.toXml())
for child in root.children:
if child.name == 'Header':
res = self.services[
child.children[0]['serviceId']]['app'].handle_rpc(
root.toXml(), child.children[0]['serviceId'])
elif child.name == 'Body':
decomposed = child.children[0].uri.split(':')
guessed_id = ':'.join(
(decomposed[0], decomposed[1],
decomposed[2] + 'Id', decomposed[3]))
res = self.services[str(guessed_id)]['app'].handle_rpc(
root.toXml(), str(guessed_id))
else:
self.log.warn('bad iq request: %s' % child.name)
continue
res.addCallback(self.respond_rpc, iq['from'], iq['id'])
def respond_rpc(self, resp, to, queryID):
# print('send: %s' % resp)
# self.log.debug('respond rpc: %s' % resp[0][39:])
res = IQ(self.xmlstream, 'result')
res['id'] = queryID
if resp:
for item in resp:
res.addRawXml(item[39:].decode('utf-8')) # Skip the xml header
res.send(to=to)
def on_presence(self, presence):
self.log.debug('received presence: %s' %
presence.toXml().encode('utf-8'))
if presence.hasAttribute('from'):
user, host, res = parse(presence['from'])
if presence['from'] in self.active_controllers:
if presence.hasAttribute('type'):
if presence['type'] == 'unavailable':
self.active_controllers.remove(presence['from'])
self.log.info('User {_from} disconnected',
_from=presence['from'])
return
elif 'ControlPoint' in res:
if presence.hasAttribute('type'):
if presence['type'] == 'unavailable':
return
self.log.info('control point %s added' % presence['from'])
if len(self.active_controllers) == 0:
self.check_ps_nodes()
self.active_controllers.append(presence['from'])
del (res)
jid = '@'.join((user, host))
if presence.hasAttribute('type'):
if presence['type'] == 'subscribe':
if jid in self.users:
self.log.info('received subscription from %s' % jid)
if self.users[jid] is False:
iq = IQ(self.xmlstream, 'set')
query = domish.Element(
('jabber:iq:roster', 'query'))
item = domish.Element((None, 'item'))
item['jid'] = jid
item['name'] = jid
item.addElement('group', content='UPnPCloud')
query.addChild(item)
iq.addChild(query)
iq.addCallback(self.subscribed, jid)
iq.send()
else:
self.log.error('subscription for user %s failed: %s' %
(jid, 'Not in user list'))
pres = domish.Element((None, 'presence'))
pres['type'] = 'unsubscribed'
pres['to'] = presence['from']
self.xmlstream.send(pres)
def subscribed(self, jid, result):
if result['type'] == 'result':
self.log.info('user %s successfully suscribed' % jid)
self.users.update({jid: True})
pres = domish.Element((None, 'presence'))
pres['type'] = 'subscribed'
pres['to'] = jid
self.xmlstream.send(pres)
else:
self.log.error('subscription for user %s failed: %s' %
(jid, result.toXml()))
pres = domish.Element((None, 'presence'))
pres['type'] = 'unsubscribed'
pres['to'] = jid
self.xmlstream.send(pres)
def init_failed(self, failure):
print "Initialization failed."
print failure
def register_art_url(self, url, cloud=False):
if cloud:
return None
newurl = hashlib.md5(url).hexdigest() + url[-4:]
self.resource.putChild(newurl, static.File(url))
return self.webserver.weburl % get_default_v4_address() + '/' + newurl
class RemoteClientBase( # pylint: disable=too-many-instance-attributes
pb.Referenceable):
PING_DELAY = 60
PING_MAX_FAILURES = 3
DEBUG = False
def __init__(self):
self.log_level = LogLevel.warn
self.log = Logger(namespace="remote", observer=self._log_observer)
self.id = app.get_host_id()
self.name = app.get_host_name()
self.join_options = {"corever": __version__}
self.perspective = None
self.agentpool = None
self._ping_id = 0
self._ping_caller = None
self._ping_counter = 0
self._reactor_stopped = False
self._exit_code = 0
@provider(ILogObserver)
def _log_observer(self, event):
if not self.DEBUG and (event["log_namespace"] != self.log.namespace
or self.log_level > event["log_level"]):
return
msg = formatEvent(event)
click.echo("%s [%s] %s" % (
datetime.fromtimestamp(
event["log_time"]).strftime("%Y-%m-%d %H:%M:%S"),
event["log_level"].name,
msg,
))
def connect(self):
self.log.info("Name: {name}", name=self.name)
self.log.info("Connecting to PlatformIO Remote Development Cloud")
# pylint: disable=protected-access
proto, options = endpoints._parse(__pioremote_endpoint__)
proto = proto[0]
factory = RemoteClientFactory()
factory.remote_client = self
factory.sslContextFactory = None
if proto == "ssl":
factory.sslContextFactory = SSLContextFactory(options["host"])
reactor.connectSSL(
options["host"],
int(options["port"]),
factory,
factory.sslContextFactory,
)
elif proto == "tcp":
reactor.connectTCP(options["host"], int(options["port"]), factory)
else:
raise exception.PlatformioException(
"Unknown PIO Remote Cloud protocol")
reactor.run()
if self._exit_code != 0:
raise exception.ReturnErrorCode(self._exit_code)
def cb_client_authorization_failed(self, err):
msg = "Bad account credentials"
if err.check(pb.Error):
msg = err.getErrorMessage()
self.log.error(msg)
self.disconnect(exit_code=1)
def cb_client_authorization_made(self, perspective):
self.log.info("Successfully authorized")
self.perspective = perspective
d = perspective.callRemote("join", self.id, self.name,
self.join_options)
d.addCallback(self._cb_client_join_made)
d.addErrback(self.cb_global_error)
def _cb_client_join_made(self, result):
code = result[0]
if code == 1:
self.agentpool = result[1]
self.agent_pool_ready()
self.restart_ping()
elif code == 2:
self.remote_service(*result[1:])
def remote_service(self, command, options):
if command == "disconnect":
self.log.error("PIO Remote Cloud disconnected: {msg}",
msg=options.get("message"))
self.disconnect()
def restart_ping(self, reset_counter=True):
# stop previous ping callers
self.stop_ping(reset_counter)
self._ping_caller = reactor.callLater(self.PING_DELAY, self._do_ping)
def _do_ping(self):
self._ping_counter += 1
self._ping_id = int(time())
d = self.perspective.callRemote("service", "ping",
{"id": self._ping_id})
d.addCallback(self._cb_pong)
d.addErrback(self._cb_pong)
def stop_ping(self, reset_counter=True):
if reset_counter:
self._ping_counter = 0
if not self._ping_caller or not self._ping_caller.active():
return
self._ping_caller.cancel()
self._ping_caller = None
def _cb_pong(self, result):
if not isinstance(result, failure.Failure) and self._ping_id == result:
self.restart_ping()
return
if self._ping_counter >= self.PING_MAX_FAILURES:
self.stop_ping()
self.perspective.broker.transport.loseConnection()
else:
self.restart_ping(reset_counter=False)
def agent_pool_ready(self):
raise NotImplementedError
def disconnect(self, exit_code=None):
self.stop_ping()
if exit_code is not None:
self._exit_code = exit_code
if reactor.running and not self._reactor_stopped:
self._reactor_stopped = True
reactor.stop()
def cb_disconnected(self, _):
self.stop_ping()
self.perspective = None
self.agentpool = None
def cb_global_error(self, err):
if err.check(pb.PBConnectionLost, defer.CancelledError):
return
msg = err.getErrorMessage()
if err.check(pb.DeadReferenceError):
msg = "Remote Client has been terminated"
elif "PioAgentNotStartedError" in str(err.type):
msg = ("Could not find active agents. Please start it before on "
"a remote machine using `pio remote agent start` command.\n"
"See http://docs.platformio.org/page/plus/pio-remote.html")
else:
maintenance.on_platformio_exception(Exception(err.type))
click.secho(msg, fg="red", err=True)
self.disconnect(exit_code=1)
class Controller(service.MultiService):
targets = {}
services = []
binary_light_list = []
hvac_list = []
media_player_list = []
messager = None
server_list = []
shutter_list = []
camera_list = []
multiscreen_list = []
dimmable_light_list = []
ambi_light_list = []
event_catcher = None
cloud_event_catcher = None
subscriptions = {}
subscriptions_cloud = {}
searchables = {}
ready_to_close = False
current_device = None
cloud = False
lan = False
agent = None
def __init__(
self, parent=None, searchables=None, xmldir=None,
network='lan', cloud_user=None, cloud_servers=[],
logger=None, uid=None, messager=None):
self.connected = False
self.messager = messager
self.app_paused = False
self.fail_count = 0
if not logger:
self.log = Logger()
else:
self.log = logger
self.log.debug('UPnP controller starts')
self.xmldir = xmldir
self.devices = {}
self._services = {}
self.parent = parent
# self.amp = ControllerAmp(self)
if uid:
self.uuid = uid
else:
self.uuid = str(
uuid.uuid5(
uuid.NAMESPACE_DNS,
socket.gethostname() + 'onDemand_Controller'))
if searchables:
for typ in searchables:
self.searchables.update({typ[0]: typ[1]})
# print(self.searchables)
else:
self.searchables = {'upnp:rootdevice': self.log.debug}
if network in ('lan', 'both'):
self.log.debug('UPnP classic enabled')
self.lan = True
self.listener = ssdp.SSDP_Listener(self)
self.mcast = internet.MulticastServer( # @UndefinedVariable
SSDP_PORT,
self.listener,
listenMultiple=True,
interface=SSDP_ADDR_V4)
self.mcast.setServiceParent(self)
self.ssdp_cli = ssdp.SSDP_Client(
self, get_default_v4_address(), device=False)
self.ucast = internet.UDPServer( # @UndefinedVariable
0, self.ssdp_cli, self.ssdp_cli.interface)
self.ucast.setServiceParent(self)
# self.agent = Agent(reactor)
if network in ('cloud', 'both'):
if cloud_user:
self.log.debug('UPnP Cloud enabled')
self.cloud = True
self._jid, secret = cloud_user
self.users = {self._jid: {'state': True}}
for user in cloud_servers:
self.users.update({user: {'state': False}})
self.hosts = {}
self.resourcepart = ''.join((
'urn:schemas-upnp-org:cloud-1-0:ControlPoint:1:uuid:',
self.uuid))
full_jid = ''.join(
(self._jid, '/', self.resourcepart))
self.jid = jid = JID(full_jid)
self.reactor = reactor
f = client.XMPPClientFactory(jid, secret)
f.addBootstrap(
xmlstream.STREAM_CONNECTED_EVENT, self.cloud_connected)
f.addBootstrap(
xmlstream.STREAM_END_EVENT, self.cloud_disconnected)
f.addBootstrap(
xmlstream.STREAM_AUTHD_EVENT, self.authenticated)
f.addBootstrap(
xmlstream.INIT_FAILED_EVENT, self.cloud_failed)
self.connector = endpoints.HostnameEndpoint(
reactor, jid.host, 5222)
self.factory = f
# factory = Factory()
# factory.protocol = ControllerAmp(self)
# amp_service = internet.TCPServer( # @UndefinedVariable
# 4343, factory)
# amp_service.setServiceParent(self)
# self.connector = SRVConnector(
# reactor, 'xmpp-client', jid.host, f, defaultPort=5222)
# log.startLogging(sys.stdout)
def startService(self):
'''
'''
service.MultiService.startService(self)
if self.cloud:
self.connector.connect(self.factory)
self.log.debug('Cloud Service started')
if self.lan:
t = task.LoopingCall(self.search_devices)
t.start(15)
self.log.debug('SSDP Service started')
def resume(self):
self.app_paused = False
if not self.connected:
if self.cloud:
self.connector.connect(self.factory)
self.log.debug('Cloud Service started')
if self.lan:
t = task.LoopingCall(self.search_devices)
t.start(15)
self.log.debug('SSDP Service started')
def stopService(self):
self.log.debug('Stopping controller service...')
self.clean()
# d.addCallback(lambda ignored: service.MultiService.stopService(self))
service.MultiService.stopService(self)
# reactor.callLater(10, reactor.stop) # @UndefinedVariable
def cloud_disconnected(self, reason):
if not reason:
reason = 'Unknown'
self.log.warn('Cloud Server disconnected: %s' % reason)
self.connected = False
if not self.app_paused and self.fail_count < 10:
self.fail_count += 1
self.resume()
def cloud_failed(self, failure):
self.log.error('Cloud Login failed: %s' % str(failure))
# self.xmlstream.sendFooter()
def clean(self):
return reactor.callInThread( # @UndefinedVariable
threads.blockingCallFromThread, *(reactor, self.cleanfunc))
def cleanfunc(self):
def cleaned(res):
self.log.debug('cleaned')
if self.cloud:
self.xmlstream.sendFooter()
dl = []
if self.lan:
for name in self.subscriptions.keys():
dl.append(self.unsubscribe(name))
if self.cloud:
for name in self.subscriptions_cloud.keys():
dl.append(self.unsubscribe(name))
d = defer.DeferredList(dl)
d.addCallback(cleaned)
return d
def cloud_connected(self, xs):
self.log.debug('Cloud Connected')
self.fail_count = 0
self.connected = True
self._services = {}
self.subscriptions = {}
self.xmlstream = xs
# xs.rawDataInFn = self.rawDataIn
def authenticated(self, xs):
self.log.debug('Cloud Authenticated')
presence = domish.Element((None, 'presence'))
xs.send(presence)
xs.addObserver('/presence', self.on_presence)
xs.addObserver('/iq', self.on_iq)
xs.addObserver('/message', self.on_event)
disco = IQ(xs, 'get')
disco.addElement(('http://jabber.org/protocol/disco#items', 'query'))
disco.addCallback(self.cloud_discovered)
disco.send()
# self.reactor.callLater(120, xs.sendFooter)
self.reactor.callLater(5, self.check_users)
def check_users(self):
for user, value in self.users.items():
if value['state'] is False:
iq = IQ(self.xmlstream, 'set')
query = domish.Element(('jabber:iq:roster', 'query'))
item = domish.Element((None, 'item'))
item['name'] = user
item['jid'] = user
item.addElement('group', content='hosts')
query.addChild(item)
iq.addChild(query)
iq.addCallback(self.cloud_subscribe, user)
# print('send IQ: %s' % (iq.toXml().encode('utf-8')))
iq.send()
def cloud_subscribe(self, jid, result):
self.log.debug('Subscribe callback from %s' % jid)
presence = domish.Element((None, 'presence'))
presence['type'] = 'subscribe'
presence['to'] = jid
self.xmlstream.send(presence)
def on_event(self, message):
if not self.cloud_event_catcher:
reactor.callLater(1, self.on_event, message) # @UndefinedVariable
return
if message.name == 'iq':
if message['type'] == 'result':
try:
last = ''
for child in message.children[0].children[0].children:
last = child.children[0]
except KeyError:
return
# print(message.toXml())
# print(last.toXml())
self.cloud_event_catcher.receive(last.toXml().encode('utf-8'))
elif message.children[0].name == 'event':
evt = message.children[0]
items = evt.children[0]
node_name = str(items['node'])
if node_name in self.subscriptions_cloud:
for item in items.children:
propertyset = item.children[0]
self.cloud_event_catcher.receive(
(node_name, propertyset.toXml().encode('utf-8'),))
def rawDataIn(self, buf):
print(
"Device RECV: %s"
% unicode(buf, 'utf-8').encode('ascii', 'replace'))
def on_presence(self, resp):
self.log.debug('got presence: %s' % resp.toXml().encode('utf-8'))
# print('from :%s' % resp['from'])
user, host, res = parse(resp['from'])
jid = '@'.join((user, host))
if resp.hasAttribute('type'):
if resp['type'] == 'subscribed':
if jid in self.users:
self.users[jid].update({'state': True})
if 'services' in self.users[jid]:
self.users[jid]['services'].append(res)
else:
self.users[jid].update({'services': [res]})
presence = domish.Element((None, 'presence'))
presence['type'] = 'subscribe'
presence['to'] = resp['from']
self.xmlstream.send(presence)
else:
presence = domish.Element((None, 'presence'))
presence['type'] = 'denying'
presence['to'] = resp['from']
self.xmlstream.send(presence)
elif resp['type'] == 'unsubscribed':
if jid in self.users:
self.log.warn('subscription failed: %s' % resp['from'])
return
for child in resp.elements():
if child.name == 'ConfigIdCloud':
self.log.debug('Found UPnP Cloud device : %s type is: %s' % (
jid,
res))
info = IQ(self.xmlstream, 'get')
# info['to'] = resp['from']
query = domish.Element(
('urn:schemas-upnp-org:cloud-1-0', 'query'))
query['type'] = 'description'
query['name'] = ':'.join(res.split(':')[-2:])
info.addChild(query)
info.addCallback(self.on_description, res)
# info.send()
info.send(to=resp['from'])
def on_description(self, resource, iq):
location = iq['from']
clbk = self.searchables[
self.searchables.keys()[0]]
if iq['type'] == 'result':
if iq.children[0].name == 'query'\
and iq.children[0]['type'] == 'described':
self.update_devices(
resource,
location,
clbk,
xml=iq.children[0].children[0].toXml())
def cloud_discovered(self, iq):
self.log.debug('Discovered item: %s' % iq.toXml().encode('utf-8'))
if iq['type'] == 'result':
for child in iq.children:
if child.name == 'query':
for grandchild in child.children:
if grandchild['jid'].encode('utf-8') == self.full_jid:
continue
if grandchild['name'].encode('utf-8')\
in self.hosts:
self.hosts[
grandchild['name'].encode('utf-8')].append(
grandchild['jid'].encode('utf-8'))
else:
self.hosts.update(
{grandchild['name'].encode('utf-8'):
[grandchild['jid'].encode('utf-8')]})
# print(self.hosts)
def on_iq(self, iq):
pass
# print('got iq: %s' % iq.toXml())
# try:
# print('from :%s' % iq['from'])
# except KeyError:
# print('From I don\'t know: %s' % iq.toXml())
# print('type: %s' % iq['type'])
def search_devices(self):
for search in self.searchables:
self.ssdp_cli.send_MSEARCH(search, uuid=self.uuid)
def update_hosts(self, host, unicast=False):
if 'location' in host:
if 'usn' in host:
if host['usn'] in self.devices:
return
device = host['usn'].split('::')
if len(device) > 1:
uid = device[0].split(':')[1]
if uid in self.devices:
return
typ = device[1]
if typ in self.searchables:
self.update_devices(
uid, host['location'], self.searchables[typ])
# self.devices.append(uid)
def send_message(self, message_type, name, id_, value):
if self.messager:
if isinstance(value, dict):
self.messager.callRemote(message_type,
name=name,
id_=id_,
value=json.dumps(value))
# for v in value.iteritems():
# if not v or isinstance(v, dict):
# print('zap')
# continue
# print(v)
# self.messager.callRemote(message_type,
# name=name,
# id_=id_,
# value=':'.join((k, v)))
else:
self.messager.callRemote(message_type,
name=name,
id_=id_,
value=value)
def update_devices(self, uid, location, callback_fct, xml=None):
def device_parsed(dic):
self.devices.update(dic)
if callable(callback_fct):
callback_fct(dic)
else:
self.send_message(Event, callback_fct, uid, dic[uid])
if self.messager:
self.messager.parent.notify(
'New Device detected:', dic[uid]['name'])
uid = bytes(uid)
self.log.debug('new device %s: %s' % (uid, location))
if '@' in location:
if xml:
device_parsed(self.parse_host(xml, location, uid))
return
else:
if not self.agent:
self.agent = Agent(reactor)
d = self.agent.request('GET', location)
d.addCallback(readBody)
d.addCallback(self.parse_host, *(location, uid))
d.addCallback(device_parsed)
def parse_host(self, xml, location, uid):
typ = 'upnp'
loc = None
if '@' in location:
url_prefix = ''.join(('xmpp://', location))
net = 'cloud'
else:
url_prefix = urlparse(location).netloc
net = 'lan'
try:
root = et.fromstring(xml)
except:
self.log.error('bad xml: %s' % xml)
return {}
host = {}
icon = None
for children in root:
if children.tag.split('}')[-1] == 'device':
for att in children:
if att.tag.split('}')[-1] == 'friendlyName':
fname = att.text
if att.tag.split('}')[-1] == 'deviceType':
devtype = att.text
if 'Source' in att.text:
typ = 'oh'
if att.tag.split('}')[-1] == 'iconList':
for ico in att:
# log.debug(ico)
for info in ico:
if info.tag.split('}')[-1] == 'width':
if int(info.text) <= 96:
if ico[4].text.startswith('/'):
icon = 'http://'\
+ url_prefix\
+ ico[4].text
else:
icon = ico[4].text
if att.tag.split('}')[-1] == 'serviceList':
svc = {}
for serv in att:
d = {}
for info in serv:
if 'URL' in info.tag.split('}')[-1]:
if net == 'lan':
d.update({info.tag.split('}')[-1]:
'http://' +
url_prefix + info.text})
else:
d.update(
{info.tag.split('}')[-1]:
url_prefix + info.text})
else:
d.update(
{info.tag.split('}')[-1]: info.text})
svc.update({d['serviceType']: d})
if att.tag.split('}')[-1] == 'X_location':
loc = att.text
host.update(
{uid: {
'name': fname,
'devtype': devtype,
'icon': icon,
'services': svc,
'type': typ,
'network': net,
'location': location,
'loc': loc}})
# log.debug(host)
return host
def subscribe(self, *args, **kwargs):
if args[0][args[0].keys()[0]]['network'] == 'lan':
return self.subscribe_classic(*args, **kwargs)
else:
return self.subscribe_cloud(*args, **kwargs)
def subscribe_classic(
self, device, svc, var, callback_fct=None,
callback_args=()):
if not callback_fct:
callback_fct = self.log.debug
name = device.keys()[0]
dev = device[name]
def subscribe_failed(err, name):
self.parent.remove_device(name.split('_')[0])
def subscribed(req, raddr, host, name):
try:
uuid = req.headers.getRawHeaders('sid')[0]
print('subscription uuid = %s' % uuid)
if name in self.subscriptions:
if host in self.subscriptions[name]:
self.subscriptions[name][host].update({uuid: raddr})
else:
self.subscriptions[name].update({host: {uuid: raddr}})
else:
self.subscriptions.update({name: {host: {uuid: raddr}}})
reactor.callLater( # @UndefinedVariable
20, self.renew_subscription, uuid)
return name
except TypeError:
return subscribe_failed(None, name)
if self.event_catcher is None:
self.event_catcher = EventServer()
self.event_catcher.setServiceParent(self)
subscription_id = '_'.join((name, svc.split(':')[-2]))
childpath = '_'.join((subscription_id, 'event',))
# log.error(childpath)
if childpath in self.event_catcher.catcher.childs:
self.event_catcher.catcher.childs[childpath].update(
{var: (callback_fct, callback_args,)})
else:
self.event_catcher.catcher.childs.update(
{childpath: {var: (callback_fct, callback_args,)}})
# log.error(self.event_catcher.catcher.childs)
if subscription_id in self.subscriptions:
for k, value in self.event_catcher.catcher.unfiltered.items():
if k == var:
if value == 'False':
value = False
elif value == 'True':
value = True
if isinstance(callback_args, str)\
or isinstance(callback_args, bool):
callback_fct(value, callback_args)
else:
callback_fct(value, *callback_args)
del self.event_catcher.catcher.unfiltered[k]
return defer.succeed(None)
else:
self.subscriptions.update({subscription_id: {}})
clbk = '<' + 'http://' + get_default_v4_address() + ':' +\
str(self.event_catcher.getPort()) + '/' + childpath + '>'
# print(clbk)
headers = {'HOST': [get_default_v4_address() + ':' +
str(self.event_catcher.getPort())],
'CALLBACK': [clbk],
'NT': ['upnp:event'],
'TIMEOUT': ['Second-25']}
if svc in dev['services']:
self.log.error(svc)
addr = dev['services'][svc]['eventSubURL']
self.log.error(addr)
d = self.agent.request(
'SUBSCRIBE',
addr,
Headers(headers))
d.addCallbacks(
subscribed,
subscribe_failed,
callbackArgs=(addr, headers['HOST'][0], subscription_id),
errbackArgs=(subscription_id,))
return d
# log.error(dev['services'])
return defer.fail(Exception('Service unknow'))
def renew_subscription(self, sid):
def renewed(res):
# print('subscription %s successfully renewed' % sid)
reactor.callLater( # @UndefinedVariable
20, self.renew_subscription, sid)
def failed(res):
for name in self.subscriptions:
for host in self.subscriptions[name]:
if sid in self.subscriptions[name][host]:
del self.subscriptions[name][host][sid]
self.parent.remove_device(name.split('_')[0])
for name in self.subscriptions:
for host in self.subscriptions[name]:
if sid in self.subscriptions[name][host]:
headers = {'HOST': [host], 'SID': [sid],
'TIMEOUT': ['Second-25']}
d = self.agent.request(
'SUBSCRIBE',
self.subscriptions[name][host][sid],
Headers(headers))
d.addCallbacks(renewed, failed)
return d
def unsubscribe(self, name):
print('unsuscribe: %s' % name)
deferreds = []
if name in self.subscriptions:
for host in self.subscriptions[name]:
for sid in self.subscriptions[name][host]:
deferreds.append(self.unsubscribe_host(
sid,
host,
self.subscriptions[name][host][sid], name))
if name in self.subscriptions_cloud:
return self.unsubscribe_cloud(name)
if len(deferreds) > 0:
# print(deferreds)
d = defer.DeferredList(deferreds)
else:
d = defer.succeed('nothing to do')
return d
def unsubscribe_cloud(self, name):
def unsubscribed(name, d, res):
if res['type'] == 'result':
# print('unsubscribed: %s' % name)
del self.subscriptions_cloud[name]
print('ok')
d.callback(None)
else:
d.errback(Exception(res.toXml()))
d = defer.Deferred()
iq = IQ(self.xmlstream, 'set')
ps = domish.Element(('http://jabber.org/protocol/pubsub', 'pubsub'))
unsubscribe = domish.Element((None, 'unsubscribe'))
unsubscribe['node'] = name
unsubscribe['jid'] = self.jid.full()
ps.addChild(unsubscribe)
iq.addChild(ps)
iq.addCallback(unsubscribed, name, d)
iq.send(to='pubsub.' + self.jid.host)
return d
def unsubscribe_host(self, sid, host, addr, name=None):
# log.debug(
# 'unsubscribe uuid host addr: %s %s %s' % (sid, host, addr))
def unsubscribed(res):
# print('subscription %s successfully cancelled' % sid)
if name:
if len(self.subscriptions[name][host]) == 1:
del self.subscriptions[name]
else:
del self.subscriptions[name][host][sid]
return res
headers = {'HOST': [host], 'SID': [sid]}
d = self.agent.request(
'UNSUBSCRIBE',
addr,
Headers(headers))
d.addCallback(unsubscribed)
return d
def subscribe_cloud(
self, device, svc, var, callback_fct=None, callback_args=()):
# print('suscribe to %s' % var)
name = device.keys()[0]
dev = device[name]
if not callback_fct:
callback_fct = self.log.debug
d = defer.Deferred()
def subscribe_failed(err, name):
self.parent.remove_device(name.split('_')[0])
def subscribed(node_name, deferred, iq):
if iq['type'] == 'result':
self.subscriptions_cloud[str(node_name)] = True
# print('%s suscribed !' % str(node_name))
# iq = IQ(self.xmlstream, 'get')
# ps = domish.Element(
# ('http://jabber.org/protocol/pubsub', 'pubsub'))
# items = domish.Element((None, 'items'))
# items['node'] = node_name
# items['max_items'] = '1'
# ps.addChild(items)
# iq.addChild(ps)
# iq.addCallback(self.on_event)
# iq.send(to='pubsub.' + self.jid.host)
# print(iq.toXml())
deferred.callback(str(node_name))
else:
deferred.errback(Exception('subscription to %s failed: %s'
% (node_name, iq.toXml())))
if svc in dev['services']:
# print('service %s ok' % svc)
# print('subscriptions :%s' % self.subscriptions_cloud)
if not self.cloud_event_catcher:
self.cloud_event_catcher = CloudEventCatcher(
{}, {}, logger=self.log)
subscription_name = '/'.join((dev['location'], svc, var))
# subscription_service = svc
if subscription_name in self.cloud_event_catcher.callbacks:
self.cloud_event_catcher.callbacks[subscription_name].update(
{var: (callback_fct, callback_args,)})
else:
self.cloud_event_catcher.callbacks.update(
{subscription_name: {var: (callback_fct, callback_args,)}})
# if var in self.cloud_event_catcher.callbacks:
# self.cloud_event_catcher.callbacks[var].update(
# {var: (callback_fct, callback_args,)})
# else:
# self.cloud_event_catcher.callbacks.update(
# {var: {var: (callback_fct, callback_args,)}})
# log.error(self.event_catcher.catcher.childs)
if subscription_name in self.subscriptions_cloud:
if self.subscriptions_cloud[subscription_name]:
# print('already subscribed: %s' % subscription_name)
for k, value in\
self.cloud_event_catcher.unfiltered_dict.items():
# print('is %s == %s ?' % (k, var))
if k == var:
if value == 'False':
value = False
elif value == 'True':
value = True
if isinstance(callback_args, str)\
or isinstance(callback_args, bool):
callback_fct(value, callback_args)
else:
callback_fct(value, *callback_args)
del self.cloud_event_catcher.unfiltered_dict[k]
return defer.succeed(None)
self.subscriptions_cloud.update({str(subscription_name): False})
# print(subscription_name)
# print(subscription_service)
iq = IQ(self.xmlstream, 'set')
ps = domish.Element(
('http://jabber.org/protocol/pubsub', 'pubsub'))
subscribe = domish.Element((None, 'subscribe'))
subscribe['node'] = subscription_name
subscribe['jid'] = self.jid.full()
ps.addChild(subscribe)
iq.addChild(ps)
iq.addCallback(subscribed, subscription_name, d)
iq.send(to='pubsub.' + self.jid.host)
return d
return defer.fail(Exception('Service unknow'))
def get_client(self, device, service):
if self.xmldir is not None:
client = None
else:
import importlib
module_name = service.split(':')[-2]
app = getattr(importlib.import_module(
'upnpy_spyne.services.templates.' + module_name.lower()),
module_name)
if device['network'] == 'lan':
client = Client(
device['services'][service]['controlURL'],
Application([app], app.tns,
in_protocol=Soap11(), out_protocol=Soap11()))
client.set_options(
out_header={'Content-Type': ['text/xml;charset="utf-8"'],
'Soapaction': [app.tns]})
else:
url = (self.xmlstream, device['location'],)
client = Client(
url,
Application([app], app.tns,
in_protocol=Soap11(xml_declaration=False),
out_protocol=Soap11(xml_declaration=False)),
cloud=True)
# print('**********%s' % service)
# print(device['services'][service])
return client
def call(self, device, service, func, params=()):
if isinstance(device, dict):
devname = device.keys()[0]
dev = device[devname]
else:
devname = device
dev = self.devices[device]
if devname not in self._services:
client = self.get_client(dev, service)
self._services.update({devname: {service: client.service}})
elif service not in self._services[devname]:
client = self.get_client(dev, service)
self._services[devname].update({service: client.service})
try:
f = getattr(
self._services[devname][service], func)
except AttributeError:
self.log.error(
'function %s not found for service %s' % (func, service))
return defer.fail(Exception(
'function %s not found for service %s' % (func, service)))
try:
if len(params) > 0:
if isinstance(params, str):
d = f(params)
else:
d = f(*params)
else:
d = f()
except TypeError:
# boolean has no len
d = f(params)
d.addErrback(
lambda failure, fname: self.log.error(
'%s call failed : %s' % (fname, failure.getErrorMessage())),
func)
return d
class MpdProtocol(LineReceiver):
'''
Twisted protocol to control remote mpd server
'''
def __init__(self):
'''
doc
'''
self.log = Logger()
self.delimiter = "\n"
self.deferreds = []
self.buff = {}
self.idle = False
self.list_index = 0
def connectionLost(self, reason):
self.log.error('connection lost : {reason}', reason=reason)
self._event({'changed': 'disconnected'})
self.idle = False
try:
d = self.deferreds.pop(0)
except:
pass
else:
d.errback(reason)
def connectionMade(self):
self.log.debug('connected')
def addCallback(self, d):
self.deferreds.append(d)
def noidle(self):
d = defer.Deferred()
d.addCallback(lambda ignored: ignored)
self.deferreds.insert(0, d)
self.sendLine('noidle')
self.idle = False
# print('noidle')
def set_idle(self):
self.sendLine('idle')
self.idle = True
# print('idle')
def lineReceived(self, line):
# print(line)
if line.startswith('OK MPD'):
self._event({'changed': 'connected'})
elif line.startswith('OK'):
# print('deferred length: %d' % len(self.deferreds))
self.list_index = 1
try:
d = self.deferreds.pop(0)
except:
self.set_idle()
self._event(self.buff)
self.buff = {}
return
else:
d.callback(self.buff)
self.buff = {}
elif line.startswith('ACK'):
# print('deferred length: %d' % len(self.deferreds))
try:
d = self.deferreds.pop(0)
except:
self.set_idle()
self._event({'Error': line.split('}')[1]})
self.buff = {}
return
else:
d.errback(Exception(line.split('}')[1]))
self.buff = {}
else:
if len(line) > 0:
k = line.split(':')[0]
if isinstance(self.buff, list):
if k in self.buff[self.list_index]:
self.list_index += 1
self.buff.append({})
self.buff[self.list_index].update(
{k: ' '.join(line.split()[1:])})
else:
if k in self.buff:
self.buff = [self.buff]
self.list_index = 1
self.buff.append({k: ' '.join(line.split()[1:])})
# if str(self.list_index) + k in self.buff:
# self.list_index += 1
# self.buff.update(
# {str(self.list_index) + line.split(':')[0]:
# ' '.join(line.split()[1:])})
else:
self.buff.update({k: ' '.join(line.split()[1:])})
return
if len(self.deferreds) == 0:
self.set_idle()
class TimeSchedule:
def __init__(self, lock, host='127.0.0.1', port='6800'):
config = Config()
self.db = glv.get_value(key='sqlite_db')
self.user_name = config.get('auth_username', '')
self.user_password = config.get('auth_password', '')
self.start_time = time.strftime("%Y %m %d %H %M %S", time.localtime())
self.server_port = 'http://{}:{}/'.format(host, port)
self.schedule_post_url = 'http://{}:{}/schedule.json'.format(host, port)
self.listproject_url = 'http://{}:{}/listprojects.json'.format(host, port)
self.spider_task_dic = dict()
self.projects = None
self.db_lock = lock
self.ts_lock = threading.Lock()
self._keys_set = {
"year",
"month",
"day",
"week",
"hour",
"minute",
"second",
"y",
"m",
"d",
"w",
"H",
"M",
"S",
}
self._keys_dic = {
"y": "year",
"m": "month",
"d": "day",
"w": "week",
"H": "hour",
"M": "minute",
"S": "second",
}
self._keys_set_lis = [[y for y in x] for x in self._keys_set]
self.CPU_THRESHOLD = 93
self.MEMORY_THRESHOLD = 96
self.schedule_logger = Logger(namespace='- Scheduler -')
def run(self):
time.sleep(3)
self.projects = self.list_projects()
self.schedule_logger.info('scheduler is running')
count = 1
while True:
schedule_sta = self.task_scheduler()
if not schedule_sta and count == 1:
self.schedule_logger.info('No Scheduled Spider in Database')
count += 1
elif not schedule_sta and count != 1:
count += 1
else:
count = 1
time.sleep(1)
def task_scheduler(self):
self.ts_lock.acquire(blocking=True)
self.db_lock.acquire()
db_result = self.db.get(model_name='SpiderScheduleModel',
key_list=['hash_str', 'project', 'spider', 'schedule', 'args', 'runtime', 'status'])
self.db_lock.release()
self.ts_lock.release()
schedule_list_raw = [
{'hash_str': x.hash_str, 'project': x.project, 'spider': x.spider, 'schedule': x.schedule, 'args': x.args, 'runtime': x.runtime,
'status': x.status}
for x in db_result if int(x.status) != 0
] if db_result else []
schedule_sta = False
if schedule_list_raw:
for each_schedule in schedule_list_raw:
project = each_schedule.get('project')
runtime = int(each_schedule.get('runtime'))
if project in self.projects and runtime > 0:
schedule = each_schedule.get('schedule')
if any([x in schedule for x in self._keys_set]):
try:
schedule = json.loads(schedule)
except:
schedule = eval(schedule)
try:
if isinstance(schedule, dict):
for key in schedule.keys():
if key not in self._keys_set:
mean_key = self._check_key(key)
raise ValueError(
'found "{}" in your schedule dict, maybe you mean "{}"'.format(key, mean_key))
if key in self._keys_dic:
val = schedule.pop(key)
schedule[self._keys_dic[key]] = val
next_time_sep = self.cal_time_sep(**schedule)
else:
next_time_sep = self.cal_time_sep(schedule_str=schedule, is_str=True)
next_time_sep = int(next_time_sep) + 1
if next_time_sep > 1:
each_schedule['schedule'] = next_time_sep
item = '{}-{}'.format(each_schedule['project'], each_schedule['spider'])
self.ts_lock.acquire(blocking=True)
if self.spider_task_dic.get(item) != 'waiting':
self.spider_task_dic[item] = 'waiting'
t = threading.Thread(target=self.poster, args=(each_schedule,))
try:
t.start()
except Exception as THError:
self.schedule_logger.warn('start new job error [ {} ]: {}'.format(item, THError))
self.ts_lock.release()
except ValueError as V:
self.schedule_logger.error('spider runtime schedule error, please check the database: {}'.format(V))
schedule_sta = True
return schedule_sta
def poster(self, dic):
hash_str = dic.get('hash_str')
status = int(dic.pop('status'))
project = dic.get('project')
spider = dic.get('spider')
job_str = " %s-%s " % (project, spider)
args = dic.get('args')
try:
args = json.loads(args)
except:
args = eval(args)
wait_time = dic.get('schedule')
item = '{}-{}'.format(project, spider)
if project and spider:
data = {'project': project, 'spider': spider, 'un': self.user_name, 'pwd': self.user_password}
if args:
args = self._spider_args_method(args, hash_str)
data.update(args)
self.schedule_logger.info('job {} is waiting, countdown {}s'.format(item, wait_time))
time.sleep(wait_time - 1)
another_wait_time = 0
spider_runtime_avg = self.spiders_runtime(project=project, spider=spider)
if status == 1:
while not self.is_system_ok():
self.schedule_logger.warn('system is fully functioning, wait another 2 seconds to post schedule')
time.sleep(2)
another_wait_time += 3
if another_wait_time >= (wait_time - spider_runtime_avg):
self.schedule_logger.warning('wait too long, cancel the job %s' % job_str)
return None
res = json.loads(requests.post(url=self.schedule_post_url, data=data).content)
elif status == 2:
res = json.loads(requests.post(url=self.schedule_post_url, data=data).content)
elif status == 3:
res = json.loads(requests.post(url=self.schedule_post_url, data=data).content)
else:
res = json.loads(requests.post(url=self.schedule_post_url, data=data).content)
spider_status = res.get('status')
if spider_status != 'ok':
spider_status = 'error'
else:
self.schedule_logger.error('job project: {}, spider: {} post fail!'.format(project, spider))
spider_status = 'error'
self.ts_lock.acquire(blocking=True)
if spider_status == 'ok':
self._run_countdown(project=project, spider=spider)
self.spider_task_dic[item] = spider_status
self.ts_lock.release()
def _spider_args_method(self, args, hash_str):
args_raw = args.copy()
if args:
method = args.pop('method', 'normal')
if method == 'auto_increment':
next_args = {k: str(int(v)+1) if isinstance(v, int) or (isinstance(v, str) and v.isdigit()) else v for k, v in args.items()}
elif isinstance(method, dict):
ex_md = method.get('expression')
fc_md = method.get('function')
if ex_md:
next_args = eval(ex_md)
if fc_md:
exec(fc_md)
else:
next_args = args
next_args.update({'method': method})
self.db.update('SpiderScheduleModel', update_dic={'args': next_args}, filter_dic={"hash_str": hash_str})
return args
return args_raw
def spiders_runtime(self, project, spider):
self.db_lock.acquire()
res = self.db.get(model_name='SpiderMonitor', key_list=['runtime'],
filter_dic={'project': project, 'spider': spider})
self.db_lock.release()
spider_list = [int(x.runtime) for x in res if x.runtime.isdigit()] if res else [0]
return sum(spider_list) / len(spider_list)
def list_projects(self):
res = requests.get(url=self.listproject_url)
projects = {}
if res:
projects_list = json.loads(res.content).get('projects')
if projects_list:
projects = set(projects_list)
return projects
def cal_time_sep(self,
year='*',
month='*',
day='*',
week='*',
hour='*',
minute='*',
second='*',
schedule_str=None,
is_str=False
):
"""
"%Y-%m-%d %H:%M:%S %w"
"""
if is_str:
s = [int(x.strip()) for x in schedule_str.split(',')]
time_sep = (datetime.datetime(s[0], s[1], s[2], s[3], s[4], s[5]) - datetime.datetime.now()).total_seconds()
return time_sep
y = int(time.strftime("%Y", time.localtime()))
if year != '*' and '*' in year:
y = int(year.split('/')[-1]) + y
elif year.isdigit():
y = int(year)
if week == '*':
m = int(time.strftime("%m", time.localtime()))
if month != '*' and '*' in month:
m_raw = int(month.split('/')[-1])
if m_raw >= 12:
raise ValueError('month value is too large, please set the year instead')
m = m_raw + m
if m > 12:
y += m // 12
m = m % 12
elif month.isdigit():
m = int(month)
days_in_this_month = self.how_many_days_in_this_month(y, m)
d = int(time.strftime("%d", time.localtime()))
if day != '*' and '*' in day:
d_raw = int(day.split('/')[-1])
if d_raw > days_in_this_month:
raise ValueError('day value is too large, please set the month or the year instead')
d = d_raw + d
if d > days_in_this_month:
d = d - days_in_this_month
m += 1
if m > 12:
y += 1
m = m - 12
elif day.isdigit():
d = int(day)
days_in_this_month = self.how_many_days_in_this_month(y, m)
H = int(time.strftime("%H", time.localtime()))
if hour != '*' and '*' in hour:
H_raw = int(hour.split('/')[-1])
if H_raw > 24:
raise ValueError('hour value is too large, please set the day instead')
H = H_raw + H
if H >= 24:
H = H - 24
d += 1
if d > days_in_this_month:
d = d - days_in_this_month
m += 1
if m > 12:
y += 1
m = m - 12
elif hour.isdigit():
H = int(hour)
days_in_this_month = self.how_many_days_in_this_month(y, m)
M = int(time.strftime("%M", time.localtime()))
if minute != '*' and '*' in minute:
M_raw = int(minute.split('/')[-1])
if M_raw > 60:
raise ValueError('minute value is too large, please set the hour instead')
M = M_raw + M
if M >= 60:
M = M - 60
H += 1
if H >= 24:
H = H - 24
d += 1
if d > days_in_this_month:
d = d - days_in_this_month
m += 1
if m > 12:
y += 1
m = m - 12
elif minute.isdigit():
M = int(minute)
days_in_this_month = self.how_many_days_in_this_month(y, m)
S = int(time.strftime("%S", time.localtime()))
if second != '*' and '*' in second:
S_raw = int(second.split('/')[-1])
if S_raw > 60:
raise ValueError('second value is too large, please set the minute instead')
S = S_raw + S
if S >= 60:
S = S - 60
M += 1
if M >= 60:
M = M - 60
H += 1
if H >= 24:
H = H - 24
d += 1
if d > days_in_this_month:
d = d - days_in_this_month
m += 1
if m > 12:
y += 1
m = m - 12
elif second.isdigit():
S = int(second)
time_sep = eval(
"(datetime.datetime({},{},{}, {},{},{}) - datetime.datetime.now()).total_seconds()".format(y, m, d, H,
M, S))
else:
week_in_this_year = int(time.strftime("%U", time.localtime()))
w = int(time.strftime("%w", time.localtime()))
if '*' in week:
w_raw = int(week.split('/')[-1])
if w_raw >= 7:
raise ValueError('week value is too large, please set the day or the month instead')
if w_raw < w:
week_in_this_year += 1
w = w_raw
if week_in_this_year > 53:
y += 1
week_in_this_year = week_in_this_year - 53
elif week.isdigit():
w = int(week)
if int(week) < w:
week_in_this_year += 1
H = int(time.strftime("%H", time.localtime()))
if hour != '*' and '*' in hour:
H_raw = int(hour.split('/')[-1])
if H_raw >= 24:
raise ValueError('hour value is too large, please set the day instead')
H = H_raw + H
if H >= 24:
H = H - 24
w += 1
if w >= 7:
w = w - 7
week_in_this_year += 1
if week_in_this_year > 53:
y += 1
week_in_this_year = week_in_this_year - 53
elif hour.isdigit():
H = int(hour)
M = int(time.strftime("%M", time.localtime()))
if minute != '*' and '*' in minute:
M_raw = int(minute.split('/')[-1])
if M_raw >= 60:
raise ValueError('minute value is too large, please set the hour instead')
M = M_raw + M
if M >= 60:
M = M - 60
H += 1
if H >= 24:
H = H - 24
w += 1
if w > 7:
w = w - 7
week_in_this_year += 1
if week_in_this_year > 53:
y += 1
week_in_this_year = week_in_this_year - 53
elif minute.isdigit():
M = int(minute)
S = int(time.strftime("%S", time.localtime()))
if second != '*' and '*' in second:
S_raw = int(second.split('/')[-1])
if S_raw >= 60:
raise ValueError('second value is too large, please set the minute instead')
S = S_raw + S
if S >= 60:
S = S - 60
M += 1
if M >= 60:
M = M - 60
H += 1
if H >= 24:
H = H - 24
w += 1
if w > 7:
w = w - 7
week_in_this_year += 1
if week_in_this_year > 53:
y += 1
week_in_this_year = week_in_this_year - 53
elif second.isdigit():
S = int(second)
if S >= 60:
S = S - 60
M += 1
if M >= 60:
M = M - 60
H += 1
if H >= 24:
H = H - 24
w += 1
if w > 7:
w = w - 7
week_in_this_year += 1
if week_in_this_year > 53:
y += 1
week_in_this_year = week_in_this_year - 53
m, d = self.get_month_and_days_by_week(year=y, week_in_this_year=week_in_this_year, week=w)
time_sep = eval(
"(datetime.datetime({},{},{}, {},{},{}) - datetime.datetime.now()).total_seconds()".format(y, m, d, H,
M, S))
return time_sep
def get_month_and_days_by_week(self, year, week_in_this_year, week):
days = week_in_this_year * 7 + week
if (year % 4 == 0 and year % 100 != 0) or (year % 400 == 0):
Fe = 29
else:
Fe = 28
month_lis = [31, Fe, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]
month_count = 1
days_count = 0
for month_days in month_lis:
days = days - month_days
if days > 0:
month_count += 1
elif days == 0:
days_count = 0
month_count += 1
break
else:
days_count = days + month_days
break
return [month_count, days_count]
def how_many_days_in_this_month(self, y, m):
if m in (1, 3, 5, 7, 8, 10, 12):
days = 31
elif m in (4, 6, 9, 11):
days = 30
else:
if (y % 4 == 0 and y % 100 != 0) or (y % 400 == 0):
days = 29
else:
days = 28
return days
def is_system_ok(self):
is_pass = True
cpu_list = psutil.cpu_percent(interval=1, percpu=True)
memory_percent = psutil.virtual_memory().percent
if cpu_list and memory_percent:
is_cpu_ok = True
if min(cpu_list) > self.CPU_THRESHOLD:
is_cpu_ok = False
is_memo_ok = True
if memory_percent > self.MEMORY_THRESHOLD:
is_memo_ok = False
if not is_cpu_ok or not is_memo_ok:
is_pass = False
return is_pass
def _check_key(self, key):
key_lis = [x for x in key]
count_dic = dict()
for ksl in self._keys_set_lis:
o_key = ''.join(ksl)
score = 0
for k in key_lis:
if k in ksl:
score += 1
count_dic[o_key] = score
best_math = sorted(count_dic, key=count_dic.__getitem__, reverse=True)[0]
return best_math
def _run_countdown(self, project, spider):
db_schedule = self.db.get(model_name='SpiderScheduleModel', key_list=['id', 'runtime'],
filter_dic={'project': project, 'spider': spider})
run_time_in_db = [x.runtime for x in db_schedule][0] if db_schedule else 0
the_id = [x.id for x in db_schedule][0] if db_schedule else None
if run_time_in_db > 0 and the_id is not None:
rt = int(run_time_in_db) - 1
self.db.update(model_name='SpiderScheduleModel', update_dic={"runtime": rt}, filter_dic={"id": the_id})
class JWTClientIPAuthResource(resource.Resource):
"""
Validates JWT token passed in via a HTTP query parameter. Sets a session
cookie in order to authenticate subsequent requests.
"""
isLeaf = True
def __init__(self, param: bytes, cookie: bytes, header: bytes,
keyfile: str, sessttl: int):
self.log = Logger()
self.key = jwt.JWK()
self.param = param
self.cookie = cookie
self.header = header
# Very naive session store. Extract and improve if necessary.
self.sessttl = sessttl
self.sessions = set()
with open(keyfile, 'rb') as stream:
self.key.import_from_pem(stream.read())
def render(self, request: server.Request) -> bytes:
# Deny by default.
request.setResponseCode(401)
# Get session cookie value if any.
sessionid = request.getCookie(self.cookie)
if sessionid is not None:
if sessionid in self.sessions:
request.setResponseCode(200)
self.log.info("Session: Validation succeeded")
return b""
else:
self.log.info("Session: Invalid session id")
# Token is passed as a query parameter in the original URL.
origurl = http.urlparse(request.getHeader(self.header))
query = http.parse_qs(origurl.query)
args = query.get(self.param, [])
if len(args) != 1:
self.log.error("Request: Token {param} missing", param=self.param)
return b""
try:
token = jwt.JWT(key=self.key, jwt=args[0].decode())
except (jwt.JWTExpired, jwt.JWTNotYetValid, jwt.JWTMissingClaim,
jwt.JWTInvalidClaimValue, jwt.JWTInvalidClaimFormat,
jwt.JWTMissingKeyID, jwt.JWTMissingKey) as error:
self.log.error("JWT token: {error}", error=error)
return b""
except Exception:
self.log.failure("JWT token: Unknown exception")
return b""
try:
claims = json.loads(token.claims)
except json.JSONDecodeError as error:
self.log.failure("JWT token: Claims {error}", error=error)
return b""
# Collect session parameters from claims.
sessparams = claims.get("session", {})
kwargs = {
"expires": sessparams.get("expires", None),
"domain": sessparams.get("domain", None),
"path": sessparams.get("path", None),
"secure": sessparams.get("secure", None),
"httpOnly": sessparams.get("httpOnly", None),
"sameSite": sessparams.get("sameSite", None),
}
# Use maxAge for session ttl if it is present, convert it into a str
# type as required by the addCookie call.
if "maxAge" in sessparams:
kwargs["max_age"] = str(sessparams["maxAge"])
sessttl = int(sessparams["maxAge"])
else:
sessttl = self.sessttl
# Generate a new session id and remember it. Also clean it up after
# ttl seconds.
sessionid = secrets.token_urlsafe(nbytes=16).encode()
self.sessions.add(sessionid)
reactor.callLater(sessttl, self._session_remove, sessionid)
self.log.info("Session: Created, num sessions: {sessions}",
sessions=len(self.sessions))
# Set cookie in the browser.
request.addCookie(self.cookie, sessionid, **kwargs)
request.setResponseCode(200)
self.log.info("JWT token: Validation succeeded")
return b""
def _session_remove(self, sessionid: bytes):
self.sessions.remove(sessionid)
self.log.info("Session: Removed, num sessions: {sessions}",
sessions=len(self.sessions))
class FCMv1Router(FCMRouter):
"""FCM v1 HTTP Router Implementation
Note: FCM v1 is a newer version of the FCM HTTP API.
"""
def __init__(self, conf, router_conf, metrics):
"""Create a new FCM router and connect to FCM"""
self.conf = conf
self.router_conf = router_conf
self.metrics = metrics
self.min_ttl = router_conf.get("ttl", 60)
self.dryRun = router_conf.get("dryrun", False)
self.collapseKey = router_conf.get("collapseKey", "webpush")
self.version = router_conf["version"]
self.log = Logger()
self.clients = {}
try:
for (sid, creds) in router_conf["creds"].items():
self.clients[sid] = FCMv1(project_id=creds["projectid"],
service_cred_path=creds["auth"],
logger=self.log,
metrics=self.metrics)
except Exception as e:
self.log.error("Could not instantiate FCMv1: missing credentials,",
ex=e)
raise IOError("FCMv1 Bridge not initiated in main")
self._base_tags = ["platform:fcmv1"]
self.log.debug("Starting FCMv1 router...")
def amend_endpoint_response(self, response, router_data):
# type: (JSONDict, JSONDict) -> None
response["senderid"] = router_data["app_id"]
def register(self, uaid, router_data, app_id, *args, **kwargs):
# type: (str, JSONDict, str, *Any, **Any) -> None
"""Validate that the FCM Instance Token is in the ``router_data``"""
# "token" is the FCM token generated by the client.
if "token" not in router_data:
raise self._error("connect info missing FCM Instance 'token'",
status=401,
uri=kwargs.get('uri'),
senderid=repr(app_id))
if app_id not in self.clients:
raise self._error("Invalid SenderID", status=410, errno=105)
router_data["app_id"] = app_id
def route_notification(self, notification, uaid_data):
"""Start the FCM notification routing, returns a deferred"""
router_data = uaid_data["router_data"]
return self._route(notification, router_data)
def _route(self, notification, router_data):
"""Blocking FCM call to route the notification"""
# THIS MUST MATCH THE CHANNELID GENERATED BY THE REGISTRATION SERVICE
# Currently this value is in hex form.
data = {"chid": notification.channel_id.hex}
if not router_data.get("token"):
raise self._error(
"No registration token found. "
"Rejecting message.",
410,
errno=106,
log_exception=False)
# Payload data is optional. The endpoint handler validates that the
# correct encryption headers are included with the data.
if notification.data:
mdata = self.router_conf.get('max_data', 4096)
if notification.data_length > mdata:
raise self._error("This message is intended for a " +
"constrained device and is limited " +
"to 3070 bytes. Converted buffer too " +
"long by %d bytes" %
(notification.data_length - mdata),
413,
errno=104,
log_exception=False)
data['body'] = notification.data
data['con'] = notification.headers['encoding']
if 'encryption' in notification.headers:
data['enc'] = notification.headers['encryption']
if 'crypto_key' in notification.headers:
data['cryptokey'] = notification.headers['crypto_key']
elif 'encryption_key' in notification.headers:
data['enckey'] = notification.headers['encryption_key']
# registration_ids are the FCM instance tokens (specified during
# registration.
router_ttl = min(self.MAX_TTL, max(self.min_ttl, notification.ttl
or 0))
try:
d = self.clients[router_data["app_id"]].send(
token=router_data.get("token"),
payload={
"collapse_key": self.collapseKey,
"data_message": data,
"dry_run": self.dryRun or ('dryrun' in router_data),
"ttl": router_ttl
})
except KeyError:
raise self._error("Invalid Application ID specified",
404,
errno=106,
log_exception=False)
d.addCallback(self._process_reply, notification, router_data,
router_ttl)
d.addErrback(self._process_error)
return d
def _process_error(self, failure):
err = failure.value
if isinstance(err, FCMAuthenticationError):
self.log.error("FCM Authentication Error: {}".format(err))
raise RouterException("Server error", status_code=500, errno=901)
if isinstance(err, TimeoutError):
self.log.warn("FCM Timeout: %s" % err)
self.metrics.increment("notification.bridge.error",
tags=make_tags(self._base_tags,
reason="timeout"))
raise RouterException("Server error",
status_code=502,
errno=903,
log_exception=False)
if isinstance(err, ConnectError):
self.log.warn("FCM Unavailable: %s" % err)
self.metrics.increment("notification.bridge.error",
tags=make_tags(
self._base_tags,
reason="connection_unavailable"))
raise RouterException("Server error",
status_code=502,
errno=902,
log_exception=False)
if isinstance(err, FCMNotFoundError):
self.log.debug("FCM Recipient not found: %s" % err)
self.metrics.increment("notification.bridge.error",
tags=make_tags(self._base_tags,
reason="recpient_gone"))
raise RouterException("FCM Recipient no longer available",
status_code=404,
errno=106,
log_exception=False)
if isinstance(err, RouterException):
self.log.warn("FCM Error: {}".format(err))
self.metrics.increment("notification.bridge.error",
tags=make_tags(self._base_tags,
reason="server_error"))
return failure
def _error(self, err, status, **kwargs):
"""Error handler that raises the RouterException"""
self.log.debug(err, **kwargs)
return RouterException(err,
status_code=status,
response_body=err,
**kwargs)
def _process_reply(self, reply, notification, router_data, ttl):
"""Process FCM send reply"""
# Failures are returned as non-200 messages (404, 410, etc.)
self.metrics.increment("notification.bridge.sent",
tags=self._base_tags)
self.metrics.increment("notification.message_data",
notification.data_length,
tags=make_tags(self._base_tags,
destination="Direct"))
location = "%s/m/%s" % (self.conf.endpoint_url, notification.version)
return RouterResponse(status_code=201,
response_body="",
headers={
"TTL": ttl,
"Location": location
},
logged_status=200)
def cli():
log = Logger()
parser = argparse.ArgumentParser(prog=__version__.package)
parser.add_argument('--version',
action='version',
version=__version__.public())
parser.add_argument(
'--secrets',
help=
'Directory that contains files named "username" and "password". These files should contain your Nest username and password.'
)
parser.add_argument('--username', help='Your Nest username.')
parser.add_argument('--password', help='Your Nest password.')
parser.add_argument(
'--endpoint',
default=default_endpoint,
help=
'Twisted endpoint declaration for internal web service. Default is "{}".'
.format(default_endpoint))
options = parser.parse_args()
output = textFileLogObserver(sys.stderr)
globalLogBeginner.beginLoggingTo([output])
username = None
password = None
if options.secrets is not None:
if not os.path.isdir(options.secrets):
log.critical('{secrets:} is not a directory',
secrets=options.secrets)
sys.exit(1)
username_file = os.path.join(options.secrets, 'username')
password_file = os.path.join(options.secrets, 'password')
try:
with open(username_file, mode='r', encoding='utf-8') as f:
username = f.read().strip()
except FileNotFoundError:
log.error(
'Secrets path specified but username file {username_file:} not found!',
username_file=username_file)
except PermissionError:
log.error(
'Unable to open username file {username_file:} for reading!',
username_file=username_file)
try:
with open(password_file, mode='r', encoding='utf-8') as f:
password = f.read().strip()
except FileNotFoundError:
log.error(
'Secrets path specified but password file {password_file:} not found!',
password_file=password_file)
except PermissionError:
log.error(
'Unable to open password file {password_file:} for reading!',
password_file=password_file)
if options.username is not None:
username = options.username
if options.password is not None:
password = options.password
if username is None:
log.critical('Username must be specified!')
sys.exit(1)
if password is None:
log.critical('Password must be specified!')
sys.exit(1)
m = Main(reactor, username, password, options.endpoint)
reactor.run()
class BaseProtocol(serialBytesProtocol):
def __init__(self, shorthand=True, callback=None, escaped=False,
error_callback=None):
serialBytesProtocol.__init__(self)
if callback:
self.callbacks = [callback]
else:
self.callbacks = []
self.setRawMode()
self.shorthand = shorthand
self._escaped = escaped
self.log = Logger()
self.requests = {}
self.command_id = 0
self.buffer = None
# self.reading = False
def get_id(self):
try:
self.command_id += 1
return intToByte(self.command_id)
except ValueError:
self.command_id = 1
return intToByte(1)
def connect(self, f):
if f.callback:
self.callbacks.append(f.callback)
f.proto = self
def rawDataReceived(self, data):
for byte in data:
if self.buffer:
self.buffer.fill(byte)
if self.buffer.remaining_bytes() == 0:
try:
# Try to parse and return result
self.buffer.parse()
# Ignore empty frames
if len(self.buffer.data) == 0:
self.buffer = None
except ValueError:
# Bad frame, so restart
self.log.warn('Bad frame: %r'
% self.buffer.raw_data)
else:
self.read_frame(self.buffer.data)
self.buffer = None
# self.reading = False
else:
if byte == Frame.START_BYTE:
# self.reading == True
self.buffer = Frame(escaped=self._escaped)
def read_frame(self, frame):
"""
read_frame: binary data -> {'id':str,
'param':binary data,
...}
read_frame takes a data packet received from an XBee device
and converts it into a dictionary. This dictionary provides
names for each segment of binary data as specified in the
api_responses spec.
"""
# Fetch the first byte, identify the packet
# If the spec doesn't exist, raise exception
packet_id = frame[0:1]
try:
name = self.api_responses[packet_id]
except AttributeError:
raise NotImplementedError(
"API response specifications could not be found; " +
"use a derived class which defines 'api_responses'.")
except KeyError:
# Check to see if this ID can be found among transmittible packets
for cmd_name, cmd in list(self.api_frames.items()):
if cmd['id']['default'] == packet_id:
msg = "Incoming frame with id {packet_id} looks like a " +\
"command frame of type '{cmd_name}' (these should " +\
" not be received). Are you sure your devices " +\
"are in API mode?"
self.log.error(
msg, packet_id=bytes(frame), cmd_name=cmd_name)
return
self.log.error("Unrecognized response packet with id byte {f}",
f=frame[0])
return
# Current byte index in the data stream
packet = self.api_frames[name]
index = 0
callback = False
# Result info
info = {'id': name}
# packet_spec = packet['structure']
# Parse the packet in the order specified
if 'frame_id' in packet:
callback = True
# if packet['len'] == 'null_terminated':
# field_data = b''
# while frame[index:index + 1] != b'\x00':
# field_data += frame[index:index + 1]
# index += 1
# index += 1
# info[name]
for field, dic in packet.items():
if dic['len'] == 'null_terminated':
field_data = b''
while frame[index:index] != b'\x00':
field_data += frame[index:index]
index += 1
index += 1
info[field] = field_data
elif dic['len'] is not None:
# Store the number of bytes specified
# Are we trying to read beyond the last data element?
if index + dic['len'] > len(frame):
raise ValueError(
"Response packet was shorter than expected")
field_data = frame[index:index + dic['len']]
info[field] = field_data
index += dic['len']
# If the data field has no length specified, store any
# leftover bytes and quit
else:
field_data = frame[index:-1]
# Were there any remaining bytes?
if field_data:
# If so, store them
info[field] = field_data
index += len(field_data) + 1
break
# If there are more bytes than expected, raise an exception
if index + 1 < len(frame):
raise ValueError(
"Response packet was longer than expected; " +
"expected: %d, got: %d bytes" % (index, len(frame)))
# Apply parsing rules if any exist
if 'parsing' in packet:
for parse_rule in packet['parsing']:
# Only apply a rule if it is relevant (raw data is available)
if parse_rule[0] in info:
# Apply the parse function to the indicated field and
# replace the raw data with the result
info[parse_rule[0]] = parse_rule[1](self, info)
if callback:
if info['frame_id'] in self.requests:
self.requests[info['frame_id']].callback(info)
del self.requests[info['frame_id']]
else:
self.log.warn('Response without request: %r' % info)
elif self.callbacks:
for callback in self.callbacks:
callback(info)
else:
self.log.debug(info)
def _build_command(self, cmd, **kwargs):
"""
_build_command: string (binary data) ... -> binary data
_build_command will construct a command packet according to the
specified command's specification in api_commands. It will expect
named arguments for all fields other than those with a default
value or a length of 'None'.
Each field will be written out in the order they are defined
in the command definition.
"""
try:
cmd_spec = self.api_frames[cmd]
except AttributeError:
raise NotImplementedError(
"API command specifications could not be found; " +
"use a derived class which defines 'api_commands'.")
packet = b''
if 'frame_id' in kwargs:
fid = kwargs['frame_id']
elif cmd in ['source_route']:
fid = b'\x00'
else:
fid = self.get_id()
for name, dic in cmd_spec.items():
if name == 'frame_id':
data = fid
elif name in kwargs:
data = kwargs[name]
else:
if dic['len']:
if dic['default']:
data = dic['default']
else:
raise KeyError(
"The expected field %s of length %d was " +
"not provided" % (name, dic['len']))
else:
data = None
if dic['len'] and len(data) != dic['len']:
raise ValueError(
"The data provided for '%s' was not %d bytes long"
% (name, dic['len']))
if data:
packet += data
return packet, fid
def send(self, cmd, **kwargs):
"""
send: string param=binary data ... -> None
When send is called with the proper arguments, an API command
will be written to the serial port for this XBee device
containing the proper instructions and data.
This method must be called with named arguments in accordance
with the api_command specification. Arguments matching all
field names other than those in reserved_names (like 'id' and
'order') should be given, unless they are of variable length
(of 'None' in the specification. Those are optional).
"""
# Pass through the keyword arguments
# if self.reading:
# return task.deferLater(.5, self.send, cmd, **kwargs)
packet, fid = self._build_command(cmd, **kwargs)
d = defer.Deferred()
self.requests.update({fid: d})
f = Frame(packet).output()
self.transport.write(f)
return d
def _parse_samples_header(self, io_bytes):
"""
_parse_samples_header: binary data in XBee IO data format ->
(int, [int ...], [int ...], int, int)
_parse_samples_header will read the first three bytes of the
binary data given and will return the number of samples which
follow, a list of enabled digital inputs, a list of enabled
analog inputs, the dio_mask, and the size of the header in bytes
"""
header_size = 4
# number of samples (always 1?) is the first byte
sample_count = byteToInt(io_bytes[0])
# part of byte 1 and byte 2 are the DIO mask ( 9 bits )
dio_mask = (
byteToInt(io_bytes[1]) << 8 | byteToInt(io_bytes[2])) & 0x01FF
# upper 7 bits of byte 1 is the AIO mask
aio_mask = byteToInt(io_bytes[3]) & 0xFE >> 1
# print(byteToInt(io_bytes[3]) & 0xFE >> 1)
# print(aio_mask)
# sorted lists of enabled channels; value is position of bit in mask
dio_chans = []
aio_chans = []
for i in range(0, 9):
if dio_mask & (1 << i):
dio_chans.append(i)
dio_chans.sort()
for i in range(0, 7):
if aio_mask & (1 << i):
aio_chans.append(i)
aio_chans.sort()
return (sample_count, dio_chans, aio_chans, dio_mask, header_size)
def _parse_samples(self, io_bytes):
"""
_parse_samples: binary data in XBee IO data format ->
[ {"dio-0":True,
"dio-1":False,
"adc-0":100"}, ...]
_parse_samples reads binary data from an XBee device in the IO
data format specified by the API. It will then return a
dictionary indicating the status of each enabled IO port.
"""
sample_count, dio_chans, aio_chans, dio_mask, header_size = \
self._parse_samples_header(io_bytes)
samples = []
# split the sample data into a list, so it can be pop()'d
# self.log.debug('%r' % io_bytes)
sample_bytes = [byteToInt(c) for c in io_bytes[header_size:]]
# self.log.debug('%r' % sample_bytes)
# self.log.debug('%r' % aio_chans)
# repeat for every sample provided
for sample_ind in range(0, sample_count): # @UnusedVariable
tmp_samples = {}
if dio_chans:
# we have digital data
digital_data_set = (
sample_bytes.pop(0) << 8 | sample_bytes.pop(0))
digital_values = dio_mask & digital_data_set
for i in dio_chans:
tmp_samples['dio-{0}'.format(i)] = True if (
digital_values >> i) & 1 else False
for i in aio_chans:
analog_sample = (
sample_bytes.pop(0) << 8 | sample_bytes.pop(0))
tmp_samples['adc-{0}'.format(i)] = int(
(analog_sample * 1200.0) / 1023.0)
samples.append(tmp_samples)
return samples
def _parse_sensor_data(self, io_bytes):
# TODO
return [{'data': io_bytes}]
def __getattr__(self, name):
"""
If a method by the name of a valid api command is called,
the arguments will be automatically sent to an appropriate
send() call
"""
# If api_commands is not defined, raise NotImplementedError\
# If its not defined, _getattr__ will be called with its name
if name == 'api_frames':
raise NotImplementedError(
"API command specifications could not be found; use a " +
"derived class which defines 'api_commands'.")
# Is shorthand enabled, and is the called name a command?
if self.shorthand and name in self.api_frames:
# If so, simply return a function which passes its arguments
# to an appropriate send() call
return lambda **kwargs: self.send(name, **kwargs)
else:
raise AttributeError("XBee has no attribute '%s'" % name)
class PhotometerService(ClientService):
def __init__(self, options, reference):
self.options = options
self.namespace = 'ref.' if reference else 'test'
self.label = self.namespace.upper()
setLogLevel(namespace=self.label, levelStr=options['log_messages'])
setLogLevel(namespace=self.namespace, levelStr=options['log_level'])
self.log = Logger(namespace=self.namespace)
self.reference = reference # Flag, is this instance for the reference photometer
self.factory = self.buildFactory()
self.protocol = None
self.serport = None
self.info = None # Photometer info
self.buffer = CircularBuffer(options['size'], self.log)
parts = chop(self.options['endpoint'], sep=':')
if parts[0] == 'tcp':
endpoint = clientFromString(reactor, self.options['endpoint'])
ClientService.__init__(self,
endpoint,
self.factory,
retryPolicy=backoffPolicy(initialDelay=0.5,
factor=3.0))
@inlineCallbacks
def startService(self):
'''
Starts the photometer service listens to a TESS
Although it is technically a synchronous operation, it works well
with inline callbacks
'''
self.log.info("starting {name} service", name=self.name)
yield self.connect()
self.info = yield self.getInfo()
if self.reference:
returnValue(None)
# Now this is for the test photometer only
if self.options['dry_run']:
self.log.info('Dry run. Will stop here ...')
yield self.stopService()
elif self.info is None:
yield self.stopService()
elif self.options['zero_point'] is not None:
try:
result = yield self.protocol.writeZeroPoint(
self.options['zero_point'])
except Exception as e:
self.log.error("Timeout when updating Zero Point")
self.log.failure("{excp}", excp=e)
else:
self.log.info("[{label}] Writen ZP : {zp:0.2f}",
label=self.label,
zp=result['zp'])
finally:
yield self.stopService()
def stopService(self):
self.log.info("stopping {name} service", name=self.name)
try:
reactor.callLater(0, reactor.stop)
except Exception as e:
log.error("could not stop the reactor")
return defer.succeed(None)
# --------------
# Photometer API
# --------------
def writeZeroPoint(self, zero_point):
'''Writes Zero Point to the device. Returns a Deferred'''
return self.protocol.writeZeroPoint(zero_point)
def getPhotometerInfo(self):
if self.protocol is None:
self.log.warn("Requested photometer info but no protocol yet!")
return defer.fail()
if self.info is None:
return self.getInfo()
else:
return defer.succeed(self.info)
# --------------
# Helper methods
# ---------------
@inlineCallbacks
def connect(self):
parts = chop(self.options['endpoint'], sep=':')
if parts[0] == 'serial':
endpoint = parts[1:]
self.protocol = self.factory.buildProtocol(0)
try:
self.serport = SerialPort(self.protocol,
endpoint[0],
reactor,
baudrate=endpoint[1])
except Exception as e:
self.log.error("{excp}", excp=e)
yield self.stopService()
else:
self.gotProtocol(self.protocol)
self.log.info("Using serial port {tty} at {baud} bps",
tty=endpoint[0],
baud=endpoint[1])
else:
ClientService.startService(self)
try:
protocol = yield self.whenConnected(failAfterFailures=1)
except Exception as e:
self.log.error("{excp}", excp=e)
yield self.stopService()
else:
self.gotProtocol(protocol)
self.log.info("Using TCP endpoint {endpoint}",
endpoint=self.options['endpoint'])
@inlineCallbacks
def getInfo(self):
try:
info = yield self.protocol.readPhotometerInfo()
except Exception as e:
self.log.error("Timeout when reading photometer info")
info = self.fixIt()
returnValue(info) # May be None
else:
info['model'] = self.options['model']
info['label'] = self.label
self.log.info("[{label}] Model : {value}",
label=self.label,
value=info['model'])
self.log.info("[{label}] Name : {value}",
label=self.label,
value=info['name'])
self.log.info("[{label}] MAC : {value}",
label=self.label,
value=info['mac'])
self.log.info("[{label}] Zero Point: {value:.02f} (old)",
label=self.label,
value=info['zp'])
self.log.info("[{label}] Firmware : {value}",
label=self.label,
value=info['firmware'])
returnValue(info)
def fixIt(self):
parts = chop(self.options['endpoint'], sep=':')
if self.reference and (self.options['model']
== TESSW) and parts[0] == 'serial':
info = {
'model': TESSW,
'label': self.label,
'name': self.options['name'],
'mac': self.options['mac'],
'zp': 20.50,
'firmware': "",
}
self.log.error("Fixed photometer info with defaults {info}",
info=info)
return info
else:
return None
def limitedStart(self):
'''Detects the case where only the Test photometer service is started'''
if self.reference:
return False
return (self.options['dry_run']
or self.options['zero_point'] is not None)
def buildFactory(self):
if self.options['model'] == TESSW:
self.log.debug("Choosing a {model} factory", model=TESSW)
import zptess.tessw
factory = zptess.tessw.TESSProtocolFactory(self.label)
elif self.options['model'] == TESSP:
self.log.debug("Choosing a {model} factory", model=TESSP)
import zptess.tessp
factory = zptess.tessp.TESSProtocolFactory(self.label)
else:
self.log.debug("Choosing a {model} factory", model=TAS)
import zptess.tas
factory = zptess.tas.TESSProtocolFactory(self.label)
return factory
def gotProtocol(self, protocol):
self.log.debug("got protocol")
protocol.setContext(self.options['endpoint'])
self.buffer.registerProducer(protocol, True)
if self.limitedStart():
protocol.stopProducing(
) # We don need to feed messages to the buffer
self.protocol = protocol
class SerialClientProtocol(LineReceiver):
def connectionMade(self):
self.log = Logger(namespace="serial")
self.log.debug("serial port connected")
def lineReceived(self, line):
from . import ws_factory
self.log.debug("serial rx: {line}", line=line)
try:
data = line.strip()
if data == b"r":
if not ws_factory.client:
self.log.error("websocket client not connected")
else:
self.log.debug("serial tx: k")
self.sendLine(b"k")
data_json = json.dumps([{"data_type": "counter"}])
self.log.debug("sending data to ws: {data}",
data=data_json)
ws_factory.client.sendMessage(data_json.encode("utf8"))
else:
commands = data.split(b"#")
data_array = []
for command in commands:
values = command.split(b",")
cmd = values[0].decode()
data_dict = {"data_type": cmd}
if values[0] in [b"vel", b"pos", b"lap"]:
if len(values) > 1:
data_dict[cmd + "_1"] = int.from_bytes(
values[1], "big") if values[1] else 0
else:
data_dict[cmd + "_1"] = 0
if len(values) > 2:
data_dict[cmd + "_2"] = int.from_bytes(
values[2], "big")
else:
data_dict[cmd + "_2"] = 0
elif values[0] == b"goal":
data_dict["goal_lap"] = int.from_bytes(
values[1], "big")
elif values[0] == b"play":
if len(values) > 1:
data_dict["status"] = True if int.from_bytes(
values[1], "big") == 1 else False
else:
data_dict["status"] = False
elif values[0] == b"win":
data_dict["winner"] = int.from_bytes(values[1], "big")
elif values[0] == b"time":
if len(values) > 1:
data_dict["seconds"] = int.from_bytes(
values[1], "big") / 1000
else:
data_dict["seconds"] = 0.0
else:
self.log.error("command not supported: {cmd}",
cmd=values[0])
data_dict = None
if data_dict:
data_array.append(data_dict)
if data_array:
if not ws_factory.client:
self.log.error("websocket client not connected")
else:
data_json = json.dumps(data_array)
self.log.debug("sending data to ws: {data}",
data=data_json)
ws_factory.client.sendMessage(data_json.encode("utf8"))
pass
except Exception as e:
self.log.error("error: {e}", e=e)
def sendMessage(self, message: str):
self.log.debug("serial tx: {message}", message=message)
self.sendLine(message.encode("utf8"))
class IRCd(Service):
def __init__(self, configFileName):
self.config = Config(self, configFileName)
self.boundPorts = {}
self.loadedModules = {}
self._loadedModuleData = {}
self._unloadingModules = {}
self.commonModules = set()
self.userCommands = {}
self.serverCommands = {}
self.channelModes = ({}, {}, {}, {})
self.channelStatuses = {}
self.channelStatusSymbols = {}
self.channelStatusOrder = []
self.channelModeTypes = {}
self.userModes = ({}, {}, {}, {})
self.userModeTypes = {}
self.actions = {}
self.storage = None
self.storageSyncer = None
self.dataCache = {}
self.functionCache = {}
self.serverID = None
self.name = None
self.isupport_tokens = {
"CASEMAPPING": "strict-rfc1459",
"CHANTYPES": "#",
}
self._uid = self._genUID()
self.users = {}
self.userNicks = CaseInsensitiveDictionary()
self.channels = CaseInsensitiveDictionary(WeakValueDictionary)
self.servers = {}
self.serverNames = CaseInsensitiveDictionary()
self.recentlyQuitUsers = {}
self.recentlyQuitServers = {}
self.recentlyDestroyedChannels = CaseInsensitiveDictionary()
self.pruneRecentlyQuit = None
self.pruneRecentChannels = None
self._logFilter = LogLevelFilterPredicate()
filterObserver = FilteringLogObserver(globalLogPublisher, (self._logFilter,))
self.log = Logger("txircd", observer=filterObserver)
self.startupTime = None
def startService(self):
self.log.info("Starting up...")
self.startupTime = now()
self.log.info("Loading configuration...")
self.config.reload()
self.name = self.config["server_name"]
self.serverID = self.config["server_id"]
self.log.info("Loading storage...")
self.storage = shelve.open(self.config["datastore_path"], writeback=True)
self.storageSyncer = LoopingCall(self.storage.sync)
self.storageSyncer.start(self.config.get("storage_sync_interval", 5), now=False)
self.log.info("Starting processes...")
self.pruneRecentlyQuit = LoopingCall(self.pruneQuit)
self.pruneRecentlyQuit.start(10, now=False)
self.pruneRecentChannels = LoopingCall(self.pruneChannels)
self.pruneRecentChannels.start(15, now=False)
self.log.info("Loading modules...")
self._loadModules()
self.log.info("Binding ports...")
self._bindPorts()
self.log.info("txircd started!")
try:
self._logFilter.setLogLevelForNamespace("txircd", LogLevel.levelWithName(self.config["log_level"]))
except (KeyError, InvalidLogLevelError):
self._logFilter.setLogLevelForNamespace("txircd", LogLevel.warn)
self.runActionStandard("startup")
def stopService(self):
stopDeferreds = []
self.log.info("Disconnecting servers...")
serverList = self.servers.values() # Take the list of server objects
self.servers = {} # And then destroy the server dict to inhibit server objects generating lots of noise
for server in serverList:
if server.nextClosest == self.serverID:
stopDeferreds.append(server.disconnectedDeferred)
allUsers = self.users.keys()
for user in allUsers:
if user[:3] == server.serverID:
del self.users[user]
server.transport.loseConnection()
self.log.info("Disconnecting users...")
userList = self.users.values() # Basically do the same thing I just did with the servers
self.users = {}
for user in userList:
if user.transport:
stopDeferreds.append(user.disconnectedDeferred)
user.transport.loseConnection()
self.log.info("Unloading modules...")
moduleList = self.loadedModules.keys()
for module in moduleList:
self._unloadModule(module, False) # Incomplete unload is done to save time and because side effects are destroyed anyway
self.log.info("Stopping processes...")
if self.pruneRecentlyQuit.running:
self.pruneRecentlyQuit.stop()
if self.pruneRecentChannels.running:
self.pruneRecentChannels.stop()
self.log.info("Closing data storage...")
if self.storageSyncer.running:
self.storageSyncer.stop()
self.storage.close() # a close() will sync() also
self.log.info("Releasing ports...")
stopDeferreds.extend(self._unbindPorts())
return DeferredList(stopDeferreds)
def _loadModules(self):
for module in getPlugins(IModuleData, txircd.modules):
if module.name in self.loadedModules:
continue
if module.core or module.name in self.config["modules"]:
self._loadModuleData(module)
for moduleName in self.config["modules"]:
if moduleName not in self.loadedModules:
self.log.warn("The module {module} failed to load.", module=moduleName)
def loadModule(self, moduleName):
"""
Loads a module of the specified name.
Raises ModuleLoadError if the module cannot be loaded.
If the specified module is currently being unloaded, returns the
DeferredList specified by the module when it was unloading with a
callback to try to load the module again once it succeeds.
"""
if moduleName in self._unloadingModules:
deferList = self._unloadingModules[moduleName]
deferList.addCallback(self._tryLoadAgain, moduleName)
return deferList
for module in getPlugins(IModuleData, txircd.modules):
if module.name == moduleName:
rebuild(importlib.import_module(module.__module__)) # getPlugins doesn't recompile modules, so let's do that ourselves.
self._loadModuleData(module)
self.log.info("Loaded module {module}.", module=moduleName)
break
def _tryLoadAgain(self, _, moduleName):
self.loadModule(moduleName)
def _loadModuleData(self, module):
if not IModuleData.providedBy(module):
raise ModuleLoadError ("???", "Module does not implement module interface")
if not module.name:
raise ModuleLoadError ("???", "Module did not provide a name")
if module.name in self.loadedModules:
self.log.debug("Not loading {module.name} because it's already loaded", module=module)
return
self.log.debug("Beginning to load {module.name}...", module=module)
module.hookIRCd(self)
try:
module.verifyConfig(self.config)
except ConfigError as e:
raise ModuleLoadError(module.name, e)
self.log.debug("Loading hooks from {module.name}...", module=module)
moduleData = {
"channelmodes": module.channelModes(),
"usermodes": module.userModes(),
"actions": module.actions(),
"usercommands": module.userCommands(),
"servercommands": module.serverCommands()
}
newChannelModes = ({}, {}, {}, {})
newChannelStatuses = {}
newUserModes = ({}, {}, {}, {})
newActions = {}
newUserCommands = {}
newServerCommands = {}
common = False
self.log.debug("Processing hook data from {module.name}...", module=module)
for mode in moduleData["channelmodes"]:
if mode[0] in self.channelModeTypes:
raise ModuleLoadError (module.name, "Tries to implement channel mode +{} when that mode is already implemented.".format(mode[0]))
if not IMode.providedBy(mode[2]):
raise ModuleLoadError (module.name, "Returns a channel mode object (+{}) that doesn't implement IMode.".format(mode[0]))
if mode[1] == ModeType.Status:
if mode[4] in self.channelStatusSymbols:
raise ModuleLoadError (module.name, "Tries to create a channel rank with symbol {} when that symbol is already in use.".format(mode[4]))
try:
newChannelStatuses[mode[0]] = (mode[4], mode[3], mode[2])
except IndexError:
raise ModuleLoadError (module.name, "Specifies channel status mode {} without a rank or symbol".format(mode[0]))
else:
newChannelModes[mode[1]][mode[0]] = mode[2]
common = True
for mode in moduleData["usermodes"]:
if mode[0] in self.userModeTypes:
raise ModuleLoadError (module.name, "Tries to implement user mode +{} when that mode is already implemented.".format(mode[0]))
if not IMode.providedBy(mode[2]):
raise ModuleLoadError (module.name, "Returns a user mode object (+{}) that doesn't implement IMode.".format(mode[0]))
newUserModes[mode[1]][mode[0]] = mode[2]
common = True
for action in moduleData["actions"]:
if action[0] not in newActions:
newActions[action[0]] = [(action[2], action[1])]
else:
newActions[action[0]].append((action[2], action[1]))
for command in moduleData["usercommands"]:
if not ICommand.providedBy(command[2]):
raise ModuleLoadError (module.name, "Returns a user command object ({}) that doesn't implement ICommand.".format(command[0]))
if command[0] not in newUserCommands:
newUserCommands[command[0]] = []
newUserCommands[command[0]].append((command[2], command[1]))
for command in moduleData["servercommands"]:
if not ICommand.providedBy(command[2]):
raise ModuleLoadError (module.name, "Returns a server command object ({}) that doesnt implement ICommand.".format(command[0]))
if command[0] not in newServerCommands:
newServerCommands[command[0]] = []
newServerCommands[command[0]].append((command[2], command[1]))
common = True
if not common:
common = module.requiredOnAllServers
self.log.debug("Loaded data from {module.name}; committing data and calling hooks...", module=module)
module.load()
self.loadedModules[module.name] = module
self._loadedModuleData[module.name] = moduleData
if common:
self.commonModules.add(module.name)
self.runActionStandard("moduleload", module.name)
for modeType, typeSet in enumerate(newChannelModes):
for mode, implementation in typeSet.iteritems():
self.channelModeTypes[mode] = modeType
self.channelModes[modeType][mode] = implementation
for mode, data in newChannelStatuses.iteritems():
self.channelModeTypes[mode] = ModeType.Status
self.channelStatuses[mode] = data
self.channelStatusSymbols[data[0]] = mode
for index, status in enumerate(self.channelStatusOrder):
if self.channelStatuses[status][1] < data[1]:
self.channelStatusOrder.insert(index, mode)
break
else:
self.channelStatusOrder.append(mode)
for modeType, typeSet in enumerate(newUserModes):
for mode, implementation in typeSet.iteritems():
self.userModeTypes[mode] = modeType
self.userModes[modeType][mode] = implementation
for action, actionList in newActions.iteritems():
if action not in self.actions:
self.actions[action] = []
for actionData in actionList:
for index, handlerData in enumerate(self.actions[action]):
if handlerData[1] < actionData[1]:
self.actions[action].insert(index, actionData)
break
else:
self.actions[action].append(actionData)
for command, dataList in newUserCommands.iteritems():
if command not in self.userCommands:
self.userCommands[command] = []
for data in dataList:
for index, cmd in enumerate(self.userCommands[command]):
if cmd[1] < data[1]:
self.userCommands[command].insert(index, data)
break
else:
self.userCommands[command].append(data)
for command, dataList in newServerCommands.iteritems():
if command not in self.serverCommands:
self.serverCommands[command] = []
for data in dataList:
for index, cmd in enumerate(self.serverCommands[command]):
if cmd[1] < data[1]:
self.serverCommands[command].insert(index, data)
break
else:
self.serverCommands[command].append(data)
self.log.debug("Module {module.name} is now fully loaded.", module=module)
def unloadModule(self, moduleName):
"""
Unloads the loaded module with the given name. Raises ValueError
if the module cannot be unloaded because it's a core module.
"""
self._unloadModule(moduleName, True)
self.log.info("Unloaded module {module}.", module=moduleName)
def _unloadModule(self, moduleName, fullUnload):
unloadDeferreds = []
if moduleName not in self.loadedModules:
return
module = self.loadedModules[moduleName]
if fullUnload and module.core:
raise ValueError ("The module you're trying to unload is a core module.")
moduleData = self._loadedModuleData[moduleName]
d = module.unload()
if d is not None:
unloadDeferreds.append(d)
if fullUnload:
d = module.fullUnload()
if d is not None:
unloadDeferreds.append(d)
for modeData in moduleData["channelmodes"]:
if fullUnload: # Unset modes on full unload
if modeData[1] == ModeType.Status:
for channel in self.channels.itervalues():
removeFromChannel = []
for user, userData in channel.user.iteritems():
if modeData[0] in userData["status"]:
removeFromChannel.append((False, modeData[0], user.uuid))
channel.setModes(removeFromChannel, self.serverID)
elif modeData[1] == ModeType.List:
for channel in self.channels.itervalues():
if modeData[0] in channel.modes:
removeFromChannel = []
for paramData in channel.modes[modeData[0]]:
removeFromChannel.append((False, modeData[0], paramData[0]))
channel.setModes(removeFromChannel, self.serverID)
else:
for channel in self.channels.itervalues():
if modeData[0] in channel.modes:
channel.setModes([(False, modeData[0], channel.modes[modeData[0]])], self.serverID)
if modeData[1] == ModeType.Status:
del self.channelStatuses[modeData[0]]
del self.channelStatusSymbols[modeData[4]]
self.channelStatusOrder.remove(modeData[0])
else:
del self.channelModes[modeData[1]][modeData[0]]
del self.channelModeTypes[modeData[0]]
for modeData in moduleData["usermodes"]:
if fullUnload: # Unset modes on full unload
if modeData[1] == ModeType.List:
for user in self.users.itervalues():
if modeData[0] in user.modes:
removeFromUser = []
for paramData in user.modes[modeData[0]]:
removeFromUser.append((False, modeData[0], paramData[0]))
user.setModes(removeFromUser, self.serverID)
else:
for user in self.users.itervalues():
if modeData[0] in user.modes:
user.setModes([(False, modeData[0], user.modes[modeData[0]])], self.serverID)
del self.userModes[modeData[1]][modeData[0]]
del self.userModeTypes[modeData[0]]
for actionData in moduleData["actions"]:
self.actions[actionData[0]].remove((actionData[2], actionData[1]))
if not self.actions[actionData[0]]:
del self.actions[actionData[0]]
for commandData in moduleData["usercommands"]:
self.userCommands[commandData[0]].remove((commandData[2], commandData[1]))
if not self.userCommands[commandData[0]]:
del self.userCommands[commandData[0]]
for commandData in moduleData["servercommands"]:
self.serverCommands[commandData[0]].remove((commandData[2], commandData[1]))
if not self.serverCommands[commandData[0]]:
del self.serverCommands[commandData[0]]
del self.loadedModules[moduleName]
del self._loadedModuleData[moduleName]
if fullUnload:
self.runActionStandard("moduleunload", module.name)
if unloadDeferreds:
deferList = DeferredList(unloadDeferreds)
self._unloadingModules[moduleName] = deferList
deferList.addCallback(self._removeFromUnloadingList, moduleName)
return deferList
def _removeFromUnloadingList(self, _, moduleName):
del self._unloadingModules[moduleName]
def reloadModule(self, moduleName):
"""
Reloads the module with the given name.
Returns a DeferredList if the module unloads with one or more Deferreds.
May raise ModuleLoadError if the module cannot be loaded.
"""
deferList = self._unloadModule(moduleName, False)
if deferList is None:
deferList = self.loadModule(moduleName)
else:
deferList.addCallback(lambda result: self.loadModule(moduleName))
return deferList
def verifyConfig(self, config):
# IRCd
if "server_name" not in config:
raise ConfigValidationError("server_name", "required item not found in configuration file.")
if not isinstance(config["server_name"], basestring):
raise ConfigValidationError("server_name", "value must be a string")
if len(config["server_name"]) > 64:
config["server_name"] = config["server_name"][:64]
self.logConfigValidationWarning("server_name", "value is too long and has been truncated", config["server_name"])
if not re.match(r"^[a-zA-Z0-9.-]+\.[a-zA-Z0-9.-]+$", config["server_name"]):
raise ConfigValidationError("server_name", "server name must look like a valid hostname.")
if "server_id" in config:
if not isinstance(config["server_id"], basestring):
raise ConfigValidationError("server_id", "value must be a string")
else:
config["server_id"] = config["server_id"].upper()
else:
randFromName = random.Random(config["server_name"])
serverID = randFromName.choice(string.digits) + randFromName.choice(string.digits + string.ascii_uppercase) + randFromName.choice(string.digits + string.ascii_uppercase)
config["server_id"] = serverID
if len(config["server_id"]) != 3 or not config["server_id"].isalnum() or not config["server_id"][0].isdigit():
raise ConfigValidationError("server_id", "value must be a 3-character alphanumeric string starting with a number.")
if "server_description" not in config:
raise ConfigValidationError("server_description", "required item not found in configuration file.")
if not isinstance(config["server_description"], basestring):
raise ConfigValidationError("server_description", "value must be a string")
if not config["server_description"]:
raise ConfigValidationError("server_description", "value must not be an empty string")
if len(config["server_description"]) > 255:
config["server_description"] = config["server_description"][:255]
self.logConfigValidationWarning("server_description", "value is too long and has been truncated", config["server_description"])
if "network_name" not in config:
raise ConfigValidationError("network_name", "required item not found in configuration file.")
if not isinstance(config["network_name"], basestring):
raise ConfigValidationError("network_name", "value must be a string")
if not config["network_name"]:
raise ConfigValidationError("network_name", "value must not be an empty string")
if " " in config["network_name"]:
raise ConfigValidationError("network_name", "value cannot have spaces")
if len(config["network_name"]) > 32:
config["network_name"] = config["network_name"][:32]
self.logConfigValidationWarning("network_name", "value is too long", config["network_name"])
if "bind_client" not in config:
config["bind_client"] = [ "tcp:6667:interface={::}" ]
self.logConfigValidationWarning("bind_client", "no default client binding specified", "[ \"tcp:6667:interface={::}\" ]")
if not isinstance(config["bind_client"], list):
raise ConfigValidationError("bind_client", "value must be a list")
for bindDesc in config["bind_client"]:
if not isinstance(bindDesc, basestring):
raise ConfigValidationError("bind_client", "every entry must be a string")
if "bind_server" not in config:
config["bind_server"] = []
if not isinstance(config["bind_server"], list):
raise ConfigValidationError("bind_server", "value must be a list")
for bindDesc in config["bind_server"]:
if not isinstance(bindDesc, basestring):
raise ConfigValidationError("bind_server", "every entry must be a string")
if "modules" not in config:
config["modules"] = []
if not isinstance(config["modules"], list):
raise ConfigValidationError("modules", "value must be a list")
for module in config["modules"]:
if not isinstance(module, basestring):
raise ConfigValidationError("modules", "every entry must be a string")
if "links" in config:
if not isinstance(config["links"], dict):
raise ConfigValidationError("links", "value must be a dictionary")
for desc, server in config["links"].iteritems():
if not isinstance(desc, basestring):
raise ConfigValidationError("links", "\"{}\" is an invalid server description".format(desc))
if not isinstance(server, dict):
raise ConfigValidationError("links", "values for \"{}\" must be a dictionary".format(desc))
if "connect_descriptor" not in server:
raise ConfigValidationError("links", "server \"{}\" must contain a \"connect_descriptor\" value".format(desc))
if "in_password" in server:
if not isinstance(server["in_password"], basestring):
config["links"][desc]["in_password"] = str(server["in_password"])
if "out_password" in server:
if not isinstance(server["out_password"], basestring):
config["links"][desc]["out_password"] = str(server["out_password"])
if "datastore_path" not in config:
config["datastore_path"] = "data.db"
if "storage_sync_interval" in config and not isinstance(config["storage_sync_interval"], int):
raise ConfigValidationError(config["storage_sync_interval"], "invalid number")
# Channels
if "channel_name_length" in config:
if not isinstance(config["channel_name_length"], int) or config["channel_name_length"] < 0:
raise ConfigValidationError("channel_name_length", "invalid number")
elif config["channel_name_length"] > 64:
config["channel_name_length"] = 64
self.logConfigValidationWarning("channel_name_length", "value is too large", 64)
if "modes_per_line" in config:
if not isinstance(config["modes_per_line"], int) or config["modes_per_line"] < 0:
raise ConfigValidationError("modes_per_line", "invalid number")
elif config["modes_per_line"] > 20:
config["modes_per_line"] = 20
self.logConfigValidationWarning("modes_per_line", "value is too large", 20)
if "channel_listmode_limit" in config:
if not isinstance(config["channel_listmode_limit"], int) or config["channel_listmode_limit"] < 0:
raise ConfigValidationError("channel_listmode_limit", "invalid number")
if config["channel_listmode_limit"] > 256:
config["channel_listmode_limit"] = 256
self.logConfigValidationWarning("channel_listmode_limit", "value is too large", 256)
# Users
if "user_registration_timeout" in config:
if not isinstance(config["user_registration_timeout"], int) or config["user_registration_timeout"] < 0:
raise ConfigValidationError("user_registration_timeout", "invalid number")
elif config["user_registration_timeout"] < 10:
config["user_registration_timeout"] = 10
self.logConfigValidationWarning("user_registration_timeout", "timeout could be too short for clients to register in time", 10)
if "user_ping_frequency" in config and (not isinstance(config["user_ping_frequency"], int) or config["user_ping_frequency"] < 0):
raise ConfigValidationError("user_ping_frequency", "invalid number")
if "hostname_length" in config:
if not isinstance(config["hostname_length"], int) or config["hostname_length"] < 0:
raise ConfigValidationError("hostname_length", "invalid number")
elif config["hostname_length"] > 64:
config["hostname_length"] = 64
self.logConfigValidationWarning("hostname_length", "value is too large", 64)
elif config["hostname_length"] < 4:
config["hostname_length"] = 4
self.logConfigValidationWarning("hostname_length", "value is too small", 4)
if "ident_length" in config:
if not isinstance(config["ident_length"], int) or config["ident_length"] < 0:
raise ConfigValidationError("ident_length", "invalid number")
elif config["ident_length"] > 12:
config["ident_length"] = 12
self.logConfigValidationWarning("ident_length", "value is too large", 12)
elif config["ident_length"] < 1:
config["ident_length"] = 1
self.logConfigValidationWarning("ident_length", "value is too small", 1)
if "gecos_length" in config:
if not isinstance(config["gecos_length"], int) or config["gecos_length"] < 0:
raise ConfigValidationError("gecos_length", "invalid number")
elif config["gecos_length"] > 128:
config["gecos_length"] = 128
self.logConfigValidationWarning("gecos_length", "value is too large", 128)
elif config["gecos_length"] < 1:
config["gecos_length"] = 1
self.logConfigValidationWarning("gecos_length", "value is too small", 1)
if "user_listmode_limit" in config:
if not isinstance(config["user_listmode_limit"], int) or config["user_listmode_limit"] < 0:
raise ConfigValidationError("user_listmode_limit", "invalid number")
if config["user_listmode_limit"] > 256:
config["user_listmode_limit"] = 256
self.logConfigValidationWarning("user_listmode_limit", "value is too large", 256)
# Servers
if "server_registration_timeout" in config:
if not isinstance(config["server_registration_timeout"], int) or config["server_registration_timeout"] < 0:
raise ConfigValidationError("server_registration_timeout", "invalid number")
elif config["server_registration_timeout"] < 10:
config["server_registration_timeout"] = 10
self.logConfigValidationWarning("server_registration_timeout", "timeout could be too short for servers to register in time", 10)
if "server_ping_frequency" in config and (not isinstance(config["server_ping_frequency"], int) or config["server_ping_frequency"] < 0):
raise ConfigValidationError("server_ping_frequency", "invalid number")
for module in self.loadedModules.itervalues():
module.verifyConfig(config)
def logConfigValidationWarning(self, key, message, default):
self.log.warn("Config value \"{configKey}\" is invalid ({message}); the value has been set to a default of \"{default}\".", configKey=key, message=message, default=default)
def rehash(self):
"""
Reloads the configuration file and applies changes.
"""
self.log.info("Rehashing...")
self.config.reload()
d = self._unbindPorts() # Unbind the ports that are bound
if d: # And then bind the new ones
DeferredList(d).addCallback(lambda result: self._bindPorts())
else:
self._bindPorts()
try:
self._logFilter.setLogLevelForNamespace("txircd", LogLevel.levelWithName(self.config["log_level"]))
except (KeyError, InvalidLogLevelError):
pass # If we can't set a new log level, we'll keep the old one
for module in self.loadedModules.itervalues():
module.rehash()
def _bindPorts(self):
for bindDesc in self.config["bind_client"]:
try:
endpoint = serverFromString(reactor, unescapeEndpointDescription(bindDesc))
except ValueError as e:
self.log.error(e)
continue
listenDeferred = endpoint.listen(UserFactory(self))
listenDeferred.addCallback(self._savePort, bindDesc, "client")
listenDeferred.addErrback(self._logNotBound, bindDesc)
for bindDesc in self.config["bind_server"]:
try:
endpoint = serverFromString(reactor, unescapeEndpointDescription(bindDesc))
except ValueError as e:
self.log.error(e)
continue
listenDeferred = endpoint.listen(ServerListenFactory(self))
listenDeferred.addCallback(self._savePort, bindDesc, "server")
listenDeferred.addErrback(self._logNotBound, bindDesc)
def _unbindPorts(self):
deferreds = []
for port in self.boundPorts.itervalues():
d = port.stopListening()
if d:
deferreds.append(d)
return deferreds
def _savePort(self, port, desc, portType):
self.boundPorts[desc] = port
self.log.debug("Bound endpoint '{endpointDescription}' for {portType} connections.", endpointDescription=desc, portType=portType)
def _logNotBound(self, err, desc):
self.log.error("Could not bind '{endpointDescription}': {errorMsg}", endpointDescription=desc, errorMsg=err)
def createUUID(self):
"""
Gets the next UUID for a new client.
"""
newUUID = self.serverID + self._uid.next()
while newUUID in self.users: # It'll take over 1.5 billion connections to loop around, but we still
newUUID = self.serverID + self._uid.next() # want to be extra safe and avoid collisions
self.log.debug("Generated new UUID {uuid}", uuid=newUUID)
return newUUID
def _genUID(self):
uid = "AAAAAA"
while True:
yield uid
uid = self._incrementUID(uid)
def _incrementUID(self, uid):
if uid == "Z": # The first character must be a letter
return "A" # So wrap that around
if uid[-1] == "9":
return self._incrementUID(uid[:-1]) + "A"
if uid[-1] == "Z":
return uid[:-1] + "0"
return uid[:-1] + chr(ord(uid[-1]) + 1)
def pruneQuit(self):
compareTime = now() - timedelta(seconds=10)
remove = []
for uuid, timeQuit in self.recentlyQuitUsers.iteritems():
if timeQuit < compareTime:
remove.append(uuid)
for uuid in remove:
del self.recentlyQuitUsers[uuid]
remove = []
for serverID, timeQuit in self.recentlyQuitServers.iteritems():
if timeQuit < compareTime:
remove.append(serverID)
for serverID in remove:
del self.recentlyQuitServers[serverID]
def pruneChannels(self):
removeChannels = []
for channel, remove in self.recentlyDestroyedChannels.iteritems():
if remove:
removeChannels.append(channel)
elif channel not in self.channels:
self.recentlyDestroyedChannels[channel] = True
for channel in removeChannels:
del self.recentlyDestroyedChannels[channel]
def generateISupportList(self):
isupport = self.isupport_tokens.copy()
statusSymbolOrder = "".join([self.channelStatuses[status][0] for status in self.channelStatusOrder])
isupport["CHANMODES"] = ",".join(["".join(modes) for modes in self.channelModes])
isupport["CHANNELLEN"] = self.config.get("channel_name_length", 64)
isupport["NETWORK"] = self.config["network_name"]
isupport["PREFIX"] = "({}){}".format("".join(self.channelStatusOrder), statusSymbolOrder)
isupport["STATUSMSG"] = statusSymbolOrder
isupport["USERMODES"] = ",".join(["".join(modes) for modes in self.userModes])
self.runActionStandard("buildisupport", isupport)
isupportList = []
for key, val in isupport.iteritems():
if val is None:
isupportList.append(key)
else:
isupportList.append("{}={}".format(key, val))
return isupportList
def connectServer(self, name):
"""
Connect a server with the given name in the configuration.
Returns a Deferred for the connection when we can successfully connect
or None if the server is already connected or if we're unable to find
information for that server in the configuration.
"""
if name in self.serverNames:
return None
if name not in self.config.get("links", {}):
return None
serverConfig = self.config["links"][name]
endpoint = clientFromString(reactor, unescapeEndpointDescription(serverConfig["connect_descriptor"]))
d = endpoint.connect(ServerConnectFactory(self))
d.addCallback(self._completeServerConnection, name)
return d
def _completeServerConnection(self, result, name):
self.log.info("Connected to server {serverName}", serverName=name)
self.runActionStandard("initiateserverconnection", result)
def broadcastToServers(self, fromServer, command, *params, **kw):
"""
Broadcasts a message to all connected servers. The fromServer parameter
should be the server from which the message came; if this server is the
originating server, specify None for fromServer.
"""
for server in self.servers.itervalues():
if server.nextClosest == self.serverID and server != fromServer:
server.sendMessage(command, *params, **kw)
def _getActionModes(self, actionName, *params, **kw):
users = []
channels = []
if "users" in kw:
users = kw["users"]
if "channels" in kw:
channels = kw["channels"]
functionList = []
if users:
genericUserActionName = "modeactioncheck-user-{}".format(actionName)
genericUserActionNameWithChannel = "modeactioncheck-user-withchannel-{}".format(actionName)
for modeType in self.userModes:
for mode, modeObj in modeType.iteritems():
if actionName not in modeObj.affectedActions:
continue
priority = modeObj.affectedActions[actionName]
actionList = []
# Because Python doesn't properly capture variables in lambdas, we have to force static capture
# by wrapping lambdas in more lambdas.
# I wish Python wasn't this gross.
for action in self.actions.get("modeactioncheck-user", []):
actionList.append(((lambda action, actionName, mode: lambda user, *params: action[0](actionName, mode, user, *params))(action, actionName, mode), action[1]))
for action in self.actions.get("modeactioncheck-user-withchannel", []):
for channel in channels:
actionList.append(((lambda action, actionName, mode, channel: lambda user, *params: action[0](actionName, mode, user, channel, *params))(action, actionName, mode, channel), action[1]))
for action in self.actions.get(genericUserActionName, []):
actionList.append(((lambda action, mode: lambda user, *params: action[0](mode, user, *params))(action, mode), action[1]))
for action in self.actions.get(genericUserActionNameWithChannel, []):
for channel in channels:
actionList.append(((lambda action, mode, channel: lambda user, *params: action[0](mode, user, channel, *params))(action, mode, channel), action[1]))
modeUserActionName = "modeactioncheck-user-{}-{}".format(mode, actionName)
modeUserActionNameWithChannel = "modeactioncheck-user-withchannel-{}-{}".format(mode, actionName)
for action in self.actions.get(modeUserActionNameWithChannel, []):
for channel in channels:
actionList.append(((lambda action, channel: lambda user, *params: action[0](user, channel, *params))(action, channel), action[1]))
actionList = sorted(self.actions.get(modeUserActionName, []) + actionList, key=lambda action: action[1], reverse=True)
applyUsers = []
for user in users:
for action in actionList:
param = action[0](user, *params)
if param is not None:
if param is not False:
applyUsers.append((user, param))
break
for user, param in applyUsers:
functionList.append(((lambda modeObj, actionName, user, param: lambda *params: modeObj.apply(actionName, user, param, *params))(modeObj, actionName, user, param), priority))
if channels:
genericChannelActionName = "modeactioncheck-channel-{}".format(actionName)
genericChannelActionNameWithUser = "******".format(actionName)
for modeType in self.channelModes:
for mode, modeObj in modeType.iteritems():
if actionName not in modeObj.affectedActions:
continue
priority = modeObj.affectedActions[actionName]
actionList = []
for action in self.actions.get("modeactioncheck-channel", []):
actionList.append(((lambda action, actionName, mode: lambda channel, *params: action[0](actionName, mode, channel, *params))(action, actionName, mode), action[1]))
for action in self.actions.get("modeactioncheck-channel-withuser", []):
for user in users:
actionList.append(((lambda action, actionName, mode, user: lambda channel, *params: action[0](actionName, mode, channel, user, *params))(action, actionName, mode, user), action[1]))
for action in self.actions.get(genericChannelActionName, []):
actionList.append(((lambda action, mode: lambda channel, *params: action[0](mode, channel, *params))(action, mode), action[1]))
for action in self.actions.get(genericChannelActionNameWithUser, []):
for user in users:
actionList.append(((lambda action, mode, user: lambda channel, *params: action[0](mode, channel, user, *params))(action, mode, user), action[1]))
modeChannelActionName = "modeactioncheck-channel-{}-{}".format(mode, actionName)
modeChannelActionNameWithUser = "******".format(mode, actionName)
for action in self.actions.get(modeChannelActionNameWithUser, []):
for user in users:
actionList.append(((lambda action, user: lambda channel, *params: action[0](channel, user, *params))(action, user), action[1]))
actionList = sorted(self.actions.get(modeChannelActionName, []) + actionList, key=lambda action: action[1], reverse=True)
applyChannels = []
for channel in channels:
for action in actionList:
param = action[0](channel, *params)
if param is not None:
if param is not False:
applyChannels.append((channel, param))
break
for channel, param in applyChannels:
functionList.append(((lambda modeObj, actionName, channel, param: lambda *params: modeObj.apply(actionName, channel, param, *params))(modeObj, actionName, channel, param), priority))
return functionList
def _getActionFunctionList(self, actionName, *params, **kw):
functionList = self.actions.get(actionName, [])
functionList = functionList + self._getActionModes(actionName, *params, **kw)
return sorted(functionList, key=lambda action: action[1], reverse=True)
def _combineActionFunctionLists(self, actionLists):
"""
Combines multiple lists of action functions into one.
Assumes all lists are sorted.
Takes a dict mapping action names to their action function lists.
Returns a list in priority order (highest to lowest) of (actionName, function) tuples.
"""
fullActionList = []
for actionName, actionList in actionLists.iteritems():
insertPos = 0
for action in actionList:
try:
while fullActionList[insertPos][1] > action[1]:
insertPos += 1
fullActionList.insert(insertPos, (actionName, action[0]))
except IndexError:
fullActionList.append((actionName, action[0]))
insertPos += 1
return fullActionList
def runActionStandard(self, actionName, *params, **kw):
"""
Calls all functions for a given action with the given parameters in
priority order. Accepts the 'users' and 'channels' keyword arguments to
determine which mode handlers should be included.
"""
actionList = self._getActionFunctionList(actionName, *params, **kw)
for action in actionList:
action[0](*params)
def runActionUntilTrue(self, actionName, *params, **kw):
"""
Calls functions for a given action with the given parameters in
priority order until one of them returns a true value. Returns True
when one of the functions returned True. Accepts the 'users' and
'channels' keyword arguments to determine which mode handlers should be
included.
"""
actionList = self._getActionFunctionList(actionName, *params, **kw)
for action in actionList:
if action[0](*params):
return True
return False
def runActionUntilFalse(self, actionName, *params, **kw):
"""
Calls functions for a given action with the given parameters in
priority order until one of them returns a false value. Returns True
when one of the functions returned False. Accepts the 'users' and
'channels' keyword arguments to determine which mode handlers should be
included.
"""
actionList = self._getActionFunctionList(actionName, *params, **kw)
for action in actionList:
if not action[0](*params):
return True
return False
def runActionUntilValue(self, actionName, *params, **kw):
"""
Calls functions for a given action with the given parameters in
priority order until one of them returns a non-None value. Returns the
value returned by the function that returned a non-None value. Accepts
the 'users' and 'channels' keyword arguments to determine which mode
handlers should be included.
"""
actionList = self._getActionFunctionList(actionName, *params, **kw)
for action in actionList:
value = action[0](*params)
if value is not None:
return value
return None
def runActionFlagTrue(self, actionName, *params, **kw):
"""
Calls all functions for a given action with the given parameters in
priority order. Returns True when one of the functions returns a true
value. Accepts the 'users' and 'channels' keyword arguments to
determine which mode handlers should be included.
"""
oneIsTrue = False
actionList = self._getActionFunctionList(actionName, *params, **kw)
for action in actionList:
if action[0](*params):
oneIsTrue = True
return oneIsTrue
def runActionFlagFalse(self, actionName, *params, **kw):
"""
Calls all functions for a given action with the given parameters in
priority order. Returns True when one of the functions returns a false
value. Accepts the 'users' and 'channels' keyword arguments to
determine which mode handlers should be included.
"""
oneIsFalse = False
actionList = self._getActionFunctionList(actionName, *params, **kw)
for action in actionList:
if action[0](*params):
oneIsFalse = True
return oneIsFalse
def runActionProcessing(self, actionName, data, *params, **kw):
"""
Calls functions for a given action with the given parameters in
priority order until the provided data is all processed (the data
parameter becomes empty). Accepts 'users' and 'channels' keyword
arguments to determine which mode handlers should be included.
"""
actionList = self._getActionFunctionList(actionName, data, *params, **kw)
for action in actionList:
action[0](data, *params)
if not data:
return
def runActionProcessingMultiple(self, actionName, dataList, *params, **kw):
"""
Calls functions for a given action with the given parameters in
priority order until the provided data is all processed (all of the
data structures in the dataList parameter become empty). Accepts
'users' and 'channels' keyword arguments to determine which mode
handlers should be included.
"""
paramList = dataList + params
actionList = self._getActionFunctionList(actionName, *paramList, **kw)
for action in actionList:
action[0](*paramList)
for data in dataList:
if data:
break
else:
return
def runComboActionStandard(self, actionList, **kw):
"""
Calls all functions for the given actions with the given parameters in
priority order. Actions are specifed as a list of tuples:
[ ("action1", param1, param2, ...), ("action2", param1, param2, ...) ]
Accepts 'users' and 'channels' keyword arguments to determine which
mode handlers should be included.
"""
actionFuncLists = {}
actionParameters = {}
for action in actionList:
parameters = action[1:]
actionParameters[action[0]] = parameters
actionFuncLists[action[0]] = self._getActionFunctionList(action[0], *parameters, **kw)
funcList = self._combineActionFunctionLists(actionFuncLists)
for actionName, actionFunc in funcList:
actionFunc(*actionParameters[actionName])
def runComboActionUntilTrue(self, actionList, **kw):
"""
Calls functions for the given actions with the given parameters in
priority order until one of the functions returns a true value. Actions
are specified as a list of tuples:
[ ("action1", param1, param2, ...), ("action2", param1, param2, ...) ]
Returns True if one of the functions returned a true value. Accepts
'users' and 'channels' keyword arguments to determine which mode
handlers should be included.
"""
actionFuncLists = {}
actionParameters = {}
for action in actionList:
parameters = action[1:]
actionParameters[action[0]] = parameters
actionFuncLists[action[0]] = self._getActionFunctionList(action[0], *parameters, **kw)
funcList = self._combineActionFunctionLists(actionFuncLists)
for actionName, actionFunc in funcList:
if actionFunc(*actionParameters[actionName]):
return True
return False
def runComboActionUntilFalse(self, actionList, **kw):
"""
Calls functions for the given actions with the given parameters in
priority order until one of the functions returns a false value.
Actions are specified as a list of tuples:
[ ("action1", param1, param2, ...), ("action2", param1, param2, ...) ]
Returns True if one of the functions returned a false value. Accepts
'users' and 'channels' keyword arguments to determine which mode
handlers should be included.
"""
actionFuncLists = {}
actionParameters = {}
for action in actionList:
parameters = action[1:]
actionParameters[action[0]] = parameters
actionFuncLists[action[0]] = self._getActionFunctionList(action[0], *parameters, **kw)
funcList = self._combineActionFunctionLists(actionFuncLists)
for actionName, actionFunc in funcList:
if not actionFunc(*actionParameters[actionName]):
return True
return False
def runComboActionUntilValue(self, actionList, **kw):
"""
Calls functions for the given actions with the given parameters in
priority order until one of the functions returns a non-None value.
Actions are specified as a list of tuples:
[ ("action1", param1, param2, ...), ("action2", param1, param2, ...) ]
Returns the value returned by the function that returned a non-None
value. Accepts 'users' and 'channels' keyword arguments to determine
which mode handlers should be included.
"""
actionFuncLists = {}
actionParameters = {}
for action in actionList:
parameters = action[1:]
actionParameters[action[0]] = parameters
actionFuncLists[action[0]] = self._getActionFunctionList(action[0], *parameters, **kw)
funcList = self._combineActionFunctionLists(actionFuncLists)
for actionName, actionFunc in funcList:
value = actionFunc(*actionParameters[actionName])
if value is not None:
return value
return None
def runComboActionFlagTrue(self, actionList, **kw):
"""
Calls all functions for the given actions with the given parameters in
priority order. Actions are specified as a list of tuples:
[ ("action1", param1, param2, ...), ("action2", param1, param2, ...) ]
Returns True if any of the functions called returned a true value.
Accepts 'users' and 'channels' keyword arguments to determine which
mode handlers should be included.
"""
actionFuncLists = {}
actionParameters = {}
for action in actionList:
parameters = action[1:]
actionParameters[action[0]] = parameters
actionFuncLists[action[0]] = self._getActionFunctionList(action[0], *parameters, **kw)
funcList = self._combineActionFunctionLists(actionFuncLists)
oneIsTrue = False
for actionName, actionFunc in funcList:
if actionFunc(*actionParameters[actionName]):
oneIsTrue = True
return oneIsTrue
def runComboActionFlagFalse(self, actionList, **kw):
"""
Calls all functions for the given actions with the given parameters in
priority order. Actions are specified as a list of tuples:
[ ("action1", param1, param2, ...), ("action2", param1, param2, ...) ]
Returns True if any of the functions called returned a false value.
Accepts 'users' and 'channels' keyword arguments to determine which
mode handlers should be included.
"""
actionFuncLists = {}
actionParameters = {}
for action in actionList:
parameters = action[1:]
actionParameters[action[0]] = parameters
actionFuncLists[action[0]] = self._getActionFunctionList(action[0], *parameters, **kw)
funcList = self._combineActionFunctionLists(actionFuncLists)
oneIsFalse = False
for actionName, actionFunc in funcList:
if not actionFunc(*actionParameters[actionName]):
oneIsFalse = True
return oneIsFalse
def runComboActionProcessing(self, data, actionList, **kw):
"""
Calls functions for the given actions with the given parameters in
priority order until the data given has been processed (the data
parameter becomes empty). Actions are specified as a list of tuples:
[ ("action1", param1, param2, ...), ("action2", param1, param2, ...) ]
Accepts 'users' and 'channels' keyword arguments to determine which
mode handlers should be included.
"""
actionFuncLists = {}
actionParameters = {}
for action in actionList:
parameters = [data] + action[1:]
actionParameters[action[0]] = parameters
actionFuncLists[action[0]] = self._getActionFunctionList(action[0], *parameters, **kw)
funcList = self._combineActionFunctionLists(actionFuncLists)
for actionName, actionFunc in funcList:
actionFunc(*actionParameters[actionName])
if not data:
break
def runComboActionProcessingMultiple(self, dataList, actionList, **kw):
"""
Calls functions for the given actions with the given parameters in
priority order until the data given has been processed (all the data
items in the dataList parameter become empty). Actions are specified as
a list of tuples:
[ ("action1", param1, param2, ...), ("action2", param1, param2, ...) ]
Accepts 'users' and 'channels' keyword arguments to determine which
mode handlers should be included.
"""
actionFuncLists = {}
actionParameters = {}
for action in actionList:
parameters = dataList + action[1:]
actionParameters[action[0]] = parameters
actionFuncLists[action[0]] = self._getActionFunctionList(action[0], *parameters, **kw)
funcList = self._combineActionFunctionLists(actionFuncLists)
for actionName, actionFunc in funcList:
actionFunc(*actionParameters[actionName])
for data in dataList:
if data:
break
else:
return
class FSProtocol(basic.LineReceiver):
class State(Enum):
IDLE = 0
POLLING = 1
POLLED = 2
RAW = 3
def __init__(self):
self.log = Logger()
self.setIdleState()
def setIdleState(self):
self.my_state = self.State.IDLE
self.wait_for_list = None
self.file_transfer_peer = None
self.setLineMode()
def connectionMade(self):
self.log.info("New connection established: {peer!r}, {pid!s}",
peer=self.transport.getPeer(),
pid=id(self))
self.factory.protocols.append(self)
self.transport.write(b'Hello form server\r\n')
def send(self, command, argument):
if command != "":
command += " "
self.transport.write(command.upper().encode() + argument.encode() +
b'\r\n')
# self.log.info("Sending to {peer!r}: {command!s} {argument!s}",
# command=command, argument=argument)
def doReject(self, line):
self.log.error("We had to say no to {peer!r}: {line}",
peer=self.transport.getPeer(),
line=line)
self.transport.write(b'REJECT ' + line.encode() + b'\r\n')
# def doRequest(self, regex):
# self.log.debug("server asks for files: {regex!s}", regex=regex)
# self.transport.write(b'REQUEST ' + regex.encode() + b'\r\n')
def requestFiles(self, regex):
'''
Called by factory, if one of users requested a search
'''
if self.my_state != self.State.IDLE or self.wait_for_list is not None:
return defer.fail(RuntimeError("Connection is busy"))
# Go to POLLED state, so no one will interrupt us
self.my_state = self.State.POLLED
self.send("REQUEST", regex)
def debuglogc(data):
self.log.debug("wait_for_list:requestFiles:callback: {data!r}",
data=data)
return data
def debugloge(err):
self.log.debug("wait_for_list:requestFiles:errback: {err!r}",
err=err)
return err
self.wait_for_list = defer.Deferred().addCallbacks(
debuglogc, debugloge)
# We create Deferred, that will be fired as the response comes from user
return self.wait_for_list
def trySetStateIdle(self):
# Some operation has finished
# Check if there is file request
# if len(self.outer_req) > 0:
# # Set state
# self.state = self.State.POLLED
# # Ask peer
# regex,d = self.outer_req[0]
# self.transport.write("TODO some message" + regex + b'\r\n')
pass
def handleFind(self, regex):
'''
Hanles the FIND request from user
'''
self.my_state = self.State.POLLING
# The factory will ask all the others about our regexp
# The result here is the string
d = self.factory.getFiles(self, regex)
def response(ans):
self.log.info("FIND command returns next {filelist!s}",
filelist=ans)
self.send('RESPONSE', ans)
def error(e):
self.doReject(e.getErrorMessage())
def idle(ignore):
self.my_state = self.State.IDLE
d.addCallbacks(response, error)
d.addBoth(idle)
def lineReceived(self, line):
self.log.info("lineReceived: {peer!r} got message: {line!s}",
peer=self.transport.getPeer(),
line=line)
line = line.decode()
data = cleanInput(line)
if len(data) == 0 or data == '':
self.log.info("Oh, never mind, it's gone")
self.doReject("Seems like a bunch of tabs")
return
command = data[0].upper()
argument = data[1] if len(data) > 1 else None
# Analize the command
if command == 'FIND': # Find files from other users
if self.my_state != self.State.IDLE:
self.doReject("Another operation is in run")
return
if argument is None:
self.doReject("Regex expected")
return
self.handleFind(argument)
elif command == 'FILES': # Polled user returns list of files here
if self.my_state != self.State.POLLED:
self.doReject(
"Got list of files, but there is no request for them")
return
# self.log.debug("Got list of files from {peer!r}", peer=self.transport.getPeer())
if argument is None:
self.wait_for_list.errback(
RuntimeError("This peer has no such files"))
else:
# Return self id and list of files
self.wait_for_list.callback(':'.join([str(id(self)),
argument]))
# Cleanup
self.wait_for_list = None
self.my_state = self.State.IDLE
elif command == 'GET': # User asks for file
'''
This is a routine of connection, whitch needs file
'''
# Check if we ready
if self.my_state != self.State.IDLE:
self.doReject("Can not give you file at the moment")
return
# Get arguments
if argument is None:
self.doReject("Expected filename for GET operation")
return
try:
pid, filename = argument.split(':', 1)
except (IndexError):
self.doReject("not valid filename")
return
# Find the peer
peer = self.factory.findConnById(pid)
# Check if we are ready
if peer is None:
self.doReject("Sorry, the peer seems afk")
return
if peer.my_state != peer.State.IDLE:
self.doReject("Sorry, the peer seems busy")
return
# Ready to begin, don't interrupt us please
peer.my_state = peer.State.RAW
self.my_state = self.State.RAW
peer.file_transfer_peer = self
self.file_transfer_peer = peer
peer.send("OBTAIN", filename)
self.send("RAW", filename)
self.setRawMode()
peer.setRawMode()
elif command == 'ALICE': # User remembers how this project started
self.send('', "nice girl")
else:
self.doReject("not a protocol command: %s" % command)
def rawDataReceived(self, data):
'''
Should be called, if user sends data to peer in response
'''
if self.file_transfer_peer is None:
self.log.warning("RECIEVING RAW DATA WITHOUT PEER TO RESEND")
return
peer = self.file_transfer_peer
self.log.info("RAW: resending data: %d KB" % len(data))
peer.transport.write(data)
if not data.endswith(b'\r\n'):
# Transmission not done yet
return
self.log.info("RAW: Transmission complete")
peer.setIdleState()
self.setIdleState()
def connectionLost(self, reason):
self.log.info("Connection closed: {peer!r}, {reason!r}",
peer=self.transport.getPeer(),
reason=reason)
self.factory.protocols.remove(self)
class WebSocketDispatcherProtocol(Protocol):
"""Protocol implementing ShinySDR's WebSocket service.
This protocol's transport should be a txWS WebSocket transport.
"""
def __init__(self, caps, subscription_context):
self.__log = Logger()
self.__subscription_context = subscription_context
self._caps = caps
self._seenValues = {}
self.inner = None
def dataReceived(self, data):
"""Twisted Protocol implementation.
Additionally, txWS takes no care with exceptions here, so we catch and log."""
# pylint: disable=broad-except
try:
if self.inner is None:
# To work around txWS's lack of a notification when the URL is available, all clients send a dummy first message.
self.__dispatch_url()
else:
self.inner.dataReceived(data)
except Exception:
self.__log.failure('Error processing incoming WebSocket message')
def __dispatch_url(self):
self.__log.info('Stream connection to {url}',
url=self.transport.location)
_scheme, _netloc, path_bytes, _params, query_bytes, _fragment = urlparse(
bytes_or_ascii(self.transport.location))
# py2/3: unquote returns str in either version but we want Unicode
path = [
six.text_type(urllib.parse.unquote(x))
for x in path_bytes.split(b'/')
]
assert path[0] == ''
path[0:1] = []
cap_string = path[0]
if cap_string in self._caps:
root_object = self._caps[cap_string]
path[0:1] = []
else:
raise Exception('Unknown cap') # TODO better error reporting
if path == [AUDIO_STREAM_PATH_ELEMENT]:
options = parse_audio_stream_options(parse_qs(query_bytes, 1))
self.inner = AudioStreamInner(the_reactor, self.__send,
root_object, options.sample_rate)
elif len(path) >= 1 and path[0] == CAP_OBJECT_PATH_ELEMENT:
# note _lookup_block may throw. TODO: Better error reporting
root_object = _lookup_block(root_object, path[1:])
self.inner = StateStreamInner(
self.__send, root_object, path_bytes.decode('utf-8'),
self.__subscription_context
) # note reuse of WS path as HTTP path; probably will regret this
else:
raise Exception('Unknown path: %r' % (path, ))
def connectionMade(self):
"""twisted Protocol implementation"""
self.transport.setBinaryMode(True)
# Unfortunately, txWS calls this too soon for transport.location to be available
def connectionLost(self, reason):
# pylint: disable=signature-differs
"""twisted Protocol implementation"""
if self.inner is not None:
self.inner.connectionLost(reason)
def __send(self, message, safe_to_drop=False):
if len(self.transport.transport.dataBuffer) > 1000000:
# TODO: condition is horrible implementation-diving kludge
# Don't accumulate indefinite buffer if we aren't successfully getting it onto the network.
# TODO: There are no tests of this mechanism
if safe_to_drop:
self.__log.warn('Dropping data going to stream {url}',
url=self.transport.location)
else:
self.__log.error(
'Dropping connection due to too much data on stream {url}',
url=self.transport.location)
self.transport.close(reason='Too much data buffered')
else:
self.transport.write(message)
class Rest(object):
def __init__(
self,
host='https://developer-api.nest.com',
token=None,
event_handler=None,
net_type='lan'):
self.log = Logger()
self.host = host
self.token = token
self.event_handler = event_handler
self.pool = HTTPConnectionPool(reactor, persistent=True)
self.loc = None
self.reconnect = False
self.fail_count = 0
if event_handler:
self.reconnect = True
d = self.request(headers={'User-Agent': ['onDemand Rest Client'],
'Accept': ['text/event-stream']})
d.addCallback(self.on_disconnect)
def __getattr__(self, name):
try:
super(Rest, self).__getattr__(name)
except AttributeError:
return RestCall(self, name)
def on_disconnect(self, reason):
if not reason:
reason = {'reason': 'no_message'}
self.log.critical(
'disconnected: {reason}', reason=reason['reason'])
if self.fail_count > 10:
self.log.error('Max error count reached, aborting connection')
def test_connectivity(count):
if self.fail_count == count:
self.fail_count = 0
self.fail_count += 1
c = self.fail_count
reactor.callLater(10, test_connectivity, c) # @UndefinedVariable
if self.reconnect:
d = self.request(headers={'User-Agent': ['onDemand Rest Client'],
'Accept': ['text/event-stream']})
d.addCallback(self.on_disconnect)
def request(self, method='GET',
path='',
headers={'User-Agent': ['onDemand/1.0 (Rest_Client)'],
'Accept': ['application/json']},
body=None):
data = None
if self.loc:
host = '/'.join((self.loc, path))
else:
host = '/'.join((self.host, path))
if self.token:
host += '?auth=' + self.token
if body:
headers.update({'Content-Type': ['application/json']})
data = FileBodyProducer(StringIO(json.dumps(body)))
agent = RedirectAgent(Agent(reactor, pool=self.pool))
d = agent.request(method, host, Headers(headers), data)
def cbFail(fail):
if hasattr(fail.value, 'response'):
if hasattr(fail.value.response, 'code'):
if fail.value.response.code == 307:
loc = fail.value.response.headers.getRawHeaders(
'location')
new = urlparse(loc[0])
newhost = '://'.join((new.scheme, new.netloc))
if newhost == self.host:
self.loc = None
else:
self.loc = newhost
self.log.debug('redirect: %s' % self.loc)
data = FileBodyProducer(StringIO(json.dumps(body)))
d = agent.request(
method, loc[0], Headers(headers), data)
d.addCallbacks(cbRequest, cbFail)
return d
elif fail.value.response.code == 404 and self.loc:
self.loc = None
host = '/'.join((self.host, path))
if self.token:
host += '?auth=' + self.token
d = self.request(method, host, Headers(headers), body)
d.addCallbacks(cbRequest, cbFail)
return d
else:
print(dir(fail.value))
print(fail.value.message)
print(fail.value.args)
self.log.error('unhandled failure: %s -- %s' % (
fail.value.message, fail.value))
def cbRequest(response):
# print 'Response version:', response.version
# print 'Response code:', response.code
# print 'Response phrase:', response.phrase
# print 'Response headers:'
# print pformat(list(response.headers.getAllRawHeaders()))
finished = Deferred()
response.deliverBody(RestHandle(finished, self.event_handler))
return finished
d.addCallbacks(cbRequest, cbFail)
return d
class PostgresListenerService(Service, object):
"""Listens for NOTIFY messages from postgres.
A new connection is made to postgres with the isolation level of
autocommit. This connection is only used for listening for notifications.
Any query that needs to take place because of a notification should use
its own connection. This class runs inside of the reactor. Any long running
action that occurrs based on a notification should defer its action to a
thread to not block the reactor.
:ivar connection: A database connection within one of Django's wrapper.
:ivar connectionFileno: The fileno of the underlying database connection.
:ivar connecting: a :class:`Deferred` while connecting, `None` at all
other times.
:ivar disconnecting: a :class:`Deferred` while disconnecting, `None`
at all other times.
"""
# Seconds to wait to handle new notifications. When the notifications set
# is empty it will wait this amount of time to check again for new
# notifications.
HANDLE_NOTIFY_DELAY = 0.5
CHANNEL_REGISTRAR_DELAY = 0.5
def __init__(self, alias="default"):
self.alias = alias
self.listeners = defaultdict(list)
self.autoReconnect = False
self.connection = None
self.connectionFileno = None
self.notifications = set()
self.notifier = task.LoopingCall(self.handleNotifies)
self.notifierDone = None
self.connecting = None
self.disconnecting = None
self.registeredChannels = set()
self.channelRegistrar = task.LoopingCall(
lambda: ensureDeferred(self.registerChannels()))
self.channelRegistrarDone = None
self.log = Logger(__name__, self)
self.events = EventGroup("connected", "disconnected")
def startService(self):
"""Start the listener."""
super(PostgresListenerService, self).startService()
self.autoReconnect = True
return self.tryConnection()
def stopService(self):
"""Stop the listener."""
super(PostgresListenerService, self).stopService()
self.autoReconnect = False
return self.loseConnection()
def connected(self):
"""Return True if connected."""
if self.connection is None:
return False
if self.connection.connection is None:
return False
return self.connection.connection.closed == 0
def logPrefix(self):
"""Return nice name for twisted logging.
This is required to satisfy `IReadDescriptor`, which inherits from
`ILoggingContext`.
"""
return self.log.namespace
def isSystemChannel(self, channel):
"""Return True if channel is a system channel."""
return channel.startswith("sys_")
def doRead(self):
"""Poll the connection and process any notifications."""
try:
self.connection.connection.poll()
except Exception:
# If the connection goes down then `OperationalError` is raised.
# It contains no pgcode or pgerror to identify the reason so no
# special consideration can be made for it. Hence all errors are
# treated the same, and we assume that the connection is broken.
#
# We do NOT return a failure, which would signal to the reactor
# that the connection is broken in some way, because the reactor
# will end up removing this instance from its list of selectables
# but not from its list of readable fds, or something like that.
# The point is that the reactor's accounting gets muddled. Things
# work correctly if we manage the disconnection ourselves.
#
self.loseConnection(Failure(error.ConnectionLost()))
else:
# Add each notify to to the notifications set. This removes
# duplicate notifications when one entity in the database is
# updated multiple times in a short interval. Accumulating
# notifications and allowing the listener to pick them up in
# batches is imperfect but good enough, and simple.
notifies = self.connection.connection.notifies
if len(notifies) != 0:
for notify in notifies:
if self.isSystemChannel(notify.channel):
# System level message; pass it to the registered
# handler immediately.
if notify.channel in self.listeners:
# Be defensive in that if a handler does not exist
# for this channel then the channel should be
# unregisted and removed from listeners.
if len(self.listeners[notify.channel]) > 0:
handler = self.listeners[notify.channel][0]
handler(notify.channel, notify.payload)
else:
self.unregisterChannel(notify.channel)
del self.listeners[notify.channel]
else:
# Unregister the channel since no listener is
# registered for this channel.
self.unregisterChannel(notify.channel)
else:
# Place non-system messages into the queue to be
# processed.
self.notifications.add(
(notify.channel, notify.payload))
# Delete the contents of the connection's notifies list so
# that we don't process them a second time.
del notifies[:]
def fileno(self):
"""Return the fileno of the connection."""
return self.connectionFileno
def startReading(self):
"""Add this listener to the reactor."""
self.connectionFileno = self.connection.connection.fileno()
reactor.addReader(self)
def stopReading(self):
"""Remove this listener from the reactor."""
try:
reactor.removeReader(self)
except IOError as error:
# ENOENT here means that the fd has already been unregistered
# from the underlying poller. It is as yet unclear how we get
# into this state, so for now we ignore it. See epoll_ctl(2).
if error.errno != ENOENT:
raise
finally:
self.connectionFileno = None
def register(self, channel, handler):
"""Register listening for notifications from a channel.
When a notification is received for that `channel` the `handler` will
be called with the action and object id.
"""
handlers = self.listeners[channel]
if self.isSystemChannel(channel) and len(handlers) > 0:
# A system can only be registered once. This is because the
# message is passed directly to the handler and the `doRead`
# method does not wait for it to finish if its a defer. This is
# different from normal handlers where we will call each and wait
# for all to resolve before continuing to the next event.
raise PostgresListenerRegistrationError(
"System channel '%s' has already been registered." % channel)
else:
handlers.append(handler)
self.runChannelRegistrar()
def unregister(self, channel, handler):
"""Unregister listening for notifications from a channel.
`handler` needs to be same handler that was registered.
"""
if channel not in self.listeners:
raise PostgresListenerUnregistrationError(
"Channel '%s' is not registered with the listener." % channel)
handlers = self.listeners[channel]
if handler in handlers:
handlers.remove(handler)
else:
raise PostgresListenerUnregistrationError(
"Handler is not registered on that channel '%s'." % channel)
if len(handlers) == 0:
# Channels have already been registered. Unregister the channel.
del self.listeners[channel]
self.runChannelRegistrar()
@synchronous
def createConnection(self):
"""Create new database connection."""
db = connections.databases[self.alias]
backend = load_backend(db["ENGINE"])
return backend.DatabaseWrapper(db,
self.alias,
allow_thread_sharing=True)
@synchronous
def startConnection(self):
"""Start the database connection."""
self.connection = self.createConnection()
self.connection.connect()
self.connection.set_autocommit(True)
@synchronous
def stopConnection(self):
"""Stop database connection."""
# The connection is often in an unexpected state here -- for
# unexplained reasons -- so be careful when unpealing layers.
connection_wrapper, self.connection = self.connection, None
if connection_wrapper is not None:
connection = connection_wrapper.connection
if connection is not None and not connection.closed:
connection_wrapper.commit()
connection_wrapper.close()
def tryConnection(self):
"""Keep retrying to make the connection."""
if self.connecting is None:
if self.disconnecting is not None:
raise RuntimeError(
"Cannot attempt to make new connection before "
"pending disconnection has finished.")
def cb_connect(_):
self.log.info("Listening for database notifications.")
def eb_connect(failure):
self.log.error(
"Unable to connect to database: {error}",
error=failure.getErrorMessage(),
)
if failure.check(CancelledError):
return failure
elif self.autoReconnect:
return deferLater(reactor, 3, connect)
else:
return failure
def connect(interval=self.HANDLE_NOTIFY_DELAY):
d = deferToThread(self.startConnection)
d.addCallback(callOut, self.runChannelRegistrar)
d.addCallback(lambda result: self.channelRegistrarDone)
d.addCallback(callOut, self.events.connected.fire)
d.addCallback(callOut, self.startReading)
d.addCallback(callOut, self.runHandleNotify, interval)
# On failure ensure that the database connection is stopped.
d.addErrback(callOut, deferToThread, self.stopConnection)
d.addCallbacks(cb_connect, eb_connect)
return d
def done():
self.connecting = None
self.connecting = connect().addBoth(callOut, done)
return self.connecting
def loseConnection(self, reason=Failure(error.ConnectionDone())):
"""Request that the connection be dropped."""
if self.disconnecting is None:
self.registeredChannels.clear()
d = self.disconnecting = Deferred()
d.addBoth(callOut, self.stopReading)
d.addBoth(callOut, self.cancelChannelRegistrar)
d.addBoth(callOut, self.cancelHandleNotify)
d.addBoth(callOut, deferToThread, self.stopConnection)
d.addBoth(callOut, self.connectionLost, reason)
def done():
self.disconnecting = None
d.addBoth(callOut, done)
if self.connecting is None:
# Already/never connected: begin shutdown now.
self.disconnecting.callback(None)
else:
# Still connecting: cancel before disconnect.
self.connecting.addErrback(suppress, CancelledError)
self.connecting.chainDeferred(self.disconnecting)
self.connecting.cancel()
return self.disconnecting
def connectionLost(self, reason):
"""Reconnect when the connection is lost."""
self.connection = None
if reason.check(error.ConnectionDone):
self.log.debug("Connection closed.")
elif reason.check(error.ConnectionLost):
self.log.debug("Connection lost.")
else:
self.log.failure("Connection lost.", reason)
if self.autoReconnect:
reactor.callLater(3, self.tryConnection)
self.events.disconnected.fire(reason)
def registerChannel(self, channel):
"""Register the channel."""
with closing(self.connection.cursor()) as cursor:
if self.isSystemChannel(channel):
# This is a system channel so listen only called once.
cursor.execute("LISTEN %s;" % channel)
else:
# Not a system channel so listen called once for each action.
for action in sorted(map_enum(ACTIONS).values()):
cursor.execute("LISTEN %s_%s;" % (channel, action))
def unregisterChannel(self, channel):
"""Unregister the channel."""
with closing(self.connection.cursor()) as cursor:
if self.isSystemChannel(channel):
# This is a system channel so unlisten only called once.
cursor.execute("UNLISTEN %s;" % channel)
else:
# Not a system channel so unlisten called once for each action.
for action in sorted(map_enum(ACTIONS).values()):
cursor.execute("UNLISTEN %s_%s;" % (channel, action))
async def registerChannels(self):
"""Listen/unlisten to channels that were registered/unregistered.
When a call to register() or unregister() is made, the listeners
dict is updated, and the keys of that dict represents all the
channels that we should listen to.
The service keeps a list of channels that it already listens to
in the registeredChannels dict. We issue a call to postgres to
listen to all channels that are in listeners but not in
registeredChannels, and a call to unlisten for all channels that
are in registeredChannels but not in listeners.
"""
to_register = set(self.listeners.keys()).difference(
self.registeredChannels)
to_unregister = self.registeredChannels.difference(
set(self.listeners.keys()))
# If there's nothing to do, we can stop the loop. If there is
# any work to be done, we do the work, and then check
# whether we should stop at the beginning of the next loop
# iteration. The reason is that every time we yield, another
# deferred might call register() or unregister().
if not to_register and not to_unregister:
self.channelRegistrar.stop()
else:
for channel in to_register:
await deferToThread(self.registerChannel, channel)
self.registeredChannels.add(channel)
for channel in to_unregister:
await deferToThread(self.unregisterChannel, channel)
self.registeredChannels.remove(channel)
def convertChannel(self, channel):
"""Convert the postgres channel to a registered channel and action.
:raise PostgresListenerNotifyError: When {channel} is not registered or
{action} is not in `ACTIONS`.
"""
channel, action = channel.split("_", 1)
if channel not in self.listeners:
raise PostgresListenerNotifyError(
"%s is not a registered channel." % channel)
if action not in map_enum(ACTIONS).values():
raise PostgresListenerNotifyError("%s action is not supported." %
action)
return channel, action
def runChannelRegistrar(self):
"""Start the loop for listening to channels in postgres.
It will only start if the service is connected to postgres.
"""
if self.connection is not None and not self.channelRegistrar.running:
self.channelRegistrarDone = self.channelRegistrar.start(
self.CHANNEL_REGISTRAR_DELAY, now=True)
def cancelChannelRegistrar(self):
"""Stop the loop for listening to channels in postgres."""
if self.channelRegistrar.running:
self.channelRegistrar.stop()
return self.channelRegistrarDone
else:
return succeed(None)
def runHandleNotify(self, delay=0, clock=reactor):
"""Defer later the `handleNotify`."""
if not self.notifier.running:
self.notifierDone = self.notifier.start(delay, now=False)
def cancelHandleNotify(self):
"""Cancel the deferred `handleNotify` call."""
if self.notifier.running:
self.notifier.stop()
return self.notifierDone
else:
return succeed(None)
def handleNotifies(self, clock=reactor):
"""Process all notify message in the notifications set."""
def gen_notifications(notifications):
while len(notifications) != 0:
yield notifications.pop()
return task.coiterate(
self.handleNotify(notification, clock=clock)
for notification in gen_notifications(self.notifications))
def handleNotify(self, notification, clock=reactor):
"""Process a notify message in the notifications set."""
channel, payload = notification
try:
channel, action = self.convertChannel(channel)
except PostgresListenerNotifyError:
# Log the error and continue processing the remaining
# notifications.
self.log.failure("Failed to convert channel {channel!r}.",
channel=channel)
else:
defers = []
handlers = self.listeners[channel]
# XXX: There could be an arbitrary number of listeners. Should we
# limit concurrency here? Perhaps even do one at a time.
for handler in handlers:
d = defer.maybeDeferred(handler, action, payload)
d.addErrback(lambda failure: self.log.failure(
"Failure while handling notification to {channel!r}: "
"{payload!r}",
failure,
channel=channel,
payload=payload,
))
defers.append(d)
return defer.DeferredList(defers)
class Discord(IRCClient):
nickname = "discord"
realname = "Discord"
username = "******"
versionName = "Discord"
versionNum = "0.01"
magicFile = "true.txt"
def __init__(self, accessList):
self.logger = Logger(observer=textFileLogObserver(sys.stdout))
self.accessList = [nick.lower() for nick in accessList]
if not os.path.exists(self.magicFile):
self.logger.info("Creating magic file")
try:
with open(self.magicFile, "a"):
pass
except Exception as ex:
self.logger.error("Unable to create magic file! {0}".format(
ex.message))
reactor.stop()
self.markovGenerator = pymarkov.MarkovChainGenerator(self.magicFile)
self.channels = []
self.channelPhrasers = {}
self.logger.debug("Discord initialized")
# Maybe add hook/plugin system here?
self.commands = Commands.Commands(self)
def removeChannel(self, channel):
try:
self.channels.remove(channel)
self.channelPhrasers[channel].stop()
del self.channelPhrasers[channel]
except:
self.logger.error("Error removing {channel} from collection",
channel=channel)
def insertPhrase(self, phrase):
try:
with open(self.magicFile, "a") as magicFile:
magicFile.write("{0}\n".format(phrase))
try:
file, ext = os.path.splitext(self.magicFile)
os.remove("{0}-pickled{1}".format(file, ext))
# Simply re-populating the dictionary isn't enough for some reason
self.markovGenerator = pymarkov.MarkovChainGenerator(
self.magicFile, 2)
except IOError as ex:
self.logger.error("Unable to delete pickled file. {0}".format(
ex.message))
except Exception as ex:
self.logger.error(
"Unable to insert phrase into magic file! {0}".format(
ex.message))
def kickedFrom(self, channel, kicker, message):
self.removeChannel(channel)
self.logger.info("Kicked from {channel} by {kicker}",
channel=channel,
kicker=kicker)
def left(self, channel):
self.removeChannel(channel)
self.logger.info("Left {channel}", channel=channel)
def handleMessage(self, user, channel, message):
senderNickname = user.split("!")[0]
if message.startswith("~reload") and senderNickname in self.accessList:
self.logger.info("Reloading commands module")
self.say(channel, "Reloading.")
try:
commandsModule = reload(Commands)
self.commands = commandsModule.Commands(self)
except Exception as ex:
self.say(
channel,
"Failed to load commands module - {0}".format(ex.message))
elif message.startswith("~"):
# Don't log commands to the brain
commandMessage = message[1:]
self.commands.handleCommand(user, channel, commandMessage)
else:
self.logger.info("Adding {message!r} to brain", message=message)
# Avoid storing anything with the bot's name in it
brainMessage = message.strip(self.nickname)
self.insertPhrase(brainMessage)
try:
randomPhrase = self.generateSentence()
if self.nickname in message and channel.startswith(
"#") and self.channelPhrasers[channel].running:
phrase = "{0}, {1}".format(senderNickname, randomPhrase)
self.say(channel, phrase)
elif channel == self.nickname:
self.logger.debug("Sending message to {nickname}",
nickname=senderNickname)
self.msg(senderNickname, randomPhrase)
else:
pass
except IndexError as generationError:
self.logger.error(generationError.message)
def privmsg(self, user, channel, message):
self.logger.info("Received message from {user} in {channel}",
user=user,
channel=channel)
# deferToThread(self.handleMessage, user, channel, message)
self.handleMessage(user, channel, message)
def signedOn(self):
self.logger.info("Signed on")
self.join("#bots")
def joined(self, channel):
self.channels.append(channel)
self.logger.info("Joined channel {channel!r}", channel=channel)
channelPhraser = LoopingCall(self.sayRandomPhrase, channel)
reactor.callLater(2, channelPhraser.start, 600)
self.channelPhrasers[channel] = channelPhraser
def generateSentence(self):
try:
sentence = self.markovGenerator.generate_sentence()
sentence = sentence.strip("<{0}>".format(self.nickname))
sentence = sentence.strip(self.nickname)
return sentence
except (IndexError, ValueError) as ex:
self.logger.error(ex.message)
def sayRandomPhrase(self, channel):
sentence = self.generateSentence()
self.say(channel, sentence)
