class FeedbackConn():
def __init__(self, host=None, certfile=None, loop=None):
self.ioloop = loop or ioloop.IOLoop.instance()
self.host = host or settings.get('feedback_host')
self.certfile = certfile or settings.get('certfile')
self.connect()
def connect(self):
self.s = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
self.stream = SSLIOStream(self.s, read_chunk_size=38, ssl_options=dict(certfile=self.certfile))
self.stream.connect(self.host, self._on_connect)
self.stream.read_until_close(self._on_close, streaming_callback=self._on_read)
def _on_connect(self):
logging.info("connected to %s:%d" % (self.host[0], self.host[1]))
def _on_read(self, data):
#logging.info("read %d bytes" % len(data))
if len(data) == 38:
timestamp, toklen, token = struct.unpack_from('!IH32s', data, 0)
logging.info("flagging user %s at %d" % (token.encode('hex'), timestamp))
mc.set(token.encode('hex'), timestamp)
def _on_close(self, data):
logging.info("disconnected %s:%d" % (self.host[0], self.host[1]))
try:
self.stream.close()
self.s.close()
except:
pass
self.ioloop.add_timeout(time.time()+settings.get('feedback_reconnect_lag'), self.connect)
class _BaseSSLService(object):
"""Base SSL connection to Apple's push notification servers. Retry on
disconnect is handled via an exponential backoff.
"""
_MAX_BACKOFF_SECS = 600.0 # 10 minutes
_PUSH_TOKEN_FMT = '%s:%s'
def __init__(self, settings, host_key):
self._settings = settings
self._host = settings[host_key]
self._retries = 0
self._io_loop = IOLoop.current()
self._ResetBackoff()
self._Connect()
def IsValid(self):
return self._stream is not None
def _FormatPushToken(self, token):
return _BaseSSLService._PUSH_TOKEN_FMT % (self._settings['token-prefix'], token)
def _Connect(self):
try:
ssl_options = {'certfile': secrets.GetSecretFile(self._settings['certfile'])}
self._sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
self._stream = SSLIOStream(self._sock, io_loop=self._io_loop, ssl_options=ssl_options)
self._stream.set_close_callback(self._OnClose)
self._stream.connect(self._host, self._OnConnect)
except KeyError:
logging.warning('failed to initialize connection to APN service at %s:%d '
'whose certificate is missing from secrets/%s' %
(self._host[0], self._host[1], self._settings['certfile']))
return
except:
self._stream = None
raise
def _OnConnect(self):
logging.info("connected to %s:%d" % (self._host[0], self._host[1]))
def _ResetBackoff(self):
"""Resets backoff to 'reconnect_lag' setting."""
self._backoff = self._settings.get('reconnect_lag')
def _OnClose(self):
logging.info("disconnected from %s:%d" % (self._host[0], self._host[1]))
try:
self._stream.close()
except:
pass
finally:
self._stream = None
timeout = time.time() + self._backoff
self._io_loop.add_timeout(timeout, self._Connect)
self._backoff = min(_BaseSSLService._MAX_BACKOFF_SECS, self._backoff * 2)
def connect_to_server(self, server_cls):
server = client = None
try:
sock, port = bind_unused_port()
server = server_cls(ssl_options=_server_ssl_options())
server.add_socket(sock)
ssl_ctx = ssl.create_default_context(ssl.Purpose.SERVER_AUTH)
ssl_ctx.check_hostname = False
ssl_ctx.verify_mode = ssl.CERT_NONE
# These tests fail with ConnectionAbortedErrors with TLS
# 1.3 on windows python 3.7.4 (which includes an upgrade
# to openssl 1.1.c. Other platforms might be affected with
# newer openssl too). Disable it until we figure out
# what's up.
# Update 2021-12-28: Still happening with Python 3.10 on
# Windows. OP_NO_TLSv1_3 now raises a DeprecationWarning.
with ignore_deprecation():
ssl_ctx.options |= getattr(ssl, "OP_NO_TLSv1_3", 0)
client = SSLIOStream(socket.socket(), ssl_options=ssl_ctx)
yield client.connect(("127.0.0.1", port))
self.assertIsNotNone(client.socket.cipher())
finally:
if server is not None:
server.stop()
if client is not None:
client.close()
class SSLStreamReader(object):
def __init__(self, host, port):
self._address = (host, port)
self._stream = None
self._clients = set()
def add(self, client):
self._clients.add(client)
log.debug('Added client %s', client)
if self._stream is None:
self._connect()
self._read_stream()
def remove(self, client):
self._clients.remove(client)
log.debug('Removed client %s', client)
@staticmethod
def process(data):
return data
@staticmethod
def write_message(client, data):
pass
def _connect(self):
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
self._stream = SSLIOStream(s)
log.debug('Connecting to %s', self._address)
self._stream.connect(self._address)
def _disconnect(self):
self._stream.close()
self._stream = None
log.debug('Disconnected from %s', self._address)
@coroutine
def _read_stream(self):
log.debug('Reading stream from %s', self._address)
while self._clients:
data = yield Task(self._stream.read_until, "\n")
output = self.process(data)
for client in self._clients:
self.write_message(client, output)
self._disconnect()
return
class SSLStreamReader(object):
def __init__(self, host, port):
self._address = (host, port)
self._stream = None
self._clients = set()
def add(self, client):
self._clients.add(client)
log.debug('Added client %s', client)
if self._stream is None:
self._connect()
self._read_stream()
def remove(self, client):
self._clients.remove(client)
log.debug('Removed client %s', client)
@staticmethod
def process(data):
return data
@staticmethod
def write_message(client, data):
pass
def _connect(self):
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
self._stream = SSLIOStream(s)
log.debug('Connecting to %s', self._address)
self._stream.connect(self._address)
def _disconnect(self):
self._stream.close()
self._stream = None
log.debug('Disconnected from %s', self._address)
@coroutine
def _read_stream(self):
log.debug('Reading stream from %s', self._address)
while self._clients:
data = yield Task(self._stream.read_until, "\n")
output = self.process(data)
for client in self._clients:
self.write_message(client, output)
self._disconnect()
return
class FeedbackConn():
def __init__(self, host=None, certfile=None, loop=None):
self.ioloop = loop or ioloop.IOLoop.instance()
self.host = host or settings.get('feedback_host')
self.certfile = certfile or settings.get('certfile')
self.connect()
def connect(self):
self.s = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
self.stream = SSLIOStream(self.s,
read_chunk_size=38,
ssl_options=dict(certfile=self.certfile))
self.stream.connect(self.host, self._on_connect)
self.stream.read_until_close(self._on_close,
streaming_callback=self._on_read)
def _on_connect(self):
logging.info("connected to %s:%d" % (self.host[0], self.host[1]))
def _on_read(self, data):
#logging.info("read %d bytes" % len(data))
if len(data) == 38:
timestamp, toklen, token = struct.unpack_from('!IH32s', data, 0)
logging.info("flagging user %s at %d" %
(token.encode('hex'), timestamp))
mc.set(token.encode('hex'), timestamp)
def _on_close(self, data):
logging.info("disconnected %s:%d" % (self.host[0], self.host[1]))
try:
self.stream.close()
self.s.close()
except:
pass
self.ioloop.add_timeout(
time.time() + settings.get('feedback_reconnect_lag'), self.connect)
def connect_to_server(self, server_cls):
server = client = None
try:
sock, port = bind_unused_port()
server = server_cls(ssl_options=_server_ssl_options())
server.add_socket(sock)
client = SSLIOStream(socket.socket(),
ssl_options=dict(cert_reqs=ssl.CERT_NONE))
yield client.connect(('127.0.0.1', port))
self.assertIsNotNone(client.socket.cipher())
finally:
if server is not None:
server.stop()
if client is not None:
client.close()
def connect_to_server(self, server_cls):
server = client = None
try:
sock, port = bind_unused_port()
server = server_cls(ssl_options=_server_ssl_options())
server.add_socket(sock)
client = SSLIOStream(socket.socket(),
ssl_options=dict(cert_reqs=ssl.CERT_NONE))
yield client.connect(("10.0.0.7", port))
self.assertIsNotNone(client.socket.cipher())
finally:
if server is not None:
server.stop()
if client is not None:
client.close()
class SSLStreamReader(object):
def __init__(self, host, port):
self._address = (host, port)
self._stream = None
self._clients = set()
self._nodes=[]
self._edges=[]
def add(self, client):
self._clients.add(client)
log.debug('Added client %s', client)
if self._stream is None:
self._connect()
self._read_stream()
def remove(self, client):
self._clients.remove(client)
log.debug('Removed client %s', client)
@staticmethod
def process(data):
return data
@staticmethod
def write_message(client, data):
pass
def _connect(self):
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
self._stream = SSLIOStream(s)
log.debug('Connecting to %s', self._address)
self._stream.connect(self._address)
def _disconnect(self):
self._stream.close()
self._stream = None
log.debug('Disconnected from %s', self._address)
@coroutine
def _read_stream(self):
log.info('Reading stream from %s', self._address)
lastVertex=None
while self._clients:
data = yield Task(self._stream.read_until, "\n")
resultObject=None
if '<hwstats' in data:
result=None
if lastVertex==None:
resultObject=None
else:
result=[]
result.append('{"type":"vertex", "event":"update", ')
result.append('"id":')
result.append(lastVertex)
result.append(', "details":{"Temperature":')
Start=data.find('temperature="')+13
End=data.find('" ',Start)
result.append(data[Start:End])
result.append(', "Hostname":')
Start=data.find('hostname="')+9
End=data.find('"',Start+1)+1
result.append(data[Start:End])
result.append(', "CPU":')
Start=data.find('" cpuLoad="')+11
End=data.find('" ',Start)
result.append(data[Start:End])
result.append(', "Free Memory":')
Start=data.find('freeMem="')+9
End=data.find('" ',Start)
result.append(data[Start:End])
result.append('}}')
resultObject=result
lastVertex=None
elif '<vertex' in data or ('<edge' in data and 'type="2"' in data):
resultObject = self.process(data)
if '<vertex' in data:
if 'add' in data:
start=data.find('id="')+4
end=data.find('"',start)
lastVertex=data[start:end]
if '"add", ' in resultObject and '"vertex",' in resultObject:
for i in range(len(self._nodes)):
if (resultObject[5]==self._nodes[i][5]):
self._nodes.pop(i)
self._nodes.append(resultObject)
resultObject[3]='"update", '
elif i==len(self._nodes)-1:
self._nodes.append(resultObject)
if len(self._nodes)==0:
self._nodes.append(resultObject)
elif('"remove", ' in resultObject):
for i in range(len(self._nodes)):
if ((resultObject[5]==self._nodes[i][5])):
self._nodes.pop(i)
break
for i in range(len(self._edges)):
if resultObject[5]==self._edges[i][5] or resultObject[5]==self._edges[i][7]:
self._edges.pop(i)
elif '<edge' in data:
if 'event="add"' in resultObject and '"edge",' in resultObject:
for i in range(len(self._edges)):
if(resultObject[0:8]==self._edges[i][0:8] and resultObject!=self._edges[i] ):
self._edges[i]=resultObject
resultObject[3]='"update",'
break
elif (i==len(self._edges)-1):
self._edges.append(resultObject)
if len(self._edges)==0:
self.edges.append(resultObject)
elif '"remove", ' in resultObject and '"edge", ' in resultObject:
for i in range(len(self._edges)):
if (resultObject[5] == self._edges[i][5] and resultObject[7] == self._edges[i][7]) or (resultObject[5] == self._edges[i][7] and resultObject[7] == self._edges[i][5]):
self._edges.pop(i)
if resultObject!=None:
resultObject=''.join(resultObject)
for client in self._clients:
print('data: '+data+'\nsending: '+resultObject)
self.write_message(client, resultObject)
self._disconnect()
return
class _APNConn(object):
def __init__(self, prev_buffers, cert_file, svr_tuple, encode_func):
"""
:param cert_file: 证书文件路径
:param prev_buffers: 之前残留的推送链表
按照苹果的尿性推送协议
谁也不知道到底有没有成功
F**K JOBS!
"""
self.__io_loop = IOLoop.instance()
self.__cert = path.join(CERT_ROOT, cert_file)
self.__svr_addr = svr_tuple
self.__encode_func = encode_func
self.__stream = None
self.__connected = False
#list自身的索引作为
self.sending_buffer = []
if prev_buffers and isinstance(prev_buffers, (list, tuple)):
self.sending_buffer.extend(prev_buffers)
self.__recv_buf = ''
self.__connection_close_ts = None
#从read读到的错误消息id，便于
self.__recv_err_msgid = None
self.critical = False
def force_close(self):
self.__connected = False
self.__stream.close_fd()
self.__stream = None
self.critical = True
def connect(self):
self.__stream = SSLIOStream(
socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0),
io_loop=self.__io_loop,
ssl_options={
'ssl_version': ssl.PROTOCOL_TLSv1,
# 'ca_certs': path.join(CERT_ROOT, 'entrust_root_ca.pem'),
'certfile': self.__cert,
# 'cert_reqs': ssl.CERT_REQUIRED
})
self.__stream.set_close_callback(self._on_close)
self.__stream.connect(self.__svr_addr, self.__on_connect)
def __on_connect(self):
self.__connected = True
self.__stream.set_nodelay(True)
self.__stream.read_until_close(self._last_closd_recv, self._on_recv)
self.__send_batch()
def __do_check(self, sent_len):
if self.__recv_err_msgid:
logger.debug('CHECK: %r' % self.__recv_err_msgid)
if self.__recv_err_msgid is None:
self.sending_buffer = self.sending_buffer[sent_len:]
self.__send_batch()
return
self.sending_buffer = self.sending_buffer[self.__recv_err_msgid:]
self.__recv_err_msgid = None
self.__stream.close_fd()
self.__stream = None
self.critical = True
def __send_batch(self):
"""
连接断开后就停了
"""
if not self.__connected:
return
if not self.__stream:
return
l = self.sending_buffer[:QUEUE_CAPACITY]
for i, b in enumerate(l):
self.__stream.write(self.__encode_func(i, *b))
logger.debug('>> %d - %s' % (i, b))
self.__io_loop.add_timeout(timedelta(seconds=CHECK_TIMESPAN),
partial(self.__do_check, len(l)))
def append(self, device_token, payload):
"""
:param device_token: app从apns服务器获取的64字节串
:param payload: 报文dict
"""
self.sending_buffer.append((device_token, payload))
def _last_closd_recv(self, buf):
"""
socket关闭时最后几个字节
"""
if not buf:
return
self._on_recv(buf)
def _on_recv(self, buf):
logger.debug('<< %r' % buf)
self.__recv_buf = self.__recv_buf + buf
_ = err_resp(self.__recv_buf)
if _ is None:
return
self.__recv_buf = ''
errno, self.__recv_err_msgid = _
logger.fatal('apns err: %d - %d - %s' %
(self.__recv_err_msgid, errno, resp_errno.get(errno)))
def _on_close(self):
self.critical = True
logger.warn('closed')
self.__connected = False
self.__stream = None
class IrcConnection(object):
def __init__(self, bot, server_name, address, port, nick, user,
realname, owners, channels=None, password=None, ssl=None):
self.bot = bot
self.server_name = server_name
self.owners = owners
self.address = address
self.port = port
self.nick = nick
self.user = user
self.password = password
self.ssl = ssl
self.realname = realname
self.initial_channels = set(channels) if channels else set()
self.logger = getLogger(__name__)
self._protocol_events = dict()
self.channels = dict()
self.users = dict()
self.init_protocol_events()
def init_protocol_events(self):
self._protocol_events = {
'PRIVMSG' : self.on_privmsg,
'PING' : self.on_ping,
'JOIN' : self.on_join,
'401' : self.on_nochan,
'001' : self.on_welcome,
'KICK' : self.on_kick,
'PART' : self.on_part,
'353' : self.on_names,
'NICK' : self.on_nick
}
def connect(self, reconnecting=False):
self.logger.debug('CONNECTING...')
_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
if self.ssl is None:
self._stream = IOStream(_sock)
elif self.ssl is True:
self._stream = SSLIOStream(_sock)
else:
self._stream = SSLIOStream(_sock, ssl_options=self.ssl)
self._stream.connect((self.address, self.port), self._register)
def _register(self):
self.logger.debug('CONNECTED')
if self.password:
# TODO need to check for error responses
self.authenticate()
self.set_nick(self.nick)
self.write_raw('USER %s 0 * :%s' % (self.user, self.realname))
self._next()
def add_channel(self, channel):
self.channels[channel] = []
def remove_channel(self, channel):
del self.channels[channel]
def add_user(self, channel, user):
self.channels[channel].append(user)
self.users[user.nick] = user
def remove_user(self, channel, user):
self.channels[channels].remove(user)
del self.users[user.nick]
def user_change_nick(self, old_nick, new_nick):
user = self.users.pop(old_nick)
user.nick = new_nick
self.users[new_nick] = user
def authenticate(self):
self.logger.debug("Authenticating (%s)", self.password)
self.write_raw('PASS %s' % self.password)
def set_nick(self, nick):
if not nick:
raise IrcError('Cannot set empty nick')
self.logger.debug('SETTING NICK {nick: %s}' % nick)
self.write_raw('NICK %s' % nick)
def join_channel(self, *channels):
if not all([c for c in channels]):
raise IrcError('Empty channel')
self.logger.debug('JOINING CHANNEL(S): {channels: %s}' % repr(channels))
chan_def = ','.join(channels)
self.write_raw('JOIN %s' % chan_def)
def part_channel(self, *channels):
if not all([c for c in channels]):
raise IrcError('Empty channel')
self.logger.debug('PARTING CHANNEL: {channels: %s}' % repr(channels))
chan_def = ','.join(channels)
self.write_raw('PART :%s' % chan_def)
def private_message(self, destination, message):
if not message:
raise IrcError('Cannot send empty message')
if not destination:
raise IrcError('Cannot send to empty destination')
self.logger.debug('SENDING PRIVMSG: {destination: %s, message: %s}' % (destination, message))
self.write_raw('PRIVMSG %s :%s' % (destination, message))
def reply(self, event, message):
if event.destination and event.destination != self.nick:
destination = event.destination
else:
destination = event.nick
self.private_message(destination, message)
def reply_with_nick(self, event, message):
if event.destination:
message = '%s: %s' % (event.nick, message)
self.reply(event, message)
def kick(self, channel, user, comment=None):
if not channel:
raise IrcError('Cannot kick from empty channel')
if not user:
raise IrcError('Cannot kick empty player')
self.logger.debug('KICKING {channel: %s, user: %s}' % (channel, user))
kick_str = 'KICK %s %s' % (channel, user)
if comment:
kick_str += ' :%s' % comment
self.write_raw(kick_str)
def write_raw(self, line):
line.replace(EOL, '')
self.logger.debug('WRITE RAW: {line: %s}' % line)
self._stream.write(line + EOL)
def read_raw(self, line):
self.logger.debug('READ RAW: {line: %s}' % line.replace(EOL, ''))
event = IrcEvent(self.nick, line)
self.handle(event)
self.bot.process_hooks(self, event)
self.bot.process_plugins(self, event)
self._next()
def handle(self, event):
if event.type in self._protocol_events:
self._protocol_events[event.type](event)
def _next(self):
self._stream.read_until(EOL, self.read_raw)
def on_welcome(self, event):
self.logger.debug('RECIEVED RPL_WELCOME')
if self.initial_channels:
self.join_channel(*self.initial_channels)
def on_ping(self, event):
# One ping only, please
self.logger.debug('RECIEVED PING')
self.write_raw("PONG %s\r\n" % event.text)
def on_privmsg(self, event):
# :[email protected] PRIVMSG #xx :hi
self.logger.debug('RECIEVED PRIVMSG {destination: %s,'
' message: %s}' % (event.destination, event.text))
pass
def on_nick(self, event):
old_nick = event.nick
new_nick = event.text
self.logger.debug('RECIEVED NICK {old_nick: %s, new_nick: %s}' %
(old_nick, new_nick))
if event.nick == self.nick:
self.nick = new_nick
else:
self.user_change_nick(old_nick, new_nick)
def on_join(self, event):
channel = event.destination
nick = event.nick
self.logger.debug('RECIEVED JOIN {channel: %s, nick: %s}' % (channel, nick))
if self.nick == nick:
self.add_channel(channel)
else:
self.add_user(channel, IrcUser(nick))
def on_names(self, event):
nick_chan, nicks_raw = event.parameters_raw.split(':')
nicks = nicks_raw.split()
if '@' in nick_chan:
channel = nick_chan.split('@')[-1].strip()
elif '=' in nick_chan:
channel = nick_chan.split('=')[-1].strip()
else:
raise Exception
self.logger.debug('RECIEVED NAMES {channel: %s, nick_count: %d}' % (
channel, len(nicks)))
for nr in nicks:
if nr[0] == '@':
nick = nr[1:]
elif nr[0] == '+':
nick = nr[1:]
else:
nick = nr
user = IrcUser(nick)
self.add_user(channel, user)
def on_nochan(self, event):
channel = event.parameters[0]
self.logger.debug('RECIEVED ERR_NOSUCHCHANNEL {channel: %s}' % channel)
# :senor.crunchybueno.com 401 nodnc #xx :No such nick/channel
self.remove_channel(channel)
def on_part(self, event):
nick = event.nick
channel = event.destination
self.logger.debug('RECIEVED PART {channel: %s, nick: %s}' % (channel,
nick))
if event.nick == self.nick:
self.logger.debug('IOBot parted from %s' % channel)
self.remove_channel(channel)
def on_kick(self, event):
nick = event.parameters[0]
channel = event.destination
self.logger.debug('RECIEVED KICK {channel: %s, nick: %s}' % (channel,
nick))
if event.parameters[0] == self.nick:
self.logger.warning('IOBot was KICKed from %s' % channel)
self.remove_channel(channel)
class APNSConn():
def __init__(self, host=None, certfile=None, loop=None):
self.stats = {
'disconnects':0,
'notifications':0,
'invalid_tokens':0,
}
self.started = time.time()
self.ioloop = loop or ioloop.IOLoop.instance()
self.host = host or settings.get('apns_host')
self.certfile = certfile or settings.get('certfile')
self.write_queue = deque()
self.recent = deque(maxlen=100)
# N.B. Python upgrades ints to longs and this is a binary protocol so...
self.generation = ctypes.c_uint32(0)
self.connect()
def get_stats(self):
stats = self.stats.copy()
stats['queue_len'] = len(self.write_queue)
stats['uptime'] = time.time() - self.started
return stats
def push(self, token, alert=None, badge=None, sound=None, expiry=None, extra=None, timestamp=None):
'''
This is really the only api you want to use. Pushes a notification onto the queue
for non-flagged users. The queue is processed in the tornado event loop.
'''
flagged = mc.get(token)
if flagged:
if not timestamp or timestamp > flagged:
self.stats['invalid_tokens'] += 1
return False
self.generation.value += 1
identifier = self.generation.value
msg = create_message(token, alert=alert, badge=badge, sound=sound,
identifier=identifier, expiry=expiry, extra=extra)
if len(msg) > MAX_PAYLOAD_BYTES:
raise ValueError, u"max payload(%d) exceeded: %d" % (MAX_PAYLOAD_BYTES, len(msg))
self.write_queue.append(msg)
self.recent.append(dict(identifier=identifier, token=token))
self.ioloop.add_callback(self.push_one)
return True
def push_one(self):
if len(self.write_queue) and not self.stream.closed():
msg = self.write_queue.popleft()
try:
self.stream.write(msg)
self.stats['notifications'] += 1
except:
self.write_queue.appendleft(msg)
return False
return True
return False
def connect(self):
self.s = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
self.stream = SSLIOStream(self.s, ssl_options=dict(certfile=self.certfile))
self.stream.connect(self.host, self._on_connect)
self.stream.read_until_close(self._on_close, streaming_callback=self._on_read)
def _on_connect(self):
'''
Process the backlog, hoss.
'''
logging.info('connected to %s:%d' % (self.host[0], self.host[1]))
while self.push_one(): continue
def _on_read(self, data):
'''
The only message we expect here is an error response... sadly, followed by
a disconnect.
'''
logging.info('_on_read: %d bytes' % (len(data)))
try:
status, identifier, err_string = parse_response(data)
logging.warning('_on_read err: %d %d %s' % (status, identifier, err_string))
if status == 8:
for msg in self.recent:
if msg['identifier'] == identifier:
token = msg['token']
logging.info('flagging token: %s' % (token))
self.stats['invalid_tokens'] += 1
mc.set(token, int(time.time()))
except:
logging.info('parse_response failed')
def _on_close(self, data):
'''
This triggers the reconnect given reconnect_lag from settings. This should
probably be an exponential backoff.
'''
logging.info('_on_close')
self.stats['disconnects'] += 1
try:
self.stream.close()
self.s.close()
except:
pass
self.ioloop.add_timeout(time.time()+settings.get('apns_reconnect_lag'), self.connect)
class Client(Protocol):
'''ssl client implementation.'''
def __init__(self, uid, port):
super(Client, self).__init__('client')
self.uid = uid
self.port = port
self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.conn = SSLIOStream(self.sock, ssl_options={'ca_certs':SERVER_CRT_PATH, 'cert_reqs':ssl.CERT_REQUIRED})
self.conn.connect(('127.0.0.1', self.port), self.on_connect)
def on_connect(self):
self.conn.set_close_callback(self.on_close)
self.pubkey, self.privkey = None, None
self.init_keys()
self.write(OP_PUBKEY, self.pubkey)
# this is hardcoded currently in absence of proper tests
if self.uid == 2:
self.send_message(1, 'hello world')
logger.debug('Client %s: Sending message "Hello World" to user with uid 1' % self.uid)
self.read_line()
def read_line(self):
self.conn.read_until(CRLF, self.handle_line)
def on_close(self):
self.conn = None
self.sock = None
def init_keys(self):
if os.path.isfile(CLIENT_PUB_PATH % self.uid) and os.path.isfile(CLIENT_PRIV_PATH % self.uid):
with open(CLIENT_PUB_PATH % self.uid, 'rb') as pubfile, open(CLIENT_PRIV_PATH % self.uid, 'rb') as privfile:
self.pubkey = pubfile.read().strip()
self.privkey = privfile.read().strip()
logger.debug('read existing pub/priv key for uid %s' % self.uid)
else:
self.pubkey, self.privkey = self.generate_keys()
with open(CLIENT_PUB_PATH % self.uid, 'wb') as pubfile, open(CLIENT_PRIV_PATH % self.uid, 'wb') as privfile:
pubfile.write(self.pubkey)
privfile.write(self.privkey)
logger.debug('written pub/priv key for uid %s' % self.uid)
def write(self, *messages):
for message in messages:
self.conn.write('%s%s' % (message, CRLF))
def send_message(self, uid, message):
with open(CLIENT_PUB_PATH % uid, 'rb') as pubfile:
pubkey = pubfile.read()
# encrypt message using receiver public key
enc = self.encrypt_message(message, pubkey)
# sign encrypted message for digital verification
sig = self.generate_signature(enc[0])
message = (OP_MESSAGE, pubkey, enc[0], sig[0],)
self.write(*message)
@staticmethod
def generate_keys():
random_generator = Random.new().read
priv = RSA.generate(1024, random_generator)
pub = priv.publickey()
return (pub.exportKey().strip(), priv.exportKey().strip())
def generate_signature(self, message):
'''sign messaging using our priv key'''
k = RSA.importKey(self.privkey)
h = MD5.new(message).digest()
return k.sign(h, '')
@staticmethod
def verify_signature(pubkey, signature, message):
'''verify signature using signing user public key'''
k = RSA.importKey(pubkey)
h = MD5.new(message).digest()
return k.verify(h, signature)
@staticmethod
def encrypt_message(message, pubkey):
'''encrypt message using receiving user public key'''
k = RSA.importKey(pubkey)
return k.encrypt(message, 32)
def decrypt_message(self, enc):
'''decrypt message using our priv key'''
k = RSA.importKey(self.privkey)
return k.decrypt(enc)
class _BaseSSLService(object):
"""Base SSL connection to Apple's push notification servers. Retry on
disconnect is handled via an exponential backoff.
"""
_MAX_BACKOFF_SECS = 600.0 # 10 minutes
_PUSH_TOKEN_FMT = '%s:%s'
def __init__(self, settings, host_key):
self._settings = settings
self._host = settings[host_key]
self._retries = 0
self._io_loop = IOLoop.current()
self._ResetBackoff()
self._Connect()
def IsValid(self):
return self._stream is not None
def _FormatPushToken(self, token):
return _BaseSSLService._PUSH_TOKEN_FMT % (
self._settings['token-prefix'], token)
def _Connect(self):
try:
ssl_options = {
'certfile': secrets.GetSecretFile(self._settings['certfile'])
}
self._sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
self._stream = SSLIOStream(self._sock,
io_loop=self._io_loop,
ssl_options=ssl_options)
self._stream.set_close_callback(self._OnClose)
self._stream.connect(self._host, self._OnConnect)
except KeyError:
logging.warning(
'failed to initialize connection to APN service at %s:%d '
'whose certificate is missing from secrets/%s' %
(self._host[0], self._host[1], self._settings['certfile']))
return
except:
self._stream = None
raise
def _OnConnect(self):
logging.info("connected to %s:%d" % (self._host[0], self._host[1]))
def _ResetBackoff(self):
"""Resets backoff to 'reconnect_lag' setting."""
self._backoff = self._settings.get('reconnect_lag')
def _OnClose(self):
logging.info("disconnected from %s:%d" %
(self._host[0], self._host[1]))
try:
self._stream.close()
except:
pass
finally:
self._stream = None
timeout = time.time() + self._backoff
self._io_loop.add_timeout(timeout, self._Connect)
self._backoff = min(_BaseSSLService._MAX_BACKOFF_SECS,
self._backoff * 2)
class APNSConn():
def __init__(self, host=None, certfile=None, loop=None):
self.stats = {
'disconnects': 0,
'notifications': 0,
'invalid_tokens': 0,
}
self.started = time.time()
self.ioloop = loop or ioloop.IOLoop.instance()
self.host = host or settings.get('apns_host')
self.certfile = certfile or settings.get('certfile')
self.write_queue = deque()
self.recent = deque(maxlen=100)
# N.B. Python upgrades ints to longs and this is a binary protocol so...
self.generation = ctypes.c_uint32(0)
self.connect()
def get_stats(self):
stats = self.stats.copy()
stats['queue_len'] = len(self.write_queue)
stats['uptime'] = time.time() - self.started
return stats
def push(self,
token,
alert=None,
badge=None,
sound=None,
expiry=None,
extra=None,
timestamp=None):
'''
This is really the only api you want to use. Pushes a notification onto the queue
for non-flagged users. The queue is processed in the tornado event loop.
'''
flagged = mc.get(token)
if flagged:
if not timestamp or timestamp > flagged:
self.stats['invalid_tokens'] += 1
return False
self.generation.value += 1
identifier = self.generation.value
msg = create_message(token,
alert=alert,
badge=badge,
sound=sound,
identifier=identifier,
expiry=expiry,
extra=extra)
if len(msg) > MAX_PAYLOAD_BYTES:
raise ValueError, u"max payload(%d) exceeded: %d" % (
MAX_PAYLOAD_BYTES, len(msg))
self.write_queue.append(msg)
self.recent.append(dict(identifier=identifier, token=token))
self.ioloop.add_callback(self.push_one)
return True
def push_one(self):
if len(self.write_queue) and not self.stream.closed():
msg = self.write_queue.popleft()
try:
self.stream.write(msg)
self.stats['notifications'] += 1
except:
self.write_queue.appendleft(msg)
return False
return True
return False
def connect(self):
self.s = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
self.stream = SSLIOStream(self.s,
ssl_options=dict(certfile=self.certfile))
self.stream.connect(self.host, self._on_connect)
self.stream.read_until_close(self._on_close,
streaming_callback=self._on_read)
def _on_connect(self):
'''
Process the backlog, hoss.
'''
logging.info('connected to %s:%d' % (self.host[0], self.host[1]))
while self.push_one():
continue
def _on_read(self, data):
'''
The only message we expect here is an error response... sadly, followed by
a disconnect.
'''
logging.info('_on_read: %d bytes' % (len(data)))
try:
status, identifier, err_string = parse_response(data)
logging.warning('_on_read err: %d %d %s' %
(status, identifier, err_string))
if status == 8:
for msg in self.recent:
if msg['identifier'] == identifier:
token = msg['token']
logging.info('flagging token: %s' % (token))
self.stats['invalid_tokens'] += 1
mc.set(token, int(time.time()))
except:
logging.info('parse_response failed')
def _on_close(self, data):
'''
This triggers the reconnect given reconnect_lag from settings. This should
probably be an exponential backoff.
'''
logging.info('_on_close')
self.stats['disconnects'] += 1
try:
self.stream.close()
self.s.close()
except:
pass
self.ioloop.add_timeout(
time.time() + settings.get('apns_reconnect_lag'), self.connect)
class H2Client(object):
ALPN_HTTP2_PROTOCOL = b'h2'
USER_AGENT = 'Tornado 4.3 hyper-h2/1.0.0'
def __init__(self, io_loop=None, config=None):
self.io_loop = io_loop or IOLoop.current()
self.conn = H2Connection()
self.known_proto = None
self.authority = None
self.io_stream = None
self.ssl_context = ssl.create_default_context(
purpose=ssl.Purpose.SERVER_AUTH)
self.ssl_context.check_hostname = False
self.ssl_context.verify_mode = ssl.CERT_NONE
self.ssl_context.set_alpn_protocols([self.ALPN_HTTP2_PROTOCOL])
self.are_settings_acked = False
self.pending_requests = []
#self.settings_acked_future = Future()
self.responses = {}
self.pushes = {}
self.data_received_size = 0
self.last_time_data_recvd = None
@gen.coroutine
def connect(self, host, port):
self.authority = host
s = socket.socket()
self.io_stream = SSLIOStream(s, ssl_options=self.ssl_context)
yield self.io_stream.connect((host, port), server_hostname=host)
logger.debug("Connected!")
self.known_proto = self.io_stream.socket.selected_alpn_protocol()
assert self.known_proto == self.ALPN_HTTP2_PROTOCOL, "ALPN protocol was not h2, was {} instead".format(
self.known_proto)
self.io_stream.set_close_callback(self.connection_lost)
logger.debug("Talking to a valid HTTP2 server! Sending preamble")
self.conn.initiate_connection()
self.io_stream.read_until_close(streaming_callback=self.data_received)
data = self.conn.data_to_send()
yield self.io_stream.write(data)
logger.debug("Preamble Sent! Should be connected now")
@gen.coroutine
def close_connection(self):
self.io_stream.set_close_callback(lambda: None)
self.conn.close_connection()
data = self.conn.data_to_send()
yield self.io_stream.write(data)
# @gen.coroutine
def data_received(self, data):
"""
Called by Tornado when data is received on the connection.
We need to check a few things here. Firstly, we want to validate that
we actually negotiated HTTP/2: if we didn't, we shouldn't proceed!
Then, we want to pass the data to the protocol stack and check what
events occurred.
"""
self.data_received_size += len(data)
self.last_time_data_recvd = time.time()
if not self.known_proto:
self.known_proto = self.io_stream.socket.selected_alpn_protocol()
assert self.known_proto == b'h2'
events = self.conn.receive_data(data)
for event in events:
#print("Processing event: {}".format(event))
if isinstance(event, ResponseReceived):
self.handle_response(event)
elif isinstance(event, DataReceived):
self.handle_data(event)
elif isinstance(event, StreamEnded):
#self.end_stream(event)
logger.debug("Got event Stream ended for stream {}".format(
event.stream_id))
elif isinstance(event, SettingsAcknowledged):
self.settings_acked(event)
elif isinstance(event, StreamReset):
logger.debug("A Stream reset!: %d" % event.error_code)
elif isinstance(event, WindowUpdated):
self.window_updated(event)
elif isinstance(event, PushedStreamReceived):
self.handle_push(event)
elif isinstance(event, TrailersReceived):
self.handle_response(event, response_type="trailers")
else:
logger.debug(
"Received an event we don't handle: {}".format(event))
data = self.conn.data_to_send()
if data:
#print("Responding to the server: {}".format(data))
self.io_stream.write(data)
def settings_acked(self, event):
"""
Called when the remote party ACKs our settings. We send a SETTINGS
frame as part of the preamble, so if we want to be very polite we can
wait until the ACK for that frame comes before we start sending our
request.
"""
self.are_settings_acked = True
#self.settings_acked_future.set_result(True)
def _get_stream_response_holder(self, stream_id):
try:
return self.responses[stream_id]
except KeyError:
return self.pushes[stream_id]
def handle_response(self, event, response_type="headers"):
"""
Handle the response by storing the response headers.
"""
try:
response = self._get_stream_response_holder(event.stream_id)
except KeyError:
logger.exception(
"Unable to find a response future for stream {} while handling a response"
.format(event.stream_id))
else:
response[response_type] = event.headers
if event.stream_ended is not None:
try:
future = response.pop("future")
except KeyError:
logger.exception(
"No future associated with the response for stream {} while handling a response"
.format(event.stream_id))
else:
future.set_result(response)
try:
self.end_stream(event.stream_ended)
except H2StreamClosedError as e:
logger.exception(
"Got an exception trying to end a stream after handling a response for stream: {}"
.format(e.stream_id))
def handle_data(self, event):
"""
We handle data that's received
"""
self.conn.acknowledge_received_data(event.flow_controlled_length,
event.stream_id)
try:
response = self._get_stream_response_holder(event.stream_id)
except KeyError:
logger.debug(
"Unable to find a response future for stream {} while handling a data. Adding one now"
.format(event.stream_id))
# response = {'future': Future()}
# self.responses[event.stream_id] = response
else:
if "data" not in response:
response["data"] = event.data
else:
response["data"] = response["data"] + event.data
if event.stream_ended is not None:
try:
future = response.pop("future")
except KeyError:
logger.debug(
"No future associated with the response for stream {} while handling data"
.format(event.stream_id))
else:
future.set_result(response)
try:
self.end_stream(event.stream_ended)
except H2StreamClosedError as e:
logger.exception(
"Got an exception trying to end a stream after handling a response for stream: {}"
.format(e.stream_id))
def handle_push(self, event):
self.pushes[event.pushed_stream_id] = {
"parent_stream_id": event.parent_stream_id,
"request_headers": event.headers
}
# @gen.coroutine
def end_stream(self, event):
"""
We call this when the stream is cleanly ended by the remote peer. That
means that the response is complete.
"""
self.conn.end_stream(event.stream_id)
yield self.io_stream.write(self.conn.data_to_send())
logger.debug("Closed Stream {}".format(event.stream_id))
def window_updated(self, event):
"""
I don't think I actually need to do anything with this in Tornado, since sending data uses futures to continue
"""
pass
def connection_lost(self):
"""
Called by Twisted when the connection is gone. Regardless of whether
it was clean or not, we want to stop the reactor.
"""
logger.debug("Connection was lost!")
def get_request(self, path):
return self.send_bodyless_request(path, "GET")
fetch = get_request
def head_request(self, path):
return self.send_bodyless_request(path, "HEAD")
@gen.coroutine
def send_bodyless_request(self, path, method):
# if not self.settings_acked_future.done():
# print("Settings haven't been acked, yield until they are")
# yield self.settings_acked_future
# print("Settings acked! Let's send this pending request")
request_headers = [
(':method', method),
(':authority', self.authority),
(':scheme', 'https'),
(':path', path),
('user-agent', self.USER_AGENT),
]
stream_id = self.conn.get_next_available_stream_id()
logger.debug("Generating HEADER frame to send for stream_id {}".format(
stream_id))
self.conn.send_headers(stream_id, request_headers, end_stream=True)
response_future = Future()
self.responses[stream_id] = {'future': response_future}
logger.debug("Writing out to the stream")
yield self.io_stream.write(self.conn.data_to_send())
logger.debug("Request sent! Waiting for response")
result = yield response_future
logger.debug("Got a result")
raise gen.Return(result)
def post_file_request(self, path, file_path):
# First, we need to work out how large the file is.
file_size = os.stat(file_path).st_size
# Next, we want to guess a content-type and content-encoding.
content_type, content_encoding = mimetypes.guess_type(file_path)
# We can now open the file.
file_obj = open(file_path, 'rb')
self.post_request(path=path,
file_obj=file_obj,
file_size=file_size,
content_type=content_type,
content_encoding=content_encoding)
def post_request(self,
path,
file_obj,
file_size=None,
content_type=None,
content_encoding=None):
"""
Send the POST request.
A POST request is made up of one headers frame, and then 0+ data
frames. This method begins by sending the headers, and then starts a
series of calls to send data.
"""
# if not self.settings_acked_future.done():
# print("Settings haven't been acked, yield until they are")
# yield self.settings_acked_future
# print("Settings acked! Let's send this pending post request")
if type(file_obj) is str:
file_size = len(file_obj)
file_obj = StringIO(file_obj)
# Now we can build a header block.
request_headers = [
(':method', 'POST'),
(':authority', self.authority),
(':scheme', 'https'),
(':path', path),
('user-agent', self.USER_AGENT),
('content-length', str(file_size)),
]
if content_type is not None:
request_headers.append(('content-type', content_type))
if content_encoding is not None:
request_headers.append(('content-encoding', content_encoding))
stream_id = self.conn.get_next_available_stream_id()
self.conn.send_headers(stream_id, request_headers)
# We now need to send all the relevant data. We do this by checking
# what the acceptable amount of data is to send, and sending it. If we
# find ourselves blocked behind flow control, we then place a deferred
# and wait until that deferred fires.
response_future = Future()
self.responses[stream_id] = {'future': response_future}
# We now need to send a number of data frames.
try:
while file_size > 0:
# Firstly, check what the flow control window is for the current stream.
window_size = self.conn.local_flow_control_window(
stream_id=stream_id)
# Next, check what the maximum frame size is.
max_frame_size = self.conn.max_outbound_frame_size
# We will send no more than the window size or the remaining file size
# of data in this call, whichever is smaller.
bytes_to_send = min(window_size, file_size)
while bytes_to_send > 0:
chunk_size = min(bytes_to_send, max_frame_size)
data_chunk = file_obj.read(chunk_size)
self.conn.send_data(stream_id=stream_id, data=data_chunk)
yield self.io_stream.write(self.conn.data_to_send())
bytes_to_send -= chunk_size
file_size -= chunk_size
except StreamClosedError:
logger.warning(
"Connection was lost while sending stream {}".format(
stream_id))
else:
self.conn.end_stream(stream_id=stream_id)
finally:
file_obj.close()
result = yield response_future
raise gen.Return(result)
@gen.coroutine
def update_settings(self, new_settings):
self.conn.update_settings(new_settings=new_settings)
data = self.conn.data_to_send()
if data:
self.io_stream.write(data)
class Client(Protocol):
'''ssl client implementation.'''
def __init__(self, uid, port):
super(Client, self).__init__('client')
self.uid = uid
self.port = port
self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.conn = SSLIOStream(self.sock,
ssl_options={
'ca_certs': SERVER_CRT_PATH,
'cert_reqs': ssl.CERT_REQUIRED
})
self.conn.connect(('127.0.0.1', self.port), self.on_connect)
def on_connect(self):
self.conn.set_close_callback(self.on_close)
self.pubkey, self.privkey = None, None
self.init_keys()
self.write(OP_PUBKEY, self.pubkey)
# this is hardcoded currently in absence of proper tests
if self.uid == 2:
self.send_message(1, 'hello world')
logger.debug(
'Client %s: Sending message "Hello World" to user with uid 1' %
self.uid)
self.read_line()
def read_line(self):
self.conn.read_until(CRLF, self.handle_line)
def on_close(self):
self.conn = None
self.sock = None
def init_keys(self):
if os.path.isfile(CLIENT_PUB_PATH % self.uid) and os.path.isfile(
CLIENT_PRIV_PATH % self.uid):
with open(CLIENT_PUB_PATH % self.uid,
'rb') as pubfile, open(CLIENT_PRIV_PATH % self.uid,
'rb') as privfile:
self.pubkey = pubfile.read().strip()
self.privkey = privfile.read().strip()
logger.debug('read existing pub/priv key for uid %s' %
self.uid)
else:
self.pubkey, self.privkey = self.generate_keys()
with open(CLIENT_PUB_PATH % self.uid,
'wb') as pubfile, open(CLIENT_PRIV_PATH % self.uid,
'wb') as privfile:
pubfile.write(self.pubkey)
privfile.write(self.privkey)
logger.debug('written pub/priv key for uid %s' % self.uid)
def write(self, *messages):
for message in messages:
self.conn.write('%s%s' % (message, CRLF))
def send_message(self, uid, message):
with open(CLIENT_PUB_PATH % uid, 'rb') as pubfile:
pubkey = pubfile.read()
# encrypt message using receiver public key
enc = self.encrypt_message(message, pubkey)
# sign encrypted message for digital verification
sig = self.generate_signature(enc[0])
message = (
OP_MESSAGE,
pubkey,
enc[0],
sig[0],
)
self.write(*message)
@staticmethod
def generate_keys():
random_generator = Random.new().read
priv = RSA.generate(1024, random_generator)
pub = priv.publickey()
return (pub.exportKey().strip(), priv.exportKey().strip())
def generate_signature(self, message):
'''sign messaging using our priv key'''
k = RSA.importKey(self.privkey)
h = MD5.new(message).digest()
return k.sign(h, '')
@staticmethod
def verify_signature(pubkey, signature, message):
'''verify signature using signing user public key'''
k = RSA.importKey(pubkey)
h = MD5.new(message).digest()
return k.verify(h, signature)
@staticmethod
def encrypt_message(message, pubkey):
'''encrypt message using receiving user public key'''
k = RSA.importKey(pubkey)
return k.encrypt(message, 32)
def decrypt_message(self, enc):
'''decrypt message using our priv key'''
k = RSA.importKey(self.privkey)
return k.decrypt(enc)
