class ThreadedUrllib2TestMixin(object):
def setUp(self):
self._semaphore = DeferredSemaphore(2)
def tearDown(self):
pass
def getPages(self, count, url):
return gatherResults([self.getPage(url) for i in xrange(0, count)])
@inlineCallbacks
def getPage(self, url):
yield self._semaphore.acquire()
page = yield deferToThread(self._openPage, url)
self._semaphore.release()
returnValue(page)
def _openPage(self, url):
log.msg("Opening url: %r" % url)
return urlopen(url).read()
@inlineCallbacks
def getPageLength(self, url):
response = yield self.getPage(url)
returnValue(len(response))
class PlotlyStreamProducer(object):
"""Implements a producer that copies from a buffer to a plot.ly
connection.
"""
implements(IBodyProducer)
length = UNKNOWN_LENGTH
def __init__(self, buffer, start_callback=None):
self.buffer = buffer
self._done = False
self._flush = DeferredSemaphore(1)
self._waiter = DeferredSemaphore(1)
self._flush.acquire()
self._started = start_callback
self._keepalive = LoopingCall(self._send_keepalive)
@inlineCallbacks
def startProducing(self, consumer):
self._keepalive.start(60)
self._started.callback(None)
while True:
# if paused, this will block
yield self._waiter.acquire()
while len(self.buffer):
v = self.buffer.pop(0)
if v is not None:
consumer.write(json.dumps(v))
consumer.write("\n")
yield self._waiter.release()
if self._done:
return
yield self._flush.acquire()
def pauseProducing(self):
return self._waiter.acquire()
def resumeProducing(self):
return self._waiter.release()
def stopProducing(self):
self._done = True
if self._keepalive.running:
self._keepalive.stop()
def _send_keepalive(self):
self.buffer.append(None)
self.flush()
def flush(self):
if self._flush.tokens == 0:
self._flush.release()
class TwistedWebTestMixin(object):
def setUp(self):
self._semaphore = DeferredSemaphore(2)
def tearDown(self):
pass
@inlineCallbacks
def getPages(self, count, url):
return gatherResults([self.getPage(url) for i in xrange(0, count)])
@inlineCallbacks
def getPage(self, url):
yield self._semaphore.acquire()
page = yield tx_getPage(url)
self._semaphore.release()
returnValue(page)
@inlineCallbacks
def getPageLength(self, url):
response = yield self.getPage(url)
returnValue(len(response))
class GcmPushkin(ConcurrencyLimitedPushkin):
"""
Pushkin that relays notifications to Google/Firebase Cloud Messaging.
"""
UNDERSTOOD_CONFIG_FIELDS = {
"type",
"api_key",
"fcm_options",
"max_connections",
} | ConcurrencyLimitedPushkin.UNDERSTOOD_CONFIG_FIELDS
def __init__(self, name: str, sygnal: "Sygnal", config: Dict[str,
Any]) -> None:
super().__init__(name, sygnal, config)
nonunderstood = set(self.cfg.keys()).difference(
self.UNDERSTOOD_CONFIG_FIELDS)
if len(nonunderstood) > 0:
logger.warning(
"The following configuration fields are not understood: %s",
nonunderstood,
)
self.http_pool = HTTPConnectionPool(reactor=sygnal.reactor)
self.max_connections = self.get_config("max_connections", int,
DEFAULT_MAX_CONNECTIONS)
self.connection_semaphore = DeferredSemaphore(self.max_connections)
self.http_pool.maxPersistentPerHost = self.max_connections
tls_client_options_factory = ClientTLSOptionsFactory()
# use the Sygnal global proxy configuration
proxy_url = sygnal.config.get("proxy")
self.http_agent = ProxyAgent(
reactor=sygnal.reactor,
pool=self.http_pool,
contextFactory=tls_client_options_factory,
proxy_url_str=proxy_url,
)
self.api_key = self.get_config("api_key", str)
if not self.api_key:
raise PushkinSetupException("No API key set in config")
# Use the fcm_options config dictionary as a foundation for the body;
# this lets the Sygnal admin choose custom FCM options
# (e.g. content_available).
self.base_request_body = self.get_config("fcm_options", dict, {})
if not isinstance(self.base_request_body, dict):
raise PushkinSetupException(
"Config field fcm_options, if set, must be a dictionary of options"
)
@classmethod
async def create(cls, name: str, sygnal: "Sygnal",
config: Dict[str, Any]) -> "GcmPushkin":
"""
Override this if your pushkin needs to call async code in order to
be constructed. Otherwise, it defaults to just invoking the Python-standard
__init__ constructor.
Returns:
an instance of this Pushkin
"""
return cls(name, sygnal, config)
async def _perform_http_request(
self, body: Dict,
headers: Dict[AnyStr, List[AnyStr]]) -> Tuple[IResponse, str]:
"""
Perform an HTTP request to the FCM server with the body and headers
specified.
Args:
body: Body. Will be JSON-encoded.
headers: HTTP Headers.
Returns:
"""
body_producer = FileBodyProducer(BytesIO(json.dumps(body).encode()))
# we use the semaphore to actually limit the number of concurrent
# requests, since the HTTPConnectionPool will actually just lead to more
# requests being created but not pooled – it does not perform limiting.
with QUEUE_TIME_HISTOGRAM.time():
with PENDING_REQUESTS_GAUGE.track_inprogress():
await self.connection_semaphore.acquire()
try:
with SEND_TIME_HISTOGRAM.time():
with ACTIVE_REQUESTS_GAUGE.track_inprogress():
response = await self.http_agent.request(
b"POST",
GCM_URL,
headers=Headers(headers),
bodyProducer=body_producer,
)
response_text = (await readBody(response)).decode()
except Exception as exception:
raise TemporaryNotificationDispatchException(
"GCM request failure") from exception
finally:
self.connection_semaphore.release()
return response, response_text
async def _request_dispatch(
self,
n: Notification,
log: NotificationLoggerAdapter,
body: dict,
headers: Dict[AnyStr, List[AnyStr]],
pushkeys: List[str],
span: Span,
) -> Tuple[List[str], List[str]]:
poke_start_time = time.time()
failed = []
response, response_text = await self._perform_http_request(
body, headers)
RESPONSE_STATUS_CODES_COUNTER.labels(pushkin=self.name,
code=response.code).inc()
log.debug("GCM request took %f seconds", time.time() - poke_start_time)
span.set_tag(tags.HTTP_STATUS_CODE, response.code)
if 500 <= response.code < 600:
log.debug("%d from server, waiting to try again", response.code)
retry_after = None
for header_value in response.headers.getRawHeaders(b"retry-after",
default=[]):
retry_after = int(header_value)
span.log_kv({
"event": "gcm_retry_after",
"retry_after": retry_after
})
raise TemporaryNotificationDispatchException(
"GCM server error, hopefully temporary.",
custom_retry_delay=retry_after)
elif response.code == 400:
log.error(
"%d from server, we have sent something invalid! Error: %r",
response.code,
response_text,
)
# permanent failure: give up
raise NotificationDispatchException("Invalid request")
elif response.code == 401:
log.error("401 from server! Our API key is invalid? Error: %r",
response_text)
# permanent failure: give up
raise NotificationDispatchException("Not authorised to push")
elif response.code == 404:
# assume they're all failed
log.info("Reg IDs %r get 404 response; assuming unregistered",
pushkeys)
return pushkeys, []
elif 200 <= response.code < 300:
try:
resp_object = json_decoder.decode(response_text)
except ValueError:
raise NotificationDispatchException(
"Invalid JSON response from GCM.")
if "results" not in resp_object:
log.error(
"%d from server but response contained no 'results' key: %r",
response.code,
response_text,
)
if len(resp_object["results"]) < len(pushkeys):
log.error(
"Sent %d notifications but only got %d responses!",
len(n.devices),
len(resp_object["results"]),
)
span.log_kv({
logs.EVENT: "gcm_response_mismatch",
"num_devices": len(n.devices),
"num_results": len(resp_object["results"]),
})
# determine which pushkeys to retry or forget about
new_pushkeys = []
for i, result in enumerate(resp_object["results"]):
if "error" in result:
log.warning("Error for pushkey %s: %s", pushkeys[i],
result["error"])
span.set_tag("gcm_error", result["error"])
if result["error"] in BAD_PUSHKEY_FAILURE_CODES:
log.info(
"Reg ID %r has permanently failed with code %r: "
"rejecting upstream",
pushkeys[i],
result["error"],
)
failed.append(pushkeys[i])
elif result["error"] in BAD_MESSAGE_FAILURE_CODES:
log.info(
"Message for reg ID %r has permanently failed with code %r",
pushkeys[i],
result["error"],
)
else:
log.info(
"Reg ID %r has temporarily failed with code %r",
pushkeys[i],
result["error"],
)
new_pushkeys.append(pushkeys[i])
return failed, new_pushkeys
else:
raise NotificationDispatchException(
f"Unknown GCM response code {response.code}")
async def _dispatch_notification_unlimited(
self, n: Notification, device: Device,
context: NotificationContext) -> List[str]:
log = NotificationLoggerAdapter(logger,
{"request_id": context.request_id})
# `_dispatch_notification_unlimited` gets called once for each device in the
# `Notification` with a matching app ID. We do something a little dirty and
# perform all of our dispatches the first time we get called for a
# `Notification` and do nothing for the rest of the times we get called.
pushkeys = [
device.pushkey for device in n.devices
if self.handles_appid(device.app_id)
]
# `pushkeys` ought to never be empty here. At the very least it should contain
# `device`'s pushkey.
if pushkeys[0] != device.pushkey:
# We've already been asked to dispatch for this `Notification` and have
# previously sent out the notification to all devices.
return []
# The pushkey is kind of secret because you can use it to send push
# to someone.
# span_tags = {"pushkeys": pushkeys}
span_tags = {"gcm_num_devices": len(pushkeys)}
with self.sygnal.tracer.start_span(
"gcm_dispatch", tags=span_tags,
child_of=context.opentracing_span) as span_parent:
# TODO: Implement collapse_key to queue only one message per room.
failed: List[str] = []
data = GcmPushkin._build_data(n, device)
# Reject pushkey if default_payload is misconfigured
if data is None:
failed.append(device.pushkey)
headers = {
"User-Agent": ["sygnal"],
"Content-Type": ["application/json"],
"Authorization": ["key=%s" % (self.api_key, )],
}
body = self.base_request_body.copy()
body["data"] = data
body["priority"] = "normal" if n.prio == "low" else "high"
for retry_number in range(0, MAX_TRIES):
if len(pushkeys) == 1:
body["to"] = pushkeys[0]
else:
body["registration_ids"] = pushkeys
log.info("Sending (attempt %i) => %r", retry_number, pushkeys)
try:
span_tags = {"retry_num": retry_number}
with self.sygnal.tracer.start_span(
"gcm_dispatch_try",
tags=span_tags,
child_of=span_parent) as span:
new_failed, new_pushkeys = await self._request_dispatch(
n, log, body, headers, pushkeys, span)
pushkeys = new_pushkeys
failed += new_failed
if len(pushkeys) == 0:
break
except TemporaryNotificationDispatchException as exc:
retry_delay = RETRY_DELAY_BASE * (2**retry_number)
if exc.custom_retry_delay is not None:
retry_delay = exc.custom_retry_delay
log.warning(
"Temporary failure, will retry in %d seconds",
retry_delay,
exc_info=True,
)
span_parent.log_kv({
"event": "temporary_fail",
"retrying_in": retry_delay
})
await twisted_sleep(retry_delay,
twisted_reactor=self.sygnal.reactor)
if len(pushkeys) > 0:
log.info("Gave up retrying reg IDs: %r", pushkeys)
# Count the number of failed devices.
span_parent.set_tag("gcm_num_failed", len(failed))
return failed
@staticmethod
def _build_data(n: Notification,
device: Device) -> Optional[Dict[str, Any]]:
"""
Build the payload data to be sent.
Args:
n: Notification to build the payload for.
device: Device information to which the constructed payload
will be sent.
Returns:
JSON-compatible dict or None if the default_payload is misconfigured
"""
data = {}
if device.data:
default_payload = device.data.get("default_payload", {})
if isinstance(default_payload, dict):
data.update(default_payload)
else:
logger.error(
"default_payload was misconfigured, this value must be a dict."
)
return None
for attr in [
"event_id",
"type",
"sender",
"room_name",
"room_alias",
"membership",
"sender_display_name",
"content",
"room_id",
]:
if hasattr(n, attr):
data[attr] = getattr(n, attr)
# Truncate fields to a sensible maximum length. If the whole
# body is too long, GCM will reject it.
if data[attr] is not None and len(
data[attr]) > MAX_BYTES_PER_FIELD:
data[attr] = data[attr][0:MAX_BYTES_PER_FIELD]
data["prio"] = "high"
if n.prio == "low":
data["prio"] = "normal"
if getattr(n, "counts", None):
data["unread"] = n.counts.unread
data["missed_calls"] = n.counts.missed_calls
return data
class WebpushPushkin(ConcurrencyLimitedPushkin):
"""
Pushkin that relays notifications to Google/Firebase Cloud Messaging.
"""
UNDERSTOOD_CONFIG_FIELDS = {
"type",
"max_connections",
"vapid_private_key",
"vapid_contact_email",
"allowed_endpoints",
"ttl",
} | ConcurrencyLimitedPushkin.UNDERSTOOD_CONFIG_FIELDS
def __init__(self, name, sygnal, config):
super(WebpushPushkin, self).__init__(name, sygnal, config)
nonunderstood = self.cfg.keys() - self.UNDERSTOOD_CONFIG_FIELDS
if nonunderstood:
logger.warning(
"The following configuration fields are not understood: %s",
nonunderstood,
)
self.http_pool = HTTPConnectionPool(reactor=sygnal.reactor)
self.max_connections = self.get_config("max_connections",
DEFAULT_MAX_CONNECTIONS)
self.connection_semaphore = DeferredSemaphore(self.max_connections)
self.http_pool.maxPersistentPerHost = self.max_connections
tls_client_options_factory = ClientTLSOptionsFactory()
# use the Sygnal global proxy configuration
proxy_url = sygnal.config.get("proxy")
self.http_agent = ProxyAgent(
reactor=sygnal.reactor,
pool=self.http_pool,
contextFactory=tls_client_options_factory,
proxy_url_str=proxy_url,
)
self.http_agent_wrapper = HttpAgentWrapper(self.http_agent)
self.allowed_endpoints = None # type: Optional[List[Pattern]]
allowed_endpoints = self.get_config("allowed_endpoints")
if allowed_endpoints:
if not isinstance(allowed_endpoints, list):
raise PushkinSetupException(
"'allowed_endpoints' should be a list or not set")
self.allowed_endpoints = list(map(glob_to_regex,
allowed_endpoints))
privkey_filename = self.get_config("vapid_private_key")
if not privkey_filename:
raise PushkinSetupException(
"'vapid_private_key' not set in config")
if not os.path.exists(privkey_filename):
raise PushkinSetupException(
"path in 'vapid_private_key' does not exist")
try:
self.vapid_private_key = Vapid.from_file(
private_key_file=privkey_filename)
except VapidException as e:
raise PushkinSetupException(
"invalid 'vapid_private_key' file") from e
self.vapid_contact_email = self.get_config("vapid_contact_email")
if not self.vapid_contact_email:
raise PushkinSetupException(
"'vapid_contact_email' not set in config")
self.ttl = self.get_config("ttl", DEFAULT_TTL)
if not isinstance(self.ttl, int):
raise PushkinSetupException("'ttl' must be an int if set")
async def _dispatch_notification_unlimited(self, n, device, context):
p256dh = device.pushkey
if not isinstance(device.data, dict):
logger.warn("Rejecting pushkey %s; device.data is not a dict",
device.pushkey)
return [device.pushkey]
endpoint = device.data.get("endpoint")
auth = device.data.get("auth")
endpoint_domain = urlparse(endpoint).netloc
if self.allowed_endpoints:
allowed = any(
regex.fullmatch(endpoint_domain)
for regex in self.allowed_endpoints)
if not allowed:
logger.error(
"push gateway %s is not in allowed_endpoints, blocking request",
endpoint_domain,
)
# abort, but don't reject push key
return []
if not p256dh or not endpoint or not auth:
logger.warn(
"Rejecting pushkey; subscription info incomplete " +
"(p256dh: %s, endpoint: %s, auth: %s)",
p256dh,
endpoint,
auth,
)
return [device.pushkey]
subscription_info = {
"endpoint": endpoint,
"keys": {
"p256dh": p256dh,
"auth": auth
},
}
payload = WebpushPushkin._build_payload(n, device)
data = json.dumps(payload)
# note that webpush modifies vapid_claims, so make sure it's only used once
vapid_claims = {
"sub": "mailto:{}".format(self.vapid_contact_email),
}
# we use the semaphore to actually limit the number of concurrent
# requests, since the HTTPConnectionPool will actually just lead to more
# requests being created but not pooled – it does not perform limiting.
with QUEUE_TIME_HISTOGRAM.time():
with PENDING_REQUESTS_GAUGE.track_inprogress():
await self.connection_semaphore.acquire()
try:
with SEND_TIME_HISTOGRAM.time():
with ACTIVE_REQUESTS_GAUGE.track_inprogress():
response_wrapper = webpush(
subscription_info=subscription_info,
data=data,
ttl=self.ttl,
vapid_private_key=self.vapid_private_key,
vapid_claims=vapid_claims,
requests_session=self.http_agent_wrapper,
)
response = await response_wrapper.deferred
response_text = (await readBody(response)).decode()
finally:
self.connection_semaphore.release()
# assume 4xx is permanent and 5xx is temporary
if 400 <= response.code < 500:
logger.warn(
"Rejecting pushkey %s; gateway %s failed with %d: %s",
device.pushkey,
endpoint_domain,
response.code,
response_text,
)
return [device.pushkey]
return []
@staticmethod
def _build_payload(n, device):
"""
Build the payload data to be sent.
Args:
n: Notification to build the payload for.
device (Device): Device information to which the constructed payload
will be sent.
Returns:
JSON-compatible dict
"""
payload = {}
default_payload = device.data.get("default_payload")
if isinstance(default_payload, dict):
payload.update(default_payload)
for attr in [
"room_id",
"room_name",
"room_alias",
"membership",
"event_id",
"sender",
"sender_display_name",
"user_is_target",
"type",
"content",
]:
value = getattr(n, attr, None)
if value:
payload[attr] = value
counts = getattr(n, "counts", None)
if counts is not None:
for attr in ["unread", "missed_calls"]:
count_value = getattr(counts, attr, None)
if count_value is not None:
payload[attr] = count_value
return payload
class IndxConnectionPool:
""" A wrapper for txpostgres connection pools, which auto-reconnects. """
def __init__(self):
logging.debug("IndxConnectionPool starting. ")
self.connections = {} # by connection string
self.conn_strs = {} # by db_name
self.semaphore = DeferredSemaphore(1)
self.subscribers = {} # by db name
def removeAll(self, db_name):
""" Remove all connections for a named database - used before deleting that database. """
logging.debug("IndxConnectionPool removeAll {0}".format(db_name))
d_list = []
if db_name in self.conn_strs:
for conn_str in self.conn_strs[db_name]:
for conn in self.connections[conn_str].getInuse():
d_list.append(conn.close())
for conn in self.connections[conn_str].getFree():
d_list.append(conn.close())
del self.connections[conn_str]
del self.conn_strs[db_name]
dl = DeferredList(d_list)
return dl
def connect(self, db_name, db_user, db_pass, db_host, db_port):
""" Returns an IndxConnection (Actual connection and pool made when query is made). """
return_d = Deferred()
log_conn_str = "dbname='{0}' user='******' password='******' host='{3}' port='{4}' application_name='{5}'".format(
db_name, db_user, "XXXX", db_host, db_port,
indx_pg2.APPLICATION_NAME)
conn_str = "dbname='{0}' user='******' password='******' host='{3}' port='{4}' application_name='{5}'".format(
db_name, db_user, db_pass, db_host, db_port,
indx_pg2.APPLICATION_NAME)
logging.debug("IndxConnectionPool connect: {0}".format(log_conn_str))
if db_name not in self.conn_strs:
self.conn_strs[db_name] = []
self.conn_strs[db_name].append(conn_str)
def free_cb(conn):
""" Called back when this IndxConnection has finished querying, so
we put the real connection back into the pool. """
logging.debug("IndxConnectionPool free_cb, conn: {0}".format(conn))
self.connections[conn_str].freeConnection(
conn) # no dealing with callbacks, just carry on
def alloc_cb(conn_str):
# a query was called - allocate a connection now and pass it back
return self._connect(conn_str)
indx_connection = IndxConnection(conn_str, alloc_cb, free_cb)
return_d.callback(indx_connection)
return return_d
def _connect(self, conn_str):
""" Connect and return a free Connection.
Figures out whether to make new connections, use the pool, or wait in a queue.
"""
logging.debug("IndxConnectionPool _connect ({0})".format(conn_str))
return_d = Deferred()
def err_cb(failure):
logging.error(
"IndxConnectionPool _connect err_cb: {0}".format(failure))
self.semaphore.release()
return_d.errback(failure)
def succeed_cb(empty):
logging.debug("IndxConnectionPool _connect succeed_cb")
# TODO pass a Connection back
if len(self.connections[conn_str].getFree()) > 0:
# free connection, use it straight away
conn = self.connections[conn_str].getFree().pop()
self.connections[conn_str].getInuse().append(conn)
self.semaphore.release()
return_d.callback(conn)
return
if len(self.connections[conn_str].getInuse()) < MAX_CONNS:
# not at max connections for this conn_str
# create a new one
d = self._newConnection(conn_str)
def connected_cb(indx_conn):
logging.debug(
"IndxConnectionPool _connect connected_cb ({0})".
format(indx_conn))
self.connections[conn_str].getFree().remove(indx_conn)
self.connections[conn_str].getInuse().append(indx_conn)
self.semaphore.release()
return_d.callback(indx_conn)
return
d.addCallbacks(connected_cb, err_cb)
return
# wait for a connection
def wait_cb(conn):
logging.debug(
"IndxConnectionPool _connect wait_cb ({0})".format(conn))
# already put in 'inuse'
return_d.callback(conn)
return
self.semaphore.release()
self.connections[conn_str].getWaiting().append(wait_cb)
return
def locked_cb(empty):
logging.debug("IndxConnectionPool _connect locked_cb")
if conn_str not in self.connections:
self._newConnections(conn_str).addCallbacks(succeed_cb, err_cb)
else:
threads.deferToThread(succeed_cb, None)
# succeed_cb(None)
self.semaphore.acquire().addCallbacks(locked_cb, err_cb)
return return_d
def _closeOldConnection(self):
""" Close the oldest connection, so we can open a new one up. """
# is already in a semaphore lock, from _newConnection
logging.debug("IndxConnectionPool _closeOldConnection")
### we could force quite them through postgresql like this - but instead we kill them from inside
#query = "SELECT * FROM pg_stat_activity WHERE state = 'idle' AND application_name = %s AND query != 'LISTEN wb_new_version' ORDER BY state_change LIMIT 1;"
#params = [indx_pg2.APPLICATION_NAME]
return_d = Deferred()
def err_cb(failure):
return_d.errback(failure)
ages = {}
for conn_str, dbpool in self.connections.items():
lastused = dbpool.getTime()
if lastused not in ages:
ages[lastused] = []
ages[lastused].append(dbpool)
times = ages.keys()
times.sort()
pool_queue = []
for timekey in times:
pools = ages[timekey]
pool_queue.extend(pools)
def removed_cb(count):
if count < REMOVE_AT_ONCE and len(pool_queue) > 0:
pool = pool_queue.pop(0)
pool.getFree()
pool.removeAll(count).addCallbacks(removed_cb, err_cb)
else:
return_d.callback(None)
removed_cb(0)
return return_d
def _newConnection(self, conn_str):
""" Makes a new connection to the DB
and then puts it in the 'free' pool of this conn_str.
"""
logging.debug("IndxConnectionPool _newConnection")
# lock with the semaphore before calling this
return_d = Deferred()
def close_old_cb(failure):
failure.trap(psycopg2.OperationalError, Exception)
# couldn't connect, so close an old connection first
logging.error(
"IndxConnectionPool error close_old_cb: {0} - state of conns is: {1}"
.format(failure.value, self.connections))
logging.error("IndxConnectionPool connections: {0}".format(
"\n".join(
map(lambda name: self.connections[name].__str__(),
self.connections))))
def closed_cb(empty):
# closed, so try connecting again
self._newConnection(conn_str).addCallbacks(
return_d.callback, return_d.errback)
closed_d = self._closeOldConnection()
closed_d.addCallbacks(closed_cb, return_d.errback)
try:
# try to connect
def connected_cb(connection):
logging.debug(
"IndxConnectionPool _newConnection connected_cb, connection: {0}"
.format(connection))
self.connections[conn_str].getFree().append(connection)
return_d.callback(connection)
conn = txpostgres.Connection()
connection_d = conn.connect(conn_str)
connection_d.addCallbacks(connected_cb, close_old_cb)
except Exception as e:
# close an old connection first
logging.debug(
"IndxConnectionPool Exception, going to call close_old_cb: ({0})"
.format(e))
close_old_cb(Failure(e))
return return_d
def _newConnections(self, conn_str):
""" Make a pool of new connections. """
# lock with the semaphore before calling this
logging.debug("IndxConnectionPool _newConnections")
return_d = Deferred()
self.connections[conn_str] = DBConnectionPool(conn_str)
try:
d_list = []
for i in range(MIN_CONNS):
connection_d = self._newConnection(conn_str)
d_list.append(connection_d)
dl = DeferredList(d_list)
dl.addCallbacks(return_d.callback, return_d.errback)
except Exception as e:
logging.error(
"IndxConnectionPool error in _newConnections: {0}".format(e))
return_d.errback(Failure(e))
return return_d
class DBConnectionPool():
""" A pool of DB connections for a specific connection string / DB. """
def __init__(self, conn_str):
self.waiting = []
self.inuse = []
self.free = []
self.semaphore = DeferredSemaphore(1)
self.updateTime()
def __unicode__(self):
return self.__str__()
def __str__(self):
return "waiting: {0}, inuse: {1}, free: {2}, semaphore: {3}, lastused: {4}".format(
self.waiting, self.inuse, self.free, self.semaphore, self.lastused)
def updateTime(self):
self.lastused = time.mktime(time.gmtime()) # epoch time
def getTime(self):
return self.lastused
def getWaiting(self):
self.updateTime()
return self.waiting
def getInuse(self):
self.updateTime()
return self.inuse
def getFree(self):
self.updateTime()
return self.free
def freeConnection(self, conn):
""" Free a connection from this DBPool. """
def locked_cb(empty):
logging.debug("DBConnectionPool locked_cb")
self.getInuse().remove(conn)
if len(self.getWaiting()) > 0:
callback = self.getWaiting().pop()
self.getInuse().append(conn)
self.semaphore.release()
callback(conn)
else:
self.getFree().append(conn)
self.semaphore.release()
def err_cb(failure):
failure.trap(Exception)
logging.error("DBConnectionPool free, err_cb: {0}".format(
failure.value))
self.semaphore.release()
self.semaphore.acquire().addCallbacks(locked_cb, err_cb)
def removeAll(self, count):
""" Remove all free connections (usually because they're old and we're in
a freeing up period.
"""
logging.debug(
"DBConnectionPool removeAll called, count: {0}".format(count))
return_d = Deferred()
self.updateTime()
def err_cb(failure):
self.semaphore.release()
return_d.errback(failure)
def locked_cb(count):
# immediately close the free connections
while len(self.free) > 0:
conn = self.free.pop(0)
conn.close()
count += 1
self.semaphore.release()
return_d.callback(count)
self.semaphore.acquire().addCallbacks(lambda s: locked_cb(count),
err_cb)
return return_d
class RateLimitedClient(object):
"""A Web client with per-second request limit.
"""
# Max number of requests per second (can be < 1.0)
rate_limit = None
# Grace delay (seconds) when the server throttles us
grace_delay = 30
# Max number of parallel requests
max_concurrency = 5
def __init__(self, time=None):
self.sem = DeferredSemaphore(self.max_concurrency)
self.grace_deferred = None
self.logger = logging.getLogger("webclient")
self.time = time or reactor
self.last_request = 0.0
def _enable_grace_delay(self, delay):
if self.grace_deferred:
# Already enabled by an earlier concurrent request
return
self.grace_deferred = Deferred()
def expire():
g = self.grace_deferred
self.grace_deferred = None
g.callback(None)
reactor.callLater(self.grace_delay, expire)
def _delay_if_necessary(self, func, *args, **kwargs):
d = Deferred()
d.addCallback(lambda _: func(*args, **kwargs))
trigger = None
if self.grace_deferred:
trigger = self.grace_deferred
elif self.rate_limit:
delay = (self.last_request + 1.0 / self.rate_limit) - self.time.seconds()
if delay > 0:
self.logger.debug("inserting rate limit delay of %.1f", delay)
trigger = Deferred()
self.time.callLater(delay, trigger.callback, None)
(trigger or maybeDeferred(lambda: None)).chainDeferred(d)
return d
def get_page(self, url, *args, **kwargs):
if isinstance(url, unicode):
url = url.encode("utf8")
def schedule_request(_):
return self._delay_if_necessary(issue_request, None)
def issue_request(_):
self.last_request = self.time.seconds()
self.logger.debug("fetching %r", url)
return getPage(url, *args, **kwargs)
def handle_success(value):
self.sem.release()
self.logger.debug("got %d bytes for %r", len(value), url)
return value
def handle_error(failure):
self.sem.release()
failure.trap(HTTPError)
self.logger.debug("got HTTP error %s", failure.value)
self.trap_throttling(failure)
delay = self.grace_delay
self.logger.warning("we are throttled, delaying by %.1f seconds", delay)
self._enable_grace_delay(delay)
# auto-retry
return do_get_page()
def do_get_page():
# We acquire the semaphore *before* seeing if we should delay
# the request, so that we avoid pounding on the server when
# the grace period is entered.
d = self.sem.acquire()
d.addCallback(schedule_request)
d.addCallbacks(handle_success, handle_error)
return d
return do_get_page()
def trap_throttling(self, failure):
"""Trap HTTP failures and return if we are
throttled by the distant site, else re-raise.
"""
e = failure.value
if e.status in ("400", "420", "500", "503"):
return
failure.raiseException()
class GcmPushkin(Pushkin):
"""
Pushkin that relays notifications to Google/Firebase Cloud Messaging.
"""
UNDERSTOOD_CONFIG_FIELDS = {"type", "api_key"}
def __init__(self, name, sygnal, config, canonical_reg_id_store):
super(GcmPushkin, self).__init__(name, sygnal, config)
nonunderstood = set(self.cfg.keys()).difference(
self.UNDERSTOOD_CONFIG_FIELDS)
if len(nonunderstood) > 0:
logger.warning(
"The following configuration fields are not understood: %s",
nonunderstood,
)
self.http_pool = HTTPConnectionPool(reactor=sygnal.reactor)
self.max_connections = self.get_config("max_connections",
DEFAULT_MAX_CONNECTIONS)
self.connection_semaphore = DeferredSemaphore(self.max_connections)
self.http_pool.maxPersistentPerHost = self.max_connections
tls_client_options_factory = ClientTLSOptionsFactory()
self.http_agent = Agent(
reactor=sygnal.reactor,
pool=self.http_pool,
contextFactory=tls_client_options_factory,
)
self.db = sygnal.database
self.canonical_reg_id_store = canonical_reg_id_store
self.api_key = self.get_config("api_key")
if not self.api_key:
raise PushkinSetupException("No API key set in config")
@classmethod
async def create(cls, name, sygnal, config):
"""
Override this if your pushkin needs to call async code in order to
be constructed. Otherwise, it defaults to just invoking the Python-standard
__init__ constructor.
Returns:
an instance of this Pushkin
"""
logger.debug("About to set up CanonicalRegId Store")
canonical_reg_id_store = CanonicalRegIdStore()
await canonical_reg_id_store.setup(sygnal.database,
sygnal.database_engine)
logger.debug("Finished setting up CanonicalRegId Store")
return cls(name, sygnal, config, canonical_reg_id_store)
async def _perform_http_request(self, body, headers):
"""
Perform an HTTP request to the FCM server with the body and headers
specified.
Args:
body (nested dict): Body. Will be JSON-encoded.
headers (Headers): HTTP Headers.
Returns:
"""
body_producer = FileBodyProducer(BytesIO(json.dumps(body).encode()))
# we use the semaphore to actually limit the number of concurrent
# requests, since the HTTPConnectionPool will actually just lead to more
# requests being created but not pooled – it does not perform limiting.
with QUEUE_TIME_HISTOGRAM.time():
with PENDING_REQUESTS_GAUGE.track_inprogress():
await self.connection_semaphore.acquire()
try:
with SEND_TIME_HISTOGRAM.time():
with ACTIVE_REQUESTS_GAUGE.track_inprogress():
response = await self.http_agent.request(
b"POST",
GCM_URL,
headers=Headers(headers),
bodyProducer=body_producer,
)
response_text = (await readBody(response)).decode()
except Exception as exception:
raise TemporaryNotificationDispatchException(
"GCM request failure") from exception
finally:
self.connection_semaphore.release()
return response, response_text
async def _request_dispatch(self, n, log, body, headers, pushkeys, span):
poke_start_time = time.time()
failed = []
response, response_text = await self._perform_http_request(
body, headers)
RESPONSE_STATUS_CODES_COUNTER.labels(pushkin=self.name,
code=response.code).inc()
log.debug("GCM request took %f seconds", time.time() - poke_start_time)
span.set_tag(tags.HTTP_STATUS_CODE, response.code)
if 500 <= response.code < 600:
log.debug("%d from server, waiting to try again", response.code)
retry_after = None
for header_value in response.headers.getRawHeaders(b"retry-after",
default=[]):
retry_after = int(header_value)
span.log_kv({
"event": "gcm_retry_after",
"retry_after": retry_after
})
raise TemporaryNotificationDispatchException(
"GCM server error, hopefully temporary.",
custom_retry_delay=retry_after)
elif response.code == 400:
log.error(
"%d from server, we have sent something invalid! Error: %r",
response.code,
response_text,
)
# permanent failure: give up
raise NotificationDispatchException("Invalid request")
elif response.code == 401:
log.error("401 from server! Our API key is invalid? Error: %r",
response_text)
# permanent failure: give up
raise NotificationDispatchException("Not authorised to push")
elif response.code == 404:
# assume they're all failed
log.info("Reg IDs %r get 404 response; assuming unregistered",
pushkeys)
return pushkeys, []
elif 200 <= response.code < 300:
try:
resp_object = json.loads(response_text)
except JSONDecodeError:
raise NotificationDispatchException(
"Invalid JSON response from GCM.")
if "results" not in resp_object:
log.error(
"%d from server but response contained no 'results' key: %r",
response.code,
response_text,
)
if len(resp_object["results"]) < len(pushkeys):
log.error(
"Sent %d notifications but only got %d responses!",
len(n.devices),
len(resp_object["results"]),
)
span.log_kv({
logs.EVENT: "gcm_response_mismatch",
"num_devices": len(n.devices),
"num_results": len(resp_object["results"]),
})
# determine which pushkeys to retry or forget about
new_pushkeys = []
for i, result in enumerate(resp_object["results"]):
span.set_tag("gcm_regid_updated", "registration_id" in result)
if "registration_id" in result:
await self.canonical_reg_id_store.set_canonical_id(
pushkeys[i], result["registration_id"])
if "error" in result:
log.warning("Error for pushkey %s: %s", pushkeys[i],
result["error"])
span.set_tag("gcm_error", result["error"])
if result["error"] in BAD_PUSHKEY_FAILURE_CODES:
log.info(
"Reg ID %r has permanently failed with code %r: "
"rejecting upstream",
pushkeys[i],
result["error"],
)
failed.append(pushkeys[i])
elif result["error"] in BAD_MESSAGE_FAILURE_CODES:
log.info(
"Message for reg ID %r has permanently failed with code %r",
pushkeys[i],
result["error"],
)
else:
log.info(
"Reg ID %r has temporarily failed with code %r",
pushkeys[i],
result["error"],
)
new_pushkeys.append(pushkeys[i])
return failed, new_pushkeys
else:
raise NotificationDispatchException(
f"Unknown GCM response code {response.code}")
async def dispatch_notification(self, n, device, context):
log = NotificationLoggerAdapter(logger,
{"request_id": context.request_id})
pushkeys = [
device.pushkey for device in n.devices
if device.app_id == self.name
]
# Resolve canonical IDs for all pushkeys
if pushkeys[0] != device.pushkey:
# Only send notifications once, to all devices at once.
return []
# The pushkey is kind of secret because you can use it to send push
# to someone.
# span_tags = {"pushkeys": pushkeys}
span_tags = {"gcm_num_devices": len(pushkeys)}
with self.sygnal.tracer.start_span(
"gcm_dispatch", tags=span_tags,
child_of=context.opentracing_span) as span_parent:
reg_id_mappings = await self.canonical_reg_id_store.get_canonical_ids(
pushkeys)
reg_id_mappings = {
reg_id: canonical_reg_id or reg_id
for (reg_id, canonical_reg_id) in reg_id_mappings.items()
}
inverse_reg_id_mappings = {
v: k
for (k, v) in reg_id_mappings.items()
}
data = GcmPushkin._build_data(n)
headers = {
b"User-Agent": ["sygnal"],
b"Content-Type": ["application/json"],
b"Authorization": ["key=%s" % (self.api_key, )],
}
# count the number of remapped registration IDs in the request
span_parent.set_tag(
"gcm_num_remapped_reg_ids_used",
[k != v for (k, v) in reg_id_mappings.items()].count(True),
)
# TODO: Implement collapse_key to queue only one message per room.
failed = []
body = {
"data": data,
"priority": "normal" if n.prio == "low" else "high"
}
for retry_number in range(0, MAX_TRIES):
mapped_pushkeys = [reg_id_mappings[pk] for pk in pushkeys]
if len(pushkeys) == 1:
body["to"] = mapped_pushkeys[0]
else:
body["registration_ids"] = mapped_pushkeys
log.info("Sending (attempt %i) => %r", retry_number,
mapped_pushkeys)
try:
span_tags = {"retry_num": retry_number}
with self.sygnal.tracer.start_span(
"gcm_dispatch_try",
tags=span_tags,
child_of=span_parent) as span:
new_failed, new_pushkeys = await self._request_dispatch(
n, log, body, headers, mapped_pushkeys, span)
pushkeys = new_pushkeys
failed += [
inverse_reg_id_mappings[canonical_pk]
for canonical_pk in new_failed
]
if len(pushkeys) == 0:
break
except TemporaryNotificationDispatchException as exc:
retry_delay = RETRY_DELAY_BASE * (2**retry_number)
if exc.custom_retry_delay is not None:
retry_delay = exc.custom_retry_delay
log.warning(
"Temporary failure, will retry in %d seconds",
retry_delay,
exc_info=True,
)
span_parent.log_kv({
"event": "temporary_fail",
"retrying_in": retry_delay
})
await twisted_sleep(retry_delay,
twisted_reactor=self.sygnal.reactor)
if len(pushkeys) > 0:
log.info("Gave up retrying reg IDs: %r", pushkeys)
# Count the number of failed devices.
span_parent.set_tag("gcm_num_failed", len(failed))
return failed
@staticmethod
def _build_data(n):
"""
Build the payload data to be sent.
Args:
n: Notification to build the payload for.
Returns:
JSON-compatible dict
"""
data = {}
for attr in [
"event_id",
"type",
"sender",
"room_name",
"room_alias",
"membership",
"sender_display_name",
"content",
"room_id",
]:
if hasattr(n, attr):
data[attr] = getattr(n, attr)
# Truncate fields to a sensible maximum length. If the whole
# body is too long, GCM will reject it.
if data[attr] is not None and len(
data[attr]) > MAX_BYTES_PER_FIELD:
data[attr] = data[attr][0:MAX_BYTES_PER_FIELD]
data["prio"] = "high"
if n.prio == "low":
data["prio"] = "normal"
if getattr(n, "counts", None):
data["unread"] = n.counts.unread
data["missed_calls"] = n.counts.missed_calls
return data
class AggregationResponseCache(object):
'''
This holds all the responses being aggregated for a single destination.
One of the main challenges here is to make sure while we're sending the responses,
we don't get a new response in and not send it.
'''
def __init__(self, numSecondsToWait, numMessagesToWaitFor, chordNode):
'''
Constructor
'''
self.numSecondsToWait = numSecondsToWait
self.numMessagesToWaitFor = numMessagesToWaitFor
self.numSecondsToWait = numSecondsToWait
self.chordNode = chordNode
self.semaphore = DeferredSemaphore(1)
self.messageList = [] # Holds tuples of (message, envelope)
# Construct a timer to wait
self.timerID = None
def addResponse(self, message, envelope):
'''We use a semaphore to ensure we don't modify the list while sending.'''
d = self.semaphore.acquire()
d.addCallback(self._addResponse, message, envelope)
def _addResponse(self, dummy_defResult, message, envelope):
'''This is called only once we have the semaphore.'''
self.messageList.append ( (message, envelope) )
print("DEBUG: AggRespCache: %s adding message %s " % (self.chordNode.nodeLocation.port, message))
if len(self.messageList) >= self.numMessagesToWaitFor:
# Send it!
self._sendResponse()
else:
# Make sure a timer is running
if self.timerID is None or not self.timerID.active():
self.timerID = reactor.callLater(self.numSecondsToWait, self.sendResponse)
# We're done.
self.semaphore.release()
def sendResponse(self):
'''Only call sendResponse when you have the lock.'''
d = self.semaphore.acquire()
d.addCallback(self._sendResponse)
def _sendResponse(self, dummy_deferResult=None):
'''Send the response but only after acquiring the semaphore
'''
# Copy the list
messagesListCopy = self.messageList
self.messageList = []
# Release the semaphore
self.semaphore.release()
# Stop the timer if it's still going
if self.timerID is not None and self.timerID.active():
self.timerID.cancel()
self.timerID = None
print("DEBUG: AggResponseCache-Sending %d Messages %s" % (len(messagesListCopy), self.chordNode.nodeLocation.port))
# Send a P2P message to the dest with all the responses
d = self.chordNode.sendSyncMultipleMessage(messagesListCopy, 'p2p') # Will this break message authentication?
d.addCallback(self.sendAcks, messagesListCopy)
d.addErrback(self.sendResponseFailed)
# def emptyMessageList(self, _):
# self.messageList = []
def sendAcks(self, resultsDict, messageList):
# Send ACK messages to the nodes for which we aggregated
for (_message, envelope) in messageList:
# Get the status to return
msgID = envelope['msgID']
if msgID not in resultsDict:
status = False
else:
status = resultsDict[msgID]
d = self.chordNode.sendSingleAck(msgID, envelope['source'], status)
d.addErrback(self.sendAckFailed, envelope['source'])
def sendAckFailed(self, fail, sourceNode):
log.err("We failed to SendAck for source %s" % sourceNode, fail)
def sendResponseFailed(self, theFailure):
log.err(theFailure)
class IndxConnectionPool:
""" A wrapper for txpostgres connection pools, which auto-reconnects. """
def __init__(self):
logging.debug("IndxConnectionPool starting. ")
self.connections = {} # by connection string
self.conn_strs = {} # by db_name
self.semaphore = DeferredSemaphore(1)
self.subscribers = {} # by db name
def removeAll(self, db_name):
""" Remove all connections for a named database - used before deleting that database. """
logging.debug("IndxConnectionPool removeAll {0}".format(db_name))
d_list = []
if db_name in self.conn_strs:
for conn_str in self.conn_strs[db_name]:
for conn in self.connections[conn_str].getInuse():
d_list.append(conn.close())
for conn in self.connections[conn_str].getFree():
d_list.append(conn.close())
del self.connections[conn_str]
del self.conn_strs[db_name]
dl = DeferredList(d_list)
return dl
def connect(self, db_name, db_user, db_pass, db_host, db_port):
""" Returns an IndxConnection (Actual connection and pool made when query is made). """
return_d = Deferred()
log_conn_str = "dbname='{0}' user='******' password='******' host='{3}' port='{4}' application_name='{5}'".format(db_name, db_user, "XXXX", db_host, db_port, indx_pg2.APPLICATION_NAME)
conn_str = "dbname='{0}' user='******' password='******' host='{3}' port='{4}' application_name='{5}'".format(db_name, db_user, db_pass, db_host, db_port, indx_pg2.APPLICATION_NAME)
logging.debug("IndxConnectionPool connect: {0}".format(log_conn_str))
if db_name not in self.conn_strs:
self.conn_strs[db_name] = []
self.conn_strs[db_name].append(conn_str)
def free_cb(conn):
""" Called back when this IndxConnection has finished querying, so
we put the real connection back into the pool. """
logging.debug("IndxConnectionPool free_cb, conn: {0}".format(conn))
self.connections[conn_str].freeConnection(conn) # no dealing with callbacks, just carry on
def alloc_cb(conn_str):
# a query was called - allocate a connection now and pass it back
return self._connect(conn_str)
indx_connection = IndxConnection(conn_str, alloc_cb, free_cb)
return_d.callback(indx_connection)
return return_d
def _connect(self, conn_str):
""" Connect and return a free Connection.
Figures out whether to make new connections, use the pool, or wait in a queue.
"""
logging.debug("IndxConnectionPool _connect ({0})".format(conn_str))
return_d = Deferred()
def err_cb(failure):
logging.error("IndxConnectionPool _connect err_cb: {0}".format(failure))
self.semaphore.release()
return_d.errback(failure)
def succeed_cb(empty):
logging.debug("IndxConnectionPool _connect succeed_cb")
# TODO pass a Connection back
if len(self.connections[conn_str].getFree()) > 0:
# free connection, use it straight away
conn = self.connections[conn_str].getFree().pop()
self.connections[conn_str].getInuse().append(conn)
self.semaphore.release()
return_d.callback(conn)
return
if len(self.connections[conn_str].getInuse()) < MAX_CONNS:
# not at max connections for this conn_str
# create a new one
d = self._newConnection(conn_str)
def connected_cb(indx_conn):
logging.debug("IndxConnectionPool _connect connected_cb ({0})".format(indx_conn))
self.connections[conn_str].getFree().remove(indx_conn)
self.connections[conn_str].getInuse().append(indx_conn)
self.semaphore.release()
return_d.callback(indx_conn)
return
d.addCallbacks(connected_cb, err_cb)
return
# wait for a connection
def wait_cb(conn):
logging.debug("IndxConnectionPool _connect wait_cb ({0})".format(conn))
# already put in 'inuse'
return_d.callback(conn)
return
self.semaphore.release()
self.connections[conn_str].getWaiting().append(wait_cb)
return
def locked_cb(empty):
logging.debug("IndxConnectionPool _connect locked_cb")
if conn_str not in self.connections:
self._newConnections(conn_str).addCallbacks(succeed_cb, err_cb)
else:
threads.deferToThread(succeed_cb, None)
# succeed_cb(None)
self.semaphore.acquire().addCallbacks(locked_cb, err_cb)
return return_d
def _closeOldConnection(self):
""" Close the oldest connection, so we can open a new one up. """
# is already in a semaphore lock, from _newConnection
logging.debug("IndxConnectionPool _closeOldConnection")
### we could force quite them through postgresql like this - but instead we kill them from inside
#query = "SELECT * FROM pg_stat_activity WHERE state = 'idle' AND application_name = %s AND query != 'LISTEN wb_new_version' ORDER BY state_change LIMIT 1;"
#params = [indx_pg2.APPLICATION_NAME]
return_d = Deferred()
def err_cb(failure):
return_d.errback(failure)
ages = {}
for conn_str, dbpool in self.connections.items():
lastused = dbpool.getTime()
if lastused not in ages:
ages[lastused] = []
ages[lastused].append(dbpool)
times = ages.keys()
times.sort()
pool_queue = []
for timekey in times:
pools = ages[timekey]
pool_queue.extend(pools)
def removed_cb(count):
if count < REMOVE_AT_ONCE and len(pool_queue) > 0:
pool = pool_queue.pop(0)
pool.getFree()
pool.removeAll(count).addCallbacks(removed_cb, err_cb)
else:
return_d.callback(None)
removed_cb(0)
return return_d
def _newConnection(self, conn_str):
""" Makes a new connection to the DB
and then puts it in the 'free' pool of this conn_str.
"""
logging.debug("IndxConnectionPool _newConnection")
# lock with the semaphore before calling this
return_d = Deferred()
def close_old_cb(failure):
failure.trap(psycopg2.OperationalError, Exception)
# couldn't connect, so close an old connection first
logging.error("IndxConnectionPool error close_old_cb: {0} - state of conns is: {1}".format(failure.value, self.connections))
logging.error("IndxConnectionPool connections: {0}".format("\n".join(map(lambda name: self.connections[name].__str__(), self.connections))))
def closed_cb(empty):
# closed, so try connecting again
self._newConnection(conn_str).addCallbacks(return_d.callback, return_d.errback)
closed_d = self._closeOldConnection()
closed_d.addCallbacks(closed_cb, return_d.errback)
try:
# try to connect
def connected_cb(connection):
logging.debug("IndxConnectionPool _newConnection connected_cb, connection: {0}".format(connection))
self.connections[conn_str].getFree().append(connection)
return_d.callback(connection)
conn = txpostgres.Connection()
connection_d = conn.connect(conn_str)
connection_d.addCallbacks(connected_cb, close_old_cb)
except Exception as e:
# close an old connection first
logging.debug("IndxConnectionPool Exception, going to call close_old_cb: ({0})".format(e))
close_old_cb(Failure(e))
return return_d
def _newConnections(self, conn_str):
""" Make a pool of new connections. """
# lock with the semaphore before calling this
logging.debug("IndxConnectionPool _newConnections")
return_d = Deferred()
self.connections[conn_str] = DBConnectionPool(conn_str)
try:
d_list = []
for i in range(MIN_CONNS):
connection_d = self._newConnection(conn_str)
d_list.append(connection_d)
dl = DeferredList(d_list)
dl.addCallbacks(return_d.callback, return_d.errback)
except Exception as e:
logging.error("IndxConnectionPool error in _newConnections: {0}".format(e))
return_d.errback(Failure(e))
return return_d
class DBConnectionPool():
""" A pool of DB connections for a specific connection string / DB. """
def __init__(self, conn_str):
self.waiting = []
self.inuse = []
self.free = []
self.semaphore = DeferredSemaphore(1)
self.updateTime()
def __unicode__(self):
return self.__str__()
def __str__(self):
return "waiting: {0}, inuse: {1}, free: {2}, semaphore: {3}, lastused: {4}".format(self.waiting, self.inuse, self.free, self.semaphore, self.lastused)
def updateTime(self):
self.lastused = time.mktime(time.gmtime()) # epoch time
def getTime(self):
return self.lastused
def getWaiting(self):
self.updateTime()
return self.waiting
def getInuse(self):
self.updateTime()
return self.inuse
def getFree(self):
self.updateTime()
return self.free
def freeConnection(self, conn):
""" Free a connection from this DBPool. """
def locked_cb(empty):
logging.debug("DBConnectionPool locked_cb")
self.getInuse().remove(conn)
if len(self.getWaiting()) > 0:
callback = self.getWaiting().pop()
self.getInuse().append(conn)
self.semaphore.release()
callback(conn)
else:
self.getFree().append(conn)
self.semaphore.release()
def err_cb(failure):
failure.trap(Exception)
logging.error("DBConnectionPool free, err_cb: {0}".format(failure.value))
self.semaphore.release()
self.semaphore.acquire().addCallbacks(locked_cb, err_cb)
def removeAll(self, count):
""" Remove all free connections (usually because they're old and we're in
a freeing up period.
"""
logging.debug("DBConnectionPool removeAll called, count: {0}".format(count))
return_d = Deferred()
self.updateTime()
def err_cb(failure):
self.semaphore.release()
return_d.errback(failure)
def locked_cb(count):
# immediately close the free connections
while len(self.free) > 0:
conn = self.free.pop(0)
conn.close()
count += 1
self.semaphore.release()
return_d.callback(count)
self.semaphore.acquire().addCallbacks(lambda s: locked_cb(count), err_cb)
return return_d
class WebpushPushkin(ConcurrencyLimitedPushkin):
"""
Pushkin that relays notifications to Google/Firebase Cloud Messaging.
"""
UNDERSTOOD_CONFIG_FIELDS = {
"type",
"max_connections",
"vapid_private_key",
"vapid_contact_email",
"allowed_endpoints",
"ttl",
} | ConcurrencyLimitedPushkin.UNDERSTOOD_CONFIG_FIELDS
def __init__(self, name: str, sygnal: "Sygnal", config: Dict[str, Any]):
super().__init__(name, sygnal, config)
nonunderstood = self.cfg.keys() - self.UNDERSTOOD_CONFIG_FIELDS
if nonunderstood:
logger.warning(
"The following configuration fields are not understood: %s",
nonunderstood,
)
self.http_pool = HTTPConnectionPool(reactor=sygnal.reactor)
self.max_connections = self.get_config(
"max_connections", int, DEFAULT_MAX_CONNECTIONS
)
self.connection_semaphore = DeferredSemaphore(self.max_connections)
self.http_pool.maxPersistentPerHost = self.max_connections
tls_client_options_factory = ClientTLSOptionsFactory()
# use the Sygnal global proxy configuration
proxy_url = sygnal.config.get("proxy")
self.http_agent = ProxyAgent(
reactor=sygnal.reactor,
pool=self.http_pool,
contextFactory=tls_client_options_factory,
proxy_url_str=proxy_url,
)
self.http_request_factory = HttpRequestFactory()
self.allowed_endpoints: Optional[List[Pattern[str]]] = None
allowed_endpoints = self.get_config("allowed_endpoints", list)
if allowed_endpoints:
self.allowed_endpoints = list(map(glob_to_regex, allowed_endpoints))
privkey_filename = self.get_config("vapid_private_key", str)
if not privkey_filename:
raise PushkinSetupException("'vapid_private_key' not set in config")
if not os.path.exists(privkey_filename):
raise PushkinSetupException("path in 'vapid_private_key' does not exist")
try:
self.vapid_private_key = Vapid.from_file(private_key_file=privkey_filename)
except VapidException as e:
raise PushkinSetupException("invalid 'vapid_private_key' file") from e
self.vapid_contact_email = self.get_config("vapid_contact_email", str)
if not self.vapid_contact_email:
raise PushkinSetupException("'vapid_contact_email' not set in config")
self.ttl = self.get_config("ttl", int, DEFAULT_TTL)
async def _dispatch_notification_unlimited(
self, n: Notification, device: Device, context: NotificationContext
) -> List[str]:
p256dh = device.pushkey
if not isinstance(device.data, dict):
logger.warn(
"Rejecting pushkey %s; device.data is not a dict", device.pushkey
)
return [device.pushkey]
# drop notifications without an event id if requested,
# see https://github.com/matrix-org/sygnal/issues/186
if device.data.get("events_only") is True and not n.event_id:
return []
endpoint = device.data.get("endpoint")
auth = device.data.get("auth")
if not p256dh or not isinstance(endpoint, str) or not isinstance(auth, str):
logger.warn(
"Rejecting pushkey; subscription info incomplete or invalid "
+ "(p256dh: %s, endpoint: %r, auth: %r)",
p256dh,
endpoint,
auth,
)
return [device.pushkey]
endpoint_domain = urlparse(endpoint).netloc
if self.allowed_endpoints:
allowed = any(
regex.fullmatch(endpoint_domain) for regex in self.allowed_endpoints
)
if not allowed:
logger.error(
"push gateway %s is not in allowed_endpoints, blocking request",
endpoint_domain,
)
# abort, but don't reject push key
return []
subscription_info = {
"endpoint": endpoint,
"keys": {"p256dh": p256dh, "auth": auth},
}
payload = WebpushPushkin._build_payload(n, device)
data = json.dumps(payload)
# web push only supports normal and low priority, so assume normal if absent
low_priority = n.prio == "low"
# allow dropping earlier notifications in the same room if requested
topic = None
if n.room_id and device.data.get("only_last_per_room") is True:
# ask for a 22 byte hash, so the base64 of it is 32,
# the limit webpush allows for the topic
topic = urlsafe_b64encode(
blake2s(n.room_id.encode(), digest_size=22).digest()
)
# note that webpush modifies vapid_claims, so make sure it's only used once
vapid_claims = {
"sub": "mailto:{}".format(self.vapid_contact_email),
}
# we use the semaphore to actually limit the number of concurrent
# requests, since the HTTPConnectionPool will actually just lead to more
# requests being created but not pooled – it does not perform limiting.
with QUEUE_TIME_HISTOGRAM.time():
with PENDING_REQUESTS_GAUGE.track_inprogress():
await self.connection_semaphore.acquire()
try:
with SEND_TIME_HISTOGRAM.time():
with ACTIVE_REQUESTS_GAUGE.track_inprogress():
request = webpush(
subscription_info=subscription_info,
data=data,
ttl=self.ttl,
vapid_private_key=self.vapid_private_key,
vapid_claims=vapid_claims,
requests_session=self.http_request_factory,
)
response = await request.execute(
self.http_agent, low_priority, topic
)
response_text = (await readBody(response)).decode()
finally:
self.connection_semaphore.release()
reject_pushkey = self._handle_response(
response, response_text, device.pushkey, endpoint_domain
)
if reject_pushkey:
return [device.pushkey]
return []
@staticmethod
def _build_payload(n: Notification, device: Device) -> Dict[str, Any]:
"""
Build the payload data to be sent.
Args:
n: Notification to build the payload for.
device: Device information to which the constructed payload
will be sent.
Returns:
JSON-compatible dict
"""
payload = {}
if device.data:
default_payload = device.data.get("default_payload")
if isinstance(default_payload, dict):
payload.update(default_payload)
for attr in [
"room_id",
"room_name",
"room_alias",
"membership",
"event_id",
"sender",
"sender_display_name",
"user_is_target",
"type",
]:
value = getattr(n, attr, None)
if value:
payload[attr] = value
counts = getattr(n, "counts", None)
if counts is not None:
for attr in ["unread", "missed_calls"]:
count_value = getattr(counts, attr, None)
if count_value is not None:
payload[attr] = count_value
if n.content and isinstance(n.content, dict):
content = n.content.copy()
# we can't show formatted_body in a notification anyway on web
# so remove it
content.pop("formatted_body", None)
body = content.get("body")
# make some attempts to not go over the max payload length
if isinstance(body, str) and len(body) > MAX_BODY_LENGTH:
content["body"] = body[0 : MAX_BODY_LENGTH - 1] + "…"
ciphertext = content.get("ciphertext")
if isinstance(ciphertext, str) and len(ciphertext) > MAX_CIPHERTEXT_LENGTH:
content.pop("ciphertext", None)
payload["content"] = content
return payload
def _handle_response(
self,
response: IResponse,
response_text: str,
pushkey: str,
endpoint_domain: str,
) -> bool:
"""
Logs and determines the outcome of the response
Returns:
Boolean whether the puskey should be rejected
"""
ttl_response_headers = response.headers.getRawHeaders(b"TTL")
if ttl_response_headers:
try:
ttl_given = int(ttl_response_headers[0])
if ttl_given != self.ttl:
logger.info(
"requested TTL of %d to endpoint %s but got %d",
self.ttl,
endpoint_domain,
ttl_given,
)
except ValueError:
pass
# permanent errors
if response.code == 404 or response.code == 410:
logger.warn(
"Rejecting pushkey %s; subscription is invalid on %s: %d: %s",
pushkey,
endpoint_domain,
response.code,
response_text,
)
return True
# and temporary ones
if response.code >= 400:
logger.warn(
"webpush request failed for pushkey %s; %s responded with %d: %s",
pushkey,
endpoint_domain,
response.code,
response_text,
)
elif response.code != 201:
logger.info(
"webpush request for pushkey %s didn't respond with 201; "
+ "%s responded with %d: %s",
pushkey,
endpoint_domain,
response.code,
response_text,
)
return False
class BaseQtWebKitMiddleware(object):
nam_cls = ScrapyNetworkAccessManager
@classmethod
def from_crawler(cls, crawler):
settings = crawler.settings
if crawler.settings.getbool('QTWEBKIT_COOKIES_ENABLED', False):
cookies_middleware = CookiesMiddleware(
crawler.settings.getbool('COOKIES_DEBUG')
)
else:
cookies_middleware = None
qt_platform = settings.get("QTWEBKIT_QT_PLATFORM", "minimal")
if qt_platform == "default":
qt_platform = None
ext = cls(
crawler,
show_window=settings.getbool("QTWEBKIT_SHOW_WINDOW", False),
qt_platform=qt_platform,
enable_webkit_dev_tools=settings.get("QTWEBKIT_ENABLE_DEV_TOOLS",
False),
page_limit=settings.getint("QTWEBKIT_PAGE_LIMIT", 4),
cookies_middleware=cookies_middleware
)
return ext
@staticmethod
def engine_stopped():
if QApplication.instance():
QApplication.instance().quit()
def __init__(self, crawler, show_window=False, qt_platform="minimal",
enable_webkit_dev_tools=False, page_limit=4,
cookies_middleware=None):
super(BaseQtWebKitMiddleware, self).__init__()
self._crawler = crawler
self.show_window = show_window
self.qt_platform = qt_platform
self.enable_webkit_dev_tools = enable_webkit_dev_tools
if page_limit != 1:
if QWebSettings is not None:
QWebSettings.setObjectCacheCapacities(0, 0, 0)
if page_limit is None:
self.semaphore = DummySemaphore()
else:
self.semaphore = DeferredSemaphore(page_limit)
self.cookies_middleware = cookies_middleware
self._references = set()
@staticmethod
def _schedule_qt_event_loop(app):
"""
Schedule a QApplication's event loop within Twisted. Should be called
at most once per QApplication.
"""
# XXX: This is ugly but I don't know another way to do it.
call = LoopingCall(app.processEvents)
call.start(0.02, False)
app.aboutToQuit.connect(call.stop)
def _setup_page(self, page, extra_settings):
settings = page.settings()
settings.setAttribute(QWebSettings.JavaEnabled, False)
settings.setAttribute(QWebSettings.PluginsEnabled, False)
settings.setAttribute(QWebSettings.PrivateBrowsingEnabled, True)
settings.setAttribute(QWebSettings.LocalStorageEnabled, True)
settings.setAttribute(QWebSettings.LocalContentCanAccessRemoteUrls,
True)
settings.setAttribute(QWebSettings.LocalContentCanAccessFileUrls,
True)
settings.setAttribute(QWebSettings.NotificationsEnabled, False)
settings.setAttribute(QWebSettings.DeveloperExtrasEnabled,
self.enable_webkit_dev_tools)
for setting, value in extra_settings.items():
settings.setAttribute(setting, value)
@staticmethod
def _make_qt_request(scrapy_request):
"""Build a QNetworkRequest from a Scrapy request."""
qt_request = QNetworkRequest(QUrl(scrapy_request.url))
for header, values in scrapy_request.headers.items():
qt_request.setRawHeader(header, b', '.join(values))
try:
operation = HTTP_METHOD_TO_QT_OPERATION[scrapy_request.method]
except KeyError:
operation = QNetworkAccessManager.CustomOperation
qt_request.setAttribute(QNetworkRequest.CustomVerbAttribute,
scrapy_request.method)
qt_request.setAttribute(QNetworkRequest.CacheSaveControlAttribute,
False)
req_body = QByteArray(scrapy_request.body)
return qt_request, operation, req_body
@inlineCallbacks
def process_request(self, request, spider):
if self.cookies_middleware:
yield self.cookies_middleware.process_request(request, spider)
if isinstance(request, QtWebKitRequest):
if request.webpage:
# Request is to continue processing with an existing webpage
# object.
webpage = request.webpage
request = request.replace(webpage=None)
webpage.networkAccessManager().request = request
returnValue(self._handle_page_request(spider, request,
webpage))
else:
yield self.semaphore.acquire()
response = yield self.create_page(request, spider)
returnValue(response)
def process_response(self, request, response, spider):
if self.cookies_middleware:
return self.cookies_middleware.process_response(request, response,
spider)
else:
return response
def ensure_qapplication(self):
"""Create and setup a QApplication if one does not already exist."""
if not QApplication.instance():
args = ["scrapy"]
if self.qt_platform is not None:
args.extend(["-platform", self.qt_platform])
app = QApplication(args)
self._schedule_qt_event_loop(app)
_QApplicationStopper(self._crawler.signals, app)
def create_page(self, request, spider):
"""
Create a webpage object, load a request on it, return a deferred that
fires with a response on page load.
"""
self.ensure_qapplication()
webpage = WebPage()
self._setup_page(webpage,
request.meta.get('qwebsettings_settings', {}))
self._references.add(webpage)
if self.show_window:
webview = QWebView()
webview.setPage(webpage)
webpage.webview = webview
self._add_webview_to_window(webview, spider.name)
if request.meta.get('qtwebkit_user_agent', False):
request.headers['User-Agent'] = webpage.userAgentForUrl(
QUrl(request.url)
)
nam = self.nam_cls(spider, request, request.headers.get('User-Agent'),
parent=webpage)
if ((self.cookies_middleware and
'dont_merge_cookies' not in request.meta)):
cookiejarkey = request.meta.get("cookiejar")
cookiejar = ScrapyAwareCookieJar(self.cookies_middleware,
cookiejarkey, parent=nam)
nam.setCookieJar(cookiejar)
webpage.setNetworkAccessManager(nam)
d = deferred_for_signal(webpage.load_finished_with_error)
d.addCallback(partial(self._handle_page_request, spider, request,
webpage))
webpage.mainFrame().load(*self._make_qt_request(request))
return d
def _add_webview_to_window(self, webview, title=""):
pass
def _remove_webview_from_window(self, webview):
pass
def _handle_page_request(self, spider, request, webpage,
load_result=(True, None)):
"""
Handle a request for a web page, either a page load or a request to
continue using an existing page object.
"""
try:
ok, error = load_result
if ok:
# The error object is not available if a page load was not
# requested.
if error and error.domain == QWebPage.Http:
status = error.error
else:
status = 200
if error:
url = error.url
else:
url = webpage.mainFrame().url()
qwebpage_response = request.meta.get('qwebpage_response', False)
if qwebpage_response:
respcls = QtWebKitResponse
else:
respcls = HtmlResponse
response = respcls(status=status,
url=url.toString(),
headers=error.headers,
body=webpage.mainFrame().toHtml(),
encoding='utf-8',
request=request)
if qwebpage_response:
response.webpage = webpage
request.callback = partial(self._request_callback, spider,
request.callback or 'parse')
else:
self._close_page(webpage)
else:
raise self._exception_from_errorpageextensionoption(error)
except Exception as err:
response = Failure(err)
return response
@inlineCallbacks
def _request_callback(self, spider, original_callback, response):
"""
Close the page (lose the reference to it so it is garbage collected)
when the callback returns.
The original callback may prevent page closing by setting the
should_close_webpage attribute in responses. This is useful for
example if the page is stored somewhere else (e.g. request meta) to be
used later. The page then needs to be closed manually at some point by
calling its close_page() function, which is created here.
"""
if isinstance(original_callback, basestring):
original_callback = getattr(spider, original_callback)
webpage = response.webpage
response.should_close_webpage = True
try:
returnValue(arg_to_iter((yield maybeDeferred(original_callback,
response))))
finally:
# FIXME: sometimes this section is reached before the wrapped
# callback finishes, when it returns a Deferred.
if response.should_close_webpage:
self._close_page(webpage)
else:
webpage.close_page = partial(self._close_page, webpage)
webpage.close_page.__doc__ = ("Lose the reference to the "
"webpage object and allow it "
"to be garbage collected.")
def _close_page(self, webpage):
self._references.remove(webpage)
# Resetting the main frame URL prevents it from making any more
# requests, which would cause Qt errors after the webpage is deleted.
webpage.mainFrame().setUrl(QUrl())
if webpage.webview is not None:
self._remove_webview_from_window(webpage.webview)
self.semaphore.release()
_qt_error_exc_mapping = {
QNetworkReply.ConnectionRefusedError: ConnectionRefusedError,
QNetworkReply.RemoteHostClosedError: ConnectionLost,
QNetworkReply.HostNotFoundError: DNSLookupError,
QNetworkReply.TimeoutError: TimeoutError,
QNetworkReply.OperationCanceledError: ConnectingCancelledError,
QNetworkReply.SslHandshakeFailedError: SSLError,
QNetworkReply.ProtocolUnknownError: NotSupported
}
def _exception_from_errorpageextensionoption(self, option):
if option.domain == QWebPage.QtNetwork:
exc_cls = self._qt_error_exc_mapping.get(option.error,
ConnectError)
# elif option.domain == QWebPage.WebKit:
# exc_cls = Exception
else:
exc_cls = Exception
return exc_cls(option.errorString)
