def publish_search(self, path, req): """ Publish a verify request on the queue to gate engine """ headers = HeaderKeyDict(req.headers) #self.logger.debug("SWIFTSEARCH avaiable headers: %s" % (headers.items())) # TODO(mlopezc1) is this actually faster than a regular regex with match? # swift code loves this pattern (ex tempurl middleware) but not sure how # will this actually perform in high use. Perf comparison later? for k, v in headers.items(): # if header not in the whitelist of allowed full header names or * if k not in self.index_headers: #self.logger.debug("SWIFTSEARCH k=%s not in %s" % (k, self.index_headers)) for h in self.index_headers_startwith: if not k.startswith(h): #self.logger.debug("SWIFTSEARCH k=%s not allowed" % (k)) del headers[k] self.logger.debug("SWIFTSEARCH sending metadata for indexing: %s" % (headers.items())) # TODO(mlopez1) what about renaming keys to something more human ? the X- and title format is kinda weird exchange = kombu.Exchange(self.exc_str, self.exc_type, durable=self.exc_durable) queue = kombu.Queue('search', exchange=exchange, routing_key='search') with kombu.Connection(self.conn_str) as connection: with connection.Producer(serializer='json') as producer: producer.publish({'id': b64encode(path), 'path': path, 'metadata': headers}, exchange=exchange, routing_key='search', declare=[queue]) return True
def test_clean_outgoing_headers(self): orh = '' oah = '' hdrs = {'test-header': 'value'} hdrs = HeaderKeyDict(tempurl.TempURL( None, {'outgoing_remove_headers': orh, 'outgoing_allow_headers': oah} )._clean_outgoing_headers(hdrs.items())) self.assertTrue('test-header' in hdrs) orh = 'test-header' oah = '' hdrs = {'test-header': 'value'} hdrs = HeaderKeyDict(tempurl.TempURL( None, {'outgoing_remove_headers': orh, 'outgoing_allow_headers': oah} )._clean_outgoing_headers(hdrs.items())) self.assertTrue('test-header' not in hdrs) orh = 'test-header-*' oah = '' hdrs = {'test-header-one': 'value', 'test-header-two': 'value'} hdrs = HeaderKeyDict(tempurl.TempURL( None, {'outgoing_remove_headers': orh, 'outgoing_allow_headers': oah} )._clean_outgoing_headers(hdrs.items())) self.assertTrue('test-header-one' not in hdrs) self.assertTrue('test-header-two' not in hdrs) orh = 'test-header-*' oah = 'test-header-two' hdrs = {'test-header-one': 'value', 'test-header-two': 'value'} hdrs = HeaderKeyDict(tempurl.TempURL( None, {'outgoing_remove_headers': orh, 'outgoing_allow_headers': oah} )._clean_outgoing_headers(hdrs.items())) self.assertTrue('test-header-one' not in hdrs) self.assertTrue('test-header-two' in hdrs) orh = 'test-header-* test-other-header' oah = 'test-header-two test-header-yes-*' hdrs = {'test-header-one': 'value', 'test-header-two': 'value', 'test-other-header': 'value', 'test-header-yes': 'value', 'test-header-yes-this': 'value'} hdrs = HeaderKeyDict(tempurl.TempURL( None, {'outgoing_remove_headers': orh, 'outgoing_allow_headers': oah} )._clean_outgoing_headers(hdrs.items())) self.assertTrue('test-header-one' not in hdrs) self.assertTrue('test-header-two' in hdrs) self.assertTrue('test-other-header' not in hdrs) self.assertTrue('test-header-yes' not in hdrs) self.assertTrue('test-header-yes-this' in hdrs)
def _clean_outgoing_headers(self, headers): """ Removes any headers as per the middleware configuration for outgoing responses. :param headers: A WSGI start_response style list of headers, [('header1', 'value), ('header2', 'value), ...] :returns: The same headers list, but with some headers removed as per the middlware configuration for outgoing responses. """ headers = HeaderKeyDict(headers) for h in headers.keys(): remove = h in self.outgoing_remove_headers if not remove: for p in self.outgoing_remove_headers_startswith: if h.startswith(p): remove = True break if remove: if h in self.outgoing_allow_headers: remove = False if remove: for p in self.outgoing_allow_headers_startswith: if h.startswith(p): remove = False break if remove: del headers[h] return headers.items()
class FakeConn(object): def __init__(self, status, headers=None, body='', **kwargs): self.status = status try: self.reason = RESPONSE_REASONS[self.status][0] except Exception: self.reason = 'Fake' self.body = body self.resp_headers = HeaderKeyDict() if headers: self.resp_headers.update(headers) self.with_exc = False self.etag = None def _update_raw_call_args(self, *args, **kwargs): capture_attrs = ('host', 'port', 'method', 'path', 'req_headers', 'query_string') for attr, value in zip(capture_attrs, args[:len(capture_attrs)]): setattr(self, attr, value) return self def getresponse(self): if self.etag: self.resp_headers['etag'] = str(self.etag.hexdigest()) if self.with_exc: raise Exception('test') return self def getheader(self, header, default=None): return self.resp_headers.get(header, default) def getheaders(self): return self.resp_headers.items() def read(self, amt=None): if amt is None: return self.body elif isinstance(self.body, six.StringIO): return self.body.read(amt) else: return Exception('Not a StringIO entry') def send(self, data): if not self.etag: self.etag = md5() self.etag.update(data)
class FakeConn(object): def __init__(self, status, headers=None, body='', **kwargs): self.status = status try: self.reason = RESPONSE_REASONS[self.status][0] except Exception: self.reason = 'Fake' self.body = body self.resp_headers = HeaderKeyDict() if headers: self.resp_headers.update(headers) self.with_exc = False self.etag = None def _update_raw_call_args(self, *args, **kwargs): capture_attrs = ('host', 'port', 'method', 'path', 'req_headers', 'query_string') for attr, value in zip(capture_attrs, args[:len(capture_attrs)]): setattr(self, attr, value) return self def getresponse(self): if self.etag: self.resp_headers['etag'] = str(self.etag.hexdigest()) if self.with_exc: raise Exception('test') return self def getheader(self, header, default=None): return self.resp_headers.get(header, default) def getheaders(self): return self.resp_headers.items() def read(self, amt=None): if isinstance(self.body, six.StringIO): return self.body.read(amt) elif amt is None: return self.body else: return Exception('Not a StringIO entry') def send(self, data): if not self.etag: self.etag = md5() self.etag.update(data)
def test_clean_outgoing_headers(self): orh = "" oah = "" hdrs = {"test-header": "value"} hdrs = HeaderKeyDict( tempurl.TempURL( None, {"outgoing_remove_headers": orh, "outgoing_allow_headers": oah} )._clean_outgoing_headers(hdrs.items()) ) self.assertTrue("test-header" in hdrs) orh = "test-header" oah = "" hdrs = {"test-header": "value"} hdrs = HeaderKeyDict( tempurl.TempURL( None, {"outgoing_remove_headers": orh, "outgoing_allow_headers": oah} )._clean_outgoing_headers(hdrs.items()) ) self.assertTrue("test-header" not in hdrs) orh = "test-header-*" oah = "" hdrs = {"test-header-one": "value", "test-header-two": "value"} hdrs = HeaderKeyDict( tempurl.TempURL( None, {"outgoing_remove_headers": orh, "outgoing_allow_headers": oah} )._clean_outgoing_headers(hdrs.items()) ) self.assertTrue("test-header-one" not in hdrs) self.assertTrue("test-header-two" not in hdrs) orh = "test-header-*" oah = "test-header-two" hdrs = {"test-header-one": "value", "test-header-two": "value"} hdrs = HeaderKeyDict( tempurl.TempURL( None, {"outgoing_remove_headers": orh, "outgoing_allow_headers": oah} )._clean_outgoing_headers(hdrs.items()) ) self.assertTrue("test-header-one" not in hdrs) self.assertTrue("test-header-two" in hdrs) orh = "test-header-* test-other-header" oah = "test-header-two test-header-yes-*" hdrs = { "test-header-one": "value", "test-header-two": "value", "test-other-header": "value", "test-header-yes": "value", "test-header-yes-this": "value", } hdrs = HeaderKeyDict( tempurl.TempURL( None, {"outgoing_remove_headers": orh, "outgoing_allow_headers": oah} )._clean_outgoing_headers(hdrs.items()) ) self.assertTrue("test-header-one" not in hdrs) self.assertTrue("test-header-two" in hdrs) self.assertTrue("test-other-header" not in hdrs) self.assertTrue("test-header-yes" not in hdrs) self.assertTrue("test-header-yes-this" in hdrs)