def publish_search(self, path, req):
""" Publish a verify request on the queue to gate engine """
headers = HeaderKeyDict(req.headers)
#self.logger.debug("SWIFTSEARCH avaiable headers: %s" % (headers.items()))
# TODO(mlopezc1) is this actually faster than a regular regex with match?
# swift code loves this pattern (ex tempurl middleware) but not sure how
# will this actually perform in high use. Perf comparison later?
for k, v in headers.items():
# if header not in the whitelist of allowed full header names or *
if k not in self.index_headers:
#self.logger.debug("SWIFTSEARCH k=%s not in %s" % (k, self.index_headers))
for h in self.index_headers_startwith:
if not k.startswith(h):
#self.logger.debug("SWIFTSEARCH k=%s not allowed" % (k))
del headers[k]
self.logger.debug("SWIFTSEARCH sending metadata for indexing: %s" % (headers.items()))
# TODO(mlopez1) what about renaming keys to something more human ? the X- and title format is kinda weird
exchange = kombu.Exchange(self.exc_str, self.exc_type, durable=self.exc_durable)
queue = kombu.Queue('search', exchange=exchange, routing_key='search')
with kombu.Connection(self.conn_str) as connection:
with connection.Producer(serializer='json') as producer:
producer.publish({'id': b64encode(path), 'path': path, 'metadata': headers},
exchange=exchange, routing_key='search', declare=[queue])
return True
def test_clean_outgoing_headers(self):
orh = ''
oah = ''
hdrs = {'test-header': 'value'}
hdrs = HeaderKeyDict(tempurl.TempURL(
None,
{'outgoing_remove_headers': orh, 'outgoing_allow_headers': oah}
)._clean_outgoing_headers(hdrs.items()))
self.assertTrue('test-header' in hdrs)
orh = 'test-header'
oah = ''
hdrs = {'test-header': 'value'}
hdrs = HeaderKeyDict(tempurl.TempURL(
None,
{'outgoing_remove_headers': orh, 'outgoing_allow_headers': oah}
)._clean_outgoing_headers(hdrs.items()))
self.assertTrue('test-header' not in hdrs)
orh = 'test-header-*'
oah = ''
hdrs = {'test-header-one': 'value',
'test-header-two': 'value'}
hdrs = HeaderKeyDict(tempurl.TempURL(
None,
{'outgoing_remove_headers': orh, 'outgoing_allow_headers': oah}
)._clean_outgoing_headers(hdrs.items()))
self.assertTrue('test-header-one' not in hdrs)
self.assertTrue('test-header-two' not in hdrs)
orh = 'test-header-*'
oah = 'test-header-two'
hdrs = {'test-header-one': 'value',
'test-header-two': 'value'}
hdrs = HeaderKeyDict(tempurl.TempURL(
None,
{'outgoing_remove_headers': orh, 'outgoing_allow_headers': oah}
)._clean_outgoing_headers(hdrs.items()))
self.assertTrue('test-header-one' not in hdrs)
self.assertTrue('test-header-two' in hdrs)
orh = 'test-header-* test-other-header'
oah = 'test-header-two test-header-yes-*'
hdrs = {'test-header-one': 'value',
'test-header-two': 'value',
'test-other-header': 'value',
'test-header-yes': 'value',
'test-header-yes-this': 'value'}
hdrs = HeaderKeyDict(tempurl.TempURL(
None,
{'outgoing_remove_headers': orh, 'outgoing_allow_headers': oah}
)._clean_outgoing_headers(hdrs.items()))
self.assertTrue('test-header-one' not in hdrs)
self.assertTrue('test-header-two' in hdrs)
self.assertTrue('test-other-header' not in hdrs)
self.assertTrue('test-header-yes' not in hdrs)
self.assertTrue('test-header-yes-this' in hdrs)
def _clean_outgoing_headers(self, headers):
"""
Removes any headers as per the middleware configuration for
outgoing responses.
:param headers: A WSGI start_response style list of headers,
[('header1', 'value), ('header2', 'value),
...]
:returns: The same headers list, but with some headers
removed as per the middlware configuration for
outgoing responses.
"""
headers = HeaderKeyDict(headers)
for h in headers.keys():
remove = h in self.outgoing_remove_headers
if not remove:
for p in self.outgoing_remove_headers_startswith:
if h.startswith(p):
remove = True
break
if remove:
if h in self.outgoing_allow_headers:
remove = False
if remove:
for p in self.outgoing_allow_headers_startswith:
if h.startswith(p):
remove = False
break
if remove:
del headers[h]
return headers.items()
def _clean_outgoing_headers(self, headers):
"""
Removes any headers as per the middleware configuration for
outgoing responses.
:param headers: A WSGI start_response style list of headers,
[('header1', 'value), ('header2', 'value),
...]
:returns: The same headers list, but with some headers
removed as per the middlware configuration for
outgoing responses.
"""
headers = HeaderKeyDict(headers)
for h in headers.keys():
remove = h in self.outgoing_remove_headers
if not remove:
for p in self.outgoing_remove_headers_startswith:
if h.startswith(p):
remove = True
break
if remove:
if h in self.outgoing_allow_headers:
remove = False
if remove:
for p in self.outgoing_allow_headers_startswith:
if h.startswith(p):
remove = False
break
if remove:
del headers[h]
return headers.items()
class FakeConn(object):
def __init__(self, status, headers=None, body='', **kwargs):
self.status = status
try:
self.reason = RESPONSE_REASONS[self.status][0]
except Exception:
self.reason = 'Fake'
self.body = body
self.resp_headers = HeaderKeyDict()
if headers:
self.resp_headers.update(headers)
self.with_exc = False
self.etag = None
def _update_raw_call_args(self, *args, **kwargs):
capture_attrs = ('host', 'port', 'method', 'path', 'req_headers',
'query_string')
for attr, value in zip(capture_attrs, args[:len(capture_attrs)]):
setattr(self, attr, value)
return self
def getresponse(self):
if self.etag:
self.resp_headers['etag'] = str(self.etag.hexdigest())
if self.with_exc:
raise Exception('test')
return self
def getheader(self, header, default=None):
return self.resp_headers.get(header, default)
def getheaders(self):
return self.resp_headers.items()
def read(self, amt=None):
if amt is None:
return self.body
elif isinstance(self.body, six.StringIO):
return self.body.read(amt)
else:
return Exception('Not a StringIO entry')
def send(self, data):
if not self.etag:
self.etag = md5()
self.etag.update(data)
class FakeConn(object):
def __init__(self, status, headers=None, body='', **kwargs):
self.status = status
try:
self.reason = RESPONSE_REASONS[self.status][0]
except Exception:
self.reason = 'Fake'
self.body = body
self.resp_headers = HeaderKeyDict()
if headers:
self.resp_headers.update(headers)
self.with_exc = False
self.etag = None
def _update_raw_call_args(self, *args, **kwargs):
capture_attrs = ('host', 'port', 'method', 'path', 'req_headers',
'query_string')
for attr, value in zip(capture_attrs, args[:len(capture_attrs)]):
setattr(self, attr, value)
return self
def getresponse(self):
if self.etag:
self.resp_headers['etag'] = str(self.etag.hexdigest())
if self.with_exc:
raise Exception('test')
return self
def getheader(self, header, default=None):
return self.resp_headers.get(header, default)
def getheaders(self):
return self.resp_headers.items()
def read(self, amt=None):
if isinstance(self.body, six.StringIO):
return self.body.read(amt)
elif amt is None:
return self.body
else:
return Exception('Not a StringIO entry')
def send(self, data):
if not self.etag:
self.etag = md5()
self.etag.update(data)
def test_clean_outgoing_headers(self):
orh = ""
oah = ""
hdrs = {"test-header": "value"}
hdrs = HeaderKeyDict(
tempurl.TempURL(
None, {"outgoing_remove_headers": orh, "outgoing_allow_headers": oah}
)._clean_outgoing_headers(hdrs.items())
)
self.assertTrue("test-header" in hdrs)
orh = "test-header"
oah = ""
hdrs = {"test-header": "value"}
hdrs = HeaderKeyDict(
tempurl.TempURL(
None, {"outgoing_remove_headers": orh, "outgoing_allow_headers": oah}
)._clean_outgoing_headers(hdrs.items())
)
self.assertTrue("test-header" not in hdrs)
orh = "test-header-*"
oah = ""
hdrs = {"test-header-one": "value", "test-header-two": "value"}
hdrs = HeaderKeyDict(
tempurl.TempURL(
None, {"outgoing_remove_headers": orh, "outgoing_allow_headers": oah}
)._clean_outgoing_headers(hdrs.items())
)
self.assertTrue("test-header-one" not in hdrs)
self.assertTrue("test-header-two" not in hdrs)
orh = "test-header-*"
oah = "test-header-two"
hdrs = {"test-header-one": "value", "test-header-two": "value"}
hdrs = HeaderKeyDict(
tempurl.TempURL(
None, {"outgoing_remove_headers": orh, "outgoing_allow_headers": oah}
)._clean_outgoing_headers(hdrs.items())
)
self.assertTrue("test-header-one" not in hdrs)
self.assertTrue("test-header-two" in hdrs)
orh = "test-header-* test-other-header"
oah = "test-header-two test-header-yes-*"
hdrs = {
"test-header-one": "value",
"test-header-two": "value",
"test-other-header": "value",
"test-header-yes": "value",
"test-header-yes-this": "value",
}
hdrs = HeaderKeyDict(
tempurl.TempURL(
None, {"outgoing_remove_headers": orh, "outgoing_allow_headers": oah}
)._clean_outgoing_headers(hdrs.items())
)
self.assertTrue("test-header-one" not in hdrs)
self.assertTrue("test-header-two" in hdrs)
self.assertTrue("test-other-header" not in hdrs)
self.assertTrue("test-header-yes" not in hdrs)
self.assertTrue("test-header-yes-this" in hdrs)
