def test_deny_all(self):
filter = PathFilter(filter_policy=PathFilter.DENY_FILTER_ALLOW)
test_paths = [
('/neodc/sentinel1b/data/TC_Sentinel_Data_31072014.pdf', False),
('/neodc/esacci/biomass/data/agb/maps/v2.0/00README_catalogue_and_licence.txt',
False)
]
for test_path, expected in test_paths:
match = filter.allow_path(test_path)
self.assertEqual(match, expected)
class FacetScannerQueueConsumer(QueueHandler):
"""
RabbitMQ queue consumer to extract facets from the CEDA Archive for faceted search
"""
HANDLER_CLASS = FacetScannerUpdateHandler
def __init__(self, conf):
super().__init__(conf)
filter_kwargs = self.conf.get('indexer', 'path_filter', default={})
self.path_filter = PathFilter(**filter_kwargs)
def callback(self, ch, method, properties, body, connection):
"""
Callback to run during basic consume routine.
Arguments provided by pika standard message callback method
:param ch: Channel
:param method: pika method
:param properties: pika header properties
:param body: Message body
:param connection: Pika connection
"""
try:
message = self.decode_message(body)
except IndexError:
# Acknowledge message if the message is not compliant
self.acknowledge_message(ch, method.delivery_tag, connection)
return
# Check if there are any filters
allowed = self.path_filter.allow_path(message.filepath)
if not allowed:
self.acknowledge_message(ch, method.delivery_tag, connection)
return
# Try to extract the facet tags
try:
if message.action in ['DEPOSIT']:
self.queue_handler.process_event(message)
# Acknowledge message
self.acknowledge_message(ch, method.delivery_tag, connection)
except Exception as e:
# Catch all exceptions in the scanning code and log them
logger.error(f'Error occurred while scanning: {message}',
exc_info=e)
raise
class FBIQueueConsumer(QueueHandler):
"""
Provides the callback function for the FBS scanning
"""
def __init__(self, conf):
super().__init__(conf)
filter_kwargs = self.conf.get('indexer', 'path_filter', default={})
self.path_filter = PathFilter(**filter_kwargs)
def callback(self, ch, method, properties, body, connection):
"""
Callback to run during basic consume routine.
Arguments provided by pika standard message callback method
:param ch: Channel
:param method: pika method
:param properties: pika header properties
:param body: Message body
:param connection: Pika connection
"""
try:
message = self.decode_message(body)
except IndexError:
# Acknowledge message if the message is not compliant
self.acknowledge_message(ch, method.delivery_tag, connection)
return
# Check if there are any path filters
allowed = self.path_filter.allow_path(message.filepath)
if not allowed:
self.acknowledge_message(ch, method.delivery_tag, connection)
return
# Try to processs the event
try:
if message.action in ['DEPOSIT', 'REMOVE']:
self.queue_handler.process_event(message)
# Acknowledge message
self.acknowledge_message(ch, method.delivery_tag, connection)
except Exception as e:
# Catch all exceptions in the scanning code and log them
logger.error(f'Error occurred while scanning: {message}',
exc_info=e)
raise
def test_error_raised_on_invalid_mode(self):
with self.assertRaises(ValueError):
PathFilter(filter_policy=3)
def __init__(self, conf):
super().__init__(conf)
filter_kwargs = self.conf.get('indexer', 'path_filter', default={})
self.path_filter = PathFilter(**filter_kwargs)