def __init__(self,
name,
parent_logger=None,
enable_service_recovery=False):
BaseService.__init__(self,
name,
parent_logger=parent_logger,
enable_service_recovery=enable_service_recovery)
self.response_parser = ResponseParser("response-parser",
parent_logger=self.log)
self.db = None
self.queue = None
class ResponseParserService(BaseService):
def __init__(self,
name,
parent_logger=None,
enable_service_recovery=False):
BaseService.__init__(self,
name,
parent_logger=parent_logger,
enable_service_recovery=enable_service_recovery)
self.response_parser = ResponseParser("response-parser",
parent_logger=self.log)
self.db = None
self.queue = None
def set_queue(self, queue):
self.queue = queue
def register(self):
self.db = self.get_directory_service_proxy().get_service(
"database-service")
self.queue = self.get_directory_service_proxy().get_service(
"queue-service")
def event_loop(self):
"""
The event loop.
"""
while self.should_loop():
resource, response = self.queue.get_publish() # pop from queue
if resource is not None and response is not None: # if an item exists
self.log.debug("found resource, parsing...")
parse_success = self.response_parser.parse(response, resource)
if parse_success:
self.queue.put_analyze(resource)
self.log.debug(
"resource id:[%s] parsed, and put on analyze queue for analysis, size: [%d]"
% (str(resource.id), self.queue.analyze_size()))
else: # serious error if we got here
# log the error and put on the publish error queue
error_msg = "Found either resource or response that was None, resource: [%s], response: [%s]"
self.log.error(error_msg % (resource, response))
self.queue.put_publish_error((resource, response))
self.log.error(
"resource put on publish error queue, size: [%d]" %
self.queue.get_publish_error())
gevent.sleep(2)
gevent.idle()
def test_response_parser():
"""
This test will execute a mock request with mock results.
:return:
"""
uri = "mock://github/events/statustest"
session = mock_requests.create_mock_session()
response = session.get(uri)
timings = ResourceTimings()
resource = Resource(uri, timings, json=True)
# parse out timings from headers
# don't worry about getting the body to publish,
# that is something the service should worry about
parser = ResponseParser("response-parser")
parser.parse(response, resource)
# asserts new timings
assert timings.interval == int(mock_requests.GLOBAL_MOCK_REQUEST_INTERVAL) * 1000
assert timings.rate_limit == int(mock_requests.GLOBAL_MOCK_REQUEST_RATELIMIT)
assert timings.rate_limit_remaining == int(mock_requests.GLOBAL_MOCK_REQUEST_REMAINING)
assert timings.etag == mock_requests.GLOBAL_MOCK_REQUEST_ETAG1
assert timings.time_to_reset == int(mock_requests.GLOBAL_MOCK_REQUEST_RESET) * 1000
class ResponseParserService(BaseService):
def __init__(self, name, parent_logger=None, enable_service_recovery=False):
BaseService.__init__(self, name, parent_logger=parent_logger, enable_service_recovery=enable_service_recovery)
self.response_parser = ResponseParser("response-parser", parent_logger=self.log)
self.db = None
self.queue = None
def set_queue(self, queue):
self.queue = queue
def register(self):
self.db = self.get_directory_service_proxy().get_service("database-service")
self.queue = self.get_directory_service_proxy().get_service("queue-service")
def event_loop(self):
"""
The event loop.
"""
while self.should_loop():
resource, response = self.queue.get_publish() # pop from queue
if resource is not None and response is not None: # if an item exists
self.log.debug("found resource, parsing...")
parse_success = self.response_parser.parse(response, resource)
if parse_success:
self.queue.put_analyze(resource)
self.log.debug("resource id:[%s] parsed, and put on analyze queue for analysis, size: [%d]" %
(str(resource.id), self.queue.analyze_size()))
else: # serious error if we got here
# log the error and put on the publish error queue
error_msg = "Found either resource or response that was None, resource: [%s], response: [%s]"
self.log.error(error_msg % (resource, response))
self.queue.put_publish_error((resource, response))
self.log.error("resource put on publish error queue, size: [%d]" %
self.queue.get_publish_error())
gevent.sleep(2)
gevent.idle()
def __init__(self, name, parent_logger=None, enable_service_recovery=False):
BaseService.__init__(self, name, parent_logger=parent_logger, enable_service_recovery=enable_service_recovery)
self.response_parser = ResponseParser("response-parser", parent_logger=self.log)
self.db = None
self.queue = None
