def __init__(self, name, parent_logger=None, enable_service_recovery=False): BaseService.__init__(self, name, parent_logger=parent_logger, enable_service_recovery=enable_service_recovery) self.response_parser = ResponseParser("response-parser", parent_logger=self.log) self.db = None self.queue = None
class ResponseParserService(BaseService): def __init__(self, name, parent_logger=None, enable_service_recovery=False): BaseService.__init__(self, name, parent_logger=parent_logger, enable_service_recovery=enable_service_recovery) self.response_parser = ResponseParser("response-parser", parent_logger=self.log) self.db = None self.queue = None def set_queue(self, queue): self.queue = queue def register(self): self.db = self.get_directory_service_proxy().get_service( "database-service") self.queue = self.get_directory_service_proxy().get_service( "queue-service") def event_loop(self): """ The event loop. """ while self.should_loop(): resource, response = self.queue.get_publish() # pop from queue if resource is not None and response is not None: # if an item exists self.log.debug("found resource, parsing...") parse_success = self.response_parser.parse(response, resource) if parse_success: self.queue.put_analyze(resource) self.log.debug( "resource id:[%s] parsed, and put on analyze queue for analysis, size: [%d]" % (str(resource.id), self.queue.analyze_size())) else: # serious error if we got here # log the error and put on the publish error queue error_msg = "Found either resource or response that was None, resource: [%s], response: [%s]" self.log.error(error_msg % (resource, response)) self.queue.put_publish_error((resource, response)) self.log.error( "resource put on publish error queue, size: [%d]" % self.queue.get_publish_error()) gevent.sleep(2) gevent.idle()
def test_response_parser(): """ This test will execute a mock request with mock results. :return: """ uri = "mock://github/events/statustest" session = mock_requests.create_mock_session() response = session.get(uri) timings = ResourceTimings() resource = Resource(uri, timings, json=True) # parse out timings from headers # don't worry about getting the body to publish, # that is something the service should worry about parser = ResponseParser("response-parser") parser.parse(response, resource) # asserts new timings assert timings.interval == int(mock_requests.GLOBAL_MOCK_REQUEST_INTERVAL) * 1000 assert timings.rate_limit == int(mock_requests.GLOBAL_MOCK_REQUEST_RATELIMIT) assert timings.rate_limit_remaining == int(mock_requests.GLOBAL_MOCK_REQUEST_REMAINING) assert timings.etag == mock_requests.GLOBAL_MOCK_REQUEST_ETAG1 assert timings.time_to_reset == int(mock_requests.GLOBAL_MOCK_REQUEST_RESET) * 1000
class ResponseParserService(BaseService): def __init__(self, name, parent_logger=None, enable_service_recovery=False): BaseService.__init__(self, name, parent_logger=parent_logger, enable_service_recovery=enable_service_recovery) self.response_parser = ResponseParser("response-parser", parent_logger=self.log) self.db = None self.queue = None def set_queue(self, queue): self.queue = queue def register(self): self.db = self.get_directory_service_proxy().get_service("database-service") self.queue = self.get_directory_service_proxy().get_service("queue-service") def event_loop(self): """ The event loop. """ while self.should_loop(): resource, response = self.queue.get_publish() # pop from queue if resource is not None and response is not None: # if an item exists self.log.debug("found resource, parsing...") parse_success = self.response_parser.parse(response, resource) if parse_success: self.queue.put_analyze(resource) self.log.debug("resource id:[%s] parsed, and put on analyze queue for analysis, size: [%d]" % (str(resource.id), self.queue.analyze_size())) else: # serious error if we got here # log the error and put on the publish error queue error_msg = "Found either resource or response that was None, resource: [%s], response: [%s]" self.log.error(error_msg % (resource, response)) self.queue.put_publish_error((resource, response)) self.log.error("resource put on publish error queue, size: [%d]" % self.queue.get_publish_error()) gevent.sleep(2) gevent.idle()