def __init__(self): self.cores = {} for core_name in SCHEMA.keys(): self.cores[core_name] = solr_connection(core_name) solr_version_check(core_name) # Used to define the batch size of the pending messages list try: self.batch_size = config.CFG.getint("sir", "live_index_batch_size") except (NoOptionError, AttributeError): self.batch_size = 1 # Defines how long the handler should wait before processing messages. # Used to trigger the process_message callback to prevent starvation # in pending_messages in case it doesn't fill up to batch_size try: self.process_delay = config.CFG.getint("sir", "process_delay") except (NoOptionError, AttributeError): self.process_delay = 120 logger.info("Batch size is set to %s", self.batch_size) logger.info("Process delay is set to %s seconds", self.process_delay) self.db_session = db_session() self.pending_messages = [] self.pending_entities = defaultdict(set) self.process_timer = ReusableTimer(self.process_delay, self.process_messages) self.queue_lock = Lock()
def __init__(self): self.cores = {} #: Maps entity type names to Solr cores for corename in SCHEMA.keys(): self.cores[corename] = solr_connection(corename) solr_version_check(corename) self.session = db_session( ) #: The database session used by the handler
def __init__(self, entities): self.cores = {} for core_name in entities: self.cores[core_name] = solr_connection(core_name) solr_version_check(core_name) # Used to define the batch size of the pending messages list try: self.batch_size = config.CFG.getint("sir", "live_index_batch_size") except (NoOptionError, AttributeError): self.batch_size = 1 # Defines how long the handler should wait before processing messages. # Used to trigger the process_message callback to prevent starvation # in pending_messages in case it doesn't fill up to batch_size try: self.process_delay = config.CFG.getint("sir", "process_delay") except (NoOptionError, AttributeError): self.process_delay = 120 # Used to limit the number of queried rows from PGSQL. Anything above this limit # raises a INDEX_LIMIT_EXCEEDED error try: self.index_limit = config.CFG.getint("sir", "index_limit") except (NoOptionError, AttributeError): self.index_limit = 40000 logger.info("Batch size is set to %s", self.batch_size) logger.info("Process delay is set to %s seconds", self.process_delay) logger.info("Index limit is set to %s rows", self.index_limit) self.db_session = db_session() self.pending_messages = [] self.pending_entities = defaultdict(set) self.processing = False self.channel = None self.connection = None self.last_message = time.time()
def __init__(self): self.cores = {} for core_name in SCHEMA.keys(): self.cores[core_name] = solr_connection(core_name) solr_version_check(core_name) # Used to define the batch size of the pending messages list try: self.batch_size = config.CFG.getint("sir", "live_index_batch_size") except (NoOptionError, AttributeError): self.batch_size = 1 # Defines how long the handler should wait before processing messages. # Used to trigger the process_message callback to prevent starvation # in pending_messages in case it doesn't fill up to batch_size try: self.process_delay = config.CFG.getint("sir", "process_delay") except (NoOptionError, AttributeError): self.process_delay = 120 # Used to limit the number of queried rows from PGSQL. Anything above this limit # raises a INDEX_LIMIT_EXCEEDED error try: self.index_limit = config.CFG.getint("sir", "index_limit") except (NoOptionError, AttributeError): self.index_limit = 40000 logger.info("Batch size is set to %s", self.batch_size) logger.info("Process delay is set to %s seconds", self.process_delay) logger.info("Index limit is set to %s rows", self.index_limit) self.db_session = db_session() self.pending_messages = [] self.pending_entities = defaultdict(set) self.processing = False self.channel = None self.connection = None self.last_message = time.time()
def test_matching_version(self): self.read.return_value = dumps({"version": 1.1}) util.solr_version_check("testcore")
def test_matching_version(self): self.read.return_value = dumps({"version": 1.1}) util.solr_version_check("testcore")