def test_index_by_fk_1(self): columns = {'id': '1', 'area': '2', 'type': '3'} parsed_message = Message(1, 'area_alias', columns, 'delete') handler.SCHEMA = SCHEMA self.handler = handler.Handler(SCHEMA.keys()) for entity_type, entity in SCHEMA.items(): self.handler.cores[entity_type] = mock.Mock() self.handler._index_by_fk(parsed_message) calls = self.handler.db_session().execute.call_args_list self.assertEqual(len(calls), 6) actual_queries = [str(call[0][0]) for call in calls] expected_queries = [ 'SELECT place_1.id AS place_1_id \n' 'FROM musicbrainz.place AS place_1 JOIN musicbrainz.area ON musicbrainz.area.id = place_1.area \n' 'WHERE musicbrainz.area.id = :id_1', 'SELECT label_1.id AS label_1_id \n' 'FROM musicbrainz.label AS label_1 JOIN musicbrainz.area ON musicbrainz.area.id = label_1.area \n' 'WHERE musicbrainz.area.id = :id_1', 'SELECT artist_1.id AS artist_1_id \n' 'FROM musicbrainz.artist AS artist_1 JOIN musicbrainz.area ON musicbrainz.area.id = artist_1.end_area \n' 'WHERE musicbrainz.area.id = :id_1', 'SELECT artist_1.id AS artist_1_id \n' 'FROM musicbrainz.artist AS artist_1 JOIN musicbrainz.area ON musicbrainz.area.id = artist_1.area \n' 'WHERE musicbrainz.area.id = :id_1', 'SELECT artist_1.id AS artist_1_id \n' 'FROM musicbrainz.artist AS artist_1 JOIN musicbrainz.area ON musicbrainz.area.id = artist_1.begin_area \n' 'WHERE musicbrainz.area.id = :id_1', 'SELECT musicbrainz.area.id AS musicbrainz_area_id \n' 'FROM musicbrainz.area \n' 'WHERE musicbrainz.area.id = :id_1' ] self.assertEqual(expected_queries, actual_queries)
def __init__(self): self.cores = {} for core_name in SCHEMA.keys(): self.cores[core_name] = solr_connection(core_name) solr_version_check(core_name) # Used to define the batch size of the pending messages list try: self.batch_size = config.CFG.getint("sir", "live_index_batch_size") except (NoOptionError, AttributeError): self.batch_size = 1 # Defines how long the handler should wait before processing messages. # Used to trigger the process_message callback to prevent starvation # in pending_messages in case it doesn't fill up to batch_size try: self.process_delay = config.CFG.getint("sir", "process_delay") except (NoOptionError, AttributeError): self.process_delay = 120 logger.info("Batch size is set to %s", self.batch_size) logger.info("Process delay is set to %s seconds", self.process_delay) self.db_session = db_session() self.pending_messages = [] self.pending_entities = defaultdict(set) self.process_timer = ReusableTimer(self.process_delay, self.process_messages) self.queue_lock = Lock()
def __init__(self): self.cores = {} #: Maps entity type names to Solr cores for corename in SCHEMA.keys(): self.cores[corename] = solr_connection(corename) solr_version_check(corename) self.session = db_session( ) #: The database session used by the handler
def generate_func(args): """ This is the entry point for this trigger_generation module. This function gets called from :func:`~sir.__main__.main`. """ generate(trigger_filename=args["trigger_file"], function_filename=args["function_file"], broker_id=args["broker_id"], entities=args["entity_type"] or SCHEMA.keys())
def test_index_by_fk_3(self): columns = {'release_group': 1} parsed_message = Message(1, 'release', columns, 'delete') handler.SCHEMA = SCHEMA self.handler = handler.Handler(SCHEMA.keys()) for entity_type, entity in SCHEMA.items(): self.handler.cores[entity_type] = mock.Mock() self.handler._index_by_fk(parsed_message) calls = self.handler.db_session().execute.call_args_list self.assertEqual(len(calls), 1) actual_queries = [str(call[0][0]) for call in calls] expected_queries = [ 'SELECT musicbrainz.release_group.id AS musicbrainz_release_group_id \n' 'FROM musicbrainz.release_group \n' 'WHERE musicbrainz.release_group.id = :id_1' ] self.assertEqual(expected_queries, actual_queries)
def watch(args): """ Watch AMQP queues for messages. :param [str] entity_type: Entity types to watch. """ try: create_amqp_connection() except socket_error as e: logger.error("Couldn't connect to RabbitMQ, check your settings. %s", e) exit(1) try: entities = args["entity_type"] or SCHEMA.keys() _watch_impl(entities) except URLError as e: logger.error("Connecting to Solr failed: %s", e) exit(1)
def from_amqp_message(cls, queue, amqp_message): """ Parses an AMQP message. :param str queue: The queue name :param amqp.basic_message.Message amqp_message: :rtype: :class:`sir.amqp.message.Message` :raises sir.amqp.message.InvalidMessageContentException: If the message content could not be parsed :raises ValueError: If the entity type in the message was invalid or any of the IDs was not numeric (in case ``type`` is not :data:`MESSAGE_TYPES.delete`) or the queue is unknown """ if queue not in QUEUE_TO_TYPE.keys(): raise ValueError("%s is not a valid queue name" % queue) else: message_type = QUEUE_TO_TYPE[queue] dbg_msg = amqp_message.body if len(dbg_msg) > 20: dbg_msg = dbg_msg[:20] + "..." logger.debug("Recieved message from queue %s: %s" % (queue, dbg_msg)) split_message = amqp_message.body.split(" ") if not len(split_message) >= 2: raise InvalidMessageContentException("AMQP messages must at least " "contain 2 entries separated" " by spaces") entity_type = split_message[0].replace("_", "-") if entity_type == "release-raw": # See https://git.io/vDcdo entity_type = "cdstub" if entity_type not in SCHEMA.keys(): raise ValueError("Received a message with the invalid entity type " "%s" % entity_type) ids = split_message[1:] return cls(message_type, entity_type, ids)
def __init__(self): self.cores = {} for core_name in SCHEMA.keys(): self.cores[core_name] = solr_connection(core_name) solr_version_check(core_name) # Used to define the batch size of the pending messages list try: self.batch_size = config.CFG.getint("sir", "live_index_batch_size") except (NoOptionError, AttributeError): self.batch_size = 1 # Defines how long the handler should wait before processing messages. # Used to trigger the process_message callback to prevent starvation # in pending_messages in case it doesn't fill up to batch_size try: self.process_delay = config.CFG.getint("sir", "process_delay") except (NoOptionError, AttributeError): self.process_delay = 120 # Used to limit the number of queried rows from PGSQL. Anything above this limit # raises a INDEX_LIMIT_EXCEEDED error try: self.index_limit = config.CFG.getint("sir", "index_limit") except (NoOptionError, AttributeError): self.index_limit = 40000 logger.info("Batch size is set to %s", self.batch_size) logger.info("Process delay is set to %s seconds", self.process_delay) logger.info("Index limit is set to %s rows", self.index_limit) self.db_session = db_session() self.pending_messages = [] self.pending_entities = defaultdict(set) self.processing = False self.channel = None self.connection = None self.last_message = time.time()
def __init__(self): self.cores = {} for core_name in SCHEMA.keys(): self.cores[core_name] = solr_connection(core_name) solr_version_check(core_name) # Used to define the batch size of the pending messages list try: self.batch_size = config.CFG.getint("sir", "live_index_batch_size") except (NoOptionError, AttributeError): self.batch_size = 1 # Defines how long the handler should wait before processing messages. # Used to trigger the process_message callback to prevent starvation # in pending_messages in case it doesn't fill up to batch_size try: self.process_delay = config.CFG.getint("sir", "process_delay") except (NoOptionError, AttributeError): self.process_delay = 120 # Used to limit the number of queried rows from PGSQL. Anything above this limit # raises a INDEX_LIMIT_EXCEEDED error try: self.index_limit = config.CFG.getint("sir", "index_limit") except (NoOptionError, AttributeError): self.index_limit = 40000 logger.info("Batch size is set to %s", self.batch_size) logger.info("Process delay is set to %s seconds", self.process_delay) logger.info("Index limit is set to %s rows", self.index_limit) self.db_session = db_session() self.pending_messages = [] self.pending_entities = defaultdict(set) self.processing = False self.channel = None self.connection = None self.last_message = time.time()
def setUp(self): super(CallbackWrapperTest, self).setUp() self.handler = handler.Handler(SCHEMA.keys()) self.channel = self.handler.channel = mock.MagicMock() self.handler.connection = mock.MagicMock()
def main(): parser = argparse.ArgumentParser() parser.add_argument("-d", "--debug", action="store_true") parser.add_argument("--sqltimings", action="store_true") subparsers = parser.add_subparsers() reindex_parser = subparsers.add_parser("reindex", help="Reindexes all or a single " "entity type") reindex_parser.set_defaults(func=reindex) reindex_parser.add_argument('--entity-type', action='append', help="Which entity types to index.", choices=SCHEMA.keys()) generate_trigger_parser = subparsers.add_parser("triggers", help="Generate triggers") generate_trigger_parser.set_defaults(func=generate_triggers) generate_trigger_parser.add_argument('-t', '--trigger-file', action="store", default="sql/CreateTriggers.sql", help="The filename to save the " "triggers into") generate_trigger_parser.add_argument('-f', '--function-file', action="store", default="sql/CreateFunctions.sql", help="The filename to save the " "functions into") generate_trigger_parser.add_argument('-bid', '--broker-id', action="store", default="1", help="ID of the AMQP broker row " "in the database.") amqp_setup_parser = subparsers.add_parser("amqp_setup", help="Set up AMQP exchanges and " "queues") amqp_setup_parser.set_defaults(func=setup_rabbitmq) amqp_watch_parser = subparsers.add_parser("amqp_watch", help="Watch AMQP queues for " "changes") amqp_watch_parser.set_defaults(func=watch) args = parser.parse_args() if args.debug: logger.setLevel(logging.DEBUG) else: logger.setLevel(logging.INFO) loghandler = logging.StreamHandler() if args.debug: formatter = logging.Formatter(fmt="%(processName)s %(asctime)s " "%(levelname)s: %(message)s") else: formatter = logging.Formatter(fmt="%(asctime)s: %(message)s") loghandler.setFormatter(formatter) logger.addHandler(loghandler) mplogger = multiprocessing.get_logger() mplogger.setLevel(logging.ERROR) mplogger.addHandler(loghandler) if args.sqltimings: from sqlalchemy import event from sqlalchemy.engine import Engine import time sqltimelogger = logging.getLogger("sqltimer") sqltimelogger.setLevel(logging.DEBUG) sqltimelogger.addHandler(loghandler) @event.listens_for(Engine, "before_cursor_execute") def before_cursor_execute(conn, cursor, statement, parameters, context, executemany): conn.info.setdefault('query_start_time', []).append(time.time()) sqltimelogger.debug("Start Query: %s", statement) @event.listens_for(Engine, "after_cursor_execute") def after_cursor_execute(conn, cursor, statement, parameters, context, executemany): total = time.time() - conn.info['query_start_time'].pop(-1) sqltimelogger.debug("Query Complete!") sqltimelogger.debug("Total Time: %f", total) config.read_config() try: init_raven_client(config.CFG.get("sentry", "dsn")) except ConfigParser.Error as e: logger.info("Skipping Raven client initialization. Configuration issue: %s", e) func = args.func args = vars(args) func(args)