def _index_by_pk(self, parsed_message):
for core_name, path in update_map[parsed_message.table_name]:
# Going through each core/entity that needs to be updated
# depending on which table we receive a message from.
entity = SCHEMA[core_name]
with db_session_ctx(self.db_session) as session:
if path is None:
# If `path` is `None` then we received a message for an entity itself
ids = [parsed_message.columns["id"]]
else:
# otherwise it's a different table...
logger.info(
"Generating SELECT statement for %s with path '%s'" %
(entity.model, path))
select_query = generate_filtered_query(
entity.model, path, parsed_message.columns)
if select_query is None:
logger.warning("SELECT is `None`")
continue
else:
logger.debug("SQL: %s" % select_query)
ids = [
row[0] for row in session.execute(
select_query).fetchall()
]
# Retrieving actual data
self._index_data(core_name, ids)
def _index_by_pk(self, parsed_message):
for core_name, path in update_map[parsed_message.table_name]:
# Going through each core/entity that needs to be updated
# depending on which table we receive a message from.
entity = SCHEMA[core_name]
with db_session_ctx(self.db_session) as session:
select_query = None
if path is None:
# If `path` is `None` then we received a message for an entity itself
ids = [parsed_message.columns["id"]]
else:
# otherwise it's a different table...
logger.debug("Generating SELECT statement for %s with path '%s'" % (entity.model, path))
select_query = generate_filtered_query(entity.model, path, parsed_message.columns)
if select_query is None:
logger.warning("SELECT is `None`")
continue
else:
logger.debug("SQL: %s" % select_query)
ids = [row[0] for row in session.execute(select_query).fetchall()]
# Retrieving actual data
extra_data = {'table_name': parsed_message.table_name,
'path': path,
'select_query': str(select_query),
}
self._index_data(core_name, ids, extra_data)
def _index_by_fk(self, parsed_message):
index_model = model_map[parsed_message.table_name]
# We need to construct this since we only need to update tables which
# have 'many to one' relationship with the table represented by `index_model`,
# since index_by_fk is only called when an entity is deleted and we need
# to update the related entities. For 'one to many' relationships, the related
# entity would have had an update trigger firing off to unlink the `index_entity`
# before `index_entity` itself is deleted, so we can ignore those.
relevant_rels = dict((r.table.name, (list(r.local_columns)[0].name, list(r.remote_side)[0]))
for r in class_mapper(index_model).mapper.relationships
if r.direction.name == 'MANYTOONE')
for core_name, path in update_map[parsed_message.table_name]:
if not core_name in self.cores:
continue
# Going through each core/entity that needs to be updated
# depending on original index model
entity = SCHEMA[core_name]
# We are finding the second last model in path, since the rows related to
# `index_model` are deleted and the sql query generated from that path
# returns no ids, because of the way select query is generated.
# We generate sql queries with the second last model in path, since that
# will be related to the `index_model` by a FK and we can thus determine
# the tables to be updated from the FK values emitted by the delete triggers
related_model, new_path = second_last_model_in_path(entity.model, path)
related_table_name = ""
if related_model:
related_table_name = class_mapper(related_model).mapped_table.name
if related_table_name in relevant_rels:
with db_session_ctx(self.db_session) as session:
select_query = None
if new_path is None:
# If `path` is `None` then we received a message for an entity itself
ids = [parsed_message.columns['id']]
else:
logger.debug("Generating SELECT statement for %s with path '%s'" % (entity.model, new_path))
fk_name, remote_key = relevant_rels[related_table_name]
filter_expression = remote_key.__eq__(parsed_message.columns[fk_name])
# If `new_path` is blank, then the given table, was directly related to the
# `index_model` by a FK.
select_query = generate_query(entity.model, new_path, filter_expression)
if select_query is None:
logger.warning("SELECT is `None`")
continue
ids = [row[0] for row in session.execute(select_query).fetchall()]
logger.debug("SQL: %s" % (select_query))
# Retrieving actual data
extra_data = {'table_name': parsed_message.table_name,
'path': path,
'related_table_name': related_table_name,
'new_path': new_path,
'select_query': str(select_query),
}
self._index_data(core_name, ids, extra_data)
def _index_by_fk(self, parsed_message):
index_model = model_map[parsed_message.table_name]
# We need to construct this since we only need to update tables which
# have 'many to one' relationship with the table represented by `index_model`,
# since index_by_fk is only called when an entity is deleted and we need
# to update the related entities. For 'one to many' relationships, the related
# entity would have had an update trigger firing off to unlink the `index_entity`
# before `index_entity` itself is deleted, so we can ignore those.
relevant_rels = dict((r.table.name, (list(r.local_columns)[0].name, list(r.remote_side)[0]))
for r in class_mapper(index_model).mapper.relationships
if r.direction.name == 'MANYTOONE')
for core_name, path in update_map[parsed_message.table_name]:
# Going through each core/entity that needs to be updated
# depending on original index model
entity = SCHEMA[core_name]
# We are finding the second last model in path, since the rows related to
# `index_model` are deleted and the sql query generated from that path
# returns no ids, because of the way select query is generated.
# We generate sql queries with the second last model in path, since that
# will be related to the `index_model` by a FK and we can thus determine
# the tables to be updated from the FK values emitted by the delete triggers
related_model, new_path = second_last_model_in_path(entity.model, path)
related_table_name = ""
if related_model:
related_table_name = class_mapper(related_model).mapped_table.name
if related_table_name in relevant_rels:
with db_session_ctx(self.db_session) as session:
select_query = None
if new_path is None:
# If `path` is `None` then we received a message for an entity itself
ids = [parsed_message.columns['id']]
else:
logger.debug("Generating SELECT statement for %s with path '%s'" % (entity.model, new_path))
fk_name, remote_key = relevant_rels[related_table_name]
filter_expression = remote_key.__eq__(parsed_message.columns[fk_name])
# If `new_path` is blank, then the given table, was directly related to the
# `index_model` by a FK.
select_query = generate_query(entity.model, new_path, filter_expression)
if select_query is None:
logger.warning("SELECT is `None`")
continue
ids = [row[0] for row in session.execute(select_query).fetchall()]
logger.debug("SQL: %s" % (select_query))
# Retrieving actual data
extra_data = {'table_name': parsed_message.table_name,
'path': path,
'related_table_name': related_table_name,
'new_path': new_path,
'select_query': str(select_query),
}
self._index_data(core_name, ids, extra_data)
def index_callback(self, parsed_message):
logger.debug("Processing `index` message for entity: %s" %
parsed_message.entity_type)
entity = SCHEMA[parsed_message.entity_type]
converter = entity.query_result_to_dict
query = entity.query
condition = and_(entity.model.id.in_(parsed_message.ids))
with db_session_ctx(self.session) as session:
query = query.filter(condition).with_session(session)
send_data_to_solr(self.cores[parsed_message.entity_type],
[converter(obj) for obj in query.all()])
def publish(self):
try:
messages_pending = True
while messages_pending:
with db_session_ctx(self.db_session) as session:
for msg in self._sir_message_query(session).limit(
_SIR_MESSAGE_BATCH_SIZE):
logger.debug("Publishing sid.message ID: %s", msg.id)
try:
self._queue_message(msg.to_amqp_message(),
msg.exchange, msg.routing_key)
session.delete(msg)
except Exception as e:
logger.error(format_exc(e))
raise
num_remaining = self._sir_message_query(session).count()
logger.info("%s messages pending", num_remaining)
messages_pending = _PROCESS_FLAG and num_remaining > 0
except Exception as e:
logger.error(format_exc(e))
