def fetch_search_results(self, query, offset=0, limit=0):
start_time = timeit.default_timer()
# Pre-process the query.
# We will return the query to the client, including canonical forms of
# special commands.
return_query, fts_query, suffixes_query, suffixes_full_terms = \
self._process_query(query)
logger.info("Searching -- FTS:'{}', Suffixes:'{}'.".format(
fts_query, suffixes_query))
try:
# Get the results for the query, hitting the memoisation caches
# if we can.
count = self._cached_search_results_count(fts_query,
suffixes_query,
suffixes_full_terms)
results = self._cached_search_results(fts_query, suffixes_query,
suffixes_full_terms, offset,
limit)
elapsed = "{:.3f}".format(timeit.default_timer() - start_time)
return {
'total': count,
'results': results,
'query': return_query,
'elapsed': elapsed,
'offset': offset
}
except (sqlalchemy.exc.SQLAlchemyError,
oce.exceptions.CustomError) as e:
# Whoops.
logger.error(e)
return {'total': 0, 'results': 'error'}
def _receive_to_queue(self):
try:
while True:
msg = yield from self.websocket.recv()
if msg is None:
logger.info("[{}] Client connection closed.".format(
self.websocket.remote_ip))
break
# Attempt to parse JSON
try:
msg = json.loads(msg)
except ValueError:
logger.error("[{}] Bad input from client. "
"(Could not parse JSON)".format(
self.websocket.remote_ip))
break
yield from self.input_queue.put(msg)
logger.info("[{}] [RECV] {}".format(self.websocket.remote_ip,
msg))
except CancelledError:
logger.debug("[{}] CancelledError on receiver -- "
"Should not be happening.".format(
self.websocket.remote_ip))
def _receive_to_queue(self):
try:
while True:
msg = yield from self.websocket.recv()
if msg is None:
logger.info(
"[{}] Client connection closed.".format(
self.websocket.remote_ip)
)
break
# Attempt to parse JSON
try:
msg = json.loads(msg)
except ValueError:
logger.error(
"[{}] Bad input from client. "
"(Could not parse JSON)".format(
self.websocket.remote_ip)
)
break
yield from self.input_queue.put(msg)
logger.info("[{}] [RECV] {}".format(
self.websocket.remote_ip,
msg)
)
except CancelledError:
logger.debug(
"[{}] CancelledError on receiver -- "
"Should not be happening.".format(self.websocket.remote_ip)
)
def execute_retag(self, old_tag, new_tag):
"""
Replaces the old tag with the new one on all applicable records.
"""
if re.search(r',', "{}{}".format(old_tag, new_tag)):
logger.error("Asked to create a tag that contains a comma. "
"Aborting.")
return 'error'
search = self.fetch_search_results("tag:{}".format(old_tag))
records = [(x['rowid'], x['tag']) for x in search['results']]
for rowid, tag in records:
new_tags = tag.split(",")
new_tags[new_tags.index(old_tag)] = new_tag
new_tags = ','.join(new_tags)
self.update_record(rowid, 'tag', new_tags)
return 'success'
def _send_from_queue(self):
try:
while True:
msg = yield from self.output_queue.get()
msg = json.dumps(msg)
msg_preview = msg[0:80]
msg = base64.b64encode(zlib.compress(msg.encode())).decode()
if not self.websocket.open:
logger.error(
"[{}] Send error: Socket closed unexpectedly.".format(
self.websocket.remote_ip))
break
yield from self.websocket.send(msg)
logger.info("[{}] [SEND] {}...".format(
self.websocket.remote_ip, msg_preview))
except CancelledError:
logger.debug("[{}] Cancelling sender...".format(
self.websocket.remote_ip))
def execute_orm_filter(self, where_conditions, table="Records"):
"""
Executes a select on the specified table with a literal where clause.
Uses the SQLAlchemy ORM, so results are dictionaries.
"""
try:
_table = globals()[table]
if not isinstance(_table,
sqlalchemy.ext.declarative.api.DeclarativeMeta):
raise KeyError
except KeyError:
# The specified table wasn't properly defined.
logger.error(
"Could not find ORM mappings for table: {}".format(table))
return "error"
return [
row.dictionary for row in self.session.query(_table).filter(
sqlalchemy.sql.expression.text(where_conditions))
]
def update_record(self, row_id, field, value):
try:
for row in self.session.query(Records) \
.filter(Records.rowid == row_id):
if not hasattr(row, field):
logger.warning(
"Invalid field name given by client: '{}'".format(
field))
return 'invalid_field'
# ====================
# Field-specific hooks
# ====================
# Process record tags to update tweets_tags table
if field == 'tag':
old_tags = fts_detag('tag', row.tag)
self._update_tags(value, old_tags)
elif field == 'language':
value = langid_normalise_language(value)
# ====================
# Add FTS tags if needed
value = fts_tag(field, value)
logger.info("Updating '{0}' on row {1}: {2} -> {3}".format(
field, row.rowid,
str(getattr(row, field)).replace('\n', '\\n'),
str(value).replace('\n', '\\n')))
setattr(row, field, value)
self.session.commit()
# Also clear all memoisation caches, in case the update
# invalidates their results
self._clear_caches()
return 'success'
except sqlalchemy.exc.SQLAlchemyError as e:
# Uh oh.
logger.error(e)
return 'error'
def _send_from_queue(self):
try:
while True:
msg = yield from self.output_queue.get()
msg = json.dumps(msg)
msg_preview = msg[0:80]
msg = base64.b64encode(zlib.compress(msg.encode())).decode()
if not self.websocket.open:
logger.error(
"[{}] Send error: Socket closed unexpectedly.".format(
self.websocket.remote_ip))
break
yield from self.websocket.send(msg)
logger.info("[{}] [SEND] {}...".format(
self.websocket.remote_ip,
msg_preview)
)
except CancelledError:
logger.debug("[{}] Cancelling sender...".format(
self.websocket.remote_ip))
