def test_account_delta_for_different_namespace_id(db):
from inbox.transactions.delta_sync import format_transactions_after_pointer
account = add_account_with_different_namespace_id(db.session)
namespace = account.namespace
# Create an `account` delta
account.sync_state = "invalid"
db.session.commit()
# Verify `account` delta is not returned when exclude_account=True
txns, _ = format_transactions_after_pointer(namespace,
0,
db.session,
10,
exclude_account=True)
assert not txns
# Verify `account` delta is returned when exclude_account=False
txns, _ = format_transactions_after_pointer(namespace,
0,
db.session,
10,
exclude_account=False)
assert txns
def pull():
start_time = time.time()
pointer = int(request.values.get('pointer'))
result_limit = int(
request.values.get('limit')) if 'limit' in request.values else 100
output = {}
if result_limit < 0 or result_limit > 10000:
return default_json_error('Invalid value for result_lmit')
output['pointer_start'] = pointer
with session_scope(0) as db_session:
deltas, pointer_end = delta_sync.format_transactions_after_pointer(
None,
pointer,
db_session,
result_limit,
filter_for_namespace=False)
output['deltas'] = deltas
output['pointer_end'] = pointer_end
time_diff = (time.time() - start_time)
output['runtime'] = time_diff
return jsonify(output)
def sync_deltas():
g.parser.add_argument('cursor', type=valid_public_id, location='args',
required=True)
args = strict_parse_args(g.parser, request.args)
cursor = args['cursor']
if cursor == '0':
start_pointer = 0
else:
try:
start_pointer, = g.db_session.query(Transaction.id). \
filter(Transaction.public_id == cursor,
Transaction.namespace_id == g.namespace.id).one()
except NoResultFound:
return err(404, 'Invalid cursor parameter')
deltas, _ = delta_sync.format_transactions_after_pointer(
g.namespace.id, start_pointer, g.db_session, args['limit'])
response = {
'cursor_start': cursor,
'deltas': deltas,
}
if deltas:
response['cursor_end'] = deltas[-1]['cursor']
else:
# No changes.
response['cursor_end'] = cursor
return g.encoder.jsonify(response)
def sync_deltas():
g.parser.add_argument('cursor',
type=valid_public_id,
location='args',
required=True)
g.parser.add_argument('exclude_types',
type=valid_delta_object_types,
location='args')
args = strict_parse_args(g.parser, request.args)
cursor = args['cursor']
if cursor == '0':
start_pointer = 0
else:
try:
start_pointer, = g.db_session.query(Transaction.id). \
filter(Transaction.public_id == cursor,
Transaction.namespace_id == g.namespace.id).one()
except NoResultFound:
raise InputError('Invalid cursor parameter')
exclude_types = args.get('exclude_types')
deltas, _ = delta_sync.format_transactions_after_pointer(
g.namespace.id, start_pointer, g.db_session, args['limit'],
delta_sync._format_transaction_for_delta_sync, exclude_types)
response = {
'cursor_start': cursor,
'deltas': deltas,
}
if deltas:
response['cursor_end'] = deltas[-1]['cursor']
else:
# No changes.
response['cursor_end'] = cursor
return g.encoder.jsonify(response)
def sync_deltas():
g.parser.add_argument('cursor', type=valid_public_id, location='args',
required=True)
g.parser.add_argument('exclude_types', type=valid_delta_object_types,
location='args')
g.parser.add_argument('include_types', type=valid_delta_object_types,
location='args')
g.parser.add_argument('timeout', type=int,
default=LONG_POLL_REQUEST_TIMEOUT, location='args')
# TODO(emfree): should support `expand` parameter in delta endpoints.
args = strict_parse_args(g.parser, request.args)
exclude_types = args.get('exclude_types')
include_types = args.get('include_types')
cursor = args['cursor']
timeout = args['timeout']
if include_types and exclude_types:
return err(400, "Invalid Request. Cannot specify both include_types"
"and exclude_types")
if cursor == '0':
start_pointer = 0
else:
try:
start_pointer, = g.db_session.query(Transaction.id). \
filter(Transaction.public_id == cursor,
Transaction.namespace_id == g.namespace.id).one()
except NoResultFound:
raise InputError('Invalid cursor parameter')
# The client wants us to wait until there are changes
g.db_session.close() # hack to close the flask session
poll_interval = 1
start_time = time.time()
while time.time() - start_time < timeout:
with session_scope() as db_session:
deltas, _ = delta_sync.format_transactions_after_pointer(
g.namespace, start_pointer, db_session, args['limit'],
exclude_types, include_types)
response = {
'cursor_start': cursor,
'deltas': deltas,
}
if deltas:
response['cursor_end'] = deltas[-1]['cursor']
return g.encoder.jsonify(response)
# No changes. perhaps wait
elif '/delta/longpoll' in request.url_rule.rule:
gevent.sleep(poll_interval)
else: # Return immediately
response['cursor_end'] = cursor
return g.encoder.jsonify(response)
# If nothing happens until timeout, just return the end of the cursor
response['cursor_end'] = cursor
return g.encoder.jsonify(response)
def sync_deltas():
g.parser.add_argument('cursor', type=valid_public_id, location='args',
required=True)
g.parser.add_argument('exclude_types', type=valid_delta_object_types,
location='args')
g.parser.add_argument('include_types', type=valid_delta_object_types,
location='args')
g.parser.add_argument('timeout', type=int,
default=LONG_POLL_REQUEST_TIMEOUT, location='args')
# TODO(emfree): should support `expand` parameter in delta endpoints.
args = strict_parse_args(g.parser, request.args)
exclude_types = args.get('exclude_types')
include_types = args.get('include_types')
cursor = args['cursor']
timeout = args['timeout']
if include_types and exclude_types:
return err(400, "Invalid Request. Cannot specify both include_types"
"and exclude_types")
if cursor == '0':
start_pointer = 0
else:
try:
start_pointer, = g.db_session.query(Transaction.id). \
filter(Transaction.public_id == cursor,
Transaction.namespace_id == g.namespace.id).one()
except NoResultFound:
raise InputError('Invalid cursor parameter')
# The client wants us to wait until there are changes
g.db_session.close() # hack to close the flask session
poll_interval = 1
start_time = time.time()
while time.time() - start_time < timeout:
with session_scope() as db_session:
deltas, _ = delta_sync.format_transactions_after_pointer(
g.namespace, start_pointer, db_session, args['limit'],
exclude_types, include_types)
response = {
'cursor_start': cursor,
'deltas': deltas,
}
if deltas:
response['cursor_end'] = deltas[-1]['cursor']
return g.encoder.jsonify(response)
# No changes. perhaps wait
elif '/delta/longpoll' in request.url_rule.rule:
gevent.sleep(poll_interval)
else: # Return immediately
response['cursor_end'] = cursor
return g.encoder.jsonify(response)
# If nothing happens until timeout, just return the end of the cursor
response['cursor_end'] = cursor
return g.encoder.jsonify(response)
def test_account_delta_for_different_namespace_id(db):
from inbox.transactions.delta_sync import format_transactions_after_pointer
account = add_account_with_different_namespace_id(db.session)
namespace = account.namespace
# Create an `account` delta
account.sync_state = 'invalid'
db.session.commit()
# Verify `account` delta is not returned when exclude_account=True
txns, _ = format_transactions_after_pointer(namespace, 0, db.session, 10,
exclude_account=True)
assert not txns
# Verify `account` delta is returned when exclude_account=False
txns, _ = format_transactions_after_pointer(namespace, 0, db.session, 10,
exclude_account=False)
assert txns
def _run(self):
"""
Index into Elasticsearch the threads, messages of all namespaces.
"""
# Indexing is namespace agnostic.
# Note that although this means we do not restrict the Transaction
# table query (via the format_transactions_after_pointer() call below)
# to a namespace, since we pass a `result_limit` (== chunk_size)
# argument, the query should still be performant.
namespace_id = None
# Only index messages, threads.
object_types = transaction_objects()
exclude_types = [
api_name for model_name, api_name in object_types.iteritems()
if model_name not in ['message', 'thread']
]
with session_scope() as db_session:
pointer = db_session.query(SearchIndexCursor).first()
self.transaction_pointer = pointer.transaction_id if pointer else 0
self.log.info('Starting search-index service',
transaction_pointer=self.transaction_pointer)
while True:
with session_scope() as db_session:
deltas, new_pointer = format_transactions_after_pointer(
namespace_id, self.transaction_pointer, db_session,
self.chunk_size, _format_transaction_for_search,
exclude_types)
# TODO[k]: We ideally want to index chunk_size at a time.
# This currently indexes <= chunk_size, and it varies each time.
if new_pointer is not None and \
new_pointer != self.transaction_pointer:
self.index(deltas)
self.update_pointer(new_pointer)
else:
sleep(self.poll_interval)
def _run(self):
"""
Index into Elasticsearch the threads, messages of all namespaces.
"""
# Indexing is namespace agnostic.
# Note that although this means we do not restrict the Transaction
# table query (via the format_transactions_after_pointer() call below)
# to a namespace, since we pass a `result_limit` (== chunk_size)
# argument, the query should still be performant.
namespace_id = None
# Only index messages, threads.
object_types = transaction_objects()
exclude_types = [api_name for model_name, api_name in
object_types.iteritems() if model_name not in
['message', 'thread']]
with session_scope() as db_session:
pointer = db_session.query(SearchIndexCursor).first()
self.transaction_pointer = pointer.transaction_id if pointer else 0
self.log.info('Starting search-index service',
transaction_pointer=self.transaction_pointer)
while True:
with session_scope() as db_session:
deltas, new_pointer = format_transactions_after_pointer(
namespace_id, self.transaction_pointer, db_session,
self.chunk_size, _format_transaction_for_search,
exclude_types)
# TODO[k]: We ideally want to index chunk_size at a time.
# This currently indexes <= chunk_size, and it varies each time.
if new_pointer is not None and \
new_pointer != self.transaction_pointer:
self.index(deltas)
self.update_pointer(new_pointer)
else:
sleep(self.poll_interval)
