def enque(self, resultList: list, future: asyncio.Future):
"""
Push a result into the queue for saving to
the db server. Once the batch size has been reached,
it will be sent to the server
:type resultList: list
:param resultList:
:type future: asyncio.Future
"""
# Handle logging
if environment.TIME_LOGGING:
timestamp_writer(environment.CLIENT_ENQUE_TIMESTAMP_LOG_FILE)
if environment.INTEGRITY_LOGGING:
[
timestamped_count_writer(environment.CLIENT_ENQUE_LOG_FILE,
r.id, 'userid') for r in resultList
]
# Actually enque the item
with (yield lock.acquire()):
[self.store.appendleft(r) for r in resultList]
# if we've reached the batch size, send to the server
# needs to be greater in case hit limit in middle of list
if len(self.store) >= self.batch_size:
yield from self.handle_send(future)
else:
future.set_result(True)
return future
def post( self ):
"""Handles the submision of a list of
new tweets
"""
if environment.TIME_LOGGING:
timestamp_writer( environment.SERVER_RECEIVE_TIMESTAMP_LOG_FILE )
type( self ).increment_request_count()
try:
# decode json
payload = Helpers.decode_payload( self.request.body )
# The payload is a list containing dictionaries
tweets = [ TweetFactory( p ) for p in payload ]
# now extract users from the tweet objects' other_data field
users = [ add_audit_data_to_user( UserFactory( extract_user_dict_from_tweet( tweet ) ), tweet.tweetID ) for
tweet in tweets ]
# add the users to the tweets list since the save handler in the queue
# is agnostic on what kind of orm object it receives
tweets += users
with self.make_session() as session:
yield from asyncio.ensure_future( type( self ).q.enque( tweets, session ) )
self.write( "success" )
except DBExceptions as e:
print( 'db error: %s' % e.message )
# self.logger.log_error('db error: %s' % e.message)
# no need for the client to know
self.write( "success" )
def post(self):
"""Handles the submision of a list of
new user-word records.
"""
if environment.TIME_LOGGING:
timestamp_writer(environment.SERVER_RECEIVE_TIMESTAMP_LOG_FILE)
type(self).increment_request_count()
try:
# decode json
payload = Helpers.decode_payload(self.request.body)
# The payload is a list containing
# a batch of results.
# Thus we need to iterate over it
for p in payload:
# convert to a Result
result = Helpers.make_result_from_decoded_payload(p)
if environment.INTEGRITY_LOGGING:
timestamped_count_writer(
environment.SERVER_RECEIVE_LOG_FILE, result.id,
'userid')
# push it into the queue
yield from type(self).q.enqueue(result)
# Send success response
yield self.write("success")
except DBExceptions as e:
# self.logger.log_error('db error: %s' % e.message)
self.write("error")
def make_and_enque_result(self, sentenceIndex: int, wordIndex: int,
text: str, objId: int):
"""Makes the result and hands it off to the queue handler
:param objId: The user or tweet's id
:param text: The profile or tweet text
:param wordIndex: The ordinal position of the word in the sentence
:param sentenceIndex: The ordinal position of the sentence in the profile
"""
if text is not None:
result = self.make_result(sentenceIndex, wordIndex, text, objId)
# write the timestamp to file
timestamp_writer(log_file)
self.QueueHandler.enque(result)
def post(self):
"""Handles the submision of a list of
new user-word records.
"""
if environment.TIME_LOGGING:
timestamp_writer(environment.SERVER_RECEIVE_TIMESTAMP_LOG_FILE)
type(self).increment_request_count()
try:
# decode json
payload = Helpers.decode_payload(self.request.body)
# The payload is a list containing dictionaries
users = [UserFactory(p) for p in payload]
with self.make_session() as session:
yield from asyncio.ensure_future(
type(self).q.enque(users, session))
self.write("success")
except DBExceptions as e:
# self.logger.log_error('db error: %s' % e.message)
self.write("error")