def _process_product_usage_upload(addr, port, url, limit, gzip_body=True):
"""
@param addr: address of remote server
@param port: port of remote server
@param url: url of remote server
@param limit: max amount of objects to process per request
@param gzip_body: defaults to True, will gzip the request body
@return: True on success, False on failure
"""
url = url + "/productusage/" #must end in '/'
time_a = time.time()
cursor = _get_product_usage_data(addr, limit)
time_b = time.time()
pu_data = list(cursor)
time_c = time.time()
if not pu_data:
_LOG.info("No new product usage data to upload")
return True
#last_timestamp = pu_data[-1].date
try:
_LOG.info("Uploading %s ProductUsage entries to %s:%s/%s" % (len(pu_data), addr, port, url))
# TODO:
# Examine return values and determine, what/if any objects were not successfuly uploaded.
time_d = time.time()
splice_server_client.upload_product_usage_data(addr, port, url, pu_data, gzip_body=gzip_body)
time_e = time.time()
# Mark the successfully uploaded objects as transferred
# TODO: Update logic to account for return value from upload call
object_ids = [x.id for x in pu_data]
_mark_sent(object_ids, addr)
time_f = time.time()
_LOG.info("%s seconds to fetch/upload %s ProductUsage entries to %s:%s/%s" % (time_f-time_a, len(pu_data), addr, port, url))
_LOG.info(" %s seconds to fetch %s ProductUsage entries, %s for initial mongo query %s seconds to convert to list" % \
(time_c-time_a, len(pu_data), time_b-time_a, time_c-time_b))
_LOG.info(" %s seconds to upload %s ProductUsage entries, %s seconds to update tracker" % (time_e-time_d, len(pu_data), time_f-time_e))
# Log items unsuccessful and retry upload
except RequestException, e:
_LOG.exception("Received exception attempting to send %s records to %s:%s\%s" % (len(pu_data), addr, port, url))
return False
def send(host, port, url, data, batch_size=5000, gzip_body=False):
# Transfer data to SpliceServer
start_index = 0
end_index = 0
while True:
end_index += batch_size
if end_index > len(data):
end_index = len(data)
sub_data = data[start_index:end_index]
start = time.time()
resp = splice_server_client.upload_product_usage_data(host, port, url, {"objects": sub_data}, gzip_body=gzip_body)
end = time.time()
print "Sent %s items in %.4f seconds" % (len(sub_data), end-start)
start_index += batch_size
if end_index >= len(data):
break
