def _process_product_usage_upload(addr, port, url, limit, gzip_body=True): """ @param addr: address of remote server @param port: port of remote server @param url: url of remote server @param limit: max amount of objects to process per request @param gzip_body: defaults to True, will gzip the request body @return: True on success, False on failure """ url = url + "/productusage/" #must end in '/' time_a = time.time() cursor = _get_product_usage_data(addr, limit) time_b = time.time() pu_data = list(cursor) time_c = time.time() if not pu_data: _LOG.info("No new product usage data to upload") return True #last_timestamp = pu_data[-1].date try: _LOG.info("Uploading %s ProductUsage entries to %s:%s/%s" % (len(pu_data), addr, port, url)) # TODO: # Examine return values and determine, what/if any objects were not successfuly uploaded. time_d = time.time() splice_server_client.upload_product_usage_data(addr, port, url, pu_data, gzip_body=gzip_body) time_e = time.time() # Mark the successfully uploaded objects as transferred # TODO: Update logic to account for return value from upload call object_ids = [x.id for x in pu_data] _mark_sent(object_ids, addr) time_f = time.time() _LOG.info("%s seconds to fetch/upload %s ProductUsage entries to %s:%s/%s" % (time_f-time_a, len(pu_data), addr, port, url)) _LOG.info(" %s seconds to fetch %s ProductUsage entries, %s for initial mongo query %s seconds to convert to list" % \ (time_c-time_a, len(pu_data), time_b-time_a, time_c-time_b)) _LOG.info(" %s seconds to upload %s ProductUsage entries, %s seconds to update tracker" % (time_e-time_d, len(pu_data), time_f-time_e)) # Log items unsuccessful and retry upload except RequestException, e: _LOG.exception("Received exception attempting to send %s records to %s:%s\%s" % (len(pu_data), addr, port, url)) return False
def send(host, port, url, data, batch_size=5000, gzip_body=False): # Transfer data to SpliceServer start_index = 0 end_index = 0 while True: end_index += batch_size if end_index > len(data): end_index = len(data) sub_data = data[start_index:end_index] start = time.time() resp = splice_server_client.upload_product_usage_data(host, port, url, {"objects": sub_data}, gzip_body=gzip_body) end = time.time() print "Sent %s items in %.4f seconds" % (len(sub_data), end-start) start_index += batch_size if end_index >= len(data): break