def poll(uri, predicate=None, start_interval_secs=None, max_interval_secs=None, backoff_factor=None):
"""
Issues GET methods on the given command uri until the response
command_info cause the predicate to evalute True. Exponential retry
backoff
Parameters
----------
uri : str
The uri of the command
predicate : function
Function with a single CommandInfo parameter which evaluates to True
when the polling should stop.
start_interval_secs : float
Initial sleep interval for the polling, in seconds
max_interval_secs : float
Maximum sleep interval for the polling, in seconds
backoff_factor : float
Factor to increase the sleep interval on subsequent retries
"""
if predicate is None:
predicate = Polling._get_completion_status
if start_interval_secs is None:
start_interval_secs = config.polling_defaults.start_interval_secs
if backoff_factor is None:
backoff_factor = config.polling_defaults.backoff_factor
if max_interval_secs is None:
max_interval_secs = config.polling_defaults.max_interval_secs
if not CommandInfo.is_valid_command_uri(uri):
raise ValueError("Cannot poll " + uri + " - a /commands/{number} uri is required")
interval_secs = start_interval_secs
command_info = None
printer = ProgressPrinter()
start_time = time.time()
while True:
command_info = Polling._get_command_info(uri)
finish = predicate(command_info)
progress = command_info.progress
printer.update(progress)
if finish:
break
# if len(last_progress) == len(progress) and interval_secs < max_interval_secs:
# interval_secs = min(max_interval_secs, interval_secs * backoff_factor)
# last_progress = progress
uri = command_info.uri
time.sleep(interval_secs)
logger.info("polling %s completed after %0.3f seconds" % (uri, time.time() - start_time))
if not command_info.error:
printer.update("Done [=========================] 100.00%", True)
return command_info
def query(self, query_url):
"""
Issues the query_request to the server
"""
logger.info("Issuing query " + query_url)
try:
response = server.get(query_url)
except:
# do a single retry
response = server.get(query_url)
response_json = response.json()
schema = response_json["result"]["schema"]['columns']
if response_json["complete"]:
data = response_json["result"]["data"]
return QueryResult(data, schema)
else:
command = self.poll_command_info(response)
#retreive the data
printer = ProgressPrinter()
total_pages = command.result["total_pages"] + 1
start = 1
data = []
for i in range(start, total_pages):
next_partition = self.get_query_response(command.id_number, i)
data_in_page = next_partition.result["data"]
data.extend(data_in_page)
#if the total pages is greater than 10 display a progress bar
if total_pages > 5:
finished = i == (total_pages - 1)
if not finished:
time.sleep(.5)
progress = [{
"progress": (float(i)/(total_pages - 1)) * 100,
"tasks_info": {
"retries": 0
}
}]
printer.print_progress(progress, finished)
return QueryResult(data, schema)
def query(self, query_url):
"""
Issues the query_request to the server
"""
logger.info("Issuing query " + query_url)
try:
response = server.get(query_url)
except:
# do a single retry
response = server.get(query_url)
response_json = response.json()
schema = response_json["result"]["schema"]['columns']
if response_json["complete"]:
data = response_json["result"]["data"]
return QueryResult(data, schema)
else:
command = self.poll_command_info(response)
#retreive the data
printer = ProgressPrinter()
total_pages = command.result["total_pages"] + 1
start = 1
data = []
for i in range(start, total_pages):
next_partition = self.get_query_response(command.id_number, i)
data_in_page = next_partition.result["data"]
data.extend(data_in_page)
#if the total pages is greater than 10 display a progress bar
if total_pages > 5:
finished = i == (total_pages - 1)
if not finished:
time.sleep(.5)
progress = [{
"progress": (float(i) / (total_pages - 1)) * 100,
"tasks_info": {
"retries": 0
}
}]
printer.print_progress(progress, finished)
return QueryResult(data, schema)
def poll(uri,
predicate=None,
start_interval_secs=None,
max_interval_secs=None,
backoff_factor=None):
"""
Issues GET methods on the given command uri until the response
command_info cause the predicate to evalute True. Exponential retry
backoff
Parameters
----------
uri : str
The uri of the command
predicate : function
Function with a single CommandInfo parameter which evaluates to True
when the polling should stop.
start_interval_secs : float
Initial sleep interval for the polling, in seconds
max_interval_secs : float
Maximum sleep interval for the polling, in seconds
backoff_factor : float
Factor to increase the sleep interval on subsequent retries
"""
if predicate is None:
predicate = Polling._get_completion_status
if start_interval_secs is None:
start_interval_secs = config.polling_defaults.start_interval_secs
if backoff_factor is None:
backoff_factor = config.polling_defaults.backoff_factor
if max_interval_secs is None:
max_interval_secs = config.polling_defaults.max_interval_secs
if not CommandInfo.is_valid_command_uri(uri):
raise ValueError('Cannot poll ' + uri +
' - a /commands/{number} uri is required')
interval_secs = start_interval_secs
command_info = None
printer = ProgressPrinter()
start_time = time.time()
while True:
command_info = Polling._get_command_info(uri)
finish = predicate(command_info)
progress = command_info.progress
printer.update(progress)
if finish:
break
#if len(last_progress) == len(progress) and interval_secs < max_interval_secs:
# interval_secs = min(max_interval_secs, interval_secs * backoff_factor)
#last_progress = progress
uri = command_info.uri
time.sleep(interval_secs)
logger.info("polling %s completed after %0.3f seconds" %
(uri, time.time() - start_time))
if not command_info.error:
printer.update("Done [=========================] 100.00%", True)
return command_info
def poll(uri, predicate=None, start_interval_secs=None, max_interval_secs=None, backoff_factor=None):
"""
Issues GET methods on the given command uri until the response
command_info cause the predicate to evalute True. Exponential retry
backoff
Parameters
----------
uri : str
The uri of the command
predicate : function
Function with a single CommandInfo parameter which evaluates to True
when the polling should stop.
start_interval_secs : float
Initial sleep interval for the polling, in seconds
max_interval_secs : float
Maximum sleep interval for the polling, in seconds
backoff_factor : float
Factor to increase the sleep interval on subsequent retries
"""
if predicate is None:
predicate = Polling._get_completion_status
if start_interval_secs is None:
start_interval_secs = config.polling_defaults.start_interval_secs
if backoff_factor is None:
backoff_factor = config.polling_defaults.backoff_factor
if max_interval_secs is None:
max_interval_secs = config.polling_defaults.max_interval_secs
if not CommandInfo.is_valid_command_uri(uri):
raise ValueError('Cannot poll ' + uri + ' - a /commands/{number} uri is required')
interval_secs = start_interval_secs
command_info = Polling._get_command_info(uri)
printer = ProgressPrinter()
if not predicate(command_info):
last_progress = []
next_poll_time = time.time()
start_time = time.time()
while True:
if time.time() < next_poll_time:
time.sleep(start_interval_secs)
continue
command_info = Polling._get_command_info(command_info.uri)
finish = predicate(command_info)
next_poll_time = time.time() + interval_secs
progress = command_info.progress
printer.print_progress(progress, finish)
if finish:
break
if last_progress == progress and interval_secs < max_interval_secs:
interval_secs = min(max_interval_secs, interval_secs * backoff_factor)
last_progress = progress
end_time = time.time()
logger.info("polling %s completed after %0.2f seconds" % (uri, end_time - start_time))
return command_info
