def handle_rest_timeout(ex, stream, state, current_window, start_pen):
if stream.get("replication_key"):
LOGGER.info(
"Export timed out, reducing query window and writing state.")
new_window = current_window // 2
if new_window == 0:
raise apis.ExportFailed(
"Export too large for smallest possible query window. " +
"Cannot subdivide any further. ({}: {})".format(
stream["replication_key"], start_pen)) from ex
state["bookmarks"][
stream["tap_stream_id"]]["window_length"] = new_window
singer.write_state(state)
return new_window
def handle_rest_timeout(ex, stream, state, current_window, start_pen):
if stream.get("replication_key"):
LOGGER.info("Export timed out, reducing query window and writing state.")
new_window = current_window // 2
if new_window == 0:
raise apis.ExportFailed("Export too large for smallest possible query window. " +
"Cannot subdivide any further. ({}: {})"
.format(stream["replication_key"], start_pen)) from ex
state["bookmarks"][stream["tap_stream_id"]]["window_length"] = new_window
singer.write_state(state)
return new_window
# NB: Pylint caught this, since no return existed. Returning `None` to not change
# the existing return value, but it may not make sense for usage, or never
# get hit (defensive coding might not be necessary above)
return None
def handle_aqua_timeout(ex, stream, state):
if stream.get("replication_key"):
LOGGER.info("Export timed out, reducing query window and writing state.")
window_bookmark = state["bookmarks"][stream["tap_stream_id"]].get("current_window_end")
previous_window_end = pendulum.parse(window_bookmark) if window_bookmark else pendulum.utcnow()
window_start = pendulum.parse(state["bookmarks"][stream["tap_stream_id"]][stream["replication_key"]])
if previous_window_end == window_start:
raise apis.ExportFailed("Export too large for smallest possible query window. " +
"Cannot subdivide any further. ({}: {})"
.format(stream["replication_key"], window_start)) from ex
half_day_range = (previous_window_end - window_start) // 2
current_window_end = previous_window_end - half_day_range
state["bookmarks"][stream["tap_stream_id"]]["current_window_end"] = current_window_end.strftime("%Y-%m-%dT%H:%M:%SZ")
singer.write_state(state)
