def get_date_params(stream_slice: Mapping[str, Any], cursor_field: str, end_date: pendulum.datetime = None):
end_date = end_date or pendulum.yesterday()
start_date = pendulum.parse(stream_slice.get(cursor_field))
if start_date > pendulum.now():
return start_date.to_date_string(), start_date.add(days=1).to_date_string()
end_date = min(end_date, pendulum.parse(stream_slice.get(cursor_field)).add(months=1))
return start_date.add(days=1).to_date_string(), end_date.to_date_string()
def get_date_params(stream_slice: Mapping[str, Any], cursor_field: str, end_date: pendulum.datetime = None, time_unit: str = "months"):
end_date = end_date or pendulum.yesterday()
start_date = pendulum.parse(stream_slice.get(cursor_field))
if start_date > pendulum.now():
return start_date.to_date_string(), start_date.add(days=1).to_date_string()
end_date = min(end_date, pendulum.parse(stream_slice.get(cursor_field)).add(**{time_unit: 1}))
# Fix issue #4806, start date should always be lower than end date.
if start_date.add(days=1).date() >= end_date.date():
return start_date.add(days=1).to_date_string(), start_date.add(days=2).to_date_string()
return start_date.add(days=1).to_date_string(), end_date.to_date_string()
def get_date_params(start_date: str,
time_zone=None,
range_days: int = None,
end_date: pendulum.datetime = None) -> Tuple[str, str]:
"""
Returns `start_date` and `end_date` for the given stream_slice.
If (end_date - start_date) is a big date range (>= 1 month), it can take more than 2 hours to process all the records from the given slice.
After 2 hours next page tokens will be expired, finally resulting in page token expired error
Currently this method returns `start_date` and `end_date` with 15 days difference.
"""
end_date = end_date or pendulum.yesterday(tz=time_zone)
start_date = pendulum.parse(start_date)
if start_date > pendulum.now():
return start_date.to_date_string(), start_date.add(
days=1).to_date_string()
end_date = min(end_date, start_date.add(days=range_days))
# Fix issue #4806, start date should always be lower than end date.
if start_date.add(days=1).date() >= end_date.date():
return start_date.add(days=1).to_date_string(), start_date.add(
days=2).to_date_string()
return start_date.add(days=1).to_date_string(), end_date.to_date_string()