def test_cast_datetime_to_naive():
expected = datetime(2000, 1, 2, 3, 4, 5)
assert cast_datetime_to_naive(datetime(2000, 1, 2, 3, 4, 5)) == expected
assert cast_datetime_to_naive(
datetime(2000, 1, 2, 3, 4, 5, tzinfo=timezone.utc)) == expected
assert cast_datetime_to_naive(
pytz.timezone("US/Eastern").localize(datetime(2000, 1, 1, 22, 4,
5))) == expected
def _get_ids(sql, afa_ids, start_datetime, end_datetime):
params = []
if afa_ids:
sql += " and afa_generated_unique in %s"
params.append(tuple(afa_ids))
if start_datetime:
sql += " and updated_at >= %s"
params.append(cast_datetime_to_naive(start_datetime))
if end_datetime:
sql += " and updated_at < %s"
params.append(cast_datetime_to_naive(end_datetime))
with connection.cursor() as cursor:
cursor.execute(sql, params)
return tuple(row[0] for row in cursor.fetchall())
def _get_ids(sql, submission_ids, afa_ids, start_datetime, end_datetime):
params = []
if submission_ids is not None:
sql += " and submission_id in %s"
params.append(tuple(submission_ids))
if afa_ids:
sql += " and afa_generated_unique in %s"
params.append(tuple(afa_ids))
if start_datetime:
sql += " and updated_at >= %s"
params.append(cast_datetime_to_naive(start_datetime))
if end_datetime:
sql += " and updated_at < %s"
params.append(cast_datetime_to_naive(end_datetime))
with connections["data_broker"].cursor() as cursor:
cursor.execute(sql, params)
return tuple(row[0] for row in cursor.fetchall())
def _get_ids(sql, ids, afa_ids, start_datetime, end_datetime):
params = []
if ids and afa_ids:
sql += " and (published_award_financial_assistance_id in %s or afa_generated_unique in %s)"
params.append(tuple(ids))
params.append(tuple(afa_ids))
elif ids:
sql += " and published_award_financial_assistance_id in %s"
params.append(tuple(ids))
elif afa_ids:
sql += " and afa_generated_unique in %s"
params.append(tuple(afa_ids))
if start_datetime:
sql += " and updated_at >= %s"
params.append(cast_datetime_to_naive(start_datetime))
if end_datetime:
sql += " and updated_at < %s"
params.append(cast_datetime_to_naive(end_datetime))
with connection.cursor() as cursor:
cursor.execute(sql, params)
return tuple(row[0] for row in cursor.fetchall())
def get_new_submission_ids(last_load_date):
"""
Grab all the submission ids updated since last_load_date.
"""
sql = """
select submission_id
from submission
where d2_submission is true and
publish_status_id in (2, 3) and
updated_at >= %s
"""
with connections["data_broker"].cursor() as cursor:
cursor.execute(sql, [cast_datetime_to_naive(last_load_date)])
return tuple(row[0] for row in cursor.fetchall())
def limit_objects_to_date_range(
objects: List["boto3.resources.factory.s3.ObjectSummary"],
date_format: str,
start_datetime: datetime,
end_datetime: Optional[datetime] = None,
) -> List["boto3.resources.factory.s3.ObjectSummary"]:
results = []
for obj in objects or []:
file_last_modified = cast_datetime_to_naive(obj.last_modified)
# Only keep S3 objects which fall between the provided dates
if datetime_is_lt(file_last_modified, start_datetime):
continue
if end_datetime and datetime_is_ge(file_last_modified, end_datetime):
continue
results.append(obj)
return results
def test_cast_datetime_to_naive():
expected = datetime(2000, 1, 2, 3, 4, 5)
assert cast_datetime_to_naive(datetime(2000, 1, 2, 3, 4, 5)) == expected
assert cast_datetime_to_naive(datetime(2000, 1, 2, 3, 4, 5, tzinfo=timezone.utc)) == expected
assert cast_datetime_to_naive(pytz.timezone('US/Eastern').localize(datetime(2000, 1, 1, 22, 4, 5))) == expected