def get_csv_sources(json_request):
csv_sources = []
for download_type in json_request["download_types"]:
agency_id = json_request["filters"].get("agency", "all")
filter_function = VALUE_MAPPINGS[download_type]["filter_function"]
download_type_table = VALUE_MAPPINGS[download_type]["table"]
if VALUE_MAPPINGS[download_type]["source_type"] == "award":
# Award downloads
# Use correct date range columns for advanced search
# (Will not change anything for keyword search since "time_period" is not provided))
filters = add_date_range_comparison_types(
json_request["filters"],
is_subaward=download_type != "awards",
gte_date_type="action_date",
lte_date_type="date_signed",
)
queryset = filter_function(filters)
award_type_codes = set(filters["award_type_codes"])
if award_type_codes & (set(contract_type_mapping.keys()) | set(idv_type_mapping.keys())):
# only generate d1 files if the user is asking for contract data
d1_source = CsvSource(VALUE_MAPPINGS[download_type]["table_name"], "d1", download_type, agency_id)
d1_filters = {"{}__isnull".format(VALUE_MAPPINGS[download_type]["contract_data"]): False}
d1_source.queryset = queryset & download_type_table.objects.filter(**d1_filters)
csv_sources.append(d1_source)
if award_type_codes & set(assistance_type_mapping.keys()):
# only generate d2 files if the user is asking for assistance data
d2_source = CsvSource(VALUE_MAPPINGS[download_type]["table_name"], "d2", download_type, agency_id)
d2_filters = {"{}__isnull".format(VALUE_MAPPINGS[download_type]["assistance_data"]): False}
d2_source.queryset = queryset & download_type_table.objects.filter(**d2_filters)
csv_sources.append(d2_source)
verify_requested_columns_available(tuple(csv_sources), json_request.get("columns", []))
elif VALUE_MAPPINGS[download_type]["source_type"] == "account":
# Account downloads
account_source = CsvSource(
VALUE_MAPPINGS[download_type]["table_name"], json_request["account_level"], download_type, agency_id
)
account_source.queryset = filter_function(
download_type,
VALUE_MAPPINGS[download_type]["table"],
json_request["filters"],
json_request["account_level"],
)
csv_sources.append(account_source)
return csv_sources
def post(self, request):
models = [
{"name": "subawards", "key": "subawards", "type": "boolean", "default": False},
{
"name": "object_class",
"key": "filter|object_class",
"type": "array",
"array_type": "text",
"text_type": "search",
},
{
"name": "program_activity",
"key": "filter|program_activity",
"type": "array",
"array_type": "integer",
"array_max": maxsize,
},
]
models.extend(copy.deepcopy(AWARD_FILTER))
models.extend(copy.deepcopy(PAGINATION))
json_request = TinyShield(models).block(request.data)
subawards = json_request["subawards"]
filters = add_date_range_comparison_types(
json_request.get("filters", None), subawards, gte_date_type="action_date", lte_date_type="date_signed"
)
elasticsearch = is_experimental_elasticsearch_api(request)
if elasticsearch and not subawards:
logger.info("Using experimental Elasticsearch functionality for 'spending_by_award_count'")
results = self.query_elasticsearch(filters)
return Response({"results": results, "messages": [get_time_period_message()]})
if filters is None:
raise InvalidParameterException("Missing required request parameters: 'filters'")
empty_results = {"contracts": 0, "idvs": 0, "grants": 0, "direct_payments": 0, "loans": 0, "other": 0}
if subawards:
empty_results = {"subcontracts": 0, "subgrants": 0}
if "award_type_codes" in filters and "no intersection" in filters["award_type_codes"]:
# "Special case": there will never be results when the website provides this value
return Response({"results": empty_results})
if subawards:
results = self.handle_subawards(filters)
else:
results = self.handle_awards(filters, empty_results)
return Response({"results": results, "messages": [get_time_period_message()]})
def post(self, request):
models = [
{"name": "subawards", "key": "subawards", "type": "boolean", "default": False},
{
"name": "object_class",
"key": "filter|object_class",
"type": "array",
"array_type": "text",
"text_type": "search",
},
{
"name": "program_activity",
"key": "filter|program_activity",
"type": "array",
"array_type": "integer",
"array_max": maxsize,
},
]
models.extend(copy.deepcopy(AWARD_FILTER_NO_RECIPIENT_ID))
models.extend(copy.deepcopy(PAGINATION))
self.original_filters = request.data.get("filters")
json_request = TinyShield(models).block(request.data)
subawards = json_request["subawards"]
filters = add_date_range_comparison_types(
json_request.get("filters", None), subawards, gte_date_type="action_date", lte_date_type="date_signed"
)
if filters is None:
raise InvalidParameterException("Missing required request parameters: 'filters'")
if "award_type_codes" in filters and "no intersection" in filters["award_type_codes"]:
# "Special case": there will never be results when the website provides this value
empty_results = {"contracts": 0, "idvs": 0, "grants": 0, "direct_payments": 0, "loans": 0, "other": 0}
if subawards:
empty_results = {"subcontracts": 0, "subgrants": 0}
results = empty_results
elif subawards:
results = self.handle_subawards(filters)
else:
results = self.query_elasticsearch_for_prime_awards(filters)
return Response(
{
"results": results,
"messages": get_generic_filters_message(
self.original_filters.keys(), [elem["name"] for elem in AWARD_FILTER_NO_RECIPIENT_ID]
),
}
)
def post(self, request):
"""Return all awards matching the provided filters and limits"""
self.original_filters = request.data.get("filters")
json_request = self.validate_request_data(request.data)
self.is_subaward = json_request["subawards"]
self.constants = GLOBAL_MAP[
"subaward"] if self.is_subaward else GLOBAL_MAP["award"]
self.filters = add_date_range_comparison_types(
json_request.get("filters"),
self.is_subaward,
gte_date_type="action_date",
lte_date_type="date_signed")
self.fields = json_request["fields"]
self.pagination = {
"limit": json_request["limit"],
"lower_bound": (json_request["page"] - 1) * json_request["limit"],
"page": json_request["page"],
"sort_key": json_request.get("sort") or self.fields[0],
"sort_order": json_request["order"],
"upper_bound": json_request["page"] * json_request["limit"] + 1,
}
self.elasticsearch = is_experimental_elasticsearch_api(request)
if not self.elasticsearch:
mirror_request_to_elasticsearch(request)
if self.if_no_intersection(
): # Like an exception, but API response is a HTTP 200 with a JSON payload
return Response(self.populate_response(results=[], has_next=False))
raise_if_award_types_not_valid_subset(self.filters["award_type_codes"],
self.is_subaward)
raise_if_sort_key_not_valid(self.pagination["sort_key"], self.fields,
self.is_subaward)
if self.elasticsearch and not self.is_subaward:
self.last_record_unique_id = json_request.get(
"last_record_unique_id")
self.last_record_sort_value = json_request.get(
"last_record_sort_value")
logger.info(
"Using experimental Elasticsearch functionality for 'spending_by_award'"
)
return Response(
self.construct_es_response(self.query_elasticsearch()))
return Response(self.create_response(self.construct_queryset()))
def post(self, request):
models = [{
"name": "subawards",
"key": "subawards",
"type": "boolean",
"default": False
}]
models.extend(copy.deepcopy(AWARD_FILTER))
models.extend(copy.deepcopy(PAGINATION))
json_request = TinyShield(models).block(request.data)
subawards = json_request["subawards"]
filters = add_date_range_comparison_types(json_request.get(
"filters", None),
subawards,
gte_date_type="action_date",
lte_date_type="date_signed")
if filters is None:
raise InvalidParameterException(
"Missing required request parameters: 'filters'")
empty_results = {
"contracts": 0,
"idvs": 0,
"grants": 0,
"direct_payments": 0,
"loans": 0,
"other": 0
}
if subawards:
empty_results = {"subcontracts": 0, "subgrants": 0}
if "award_type_codes" in filters and "no intersection" in filters[
"award_type_codes"]:
# "Special case": there will never be results when the website provides this value
return Response({"results": empty_results})
if subawards:
results = self.handle_subawards(filters)
else:
results = self.handle_awards(filters, empty_results)
return Response({"results": results})
def post(self, request):
"""Return all awards matching the provided filters and limits"""
json_request = self.validate_request_data(request.data)
self.is_subaward = json_request["subawards"]
self.constants = GLOBAL_MAP["subaward"] if self.is_subaward else GLOBAL_MAP["award"]
self.filters = add_date_range_comparison_types(
json_request.get("filters"), self.is_subaward, gte_date_type="action_date", lte_date_type="date_signed"
)
self.fields = json_request["fields"]
self.pagination = {
"limit": json_request["limit"],
"lower_bound": (json_request["page"] - 1) * json_request["limit"],
"page": json_request["page"],
"sort_key": json_request.get("sort") or self.fields[0],
"sort_order": json_request["order"],
"upper_bound": json_request["page"] * json_request["limit"] + 1,
}
if self.if_no_intersection(): # Like an exception, but API response is a HTTP 200 with a JSON payload
return Response(self.populate_response(results=[], has_next=False))
raise_if_award_types_not_valid_subset(self.filters["award_type_codes"], self.is_subaward)
raise_if_sort_key_not_valid(self.pagination["sort_key"], self.fields, self.is_subaward)
return Response(self.create_response(self.construct_queryset()))
def get_download_sources(json_request: dict,
origination: Optional[str] = None):
download_sources = []
for download_type in json_request["download_types"]:
agency_id = json_request.get("agency", "all")
filter_function = VALUE_MAPPINGS[download_type]["filter_function"]
download_type_table = VALUE_MAPPINGS[download_type]["table"]
if VALUE_MAPPINGS[download_type]["source_type"] == "award":
# Award downloads
# Use correct date range columns for advanced search
# (Will not change anything for keyword search since "time_period" is not provided))
filters = add_date_range_comparison_types(
json_request["filters"],
is_subaward=download_type != "awards",
gte_date_type="action_date",
lte_date_type="date_signed",
)
queryset = filter_function(filters)
if filters.get("prime_and_sub_award_types") is not None:
award_type_codes = set(
filters["prime_and_sub_award_types"][download_type])
else:
award_type_codes = set(filters["award_type_codes"])
if (award_type_codes & (set(contract_type_mapping.keys())
| set(idv_type_mapping.keys()))
or "procurement" in award_type_codes):
# only generate d1 files if the user is asking for contract data
d1_source = DownloadSource(
VALUE_MAPPINGS[download_type]["table_name"], "d1",
download_type, agency_id)
d1_filters = {
f"{VALUE_MAPPINGS[download_type]['contract_data']}__isnull":
False
}
d1_source.queryset = queryset & download_type_table.objects.filter(
**d1_filters)
download_sources.append(d1_source)
if award_type_codes & set(assistance_type_mapping.keys()) or (
"grant" in award_type_codes):
# only generate d2 files if the user is asking for assistance data
d2_source = DownloadSource(
VALUE_MAPPINGS[download_type]["table_name"], "d2",
download_type, agency_id)
d2_filters = {
f"{VALUE_MAPPINGS[download_type]['assistance_data']}__isnull":
False
}
d2_source.queryset = queryset & download_type_table.objects.filter(
**d2_filters)
download_sources.append(d2_source)
elif VALUE_MAPPINGS[download_type]["source_type"] == "account":
# Account downloads
account_source = DownloadSource(
VALUE_MAPPINGS[download_type]["table_name"],
json_request["account_level"], download_type, agency_id)
account_source.queryset = filter_function(
download_type,
VALUE_MAPPINGS[download_type]["table"],
json_request["filters"],
json_request["account_level"],
)
download_sources.append(account_source)
verify_requested_columns_available(tuple(download_sources),
json_request.get("columns", []))
return download_sources
