def filters(self):
all_def_codes = sorted(
DisasterEmergencyFundCode.objects.values_list("code", flat=True))
object_keys_lookup = {
"def_codes": {
"key": "filter|def_codes",
"name": "def_codes",
"type": "array",
"array_type": "enum",
"enum_values": all_def_codes,
"allow_nulls": False,
"optional": False,
},
"query": {
"key": "filter|query",
"name": "query",
"type": "text",
"text_type": "search",
"allow_nulls": True,
"optional": True,
},
"award_type_codes": {
"key": "filter|award_type_codes",
"name": "award_type_codes",
"type": "array",
"array_type": "enum",
"enum_values": sorted(award_type_mapping.keys()),
"allow_nulls": True,
"optional": True,
},
"_loan_award_type_codes": {
"key": "filter|award_type_codes",
"name": "award_type_codes",
"type": "array",
"array_type": "enum",
"enum_values": sorted(loan_type_mapping.keys()),
"allow_nulls": True,
"optional": True,
"default": list(loan_type_mapping.keys()),
},
"_assistance_award_type_codes": {
"key": "filter|award_type_codes",
"name": "award_type_codes",
"type": "array",
"array_type": "enum",
"enum_values": sorted(assistance_type_mapping.keys()),
"allow_nulls": True,
"optional": True,
"default": list(assistance_type_mapping.keys()),
},
}
model = [object_keys_lookup[key] for key in self.required_filters]
json_request = TinyShield(model).block(self.request.data)
return json_request["filter"]
def get_csv_sources(json_request):
csv_sources = []
for award_level in json_request['award_levels']:
queryset = VALUE_MAPPINGS[award_level]['filter_function'](json_request['filters'])
award_level_table = VALUE_MAPPINGS[award_level]['table']
award_type_codes = set(json_request['filters']['award_type_codes'])
d1_award_type_codes = set(contract_type_mapping.keys())
d2_award_type_codes = set(assistance_type_mapping.keys())
if award_type_codes & d1_award_type_codes:
# only generate d1 files if the user is asking for contract data
d1_source = CsvSource(VALUE_MAPPINGS[award_level]['table_name'], 'd1', award_level)
d1_filters = {'{}__isnull'.format(VALUE_MAPPINGS[award_level]['contract_data']): False}
d1_source.queryset = queryset & award_level_table.objects.filter(**d1_filters)
csv_sources.append(d1_source)
if award_type_codes & d2_award_type_codes:
# only generate d2 files if the user is asking for assistance data
d2_source = CsvSource(VALUE_MAPPINGS[award_level]['table_name'], 'd2', award_level)
d2_filters = {'{}__isnull'.format(VALUE_MAPPINGS[award_level]['assistance_data']): False}
d2_source.queryset = queryset & award_level_table.objects.filter(**d2_filters)
csv_sources.append(d2_source)
verify_requested_columns_available(tuple(csv_sources), json_request.get('columns', []))
return csv_sources
def subaward_types_are_valid_groups(type_list):
"""Check to ensure the award type list is a subset of one and only one award group.
Groups: are "Procurement" and "Assistance"
If false, the award type codes aren't a subset of either category.
"""
is_procurement = set(type_list).difference(set(procurement_type_mapping.keys()))
is_assistance = set(type_list).difference(set(assistance_type_mapping.keys()))
return bool(is_procurement) != bool(is_assistance) # clever XOR logic
def get_csv_sources(json_request):
csv_sources = []
for download_type in json_request['download_types']:
agency_id = json_request['filters'].get('agency', 'all')
filter_function = VALUE_MAPPINGS[download_type]['filter_function']
download_type_table = VALUE_MAPPINGS[download_type]['table']
if VALUE_MAPPINGS[download_type]['source_type'] == 'award':
# Award downloads
queryset = filter_function(json_request['filters'])
award_type_codes = set(json_request['filters']['award_type_codes'])
if award_type_codes & (set(contract_type_mapping.keys())
| set(idv_type_mapping.keys())):
# only generate d1 files if the user is asking for contract data
d1_source = CsvSource(
VALUE_MAPPINGS[download_type]['table_name'], 'd1',
download_type, agency_id)
d1_filters = {
'{}__isnull'.format(VALUE_MAPPINGS[download_type]['contract_data']):
False
}
d1_source.queryset = queryset & download_type_table.objects.filter(
**d1_filters)
csv_sources.append(d1_source)
if award_type_codes & set(assistance_type_mapping.keys()):
# only generate d2 files if the user is asking for assistance data
d2_source = CsvSource(
VALUE_MAPPINGS[download_type]['table_name'], 'd2',
download_type, agency_id)
d2_filters = {
'{}__isnull'.format(VALUE_MAPPINGS[download_type]['assistance_data']):
False
}
d2_source.queryset = queryset & download_type_table.objects.filter(
**d2_filters)
csv_sources.append(d2_source)
verify_requested_columns_available(tuple(csv_sources),
json_request.get('columns', []))
elif VALUE_MAPPINGS[download_type]['source_type'] == 'account':
# Account downloads
account_source = CsvSource(
VALUE_MAPPINGS[download_type]['table_name'],
json_request['account_level'], download_type, agency_id)
account_source.queryset = filter_function(
download_type, VALUE_MAPPINGS[download_type]['table'],
json_request['filters'], json_request['account_level'])
csv_sources.append(account_source)
return csv_sources
def get_csv_sources(json_request):
csv_sources = []
for download_type in json_request["download_types"]:
agency_id = json_request["filters"].get("agency", "all")
filter_function = VALUE_MAPPINGS[download_type]["filter_function"]
download_type_table = VALUE_MAPPINGS[download_type]["table"]
if VALUE_MAPPINGS[download_type]["source_type"] == "award":
# Award downloads
# Use correct date range columns for advanced search
# (Will not change anything for keyword search since "time_period" is not provided))
filters = add_date_range_comparison_types(
json_request["filters"],
is_subaward=download_type != "awards",
gte_date_type="action_date",
lte_date_type="date_signed",
)
queryset = filter_function(filters)
award_type_codes = set(filters["award_type_codes"])
if award_type_codes & (set(contract_type_mapping.keys()) | set(idv_type_mapping.keys())):
# only generate d1 files if the user is asking for contract data
d1_source = CsvSource(VALUE_MAPPINGS[download_type]["table_name"], "d1", download_type, agency_id)
d1_filters = {"{}__isnull".format(VALUE_MAPPINGS[download_type]["contract_data"]): False}
d1_source.queryset = queryset & download_type_table.objects.filter(**d1_filters)
csv_sources.append(d1_source)
if award_type_codes & set(assistance_type_mapping.keys()):
# only generate d2 files if the user is asking for assistance data
d2_source = CsvSource(VALUE_MAPPINGS[download_type]["table_name"], "d2", download_type, agency_id)
d2_filters = {"{}__isnull".format(VALUE_MAPPINGS[download_type]["assistance_data"]): False}
d2_source.queryset = queryset & download_type_table.objects.filter(**d2_filters)
csv_sources.append(d2_source)
verify_requested_columns_available(tuple(csv_sources), json_request.get("columns", []))
elif VALUE_MAPPINGS[download_type]["source_type"] == "account":
# Account downloads
account_source = CsvSource(
VALUE_MAPPINGS[download_type]["table_name"], json_request["account_level"], download_type, agency_id
)
account_source.queryset = filter_function(
download_type,
VALUE_MAPPINGS[download_type]["table"],
json_request["filters"],
json_request["account_level"],
)
csv_sources.append(account_source)
return csv_sources
def get_download_sources(json_request: dict,
origination: Optional[str] = None):
download_sources = []
for download_type in json_request["download_types"]:
agency_id = json_request.get("agency", "all")
filter_function = VALUE_MAPPINGS[download_type]["filter_function"]
download_type_table = VALUE_MAPPINGS[download_type]["table"]
if VALUE_MAPPINGS[download_type]["source_type"] == "award":
# Award downloads
# Use correct date range columns for advanced search
# (Will not change anything for keyword search since "time_period" is not provided))
filters = add_date_range_comparison_types(
json_request["filters"],
is_subaward=download_type != "awards",
gte_date_type="action_date",
lte_date_type="date_signed",
)
queryset = filter_function(filters)
if filters.get("prime_and_sub_award_types") is not None:
award_type_codes = set(
filters["prime_and_sub_award_types"][download_type])
else:
award_type_codes = set(filters["award_type_codes"])
if (award_type_codes & (set(contract_type_mapping.keys())
| set(idv_type_mapping.keys()))
or "procurement" in award_type_codes):
# only generate d1 files if the user is asking for contract data
d1_source = DownloadSource(
VALUE_MAPPINGS[download_type]["table_name"], "d1",
download_type, agency_id)
d1_filters = {
f"{VALUE_MAPPINGS[download_type]['contract_data']}__isnull":
False
}
d1_source.queryset = queryset & download_type_table.objects.filter(
**d1_filters)
download_sources.append(d1_source)
if award_type_codes & set(assistance_type_mapping.keys()) or (
"grant" in award_type_codes):
# only generate d2 files if the user is asking for assistance data
d2_source = DownloadSource(
VALUE_MAPPINGS[download_type]["table_name"], "d2",
download_type, agency_id)
d2_filters = {
f"{VALUE_MAPPINGS[download_type]['assistance_data']}__isnull":
False
}
d2_source.queryset = queryset & download_type_table.objects.filter(
**d2_filters)
download_sources.append(d2_source)
elif VALUE_MAPPINGS[download_type]["source_type"] == "account":
# Account downloads
account_source = DownloadSource(
VALUE_MAPPINGS[download_type]["table_name"],
json_request["account_level"], download_type, agency_id)
account_source.queryset = filter_function(
download_type,
VALUE_MAPPINGS[download_type]["table"],
json_request["filters"],
json_request["account_level"],
)
download_sources.append(account_source)
verify_requested_columns_available(tuple(download_sources),
json_request.get("columns", []))
return download_sources
from usaspending_api.common.helpers.dict_helpers import order_nested_object
from usaspending_api.common.helpers.generic_helper import generate_fiscal_year
from usaspending_api.common.sqs.sqs_handler import get_sqs_queue
from usaspending_api.download.filestreaming import download_generation
from usaspending_api.download.helpers import multipart_upload, pull_modified_agencies_cgacs
from usaspending_api.download.lookups import JOB_STATUS_DICT
from usaspending_api.download.models import DownloadJob
from usaspending_api.download.v2.request_validations import validate_award_request
from usaspending_api.download.v2.year_limited_downloads import YearLimitedDownloadViewSet
from usaspending_api.references.models import ToptierAgency
logger = logging.getLogger(__name__)
award_mappings = {
"contracts": list(procurement_type_mapping.keys()),
"assistance": list(assistance_type_mapping.keys()),
}
class Command(BaseCommand):
def download(
self,
file_name,
prime_award_types=None,
agency=None,
sub_agency=None,
date_type=None,
start_date=None,
end_date=None,
columns=[],
file_format="csv",
