def filters(self): all_def_codes = sorted( DisasterEmergencyFundCode.objects.values_list("code", flat=True)) object_keys_lookup = { "def_codes": { "key": "filter|def_codes", "name": "def_codes", "type": "array", "array_type": "enum", "enum_values": all_def_codes, "allow_nulls": False, "optional": False, }, "query": { "key": "filter|query", "name": "query", "type": "text", "text_type": "search", "allow_nulls": True, "optional": True, }, "award_type_codes": { "key": "filter|award_type_codes", "name": "award_type_codes", "type": "array", "array_type": "enum", "enum_values": sorted(award_type_mapping.keys()), "allow_nulls": True, "optional": True, }, "_loan_award_type_codes": { "key": "filter|award_type_codes", "name": "award_type_codes", "type": "array", "array_type": "enum", "enum_values": sorted(loan_type_mapping.keys()), "allow_nulls": True, "optional": True, "default": list(loan_type_mapping.keys()), }, "_assistance_award_type_codes": { "key": "filter|award_type_codes", "name": "award_type_codes", "type": "array", "array_type": "enum", "enum_values": sorted(assistance_type_mapping.keys()), "allow_nulls": True, "optional": True, "default": list(assistance_type_mapping.keys()), }, } model = [object_keys_lookup[key] for key in self.required_filters] json_request = TinyShield(model).block(self.request.data) return json_request["filter"]
def get_csv_sources(json_request): csv_sources = [] for award_level in json_request['award_levels']: queryset = VALUE_MAPPINGS[award_level]['filter_function'](json_request['filters']) award_level_table = VALUE_MAPPINGS[award_level]['table'] award_type_codes = set(json_request['filters']['award_type_codes']) d1_award_type_codes = set(contract_type_mapping.keys()) d2_award_type_codes = set(assistance_type_mapping.keys()) if award_type_codes & d1_award_type_codes: # only generate d1 files if the user is asking for contract data d1_source = CsvSource(VALUE_MAPPINGS[award_level]['table_name'], 'd1', award_level) d1_filters = {'{}__isnull'.format(VALUE_MAPPINGS[award_level]['contract_data']): False} d1_source.queryset = queryset & award_level_table.objects.filter(**d1_filters) csv_sources.append(d1_source) if award_type_codes & d2_award_type_codes: # only generate d2 files if the user is asking for assistance data d2_source = CsvSource(VALUE_MAPPINGS[award_level]['table_name'], 'd2', award_level) d2_filters = {'{}__isnull'.format(VALUE_MAPPINGS[award_level]['assistance_data']): False} d2_source.queryset = queryset & award_level_table.objects.filter(**d2_filters) csv_sources.append(d2_source) verify_requested_columns_available(tuple(csv_sources), json_request.get('columns', [])) return csv_sources
def subaward_types_are_valid_groups(type_list): """Check to ensure the award type list is a subset of one and only one award group. Groups: are "Procurement" and "Assistance" If false, the award type codes aren't a subset of either category. """ is_procurement = set(type_list).difference(set(procurement_type_mapping.keys())) is_assistance = set(type_list).difference(set(assistance_type_mapping.keys())) return bool(is_procurement) != bool(is_assistance) # clever XOR logic
def get_csv_sources(json_request): csv_sources = [] for download_type in json_request['download_types']: agency_id = json_request['filters'].get('agency', 'all') filter_function = VALUE_MAPPINGS[download_type]['filter_function'] download_type_table = VALUE_MAPPINGS[download_type]['table'] if VALUE_MAPPINGS[download_type]['source_type'] == 'award': # Award downloads queryset = filter_function(json_request['filters']) award_type_codes = set(json_request['filters']['award_type_codes']) if award_type_codes & (set(contract_type_mapping.keys()) | set(idv_type_mapping.keys())): # only generate d1 files if the user is asking for contract data d1_source = CsvSource( VALUE_MAPPINGS[download_type]['table_name'], 'd1', download_type, agency_id) d1_filters = { '{}__isnull'.format(VALUE_MAPPINGS[download_type]['contract_data']): False } d1_source.queryset = queryset & download_type_table.objects.filter( **d1_filters) csv_sources.append(d1_source) if award_type_codes & set(assistance_type_mapping.keys()): # only generate d2 files if the user is asking for assistance data d2_source = CsvSource( VALUE_MAPPINGS[download_type]['table_name'], 'd2', download_type, agency_id) d2_filters = { '{}__isnull'.format(VALUE_MAPPINGS[download_type]['assistance_data']): False } d2_source.queryset = queryset & download_type_table.objects.filter( **d2_filters) csv_sources.append(d2_source) verify_requested_columns_available(tuple(csv_sources), json_request.get('columns', [])) elif VALUE_MAPPINGS[download_type]['source_type'] == 'account': # Account downloads account_source = CsvSource( VALUE_MAPPINGS[download_type]['table_name'], json_request['account_level'], download_type, agency_id) account_source.queryset = filter_function( download_type, VALUE_MAPPINGS[download_type]['table'], json_request['filters'], json_request['account_level']) csv_sources.append(account_source) return csv_sources
def get_csv_sources(json_request): csv_sources = [] for download_type in json_request["download_types"]: agency_id = json_request["filters"].get("agency", "all") filter_function = VALUE_MAPPINGS[download_type]["filter_function"] download_type_table = VALUE_MAPPINGS[download_type]["table"] if VALUE_MAPPINGS[download_type]["source_type"] == "award": # Award downloads # Use correct date range columns for advanced search # (Will not change anything for keyword search since "time_period" is not provided)) filters = add_date_range_comparison_types( json_request["filters"], is_subaward=download_type != "awards", gte_date_type="action_date", lte_date_type="date_signed", ) queryset = filter_function(filters) award_type_codes = set(filters["award_type_codes"]) if award_type_codes & (set(contract_type_mapping.keys()) | set(idv_type_mapping.keys())): # only generate d1 files if the user is asking for contract data d1_source = CsvSource(VALUE_MAPPINGS[download_type]["table_name"], "d1", download_type, agency_id) d1_filters = {"{}__isnull".format(VALUE_MAPPINGS[download_type]["contract_data"]): False} d1_source.queryset = queryset & download_type_table.objects.filter(**d1_filters) csv_sources.append(d1_source) if award_type_codes & set(assistance_type_mapping.keys()): # only generate d2 files if the user is asking for assistance data d2_source = CsvSource(VALUE_MAPPINGS[download_type]["table_name"], "d2", download_type, agency_id) d2_filters = {"{}__isnull".format(VALUE_MAPPINGS[download_type]["assistance_data"]): False} d2_source.queryset = queryset & download_type_table.objects.filter(**d2_filters) csv_sources.append(d2_source) verify_requested_columns_available(tuple(csv_sources), json_request.get("columns", [])) elif VALUE_MAPPINGS[download_type]["source_type"] == "account": # Account downloads account_source = CsvSource( VALUE_MAPPINGS[download_type]["table_name"], json_request["account_level"], download_type, agency_id ) account_source.queryset = filter_function( download_type, VALUE_MAPPINGS[download_type]["table"], json_request["filters"], json_request["account_level"], ) csv_sources.append(account_source) return csv_sources
def get_download_sources(json_request: dict, origination: Optional[str] = None): download_sources = [] for download_type in json_request["download_types"]: agency_id = json_request.get("agency", "all") filter_function = VALUE_MAPPINGS[download_type]["filter_function"] download_type_table = VALUE_MAPPINGS[download_type]["table"] if VALUE_MAPPINGS[download_type]["source_type"] == "award": # Award downloads # Use correct date range columns for advanced search # (Will not change anything for keyword search since "time_period" is not provided)) filters = add_date_range_comparison_types( json_request["filters"], is_subaward=download_type != "awards", gte_date_type="action_date", lte_date_type="date_signed", ) queryset = filter_function(filters) if filters.get("prime_and_sub_award_types") is not None: award_type_codes = set( filters["prime_and_sub_award_types"][download_type]) else: award_type_codes = set(filters["award_type_codes"]) if (award_type_codes & (set(contract_type_mapping.keys()) | set(idv_type_mapping.keys())) or "procurement" in award_type_codes): # only generate d1 files if the user is asking for contract data d1_source = DownloadSource( VALUE_MAPPINGS[download_type]["table_name"], "d1", download_type, agency_id) d1_filters = { f"{VALUE_MAPPINGS[download_type]['contract_data']}__isnull": False } d1_source.queryset = queryset & download_type_table.objects.filter( **d1_filters) download_sources.append(d1_source) if award_type_codes & set(assistance_type_mapping.keys()) or ( "grant" in award_type_codes): # only generate d2 files if the user is asking for assistance data d2_source = DownloadSource( VALUE_MAPPINGS[download_type]["table_name"], "d2", download_type, agency_id) d2_filters = { f"{VALUE_MAPPINGS[download_type]['assistance_data']}__isnull": False } d2_source.queryset = queryset & download_type_table.objects.filter( **d2_filters) download_sources.append(d2_source) elif VALUE_MAPPINGS[download_type]["source_type"] == "account": # Account downloads account_source = DownloadSource( VALUE_MAPPINGS[download_type]["table_name"], json_request["account_level"], download_type, agency_id) account_source.queryset = filter_function( download_type, VALUE_MAPPINGS[download_type]["table"], json_request["filters"], json_request["account_level"], ) download_sources.append(account_source) verify_requested_columns_available(tuple(download_sources), json_request.get("columns", [])) return download_sources
from usaspending_api.common.helpers.dict_helpers import order_nested_object from usaspending_api.common.helpers.generic_helper import generate_fiscal_year from usaspending_api.common.sqs.sqs_handler import get_sqs_queue from usaspending_api.download.filestreaming import download_generation from usaspending_api.download.helpers import multipart_upload, pull_modified_agencies_cgacs from usaspending_api.download.lookups import JOB_STATUS_DICT from usaspending_api.download.models import DownloadJob from usaspending_api.download.v2.request_validations import validate_award_request from usaspending_api.download.v2.year_limited_downloads import YearLimitedDownloadViewSet from usaspending_api.references.models import ToptierAgency logger = logging.getLogger(__name__) award_mappings = { "contracts": list(procurement_type_mapping.keys()), "assistance": list(assistance_type_mapping.keys()), } class Command(BaseCommand): def download( self, file_name, prime_award_types=None, agency=None, sub_agency=None, date_type=None, start_date=None, end_date=None, columns=[], file_format="csv",