def get_csv_sources(json_request): csv_sources = [] for download_type in json_request['download_types']: agency_id = json_request['filters'].get('agency', 'all') filter_function = VALUE_MAPPINGS[download_type]['filter_function'] download_type_table = VALUE_MAPPINGS[download_type]['table'] if VALUE_MAPPINGS[download_type]['source_type'] == 'award': # Award downloads queryset = filter_function(json_request['filters']) award_type_codes = set(json_request['filters']['award_type_codes']) if award_type_codes & (set(contract_type_mapping.keys()) | set(idv_type_mapping.keys())): # only generate d1 files if the user is asking for contract data d1_source = CsvSource( VALUE_MAPPINGS[download_type]['table_name'], 'd1', download_type, agency_id) d1_filters = { '{}__isnull'.format(VALUE_MAPPINGS[download_type]['contract_data']): False } d1_source.queryset = queryset & download_type_table.objects.filter( **d1_filters) csv_sources.append(d1_source) if award_type_codes & set(assistance_type_mapping.keys()): # only generate d2 files if the user is asking for assistance data d2_source = CsvSource( VALUE_MAPPINGS[download_type]['table_name'], 'd2', download_type, agency_id) d2_filters = { '{}__isnull'.format(VALUE_MAPPINGS[download_type]['assistance_data']): False } d2_source.queryset = queryset & download_type_table.objects.filter( **d2_filters) csv_sources.append(d2_source) verify_requested_columns_available(tuple(csv_sources), json_request.get('columns', [])) elif VALUE_MAPPINGS[download_type]['source_type'] == 'account': # Account downloads account_source = CsvSource( VALUE_MAPPINGS[download_type]['table_name'], json_request['account_level'], download_type, agency_id) account_source.queryset = filter_function( download_type, VALUE_MAPPINGS[download_type]['table'], json_request['filters'], json_request['account_level']) csv_sources.append(account_source) return csv_sources
def get_csv_sources(json_request): csv_sources = [] for download_type in json_request["download_types"]: agency_id = json_request["filters"].get("agency", "all") filter_function = VALUE_MAPPINGS[download_type]["filter_function"] download_type_table = VALUE_MAPPINGS[download_type]["table"] if VALUE_MAPPINGS[download_type]["source_type"] == "award": # Award downloads # Use correct date range columns for advanced search # (Will not change anything for keyword search since "time_period" is not provided)) filters = add_date_range_comparison_types( json_request["filters"], is_subaward=download_type != "awards", gte_date_type="action_date", lte_date_type="date_signed", ) queryset = filter_function(filters) award_type_codes = set(filters["award_type_codes"]) if award_type_codes & (set(contract_type_mapping.keys()) | set(idv_type_mapping.keys())): # only generate d1 files if the user is asking for contract data d1_source = CsvSource(VALUE_MAPPINGS[download_type]["table_name"], "d1", download_type, agency_id) d1_filters = {"{}__isnull".format(VALUE_MAPPINGS[download_type]["contract_data"]): False} d1_source.queryset = queryset & download_type_table.objects.filter(**d1_filters) csv_sources.append(d1_source) if award_type_codes & set(assistance_type_mapping.keys()): # only generate d2 files if the user is asking for assistance data d2_source = CsvSource(VALUE_MAPPINGS[download_type]["table_name"], "d2", download_type, agency_id) d2_filters = {"{}__isnull".format(VALUE_MAPPINGS[download_type]["assistance_data"]): False} d2_source.queryset = queryset & download_type_table.objects.filter(**d2_filters) csv_sources.append(d2_source) verify_requested_columns_available(tuple(csv_sources), json_request.get("columns", [])) elif VALUE_MAPPINGS[download_type]["source_type"] == "account": # Account downloads account_source = CsvSource( VALUE_MAPPINGS[download_type]["table_name"], json_request["account_level"], download_type, agency_id ) account_source.queryset = filter_function( download_type, VALUE_MAPPINGS[download_type]["table"], json_request["filters"], json_request["account_level"], ) csv_sources.append(account_source) return csv_sources
def download(self, award_type, agency='all', generate_since=None): """ Create a delta file based on award_type, and agency_code (or all agencies) """ logger.info('Starting generation. {}, Agency: {}'.format(award_type, agency if agency == 'all' else agency['name'])) award_map = AWARD_MAPPINGS[award_type] # Create Source and update fields to include correction_delete_ind source = CsvSource('transaction', award_map['letter_name'].lower(), 'transactions', 'all' if agency == 'all' else agency['toptier_agency_id']) source.query_paths.update({ 'correction_delete_ind': award_map['correction_delete_ind'] }) if award_type == 'Contracts': # Add the agency_id column to the mappings source.query_paths.update({'agency_id': 'transaction__contract_data__agency_id'}) source.query_paths.move_to_end('agency_id', last=False) source.query_paths.move_to_end('correction_delete_ind', last=False) source.human_names = list(source.query_paths.keys()) # Apply filters to the queryset filters, agency_code = self.parse_filters(award_map['award_types'], agency) source.queryset = VALUE_MAPPINGS['transactions']['filter_function'](filters) if award_type == 'Contracts': # Derive the correction_delete_ind from the created_at of the records source.queryset = source.queryset. \ annotate(correction_delete_ind=Case(When(transaction__contract_data__created_at__lt=generate_since, then=Value('C')), default=Value(''), output_field=CharField())) source.queryset = source.queryset.filter(**{ 'transaction__{}__{}__gte'.format(award_map['model'], award_map['date_filter']): generate_since }) # Generate file file_path = self.create_local_file(award_type, source, agency_code, generate_since) if file_path is None: logger.info('No new, modified, or deleted data; discarding file') elif not settings.IS_LOCAL: # Upload file to S3 and delete local version logger.info('Uploading file to S3 bucket and deleting local copy') multipart_upload(settings.MONTHLY_DOWNLOAD_S3_BUCKET_NAME, settings.USASPENDING_AWS_REGION, file_path, os.path.basename(file_path)) os.remove(file_path) logger.info('Finished generation. {}, Agency: {}'.format(award_type, agency if agency == 'all' else agency['name']))
def download(self, award_type, agency='all', generate_since=None): """ Create a delta file based on award_type, and agency_code (or all agencies) """ logger.info('Starting generation. {}, Agency: {}'.format( award_type, agency if agency == 'all' else agency['name'])) award_map = AWARD_MAPPINGS[award_type] # Create Source and update fields to include correction_delete_ind source = CsvSource( 'transaction', award_map['letter_name'].lower(), 'transactions', 'all' if agency == 'all' else agency['toptier_agency_id']) source.query_paths.update( {'correction_delete_ind': award_map['correction_delete_ind']}) if award_type == 'Contracts': # Add the agency_id column to the mappings source.query_paths.update( {'agency_id': 'transaction__contract_data__agency_id'}) source.query_paths.move_to_end('agency_id', last=False) source.query_paths.move_to_end('correction_delete_ind', last=False) source.human_names = list(source.query_paths.keys()) # Apply filters to the queryset filters, agency_code = self.parse_filters(award_map['award_types'], agency) source.queryset = VALUE_MAPPINGS['transactions']['filter_function']( filters) if award_type == 'Contracts': # Derive the correction_delete_ind from the created_at of the records source.queryset = source.queryset. \ annotate(correction_delete_ind=Case(When(transaction__contract_data__created_at__lt=generate_since, then=Value('C')), default=Value(''), output_field=CharField())) source.queryset = source.queryset.filter( **{ 'transaction__{}__{}__gte'.format(award_map['model'], award_map['date_filter']): generate_since }) # Generate file file_path = self.create_local_file(award_type, source, agency_code, generate_since) if file_path is None: logger.info('No new, modified, or deleted data; discarding file') elif not settings.IS_LOCAL: # Upload file to S3 and delete local version logger.info('Uploading file to S3 bucket and deleting local copy') multipart_upload(settings.MONTHLY_DOWNLOAD_S3_BUCKET_NAME, settings.USASPENDING_AWS_REGION, file_path, os.path.basename(file_path)) os.remove(file_path) logger.info('Finished generation. {}, Agency: {}'.format( award_type, agency if agency == 'all' else agency['name']))
def download(self, award_type, agency="all", generate_since=None): """ Create a delta file based on award_type, and agency_code (or all agencies) """ logger.info("Starting generation. {}, Agency: {}".format( award_type, agency if agency == "all" else agency["name"])) award_map = AWARD_MAPPINGS[award_type] # Create Source and update fields to include correction_delete_ind source = CsvSource( "transaction", award_map["letter_name"].lower(), "transactions", "all" if agency == "all" else agency["toptier_agency_id"], ) source.query_paths.update( {"correction_delete_ind": award_map["correction_delete_ind"]}) if award_type == "Contracts": # Add the agency_id column to the mappings source.query_paths.update( {"agency_id": "transaction__contract_data__agency_id"}) source.query_paths.move_to_end("agency_id", last=False) source.query_paths.move_to_end("correction_delete_ind", last=False) source.human_names = list(source.query_paths.keys()) # Apply filters to the queryset filters, agency_code = self.parse_filters(award_map["award_types"], agency) source.queryset = VALUE_MAPPINGS["transactions"]["filter_function"]( filters) if award_type == "Contracts": source.queryset = source.queryset.annotate( correction_delete_ind=Case( When(transaction__contract_data__created_at__lt= generate_since, then=Value("C")), default=Value(""), output_field=CharField(), )) else: indicator_field = F( "transaction__assistance_data__correction_delete_indicatr") source.queryset = source.queryset.annotate( correction_delete_ind=Case( When(transaction__assistance_data__updated_at__gt= generate_since, then=indicator_field), When(transaction__transactiondelta__isnull=False, then=Value("C")), default=indicator_field, output_field=CharField(), )) transaction_delta_queryset = source.queryset _filter = { "transaction__{}__{}__gte".format(award_map["model"], award_map["date_filter"]): generate_since } if self.debugging_end_date: _filter["transaction__{}__{}__lt".format( award_map["model"], award_map["date_filter"])] = self.debugging_end_date source.queryset = source.queryset.filter(**_filter) # UNION the normal results to the transaction_delta results. source.queryset = source.queryset.union( transaction_delta_queryset.filter( transaction__transactiondelta__isnull=False)) # Generate file file_path = self.create_local_file(award_type, source, agency_code, generate_since) if file_path is None: logger.info("No new, modified, or deleted data; discarding file") elif not settings.IS_LOCAL: # Upload file to S3 and delete local version logger.info("Uploading file to S3 bucket and deleting local copy") multipart_upload( settings.MONTHLY_DOWNLOAD_S3_BUCKET_NAME, settings.USASPENDING_AWS_REGION, file_path, os.path.basename(file_path), ) os.remove(file_path) logger.info("Finished generation. {}, Agency: {}".format( award_type, agency if agency == "all" else agency["name"]))