def get_csv_sources(json_request):
csv_sources = []
for download_type in json_request['download_types']:
agency_id = json_request['filters'].get('agency', 'all')
filter_function = VALUE_MAPPINGS[download_type]['filter_function']
download_type_table = VALUE_MAPPINGS[download_type]['table']
if VALUE_MAPPINGS[download_type]['source_type'] == 'award':
# Award downloads
queryset = filter_function(json_request['filters'])
award_type_codes = set(json_request['filters']['award_type_codes'])
if award_type_codes & (set(contract_type_mapping.keys())
| set(idv_type_mapping.keys())):
# only generate d1 files if the user is asking for contract data
d1_source = CsvSource(
VALUE_MAPPINGS[download_type]['table_name'], 'd1',
download_type, agency_id)
d1_filters = {
'{}__isnull'.format(VALUE_MAPPINGS[download_type]['contract_data']):
False
}
d1_source.queryset = queryset & download_type_table.objects.filter(
**d1_filters)
csv_sources.append(d1_source)
if award_type_codes & set(assistance_type_mapping.keys()):
# only generate d2 files if the user is asking for assistance data
d2_source = CsvSource(
VALUE_MAPPINGS[download_type]['table_name'], 'd2',
download_type, agency_id)
d2_filters = {
'{}__isnull'.format(VALUE_MAPPINGS[download_type]['assistance_data']):
False
}
d2_source.queryset = queryset & download_type_table.objects.filter(
**d2_filters)
csv_sources.append(d2_source)
verify_requested_columns_available(tuple(csv_sources),
json_request.get('columns', []))
elif VALUE_MAPPINGS[download_type]['source_type'] == 'account':
# Account downloads
account_source = CsvSource(
VALUE_MAPPINGS[download_type]['table_name'],
json_request['account_level'], download_type, agency_id)
account_source.queryset = filter_function(
download_type, VALUE_MAPPINGS[download_type]['table'],
json_request['filters'], json_request['account_level'])
csv_sources.append(account_source)
return csv_sources
def get_csv_sources(json_request):
csv_sources = []
for download_type in json_request["download_types"]:
agency_id = json_request["filters"].get("agency", "all")
filter_function = VALUE_MAPPINGS[download_type]["filter_function"]
download_type_table = VALUE_MAPPINGS[download_type]["table"]
if VALUE_MAPPINGS[download_type]["source_type"] == "award":
# Award downloads
# Use correct date range columns for advanced search
# (Will not change anything for keyword search since "time_period" is not provided))
filters = add_date_range_comparison_types(
json_request["filters"],
is_subaward=download_type != "awards",
gte_date_type="action_date",
lte_date_type="date_signed",
)
queryset = filter_function(filters)
award_type_codes = set(filters["award_type_codes"])
if award_type_codes & (set(contract_type_mapping.keys()) | set(idv_type_mapping.keys())):
# only generate d1 files if the user is asking for contract data
d1_source = CsvSource(VALUE_MAPPINGS[download_type]["table_name"], "d1", download_type, agency_id)
d1_filters = {"{}__isnull".format(VALUE_MAPPINGS[download_type]["contract_data"]): False}
d1_source.queryset = queryset & download_type_table.objects.filter(**d1_filters)
csv_sources.append(d1_source)
if award_type_codes & set(assistance_type_mapping.keys()):
# only generate d2 files if the user is asking for assistance data
d2_source = CsvSource(VALUE_MAPPINGS[download_type]["table_name"], "d2", download_type, agency_id)
d2_filters = {"{}__isnull".format(VALUE_MAPPINGS[download_type]["assistance_data"]): False}
d2_source.queryset = queryset & download_type_table.objects.filter(**d2_filters)
csv_sources.append(d2_source)
verify_requested_columns_available(tuple(csv_sources), json_request.get("columns", []))
elif VALUE_MAPPINGS[download_type]["source_type"] == "account":
# Account downloads
account_source = CsvSource(
VALUE_MAPPINGS[download_type]["table_name"], json_request["account_level"], download_type, agency_id
)
account_source.queryset = filter_function(
download_type,
VALUE_MAPPINGS[download_type]["table"],
json_request["filters"],
json_request["account_level"],
)
csv_sources.append(account_source)
return csv_sources
def download(self, award_type, agency='all', generate_since=None):
""" Create a delta file based on award_type, and agency_code (or all agencies) """
logger.info('Starting generation. {}, Agency: {}'.format(award_type, agency if agency == 'all' else
agency['name']))
award_map = AWARD_MAPPINGS[award_type]
# Create Source and update fields to include correction_delete_ind
source = CsvSource('transaction', award_map['letter_name'].lower(), 'transactions',
'all' if agency == 'all' else agency['toptier_agency_id'])
source.query_paths.update({
'correction_delete_ind': award_map['correction_delete_ind']
})
if award_type == 'Contracts':
# Add the agency_id column to the mappings
source.query_paths.update({'agency_id': 'transaction__contract_data__agency_id'})
source.query_paths.move_to_end('agency_id', last=False)
source.query_paths.move_to_end('correction_delete_ind', last=False)
source.human_names = list(source.query_paths.keys())
# Apply filters to the queryset
filters, agency_code = self.parse_filters(award_map['award_types'], agency)
source.queryset = VALUE_MAPPINGS['transactions']['filter_function'](filters)
if award_type == 'Contracts':
# Derive the correction_delete_ind from the created_at of the records
source.queryset = source.queryset. \
annotate(correction_delete_ind=Case(When(transaction__contract_data__created_at__lt=generate_since,
then=Value('C')), default=Value(''), output_field=CharField()))
source.queryset = source.queryset.filter(**{
'transaction__{}__{}__gte'.format(award_map['model'], award_map['date_filter']): generate_since
})
# Generate file
file_path = self.create_local_file(award_type, source, agency_code, generate_since)
if file_path is None:
logger.info('No new, modified, or deleted data; discarding file')
elif not settings.IS_LOCAL:
# Upload file to S3 and delete local version
logger.info('Uploading file to S3 bucket and deleting local copy')
multipart_upload(settings.MONTHLY_DOWNLOAD_S3_BUCKET_NAME, settings.USASPENDING_AWS_REGION, file_path,
os.path.basename(file_path))
os.remove(file_path)
logger.info('Finished generation. {}, Agency: {}'.format(award_type, agency if agency == 'all' else
agency['name']))
def download(self, award_type, agency='all', generate_since=None):
""" Create a delta file based on award_type, and agency_code (or all agencies) """
logger.info('Starting generation. {}, Agency: {}'.format(
award_type, agency if agency == 'all' else agency['name']))
award_map = AWARD_MAPPINGS[award_type]
# Create Source and update fields to include correction_delete_ind
source = CsvSource(
'transaction', award_map['letter_name'].lower(), 'transactions',
'all' if agency == 'all' else agency['toptier_agency_id'])
source.query_paths.update(
{'correction_delete_ind': award_map['correction_delete_ind']})
if award_type == 'Contracts':
# Add the agency_id column to the mappings
source.query_paths.update(
{'agency_id': 'transaction__contract_data__agency_id'})
source.query_paths.move_to_end('agency_id', last=False)
source.query_paths.move_to_end('correction_delete_ind', last=False)
source.human_names = list(source.query_paths.keys())
# Apply filters to the queryset
filters, agency_code = self.parse_filters(award_map['award_types'],
agency)
source.queryset = VALUE_MAPPINGS['transactions']['filter_function'](
filters)
if award_type == 'Contracts':
# Derive the correction_delete_ind from the created_at of the records
source.queryset = source.queryset. \
annotate(correction_delete_ind=Case(When(transaction__contract_data__created_at__lt=generate_since,
then=Value('C')), default=Value(''), output_field=CharField()))
source.queryset = source.queryset.filter(
**{
'transaction__{}__{}__gte'.format(award_map['model'], award_map['date_filter']):
generate_since
})
# Generate file
file_path = self.create_local_file(award_type, source, agency_code,
generate_since)
if file_path is None:
logger.info('No new, modified, or deleted data; discarding file')
elif not settings.IS_LOCAL:
# Upload file to S3 and delete local version
logger.info('Uploading file to S3 bucket and deleting local copy')
multipart_upload(settings.MONTHLY_DOWNLOAD_S3_BUCKET_NAME,
settings.USASPENDING_AWS_REGION, file_path,
os.path.basename(file_path))
os.remove(file_path)
logger.info('Finished generation. {}, Agency: {}'.format(
award_type, agency if agency == 'all' else agency['name']))
def download(self, award_type, agency="all", generate_since=None):
""" Create a delta file based on award_type, and agency_code (or all agencies) """
logger.info("Starting generation. {}, Agency: {}".format(
award_type, agency if agency == "all" else agency["name"]))
award_map = AWARD_MAPPINGS[award_type]
# Create Source and update fields to include correction_delete_ind
source = CsvSource(
"transaction",
award_map["letter_name"].lower(),
"transactions",
"all" if agency == "all" else agency["toptier_agency_id"],
)
source.query_paths.update(
{"correction_delete_ind": award_map["correction_delete_ind"]})
if award_type == "Contracts":
# Add the agency_id column to the mappings
source.query_paths.update(
{"agency_id": "transaction__contract_data__agency_id"})
source.query_paths.move_to_end("agency_id", last=False)
source.query_paths.move_to_end("correction_delete_ind", last=False)
source.human_names = list(source.query_paths.keys())
# Apply filters to the queryset
filters, agency_code = self.parse_filters(award_map["award_types"],
agency)
source.queryset = VALUE_MAPPINGS["transactions"]["filter_function"](
filters)
if award_type == "Contracts":
source.queryset = source.queryset.annotate(
correction_delete_ind=Case(
When(transaction__contract_data__created_at__lt=
generate_since,
then=Value("C")),
default=Value(""),
output_field=CharField(),
))
else:
indicator_field = F(
"transaction__assistance_data__correction_delete_indicatr")
source.queryset = source.queryset.annotate(
correction_delete_ind=Case(
When(transaction__assistance_data__updated_at__gt=
generate_since,
then=indicator_field),
When(transaction__transactiondelta__isnull=False,
then=Value("C")),
default=indicator_field,
output_field=CharField(),
))
transaction_delta_queryset = source.queryset
_filter = {
"transaction__{}__{}__gte".format(award_map["model"], award_map["date_filter"]):
generate_since
}
if self.debugging_end_date:
_filter["transaction__{}__{}__lt".format(
award_map["model"],
award_map["date_filter"])] = self.debugging_end_date
source.queryset = source.queryset.filter(**_filter)
# UNION the normal results to the transaction_delta results.
source.queryset = source.queryset.union(
transaction_delta_queryset.filter(
transaction__transactiondelta__isnull=False))
# Generate file
file_path = self.create_local_file(award_type, source, agency_code,
generate_since)
if file_path is None:
logger.info("No new, modified, or deleted data; discarding file")
elif not settings.IS_LOCAL:
# Upload file to S3 and delete local version
logger.info("Uploading file to S3 bucket and deleting local copy")
multipart_upload(
settings.MONTHLY_DOWNLOAD_S3_BUCKET_NAME,
settings.USASPENDING_AWS_REGION,
file_path,
os.path.basename(file_path),
)
os.remove(file_path)
logger.info("Finished generation. {}, Agency: {}".format(
award_type, agency if agency == "all" else agency["name"]))