def test_spending_over_time(client, monkeypatch, elasticsearch_transaction_index, mock_tas_data): setup_elasticsearch_test(monkeypatch, elasticsearch_transaction_index) data = {"group": "fiscal_year", "filters": {"tas_codes": [{"aid": "028", "main": "8006"}]}, "subawards": False} resp = client.post("/api/v2/search/spending_over_time", content_type="application/json", data=json.dumps(data)) assert resp.status_code == status.HTTP_200_OK earliest_fiscal_year_we_care_about = datetime.strptime(settings.API_SEARCH_MIN_DATE, "%Y-%m-%d").year assert len(resp.data["results"]) == FiscalDate.today().fiscal_year - earliest_fiscal_year_we_care_about
def test_create_fiscal_year_list(): assert fyh.create_fiscal_year_list( start_year=2004, end_year=2008) == [2004, 2005, 2006, 2007] years = [ x for x in range(2000, FiscalDate.today().next_fiscal_year.fiscal_year) ] assert fyh.create_fiscal_year_list() == years
def test_contains(self, a: FiscalQuarter, f: FiscalQuarter) -> None: assert a not in f assert f in f assert FiscalDateTime(2016, 8, 1, 0, 0, 0) in a assert datetime.datetime(2016, 8, 1, 0, 0, 0) in a assert FiscalDate(2016, 8, 1) in a assert datetime.date(2016, 8, 1) in a
def test_contains(self, a: FiscalMonth, b: FiscalMonth, d: FiscalQuarter) -> None: assert b in b assert a not in d assert b in b assert FiscalDateTime(2015, 10, 1, 0, 0, 0) in a assert datetime.datetime(2015, 10, 1, 0, 0, 0) in a assert FiscalDate(2015, 10, 1) in a assert datetime.date(2015, 10, 1) in a
def test_contains(self, a: FiscalYear, b: FiscalYear, c: FiscalYear, d: FiscalYear) -> None: assert b in b assert c not in a assert c in b assert d in b assert FiscalDateTime(2016, 1, 1, 0, 0, 0) in a assert datetime.datetime(2016, 1, 1, 0, 0, 0) in a assert FiscalDate(2016, 1, 1) in a assert datetime.date(2016, 1, 1) in a
def test_leap_year(self) -> None: assert FiscalDate(2016, 1, 1).fiscal_day == 93 assert FiscalDate(2016, 2, 29).fiscal_day == 152 assert FiscalDate(2017, 3, 1).fiscal_day == 152 assert FiscalDate(2016, 9, 30).fiscal_day == 366 assert FiscalDate(2017, 9, 30).fiscal_day == 365 assert FiscalDate(2018, 9, 30).fiscal_day == 365
def create_fiscal_year_list(start_year=2000, end_year=None): """ return the list of fiscal year as integers start_year: int default 2000 FY to start at (inclusive) end_year: int default None: FY to end at (exclusive) if no end_date is provided, use the current FY """ if end_year is None: # to return the current FY, we add 1 here for the range generator below end_year = FiscalDate.today().next_fiscal_year.fiscal_year if start_year is None or start_year >= end_year: raise Exception("Invalid start_year and end_year values") return [year for year in range(start_year, end_year)]
def test_complex(self) -> None: # Test defaults day = FiscalDate(2017, 12, 1) assert day.fiscal_year == 2018 assert day.fiscal_quarter == 1 # Change fiscal year settings fiscalyear.setup_fiscal_calendar("same", 1, 1) assert day.fiscal_year == 2017 assert day.fiscal_quarter == 4 # Restore defaults and re-test fiscalyear.setup_fiscal_calendar("previous", 10, 1) assert day.fiscal_year == 2018 assert day.fiscal_quarter == 1
def generate_fiscal_quarter(date): """ Generate fiscal quarter based on the date provided """ validate_date(date) return FiscalDate(date.year, date.month, date.day).quarter
def post(self, request): """Return all budget function/subfunction titles matching the provided search text""" valid_groups = ['quarter', 'fiscal_year', 'month', 'fy', 'q', 'm'] models = [ {'name': 'subawards', 'key': 'subawards', 'type': 'boolean', 'default': False}, {'name': 'group', 'key': 'group', 'type': 'enum', 'enum_values': valid_groups, 'optional': False} ] models.extend(copy.deepcopy(AWARD_FILTER)) models.extend(copy.deepcopy(PAGINATION)) json_request = TinyShield(models).block(request.data) group = json_request['group'] subawards = json_request['subawards'] filters = json_request.get("filters", None) if filters is None: raise InvalidParameterException('Missing request parameters: filters') # define what values are needed in the sql query # we do not use matviews for Subaward filtering, just the Subaward download filters if subawards: queryset = subaward_filter(filters) else: queryset = spending_over_time(filters).values('action_date', 'generated_pragmatic_obligation') # build response response = {'group': group, 'results': []} nested_order = '' # list of time_period objects ie {"fy": "2017", "quarter": "3"} : 1000 group_results = OrderedDict() # for Subawards we extract data from action_date if subawards: data_set = queryset \ .values('award_type') \ .annotate(month=ExtractMonth('action_date'), transaction_amount=Sum('amount')) \ .values('month', 'fiscal_year', 'transaction_amount') else: # for Awards we Sum generated_pragmatic_obligation for transaction_amount queryset = queryset.values('fiscal_year') if group in ('fy', 'fiscal_year'): data_set = queryset \ .annotate(transaction_amount=Sum('generated_pragmatic_obligation')) \ .values('fiscal_year', 'transaction_amount') else: # quarterly also takes months and aggregates the data data_set = queryset \ .annotate( month=ExtractMonth('action_date'), transaction_amount=Sum('generated_pragmatic_obligation')) \ .values('fiscal_year', 'month', 'transaction_amount') for record in data_set: # generate unique key by fiscal date, depending on group key = {'fiscal_year': str(record['fiscal_year'])} if group in ('m', 'month'): # generate the fiscal month key['month'] = generate_fiscal_month(date(year=2017, day=1, month=record['month'])) nested_order = 'month' elif group in ('q', 'quarter'): # generate the fiscal quarter key['quarter'] = FiscalDate(2017, record['month'], 1).quarter nested_order = 'quarter' key = str(key) # if key exists, aggregate if group_results.get(key) is None: group_results[key] = record['transaction_amount'] else: group_results[key] = group_results.get(key) + record['transaction_amount'] # convert result into expected format, sort by key to meet front-end specs results = [] # Expected results structure # [{ # 'time_period': {'fy': '2017', 'quarter': '3'}, # 'aggregated_amount': '200000000' # }] sorted_group_results = sorted( group_results.items(), key=lambda k: ( ast.literal_eval(k[0])['fiscal_year'], int(ast.literal_eval(k[0])[nested_order])) if nested_order else (ast.literal_eval(k[0])['fiscal_year'])) for key, value in sorted_group_results: key_dict = ast.literal_eval(key) result = {'time_period': key_dict, 'aggregated_amount': float(value) if value else float(0)} results.append(result) response['results'] = results return Response(response)
def post(self, request): """Return all budget function/subfunction titles matching the provided search text""" json_request = request.data group = json_request.get('group', None) filters = json_request.get('filters', None) subawards = json_request.get('subawards', False) if group is None: raise InvalidParameterException('Missing one or more required request parameters: group') if filters is None: raise InvalidParameterException('Missing one or more required request parameters: filters') potential_groups = ['quarter', 'fiscal_year', 'month', 'fy', 'q', 'm'] if group not in potential_groups: raise InvalidParameterException('group does not have a valid value') if type(subawards) is not bool: raise InvalidParameterException('subawards does not have a valid value') # define what values are needed in the sql query # we do not use matviews for Subaward filtering, just the Subaward download filters queryset = subaward_filter(filters) if subawards else spending_over_time(filters) \ .values('action_date', 'federal_action_obligation', 'original_loan_subsidy_cost') # build response response = {'group': group, 'results': []} nested_order = '' # list of time_period objects ie {"fy": "2017", "quarter": "3"} : 1000 group_results = OrderedDict() # for Subawards we extract data from action_date, for Awards we use sum_transaction_amount if subawards: data_set = queryset.values('award_type'). \ annotate(month=ExtractMonth('action_date'), year=ExtractYear('action_date'), transaction_amount=Sum('amount')). \ values('month', 'year', 'transaction_amount') else: data_set = queryset.values('fiscal_year') if not (group == 'fy' or group == 'fiscal_year'): # quarterly also takes months and aggregates the data data_set = queryset.annotate(month=ExtractMonth('action_date')).values('fiscal_year', 'month') filter_types = filters['award_type_codes'] if 'award_type_codes' in filters else award_type_mapping data_set = sum_transaction_amount(data_set, filter_types=filter_types) for record in data_set: # create fiscal year data based on the action_date for Subawards if subawards: record['fiscal_year'] = generate_fiscal_year(date(record['year'], record['month'], 1)) # generate unique key by fiscal date, depending on group key = {'fiscal_year': str(record['fiscal_year'])} if group == 'm' or group == 'month': # generate the fiscal month key['month'] = generate_fiscal_month(date(year=2017, day=1, month=record['month'])) nested_order = 'month' elif group == 'q' or group == 'quarter': # generate the fiscal quarter key['quarter'] = FiscalDate(2017, record['month'], 1).quarter nested_order = 'quarter' key = str(key) # if key exists, aggregate if group_results.get(key) is None: group_results[key] = record['transaction_amount'] else: group_results[key] = group_results.get(key) + record['transaction_amount'] # convert result into expected format, sort by key to meet front-end specs results = [] # Expected results structure # [{ # 'time_period': {'fy': '2017', 'quarter': '3'}, # 'aggregated_amount': '200000000' # }] sorted_group_results = sorted( group_results.items(), key=lambda k: ( ast.literal_eval(k[0])['fiscal_year'], int(ast.literal_eval(k[0])[nested_order])) if nested_order else (ast.literal_eval(k[0])['fiscal_year'])) for key, value in sorted_group_results: key_dict = ast.literal_eval(key) result = {'time_period': key_dict, 'aggregated_amount': float(value) if value else float(0)} results.append(result) response['results'] = results return Response(response)
def b(self) -> FiscalDate: return FiscalDate(2017, 11, 15)
def a(self) -> FiscalDate: return FiscalDate(2017, 1, 1)
def today() -> FiscalDate: return FiscalDate(2016, 10, 1)
def test_current_fiscal_year(): current_fiscal_year = FiscalDate.today().fiscal_year fiscal_year_list = fyh.create_fiscal_year_list(start_year=2010) assert fiscal_year_list[0] == 2010 assert fiscal_year_list[-1] == current_fiscal_year
def generate_fiscal_year_and_quarter(date): validate_date(date) quarter = FiscalDate(date.year, date.month, date.day).quarter year = generate_fiscal_year(date) return "{}-Q{}".format(year, quarter)
def post(self, request): """Return all budget function/subfunction titles matching the provided search text""" json_request = request.data group = json_request.get('group', None) filters = json_request.get('filters', None) if group is None: raise InvalidParameterException( 'Missing one or more required request parameters: group') if filters is None: raise InvalidParameterException( 'Missing one or more required request parameters: filters') potential_groups = ['quarter', 'fiscal_year', 'month', 'fy', 'q', 'm'] if group not in potential_groups: raise InvalidParameterException( 'group does not have a valid value') queryset = spending_over_time(filters) filter_types = filters[ 'award_type_codes'] if 'award_type_codes' in filters else award_type_mapping # define what values are needed in the sql query queryset = queryset.values('action_date', 'federal_action_obligation', 'original_loan_subsidy_cost') # build response response = {'group': group, 'results': []} nested_order = '' group_results = OrderedDict( ) # list of time_period objects ie {"fy": "2017", "quarter": "3"} : 1000 if group == 'fy' or group == 'fiscal_year': fy_set = sum_transaction_amount(queryset.values('fiscal_year'), filter_types=filter_types) for trans in fy_set: key = {'fiscal_year': str(trans['fiscal_year'])} key = str(key) group_results[key] = trans['transaction_amount'] elif group == 'm' or group == 'month': month_set = queryset.annotate(month=ExtractMonth('action_date')) \ .values('fiscal_year', 'month') month_set = sum_transaction_amount(month_set, filter_types=filter_types) for trans in month_set: # Convert month to fiscal month fiscal_month = generate_fiscal_month( date(year=2017, day=1, month=trans['month'])) key = { 'fiscal_year': str(trans['fiscal_year']), 'month': str(fiscal_month) } key = str(key) group_results[key] = trans['transaction_amount'] nested_order = 'month' else: # quarterly, take months and add them up month_set = queryset.annotate(month=ExtractMonth('action_date')) \ .values('fiscal_year', 'month') month_set = sum_transaction_amount(month_set, filter_types=filter_types) for trans in month_set: # Convert month to quarter quarter = FiscalDate(2017, trans['month'], 1).quarter key = { 'fiscal_year': str(trans['fiscal_year']), 'quarter': str(quarter) } key = str(key) # If key exists {fy : quarter}, aggregate if group_results.get(key) is None: group_results[key] = trans['transaction_amount'] else: if trans['transaction_amount']: group_results[key] = group_results.get( key) + trans['transaction_amount'] else: group_results[key] = group_results.get(key) nested_order = 'quarter' # convert result into expected format, sort by key to meet front-end specs results = [] # Expected results structure # [{ # 'time_period': {'fy': '2017', 'quarter': '3'}, # 'aggregated_amount': '200000000' # }] sorted_group_results = sorted( group_results.items(), key=lambda k: (ast.literal_eval(k[0])['fiscal_year'], int(ast.literal_eval(k[0])[nested_order])) if nested_order else (ast.literal_eval(k[0])['fiscal_year'])) for key, value in sorted_group_results: key_dict = ast.literal_eval(key) result = { 'time_period': key_dict, 'aggregated_amount': float(value) if value else float(0) } results.append(result) response['results'] = results return Response(response)