def _check_max(rule): value = rule["value"] if rule["type"] in ("integer", "float"): if value > rule["max"]: raise UnprocessableEntityException(ABOVE_MAXIMUM_MSG.format(**rule)) if rule["type"] in ("text", "enum", "array", "object"): if len(value) > rule["max"]: raise UnprocessableEntityException(ABOVE_MAXIMUM_MSG.format(**rule) + " items")
def _check_min(rule): value = rule["value"] if rule["type"] in ("integer", "float"): if value < rule["min"]: raise UnprocessableEntityException(BELOW_MINIMUM_MSG.format(**rule)) if rule["type"] in ("text", "enum", "array", "object"): if len(value) < rule["min"]: raise UnprocessableEntityException(BELOW_MINIMUM_MSG.format(**rule) + " items")
def _check_max(rule): value = rule['value'] if rule['type'] in ('integer', 'float'): if value > rule['max']: raise UnprocessableEntityException( ABOVE_MAXIMUM_MSG.format(**rule)) if rule['type'] in ('text', 'enum', 'array', 'object'): if len(value) > rule['max']: raise UnprocessableEntityException( ABOVE_MAXIMUM_MSG.format(**rule) + ' items')
def _check_min(rule): value = rule['value'] if rule['type'] in ('integer', 'float'): if value < rule['min']: raise UnprocessableEntityException( BELOW_MINIMUM_MSG.format(**rule)) if rule['type'] in ('text', 'enum', 'array', 'object'): if len(value) < rule['min']: raise UnprocessableEntityException( BELOW_MINIMUM_MSG.format(**rule) + ' items')
def validate_object(rule): provided_object = rule["value"] if type(provided_object) is not dict: raise InvalidParameterException(INVALID_TYPE_MSG.format(**rule)) if not provided_object: raise InvalidParameterException( "'{}' is empty. Please populate object".format(rule["key"])) for field in provided_object.keys(): if field not in rule["object_keys"].keys(): raise InvalidParameterException( "Unexpected field '{}' in parameter {}".format( field, rule["key"])) for key, value in rule["object_keys"].items(): if key not in provided_object: if "optional" in value and value["optional"] is False: raise UnprocessableEntityException( "Required object fields: {}".format( rule["object_keys"].keys())) else: continue return provided_object
def search_regex_of(v): if isinstance(v, str): v = string_to_dictionary(v, "aid") code_lookup = { "ata": v["ata"] if v.get("ata") else None, "aid": v["aid"] if v.get("aid") else ".*", "main": v["main"] if v.get("main") else ".*", "sub": v["sub"] if v.get("sub") else ".*", "bpoa": v["bpoa"] if v.get("bpoa") else ".*", "epoa": v["epoa"] if v.get("epoa") else ".*", "a": v["a"] if v.get("a") else ".*", } # This is NOT the order of elements as displayed in the tas rendering label, but instead the order in the award_delta_view and transaction_delta_view search_regex = TreasuryAppropriationAccount.generate_tas_rendering_label( code_lookup["ata"], code_lookup["aid"], code_lookup["a"], code_lookup["bpoa"], code_lookup["epoa"], code_lookup["main"], code_lookup["sub"], ) # TODO: move this to a Tinyshield filter if not re.match(r"^(\d|\w|-|\*|\.)+$", search_regex): raise UnprocessableEntityException( f"Unable to parse TAS filter {search_regex}") return search_regex
def validate_object(rule): provided_object = rule['value'] if type(provided_object) is not dict: raise InvalidParameterException(INVALID_TYPE_MSG.format(**rule)) if not provided_object: raise InvalidParameterException( '\'{}\' is empty. Please populate object'.format(rule['key'])) for field in provided_object.keys(): if field not in rule['object_keys'].keys(): raise InvalidParameterException( 'Unexpected field \'{}\' in parameter {}'.format( field, rule['key'])) for key, value in rule['object_keys'].items(): if key not in provided_object: if 'optional' in value and value['optional'] is False: raise UnprocessableEntityException( 'Required object fields: {}'.format( rule['object_keys'].keys())) else: continue return provided_object
def search_regex_of(v): if isinstance(v, str): v = string_to_dictionary(v, "agency") code_lookup = { "agency": f"agency={v['agency']}" if v.get("agency") else "*", "faaid": f"faaid={v['faaid']}" if v.get("faaid") else "*", "famain": f"famain={v['famain']}" if v.get("famain") else "*", "aid": f"aid={v['aid']}" if v.get("aid") else "*", "main": f"main={v['main']}" if v.get("main") else "*", "ata": f"ata={v['ata']}" if v.get("ata") else "*", "sub": f"sub={v['sub']}" if v.get("sub") else "*", "bpoa": f"bpoa={v['bpoa']}" if v.get("bpoa") else "*", "epoa": f"epoa={v['epoa']}" if v.get("epoa") else "*", "a": f"a={v['a']}" if v.get("a") else "*", } # This is NOT the order of elements as displayed in the tas rendering label, but instead the order in the award_delta_view and transaction_delta_view search_regex = (code_lookup["agency"] + code_lookup["faaid"] + code_lookup["famain"] + code_lookup["aid"] + code_lookup["main"] + code_lookup["ata"] + code_lookup["sub"] + code_lookup["bpoa"] + code_lookup["epoa"] + code_lookup["a"]) # TODO: move this to a Tinyshield filter if not re.match(r"^(\d|\w|-|\*|=)+$", search_regex): raise UnprocessableEntityException(f"Unable to parse TAS filter") return search_regex
def tas_rendering_label_to_component_dictionary( tas_rendering_label) -> dict: try: components = tas_rendering_label.split("-") if len(components) < 4 or len(components) > 5: raise Exception # don't have to be specific here since this is being swallowed and replaced retval = {} # we go in reverse, since the first component is the only optional one retval["sub"] = components[-1] retval["main"] = components[-2] # the third component from the back can either be two years, or one character if len(components[-3]) > 1: dates = components[-3].split("/") retval["bpoa"] = dates[0] retval["epoa"] = dates[1] else: retval["a"] = components[-3] retval["aid"] = components[-4] # ata may or may not be present if len(components) > 4: retval["ata"] = components[-5] return retval except Exception: raise UnprocessableEntityException( f"Cannot parse provided TAS: {tas_rendering_label}. Valid examples: 000-2010/2010-0400-000, 009-X-1701-000, 019-011-X-1071-000" )
def fiscal_year(self): fiscal_year = str( self.request.query_params.get("fiscal_year", current_fiscal_year())) if not fullmatch("[0-9]{4}", fiscal_year): raise UnprocessableEntityException( "Unrecognized fiscal_year format. Should be YYYY.") min_fiscal_year = fy(settings.API_SEARCH_MIN_DATE) fiscal_year = int(fiscal_year) if fiscal_year < min_fiscal_year: raise UnprocessableEntityException( f"fiscal_year is currently limited to an earliest year of {min_fiscal_year}." ) if fiscal_year > current_fiscal_year(): raise UnprocessableEntityException( f"fiscal_year may not exceed current fiscal year of {current_fiscal_year()}." ) return fiscal_year
def _check_datetime_min_max(rule, value, dt_format): # DEV-4097 introduces new behavior whereby minimum/maximum date violations should raise an error. To # implement this, we added a min/max_exception property to the rule. If that property exists, we will # raise an exception, otherwise we will retain the old behavior for backward compatibility. if "min" in rule: min_cap = datetime.datetime.strptime(rule["min"], dt_format) if value < min_cap: if "min_exception" in rule: message = rule["min_exception"].format(**rule) raise UnprocessableEntityException(message) logger.info("{}. Setting to {}".format(BELOW_MINIMUM_MSG.format(**rule), min_cap)) value = min_cap if "max" in rule: max_cap = datetime.datetime.strptime(rule["max"], dt_format) if value > max_cap: if "max_exception" in rule: message = rule["max_exception"].format(**rule) raise UnprocessableEntityException(message) logger.info(ABOVE_MAXIMUM_MSG.format(**rule)) value = max_cap
def raise_if_award_types_not_valid_subset(award_type_codes, is_subaward=False): """Test to ensure the award types are a subset of only one group. For Awards: contracts, idvs, direct_payments, loans, grants, other assistance For Sub-Awards: procurement, assistance If the types are not a valid subset: Raise API exception with a JSON response describing the award type groupings """ msg_head = '"message": "\'award_type_codes\' must only contain types from one group.","award_type_groups": ' if is_subaward: if not subaward_types_are_valid_groups(award_type_codes): # Return a JSON response describing the award type groupings error_msg = ('{{{} {{ "procurement": {}, "assistance": {}}}}}').format( msg_head, json.dumps(procurement_type_mapping), json.dumps(assistance_type_mapping) ) raise UnprocessableEntityException(json.loads(error_msg)) else: if not award_types_are_valid_groups(award_type_codes): error_msg = ( "{{{} {{" '"contracts": {},' '"loans": {},' '"idvs": {},' '"grants": {},' '"other_financial_assistance": {},' '"direct_payments": {}}}}}' ).format( msg_head, json.dumps(contract_type_mapping), json.dumps(loan_type_mapping), json.dumps(idv_type_mapping), json.dumps(grant_type_mapping), json.dumps(direct_payment_type_mapping), json.dumps(other_type_mapping), ) raise UnprocessableEntityException(json.loads(error_msg))
def split_filter_values(cls, filter_values): """ Here we assume that filter_values has already been run through validate_filter_values. """ if isinstance(filter_values, list): # Legacy is treated as a "require" filter. require = [[f] for f in filter_values] exclude = [] elif isinstance(filter_values, dict): # PSCCodeObject require = filter_values.get("require") or [] exclude = filter_values.get("exclude") or [] else: raise UnprocessableEntityException( f"psc_codes must be an array or object") return require, exclude
def apply_rule(self, rule): if rule.get('allow_nulls', False) and rule['value'] is None: return rule['value'] elif rule['type'] not in ('array', 'object'): if rule['type'] in VALIDATORS: return VALIDATORS[rule['type']]['func'](rule) else: raise Exception('Invalid Type {} in rule'.format(rule['type'])) # Array is a "special" type since it is a list of other types which need to be validated elif rule['type'] == 'array': rule['array_min'] = rule.get('array_min', 1) rule['array_max'] = rule.get('array_max', MAX_ITEMS) value = VALIDATORS[rule['type']]['func'](rule) child_rule = copy.copy(rule) child_rule['type'] = rule['array_type'] child_rule['min'] = rule.get('array_min') child_rule['max'] = rule.get('array_max') child_rule = self.promote_subrules(child_rule, child_rule) array_result = [] for v in value: child_rule['value'] = v array_result.append(self.apply_rule(child_rule)) return array_result # Object is a "special" type since it is comprised of other types which need to be validated elif rule['type'] == 'object': rule['object_min'] = rule.get('object_min', 1) rule['object_max'] = rule.get('object_max', MAX_ITEMS) provided_object = VALIDATORS[rule['type']]['func'](rule) object_result = {} for k, v in rule['object_keys'].items(): try: value = provided_object[k] except KeyError as e: if "optional" in v and v['optional'] is False: raise UnprocessableEntityException( 'Required object fields: {}'.format(k)) else: continue # Start with the sub-rule definition and supplement with parent's key-values as needed child_rule = copy.copy(v) child_rule['key'] = rule['key'] child_rule['value'] = value child_rule = self.promote_subrules(child_rule, v) object_result[k] = self.apply_rule(child_rule) return object_result
def query_elasticsearch(self) -> list: filter_query = QueryWithFilters.generate_awards_elasticsearch_query( self.filters) sort_field = self.get_elastic_sort_by_fields() sorts = [{ field: self.pagination["sort_order"] } for field in sort_field] record_num = (self.pagination["page"] - 1) * self.pagination["limit"] # random page jumping was removed due to performance concerns if (self.last_record_sort_value is None and self.last_record_unique_id is not None) or (self.last_record_sort_value is not None and self.last_record_unique_id is None): # malformed request raise Exception( "Using search_after functionality in Elasticsearch requires both last_record_sort_value and last_record_unique_id." ) if record_num >= settings.ES_AWARDS_MAX_RESULT_WINDOW and ( self.last_record_unique_id is None and self.last_record_sort_value is None): raise UnprocessableEntityException( "Page #{page} with limit {limit} is over the maximum result limit {es_limit}. Please provide the 'last_record_sort_value' and 'last_record_unique_id' to paginate sequentially." .format( page=self.pagination["page"], limit=self.pagination["limit"], es_limit=settings.ES_AWARDS_MAX_RESULT_WINDOW, )) # Search_after values are provided in the API request - use search after if self.last_record_sort_value is not None and self.last_record_unique_id is not None: search = ( AwardSearch().filter(filter_query).sort(*sorts).extra( search_after=[ self.last_record_sort_value, self.last_record_unique_id ])[:self.pagination["limit"] + 1] # add extra result to check for next page ) # no values, within result window, use regular elasticsearch else: search = AwardSearch().filter(filter_query).sort( *sorts)[record_num:record_num + self.pagination["limit"]] response = search.handle_execute() return response
def parse_request(self, request): for item in self.rules: # Loop through the request to find the expected key value = request for subkey in item["key"].split(TINY_SHIELD_SEPARATOR): value = value.get(subkey, {}) if value != {}: # Key found in provided request dictionary, use the value item["value"] = value elif item["optional"] is False: # If the value is required, raise exception since key wasn't found raise UnprocessableEntityException("Missing value: '{}' is a required field".format(item["key"])) elif "default" in item: # If value wasn't found, and this is optional, use the default item["value"] = item["default"] else: # This model/field is optional, no value provided, and no default value. # Use the "hidden" feature Ellipsis since None can be a valid value provided in the request item["value"] = ...
def post(self, request): additional_models = [ { "key": "filter|award_type", "name": "award_type", "type": "enum", "enum_values": ("assistance", "procurement"), "allow_nulls": False, "optional": True, } ] f = TinyShield(additional_models).block(self.request.data).get("filter") if f: self.filters["award_type"] = f.get("award_type") if all(x in self.filters for x in ["award_type_codes", "award_type"]): raise UnprocessableEntityException("Cannot provide both 'award_type_codes' and 'award_type'") if self.count_only: return Response({"count": self.aggregation["award_count"]}) else: return Response(self.aggregation)
def validate_filter_values(cls, filter_values): """ This is validation on top of whatever TinyShield performs. """ if isinstance(filter_values, list): # Legacy. for code in filter_values: if not isinstance( code, str) or not cls.validation_pattern.fullmatch(code): raise UnprocessableEntityException( f"PSC codes must be one to four character uppercased alphanumeric strings. " f"Offending code: '{code}'.") elif isinstance(filter_values, dict): # PSCCodeObject for key in ("require", "exclude"): code_lists = filter_values.get(key) or [] if not isinstance(code_lists, list): raise UnprocessableEntityException( f"require and exclude properties must be arrays of arrays." ) for code_list in code_lists: if not isinstance(code_list, list): raise UnprocessableEntityException( f"require and exclude properties must be arrays of arrays." ) for seq, code in enumerate(code_list): if seq == 0 and code not in PSC_GROUPS: raise UnprocessableEntityException( f"Tier1 PSC filter values must be one of: {tuple(PSC_GROUPS)}. " f"Offending code: '{code}'.") elif seq > 0 and ( not isinstance(code, str) or not cls.validation_pattern.fullmatch(code)): raise UnprocessableEntityException( f"PSC codes must be one to four character uppercased alphanumeric strings. " f"Offending code: '{code}'.") else: raise UnprocessableEntityException( f"psc_codes must be an array or object")
def validate_fiscal_period(request_data): fiscal_period = request_data["fiscal_period"] if fiscal_period < 2 or fiscal_period > 12: raise UnprocessableEntityException( f"fiscal_period must be in the range 2-12")
def post(self, request): models = [{ "name": "fields", "key": "fields", "type": "array", "array_type": "text", "text_type": "search", "optional": False, }] models.extend(copy.deepcopy(AWARD_FILTER)) models.extend(copy.deepcopy(PAGINATION)) for m in models: if m["name"] in ("keywords", "award_type_codes", "sort"): m["optional"] = False validated_payload = TinyShield(models).block(request.data) record_num = (validated_payload["page"] - 1) * validated_payload["limit"] if record_num >= settings.ES_TRANSACTIONS_MAX_RESULT_WINDOW: raise UnprocessableEntityException( "Page #{page} of size {limit} is over the maximum result limit ({es_limit}). Consider using custom data downloads to obtain large data sets." .format( page=validated_payload["page"], limit=validated_payload["limit"], es_limit=settings.ES_TRANSACTIONS_MAX_RESULT_WINDOW, )) if validated_payload["sort"] not in validated_payload["fields"]: raise InvalidParameterException( "Sort value not found in fields: {}".format( validated_payload["sort"])) if "filters" in validated_payload and "no intersection" in validated_payload[ "filters"]["award_type_codes"]: # "Special case": there will never be results when the website provides this value return Response({ "limit": validated_payload["limit"], "results": [], "page_metadata": { "page": validated_payload["page"], "next": None, "previous": None, "hasNext": False, "hasPrevious": False, }, }) sorts = { TRANSACTIONS_LOOKUP[validated_payload["sort"]]: validated_payload["order"] } lower_limit = (validated_payload["page"] - 1) * validated_payload["limit"] upper_limit = ( validated_payload["page"]) * validated_payload["limit"] + 1 validated_payload["filters"]["keyword_search"] = [ es_minimal_sanitize(x) for x in validated_payload["filters"]["keywords"] ] validated_payload["filters"].pop("keywords") filter_query = QueryWithFilters.generate_transactions_elasticsearch_query( validated_payload["filters"]) search = TransactionSearch().filter(filter_query).sort( sorts)[lower_limit:upper_limit] response = search.handle_execute() return Response( self.build_elasticsearch_result(validated_payload, response))
def post(self, request): """Return all awards matching the provided filters and limits""" models = [{ 'name': 'fields', 'key': 'fields', 'type': 'array', 'array_type': 'text', 'text_type': 'search', 'min': 1 }, { 'name': 'subawards', 'key': 'subawards', 'type': 'boolean', 'default': False }] models.extend(copy.deepcopy(AWARD_FILTER)) models.extend(copy.deepcopy(PAGINATION)) for m in models: if m['name'] in ('award_type_codes', 'fields'): m['optional'] = False json_request = TinyShield(models).block(request.data) fields = json_request["fields"] filters = json_request.get("filters", {}) subawards = json_request["subawards"] order = json_request["order"] limit = json_request["limit"] page = json_request["page"] if "no intersection" in filters["award_type_codes"]: # "Special case": there will never be results when the website provides this value return Response({ "limit": limit, "results": [], "page_metadata": { "page": page, "hasNext": False }, }) sort = json_request.get("sort", fields[0]) if sort not in fields: raise InvalidParameterException( "Sort value '{}' not found in requested fields: {}".format( sort, fields)) subawards_values = list(contract_subaward_mapping.keys()) + list( grant_subaward_mapping.keys()) awards_values = list(award_contracts_mapping.keys()) + list(loan_award_mapping.keys()) + \ list(non_loan_assistance_award_mapping.keys()) + list(award_idv_mapping.keys()) msg = "Sort value '{}' not found in {{}} mappings: {{}}".format(sort) if not subawards and sort not in awards_values: raise InvalidParameterException(msg.format("award", awards_values)) elif subawards and sort not in subawards_values: raise InvalidParameterException( msg.format("subaward", subawards_values)) # build sql query filters if subawards: queryset = subaward_filter(filters) values = {'subaward_number', 'piid', 'fain', 'award_type'} for field in fields: if contract_subaward_mapping.get(field): values.add(contract_subaward_mapping.get(field)) if grant_subaward_mapping.get(field): values.add(grant_subaward_mapping.get(field)) else: queryset = matview_search_filter(filters, UniversalAwardView).values() values = {'award_id', 'piid', 'fain', 'uri', 'type'} for field in fields: if award_contracts_mapping.get(field): values.add(award_contracts_mapping.get(field)) if loan_award_mapping.get(field): values.add(loan_award_mapping.get(field)) if non_loan_assistance_award_mapping.get(field): values.add(non_loan_assistance_award_mapping.get(field)) if award_idv_mapping.get(field): values.add(award_idv_mapping.get(field)) # Modify queryset to be ordered by requested "sort" in the request or default value(s) if sort: if subawards: if set(filters["award_type_codes"]) <= set( contract_type_mapping): # Subaward contracts sort_filters = [contract_subaward_mapping[sort]] elif set(filters["award_type_codes"]) <= set( grant_type_mapping): # Subaward grants sort_filters = [grant_subaward_mapping[sort]] else: msg = 'Award Type codes limited for Subawards. Only contracts {} or grants {} are available' msg = msg.format(list(contract_type_mapping.keys()), list(grant_type_mapping.keys())) raise UnprocessableEntityException(msg) else: if set(filters["award_type_codes"]) <= set( contract_type_mapping): # contracts sort_filters = [award_contracts_mapping[sort]] elif set(filters["award_type_codes"]) <= set( loan_type_mapping): # loans sort_filters = [loan_award_mapping[sort]] elif set(filters["award_type_codes"]) <= set( idv_type_mapping): # idvs sort_filters = [award_idv_mapping[sort]] else: # assistance data sort_filters = [non_loan_assistance_award_mapping[sort]] # Explictly set NULLS LAST in the ordering to encourage the usage of the indexes if sort == "Award ID" and subawards: if order == "desc": queryset = queryset.order_by( F('award__piid').desc(nulls_last=True), F('award__fain').desc(nulls_last=True)).values( *list(values)) else: queryset = queryset.order_by( F('award__piid').asc(nulls_last=True), F('award__fain').asc(nulls_last=True)).values( *list(values)) elif sort == "Award ID": if order == "desc": queryset = queryset.order_by( F('piid').desc(nulls_last=True), F('fain').desc(nulls_last=True), F('uri').desc(nulls_last=True)).values(*list(values)) else: queryset = queryset.order_by( F('piid').asc(nulls_last=True), F('fain').asc(nulls_last=True), F('uri').asc(nulls_last=True)).values(*list(values)) elif order == "desc": queryset = queryset.order_by( F(sort_filters[0]).desc(nulls_last=True)).values( *list(values)) else: queryset = queryset.order_by( F(sort_filters[0]).asc(nulls_last=True)).values( *list(values)) limited_queryset = queryset[(page - 1) * limit:page * limit + 1] # lower limit : upper limit has_next = len(limited_queryset) > limit results = [] for award in limited_queryset[:limit]: if subawards: row = {"internal_id": award["subaward_number"]} if award['award_type'] == 'procurement': for field in fields: row[field] = award.get( contract_subaward_mapping[field]) elif award['award_type'] == 'grant': for field in fields: row[field] = award.get(grant_subaward_mapping[field]) else: row = {"internal_id": award["award_id"]} if award['type'] in loan_type_mapping: # loans for field in fields: row[field] = award.get(loan_award_mapping.get(field)) elif award[ 'type'] in non_loan_assistance_type_mapping: # assistance data for field in fields: row[field] = award.get( non_loan_assistance_award_mapping.get(field)) elif award['type'] in idv_type_mapping: for field in fields: row[field] = award.get(award_idv_mapping.get(field)) elif (award['type'] is None and award['piid']) or award['type'] in contract_type_mapping: # IDV + contract for field in fields: row[field] = award.get( award_contracts_mapping.get(field)) if "Award ID" in fields: for id_type in ["piid", "fain", "uri"]: if award[id_type]: row["Award ID"] = award[id_type] break results.append(row) return Response({ "limit": limit, "results": results, "page_metadata": { "page": page, "hasNext": has_next } })
def apply_rule(self, rule): _return = None if rule.get('allow_nulls', False) and rule['value'] is None: _return = rule['value'] elif rule['type'] not in ('array', 'object', 'any'): if rule['type'] in VALIDATORS: _return = VALIDATORS[rule['type']]['func'](rule) else: raise Exception('Invalid Type {} in rule'.format(rule['type'])) # Array is a "special" type since it is a list of other types which need to be validated elif rule['type'] == 'array': rule['array_min'] = rule.get('array_min', 1) rule['array_max'] = rule.get('array_max', MAX_ITEMS) value = VALIDATORS[rule['type']]['func'](rule) child_rule = copy.copy(rule) child_rule['type'] = rule['array_type'] child_rule['min'] = rule.get('array_min') child_rule['max'] = rule.get('array_max') child_rule = self.promote_subrules(child_rule, child_rule) array_result = [] for v in value: child_rule['value'] = v array_result.append(self.apply_rule(child_rule)) _return = array_result # Object is a "special" type since it is comprised of other types which need to be validated elif rule['type'] == 'object': rule['object_min'] = rule.get('object_min', 1) rule['object_max'] = rule.get('object_max', MAX_ITEMS) provided_object = VALIDATORS[rule['type']]['func'](rule) object_result = {} for k, v in rule['object_keys'].items(): try: value = provided_object[k] except KeyError as e: if "optional" in v and v['optional'] is False: raise UnprocessableEntityException('Required object fields: {}'.format(k)) else: continue # Start with the sub-rule definition and supplement with parent's key-values as needed child_rule = copy.copy(v) child_rule['key'] = rule['key'] child_rule['value'] = value child_rule = self.promote_subrules(child_rule, v) object_result[k] = self.apply_rule(child_rule) _return = object_result # Any is a "special" type since it is is really a collection of other rules. elif rule['type'] == 'any': for child_rule in rule['models']: child_rule['value'] = rule['value'] try: # First successful rule wins. return self.apply_rule(child_rule) except Exception: pass # No rules succeeded. raise UnprocessableEntityException(INVALID_TYPE_MSG.format( key=rule['key'], value=rule['value'], type=', '.join(sorted([m['type'] for m in rule['models']])) )) return _return
def validate_publication_sort(self, sort_key): regex_string = r"publication_date,([2-9]|1[0-2])" if not re.match(regex_string, sort_key): raise UnprocessableEntityException( "publication_date sort param must be in the format 'publication_date,<fiscal_period>' where <fiscal_period> is in the range 2-12" )
def apply_rule(self, rule): _return = None if rule.get("allow_nulls", False) and rule["value"] is None: _return = rule["value"] elif rule["type"] not in ("array", "object", "any"): if rule["type"] in VALIDATORS: _return = VALIDATORS[rule["type"]]["func"](rule) else: raise Exception("Invalid Type {} in rule".format(rule["type"])) # Array is a "special" type since it is a list of other types which need to be validated elif rule["type"] == "array": rule["array_min"] = rule.get("array_min", 1) rule["array_max"] = rule.get("array_max", MAX_ITEMS) value = VALIDATORS[rule["type"]]["func"](rule) child_rule = copy.copy(rule) child_rule["type"] = rule["array_type"] child_rule["min"] = rule.get("array_min") child_rule["max"] = rule.get("array_max") child_rule = self.promote_subrules(child_rule, child_rule) array_result = [] for v in value: child_rule["value"] = v array_result.append(self.apply_rule(child_rule)) _return = array_result # Object is a "special" type since it is comprised of other types which need to be validated elif rule["type"] == "object": rule["object_min"] = rule.get("object_min", 1) rule["object_max"] = rule.get("object_max", MAX_ITEMS) provided_object = VALIDATORS[rule["type"]]["func"](rule) object_result = {} for k, v in rule["object_keys"].items(): try: value = provided_object[k] except KeyError: if "optional" in v and v["optional"] is False: raise UnprocessableEntityException( "Required object fields: {}".format(k)) else: continue # Start with the sub-rule definition and supplement with parent's key-values as needed child_rule = copy.copy(v) child_rule["key"] = rule["key"] child_rule["value"] = value child_rule = self.promote_subrules(child_rule, v) object_result[k] = self.apply_rule(child_rule) _return = object_result # Any is a "special" type since it is is really a collection of other rules. elif rule["type"] == "any": for child_rule in rule["models"]: child_rule["value"] = rule["value"] try: # First successful rule wins. return self.apply_rule(child_rule) except Exception: pass # No rules succeeded. raise UnprocessableEntityException( INVALID_TYPE_MSG.format( key=rule["key"], value=rule["value"], type=", ".join(sorted([m["type"] for m in rule["models"]])))) return _return
def post(self, request: Request) -> Response: models = [ { "key": "geo_layer", "name": "geo_layer", "type": "enum", "enum_values": sorted([geo_layer.value for geo_layer in list(GeoLayer)]), "text_type": "search", "allow_nulls": False, "optional": False, }, { "key": "geo_layer_filters", "name": "geo_layer_filters", "type": "array", "array_type": "text", "text_type": "search", }, { "key": "spending_type", "name": "spending_type", "type": "enum", "enum_values": ["obligation", "outlay", "face_value_of_loan"], "allow_nulls": False, "optional": False, }, { "name": "scope", "key": "scope", "type": "enum", "optional": True, "enum_values": ["place_of_performance", "recipient_location"], "default": "recipient_location", }, ] # NOTE: filter object in request handled in base class: see self.filters json_request = TinyShield(models).block(request.data) agg_key_dict = { "county": "county_agg_key", "district": "congressional_agg_key", "state": "state_agg_key", } scope_dict = { "place_of_performance": "pop", "recipient_location": "recipient_location" } location_dict = { "county": "county_code", "district": "congressional_code", "state": "state_code" } self.geo_layer = GeoLayer(json_request["geo_layer"]) scope_field_name = scope_dict[json_request["scope"]] loc_field_name = location_dict[self.geo_layer.value] self.agg_key = f"{scope_field_name}_{agg_key_dict[json_request['geo_layer']]}" self.geo_layer_filters = json_request.get("geo_layer_filters") self.spending_type = json_request.get("spending_type") self.loc_lookup = f"{scope_field_name}_{loc_field_name}" # Set which field will be the aggregation amount if self.spending_type == "obligation": self.metric_field = "total_covid_obligation" elif self.spending_type == "outlay": self.metric_field = "total_covid_outlay" elif self.spending_type == "face_value_of_loan": self.metric_field = "total_loan_value" else: raise UnprocessableEntityException( f"Unrecognized value '{self.spending_type}' for field " f"'spending_type'") filter_query = QueryWithFilters.generate_awards_elasticsearch_query( self.filters) result = self.query_elasticsearch(filter_query) return Response({ "geo_layer": self.geo_layer.value, "spending_type": self.spending_type, "scope": json_request["scope"], "results": result, })
def post(self, request): """Return all budget function/subfunction titles matching the provided search text""" models = [ {'name': 'fields', 'key': 'fields', 'type': 'array', 'array_type': 'text', 'text_type': 'search'}, {'name': 'subawards', 'key': 'subawards', 'type': 'boolean', 'default': False} ] models.extend(copy.deepcopy(AWARD_FILTER)) models.extend(copy.deepcopy(PAGINATION)) for m in models: if m['name'] in ('award_type_codes', 'fields'): m['optional'] = False json_request = TinyShield(models).block(request.data) fields = json_request.get("fields", None) filters = json_request.get("filters", None) subawards = json_request["subawards"] order = json_request["order"] limit = json_request["limit"] page = json_request["page"] lower_limit = (page - 1) * limit upper_limit = page * limit sort = json_request.get("sort", fields[0]) if sort not in fields: raise InvalidParameterException("Sort value not found in fields: {}".format(sort)) subawards_values = list(contract_subaward_mapping.keys()) + list(grant_subaward_mapping.keys()) awards_values = list(award_contracts_mapping.keys()) + list(loan_award_mapping) + \ list(non_loan_assistance_award_mapping.keys()) if (subawards and sort not in subawards_values) or (not subawards and sort not in awards_values): raise InvalidParameterException("Sort value not found in award mappings: {}".format(sort)) # build sql query filters if subawards: # We do not use matviews for Subaward filtering, just the Subaward download filters queryset = subaward_filter(filters) values = {'subaward_number', 'award__piid', 'award__fain', 'award_type'} for field in fields: if contract_subaward_mapping.get(field): values.add(contract_subaward_mapping.get(field)) if grant_subaward_mapping.get(field): values.add(grant_subaward_mapping.get(field)) else: queryset = matview_search_filter(filters, UniversalAwardView).values() values = {'award_id', 'piid', 'fain', 'uri', 'type'} for field in fields: if award_contracts_mapping.get(field): values.add(award_contracts_mapping.get(field)) if loan_award_mapping.get(field): values.add(loan_award_mapping.get(field)) if non_loan_assistance_award_mapping.get(field): values.add(non_loan_assistance_award_mapping.get(field)) # Modify queryset to be ordered if we specify "sort" in the request if sort and "no intersection" not in filters["award_type_codes"]: if subawards: if set(filters["award_type_codes"]) <= set(contract_type_mapping): # Subaward contracts sort_filters = [contract_subaward_mapping[sort]] elif set(filters["award_type_codes"]) <= set(grant_type_mapping): # Subaward grants sort_filters = [grant_subaward_mapping[sort]] else: msg = 'Award Type codes limited for Subawards. Only contracts {} or grants {} are available' msg = msg.format(list(contract_type_mapping.keys()), list(grant_type_mapping.keys())) raise UnprocessableEntityException(msg) else: if set(filters["award_type_codes"]) <= set(contract_type_mapping): # contracts sort_filters = [award_contracts_mapping[sort]] elif set(filters["award_type_codes"]) <= set(loan_type_mapping): # loans sort_filters = [loan_award_mapping[sort]] else: # assistance data sort_filters = [non_loan_assistance_award_mapping[sort]] # Explictly set NULLS LAST in the ordering to encourage the usage of the indexes if sort == "Award ID" and subawards: if order == "desc": queryset = queryset.order_by( F('award__piid').desc(nulls_last=True), F('award__fain').desc(nulls_last=True)).values(*list(values)) else: queryset = queryset.order_by( F('award__piid').asc(nulls_last=True), F('award__fain').asc(nulls_last=True)).values(*list(values)) elif sort == "Award ID": if order == "desc": queryset = queryset.order_by( F('piid').desc(nulls_last=True), F('fain').desc(nulls_last=True), F('uri').desc(nulls_last=True)).values(*list(values)) else: queryset = queryset.order_by( F('piid').asc(nulls_last=True), F('fain').asc(nulls_last=True), F('uri').asc(nulls_last=True)).values(*list(values)) elif order == "desc": queryset = queryset.order_by(F(sort_filters[0]).desc(nulls_last=True)).values(*list(values)) else: queryset = queryset.order_by(F(sort_filters[0]).asc(nulls_last=True)).values(*list(values)) limited_queryset = queryset[lower_limit:upper_limit + 1] has_next = len(limited_queryset) > limit results = [] for award in limited_queryset[:limit]: if subawards: row = {"internal_id": award["subaward_number"]} if award['award_type'] == 'procurement': for field in fields: row[field] = award.get(contract_subaward_mapping[field]) elif award['award_type'] == 'grant': for field in fields: row[field] = award.get(grant_subaward_mapping[field]) else: row = {"internal_id": award["award_id"]} if award['type'] in loan_type_mapping: # loans for field in fields: row[field] = award.get(loan_award_mapping.get(field)) elif award['type'] in non_loan_assistance_type_mapping: # assistance data for field in fields: row[field] = award.get(non_loan_assistance_award_mapping.get(field)) elif (award['type'] is None and award['piid']) or award['type'] in contract_type_mapping: # IDV + contract for field in fields: row[field] = award.get(award_contracts_mapping.get(field)) if "Award ID" in fields: for id_type in ["piid", "fain", "uri"]: if award[id_type]: row["Award ID"] = award[id_type] break results.append(row) # build response response = { 'limit': limit, 'results': results, 'page_metadata': { 'page': page, 'hasNext': has_next } } return Response(response)