def spending_by_transaction_count(request_data): keyword = request_data["filters"]["keywords"] response = {} for category in INDEX_ALIASES_TO_AWARD_TYPES.keys(): total = get_total_results(keyword, category) if total is not None: if category == "directpayments": category = "direct_payments" response[category] = total else: return total return response
def spending_by_transaction_count(request_data): keyword = request_data["filters"]["keywords"] response = {} results = get_total_results(keyword) for category in INDEX_ALIASES_TO_AWARD_TYPES.keys(): if results is not None: if category == "directpayments": response["direct_payments"] = results[category]["doc_count"] else: response[category] = results[category]["doc_count"] else: return results return response
def get_download_ids(keyword, field, size=10000): """ returns a generator that yields list of transaction ids in chunksize SIZE Note: this only works for fields in ES of integer type. """ n_iter = DOWNLOAD_QUERY_SIZE // size results = get_total_results(keyword) if results is None: logger.error( "Error retrieving total results. Max number of attempts reached") return total = sum(results[category]["doc_count"] for category in INDEX_ALIASES_TO_AWARD_TYPES.keys()) required_iter = (total // size) + 1 n_iter = min(max(1, required_iter), n_iter) for i in range(n_iter): filter_query = QueryWithFilters.generate_transactions_elasticsearch_query( {"keyword_search": [es_minimal_sanitize(keyword)]}) search = TransactionSearch().filter(filter_query) group_by_agg_key_values = { "field": field, "include": { "partition": i, "num_partitions": n_iter }, "size": size, "shard_size": size, } aggs = A("terms", **group_by_agg_key_values) search.aggs.bucket("results", aggs) response = search.handle_execute() if response is None: raise Exception("Breaking generator, unable to reach cluster") results = [] for result in response["aggregations"]["results"]["buckets"]: results.append(result["key"]) yield results