def call_resource(path, qs, per_page=5000): app = task_utils.get_app() endpoint, arguments = app.url_map.bind('').match(path) resource_type = app.view_functions[endpoint].view_class if resource_type not in RESOURCE_WHITELIST: raise ValueError('Downloads on resource {} not supported'.format( resource_type.__name__)) resource = resource_type() fields, kwargs = parse_kwargs(resource, qs) kwargs = utils.extend(arguments, kwargs) for field in IGNORE_FIELDS: kwargs.pop(field, None) query, model, schema = unpack(resource.build_query(**kwargs), 3) count = counts.count_estimate(query, db.session, threshold=5000) index_column = utils.get_index_column(model or resource.model) query_kwargs = utils.extend(kwargs, {'per_page': per_page}) paginator = utils.fetch_seek_paginator(query, query_kwargs, index_column, count=count, cap=None) return { 'path': path, 'qs': qs, 'name': get_s3_name(path, qs), 'paginator': paginator, 'schema': schema or resource.schema, 'resource': resource, 'count': count, 'timestamp': datetime.datetime.utcnow(), 'fields': fields, 'kwargs': kwargs, }
def call_resource(path, qs, per_page=5000): app = task_utils.get_app() endpoint, arguments = app.url_map.bind('').match(path) resource_type = app.view_functions[endpoint].view_class if resource_type not in RESOURCE_WHITELIST: raise ValueError('Downloads on resource {} not supported'.format(resource_type.__name__)) resource = resource_type() fields, kwargs = parse_kwargs(resource, qs) kwargs = utils.extend(arguments, kwargs) for field in IGNORE_FIELDS: kwargs.pop(field, None) query, model, schema = unpack(resource.build_query(**kwargs), 3) count = counts.count_estimate(query, db.session, threshold=5000) index_column = utils.get_index_column(model or resource.model) query_kwargs = utils.extend(kwargs, {'per_page': per_page}) paginator = utils.fetch_seek_paginator(query, query_kwargs, index_column, count=count, cap=None) return { 'path': path, 'qs': qs, 'name': get_s3_name(path, qs), 'paginator': paginator, 'schema': schema or resource.schema, 'resource': resource, 'count': count, 'timestamp': datetime.datetime.utcnow(), 'fields': fields, 'kwargs': kwargs, }
def make_schema(model, class_name=None, fields=None, options=None): class_name = class_name or '{0}Schema'.format(model.__name__) Meta = type( 'Meta', (object, ), utils.extend( { 'model': model, 'sqla_session': models.db.session, 'exclude': ('idx', ), }, options or {}, ) ) mapped_schema = ( BaseSchema if not schema_map.get(model.__name__) else schema_map.get(model.__name__) ) return type( class_name, (mapped_schema, ), utils.extend({'Meta': Meta}, fields or {}), )
def _check_filter(self, field, values, base_url, alt=None, **attrs): # Build fixtures factories.CommitteeFactory(**utils.extend(attrs, {field: alt})) [ factories.CommitteeFactory(**utils.extend(attrs, {field: value})) for value in values ] # Assert that exactly one record is found for each single-valued search # (e.g. field=value1) for value in values: url = '{0}?{1}={2}'.format(base_url, field, value) results = self._results(url) self.assertEqual(len(results), 1) self.assertEqual(results[0][field], value) # Assert that `len(values)` records are found for multi-valued search # (e.g. field=value1,value2...valueN) url = '{0}?{1}'.format( base_url, urllib.parse.urlencode({field: values}, doseq=True)) results = self._results(url) self.assertEqual(len(results), len(values)) for result in results: self.assertIn(result[field], values)
def _check_filter(self, field, values, base_url, alt=None, **attrs): # Build fixtures factories.CommitteeFactory(**utils.extend(attrs, {field: alt})) [ factories.CommitteeFactory(**utils.extend(attrs, {field: value})) for value in values ] # Assert that exactly one record is found for each single-valued search # (e.g. field=value1) for value in values: url = '{0}?{1}={2}'.format(base_url, field, value) results = self._results(url) self.assertEqual(len(results), 1) self.assertEqual(results[0][field], value) # Assert that `len(values)` records are found for multi-valued search # (e.g. field=value1,value2...valueN) url = '{0}?{1}'.format( base_url, urllib.parse.urlencode({field: values}, doseq=True) ) results = self._results(url) self.assertEqual(len(results), len(values)) for result in results: self.assertIn(result[field], values)
def test_House_Senate_totals(self): committee_id = 'C8675310' transaction_coverage = factories.TransactionCoverageFactory( # noqa committee_id=committee_id, fec_election_year=2016) history = factories.CommitteeHistoryFactory( # noqa committee_id=committee_id, committee_type='S', ) house_senate_fields = { 'committee_id': committee_id, 'cycle': 2016, 'all_other_loans': 1, 'candidate_contribution': 2, 'loan_repayments': 3, 'loan_repayments_candidate_loans': 4, 'loan_repayments_other_loans': 5, 'loans': 6, 'loans_made_by_candidate': 7, 'other_receipts': 8, 'transfers_from_other_authorized_committee': 9, 'transfers_to_other_authorized_committee': 10, 'net_contributions': 127, 'net_operating_expenditures': 128, } fields = utils.extend(house_senate_fields, shared_fields) committee_total = factories.TotalsHouseSenateFactory(**fields) # noqa fields = utils.extend(fields, transaction_coverage_fields) results = self._results( api.url_for(TotalsCommitteeView, committee_id=committee_id)) self.assertEqual(results[0], fields)
def make_schema(model, class_name=None, fields=None, options=None): class_name = class_name or "{0}Schema".format(model.__name__) Meta = type( "Meta", (object,), utils.extend({"model": model, "sqla_session": models.db.session, "exclude": ("idx",)}, options or {}), ) return type(class_name, (ModelSchema,), utils.extend({"Meta": Meta}, fields or {}))
def args(self): return utils.extend( args.paging, args.rad_analyst, args.make_sort_args(validator=args.IndexValidator( models.RadAnalyst), ), )
def args(self): return utils.extend( args.itemized, args.schedule_c, args.paging, args.make_sort_args(default='incurred_date', ), )
class CCAggregatesView(AggregateResource): @property def sort_args(self): return args.make_sort_args(validator=args.IndexValidator( self.model, extra=['candidate', 'committee'], ), ) def build_query(self, **kwargs): query = super().build_query(**kwargs) cycle_column = self.model.cycle query = query.filter( cycle_column.in_(kwargs['cycle']) if kwargs.get('cycle') else True) return join_cand_cmte_names(query) model = models.CommunicationCostByCandidate schema = schemas.CCAggregatesSchema page_schema = schemas.CCAggregatesPageSchema query_args = utils.extend(args.CC_aggregates) filter_multi_fields = [ ('candidate_id', model.candidate_id), ('committee_id', model.committee_id), ] filter_match_fields = [ ('support_oppose_indicator', model.support_oppose_indicator), ]
def test_Presidential_totals(self): committee_id = 'C8675309' history = factories.CommitteeHistoryFactory( committee_id=committee_id, committee_type='P', ) presidential_fields = { 'committee_id': 'C8675309', 'cycle': 2016, 'candidate_contribution': 1, 'exempt_legal_accounting_disbursement': 2, 'federal_funds': 3, 'fundraising_disbursements': 4, 'loan_repayments_made': 16, 'loans_received': 5, 'loans_received_from_candidate': 6, 'offsets_to_fundraising_expenditures': 7, 'offsets_to_legal_accounting': 8, 'total_offsets_to_operating_expenditures': 9, 'other_loans_received': 10, 'other_receipts': 11, 'repayments_loans_made_by_candidate': 12, 'repayments_other_loans': 13, 'transfers_from_affiliated_committee': 14, 'transfers_to_other_authorized_committee': 15, } fields = utils.extend(shared_fields, presidential_fields) committee_total = factories.TotalsPresidentialFactory(**fields) results = self._results(api.url_for(TotalsView, committee_id=committee_id)) self.assertEqual(results[0], fields)
def register_resource(resource, blueprint=None): key = resource.__name__.lower() if blueprint: key = '{0}.{1}'.format(blueprint, key) rules = app.url_map._rules_by_endpoint[key] resource_doc = getattr(resource, '__apidoc__', {}) operations = {} for rule in rules: path = extract_path(rule.rule) path_params = [ utils.extend({'required': True}, each) for each in resource_doc.get('path_params', []) if each['name'] in rule.arguments ] for method in [method.lower() for method in resource.methods or []]: view = getattr(resource, method) method_doc = getattr(view, '__apidoc__', {}) docs = [method_doc, resource_doc] operations[method] = { 'tags': resolve('tags', docs, []), 'responses': resolve('responses', docs, {}), 'description': resolve('description', docs, None), 'parameters': resolve('parameters', docs, []) + path_params, } if os.getenv('PRODUCTION'): operations[method]['parameters'].insert(0, API_KEY_PARAM) spec.spec.add_path(path=path, operations=operations, view=view)
def test_House_Senate_totals(self): committee_id = 'C8675310' history = factories.CommitteeHistoryFactory( committee_id=committee_id, committee_type='S', ) house_senate_fields = { 'committee_id': committee_id, 'cycle': 2016, 'all_other_loans': 1, 'candidate_contribution': 2, 'loan_repayments': 3, 'loan_repayments_candidate_loans': 4, 'loan_repayments_other_loans': 5, 'loans': 6, 'loans_made_by_candidate': 7, 'other_receipts': 8, 'transfers_from_other_authorized_committee': 9, 'transfers_to_other_authorized_committee': 10, } fields = utils.extend(house_senate_fields, shared_fields) committee_total = factories.TotalsHouseSenateFactory(**fields) results = self._results( api.url_for(TotalsView, committee_id=committee_id)) self.assertEqual(results[0], fields)
def call_resource(path, qs): app = task_utils.get_app() endpoint, arguments = app.url_map.bind('').match(path) resource_type = app.view_functions[endpoint].view_class resource = resource_type() fields, kwargs = parse_kwargs(resource, qs) kwargs = utils.extend(arguments, kwargs) for field in IGNORE_FIELDS: kwargs.pop(field, None) query, model, schema = unpack(resource.build_query(**kwargs), 3) count = counts.count_estimate(query, db.session) return { 'path': path, 'qs': qs, 'name': get_s3_name(path, qs), 'query': query, 'schema': schema or resource.schema, 'resource': resource, 'count': count, 'timestamp': datetime.datetime.utcnow(), 'fields': fields, 'kwargs': kwargs, }
def args(self): default_sort = ['-report_year'] return utils.extend( args.paging, args.operations_log, args.make_multi_sort_args(default=default_sort, ), )
class ECAggregatesView(AggregateResource): @property def sort_args(self): return args.make_sort_args(validator=args.IndexValidator( self.model, extra=[CANDIDATE_NAME_LABEL, COMMITTEE_NAME_LABEL], ), ) def build_query(self, **kwargs): query = super().build_query(**kwargs) cycle_column = self.model.cycle query = query.filter( cycle_column.in_(kwargs['cycle']) if kwargs.get('cycle') else True) return join_cand_cmte_names(query) model = models.ElectioneeringByCandidate schema = schemas.ECAggregatesSchema page_schema = schemas.ECAggregatesPageSchema query_args = utils.extend(args.EC_aggregates) filter_multi_fields = [ ('candidate_id', model.candidate_id), ('committee_id', model.committee_id), ]
def build_committee_query(self, kwargs, committee_id): query = self.build_query(utils.extend(kwargs, {'committee_id': [committee_id]}), join=False) sort, hide_null, nulls_large = kwargs['sort'], kwargs['sort_hide_null'], kwargs['sort_nulls_large'] query, _ = sorting.sort(query, sort, model=models.ScheduleA, hide_null=hide_null, nulls_large=nulls_large) page_query = utils.fetch_seek_page(query, kwargs, self.index_column, count=-1, eager=False).results count = counts.count_estimate(query, models.db.session, threshold=5000) return page_query, count
def test_Presidential_totals(self): committee_id = 'C8675309' history = factories.CommitteeHistoryFactory( committee_id=committee_id, committee_type='P', ) presidential_fields = { 'committee_id': 'C8675309', 'cycle': 2016, 'candidate_contribution': 1, 'exempt_legal_accounting_disbursement': 2, 'federal_funds': 3, 'fundraising_disbursements': 4, 'loan_repayments_made': 16, 'loans_received': 5, 'loans_received_from_candidate': 6, 'offsets_to_fundraising_expenditures': 7, 'offsets_to_legal_accounting': 8, 'total_offsets_to_operating_expenditures': 9, 'other_loans_received': 10, 'other_receipts': 11, 'repayments_loans_made_by_candidate': 12, 'repayments_other_loans': 13, 'transfers_from_affiliated_committee': 14, 'transfers_to_other_authorized_committee': 15, } fields = utils.extend(shared_fields, presidential_fields) committee_total = factories.TotalsPresidentialFactory(**fields) results = self._results( api.url_for(TotalsView, committee_id=committee_id)) self.assertEqual(results[0], fields)
def test_House_Senate_totals(self): committee_id = 'C8675310' history = factories.CommitteeHistoryFactory( committee_id=committee_id, committee_type='S', ) house_senate_fields = { 'committee_id': committee_id, 'cycle': 2016, 'all_other_loans': 1, 'candidate_contribution': 2, 'loan_repayments': 3, 'loan_repayments_candidate_loans': 4, 'loan_repayments_other_loans': 5, 'loans': 6, 'loans_made_by_candidate': 7, 'other_receipts': 8, 'transfers_from_other_authorized_committee': 9, 'transfers_to_other_authorized_committee': 10, } fields = utils.extend(house_senate_fields, shared_fields) committee_total = factories.TotalsHouseSenateFactory(**fields) results = self._results(api.url_for(TotalsView, committee_id=committee_id)) self.assertEqual(results[0], fields)
def args(self): return utils.extend( args.paging, args.efilings, args.make_sort_args(default='-receipt_date', validator=args.IndexValidator(self.model)), )
def args(self): return utils.extend( args.paging, args.presidential_by_size, args.make_sort_args(default='size', validator=args.OptionValidator(['size'])), )
def args(self): return utils.extend( args.electioneering, args.make_seek_args(), args.make_sort_args(validator=args.IndexValidator( models.Electioneering), ), )
def register_resource(resource, blueprint=None): key = resource.__name__.lower() if blueprint: key = "{0}.{1}".format(blueprint, key) rules = app.url_map._rules_by_endpoint[key] resource_doc = getattr(resource, "__apidoc__", {}) operations = {} for rule in rules: path = extract_path(rule.rule) path_params = [ utils.extend({"required": True}, each) for each in resource_doc.get("path_params", []) if each["name"] in rule.arguments ] for method in [method.lower() for method in resource.methods or []]: view = getattr(resource, method) method_doc = getattr(view, "__apidoc__", {}) docs = [method_doc, resource_doc] operations[method] = { "tags": resolve("tags", docs, []), "responses": resolve("responses", docs, {}), "description": resolve("description", docs, None), "parameters": resolve("parameters", docs, []) + path_params, } if os.getenv("PRODUCTION"): operations[method]["parameters"].insert(0, API_KEY_PARAM) spec.spec.add_path(path=path, operations=operations, view=view)
def test_Pac_Party_totals(self): committee_id = 'C8675311' transaction_coverage = factories.TransactionCoverageFactory( committee_id=committee_id, fec_election_year=2016) history = factories.CommitteeHistoryFactory( committee_id=committee_id, committee_type='Q', ) pac_party_fields = { 'committee_id': committee_id, 'cycle': 2016, 'all_loans_received': 1, 'allocated_federal_election_levin_share': 2, 'coordinated_expenditures_by_party_committee': 3, 'fed_candidate_committee_contributions': 4, 'fed_candidate_contribution_refunds': 5, 'fed_disbursements': 6, 'fed_election_activity': 7, 'fed_operating_expenditures': 8, 'fed_receipts': 9, 'independent_expenditures': 10, 'loan_repayments_made': 11, 'loan_repayments_received': 12, 'loans_made': 13, 'non_allocated_fed_election_activity': 14, 'total_transfers': 15, 'other_fed_operating_expenditures': 16, 'other_fed_receipts': 17, 'shared_fed_activity': 18, 'shared_fed_activity_nonfed': 19, 'shared_fed_operating_expenditures': 20, 'shared_nonfed_operating_expenditures': 21, 'transfers_from_affiliated_party': 22, 'transfers_from_nonfed_account': 23, 'transfers_from_nonfed_levin': 24, 'transfers_to_affiliated_committee': 25, 'net_contributions': 127, 'net_operating_expenditures': 128, } fields = utils.extend(pac_party_fields, shared_fields) committee_total = factories.TotalsPacPartyFactory(**fields) fields = utils.extend(fields, transaction_coverage_fields) results = self._results( api.url_for(TotalsCommitteeView, committee_id=committee_id)) self.assertEqual(results[0], fields)
def args(self): return utils.extend( args.paging, args.schedule_a_by_state, args.make_sort_args( default='-total', ), )
def args(self): return utils.extend( args.paging, args.calendar_dates, args.make_sort_args( default='-start_date', ), )
def args(self): return utils.extend( args.paging, args.presidential, args.make_sort_args(default='candidate_id', validator=args.OptionValidator( ['candidate_id'])), )
def args(self): return utils.extend( args.itemized, args.schedule_b, args.make_seek_args(), args.make_sort_args(validator=args.OptionValidator( ['disbursement_date', 'disbursement_amount']), ), )
def args(self): return utils.extend( args.paging, args.itemized, args.communication_cost, args.make_sort_args(validator=args.IndexValidator( models.CommunicationCost), ), )
def args(self): return utils.extend( args.itemized, args.schedule_b, args.make_seek_args(), args.make_sort_args( default='-disbursement_date', validator=args.OptionValidator(self.sort_options), show_nulls_last_arg=False, ))
def args(self): return utils.extend( args.itemized, args.schedule_a, args.make_seek_args(), args.make_sort_args(validator=args.OptionValidator([ 'contribution_receipt_date', 'contribution_receipt_amount', 'contributor_aggregate_ytd', ]), ))
def args(self): return utils.extend( args.paging, args.rad_analyst, args.make_sort_args( validator=args.IndexValidator(models.RadAnalyst), ), )
def args(self): return utils.extend( args.paging, args.presidential, args.make_sort_args(default='-net_receipts', validator=args.OptionValidator( ['net_receipts'])), )
def args(self): return utils.extend( args.paging, args.Category, args.make_sort_args( default='primary_category_name', ), )
def args(self): return utils.extend( args.paging, args.presidential, args.make_sort_args(default='-contribution_receipt_amount', validator=args.OptionValidator( ['contribution_receipt_amount'])), )
def args(self): return utils.extend( args.electioneering, args.make_seek_args(), args.make_sort_args( validator=args.IndexValidator(models.Electioneering), ), )
def args(self): return utils.extend( args.paging, args.AuditCase, args.make_multi_sort_args( default=['-cycle', 'committee_name', ] ), )
def args(self): return utils.extend( args.schedule_a_by_state_recipient_totals, args.paging, args.make_sort_args( default='cycle', validator=args.OptionValidator( ['cycle', 'state', 'committee_type', 'total']), ))
def args(self): return utils.extend( args.itemized, args.schedule_b, args.make_seek_args(), args.make_sort_args( validator=args.OptionValidator(['disbursement_date', 'disbursement_amount']), ), )
def args(self): return utils.extend( args.paging, args.efilings, args.make_sort_args( default='-receipt_date', # validator=args.IndexValidator(self.model), ), )
def args(self): return utils.extend( args.paging, args.schedule_b_efile, args.make_sort_args( default='-disbursement_date', validator=args.OptionValidator(['disbursement_date', 'disbursement_amount']), ), )
def args(self): return utils.extend( args.itemized, args.schedule_f, args.paging, args.make_sort_args( default='expenditure_date', ) )
def args(self): return utils.extend( args.paging, args.election_dates, args.make_sort_args( default='-election_date', validator=args.IndexValidator(self.model), ), )
def args(self): return utils.extend( args.paging, args.large_aggregates, args.make_sort_args( default='date', validator=args.OptionValidator([ 'date', ]), ))
def args(self): return utils.extend( args.itemized, args.schedule_c, args.paging, args.make_sort_args( default='incurred_date', ), )
def args(self): return utils.extend( args.paging, args.committee_history, args.make_sort_args( default='-cycle', validator=args.IndexValidator(self.model), ), )
class ElectioneeringByCandidateView(CandidateAggregateResource): model = models.ElectioneeringByCandidate schema = schemas.ElectioneeringByCandidateSchema page_schema = schemas.ElectioneeringByCandidatePageSchema query_args = utils.extend(args.elections, args.electioneering_by_candidate) filter_multi_fields = [ ('candidate_id', models.ElectioneeringByCandidate.candidate_id), ]
def build_committee_query(self, kwargs, committee_id): """Build a subquery by committee. """ query = self.build_query(_apply_options=False, **utils.extend(kwargs, {'committee_id': [committee_id]})) sort, hide_null = kwargs['sort'], kwargs['sort_hide_null'] query, _ = sorting.sort(query, sort, model=self.model, hide_null=hide_null) page_query = utils.fetch_seek_page(query, kwargs, self.index_column, count=-1, eager=False).results count = counts.count_estimate(query, models.db.session) return page_query, count
def args(self): return utils.extend( args.itemized, args.communication_cost, args.make_seek_args(), args.make_sort_args( validator=args.IndexValidator(models.CommunicationCost), ), )
def args(self): return utils.extend( args.paging, args.candidate_history, args.make_sort_args( default='-two_year_period', validator=args.IndexValidator(self.model), ), )
def args(self): return utils.extend( args.paging, args.candidate_detail, args.make_sort_args( default='name', validator=args.IndexValidator(self.model), ), )
def build_committee_query(self, kwargs, committee_id): """Build a subquery by committee. """ query = self.build_query(_apply_options=False, **utils.extend(kwargs, {'committee_id': [committee_id]})) sort, hide_null, nulls_large = kwargs['sort'], kwargs['sort_hide_null'], kwargs['sort_nulls_large'] query, _ = sorting.sort(query, sort, model=self.model, hide_null=hide_null, nulls_large=nulls_large) page_query = utils.fetch_seek_page(query, kwargs, self.index_column, count=-1, eager=False).results count = counts.count_estimate(query, models.db.session, threshold=5000) return page_query, count
def args(self): #Place the sort argument in a list, as the api will return a 422 status code if it's not in a list #list is needed because multisort is used default_sort = ['-receipt_date'] return utils.extend( args.paging, args.filings, args.make_multi_sort_args( default=default_sort, validator=args.IndicesValidator(self.model) ), )
def args(self): return utils.extend( args.paging, args.candidate_list, args.candidate_detail, args.make_sort_args( default='name', validator=args.IndexValidator( models.Candidate, extra=list(self.aliases.keys()), ), ) )
def args(self): return utils.extend( args.itemized, args.schedule_a, args.make_seek_args(), args.make_sort_args( validator=args.OptionValidator([ 'contribution_receipt_date', 'contribution_receipt_amount', 'contributor_aggregate_ytd', ]), ) )
def args(self): return utils.extend( args.itemized, args.schedule_e, args.make_seek_args(), args.make_sort_args( validator=args.OptionValidator([ 'expenditure_date', 'expenditure_amount', 'office_total_ytd', ]), ), )
def args(self): return utils.extend( args.paging, args.schedule_a_e_file, args.itemized, args.make_sort_args( default='-contribution_receipt_date', validator=args.OptionValidator([ 'contribution_receipt_date', 'contribution_receipt_amount', 'contributor_aggregate_ytd', ]), ), )