def get(self, **kwargs):
"""Get itemized resources. If multiple values are passed for `committee_id`,
create a subquery for each and combine with `UNION ALL`. This is necessary
to avoid slow queries when one or more relevant committees has many
records.
"""
if kwargs.get("last_index"):
if all(
kwargs.get("last_{}".format(option)) is None
for option in self.sort_options
) and not kwargs.get("sort_null_only"):
raise exceptions.ApiError(
"When paginating through results, both values from the \
previous page's `last_indexes` object are needed. For more information, \
see https://api.open.fec.gov/developers/. Please add one of the following \
filters to your query: `sort_null_only`=True, {}".format(
", ".join("`last_" + option + "`" for option in self.sort_options)
),
status_code=422,
)
committee_ids = kwargs.get('committee_id', [])
if len(committee_ids) > 10:
raise exceptions.ApiError(
'Can only specify up to ten values for "committee_id".',
status_code=422,
)
if len(committee_ids) > 1:
query, count = self.join_committee_queries(kwargs)
return utils.fetch_seek_page(query, kwargs, self.index_column, count=count)
query = self.build_query(**kwargs)
count, _ = counts.get_count(query, models.db.session)
return utils.fetch_seek_page(query, kwargs, self.index_column, count=count, cap=self.cap)
def _validate(self, value):
super()._validate(value)
try:
value = int(value)
except (TypeError, ValueError):
raise exceptions.ApiError('District must be a number',
status_code=422)
if value < 0:
raise exceptions.ApiError('District must be a natural number',
status_code=422)
def build_query(self, **kwargs):
secondary_index_options = [
'committee_id',
'contributor_id',
'contributor_name',
'contributor_city',
'contributor_zip',
'contributor_employer',
'contributor_occupation',
'image_number',
]
two_year_transaction_periods = set(
kwargs.get('two_year_transaction_period', [])
)
if len(two_year_transaction_periods) != 1:
if not any(kwargs.get(field) for field in secondary_index_options):
raise exceptions.ApiError(
"Please choose a single `two_year_transaction_period` or \
add one of the following filters to your query: `{}`".format(
"`, `".join(secondary_index_options)
),
status_code=400,
)
query = super().build_query(**kwargs)
query = filters.filter_contributor_type(query, self.model.entity_type, kwargs)
zip_list = []
if kwargs.get('contributor_zip'):
for value in kwargs['contributor_zip']:
if re.search('[^a-zA-Z0-9-\s]', value): # noqa
raise exceptions.ApiError(
'Invalid zip code. It can not have special character',
status_code=400,
)
else:
zip_list.append(value[:5])
contributor_zip_list = {'contributor_zip': zip_list}
query = filters.filter_multi_start_with(
query, contributor_zip_list, self.filter_multi_start_with_fields
)
if kwargs.get('sub_id'):
query = query.filter_by(sub_id=int(kwargs.get('sub_id')))
if kwargs.get('line_number'):
# line_number is a composite value of 'filing_form-line_number'
if len(kwargs.get('line_number').split('-')) == 2:
form, line_no = kwargs.get('line_number').split('-')
query = query.filter_by(filing_form=form.upper())
query = query.filter_by(line_number=line_no)
else:
raise exceptions.ApiError(
exceptions.LINE_NUMBER_ERROR, status_code=400,
)
return query
def build_query(self, **kwargs):
query = super().build_query(**kwargs)
query = filters.filter_contributor_type(query, self.model.entity_type,
kwargs)
zip_list = []
if kwargs.get('contributor_zip'):
for value in kwargs['contributor_zip']:
if re.search('[^a-zA-Z0-9-\s]', value):
raise exceptions.ApiError(
'Invalid zip code. It can not have special character',
status_code=400,
)
else:
zip_list.append(value[:5])
contributor_zip_list = {'contributor_zip': zip_list}
query = filters.filter_multi_start_with(
query, contributor_zip_list,
self.filter_multi_start_with_fields)
if kwargs.get('sub_id'):
query = query.filter_by(sub_id=int(kwargs.get('sub_id')))
if kwargs.get('line_number'):
if len(kwargs.get('line_number').split('-')) == 2:
form, line_no = kwargs.get('line_number').split('-')
query = query.filter_by(filing_form=form.upper())
query = query.filter_by(line_number=line_no)
return query
def build_query(self, **kwargs):
query = super().build_query(**kwargs)
if {'receipts', '-receipts'}.intersection(kwargs.get(
'sort', [])) and 'q' not in kwargs:
raise exceptions.ApiError(
'Cannot sort on receipts when parameter "q" is not set',
status_code=422,
)
if kwargs.get('q'):
query = utils.search_text(
query.join(
models.CandidateSearch,
models.Candidate.candidate_id == models.CandidateSearch.id,
),
models.CandidateSearch.fulltxt,
kwargs['q'],
).distinct()
if kwargs.get('name'):
query = query.filter(
models.Candidate.name.ilike('%{}%'.format(kwargs['name'])))
# TODO(jmcarp) Reintroduce year filter pending accurate `load_date` and `expire_date` values
if kwargs.get('cycle'):
query = query.filter(
models.Candidate.cycles.overlap(kwargs['cycle']))
return query
def get(self, **kwargs):
"""Get itemized resources. If multiple values are passed for `committee_id`,
create a subquery for each and combine with `UNION ALL`. This is necessary
to avoid slow queries when one or more relevant committees has many
records.
"""
committee_ids = kwargs.get('committee_id', [])
if len(committee_ids) > 10:
raise exceptions.ApiError(
'Can only specify up to ten values for "committee_id".',
status_code=422,
)
if len(committee_ids) > 1:
query, count = self.join_committee_queries(kwargs)
return utils.fetch_seek_page(query,
kwargs,
self.index_column,
count=count)
query = self.build_query(**kwargs)
count, _ = counts.get_count(query, models.db.session)
return utils.fetch_seek_page(query,
kwargs,
self.index_column,
count=count,
cap=self.cap)
def build_query(self, **kwargs):
query = super().build_query(**kwargs)
if {'receipts', '-receipts'}.intersection(kwargs.get(
'sort', [])) and 'q' not in kwargs:
raise exceptions.ApiError(
'Cannot sort on receipts when parameter "q" is not set',
status_code=422,
)
if kwargs.get('candidate_id'):
query = query.filter(
models.Committee.candidate_ids.overlap(kwargs['candidate_id']))
if kwargs.get('q'):
query = query.join(
models.CommitteeSearch,
models.Committee.committee_id == models.CommitteeSearch.id,
).distinct()
if kwargs.get('year'):
query = filter_year(models.Committee, query, kwargs['year'])
if kwargs.get('cycle'):
query = query.filter(
models.Committee.cycles.overlap(kwargs['cycle']))
return query
def __call__(self, value):
for sort_column in value:
if sort_column.lstrip('-') not in self.values:
raise exceptions.ApiError(
'Cannot sort on value "{0}"'.format(sort_column),
status_code=422,
)
def build_query(self, candidate_id=None, **kwargs):
totals_class, totals_schema = candidate_totals_schema_map.get(
self._resolve_committee_type(candidate_id=candidate_id, **kwargs),
default_schemas,
)
query = totals_class.query
query = query.filter(totals_class.candidate_id == candidate_id)
if 'full_election' in kwargs.keys():
# full_election is replaced by election_full.
raise exceptions.ApiError(
exceptions.FULL_ELECTION_ERROR,
status_code=400,
)
if kwargs.get('election_full') is None:
# not pass election_full
if kwargs.get('cycle'):
# only filter by cycle
query = query.filter(totals_class.cycle.in_(kwargs['cycle']))
else:
# pass election_full (true or false)
query = query.filter(
totals_class.election_full == kwargs['election_full'])
if kwargs.get('cycle'):
if kwargs.get('election_full'):
# if election_full = true, filter by candidate_election_year = cycle
query = query.filter(
totals_class.candidate_election_year.in_(
kwargs['cycle']))
else:
# if election_full = false, filter by cycle = cycle
query = query.filter(
totals_class.cycle.in_(kwargs['cycle']))
return query, totals_class, totals_schema
def filter_election(query,
kwargs,
candidate_column,
cycle_column=None,
year_column=None):
if not kwargs['office']:
return query
if isinstance(kwargs['cycle'], list):
if len(kwargs['cycle']) != 1:
raise exceptions.ApiError(
'Must include exactly one argument "cycle"',
status_code=422,
)
kwargs['cycle'] = kwargs['cycle'][0]
utils.check_election_arguments(kwargs)
query = query.join(
models.CandidateHistory,
candidate_column == models.CandidateHistory.candidate_id,
).filter(
models.CandidateHistory.two_year_period == kwargs['cycle'],
models.CandidateHistory.office == kwargs['office'][0].upper(),
)
if cycle_column:
query = query.filter(cycle_column == kwargs['cycle'])
elif year_column:
query = query.filter(
year_column.in_(kwargs['cycle'], kwargs['cycle'] - 1))
else:
raise ValueError('Must provide `cycle_column` or `year_column`')
if kwargs['state']:
query = query.filter(models.CandidateHistory.state == kwargs['state'])
if kwargs['district']:
query = query.filter(
models.CandidateHistory.district == kwargs['district'])
return query
def check_cap(kwargs, cap):
if cap:
if not kwargs.get('per_page') or kwargs['per_page'] > cap:
raise exceptions.ApiError(
'Parameter "per_page" must be between 1 and {}'.format(cap),
status_code=422,
)
def check_cap(kwargs, cap):
if cap:
if not kwargs.get('per_page'):
raise exceptions.ApiError(
'Parameter "per_page" must be > 0'.format(cap),
status_code=422,
)
def check_election_arguments(kwargs):
for arg in office_args_required:
if kwargs.get(arg) is None:
raise exceptions.ApiError(
'Required parameter "{0}" not found.'.format(arg),
status_code=422,
)
conditional_args = office_args_map.get(kwargs['office'], [])
for arg in conditional_args:
if kwargs.get(arg) is None:
raise exceptions.ApiError(
'Must include argument "{0}" with office type "{1}"'.format(
arg,
kwargs['office'],
),
status_code=422,
)
def handle_exception(exception):
wrapped = ResponseException(str(exception), ErrorCode.INTERNAL_ERROR,
type(exception))
app.logger.error(
'An API error occurred with the status code of {status} ({exception}).'
.format(status=wrapped.status, exception=wrapped.wrappedException))
raise exceptions.ApiError('Could not process the request',
status_code=http.client.NOT_FOUND)
def __call__(self, value):
if value.lstrip('-') not in self.values:
raise exceptions.ApiError(
'Cannot sort on value "{0}". Instead choose one of: "{1}"'.format(
value, '", "'.join(self.values)
),
status_code=422,
)
def get_cycle(kwargs):
if isinstance(kwargs['cycle'], list):
if len(kwargs['cycle']) != 1:
raise exceptions.ApiError(
'Must include exactly one argument "cycle"',
status_code=422,
)
return kwargs['cycle'][0]
return kwargs['cycle']
def get(self, **kwargs):
if len(kwargs['committee_id']) > 5:
raise exceptions.ApiError(
'Can only specify up to five values for "committee_id".',
status_code=422,
)
if len(kwargs['committee_id']) > 1:
query, count = self.join_committee_queries(kwargs)
return utils.fetch_seek_page(query,
kwargs,
self.index_column,
count=count)
return super(ScheduleAView, self).get(**kwargs)
def handle_exception(exception):
wrapped = ResponseException(str(exception), ErrorCode.INTERNAL_ERROR,
type(exception))
logger.info(
'An API error occurred with the status code of {status} ({exception}).'
.format(status=wrapped.status, exception=wrapped.wrappedException))
if is_retrievable_from_cache(wrapped.status, request.path):
logger.info('Attempting to retrieving the cached request from S3...')
# Retrieve the information needed to construct a URL for the S3 bucket
# where the cached API responses live.
formatted_url = utils.format_url(request.url)
s3_bucket = utils.get_bucket()
bucket_region = env.get_credential('region')
cached_url = "http://s3-{0}.amazonaws.com/{1}/cached-calls/{2}".format(
bucket_region, s3_bucket.name, formatted_url)
# Attempt to retrieve the cached data from S3.
cached_data = utils.get_cached_request(cached_url)
# If the cached data was returned, we can return that to the client.
# Otherwise, log the error and raise an API error.
if cached_data is not None:
logger.info('Successfully retrieved cached request from S3.')
return cached_data
else:
logger.error(
'An error occured while retrieving the cached file from S3.')
raise exceptions.ApiError(
'The requested URL could not be found.'.format(request.url),
status_code=http.client.NOT_FOUND)
else:
raise exceptions.ApiError(
'The requested URL could not be found.'.format(request.url),
status_code=http.client.NOT_FOUND)
def build_query(self, **kwargs):
query = super(ScheduleH4View, self).build_query(**kwargs)
query = query.options(sa.orm.joinedload(models.ScheduleH4.committee))
if kwargs.get('sub_id'):
query = query.filter_by(sub_id=int(kwargs.get('sub_id')))
if kwargs.get('line_number'):
# line number is a composite value of 'filing_form-line_number'
if len(kwargs.get('line_number').split('-')) == 2:
form, line_no = kwargs.get('line_number').split('-')
query = query.filter_by(filing_form=form.upper())
query = query.filter_by(line_number=line_no)
else:
raise exceptions.ApiError(
exceptions.LINE_NUMBER_ERROR,
status_code=400,
)
return query
def post(self, path, filename=None, **kwargs):
parts = request.path.split('/')
parts.remove('download')
path = '/'.join(parts)
cached_file = get_cached_file(path, request.query_string, filename=filename)
if cached_file:
return {
'status': 'complete',
'url': cached_file,
}
resource = download.call_resource(path, request.query_string)
if resource['count'] > MAX_RECORDS:
raise exceptions.ApiError(
'Cannot request downloads with more than {} records'.format(MAX_RECORDS),
status_code=http.client.FORBIDDEN,
)
download.export_query.delay(path, request.query_string)
return {'status': 'queued'}
def build_query(self, committee_id=None, **kwargs):
query = super().build_query(committee_id=committee_id, **kwargs)
election_full = kwargs.get('election_full')
if election_full and not (kwargs.get('candidate_id')
or kwargs.get('office')):
raise exceptions.ApiError(
'Must include "candidate_id" or "office" argument(s)',
status_code=422,
)
cycle_column = (models.CandidateElection.cand_election_year
if election_full else self.model.cycle)
query = filters.filter_election(query, kwargs, self.model.candidate_id,
cycle_column)
query = query.filter(
cycle_column.in_(kwargs['cycle']) if kwargs.get('cycle') else True)
if election_full:
query = self.aggregate_cycles(query, cycle_column)
return self.join_entity_names(query)
def build_query(self, **kwargs):
if kwargs.get('name'):
kwargs['q'] = kwargs['name']
query = super().build_query(**kwargs)
if {'receipts', '-receipts'}.intersection(kwargs.get(
'sort', [])) and 'q' not in kwargs:
raise exceptions.ApiError(
'Cannot sort on receipts when parameter "q" is not set',
status_code=422,
)
if 'has_raised_funds' in kwargs:
query = query.filter(
models.Candidate.flags.has(
models.CandidateFlags.has_raised_funds ==
kwargs['has_raised_funds']))
if 'federal_funds_flag' in kwargs:
query = query.filter(
models.Candidate.flags.has(
models.CandidateFlags.federal_funds_flag ==
kwargs['federal_funds_flag']))
if kwargs.get('q'):
query = query.join(
models.CandidateSearch,
models.Candidate.candidate_id == models.CandidateSearch.id,
).distinct()
if kwargs.get('cycle'):
query = query.filter(
models.Candidate.cycles.overlap(kwargs['cycle']))
if kwargs.get('election_year'):
query = query.filter(
models.Candidate.election_years.overlap(
kwargs['election_year']))
return query
def build_query(self, **kwargs):
query = super().build_query(**kwargs)
query = filters.filter_contributor_type(query, self.model.entity_type,
kwargs)
if kwargs.get('contributor_zip'):
for value in kwargs['contributor_zip']:
if re.search('^-?\d{5}$', value) is None:
raise exceptions.ApiError(
'Invalid Zip code. It must be 5 digits',
status_code=400,
)
query = filters.filter_multi_start_with(
query, kwargs, self.filter_multi_start_with_fields)
if kwargs.get('sub_id'):
query = query.filter_by(sub_id=int(kwargs.get('sub_id')))
if kwargs.get('line_number'):
if len(kwargs.get('line_number').split('-')) == 2:
form, line_no = kwargs.get('line_number').split('-')
query = query.filter_by(filing_form=form.upper())
query = query.filter_by(line_number=line_no)
return query
def _validate_natural(value):
if value <= 0:
raise exceptions.ApiError(
'Must be greater than zero',
status_code=422
)
def handle_error(self, error):
message = error.messages
status_code = getattr(error, 'status_code', 422)
raise exceptions.ApiError(message, status_code)
def build_query(self, **kwargs):
if kwargs.get('name'):
kwargs['q'] = kwargs['name']
query = super().build_query(**kwargs)
candidate_detail = models.Candidate
if {'receipts', '-receipts'}.intersection(kwargs.get(
'sort', [])) and 'q' not in kwargs:
raise exceptions.ApiError(
'Cannot sort on receipts when parameter "q" is not set',
status_code=422,
)
if 'has_raised_funds' in kwargs:
query = query.filter(
candidate_detail.flags.has(
models.CandidateFlags.has_raised_funds ==
kwargs['has_raised_funds']))
if 'federal_funds_flag' in kwargs:
query = query.filter(
candidate_detail.flags.has(
models.CandidateFlags.federal_funds_flag ==
kwargs['federal_funds_flag']))
if kwargs.get('q'):
query = query.join(
models.CandidateSearch,
candidate_detail.candidate_id == models.CandidateSearch.id,
).distinct()
if kwargs.get('cycle'):
query = query.filter(
candidate_detail.cycles.overlap(kwargs['cycle']))
if kwargs.get('election_year'):
query = query.filter(
candidate_detail.election_years.overlap(
kwargs['election_year']))
if 'is_active_candidate' in kwargs and kwargs.get(
'is_active_candidate'):
# load active candidates only if True
if kwargs.get('election_year'):
query = query.filter(
sa.or_(
~(candidate_detail.inactive_election_years.contains(
kwargs['election_year'])),
candidate_detail.inactive_election_years.is_(None),
))
else:
query = query.filter(
candidate_detail.candidate_inactive == False) # noqa
elif 'is_active_candidate' in kwargs and not kwargs.get(
'is_active_candidate'):
# load inactive candidates only if False
if kwargs.get('election_year'):
query = query.filter(
candidate_detail.inactive_election_years.overlap(
kwargs['election_year']))
else:
query = query.filter(
candidate_detail.candidate_inactive == True # noqa
)
else:
# load all candidates
pass
return query
def handle_error(self, error):
logger.error(error)
message = text_type(error)
status_code = getattr(error, 'status_code', 400)
payload = getattr(error, 'data', {})
raise exceptions.ApiError(message, status_code, payload)
def __call__(self, value):
if value.lstrip('-') not in self.values:
raise exceptions.ApiError(
'Cannot sort on value "{0}"'.format(value), status_code=422)
def handle_error(self, error, req, schema, status_code, error_headers):
message = error.messages
status_code = getattr(error, 'status_code', 422)
raise exceptions.ApiError(message, status_code)
def _validate_natural(value):
if value < 0:
raise exceptions.ApiError('Must be a natural number', status_code=422)