'state': state
}
yield d
# Runs the main part of the pipeline. Errors will be tagged, clean politicians will continue on to BQ.
pol = (
p
| 'Read from CSV' >> beam.io.ReadFromText(
'{0}/tmp/senate_members/*.csv'.format(os.path.expanduser('~')),
skip_header_lines=1)
| 'Split Values' >> beam.ParDo(SplitFn())
# | 'Isolate Attributes' >> beam.ParDo(pt.IsolateAttrFn())
| 'Scrub First Name' >> beam.ParDo(pt.ScrubFnameFn(), keep_suffix=True)
| 'Fix Nicknames' >> beam.ParDo(
pt.FixNicknameFn(), n_tbl=nickname_tbl, keep_nickname=True)
| 'Scrub Last Name' >> beam.ParDo(pt.ScrubLnameFn())
| 'Fix Nones' >> beam.ParDo(pt.FixNoneFn())
| 'Tag Errors' >> beam.ParDo(pt.TagErrorsFn()).with_outputs('error_tag'))
error_pols = pol.error_tag
clean_pols = pol[None]
# A new Politician will only be published if thy are not already contained in the Politicians table. If they are new,
# they will be properly uploaded. If they are not new, then they will be ignored in this pipeline.
# new_pol = (
# clean_pols
# | 'Filter Existing Pols' >> beam.ParDo(pt.NewPolsOnlyFn(), pol_tbl=pols_tbl)
# | 'Filter Pol Keys' >> beam.ParDo(pt.FilterKeysFn(), attr_lst=pol_attr_lst)
# | 'Write Pol to BQ' >> beam.io.WriteToBigQuery(
# table=pol_spec,
'last_name': last_name,
'party': party,
'state': state,
'district': district
}
yield d
# Runs the main part of the pipeline. Errors will be tagged, clean politicians will continue on to BQ.
pol = (
p
| 'Read from CSV' >> beam.io.ReadFromText('gs://{0}/tmp/house_members/*.csv'.format(os.path.expanduser('~')),
skip_header_lines=1)
| 'Split Values' >> beam.ParDo(SplitFn())
# | 'Isolate Attributes' >> beam.ParDo(pt.IsolateAttrFn())
| 'Scrub First Name' >> beam.ParDo(pt.ScrubFnameFn(), keep_suffix=True)
| 'Fix Nicknames' >> beam.ParDo(pt.FixNicknameFn(), n_tbl=nickname_tbl, keep_nickname=True)
| 'Scrub Last Name' >> beam.ParDo(pt.ScrubLnameFn())
| 'Map States' >> beam.ParDo(StateMapFn(), tbl=state_tbl)
| 'Fix Nones' >> beam.ParDo(pt.FixNoneFn())
| 'Tag Errors' >> beam.ParDo(pt.TagErrorsFn()).with_outputs('error_tag'))
error_pols = pol.error_tag
clean_pols = pol[None]
# A new Politician will only be published if thy are not already contained in the Politicians table. If they are new,
# they will be properly uploaded. If they are not new, then they will be ignored in this pipeline.
new_pol = (clean_pols
| 'Filter Existing Pols' >> beam.ParDo(pt.NewPolsOnlyFn(), pol_tbl=pols_tbl)
| 'Filter Keys' >> beam.ParDo(pt.FilterKeysFn(), attr_lst=pol_attr_lst)
| 'Write Pol to BQ' >> beam.io.WriteToBigQuery(
table=pol_spec,
house_tbl = [{str(k):str(v) for (k,v) in d.items()} for d in house_tbl]
# Runs the main part of the pipeline. Errors will be tagged, good votes will continue on to BQ.
vote = (
p
| beam.io.gcp.pubsub.ReadFromPubSub(
topic = None,
subscription = 'projects/{0}/subscriptions/{1}'
.format(project_id, subscription_name),
with_attributes = True)
| 'Isolate Attributes' >> beam.ParDo(pt.IsolateAttrFn())
| 'Fix Value Types' >> beam.ParDo(pt.FixTypesFn(), int_lst=['vote_cast'])
| 'Fix House First Names' >> beam.ParDo(pt.FixHouseFirstNameFn(), tbl=house_tbl)
| 'Scrub First Name' >> beam.ParDo(pt.ScrubFnameFn())
| 'Fix Nicknames' >> beam.ParDo(pt.FixNicknameFn(), n_tbl=nickname_tbl)
| 'Scrub Last Name' >> beam.ParDo(pt.ScrubLnameFn())
| 'Fix Nones' >> beam.ParDo(pt.FixNoneFn())
| 'Tag Errors' >> beam.ParDo(pt.TagErrorsFn()).with_outputs('error_tag'))
# Separates the clean and error votes for proper processing.
error_votes = vote.error_tag
clean_votes = vote[None]
# The cleaned elements will be sent to the proper BQ table for storage.
(clean_votes
| 'Filter Keys' >> beam.ParDo(pt.FilterKeysFn(), attr_lst=attributes_lst)
| 'Write to BQ' >> beam.io.WriteToBigQuery(
table = bq_spec,
write_disposition = beam.io.BigQueryDisposition.WRITE_APPEND,
create_disposition = beam.io.BigQueryDisposition.CREATE_NEVER
nickname_tbl = query_job.result()
nickname_tbl = [dict(row.items()) for row in nickname_tbl]
nickname_tbl = [{str(k): str(v) for (k, v) in d.items()} for d in nickname_tbl]
cosponsor = (
p
| 'Read from PubSub' >> beam.io.gcp.pubsub.ReadFromPubSub(
topic=None,
subscription='projects/{0}/subscriptions/{1}'.format(
project_id, subscription_name),
with_attributes=True)
| 'Isolate Attributes' >> beam.ParDo(pt.IsolateAttrFn())
| 'Normalize Attributes' >> beam.ParDo(NormalizeAttributesFn())
| 'Scrub First Name' >> beam.ParDo(pt.ScrubFnameFn(), keep_suffix=False)
| 'Fix Nicknames' >> beam.ParDo(
pt.FixNicknameFn(), n_tbl=nickname_tbl, keep_nickname=False)
| 'Scrub Last Name' >> beam.ParDo(pt.ScrubLnameFn())
# | 'Revert Attributes' >> beam.ParDo(RevertAttributesFn())
| 'Fix Nones' >> beam.ParDo(pt.FixNoneFn())
| 'Tag Errors' >> beam.ParDo(pt.TagErrorsFn()).with_outputs('error_tag'))
# Separates the clean and error votes for proper processing.
error_cosponsors = cosponsor.error_tag
clean_cosponsors = cosponsor[None]
(clean_cosponsors
| 'Filter Keys' >> beam.ParDo(pt.FilterKeysFn(), attr_lst=attributes_lst)
| 'Write to BQ' >> beam.io.WriteToBigQuery(
table=bill_spec,
write_disposition=beam.io.BigQueryDisposition.WRITE_APPEND,
create_disposition=beam.io.BigQueryDisposition.CREATE_NEVER))
yield element
# Runs the main part of the pipeline. Errors will be tagged, good votes will continue on to BQ.
bill = (
p
| 'Read from PubSub' >> beam.io.gcp.pubsub.ReadFromPubSub(
topic=None,
subscription='projects/{0}/subscriptions/{1}'.format(
project_id, subscription_name),
with_attributes=True)
| 'Isolate Attributes' >> beam.ParDo(pt.IsolateAttrFn())
| 'Normalize Attributes' >> beam.ParDo(NormalizeAttributesFn())
| 'Scrub First Name' >> beam.ParDo(pt.ScrubFnameFn(), keep_suffix=False)
| 'Fix Nicknames' >> beam.ParDo(
pt.FixNicknameFn(), n_tbl=n_tbl_ex, keep_nickname=False)
| 'Scrub Last Name' >> beam.ParDo(pt.ScrubLnameFn())
| 'Revert Attributes' >> beam.ParDo(RevertAttributesFn())
| 'Fix Nones' >> beam.ParDo(pt.FixNoneFn())
| 'Tag Errors' >> beam.ParDo(pt.TagErrorsFn()).with_outputs('error_tag'))
# Separates the clean and error votes for proper processing.
error_bills = bill.error_tag
clean_bills = bill[None]
# The cleaned elements will be sent to the proper BQ table for storage.
(clean_bills
| 'Filter Keys' >> beam.ParDo(pt.FilterKeysFn(), attr_lst=attributes_lst)
| 'Write to BQ' >> beam.io.WriteToBigQuery(
table=bill_spec,
write_disposition=beam.io.BigQueryDisposition.WRITE_APPEND,