def _filings_by_key(self, key_name: str,
key_value: str) -> Iterator[FilingMetadata]:
query: str = "SELECT * FROM %s WHERE %s = ?" % (self.table_name,
key_name)
cursor: Cursor = self.conn.cursor()
for row in cursor.execute(query, (key_value, )):
yield FilingMetadata(*row)
def __iter__(self) -> Iterator[FilingMetadata]:
years: Iterator = range(EARLIEST_YEAR, datetime.now().year + 1)
#years: Iterator = range(EARLIEST_YEAR, EARLIEST_YEAR + 1)
# The IRS currently provides e-files starting with those filed in 2011. Blow up if that changes.
assert not self.bucket.exists(_json_index_key(EARLIEST_YEAR - 1))
assert self.bucket.exists(_json_index_key(EARLIEST_YEAR))
exceptions = list()
with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor:
future_to_year: Dict[Future, int] = {}
results: Deque[List[Dict]] = deque()
for year in years:
future: Future = executor.submit(self._get_for_year, year)
future_to_year[future] = year
for completed_future in as_completed(
future_to_year, timeout=DOWNLOAD_TIMEOUT): # type: Future
logging.info("Finished downloading index for %i" %
future_to_year[completed_future])
results.append(completed_future.result())
if completed_future.exception() is not None:
exceptions.append(completed_future.exception())
# NOTE: if you want to omit exceptions raising, just
# comment the folliwng lines of code
if exceptions:
raise exceptions.pop()
# NOTE: you can consider to use this outsie with-statemnt
for result in results:
for filing_spec in result:
yield FilingMetadata.from_json(filing_spec)
def reference(date_downloaded) -> FilingMetadata:
return FilingMetadata(
record_id="943041314_201012",
irs_efile_id="201120919349300412",
irs_dln="93493091004121",
ein="943041314",
period="201012",
name_org="LAW ENFORCEMENT OFFICERS AND FIREFIGHTERS HEALTH AND WELFARE TRUST",
form_type="990",
date_submitted="2011-09-28",
date_uploaded="2016-03-21T17:23:53",
date_downloaded=date_downloaded,
url="https://s3.amazonaws.com/irs-form-990/201120919349300412_public.xml"
)
def __iter__(self) -> Iterator[FilingMetadata]:
years: Iterator = range(EARLIEST_YEAR, datetime.now().year + 1)
#years: Iterator = range(EARLIEST_YEAR, EARLIEST_YEAR + 1)
# The IRS currently provides e-files starting with those filed in 2011. Blow up if that changes.
assert not self.bucket.exists(_json_index_key(EARLIEST_YEAR - 1))
assert self.bucket.exists(_json_index_key(EARLIEST_YEAR))
with ThreadPoolExecutor() as executor:
future_to_year: Dict[Future, int] = {}
results: Deque[List[Dict]] = deque()
for year in years:
future: Future = executor.submit(self._get_for_year, year)
future_to_year[future] = year
for completed_future in as_completed(
future_to_year): # type: Future
logging.info("Finished downloading index for %i" %
future_to_year[completed_future])
results.append(completed_future.result())
for result in results:
for filing_spec in result:
yield FilingMetadata.from_json(filing_spec)
def __iter__(self) -> Iterator[FilingMetadata]:
query: str = "SELECT * FROM %s" % self.table_name
cursor: Cursor = self.conn.cursor()
for row in cursor.execute(query):
yield FilingMetadata(*row)
def test_read_efile_indices(fixture_path):
index_path: str = os.path.join(fixture_path, "efile_indices",
"first_timepoint")
bucket: Bucket = file_backed_bucket(index_path)
indices: EfileIndices = EfileIndices(bucket)
expected: List = [
FilingMetadata(
record_id='364201074_201012',
irs_efile_id='201101389349300010',
irs_dln='93493138000101',
ein='364201074',
period='201012',
name_org='CENTURY WALK CORPORATION',
form_type='990',
date_submitted='2011-10-12',
date_uploaded='2016-03-21T17:23:53',
date_downloaded='the current time',
url=
'https://s3.amazonaws.com/irs-form-990/201101389349300010_public.xml'
),
FilingMetadata(
record_id='943041314_201012',
irs_efile_id='201102999349300730',
irs_dln='93493299007301',
ein='943041314',
period='201012',
name_org=
'LAW ENFORCEMENT OFFICERS AND FIREFIGHTERS HEALTH AND WELFARE TRUST',
form_type='990',
date_submitted='2011-11-15',
date_uploaded='2016-03-21T17:23:53',
date_downloaded='the current time',
url=
'https://s3.amazonaws.com/irs-form-990/201102999349300730_public.xml'
),
FilingMetadata(
record_id='943041314_201012',
irs_efile_id='201120919349300412',
irs_dln='93493091004121',
ein='943041314',
period='201012',
name_org=
'LAW ENFORCEMENT OFFICERS AND FIREFIGHTERS HEALTH AND WELFARE TRUST',
form_type='990',
date_submitted='2011-09-28',
date_uploaded='2016-03-21T17:23:53',
date_downloaded='the current time',
url=
'https://s3.amazonaws.com/irs-form-990/201120919349300412_public.xml'
),
FilingMetadata(
record_id='208419458_201012',
irs_efile_id='201121369349101317',
irs_dln='93491136013171',
ein='208419458',
period='201012',
name_org='GROSSHUTTON FAMILY FOUNDATION',
form_type='990PF',
date_submitted='2011-10-14',
date_uploaded='2016-03-21T17:23:53',
date_downloaded='the current time',
url=
'https://s3.amazonaws.com/irs-form-990/201121369349101317_public.xml'
),
FilingMetadata(
record_id='260687839_201112',
irs_efile_id='201202289349200810',
irs_dln='93492228008102',
ein='260687839',
period='201112',
name_org='COOKING WITH KIDS FOUNDATION',
form_type='990EZ',
date_submitted='2012-10-29',
date_uploaded='2016-03-21T17:23:53',
date_downloaded='the current time',
url=
'https://s3.amazonaws.com/irs-form-990/201202289349200810_public.xml'
),
FilingMetadata(
record_id='943041314_201112',
irs_efile_id='201203049349300935',
irs_dln='93493304009352',
ein='943041314',
period='201112',
name_org=
'LAW ENFORCEMENT OFFICERS AND FIREFIGHTERS HEALTH AND WELFARE TRUST',
form_type='990',
date_submitted='2012-11-13',
date_uploaded='2016-03-21T17:23:53',
date_downloaded='the current time',
url=
'https://s3.amazonaws.com/irs-form-990/201203049349300935_public.xml'
),
FilingMetadata(
record_id='208419458_201112',
irs_efile_id='201211509349100411',
irs_dln='93491150004112',
ein='208419458',
period='201112',
name_org='GROSSHUTTON FAMILY FOUNDATION',
form_type='990PF',
date_submitted='2012-12-31',
date_uploaded='2016-03-21T17:23:53',
date_downloaded='the current time',
url=
'https://s3.amazonaws.com/irs-form-990/201211509349100411_public.xml'
),
FilingMetadata(
record_id='943041314_201112',
irs_efile_id='201221359349303537',
irs_dln='93493135035372',
ein='943041314',
period='201112',
name_org=
'LAW ENFORCEMENT OFFICERS AND FIREFIGHTERS HEALTH AND WELFARE TRUST',
form_type='990',
date_submitted='2012-10-11',
date_uploaded='2016-03-21T17:23:53',
date_downloaded='the current time',
url=
'https://s3.amazonaws.com/irs-form-990/201221359349303537_public.xml'
),
FilingMetadata(
record_id='364201074_201112',
irs_efile_id='201240689349300039',
irs_dln='93493068000392',
ein='364201074',
period='201112',
name_org='CENTURY WALK CORPORATION CO W BRAND BOBOSKY',
form_type='990',
date_submitted='2012-09-18',
date_uploaded='2016-03-21T17:23:53',
date_downloaded='the current time',
url=
'https://s3.amazonaws.com/irs-form-990/201240689349300039_public.xml'
),
FilingMetadata(
record_id='943041314_201212',
irs_efile_id='201312909349300041',
irs_dln='93493290000413',
ein='943041314',
period='201212',
name_org=
'LAW ENFORCEMENT OFFICERS AND FIREFIGHTERS HEALTH AND WELFARE TRUST',
form_type='990',
date_submitted='2014-01-09',
date_uploaded='2016-03-21T17:23:53',
date_downloaded='the current time',
url=
'https://s3.amazonaws.com/irs-form-990/201312909349300041_public.xml'
),
FilingMetadata(
record_id='943041314_201212',
irs_efile_id='201320599349300107',
irs_dln='93493059001073',
ein='943041314',
period='201212',
name_org=
'LAW ENFORCEMENT OFFICERS AND FIREFIGHTERS HEALTH AND WELFARE TRUST',
form_type='990',
date_submitted='2013-07-17',
date_uploaded='2016-03-21T17:23:53',
date_downloaded='the current time',
url=
'https://s3.amazonaws.com/irs-form-990/201320599349300107_public.xml'
),
FilingMetadata(
record_id='260687839_201112',
irs_efile_id='201322319349200502',
irs_dln='93492231005023',
ein='260687839',
period='201112',
name_org='COOKING WITH KIDS FOUNDATION',
form_type='990EZ',
date_submitted='2013-09-27',
date_uploaded='2016-03-21T17:23:53',
date_downloaded='the current time',
url=
'https://s3.amazonaws.com/irs-form-990/201322319349200502_public.xml'
),
FilingMetadata(
record_id='364201074_201212',
irs_efile_id='201331349349300198',
irs_dln='93493134001983',
ein='364201074',
period='201212',
name_org='CENTURY WALK CORPORATION CO W BRAND BOBOSKY',
form_type='990',
date_submitted='2013-08-14',
date_uploaded='2016-03-21T17:23:53',
date_downloaded='the current time',
url=
'https://s3.amazonaws.com/irs-form-990/201331349349300198_public.xml'
),
FilingMetadata(
record_id='208419458_201212',
irs_efile_id='201331359349102058',
irs_dln='93491135020583',
ein='208419458',
period='201212',
name_org='GROSSHUTTON FAMILY FOUNDATION',
form_type='990PF',
date_submitted='2014-01-13',
date_uploaded='2016-03-21T17:23:53',
date_downloaded='the current time',
url=
'https://s3.amazonaws.com/irs-form-990/201331359349102058_public.xml'
),
FilingMetadata(
record_id='260687839_201012',
irs_efile_id='201332289349200818',
irs_dln='93492228008183',
ein='260687839',
period='201012',
name_org='COOKING WITH KIDS FOUNDATION',
form_type='990EZ',
date_submitted='2013-09-27',
date_uploaded='2016-03-21T17:23:53',
date_downloaded='the current time',
url=
'https://s3.amazonaws.com/irs-form-990/201332289349200818_public.xml'
),
FilingMetadata(
record_id='260687839_201212',
irs_efile_id='201342279349202019',
irs_dln='93492227020193',
ein='260687839',
period='201212',
name_org='COOKING WITH KIDS',
form_type='990EZ',
date_submitted='2013-12-04',
date_uploaded='2016-03-21T17:23:53',
date_downloaded='the current time',
url=
'https://s3.amazonaws.com/irs-form-990/201342279349202019_public.xml'
),
FilingMetadata(
record_id='943041314_201312',
irs_efile_id='201411139349300506',
irs_dln='93493113005064',
ein='943041314',
period='201312',
name_org=
'LAW ENFORCEMENT OFFICERS AND FIREFIGHTERS HEALTH AND WELFARE TRUST',
form_type='990',
date_submitted='2014-09-24',
date_uploaded='2016-03-21T17:23:53',
date_downloaded='the current time',
url=
'https://s3.amazonaws.com/irs-form-990/201411139349300506_public.xml'
),
FilingMetadata(
record_id='943041314_201312',
irs_efile_id='201413219349304036',
irs_dln='93493321040364',
ein='943041314',
period='201312',
name_org=
'LAW ENFORCEMENT OFFICERS AND FIREFIGHTERS HEALTH AND WELFARE TRUST',
form_type='990',
date_submitted='2014-12-29',
date_uploaded='2016-03-21T17:23:53',
date_downloaded='the current time',
url=
'https://s3.amazonaws.com/irs-form-990/201413219349304036_public.xml'
),
FilingMetadata(
record_id='364201074_201312',
irs_efile_id='201421349349306507',
irs_dln='93493134065074',
ein='364201074',
period='201312',
name_org='CENTURY WALK CORPORATION CO W BRAND BOBOSKY',
form_type='990',
date_submitted='2014-10-14',
date_uploaded='2016-03-21T17:23:53',
date_downloaded='the current time',
url=
'https://s3.amazonaws.com/irs-form-990/201421349349306507_public.xml'
),
FilingMetadata(
record_id='260687839_201112',
irs_efile_id='201422239349200512',
irs_dln='93492223005124',
ein='260687839',
period='201112',
name_org='COOKING WITH KIDS FOUNDATION',
form_type='990EZ',
date_submitted='2014-10-01',
date_uploaded='2016-03-21T17:23:53',
date_downloaded='the current time',
url=
'https://s3.amazonaws.com/irs-form-990/201422239349200512_public.xml'
),
FilingMetadata(
record_id='208419458_201312',
irs_efile_id='201422309349100612',
irs_dln='93491230006124',
ein='208419458',
period='201312',
name_org='GROSSHUTTON FAMILY FOUNDATION',
form_type='990PF',
date_submitted='2014-10-03',
date_uploaded='2016-03-21T17:23:53',
date_downloaded='the current time',
url=
'https://s3.amazonaws.com/irs-form-990/201422309349100612_public.xml'
),
FilingMetadata(
record_id='260687839_201312',
irs_efile_id='201432239349200108',
irs_dln='93492223001084',
ein='260687839',
period='201312',
name_org='COOKING WITH KIDS FOUNDATION',
form_type='990EZ',
date_submitted='2014-10-21',
date_uploaded='2016-03-21T17:23:53',
date_downloaded='the current time',
url=
'https://s3.amazonaws.com/irs-form-990/201432239349200108_public.xml'
),
FilingMetadata(
record_id='260687839_201212',
irs_efile_id='201432239349200513',
irs_dln='93492223005134',
ein='260687839',
period='201212',
name_org='COOKING WITH KIDS',
form_type='990EZ',
date_submitted='2014-10-20',
date_uploaded='2016-03-21T17:23:53',
date_downloaded='the current time',
url=
'https://s3.amazonaws.com/irs-form-990/201432239349200513_public.xml'
),
FilingMetadata(
record_id='208419458_201312',
irs_efile_id='201441399349100554',
irs_dln='93491139005544',
ein='208419458',
period='201312',
name_org='GROSSHUTTON FAMILY FOUNDATION',
form_type='990PF',
date_submitted='2014-11-07',
date_uploaded='2016-03-21T17:23:53',
date_downloaded='the current time',
url=
'https://s3.amazonaws.com/irs-form-990/201441399349100554_public.xml'
),
FilingMetadata(
record_id='943041314_201412',
irs_efile_id='201520579349300422',
irs_dln='93493057004225',
ein='943041314',
period='201412',
name_org=
'LAW ENFORCEMENT OFFICERS AND FIREFIGHTERS HEALTH AND WELFARE TRUST',
form_type='990',
date_submitted='2015-07-17',
date_uploaded='2016-03-21T17:23:53',
date_downloaded='the current time',
url=
'https://s3.amazonaws.com/irs-form-990/201520579349300422_public.xml'
),
FilingMetadata(
record_id='364201074_201412',
irs_efile_id='201521949349301272',
irs_dln='93493194012725',
ein='364201074',
period='201412',
name_org='CENTURY WALK CORPORATION CO W BRAND BOBOSKY',
form_type='990',
date_submitted='2015-07-27',
date_uploaded='2016-03-21T17:23:53',
date_downloaded='the current time',
url=
'https://s3.amazonaws.com/irs-form-990/201521949349301272_public.xml'
),
FilingMetadata(
record_id='208419458_201412',
irs_efile_id='201541349349101014',
irs_dln='93491134010145',
ein='208419458',
period='201412',
name_org='GROSSHUTTON FAMILY FOUNDATION',
form_type='990PF',
date_submitted='2015-08-06',
date_uploaded='2016-03-21T17:23:53',
date_downloaded='the current time',
url=
'https://s3.amazonaws.com/irs-form-990/201541349349101014_public.xml'
),
FilingMetadata(
record_id='260687839_201412',
irs_efile_id='201541359349200429',
irs_dln='93492135004295',
ein='260687839',
period='201412',
name_org='COOKING WITH KIDS FOUNDATION',
form_type='990EZ',
date_submitted='2015-08-14',
date_uploaded='2016-04-29T13:40:20',
date_downloaded='the current time',
url=
'https://s3.amazonaws.com/irs-form-990/201541359349200429_public.xml'
),
FilingMetadata(
record_id='943041314_201412',
irs_efile_id='201600639349300035',
irs_dln='93493063000356',
ein='943041314',
period='201412',
name_org=
'LAW ENFORCEMENT OFFICERS AND FIREFIGHTERS HEALTH AND WELFARE TRUST',
form_type='990',
date_submitted='2016-07-21',
date_uploaded='2016-09-09T23:14:27',
date_downloaded='the current time',
url=
'https://s3.amazonaws.com/irs-form-990/201600639349300035_public.xml'
),
FilingMetadata(
record_id='260687839_201512',
irs_efile_id='201601309349200540',
irs_dln='93492130005406',
ein='260687839',
period='201512',
name_org='COOKING WITH KIDS FOUNDATION',
form_type='990EZ',
date_submitted='2016-09-06',
date_uploaded='2016-10-12T19:43:10',
date_downloaded='the current time',
url=
'https://s3.amazonaws.com/irs-form-990/201601309349200540_public.xml'
),
FilingMetadata(
record_id='364201074_201512',
irs_efile_id='201621379349301212',
irs_dln='93493137012126',
ein='364201074',
period='201512',
name_org='CENTURY WALK CORPORATION CO W BRAND BOBOSKY',
form_type='990',
date_submitted='2016-12-27',
date_uploaded='2017-01-11T22:15:19.5060064Z',
date_downloaded='the current time',
url=
'https://s3.amazonaws.com/irs-form-990/201621379349301212_public.xml'
),
FilingMetadata(
record_id='943041314_201512',
irs_efile_id='201640639349300634',
irs_dln='93493063006346',
ein='943041314',
period='201512',
name_org=
'LAW ENFORCEMENT OFFICERS AND FIREFIGHTERS HEALTH AND WELFARE TRUST',
form_type='990',
date_submitted='2016-08-16',
date_uploaded='2016-09-09T23:14:27',
date_downloaded='the current time',
url=
'https://s3.amazonaws.com/irs-form-990/201640639349300634_public.xml'
),
FilingMetadata(
record_id='208419458_201512',
irs_efile_id='201642429349100559',
irs_dln='93491242005596',
ein='208419458',
period='201512',
name_org='GROSSHUTTON FAMILY FOUNDATION',
form_type='990PF',
date_submitted='2017-01-05',
date_uploaded='2017-01-11T22:15:15',
date_downloaded='the current time',
url=
'https://s3.amazonaws.com/irs-form-990/201642429349100559_public.xml'
),
FilingMetadata(
record_id='260687839_201612',
irs_efile_id='201702429349200215',
irs_dln='93492242002157',
ein='260687839',
period='201612',
name_org='COOKING WITH KIDS FOUNDATION',
form_type='990EZ',
date_submitted='2017-10-13',
date_uploaded='2017-11-15T14:51:02',
date_downloaded='the current time',
url=
'https://s3.amazonaws.com/irs-form-990/201702429349200215_public.xml'
),
FilingMetadata(
record_id='943041314_201612',
irs_efile_id='201710309349300846',
irs_dln='93493030008467',
ein='943041314',
period='201612',
name_org=
'LAW ENFORCEMENT OFFICERS AND FIREFIGHTERS HEALTH AND WELFARE TRUST',
form_type='990',
date_submitted='2017-08-03',
date_uploaded='2017-09-13T21:58:58',
date_downloaded='the current time',
url=
'https://s3.amazonaws.com/irs-form-990/201710309349300846_public.xml'
),
FilingMetadata(
record_id='208419458_201612',
irs_efile_id='201721359349103242',
irs_dln='93491135032427',
ein='208419458',
period='201612',
name_org='GROSSHUTTON FAMILY FOUNDATION',
form_type='990PF',
date_submitted='2017-09-26',
date_uploaded='2017-10-13T18:40:16',
date_downloaded='the current time',
url=
'https://s3.amazonaws.com/irs-form-990/201721359349103242_public.xml'
),
FilingMetadata(
record_id='943041314_201512',
irs_efile_id='201730249349300103',
irs_dln='93493024001037',
ein='943041314',
period='201512',
name_org=
'LAW ENFORCEMENT OFFICERS AND FIREFIGHTERS HEALTH AND WELFARE TRUST',
form_type='990',
date_submitted='2017-07-18',
date_uploaded='2017-09-13T21:58:58',
date_downloaded='the current time',
url=
'https://s3.amazonaws.com/irs-form-990/201730249349300103_public.xml'
),
FilingMetadata(
record_id='943041314_201412',
irs_efile_id='201730249349300543',
irs_dln='93493024005437',
ein='943041314',
period='201412',
name_org=
'LAW ENFORCEMENT OFFICERS AND FIREFIGHTERS HEALTH AND WELFARE TRUST',
form_type='990',
date_submitted='2017-07-18',
date_uploaded='2017-09-13T21:58:58',
date_downloaded='the current time',
url=
'https://s3.amazonaws.com/irs-form-990/201730249349300543_public.xml'
),
FilingMetadata(
record_id='364201074_201612',
irs_efile_id='201742349349300959',
irs_dln='93493234009597',
ein='364201074',
period='201612',
name_org='CENTURY WALK CORPORATION',
form_type='990',
date_submitted='2017-10-13',
date_uploaded='2017-11-15T14:51:02',
date_downloaded='the current time',
url=
'https://s3.amazonaws.com/irs-form-990/201742349349300959_public.xml'
),
FilingMetadata(
record_id='943041314_201712',
irs_efile_id='201820589349300227',
irs_dln='93493058002278',
ein='943041314',
period='201712',
name_org=
'LAW ENFORCEMENT OFFICERS AND FIREFIGHTERS HEALTH AND WELFARE TRUST',
form_type='990',
date_submitted='2018-08-02',
date_uploaded='2018-10-16T15:38:26',
date_downloaded='the current time',
url=
'https://s3.amazonaws.com/irs-form-990/201820589349300227_public.xml'
),
FilingMetadata(
record_id='364201074_201712',
irs_efile_id='201842359349300914',
irs_dln='93493235009148',
ein='364201074',
period='201712',
name_org='CENTURY WALK CORPORATION',
form_type='990',
date_submitted='2018-10-19',
date_uploaded='2018-11-15T17:06:31',
date_downloaded='the current time',
url=
'https://s3.amazonaws.com/irs-form-990/201842359349300914_public.xml'
)
]
actual: List = sorted(indices, key=lambda filing: filing.irs_efile_id)
for a in actual:
a.date_downloaded = 'the current time'
assert expected == actual
def test_from_json(reference, date_downloaded, filing_original_dict):
actual: FilingMetadata = FilingMetadata.from_json(filing_original_dict)
actual.date_downloaded = date_downloaded # Too hard to monkeypatch datetime.now
assert actual == reference
def ret(filing_dict: Dict) -> FilingMetadata:
filing: FilingMetadata = FilingMetadata.from_json(filing_dict)
filing.date_downloaded = date_downloaded
return filing
def alpha_filing_2() -> FilingMetadata:
return FilingMetadata('0123456789_201112', 'zyxwvutsrqponmlkjihgfedcba',
'zyxwvutsrqp', '0123456789', '201112',
'The Alphabet Company', '990', '2012-04-15',
'2016-05-02', '2019-08-08',
'https://example.com/bar')
def alpha_filing_1() -> FilingMetadata:
return FilingMetadata('0123456789_201012', 'abcdefghijklmnopqrstuvwxyz',
'abcdefghijk', '0123456789', '201012',
'The Alphabet Company', '990', '2011-04-15',
'2016-05-01', '2019-08-07',
'https://example.com/foo')