def make_compose(tp_path: str) -> ComposeEfiles: efile_xml_path: str = os.path.join(fixture_path, "efile_xml") xml_bucket: Bucket = file_backed_bucket(efile_xml_path) retrieve: RetrieveEfiles = RetrieveEfiles(xml_bucket) path_mgr: EINPathManager = EINPathManager(tp_path) compose: ComposeEfiles = ComposeEfiles(retrieve, path_mgr) return compose
def test_read_efile_indices(fixture_path): index_path: str = os.path.join(fixture_path, "efile_indices", "first_timepoint") bucket: Bucket = file_backed_bucket(index_path) indices: EfileIndices = EfileIndices(bucket) expected: List = [ FilingMetadata( record_id='364201074_201012', irs_efile_id='201101389349300010', irs_dln='93493138000101', ein='364201074', period='201012', name_org='CENTURY WALK CORPORATION', form_type='990', date_submitted='2011-10-12', date_uploaded='2016-03-21T17:23:53', date_downloaded='the current time', url= 'https://s3.amazonaws.com/irs-form-990/201101389349300010_public.xml' ), FilingMetadata( record_id='943041314_201012', irs_efile_id='201102999349300730', irs_dln='93493299007301', ein='943041314', period='201012', name_org= 'LAW ENFORCEMENT OFFICERS AND FIREFIGHTERS HEALTH AND WELFARE TRUST', form_type='990', date_submitted='2011-11-15', date_uploaded='2016-03-21T17:23:53', date_downloaded='the current time', url= 'https://s3.amazonaws.com/irs-form-990/201102999349300730_public.xml' ), FilingMetadata( record_id='943041314_201012', irs_efile_id='201120919349300412', irs_dln='93493091004121', ein='943041314', period='201012', name_org= 'LAW ENFORCEMENT OFFICERS AND FIREFIGHTERS HEALTH AND WELFARE TRUST', form_type='990', date_submitted='2011-09-28', date_uploaded='2016-03-21T17:23:53', date_downloaded='the current time', url= 'https://s3.amazonaws.com/irs-form-990/201120919349300412_public.xml' ), FilingMetadata( record_id='208419458_201012', irs_efile_id='201121369349101317', irs_dln='93491136013171', ein='208419458', period='201012', name_org='GROSSHUTTON FAMILY FOUNDATION', form_type='990PF', date_submitted='2011-10-14', date_uploaded='2016-03-21T17:23:53', date_downloaded='the current time', url= 'https://s3.amazonaws.com/irs-form-990/201121369349101317_public.xml' ), FilingMetadata( record_id='260687839_201112', irs_efile_id='201202289349200810', irs_dln='93492228008102', ein='260687839', period='201112', name_org='COOKING WITH KIDS FOUNDATION', form_type='990EZ', date_submitted='2012-10-29', date_uploaded='2016-03-21T17:23:53', date_downloaded='the current time', url= 'https://s3.amazonaws.com/irs-form-990/201202289349200810_public.xml' ), FilingMetadata( record_id='943041314_201112', irs_efile_id='201203049349300935', irs_dln='93493304009352', ein='943041314', period='201112', name_org= 'LAW ENFORCEMENT OFFICERS AND FIREFIGHTERS HEALTH AND WELFARE TRUST', form_type='990', date_submitted='2012-11-13', date_uploaded='2016-03-21T17:23:53', date_downloaded='the current time', url= 'https://s3.amazonaws.com/irs-form-990/201203049349300935_public.xml' ), FilingMetadata( record_id='208419458_201112', irs_efile_id='201211509349100411', irs_dln='93491150004112', ein='208419458', period='201112', name_org='GROSSHUTTON FAMILY FOUNDATION', form_type='990PF', date_submitted='2012-12-31', date_uploaded='2016-03-21T17:23:53', date_downloaded='the current time', url= 'https://s3.amazonaws.com/irs-form-990/201211509349100411_public.xml' ), FilingMetadata( record_id='943041314_201112', irs_efile_id='201221359349303537', irs_dln='93493135035372', ein='943041314', period='201112', name_org= 'LAW ENFORCEMENT OFFICERS AND FIREFIGHTERS HEALTH AND WELFARE TRUST', form_type='990', date_submitted='2012-10-11', date_uploaded='2016-03-21T17:23:53', date_downloaded='the current time', url= 'https://s3.amazonaws.com/irs-form-990/201221359349303537_public.xml' ), FilingMetadata( record_id='364201074_201112', irs_efile_id='201240689349300039', irs_dln='93493068000392', ein='364201074', period='201112', name_org='CENTURY WALK CORPORATION CO W BRAND BOBOSKY', form_type='990', date_submitted='2012-09-18', date_uploaded='2016-03-21T17:23:53', date_downloaded='the current time', url= 'https://s3.amazonaws.com/irs-form-990/201240689349300039_public.xml' ), FilingMetadata( record_id='943041314_201212', irs_efile_id='201312909349300041', irs_dln='93493290000413', ein='943041314', period='201212', name_org= 'LAW ENFORCEMENT OFFICERS AND FIREFIGHTERS HEALTH AND WELFARE TRUST', form_type='990', date_submitted='2014-01-09', date_uploaded='2016-03-21T17:23:53', date_downloaded='the current time', url= 'https://s3.amazonaws.com/irs-form-990/201312909349300041_public.xml' ), FilingMetadata( record_id='943041314_201212', irs_efile_id='201320599349300107', irs_dln='93493059001073', ein='943041314', period='201212', name_org= 'LAW ENFORCEMENT OFFICERS AND FIREFIGHTERS HEALTH AND WELFARE TRUST', form_type='990', date_submitted='2013-07-17', date_uploaded='2016-03-21T17:23:53', date_downloaded='the current time', url= 'https://s3.amazonaws.com/irs-form-990/201320599349300107_public.xml' ), FilingMetadata( record_id='260687839_201112', irs_efile_id='201322319349200502', irs_dln='93492231005023', ein='260687839', period='201112', name_org='COOKING WITH KIDS FOUNDATION', form_type='990EZ', date_submitted='2013-09-27', date_uploaded='2016-03-21T17:23:53', date_downloaded='the current time', url= 'https://s3.amazonaws.com/irs-form-990/201322319349200502_public.xml' ), FilingMetadata( record_id='364201074_201212', irs_efile_id='201331349349300198', irs_dln='93493134001983', ein='364201074', period='201212', name_org='CENTURY WALK CORPORATION CO W BRAND BOBOSKY', form_type='990', date_submitted='2013-08-14', date_uploaded='2016-03-21T17:23:53', date_downloaded='the current time', url= 'https://s3.amazonaws.com/irs-form-990/201331349349300198_public.xml' ), FilingMetadata( record_id='208419458_201212', irs_efile_id='201331359349102058', irs_dln='93491135020583', ein='208419458', period='201212', name_org='GROSSHUTTON FAMILY FOUNDATION', form_type='990PF', date_submitted='2014-01-13', date_uploaded='2016-03-21T17:23:53', date_downloaded='the current time', url= 'https://s3.amazonaws.com/irs-form-990/201331359349102058_public.xml' ), FilingMetadata( record_id='260687839_201012', irs_efile_id='201332289349200818', irs_dln='93492228008183', ein='260687839', period='201012', name_org='COOKING WITH KIDS FOUNDATION', form_type='990EZ', date_submitted='2013-09-27', date_uploaded='2016-03-21T17:23:53', date_downloaded='the current time', url= 'https://s3.amazonaws.com/irs-form-990/201332289349200818_public.xml' ), FilingMetadata( record_id='260687839_201212', irs_efile_id='201342279349202019', irs_dln='93492227020193', ein='260687839', period='201212', name_org='COOKING WITH KIDS', form_type='990EZ', date_submitted='2013-12-04', date_uploaded='2016-03-21T17:23:53', date_downloaded='the current time', url= 'https://s3.amazonaws.com/irs-form-990/201342279349202019_public.xml' ), FilingMetadata( record_id='943041314_201312', irs_efile_id='201411139349300506', irs_dln='93493113005064', ein='943041314', period='201312', name_org= 'LAW ENFORCEMENT OFFICERS AND FIREFIGHTERS HEALTH AND WELFARE TRUST', form_type='990', date_submitted='2014-09-24', date_uploaded='2016-03-21T17:23:53', date_downloaded='the current time', url= 'https://s3.amazonaws.com/irs-form-990/201411139349300506_public.xml' ), FilingMetadata( record_id='943041314_201312', irs_efile_id='201413219349304036', irs_dln='93493321040364', ein='943041314', period='201312', name_org= 'LAW ENFORCEMENT OFFICERS AND FIREFIGHTERS HEALTH AND WELFARE TRUST', form_type='990', date_submitted='2014-12-29', date_uploaded='2016-03-21T17:23:53', date_downloaded='the current time', url= 'https://s3.amazonaws.com/irs-form-990/201413219349304036_public.xml' ), FilingMetadata( record_id='364201074_201312', irs_efile_id='201421349349306507', irs_dln='93493134065074', ein='364201074', period='201312', name_org='CENTURY WALK CORPORATION CO W BRAND BOBOSKY', form_type='990', date_submitted='2014-10-14', date_uploaded='2016-03-21T17:23:53', date_downloaded='the current time', url= 'https://s3.amazonaws.com/irs-form-990/201421349349306507_public.xml' ), FilingMetadata( record_id='260687839_201112', irs_efile_id='201422239349200512', irs_dln='93492223005124', ein='260687839', period='201112', name_org='COOKING WITH KIDS FOUNDATION', form_type='990EZ', date_submitted='2014-10-01', date_uploaded='2016-03-21T17:23:53', date_downloaded='the current time', url= 'https://s3.amazonaws.com/irs-form-990/201422239349200512_public.xml' ), FilingMetadata( record_id='208419458_201312', irs_efile_id='201422309349100612', irs_dln='93491230006124', ein='208419458', period='201312', name_org='GROSSHUTTON FAMILY FOUNDATION', form_type='990PF', date_submitted='2014-10-03', date_uploaded='2016-03-21T17:23:53', date_downloaded='the current time', url= 'https://s3.amazonaws.com/irs-form-990/201422309349100612_public.xml' ), FilingMetadata( record_id='260687839_201312', irs_efile_id='201432239349200108', irs_dln='93492223001084', ein='260687839', period='201312', name_org='COOKING WITH KIDS FOUNDATION', form_type='990EZ', date_submitted='2014-10-21', date_uploaded='2016-03-21T17:23:53', date_downloaded='the current time', url= 'https://s3.amazonaws.com/irs-form-990/201432239349200108_public.xml' ), FilingMetadata( record_id='260687839_201212', irs_efile_id='201432239349200513', irs_dln='93492223005134', ein='260687839', period='201212', name_org='COOKING WITH KIDS', form_type='990EZ', date_submitted='2014-10-20', date_uploaded='2016-03-21T17:23:53', date_downloaded='the current time', url= 'https://s3.amazonaws.com/irs-form-990/201432239349200513_public.xml' ), FilingMetadata( record_id='208419458_201312', irs_efile_id='201441399349100554', irs_dln='93491139005544', ein='208419458', period='201312', name_org='GROSSHUTTON FAMILY FOUNDATION', form_type='990PF', date_submitted='2014-11-07', date_uploaded='2016-03-21T17:23:53', date_downloaded='the current time', url= 'https://s3.amazonaws.com/irs-form-990/201441399349100554_public.xml' ), FilingMetadata( record_id='943041314_201412', irs_efile_id='201520579349300422', irs_dln='93493057004225', ein='943041314', period='201412', name_org= 'LAW ENFORCEMENT OFFICERS AND FIREFIGHTERS HEALTH AND WELFARE TRUST', form_type='990', date_submitted='2015-07-17', date_uploaded='2016-03-21T17:23:53', date_downloaded='the current time', url= 'https://s3.amazonaws.com/irs-form-990/201520579349300422_public.xml' ), FilingMetadata( record_id='364201074_201412', irs_efile_id='201521949349301272', irs_dln='93493194012725', ein='364201074', period='201412', name_org='CENTURY WALK CORPORATION CO W BRAND BOBOSKY', form_type='990', date_submitted='2015-07-27', date_uploaded='2016-03-21T17:23:53', date_downloaded='the current time', url= 'https://s3.amazonaws.com/irs-form-990/201521949349301272_public.xml' ), FilingMetadata( record_id='208419458_201412', irs_efile_id='201541349349101014', irs_dln='93491134010145', ein='208419458', period='201412', name_org='GROSSHUTTON FAMILY FOUNDATION', form_type='990PF', date_submitted='2015-08-06', date_uploaded='2016-03-21T17:23:53', date_downloaded='the current time', url= 'https://s3.amazonaws.com/irs-form-990/201541349349101014_public.xml' ), FilingMetadata( record_id='260687839_201412', irs_efile_id='201541359349200429', irs_dln='93492135004295', ein='260687839', period='201412', name_org='COOKING WITH KIDS FOUNDATION', form_type='990EZ', date_submitted='2015-08-14', date_uploaded='2016-04-29T13:40:20', date_downloaded='the current time', url= 'https://s3.amazonaws.com/irs-form-990/201541359349200429_public.xml' ), FilingMetadata( record_id='943041314_201412', irs_efile_id='201600639349300035', irs_dln='93493063000356', ein='943041314', period='201412', name_org= 'LAW ENFORCEMENT OFFICERS AND FIREFIGHTERS HEALTH AND WELFARE TRUST', form_type='990', date_submitted='2016-07-21', date_uploaded='2016-09-09T23:14:27', date_downloaded='the current time', url= 'https://s3.amazonaws.com/irs-form-990/201600639349300035_public.xml' ), FilingMetadata( record_id='260687839_201512', irs_efile_id='201601309349200540', irs_dln='93492130005406', ein='260687839', period='201512', name_org='COOKING WITH KIDS FOUNDATION', form_type='990EZ', date_submitted='2016-09-06', date_uploaded='2016-10-12T19:43:10', date_downloaded='the current time', url= 'https://s3.amazonaws.com/irs-form-990/201601309349200540_public.xml' ), FilingMetadata( record_id='364201074_201512', irs_efile_id='201621379349301212', irs_dln='93493137012126', ein='364201074', period='201512', name_org='CENTURY WALK CORPORATION CO W BRAND BOBOSKY', form_type='990', date_submitted='2016-12-27', date_uploaded='2017-01-11T22:15:19.5060064Z', date_downloaded='the current time', url= 'https://s3.amazonaws.com/irs-form-990/201621379349301212_public.xml' ), FilingMetadata( record_id='943041314_201512', irs_efile_id='201640639349300634', irs_dln='93493063006346', ein='943041314', period='201512', name_org= 'LAW ENFORCEMENT OFFICERS AND FIREFIGHTERS HEALTH AND WELFARE TRUST', form_type='990', date_submitted='2016-08-16', date_uploaded='2016-09-09T23:14:27', date_downloaded='the current time', url= 'https://s3.amazonaws.com/irs-form-990/201640639349300634_public.xml' ), FilingMetadata( record_id='208419458_201512', irs_efile_id='201642429349100559', irs_dln='93491242005596', ein='208419458', period='201512', name_org='GROSSHUTTON FAMILY FOUNDATION', form_type='990PF', date_submitted='2017-01-05', date_uploaded='2017-01-11T22:15:15', date_downloaded='the current time', url= 'https://s3.amazonaws.com/irs-form-990/201642429349100559_public.xml' ), FilingMetadata( record_id='260687839_201612', irs_efile_id='201702429349200215', irs_dln='93492242002157', ein='260687839', period='201612', name_org='COOKING WITH KIDS FOUNDATION', form_type='990EZ', date_submitted='2017-10-13', date_uploaded='2017-11-15T14:51:02', date_downloaded='the current time', url= 'https://s3.amazonaws.com/irs-form-990/201702429349200215_public.xml' ), FilingMetadata( record_id='943041314_201612', irs_efile_id='201710309349300846', irs_dln='93493030008467', ein='943041314', period='201612', name_org= 'LAW ENFORCEMENT OFFICERS AND FIREFIGHTERS HEALTH AND WELFARE TRUST', form_type='990', date_submitted='2017-08-03', date_uploaded='2017-09-13T21:58:58', date_downloaded='the current time', url= 'https://s3.amazonaws.com/irs-form-990/201710309349300846_public.xml' ), FilingMetadata( record_id='208419458_201612', irs_efile_id='201721359349103242', irs_dln='93491135032427', ein='208419458', period='201612', name_org='GROSSHUTTON FAMILY FOUNDATION', form_type='990PF', date_submitted='2017-09-26', date_uploaded='2017-10-13T18:40:16', date_downloaded='the current time', url= 'https://s3.amazonaws.com/irs-form-990/201721359349103242_public.xml' ), FilingMetadata( record_id='943041314_201512', irs_efile_id='201730249349300103', irs_dln='93493024001037', ein='943041314', period='201512', name_org= 'LAW ENFORCEMENT OFFICERS AND FIREFIGHTERS HEALTH AND WELFARE TRUST', form_type='990', date_submitted='2017-07-18', date_uploaded='2017-09-13T21:58:58', date_downloaded='the current time', url= 'https://s3.amazonaws.com/irs-form-990/201730249349300103_public.xml' ), FilingMetadata( record_id='943041314_201412', irs_efile_id='201730249349300543', irs_dln='93493024005437', ein='943041314', period='201412', name_org= 'LAW ENFORCEMENT OFFICERS AND FIREFIGHTERS HEALTH AND WELFARE TRUST', form_type='990', date_submitted='2017-07-18', date_uploaded='2017-09-13T21:58:58', date_downloaded='the current time', url= 'https://s3.amazonaws.com/irs-form-990/201730249349300543_public.xml' ), FilingMetadata( record_id='364201074_201612', irs_efile_id='201742349349300959', irs_dln='93493234009597', ein='364201074', period='201612', name_org='CENTURY WALK CORPORATION', form_type='990', date_submitted='2017-10-13', date_uploaded='2017-11-15T14:51:02', date_downloaded='the current time', url= 'https://s3.amazonaws.com/irs-form-990/201742349349300959_public.xml' ), FilingMetadata( record_id='943041314_201712', irs_efile_id='201820589349300227', irs_dln='93493058002278', ein='943041314', period='201712', name_org= 'LAW ENFORCEMENT OFFICERS AND FIREFIGHTERS HEALTH AND WELFARE TRUST', form_type='990', date_submitted='2018-08-02', date_uploaded='2018-10-16T15:38:26', date_downloaded='the current time', url= 'https://s3.amazonaws.com/irs-form-990/201820589349300227_public.xml' ), FilingMetadata( record_id='364201074_201712', irs_efile_id='201842359349300914', irs_dln='93493235009148', ein='364201074', period='201712', name_org='CENTURY WALK CORPORATION', form_type='990', date_submitted='2018-10-19', date_uploaded='2018-11-15T17:06:31', date_downloaded='the current time', url= 'https://s3.amazonaws.com/irs-form-990/201842359349300914_public.xml' ) ] actual: List = sorted(indices, key=lambda filing: filing.irs_efile_id) for a in actual: a.date_downloaded = 'the current time' assert expected == actual
def get_bucket(): efile_xml_path: str = os.path.join(BASEPATH, "fixtures", "efile_xml") return file_backed_bucket(efile_xml_path)
def get_bucket(): efile_xml_path: str = os.path.join(fixture_path, "efile_xml") return file_backed_bucket(efile_xml_path)
def make_indices(timepoint: str) -> EfileIndices: efile_index_path: str = os.path.join(fixture_path, "efile_indices", "%s_timepoint" % timepoint) index_bucket: Bucket = file_backed_bucket(efile_index_path) indices: EfileIndices = EfileIndices(index_bucket) return indices
BASEPATH = "/dmz/github/analysis/composer/" def get_composite_items( idx: EfileMetadataIndex ) -> Iterator[Tuple[str, Dict[str, FilingMetadata]]]: for ein in idx.latest_filings.eins: observations: Dict = { o.period: o for o in idx.latest_filings.filings_for_ein(ein) } o_sorted: OrderedDict = OrderedDict() for period, filing in sorted( zip(observations.keys(), observations.values())): o_sorted[period] = filing yield ein, o_sorted for timepoint in ["first", "second"]: conn: Connection = connect("%s/fixtures/sqlite/%s_timepoint.sqlite" % (BASEPATH, timepoint)) index: EfileMetadataIndex = EfileMetadataIndex.build(conn) bucket = file_backed_bucket("%s/fixtures/efile_xml" % BASEPATH) s3_filings: EfileFilings = EfileFilings(bucket) path_mgr: EINPathManager = EINPathManager( "%s/fixtures/efile_composites/%s_timepoint" % (BASEPATH, timepoint)) compose: ComposeEfiles = ComposeEfiles(s3_filings, path_mgr) to_record: Iterator = get_composite_items(index) compose(to_record)