def test_get_most_recent_filing_by_legal_type_in_json(session): """Assert that the most recent legal filing can be retrieved.""" business = factory_business('CP1234567') uow = versioning_manager.unit_of_work(session) transaction = uow.create_transaction(session) for i in range(1, 5): effective_date = f'200{i}-07-01T00:00:00+00:00' completion_date = datetime.datetime.fromisoformat(effective_date) base_filing = copy.deepcopy(ANNUAL_REPORT) cod = copy.deepcopy(CHANGE_OF_DIRECTORS) base_filing['filing']['changeOfDirectors'] = cod base_filing['filing']['header']['effectiveDate'] = effective_date filing = Filing() filing._filing_date = completion_date filing.business_id = business.id filing.filing_json = base_filing filing.effective_date = datetime.datetime.fromisoformat(effective_date) filing.payment_token = 'token' filing.transaction_id = transaction.id filing.payment_completion_date = completion_date filing.save() f = Filing.get_most_recent_legal_filing(business.id, 'changeOfDirectors') assert f.effective_date == datetime.datetime.fromisoformat(effective_date) assert f.filing_type == 'annualReport' assert f.id == filing.id
def factory_business(identifier, founding_date=EPOCH_DATETIME, last_ar_date=None, entity_type=Business.LegalTypes.COOP.value): """Create a business entity with a versioned business.""" last_ar_year = None if last_ar_date: last_ar_year = last_ar_date.year business = Business( legal_name=f'legal_name-{identifier}', founding_date=founding_date, last_ar_date=last_ar_date, last_ar_year=last_ar_year, last_ledger_timestamp=EPOCH_DATETIME, # dissolution_date=EPOCH_DATETIME, identifier=identifier, tax_id='BN123456789', fiscal_year_end_date=FROZEN_DATETIME, legal_type=entity_type) # Versioning business uow = versioning_manager.unit_of_work(db.session) uow.create_transaction(db.session) business.save() return business
def test_get_filings_by_status_before_go_live_date(session, test_type, days, expected, status): """Assert that a filing can be retrieved by status.""" import copy uow = versioning_manager.unit_of_work(session) transaction = uow.create_transaction(session) business = factory_business('CP1234567') payment_token = '1000' ar = copy.deepcopy(ANNUAL_REPORT) go_live_date = datetime.date.fromisoformat(current_app.config.get('GO_LIVE_DATE')) filing_date = go_live_date + datetime.timedelta(days=days) filing = Filing() filing.filing_date = filing_date filing.business_id = business.id filing.filing_json = ar filing.payment_token = payment_token filing.transaction_id = transaction.id filing.payment_completion_date = datetime.datetime.utcnow() filing.save() rv = Filing.get_filings_by_status(business.id, [Filing.Status.COMPLETED.value], go_live_date) assert eval(expected) # pylint: disable=eval-used; useful for parameterized tests if rv: assert rv[0].status == status
def prep_incorporation_correction_filing(session, business, original_filing_id, payment_id, option, name_change_with_new_nr): """Return a new incorporation correction filing prepped for email notification.""" filing_template = copy.deepcopy(CORRECTION_INCORPORATION) filing_template['filing']['business'] = {'identifier': business.identifier} for party in filing_template['filing']['incorporationApplication'][ 'parties']: for role in party['roles']: if role['roleType'] == 'Completing Party': party['officer']['email'] = '*****@*****.**' filing_template['filing']['incorporationApplication']['contactPoint'] = {} filing_template['filing']['incorporationApplication']['contactPoint'][ 'email'] = '*****@*****.**' filing_template['filing']['correction'][ 'correctedFilingId'] = original_filing_id if not name_change_with_new_nr: del filing_template['filing']['incorporationApplication'][ 'nameRequest']['legalName'] else: filing_template['filing']['incorporationApplication']['nameRequest'][ 'nrNumber'] = 'NR 1234567' filing = create_filing(token=payment_id, filing_json=filing_template, business_id=business.id) filing.payment_completion_date = filing.filing_date filing.save() if option in ['COMPLETED', 'bn']: uow = versioning_manager.unit_of_work(session) transaction = uow.create_transaction(session) filing.transaction_id = transaction.id filing.save() return filing
def __enter__(self): self.session = s = object_session(node) if s.new or s.dirty: raise Exception("Refusing to create a new tagged node version. Session must be clean!") uow = versioning_manager.unit_of_work(s) tx = uow.create_transaction(s) if user is not None: tx.user = user if tag: if node.get_tagged_version(tag): raise ValueError("tag already exists") tx.meta[u"tag"] = tag elif publish: if node.get_published_version(): raise ValueError("publish version already exists") tx.meta[u"publish"] = publish else: NodeVersion = version_class(node.__class__) # in case you were wondering: order_by(None) resets the default order_by last_tagged_version = node.tagged_versions.order_by(None).order_by(NodeVersion.transaction_id.desc()).first() if last_tagged_version is not None: next_version = int(last_tagged_version.tag) + 1 else: node.versions[-1].tag = u"1" next_version = 2 tx.meta[u"tag"] = unicode(next_version) if comment: tx.meta[u"comment"] = comment return tx
def factory_completed_filing(business, data_dict, filing_date=FROZEN_DATETIME, payment_token=None, colin_id=None): """Create a completed filing.""" if not payment_token: payment_token = str(base64.urlsafe_b64encode(uuid.uuid4().bytes)).replace('=', '') with freeze_time(filing_date): filing = Filing() filing.business_id = business.id filing.filing_date = filing_date filing.filing_json = data_dict filing.save() uow = versioning_manager.unit_of_work(db.session) transaction = uow.create_transaction(db.session) filing.transaction_id = transaction.id filing.payment_token = payment_token filing.effective_date = filing_date filing.payment_completion_date = filing_date if colin_id: colin_event = ColinEventId() colin_event.colin_event_id = colin_id colin_event.filing_id = filing.id colin_event.save() filing.save() return filing
def test_get_most_recent_filing_by_legal_type_db_field(session): """Assert that the most recent legal filing can be retrieved. Create 3 filings, find the 2 one by the type only. """ business = factory_business('CP1234567') uow = versioning_manager.unit_of_work(session) transaction = uow.create_transaction(session) # filing 1 effective_date = '2001-07-01T00:00:00+00:00' completion_date = datetime.datetime.fromisoformat(effective_date) base_filing = copy.deepcopy(ANNUAL_REPORT) base_filing['filing']['header']['effectiveDate'] = effective_date filing1 = Filing() filing1._filing_date = completion_date filing1.business_id = business.id filing1.filing_json = base_filing filing1.effective_date = datetime.datetime.fromisoformat(effective_date) filing1.payment_token = 'token' filing1.transaction_id = transaction.id filing1.payment_completion_date = completion_date filing1.save() # filing 2 <- target effective_date = '2002-07-01T00:00:00+00:00' completion_date = datetime.datetime.fromisoformat(effective_date) base_filing = copy.deepcopy(FILING_HEADER) base_filing['filing']['header']['effectiveDate'] = effective_date base_filing['filing']['header']['name'] = 'changeOfDirectors' base_filing['filing']['header']['availableOnPaperOnly'] = True filing2 = Filing() filing2._filing_date = completion_date filing2.business_id = business.id filing2.filing_json = base_filing filing2.effective_date = datetime.datetime.fromisoformat(effective_date) filing2.payment_token = 'token' filing2.transaction_id = transaction.id filing2.payment_completion_date = completion_date filing2.save() # filing 3 effective_date = '2003-07-01T00:00:00+00:00' completion_date = datetime.datetime.fromisoformat(effective_date) base_filing = copy.deepcopy(ANNUAL_REPORT) base_filing['filing']['header']['effectiveDate'] = effective_date filing3 = Filing() filing3._filing_date = completion_date filing3.business_id = business.id filing3.filing_json = base_filing filing3.effective_date = datetime.datetime.fromisoformat(effective_date) filing3.payment_token = 'token' filing3.transaction_id = transaction.id filing3.payment_completion_date = completion_date filing3.save() f = Filing.get_most_recent_legal_filing(business.id, 'changeOfDirectors') assert f.filing_type == 'changeOfDirectors' assert f.id == filing2.id
def process_filing(payment_token, flask_app): """Render the filings contained in the submission.""" if not flask_app: raise QueueException('Flask App not available.') with flask_app.app_context(): # try to find the filing 5 times before putting back on the queue - in case payment token ends up on the queue # before it is assigned to filing. counter = 1 filing_submission = None while not filing_submission and counter <= 5: filing_submission = get_filing_by_payment_id(payment_token['paymentToken'].get('id')) counter += 1 sleep(0.2) if not filing_submission: raise FilingException if filing_submission.status == Filing.Status.COMPLETED.value: logger.warning('Queue: Attempting to reprocess business.id=%s, filing.id=%s payment=%s', filing_submission.business_id, filing_submission.id, payment_token) return if payment_token['paymentToken'].get('statusCode') == 'TRANSACTION_FAILED': # TODO - need to surface TRANSACTION_FAILED, but Filings manages its own status # filing_submission.status = Filing.Status.ERROR filing_submission.payment_completion_date = datetime.datetime.utcnow() db.session.add(filing_submission) db.session.commit() return legal_filings = filing_submission.legal_filings() # TODO: handle case where there are no legal_filings uow = versioning_manager.unit_of_work(db.session) transaction = uow.create_transaction(db.session) if not payment_token['paymentToken'].get('statusCode') == 'TRANSACTION_FAILED': if not payment_token['paymentToken'].get('statusCode') == Filing.Status.COMPLETED.value: logger.error('Unknown payment status given: %s', payment_token['paymentToken'].get('statusCode')) raise QueueException business = Business.find_by_internal_id(filing_submission.business_id) for filing in legal_filings: if filing.get('annualReport'): annual_report.process(business, filing) if filing.get('changeOfAddress'): change_of_address.process(business, filing) if filing.get('changeOfDirectors'): change_of_directors.process(business, filing) filing_submission.transaction_id = transaction.id db.session.add(business) filing_submission.payment_completion_date = datetime.datetime.utcnow() db.session.add(filing_submission) db.session.commit() return
def test_create_transaction_with_scoped_session(self): article = self.Article() article.name = u"Some article" article.content = u"Some content" self.db.session.add(article) uow = versioning_manager.unit_of_work(self.db.session) transaction = uow.create_transaction(self.db.session) assert transaction.id
def test_create_transaction_with_scoped_session(self): article = self.Article() article.name = u'Some article' article.content = u'Some content' self.db.session.add(article) uow = versioning_manager.unit_of_work(self.db.session) transaction = uow.create_transaction(self.db.session) assert transaction.id
def process_filing(filing_msg: Dict, flask_app: Flask): """Render the filings contained in the submission.""" if not flask_app: raise QueueException('Flask App not available.') with flask_app.app_context(): filing_submission = Filing.find_by_id(filing_msg['filing']['id']) if not filing_submission: raise QueueException if filing_submission.status == Filing.Status.COMPLETED.value: logger.warning('QueueFiler: Attempting to reprocess business.id=%s, filing.id=%s filing=%s', filing_submission.business_id, filing_submission.id, filing_msg) return legal_filings = filing_submission.legal_filings() if legal_filings: uow = versioning_manager.unit_of_work(db.session) transaction = uow.create_transaction(db.session) business = Business.find_by_internal_id(filing_submission.business_id) for filing in legal_filings: if filing.get('annualReport'): annual_report.process(business, filing) elif filing.get('changeOfAddress'): change_of_address.process(business, filing) elif filing.get('changeOfDirectors'): filing['colinIds'] = filing_submission.colin_event_ids change_of_directors.process(business, filing) elif filing.get('changeOfName'): change_of_name.process(business, filing) elif filing.get('specialResolution'): pass # nothing to do here elif filing.get('voluntaryDissolution'): voluntary_dissolution.process(business, filing) elif filing.get('incorporationApplication'): incorporation_filing.process(business, filing, flask_app) filing_submission.transaction_id = transaction.id filing_submission.set_processed() db.session.add(business) db.session.add(filing_submission) db.session.commit() publish_event(business, filing_submission) return
def __add_filings(self, business, book): # Get the filings properties and create the filings filings_sheet = book.sheet_by_name(SheetName.FILING.value) iter_filings_rows = iter(filings_sheet.get_rows()) # (skipping the header line) next(iter_filings_rows) for filing_row in iter_filings_rows: transaction_id = None if filing_row[0].value == business.identifier: # If the filing is completed, it has to contain a transaction ID status = self.__get_value_from_row(filing_row, 9) if Filing.Status.COMPLETED.value == status: uow = versioning_manager.unit_of_work(db.session) transaction = uow.create_transaction(db.session) transaction_id = transaction.id filing = Filing( _completion_date=self.__get_value_from_row(filing_row, 2), _filing_date=self.__get_value_from_row(filing_row, 3), _filing_type=self.__get_value_from_row(filing_row, 4), effective_date=self.__get_value_from_row(filing_row, 5), _payment_token=self.__get_value_from_row(filing_row, 6), _payment_completion_date=self.__get_value_from_row( filing_row, 7), _status=status, paper_only=self.__get_value_from_row(filing_row, 10), # transaction_id comes from continuuum transaction_id=transaction_id ) filing.business_id = business.id # need to convert this first before storing filing_value = self.__get_value_from_row(filing_row, 1) if filing_value: filing._filing_json = json.loads(filing_value) # pylint: disable=protected-access business.filings.append(filing) db.session.add(filing) db.session.commit() # add colin event ids colin_event_ids = self.__get_value_from_row(filing_row, 8) if colin_event_ids: # convert string to list colin_event_ids = eval(colin_event_ids) for colin_event_id in colin_event_ids: colin_event_id_obj = ColinEventId( colin_event_id=colin_event_id, filing_id=filing.id ) filing.colin_event_ids.append(colin_event_id_obj) db.session.add(filing) db.session.commit()
def test_assign_meta_to_transaction(self): self.article.name = u'Some update article' meta = {u'some_key': u'some_value'} uow = versioning_manager.unit_of_work(self.session) tx = uow.create_transaction(self.session) tx.meta = meta self.session.commit() tx = self.article.versions[-1].transaction assert tx.meta[u'some_key'] == u'some_value'
def content_node_versioned_with_alias_id(session, content_node): from sqlalchemy_continuum import versioning_manager session.commit() content_node.orderpos = 42 session.commit() uow = versioning_manager.unit_of_work(session) tx = uow.create_transaction(session) tx.meta = {u"alias_id": 23} content_node.orderpos = 23 session.commit() return content_node
def factory_epoch_filing(business, filing_date=FROZEN_DATETIME): """Create an error filing.""" filing = Filing() filing.business_id = business.id uow = versioning_manager.unit_of_work(db.session) transaction = uow.create_transaction(db.session) filing.transaction_id = transaction.id filing.filing_date = filing_date filing.filing_json = {'filing': {'header': {'name': 'lear_epoch'}}} filing.save() return filing
def load_historic_filings(corp_num: str, business: Business, legal_type: str): """Load historic filings for a business.""" try: # get historic filings r = requests.get( f'{COLIN_API}/api/v1/businesses/{legal_type}/{corp_num}/filings/historic', timeout=TIMEOUT) if r.status_code != HTTPStatus.OK or not r.json(): print( f'skipping history for {corp_num} historic filings not found') else: for historic_filing in r.json(): uow = versioning_manager.unit_of_work(db.session) transaction = uow.create_transaction(db.session) filing = Filing() filing_date = historic_filing['filing']['header']['date'] filing.filing_date = datetime.datetime.strptime( filing_date, '%Y-%m-%d') filing.business_id = business.id filing.filing_json = historic_filing for colin_id in filing.filing_json['filing']['header'][ 'colinIds']: colin_event_id = ColinEventId() colin_event_id.colin_event_id = colin_id filing.colin_event_ids.append(colin_event_id) filing.transaction_id = transaction.id filing._filing_type = historic_filing['filing']['header'][ 'name'] filing.paper_only = True filing.effective_date = datetime.datetime.strptime( historic_filing['filing']['header']['effectiveDate'], '%Y-%m-%d') updater_user = User.find_by_username(UPDATER_USERNAME) filing.submitter_id = updater_user.id filing.source = Filing.Source.COLIN.value db.session.add(filing) # only commit after all historic filings were added successfully db.session.commit() LOADED_FILING_HISTORY.append(corp_num) except requests.exceptions.Timeout: print('rolling back partial changes...') db.session.rollback() FAILED_FILING_HISTORY.append(corp_num) print('colin_api request timed out getting historic filings.') except Exception as err: print('rolling back partial changes...') db.session.rollback() FAILED_FILING_HISTORY.append(corp_num) raise err
def test_manual_transaction_creation(self): uow = versioning_manager.unit_of_work(self.session) transaction = uow.create_transaction(self.session) self.session.flush() assert transaction.id article = self.Article(name=u'Session1 article') self.session.add(article) self.session.flush() assert uow.current_transaction.id self.session.commit() assert article.versions[-1].transaction_id
def factory_completed_filing(business, data_dict, filing_date=FROZEN_DATETIME): """Create a completed filing.""" filing = Filing() filing.business_id = business.id filing.filing_date = filing_date filing.filing_json = data_dict filing.save() uow = versioning_manager.unit_of_work(db.session) transaction = uow.create_transaction(db.session) filing.transaction_id = transaction.id filing.payment_token = 1 filing.payment_completion_date = datetime.now() return filing
def test_get_filings_by_status__default_order(session): """Assert that a filing can be retrieved. by status and is returned in the default order. default order is submission_date, and then effective_date. """ # setup base_filing = copy.deepcopy(FILING_HEADER) base_filing['specialResolution'] = SPECIAL_RESOLUTION uow = versioning_manager.unit_of_work(session) business = factory_business('CP1234567') completion_date = datetime.datetime.utcnow().replace( tzinfo=datetime.timezone.utc) # setup - create multiple filings on the same day & time filing_ids = [] file_counter = -1 with freeze_time(completion_date): for i in range(0, 5): transaction = uow.create_transaction(session) payment_token = str(i) effective_date = f'200{i}-04-15T00:00:00+00:00' base_filing['filing']['header']['effectiveDate'] = effective_date filing = Filing() filing._filing_date = completion_date filing.business_id = business.id filing.filing_json = base_filing filing.effective_date = datetime.datetime.fromisoformat( effective_date) filing.payment_token = payment_token filing.transaction_id = transaction.id filing.payment_completion_date = completion_date filing.save() filing_ids.append(filing.id) file_counter += 1 # test rv = Filing.get_filings_by_status(business.id, [Filing.Status.COMPLETED.value]) # check assert rv # filings should be in newest to oldest effective date order for filing in rv: assert filing.id == filing_ids[file_counter] file_counter -= 1
def test_get_filings_by_status(session): """Assert that a filing can be retrieved by status.""" uow = versioning_manager.unit_of_work(session) transaction = uow.create_transaction(session) business = factory_business('CP1234567') payment_token = '1000' filing = Filing() filing.business_id = business.id filing.filing_json = ANNUAL_REPORT filing.payment_token = payment_token filing.transaction_id = transaction.id filing.payment_completion_date = datetime.datetime.utcnow() filing.save() rv = Filing.get_filings_by_status(business.id, [Filing.Status.COMPLETED.value]) assert rv assert rv[0].status == Filing.Status.COMPLETED.value
def email_prepped_filing(session, identifier, payment_id, option): """Return a test filing ready for email notification.""" business = create_business(identifier) filing_template = copy.deepcopy(INCORPORATION_FILING_TEMPLATE) filing_template['filing']['business'] = {'identifier': business.identifier} for party in filing_template['filing']['incorporationApplication']['parties']: for role in party['roles']: if role['roleType'] == 'Completing Party': party['officer']['email'] = '*****@*****.**' filing_template['filing']['incorporationApplication']['contactPoint']['email'] = '*****@*****.**' filing = create_filing(token=payment_id, json_filing=filing_template, business_id=business.id) filing.payment_completion_date = filing.filing_date filing.save() if option in ['registered', 'bn']: uow = versioning_manager.unit_of_work(session) transaction = uow.create_transaction(session) filing.transaction_id = transaction.id filing.save() return filing
def __enter__(self): self.session = s = object_session(node) if s.new or s.dirty: raise Exception( "Refusing to create a new tagged node version. Session must be clean!" ) uow = versioning_manager.unit_of_work(s) tx = uow.create_transaction(s) if user is not None: tx.user = user if tag: if node.get_tagged_version(tag): raise ValueError("tag already exists") tx.meta[u"tag"] = tag elif publish: if node.get_published_version(): raise ValueError("publish version already exists") tx.meta[u"publish"] = publish else: NodeVersion = version_class(node.__class__) # in case you were wondering: order_by(None) resets the default order_by last_tagged_version = node.tagged_versions.order_by( None).order_by( NodeVersion.transaction_id.desc()).first() if last_tagged_version is not None: next_version = int(last_tagged_version.tag) + 1 else: node.versions[-1].tag = u"1" next_version = 2 tx.meta[u"tag"] = unicode(next_version) if comment: tx.meta[u"comment"] = comment return tx
def create(source, ids, force): """Create new bibliographic record(s).""" # Make sure that all imports are done with application context. from sqlalchemy_continuum import versioning_manager from .api import Record from .models import RecordMetadata data = json.load(source) if isinstance(data, dict): data = [data] if ids: assert len(ids) == len(data), 'Not enough identifiers.' for record, id_ in zip_longest(data, ids): try: click.echo(Record.create(record, id_=id_).id) except exc.IntegrityError: if force: current_app.logger.warning( "Trying to force insert: {0}".format(id_)) # IMPORTANT: We need to create new transtaction for # SQLAlchemy-Continuum as we are using no autoflush # in Record.get_record. uow = versioning_manager.unit_of_work(db.session) transaction = uow.create_transaction(db.session) # Use low-level database model to retreive an instance. model = RecordMetadata.query.get(id_) click.echo(Record(record, model=model).commit().id) else: raise click.BadParameter( 'Record with id={0} already exists. If you want to ' 'override its data use --force.'.format(id_), param_hint='ids', ) db.session.flush() db.session.commit()
def test_filing_orm_delete_blocked_if_completed(session): """Assert that attempting to delete a filing will raise a BusinessException.""" from legal_api.exceptions import BusinessException uow = versioning_manager.unit_of_work(session) transaction = uow.create_transaction(session) b = factory_business('CP1234567') filing = Filing() filing.business_id = b.id filing.filing_date = datetime.datetime.utcnow() filing.filing_json = ANNUAL_REPORT filing.payment_token = 'a token' filing.payment_completion_date = datetime.datetime.utcnow() filing.transaction_id = transaction.id filing.save() with pytest.raises(BusinessException) as excinfo: session.delete(filing) session.commit() assert excinfo.value.status_code == HTTPStatus.FORBIDDEN assert excinfo.value.error == 'Deletion not allowed.'
def test_with_entity_arg(self): article = self.Article() self.session.add(article) uow = versioning_manager.unit_of_work(article) assert isinstance(uow, UnitOfWork)
def test_raises_type_error_for_unknown_type(self): with raises(TypeError): versioning_manager.unit_of_work(None)
def update_business_legal_name(business, legal_name): """Update business legal name.""" uow = versioning_manager.unit_of_work(db.session) uow.create_transaction(db.session) business.legal_name = legal_name business.save()
def load_corps(csv_filepath: str = 'corp_nums/corps_to_load.csv'): """Load corps in given csv file from oracle into postgres.""" global ROWCOUNT with open(csv_filepath, 'r') as csvfile: reader = csv.DictReader(csvfile) with FLASK_APP.app_context(): for row in reader: corp_num = row['CORP_NUM'] print('loading: ', corp_num) added = False ROWCOUNT += 1 try: legal_type = Business.LegalTypes.COOP.value if corp_num[:2] != Business.LegalTypes.COOP.value: legal_type = Business.LegalTypes.BCOMP.value corp_num = 'BC' + corp_num[-7:] business = Business.find_by_identifier(corp_num) if business: added = True print( '-> business info already exists -- skipping corp load' ) else: try: # get current company info business_current_info = {} for info_type in BUSINESS_MODEL_INFO_TYPES[ legal_type]: business_current_info[ info_type] = get_oracle_info( corp_num=corp_num, legal_type=legal_type, info_type=info_type) if business_current_info[info_type].get( 'failed', False): raise Exception( f'could not load {info_type}') except requests.exceptions.Timeout: FAILED_CORPS.append(corp_num) print( 'colin_api request timed out getting corporation details.' ) continue except Exception as err: print(f'exception: {err}') print( f'skipping load for {corp_num}, exception occurred getting company info' ) continue uow = versioning_manager.unit_of_work(db.session) transaction = uow.create_transaction(db.session) try: # add BC prefix to non coop identifiers if legal_type != Business.LegalTypes.COOP.value: business_current_info['business']['business']['identifier'] = 'BC' + \ business_current_info['business']['business']['identifier'] # add company to postgres db business = create_business( business_current_info['business']) add_business_offices( business, business_current_info['office']) add_business_directors( business, business_current_info['parties']) if legal_type == Business.LegalTypes.BCOMP.value: add_business_shares( business, business_current_info['sharestructure']) add_business_resolutions( business, business_current_info['resolutions']) add_business_aliases( business, business_current_info['aliases']) filing = Filing() filing.filing_json = { 'filing': { 'header': { 'name': 'lear_epoch' }, 'business': business.json() } } filing._filing_type = 'lear_epoch' filing.source = Filing.Source.COLIN.value filing.transaction_id = transaction.id business.filings.append(filing) business.save() added = True NEW_CORPS.append(corp_num) except Exception as err: print(err) print(f'skipping {corp_num} missing info') FAILED_CORPS.append(corp_num) if added and history_needed(business=business): load_historic_filings(corp_num=corp_num, business=business, legal_type=legal_type) else: print( '-> historic filings not needed - skipping history load' ) except Exception as err: print(err) exit(-1)
r = requests.get(COLIN_API + '/api/v1/businesses/' + row['CORP_NUM'] + '/directors', timeout=TIMEOUT) if r.status_code != HTTPStatus.OK \ or not r.json(): print('skipping ' + row['CORP_NUM'] + ' business directors not found') continue directors_json = r.json() except requests.exceptions.Timeout as timeout: print( 'colin_api request timed out getting corporation details.' ) continue uow = versioning_manager.unit_of_work(db.session) transaction = uow.create_transaction(db.session) business = create_business(db, business_json) add_business_addresses(business, offices_json) add_business_directors(business, directors_json) db.session.add(business) filing = Filing() # filing.filing_date = datetime.datetime.utcnow filing.filing_json = { 'filing': { 'header': { 'name': 'lear_epoch' } }
async def process_filing(filing_msg: Dict, flask_app: Flask): # pylint: disable=too-many-branches,too-many-statements """Render the filings contained in the submission.""" if not flask_app: raise QueueException('Flask App not available.') with flask_app.app_context(): filing_submission = Filing.find_by_id(filing_msg['filing']['id']) if not filing_submission: raise QueueException if filing_submission.status == Filing.Status.COMPLETED.value: logger.warning( 'QueueFiler: Attempting to reprocess business.id=%s, filing.id=%s filing=%s', filing_submission.business_id, filing_submission.id, filing_msg) return None, None if legal_filings := filing_submission.legal_filings(): uow = versioning_manager.unit_of_work(db.session) transaction = uow.create_transaction(db.session) business = Business.find_by_internal_id( filing_submission.business_id) for filing in legal_filings: if filing.get('alteration'): alteration.process(business, filing_submission, filing) if filing.get('annualReport'): annual_report.process(business, filing) elif filing.get('changeOfAddress'): change_of_address.process(business, filing) elif filing.get('changeOfDirectors'): filing['colinIds'] = filing_submission.colin_event_ids change_of_directors.process(business, filing) elif filing.get('changeOfName'): change_of_name.process(business, filing) elif filing.get('voluntaryDissolution'): voluntary_dissolution.process(business, filing) elif filing.get('incorporationApplication'): business, filing_submission = incorporation_filing.process( business, filing, filing_submission) if filing.get('correction'): filing_submission = correction.process( filing_submission, filing) if filing.get('transition'): filing_submission = transition.process( business, filing_submission, filing) filing_submission.transaction_id = transaction.id filing_submission.set_processed() db.session.add(business) db.session.add(filing_submission) db.session.commit() # post filing changes to other services if any('alteration' in x for x in legal_filings): if name_request.has_new_nr_for_alteration( business, filing_submission.filing_json): name_request.consume_nr( business, filing_submission, '/filing/alteration/nameRequest/nrNumber') alteration.post_process(business, filing_submission) db.session.add(business) db.session.commit() AccountService.update_entity( business_registration=business.identifier, business_name=business.legal_name, corp_type_code=business.legal_type) if any('incorporationApplication' in x for x in legal_filings): if any('correction' in x for x in legal_filings): if name_request.has_new_nr_for_correction( filing_submission.filing_json): name_request.consume_nr(business, filing_submission) else: filing_submission.business_id = business.id db.session.add(filing_submission) db.session.commit() incorporation_filing.update_affiliation( business, filing_submission) name_request.consume_nr(business, filing_submission) incorporation_filing.post_process(business, filing_submission) try: await publish_email_message( qsm, APP_CONFIG.EMAIL_PUBLISH_OPTIONS['subject'], filing_submission, 'mras') except Exception as err: # pylint: disable=broad-except, unused-variable # noqa F841; # mark any failure for human review capture_message( f'Queue Error: Failed to place email for filing:{filing_submission.id}' f'on Queue with error:{err}', level='error') try: await publish_email_message( qsm, APP_CONFIG.EMAIL_PUBLISH_OPTIONS['subject'], filing_submission, filing_submission.status) except Exception as err: # pylint: disable=broad-except, unused-variable # noqa F841; # mark any failure for human review capture_message( f'Queue Error: Failed to place email for filing:{filing_submission.id}' f'on Queue with error:{err}', level='error') try: await publish_event(business, filing_submission) except Exception as err: # pylint: disable=broad-except, unused-variable # noqa F841; # mark any failure for human review capture_message( f'Queue Error: Failed to publish event for filing:{filing_submission.id}' f'on Queue with error:{err}', level='error')
def test_with_session_arg(self): uow = versioning_manager.unit_of_work(self.session) assert isinstance(uow, UnitOfWork)
def upgrade(): # ensure new transaction/version tables exist db.create_all() # update CVE table op.add_column( 'cve', Column('created', DateTime, default=datetime.utcnow, nullable=True, index=True)) op.add_column( 'cve', Column('changed', DateTime, default=datetime.utcnow, nullable=True, index=True)) for cve in CVE.query.all(): cve.created = datetime.utcnow() cve.changed = cve.created db.session.commit() db.session.flush() # update AVG table op.add_column( 'cve_group', Column('changed', DateTime, default=datetime.utcnow, nullable=True, index=True)) for group in CVEGroup.query.all(): group.changed = group.created db.session.commit() db.session.flush() VersionClassGroup = version_class(CVEGroup) uow = versioning_manager.unit_of_work(db.session) uow.create_transaction(db.session) for group in VersionClassGroup.query.all(): for package in CVEGroupPackage.query.filter( CVEGroupPackage.group_id == group.id).all(): package_version = uow.get_or_create_version_object(package) package_version.group_id = group.id package_version.pkgname = package.pkgname package_version.transaction_id = group.transaction_id package_version.end_transaction_id = group.end_transaction_id package_version.operation_type = Operation.INSERT package_version.group_id_mod = 1 package_version.pkgname_mod = 1 uow.process_operation(Operation(package, Operation.INSERT)) for cve in CVEGroupEntry.query.filter( CVEGroupEntry.group_id == group.id).all(): cve_version = uow.get_or_create_version_object(cve) cve_version.group_id = group.id cve_version.cve_id = cve.cve_id cve_version.transaction_id = group.transaction_id cve_version.end_transaction_id = group.end_transaction_id cve_version.operation_type = Operation.INSERT cve_version.group_id_mod = 1 cve_version.cve_id_mod = 1 uow.process_operation(Operation(cve, Operation.INSERT)) uow.make_versions(db.session) db.session.commit() db.session.flush() with op.batch_alter_table('cve_group', schema=None) as batch_op: batch_op.alter_column('changed', nullable=False) # update advisory table op.add_column( 'advisory', Column('changed', DateTime, default=datetime.utcnow, nullable=True, index=True)) for advisory in Advisory.query.all(): advisory.changed = group.created db.session.commit() db.session.flush() with op.batch_alter_table('advisory', schema=None) as batch_op: batch_op.alter_column('changed', nullable=False) # set all fields to modified for initial insert VersionClassCVE = version_class(CVE) VersionClassCVE.query.update({ VersionClassCVE.operation_type: Operation.INSERT, VersionClassCVE.issue_type_mod: 1, VersionClassCVE.description_mod: 1, VersionClassCVE.severity_mod: 1, VersionClassCVE.remote_mod: 1, VersionClassCVE.reference_mod: 1, VersionClassCVE.notes_mod: 1 }) VersionClassGroup = version_class(CVEGroup) VersionClassGroup.query.update({ VersionClassGroup.operation_type: Operation.INSERT, VersionClassGroup.status_mod: 1, VersionClassGroup.severity_mod: 1, VersionClassGroup.affected_mod: 1, VersionClassGroup.fixed_mod: 1, VersionClassGroup.bug_ticket_mod: 1, VersionClassGroup.reference_mod: 1, VersionClassGroup.notes_mod: 1, VersionClassGroup.created_mod: 1, VersionClassGroup.changed_mod: 1, VersionClassGroup.advisory_qualified_mod: 1 }) VersionClassAdvisory = version_class(Advisory) VersionClassAdvisory.query.update({ VersionClassAdvisory.operation_type: Operation.INSERT, VersionClassAdvisory.group_package_id_mod: 1, VersionClassAdvisory.advisory_type_mod: 1, VersionClassAdvisory.publication_mod: 1, VersionClassAdvisory.workaround_mod: 1, VersionClassAdvisory.impact_mod: 1, VersionClassAdvisory.content_mod: 1, VersionClassAdvisory.created_mod: 1, VersionClassAdvisory.changed_mod: 1, VersionClassAdvisory.reference_mod: 1 }) db.session.commit()
def test_manual_tx_creation_with_no_actual_changes(self): self.article.name = u'Some article' uow = versioning_manager.unit_of_work(self.session) uow.create_transaction(self.session) self.session.flush()