def test_get_most_recent_filing_by_legal_type_in_json(session):
"""Assert that the most recent legal filing can be retrieved."""
business = factory_business('CP1234567')
uow = versioning_manager.unit_of_work(session)
transaction = uow.create_transaction(session)
for i in range(1, 5):
effective_date = f'200{i}-07-01T00:00:00+00:00'
completion_date = datetime.datetime.fromisoformat(effective_date)
base_filing = copy.deepcopy(ANNUAL_REPORT)
cod = copy.deepcopy(CHANGE_OF_DIRECTORS)
base_filing['filing']['changeOfDirectors'] = cod
base_filing['filing']['header']['effectiveDate'] = effective_date
filing = Filing()
filing._filing_date = completion_date
filing.business_id = business.id
filing.filing_json = base_filing
filing.effective_date = datetime.datetime.fromisoformat(effective_date)
filing.payment_token = 'token'
filing.transaction_id = transaction.id
filing.payment_completion_date = completion_date
filing.save()
f = Filing.get_most_recent_legal_filing(business.id, 'changeOfDirectors')
assert f.effective_date == datetime.datetime.fromisoformat(effective_date)
assert f.filing_type == 'annualReport'
assert f.id == filing.id
def factory_business(identifier,
founding_date=EPOCH_DATETIME,
last_ar_date=None,
entity_type=Business.LegalTypes.COOP.value):
"""Create a business entity with a versioned business."""
last_ar_year = None
if last_ar_date:
last_ar_year = last_ar_date.year
business = Business(
legal_name=f'legal_name-{identifier}',
founding_date=founding_date,
last_ar_date=last_ar_date,
last_ar_year=last_ar_year,
last_ledger_timestamp=EPOCH_DATETIME,
# dissolution_date=EPOCH_DATETIME,
identifier=identifier,
tax_id='BN123456789',
fiscal_year_end_date=FROZEN_DATETIME,
legal_type=entity_type)
# Versioning business
uow = versioning_manager.unit_of_work(db.session)
uow.create_transaction(db.session)
business.save()
return business
def test_get_filings_by_status_before_go_live_date(session, test_type, days, expected, status):
"""Assert that a filing can be retrieved by status."""
import copy
uow = versioning_manager.unit_of_work(session)
transaction = uow.create_transaction(session)
business = factory_business('CP1234567')
payment_token = '1000'
ar = copy.deepcopy(ANNUAL_REPORT)
go_live_date = datetime.date.fromisoformat(current_app.config.get('GO_LIVE_DATE'))
filing_date = go_live_date + datetime.timedelta(days=days)
filing = Filing()
filing.filing_date = filing_date
filing.business_id = business.id
filing.filing_json = ar
filing.payment_token = payment_token
filing.transaction_id = transaction.id
filing.payment_completion_date = datetime.datetime.utcnow()
filing.save()
rv = Filing.get_filings_by_status(business.id, [Filing.Status.COMPLETED.value], go_live_date)
assert eval(expected) # pylint: disable=eval-used; useful for parameterized tests
if rv:
assert rv[0].status == status
def prep_incorporation_correction_filing(session, business, original_filing_id,
payment_id, option,
name_change_with_new_nr):
"""Return a new incorporation correction filing prepped for email notification."""
filing_template = copy.deepcopy(CORRECTION_INCORPORATION)
filing_template['filing']['business'] = {'identifier': business.identifier}
for party in filing_template['filing']['incorporationApplication'][
'parties']:
for role in party['roles']:
if role['roleType'] == 'Completing Party':
party['officer']['email'] = '*****@*****.**'
filing_template['filing']['incorporationApplication']['contactPoint'] = {}
filing_template['filing']['incorporationApplication']['contactPoint'][
'email'] = '*****@*****.**'
filing_template['filing']['correction'][
'correctedFilingId'] = original_filing_id
if not name_change_with_new_nr:
del filing_template['filing']['incorporationApplication'][
'nameRequest']['legalName']
else:
filing_template['filing']['incorporationApplication']['nameRequest'][
'nrNumber'] = 'NR 1234567'
filing = create_filing(token=payment_id,
filing_json=filing_template,
business_id=business.id)
filing.payment_completion_date = filing.filing_date
filing.save()
if option in ['COMPLETED', 'bn']:
uow = versioning_manager.unit_of_work(session)
transaction = uow.create_transaction(session)
filing.transaction_id = transaction.id
filing.save()
return filing
def __enter__(self):
self.session = s = object_session(node)
if s.new or s.dirty:
raise Exception("Refusing to create a new tagged node version. Session must be clean!")
uow = versioning_manager.unit_of_work(s)
tx = uow.create_transaction(s)
if user is not None:
tx.user = user
if tag:
if node.get_tagged_version(tag):
raise ValueError("tag already exists")
tx.meta[u"tag"] = tag
elif publish:
if node.get_published_version():
raise ValueError("publish version already exists")
tx.meta[u"publish"] = publish
else:
NodeVersion = version_class(node.__class__)
# in case you were wondering: order_by(None) resets the default order_by
last_tagged_version = node.tagged_versions.order_by(None).order_by(NodeVersion.transaction_id.desc()).first()
if last_tagged_version is not None:
next_version = int(last_tagged_version.tag) + 1
else:
node.versions[-1].tag = u"1"
next_version = 2
tx.meta[u"tag"] = unicode(next_version)
if comment:
tx.meta[u"comment"] = comment
return tx
def factory_completed_filing(business, data_dict, filing_date=FROZEN_DATETIME, payment_token=None, colin_id=None):
"""Create a completed filing."""
if not payment_token:
payment_token = str(base64.urlsafe_b64encode(uuid.uuid4().bytes)).replace('=', '')
with freeze_time(filing_date):
filing = Filing()
filing.business_id = business.id
filing.filing_date = filing_date
filing.filing_json = data_dict
filing.save()
uow = versioning_manager.unit_of_work(db.session)
transaction = uow.create_transaction(db.session)
filing.transaction_id = transaction.id
filing.payment_token = payment_token
filing.effective_date = filing_date
filing.payment_completion_date = filing_date
if colin_id:
colin_event = ColinEventId()
colin_event.colin_event_id = colin_id
colin_event.filing_id = filing.id
colin_event.save()
filing.save()
return filing
def test_get_most_recent_filing_by_legal_type_db_field(session):
"""Assert that the most recent legal filing can be retrieved.
Create 3 filings, find the 2 one by the type only.
"""
business = factory_business('CP1234567')
uow = versioning_manager.unit_of_work(session)
transaction = uow.create_transaction(session)
# filing 1
effective_date = '2001-07-01T00:00:00+00:00'
completion_date = datetime.datetime.fromisoformat(effective_date)
base_filing = copy.deepcopy(ANNUAL_REPORT)
base_filing['filing']['header']['effectiveDate'] = effective_date
filing1 = Filing()
filing1._filing_date = completion_date
filing1.business_id = business.id
filing1.filing_json = base_filing
filing1.effective_date = datetime.datetime.fromisoformat(effective_date)
filing1.payment_token = 'token'
filing1.transaction_id = transaction.id
filing1.payment_completion_date = completion_date
filing1.save()
# filing 2 <- target
effective_date = '2002-07-01T00:00:00+00:00'
completion_date = datetime.datetime.fromisoformat(effective_date)
base_filing = copy.deepcopy(FILING_HEADER)
base_filing['filing']['header']['effectiveDate'] = effective_date
base_filing['filing']['header']['name'] = 'changeOfDirectors'
base_filing['filing']['header']['availableOnPaperOnly'] = True
filing2 = Filing()
filing2._filing_date = completion_date
filing2.business_id = business.id
filing2.filing_json = base_filing
filing2.effective_date = datetime.datetime.fromisoformat(effective_date)
filing2.payment_token = 'token'
filing2.transaction_id = transaction.id
filing2.payment_completion_date = completion_date
filing2.save()
# filing 3
effective_date = '2003-07-01T00:00:00+00:00'
completion_date = datetime.datetime.fromisoformat(effective_date)
base_filing = copy.deepcopy(ANNUAL_REPORT)
base_filing['filing']['header']['effectiveDate'] = effective_date
filing3 = Filing()
filing3._filing_date = completion_date
filing3.business_id = business.id
filing3.filing_json = base_filing
filing3.effective_date = datetime.datetime.fromisoformat(effective_date)
filing3.payment_token = 'token'
filing3.transaction_id = transaction.id
filing3.payment_completion_date = completion_date
filing3.save()
f = Filing.get_most_recent_legal_filing(business.id, 'changeOfDirectors')
assert f.filing_type == 'changeOfDirectors'
assert f.id == filing2.id
def process_filing(payment_token, flask_app):
"""Render the filings contained in the submission."""
if not flask_app:
raise QueueException('Flask App not available.')
with flask_app.app_context():
# try to find the filing 5 times before putting back on the queue - in case payment token ends up on the queue
# before it is assigned to filing.
counter = 1
filing_submission = None
while not filing_submission and counter <= 5:
filing_submission = get_filing_by_payment_id(payment_token['paymentToken'].get('id'))
counter += 1
sleep(0.2)
if not filing_submission:
raise FilingException
if filing_submission.status == Filing.Status.COMPLETED.value:
logger.warning('Queue: Attempting to reprocess business.id=%s, filing.id=%s payment=%s',
filing_submission.business_id, filing_submission.id, payment_token)
return
if payment_token['paymentToken'].get('statusCode') == 'TRANSACTION_FAILED':
# TODO - need to surface TRANSACTION_FAILED, but Filings manages its own status
# filing_submission.status = Filing.Status.ERROR
filing_submission.payment_completion_date = datetime.datetime.utcnow()
db.session.add(filing_submission)
db.session.commit()
return
legal_filings = filing_submission.legal_filings()
# TODO: handle case where there are no legal_filings
uow = versioning_manager.unit_of_work(db.session)
transaction = uow.create_transaction(db.session)
if not payment_token['paymentToken'].get('statusCode') == 'TRANSACTION_FAILED':
if not payment_token['paymentToken'].get('statusCode') == Filing.Status.COMPLETED.value:
logger.error('Unknown payment status given: %s', payment_token['paymentToken'].get('statusCode'))
raise QueueException
business = Business.find_by_internal_id(filing_submission.business_id)
for filing in legal_filings:
if filing.get('annualReport'):
annual_report.process(business, filing)
if filing.get('changeOfAddress'):
change_of_address.process(business, filing)
if filing.get('changeOfDirectors'):
change_of_directors.process(business, filing)
filing_submission.transaction_id = transaction.id
db.session.add(business)
filing_submission.payment_completion_date = datetime.datetime.utcnow()
db.session.add(filing_submission)
db.session.commit()
return
def test_create_transaction_with_scoped_session(self):
article = self.Article()
article.name = u"Some article"
article.content = u"Some content"
self.db.session.add(article)
uow = versioning_manager.unit_of_work(self.db.session)
transaction = uow.create_transaction(self.db.session)
assert transaction.id
def test_create_transaction_with_scoped_session(self):
article = self.Article()
article.name = u'Some article'
article.content = u'Some content'
self.db.session.add(article)
uow = versioning_manager.unit_of_work(self.db.session)
transaction = uow.create_transaction(self.db.session)
assert transaction.id
def process_filing(filing_msg: Dict, flask_app: Flask):
"""Render the filings contained in the submission."""
if not flask_app:
raise QueueException('Flask App not available.')
with flask_app.app_context():
filing_submission = Filing.find_by_id(filing_msg['filing']['id'])
if not filing_submission:
raise QueueException
if filing_submission.status == Filing.Status.COMPLETED.value:
logger.warning('QueueFiler: Attempting to reprocess business.id=%s, filing.id=%s filing=%s',
filing_submission.business_id, filing_submission.id, filing_msg)
return
legal_filings = filing_submission.legal_filings()
if legal_filings:
uow = versioning_manager.unit_of_work(db.session)
transaction = uow.create_transaction(db.session)
business = Business.find_by_internal_id(filing_submission.business_id)
for filing in legal_filings:
if filing.get('annualReport'):
annual_report.process(business, filing)
elif filing.get('changeOfAddress'):
change_of_address.process(business, filing)
elif filing.get('changeOfDirectors'):
filing['colinIds'] = filing_submission.colin_event_ids
change_of_directors.process(business, filing)
elif filing.get('changeOfName'):
change_of_name.process(business, filing)
elif filing.get('specialResolution'):
pass # nothing to do here
elif filing.get('voluntaryDissolution'):
voluntary_dissolution.process(business, filing)
elif filing.get('incorporationApplication'):
incorporation_filing.process(business, filing, flask_app)
filing_submission.transaction_id = transaction.id
filing_submission.set_processed()
db.session.add(business)
db.session.add(filing_submission)
db.session.commit()
publish_event(business, filing_submission)
return
def __add_filings(self, business, book):
# Get the filings properties and create the filings
filings_sheet = book.sheet_by_name(SheetName.FILING.value)
iter_filings_rows = iter(filings_sheet.get_rows())
# (skipping the header line)
next(iter_filings_rows)
for filing_row in iter_filings_rows:
transaction_id = None
if filing_row[0].value == business.identifier:
# If the filing is completed, it has to contain a transaction ID
status = self.__get_value_from_row(filing_row, 9)
if Filing.Status.COMPLETED.value == status:
uow = versioning_manager.unit_of_work(db.session)
transaction = uow.create_transaction(db.session)
transaction_id = transaction.id
filing = Filing(
_completion_date=self.__get_value_from_row(filing_row, 2),
_filing_date=self.__get_value_from_row(filing_row, 3),
_filing_type=self.__get_value_from_row(filing_row, 4),
effective_date=self.__get_value_from_row(filing_row, 5),
_payment_token=self.__get_value_from_row(filing_row, 6),
_payment_completion_date=self.__get_value_from_row(
filing_row, 7),
_status=status,
paper_only=self.__get_value_from_row(filing_row, 10),
# transaction_id comes from continuuum
transaction_id=transaction_id
)
filing.business_id = business.id
# need to convert this first before storing
filing_value = self.__get_value_from_row(filing_row, 1)
if filing_value:
filing._filing_json = json.loads(filing_value) # pylint: disable=protected-access
business.filings.append(filing)
db.session.add(filing)
db.session.commit()
# add colin event ids
colin_event_ids = self.__get_value_from_row(filing_row, 8)
if colin_event_ids:
# convert string to list
colin_event_ids = eval(colin_event_ids)
for colin_event_id in colin_event_ids:
colin_event_id_obj = ColinEventId(
colin_event_id=colin_event_id,
filing_id=filing.id
)
filing.colin_event_ids.append(colin_event_id_obj)
db.session.add(filing)
db.session.commit()
def test_assign_meta_to_transaction(self):
self.article.name = u'Some update article'
meta = {u'some_key': u'some_value'}
uow = versioning_manager.unit_of_work(self.session)
tx = uow.create_transaction(self.session)
tx.meta = meta
self.session.commit()
tx = self.article.versions[-1].transaction
assert tx.meta[u'some_key'] == u'some_value'
def content_node_versioned_with_alias_id(session, content_node):
from sqlalchemy_continuum import versioning_manager
session.commit()
content_node.orderpos = 42
session.commit()
uow = versioning_manager.unit_of_work(session)
tx = uow.create_transaction(session)
tx.meta = {u"alias_id": 23}
content_node.orderpos = 23
session.commit()
return content_node
def factory_epoch_filing(business, filing_date=FROZEN_DATETIME):
"""Create an error filing."""
filing = Filing()
filing.business_id = business.id
uow = versioning_manager.unit_of_work(db.session)
transaction = uow.create_transaction(db.session)
filing.transaction_id = transaction.id
filing.filing_date = filing_date
filing.filing_json = {'filing': {'header': {'name': 'lear_epoch'}}}
filing.save()
return filing
def load_historic_filings(corp_num: str, business: Business, legal_type: str):
"""Load historic filings for a business."""
try:
# get historic filings
r = requests.get(
f'{COLIN_API}/api/v1/businesses/{legal_type}/{corp_num}/filings/historic',
timeout=TIMEOUT)
if r.status_code != HTTPStatus.OK or not r.json():
print(
f'skipping history for {corp_num} historic filings not found')
else:
for historic_filing in r.json():
uow = versioning_manager.unit_of_work(db.session)
transaction = uow.create_transaction(db.session)
filing = Filing()
filing_date = historic_filing['filing']['header']['date']
filing.filing_date = datetime.datetime.strptime(
filing_date, '%Y-%m-%d')
filing.business_id = business.id
filing.filing_json = historic_filing
for colin_id in filing.filing_json['filing']['header'][
'colinIds']:
colin_event_id = ColinEventId()
colin_event_id.colin_event_id = colin_id
filing.colin_event_ids.append(colin_event_id)
filing.transaction_id = transaction.id
filing._filing_type = historic_filing['filing']['header'][
'name']
filing.paper_only = True
filing.effective_date = datetime.datetime.strptime(
historic_filing['filing']['header']['effectiveDate'],
'%Y-%m-%d')
updater_user = User.find_by_username(UPDATER_USERNAME)
filing.submitter_id = updater_user.id
filing.source = Filing.Source.COLIN.value
db.session.add(filing)
# only commit after all historic filings were added successfully
db.session.commit()
LOADED_FILING_HISTORY.append(corp_num)
except requests.exceptions.Timeout:
print('rolling back partial changes...')
db.session.rollback()
FAILED_FILING_HISTORY.append(corp_num)
print('colin_api request timed out getting historic filings.')
except Exception as err:
print('rolling back partial changes...')
db.session.rollback()
FAILED_FILING_HISTORY.append(corp_num)
raise err
def test_manual_transaction_creation(self):
uow = versioning_manager.unit_of_work(self.session)
transaction = uow.create_transaction(self.session)
self.session.flush()
assert transaction.id
article = self.Article(name=u'Session1 article')
self.session.add(article)
self.session.flush()
assert uow.current_transaction.id
self.session.commit()
assert article.versions[-1].transaction_id
def factory_completed_filing(business, data_dict, filing_date=FROZEN_DATETIME):
"""Create a completed filing."""
filing = Filing()
filing.business_id = business.id
filing.filing_date = filing_date
filing.filing_json = data_dict
filing.save()
uow = versioning_manager.unit_of_work(db.session)
transaction = uow.create_transaction(db.session)
filing.transaction_id = transaction.id
filing.payment_token = 1
filing.payment_completion_date = datetime.now()
return filing
def test_get_filings_by_status__default_order(session):
"""Assert that a filing can be retrieved.
by status and is returned in the default order.
default order is submission_date, and then effective_date.
"""
# setup
base_filing = copy.deepcopy(FILING_HEADER)
base_filing['specialResolution'] = SPECIAL_RESOLUTION
uow = versioning_manager.unit_of_work(session)
business = factory_business('CP1234567')
completion_date = datetime.datetime.utcnow().replace(
tzinfo=datetime.timezone.utc)
# setup - create multiple filings on the same day & time
filing_ids = []
file_counter = -1
with freeze_time(completion_date):
for i in range(0, 5):
transaction = uow.create_transaction(session)
payment_token = str(i)
effective_date = f'200{i}-04-15T00:00:00+00:00'
base_filing['filing']['header']['effectiveDate'] = effective_date
filing = Filing()
filing._filing_date = completion_date
filing.business_id = business.id
filing.filing_json = base_filing
filing.effective_date = datetime.datetime.fromisoformat(
effective_date)
filing.payment_token = payment_token
filing.transaction_id = transaction.id
filing.payment_completion_date = completion_date
filing.save()
filing_ids.append(filing.id)
file_counter += 1
# test
rv = Filing.get_filings_by_status(business.id,
[Filing.Status.COMPLETED.value])
# check
assert rv
# filings should be in newest to oldest effective date order
for filing in rv:
assert filing.id == filing_ids[file_counter]
file_counter -= 1
def test_get_filings_by_status(session):
"""Assert that a filing can be retrieved by status."""
uow = versioning_manager.unit_of_work(session)
transaction = uow.create_transaction(session)
business = factory_business('CP1234567')
payment_token = '1000'
filing = Filing()
filing.business_id = business.id
filing.filing_json = ANNUAL_REPORT
filing.payment_token = payment_token
filing.transaction_id = transaction.id
filing.payment_completion_date = datetime.datetime.utcnow()
filing.save()
rv = Filing.get_filings_by_status(business.id, [Filing.Status.COMPLETED.value])
assert rv
assert rv[0].status == Filing.Status.COMPLETED.value
def email_prepped_filing(session, identifier, payment_id, option):
"""Return a test filing ready for email notification."""
business = create_business(identifier)
filing_template = copy.deepcopy(INCORPORATION_FILING_TEMPLATE)
filing_template['filing']['business'] = {'identifier': business.identifier}
for party in filing_template['filing']['incorporationApplication']['parties']:
for role in party['roles']:
if role['roleType'] == 'Completing Party':
party['officer']['email'] = '*****@*****.**'
filing_template['filing']['incorporationApplication']['contactPoint']['email'] = '*****@*****.**'
filing = create_filing(token=payment_id, json_filing=filing_template, business_id=business.id)
filing.payment_completion_date = filing.filing_date
filing.save()
if option in ['registered', 'bn']:
uow = versioning_manager.unit_of_work(session)
transaction = uow.create_transaction(session)
filing.transaction_id = transaction.id
filing.save()
return filing
def __enter__(self):
self.session = s = object_session(node)
if s.new or s.dirty:
raise Exception(
"Refusing to create a new tagged node version. Session must be clean!"
)
uow = versioning_manager.unit_of_work(s)
tx = uow.create_transaction(s)
if user is not None:
tx.user = user
if tag:
if node.get_tagged_version(tag):
raise ValueError("tag already exists")
tx.meta[u"tag"] = tag
elif publish:
if node.get_published_version():
raise ValueError("publish version already exists")
tx.meta[u"publish"] = publish
else:
NodeVersion = version_class(node.__class__)
# in case you were wondering: order_by(None) resets the default order_by
last_tagged_version = node.tagged_versions.order_by(
None).order_by(
NodeVersion.transaction_id.desc()).first()
if last_tagged_version is not None:
next_version = int(last_tagged_version.tag) + 1
else:
node.versions[-1].tag = u"1"
next_version = 2
tx.meta[u"tag"] = unicode(next_version)
if comment:
tx.meta[u"comment"] = comment
return tx
def create(source, ids, force):
"""Create new bibliographic record(s)."""
# Make sure that all imports are done with application context.
from sqlalchemy_continuum import versioning_manager
from .api import Record
from .models import RecordMetadata
data = json.load(source)
if isinstance(data, dict):
data = [data]
if ids:
assert len(ids) == len(data), 'Not enough identifiers.'
for record, id_ in zip_longest(data, ids):
try:
click.echo(Record.create(record, id_=id_).id)
except exc.IntegrityError:
if force:
current_app.logger.warning(
"Trying to force insert: {0}".format(id_))
# IMPORTANT: We need to create new transtaction for
# SQLAlchemy-Continuum as we are using no autoflush
# in Record.get_record.
uow = versioning_manager.unit_of_work(db.session)
transaction = uow.create_transaction(db.session)
# Use low-level database model to retreive an instance.
model = RecordMetadata.query.get(id_)
click.echo(Record(record, model=model).commit().id)
else:
raise click.BadParameter(
'Record with id={0} already exists. If you want to '
'override its data use --force.'.format(id_),
param_hint='ids',
)
db.session.flush()
db.session.commit()
def test_filing_orm_delete_blocked_if_completed(session):
"""Assert that attempting to delete a filing will raise a BusinessException."""
from legal_api.exceptions import BusinessException
uow = versioning_manager.unit_of_work(session)
transaction = uow.create_transaction(session)
b = factory_business('CP1234567')
filing = Filing()
filing.business_id = b.id
filing.filing_date = datetime.datetime.utcnow()
filing.filing_json = ANNUAL_REPORT
filing.payment_token = 'a token'
filing.payment_completion_date = datetime.datetime.utcnow()
filing.transaction_id = transaction.id
filing.save()
with pytest.raises(BusinessException) as excinfo:
session.delete(filing)
session.commit()
assert excinfo.value.status_code == HTTPStatus.FORBIDDEN
assert excinfo.value.error == 'Deletion not allowed.'
def test_with_entity_arg(self):
article = self.Article()
self.session.add(article)
uow = versioning_manager.unit_of_work(article)
assert isinstance(uow, UnitOfWork)
def test_raises_type_error_for_unknown_type(self):
with raises(TypeError):
versioning_manager.unit_of_work(None)
def update_business_legal_name(business, legal_name):
"""Update business legal name."""
uow = versioning_manager.unit_of_work(db.session)
uow.create_transaction(db.session)
business.legal_name = legal_name
business.save()
def test_with_entity_arg(self):
article = self.Article()
self.session.add(article)
uow = versioning_manager.unit_of_work(article)
assert isinstance(uow, UnitOfWork)
def test_raises_type_error_for_unknown_type(self):
with raises(TypeError):
versioning_manager.unit_of_work(None)
def load_corps(csv_filepath: str = 'corp_nums/corps_to_load.csv'):
"""Load corps in given csv file from oracle into postgres."""
global ROWCOUNT
with open(csv_filepath, 'r') as csvfile:
reader = csv.DictReader(csvfile)
with FLASK_APP.app_context():
for row in reader:
corp_num = row['CORP_NUM']
print('loading: ', corp_num)
added = False
ROWCOUNT += 1
try:
legal_type = Business.LegalTypes.COOP.value
if corp_num[:2] != Business.LegalTypes.COOP.value:
legal_type = Business.LegalTypes.BCOMP.value
corp_num = 'BC' + corp_num[-7:]
business = Business.find_by_identifier(corp_num)
if business:
added = True
print(
'-> business info already exists -- skipping corp load'
)
else:
try:
# get current company info
business_current_info = {}
for info_type in BUSINESS_MODEL_INFO_TYPES[
legal_type]:
business_current_info[
info_type] = get_oracle_info(
corp_num=corp_num,
legal_type=legal_type,
info_type=info_type)
if business_current_info[info_type].get(
'failed', False):
raise Exception(
f'could not load {info_type}')
except requests.exceptions.Timeout:
FAILED_CORPS.append(corp_num)
print(
'colin_api request timed out getting corporation details.'
)
continue
except Exception as err:
print(f'exception: {err}')
print(
f'skipping load for {corp_num}, exception occurred getting company info'
)
continue
uow = versioning_manager.unit_of_work(db.session)
transaction = uow.create_transaction(db.session)
try:
# add BC prefix to non coop identifiers
if legal_type != Business.LegalTypes.COOP.value:
business_current_info['business']['business']['identifier'] = 'BC' + \
business_current_info['business']['business']['identifier']
# add company to postgres db
business = create_business(
business_current_info['business'])
add_business_offices(
business, business_current_info['office'])
add_business_directors(
business, business_current_info['parties'])
if legal_type == Business.LegalTypes.BCOMP.value:
add_business_shares(
business,
business_current_info['sharestructure'])
add_business_resolutions(
business,
business_current_info['resolutions'])
add_business_aliases(
business, business_current_info['aliases'])
filing = Filing()
filing.filing_json = {
'filing': {
'header': {
'name': 'lear_epoch'
},
'business': business.json()
}
}
filing._filing_type = 'lear_epoch'
filing.source = Filing.Source.COLIN.value
filing.transaction_id = transaction.id
business.filings.append(filing)
business.save()
added = True
NEW_CORPS.append(corp_num)
except Exception as err:
print(err)
print(f'skipping {corp_num} missing info')
FAILED_CORPS.append(corp_num)
if added and history_needed(business=business):
load_historic_filings(corp_num=corp_num,
business=business,
legal_type=legal_type)
else:
print(
'-> historic filings not needed - skipping history load'
)
except Exception as err:
print(err)
exit(-1)
r = requests.get(COLIN_API + '/api/v1/businesses/' +
row['CORP_NUM'] + '/directors',
timeout=TIMEOUT)
if r.status_code != HTTPStatus.OK \
or not r.json():
print('skipping ' + row['CORP_NUM'] +
' business directors not found')
continue
directors_json = r.json()
except requests.exceptions.Timeout as timeout:
print(
'colin_api request timed out getting corporation details.'
)
continue
uow = versioning_manager.unit_of_work(db.session)
transaction = uow.create_transaction(db.session)
business = create_business(db, business_json)
add_business_addresses(business, offices_json)
add_business_directors(business, directors_json)
db.session.add(business)
filing = Filing()
# filing.filing_date = datetime.datetime.utcnow
filing.filing_json = {
'filing': {
'header': {
'name': 'lear_epoch'
}
}
async def process_filing(filing_msg: Dict, flask_app: Flask): # pylint: disable=too-many-branches,too-many-statements
"""Render the filings contained in the submission."""
if not flask_app:
raise QueueException('Flask App not available.')
with flask_app.app_context():
filing_submission = Filing.find_by_id(filing_msg['filing']['id'])
if not filing_submission:
raise QueueException
if filing_submission.status == Filing.Status.COMPLETED.value:
logger.warning(
'QueueFiler: Attempting to reprocess business.id=%s, filing.id=%s filing=%s',
filing_submission.business_id, filing_submission.id,
filing_msg)
return None, None
if legal_filings := filing_submission.legal_filings():
uow = versioning_manager.unit_of_work(db.session)
transaction = uow.create_transaction(db.session)
business = Business.find_by_internal_id(
filing_submission.business_id)
for filing in legal_filings:
if filing.get('alteration'):
alteration.process(business, filing_submission, filing)
if filing.get('annualReport'):
annual_report.process(business, filing)
elif filing.get('changeOfAddress'):
change_of_address.process(business, filing)
elif filing.get('changeOfDirectors'):
filing['colinIds'] = filing_submission.colin_event_ids
change_of_directors.process(business, filing)
elif filing.get('changeOfName'):
change_of_name.process(business, filing)
elif filing.get('voluntaryDissolution'):
voluntary_dissolution.process(business, filing)
elif filing.get('incorporationApplication'):
business, filing_submission = incorporation_filing.process(
business, filing, filing_submission)
if filing.get('correction'):
filing_submission = correction.process(
filing_submission, filing)
if filing.get('transition'):
filing_submission = transition.process(
business, filing_submission, filing)
filing_submission.transaction_id = transaction.id
filing_submission.set_processed()
db.session.add(business)
db.session.add(filing_submission)
db.session.commit()
# post filing changes to other services
if any('alteration' in x for x in legal_filings):
if name_request.has_new_nr_for_alteration(
business, filing_submission.filing_json):
name_request.consume_nr(
business, filing_submission,
'/filing/alteration/nameRequest/nrNumber')
alteration.post_process(business, filing_submission)
db.session.add(business)
db.session.commit()
AccountService.update_entity(
business_registration=business.identifier,
business_name=business.legal_name,
corp_type_code=business.legal_type)
if any('incorporationApplication' in x for x in legal_filings):
if any('correction' in x for x in legal_filings):
if name_request.has_new_nr_for_correction(
filing_submission.filing_json):
name_request.consume_nr(business, filing_submission)
else:
filing_submission.business_id = business.id
db.session.add(filing_submission)
db.session.commit()
incorporation_filing.update_affiliation(
business, filing_submission)
name_request.consume_nr(business, filing_submission)
incorporation_filing.post_process(business,
filing_submission)
try:
await publish_email_message(
qsm, APP_CONFIG.EMAIL_PUBLISH_OPTIONS['subject'],
filing_submission, 'mras')
except Exception as err: # pylint: disable=broad-except, unused-variable # noqa F841;
# mark any failure for human review
capture_message(
f'Queue Error: Failed to place email for filing:{filing_submission.id}'
f'on Queue with error:{err}',
level='error')
try:
await publish_email_message(
qsm, APP_CONFIG.EMAIL_PUBLISH_OPTIONS['subject'],
filing_submission, filing_submission.status)
except Exception as err: # pylint: disable=broad-except, unused-variable # noqa F841;
# mark any failure for human review
capture_message(
f'Queue Error: Failed to place email for filing:{filing_submission.id}'
f'on Queue with error:{err}',
level='error')
try:
await publish_event(business, filing_submission)
except Exception as err: # pylint: disable=broad-except, unused-variable # noqa F841;
# mark any failure for human review
capture_message(
f'Queue Error: Failed to publish event for filing:{filing_submission.id}'
f'on Queue with error:{err}',
level='error')
def test_with_session_arg(self):
uow = versioning_manager.unit_of_work(self.session)
assert isinstance(uow, UnitOfWork)
def upgrade():
# ensure new transaction/version tables exist
db.create_all()
# update CVE table
op.add_column(
'cve',
Column('created',
DateTime,
default=datetime.utcnow,
nullable=True,
index=True))
op.add_column(
'cve',
Column('changed',
DateTime,
default=datetime.utcnow,
nullable=True,
index=True))
for cve in CVE.query.all():
cve.created = datetime.utcnow()
cve.changed = cve.created
db.session.commit()
db.session.flush()
# update AVG table
op.add_column(
'cve_group',
Column('changed',
DateTime,
default=datetime.utcnow,
nullable=True,
index=True))
for group in CVEGroup.query.all():
group.changed = group.created
db.session.commit()
db.session.flush()
VersionClassGroup = version_class(CVEGroup)
uow = versioning_manager.unit_of_work(db.session)
uow.create_transaction(db.session)
for group in VersionClassGroup.query.all():
for package in CVEGroupPackage.query.filter(
CVEGroupPackage.group_id == group.id).all():
package_version = uow.get_or_create_version_object(package)
package_version.group_id = group.id
package_version.pkgname = package.pkgname
package_version.transaction_id = group.transaction_id
package_version.end_transaction_id = group.end_transaction_id
package_version.operation_type = Operation.INSERT
package_version.group_id_mod = 1
package_version.pkgname_mod = 1
uow.process_operation(Operation(package, Operation.INSERT))
for cve in CVEGroupEntry.query.filter(
CVEGroupEntry.group_id == group.id).all():
cve_version = uow.get_or_create_version_object(cve)
cve_version.group_id = group.id
cve_version.cve_id = cve.cve_id
cve_version.transaction_id = group.transaction_id
cve_version.end_transaction_id = group.end_transaction_id
cve_version.operation_type = Operation.INSERT
cve_version.group_id_mod = 1
cve_version.cve_id_mod = 1
uow.process_operation(Operation(cve, Operation.INSERT))
uow.make_versions(db.session)
db.session.commit()
db.session.flush()
with op.batch_alter_table('cve_group', schema=None) as batch_op:
batch_op.alter_column('changed', nullable=False)
# update advisory table
op.add_column(
'advisory',
Column('changed',
DateTime,
default=datetime.utcnow,
nullable=True,
index=True))
for advisory in Advisory.query.all():
advisory.changed = group.created
db.session.commit()
db.session.flush()
with op.batch_alter_table('advisory', schema=None) as batch_op:
batch_op.alter_column('changed', nullable=False)
# set all fields to modified for initial insert
VersionClassCVE = version_class(CVE)
VersionClassCVE.query.update({
VersionClassCVE.operation_type: Operation.INSERT,
VersionClassCVE.issue_type_mod: 1,
VersionClassCVE.description_mod: 1,
VersionClassCVE.severity_mod: 1,
VersionClassCVE.remote_mod: 1,
VersionClassCVE.reference_mod: 1,
VersionClassCVE.notes_mod: 1
})
VersionClassGroup = version_class(CVEGroup)
VersionClassGroup.query.update({
VersionClassGroup.operation_type:
Operation.INSERT,
VersionClassGroup.status_mod:
1,
VersionClassGroup.severity_mod:
1,
VersionClassGroup.affected_mod:
1,
VersionClassGroup.fixed_mod:
1,
VersionClassGroup.bug_ticket_mod:
1,
VersionClassGroup.reference_mod:
1,
VersionClassGroup.notes_mod:
1,
VersionClassGroup.created_mod:
1,
VersionClassGroup.changed_mod:
1,
VersionClassGroup.advisory_qualified_mod:
1
})
VersionClassAdvisory = version_class(Advisory)
VersionClassAdvisory.query.update({
VersionClassAdvisory.operation_type:
Operation.INSERT,
VersionClassAdvisory.group_package_id_mod:
1,
VersionClassAdvisory.advisory_type_mod:
1,
VersionClassAdvisory.publication_mod:
1,
VersionClassAdvisory.workaround_mod:
1,
VersionClassAdvisory.impact_mod:
1,
VersionClassAdvisory.content_mod:
1,
VersionClassAdvisory.created_mod:
1,
VersionClassAdvisory.changed_mod:
1,
VersionClassAdvisory.reference_mod:
1
})
db.session.commit()
def test_manual_tx_creation_with_no_actual_changes(self):
self.article.name = u'Some article'
uow = versioning_manager.unit_of_work(self.session)
uow.create_transaction(self.session)
self.session.flush()
