def test_fetch_and_replace_batch_overrides_data(no_award_db, monkeypatch):
def fetch_duns(award_id):
test_result = no_award_db.query(FSRSGrant.duns).filter(
FSRSGrant.id == award_id).one_or_none()
if test_result:
return test_result[0]
return None
award1 = FSRSGrantFactory(id=1, internal_id='12345', duns='To Be Replaced')
award1.subawards = [FSRSSubgrantFactory() for _ in range(4)]
award2 = FSRSGrantFactory(id=2, internal_id='54321', duns='Not Altered')
award2.subawards = [FSRSSubgrantFactory()]
monkeypatch.setattr(fsrs, 'retrieve_batch', Mock(return_value=[award1, award2]))
fsrs.fetch_and_replace_batch(no_award_db, fsrs.GRANT)
assert fetch_duns(1) == 'To Be Replaced'
assert fetch_duns(2) == 'Not Altered'
# 5 subawards, 4 from award1 and 1 from award2
assert no_award_db.query(FSRSSubgrant).count() == 5
no_award_db.expunge_all() # Reset the model cache
award3 = FSRSGrantFactory(id=3, internal_id='12345', duns='Replaced')
award3.subawards = [FSRSSubgrantFactory() for _ in range(2)]
monkeypatch.setattr(fsrs, 'retrieve_batch', Mock(return_value=[award3]))
fsrs.fetch_and_replace_batch(no_award_db, fsrs.GRANT)
assert fetch_duns(1) is None
assert fetch_duns(2) == 'Not Altered'
assert fetch_duns(3) == 'Replaced'
# 3 subawards, 4 from award1/award3 and 1 from award2
assert no_award_db.query(FSRSSubgrant).count() == 3
def test_fetch_and_replace_batch_overrides_data(no_award_db, monkeypatch):
def fetch_duns(award_id):
test_result = no_award_db.query(
FSRSGrant.duns).filter(FSRSGrant.id == award_id).one_or_none()
if test_result:
return test_result[0]
return None
award1 = FSRSGrantFactory(id=1, internal_id='12345', duns='To Be Replaced')
award1.subawards = [FSRSSubgrantFactory() for _ in range(4)]
award2 = FSRSGrantFactory(id=2, internal_id='54321', duns='Not Altered')
award2.subawards = [FSRSSubgrantFactory()]
monkeypatch.setattr(fsrs, 'retrieve_batch',
Mock(return_value=[award1, award2]))
fsrs.fetch_and_replace_batch(no_award_db, fsrs.GRANT)
assert fetch_duns(1) == 'To Be Replaced'
assert fetch_duns(2) == 'Not Altered'
# 5 subawards, 4 from award1 and 1 from award2
assert no_award_db.query(FSRSSubgrant).count() == 5
no_award_db.expunge_all() # Reset the model cache
award3 = FSRSGrantFactory(id=3, internal_id='12345', duns='Replaced')
award3.subawards = [FSRSSubgrantFactory() for _ in range(2)]
monkeypatch.setattr(fsrs, 'retrieve_batch', Mock(return_value=[award3]))
fsrs.fetch_and_replace_batch(no_award_db, fsrs.GRANT)
assert fetch_duns(1) is None
assert fetch_duns(2) == 'Not Altered'
assert fetch_duns(3) == 'Replaced'
# 3 subawards, 4 from award1/award3 and 1 from award2
assert no_award_db.query(FSRSSubgrant).count() == 3
def test_fetch_and_replace_batch_saves_data(no_award_db, monkeypatch):
award1 = FSRSProcurementFactory()
award1.subawards = [FSRSSubcontractFactory() for _ in range(4)]
award2 = FSRSProcurementFactory()
award2.subawards = [FSRSSubcontractFactory()]
monkeypatch.setattr(fsrs, 'retrieve_batch', Mock(return_value=[award1, award2]))
assert no_award_db.query(FSRSProcurement).count() == 0
fsrs.fetch_and_replace_batch(no_award_db, fsrs.PROCUREMENT)
assert no_award_db.query(FSRSProcurement).count() == 2
assert no_award_db.query(FSRSSubcontract).count() == 5
def test_fetch_and_replace_batch_saves_data(no_award_db, monkeypatch):
award1 = FSRSProcurementFactory()
award1.subawards = [FSRSSubcontractFactory() for _ in range(4)]
award2 = FSRSProcurementFactory()
award2.subawards = [FSRSSubcontractFactory()]
monkeypatch.setattr(fsrs, 'retrieve_batch',
Mock(return_value=[award1, award2]))
assert no_award_db.query(FSRSProcurement).count() == 0
fsrs.fetch_and_replace_batch(no_award_db, fsrs.PROCUREMENT)
assert no_award_db.query(FSRSProcurement).count() == 2
assert no_award_db.query(FSRSSubcontract).count() == 5
args = parser.parse_args()
# Setups state mapping for deriving state name
config_state_mappings(sess, init=True)
if not config_valid():
logger.error("No config for broker/fsrs/[service]/wsdl")
sys.exit(1)
elif args.procurement and args.grants and args.ids:
logger.error("Cannot run both procurement and grant loads when specifying FSRS ids")
else:
# Regular FSRS data load, starts where last load left off
if len(sys.argv) <= 1:
awards = ['Starting']
while len(awards) > 0:
procs = fetch_and_replace_batch(sess, PROCUREMENT)
grants = fetch_and_replace_batch(sess, GRANT)
awards = procs + grants
log_fsrs_counts(awards)
elif args.procurement and args.ids:
for procurement_id in args.ids:
logger.info('Begin loading FSRS reports for procurement id {}'.format(procurement_id))
procs = fetch_and_replace_batch(sess, PROCUREMENT, procurement_id)
log_fsrs_counts(procs)
elif args.grants and args.ids:
for grant_id in args.ids:
logger.info('Begin loading FSRS reports for grant id {}'.format(grant_id))
grants = fetch_and_replace_batch(sess, GRANT, grant_id)
log_fsrs_counts(grants)
sys.exit(1)
if max_proc_id:
logger.info('Loading FSRS contracts from {} up to {}'.format(
next_proc_id, max_proc_id))
if max_grant_id:
logger.info('Loading FSRS grants from {} up to {}'.format(
next_grant_id, max_grant_id))
awards = ['Starting']
while len(awards) > 0:
procs = []
if max_proc_id:
procs = fetch_and_replace_batch(sess,
PROCUREMENT,
next_proc_id,
min_id=True,
max_id=max_proc_id)
if procs:
next_proc_id = procs[-1].id + 1
grants = []
if max_grant_id:
grants = fetch_and_replace_batch(sess,
GRANT,
next_grant_id,
min_id=True,
max_id=max_grant_id)
if grants:
next_grant_id = grants[-1].id + 1
awards = procs + grants
logger.info('Finished FSRS backfill.')
last_updated_at = sess.query(func.max(
Subaward.updated_at)).one_or_none()[0]
if len(sys.argv) <= 1:
# there may be more transaction data since we've last run, let's fix any links before importing new data
if last_updated_at:
fix_broken_links(sess,
PROCUREMENT,
min_date=last_updated_at)
fix_broken_links(sess, GRANT, min_date=last_updated_at)
awards = ['Starting']
logger.info('Loading latest FSRS reports')
while len(awards) > 0:
procs = fetch_and_replace_batch(
sess,
PROCUREMENT,
SERVICE_MODEL[PROCUREMENT].next_id(sess),
min_id=True)
grants = fetch_and_replace_batch(
sess,
GRANT,
SERVICE_MODEL[GRANT].next_id(sess),
min_id=True)
updated_proc_internal_ids.extend(
[proc.internal_id for proc in procs])
updated_grant_internal_ids.extend(
[grant.internal_id for grant in grants])
awards = procs + grants
log_fsrs_counts(awards)
metric_counts(procs, 'procurement', metrics_json)
metric_counts(grants, 'grant', metrics_json)
import logging
import sys
from dataactcore.interfaces.db import GlobalDB
from dataactcore.logging import configure_logging
from dataactbroker.fsrs import config_valid, fetch_and_replace_batch, GRANT, PROCUREMENT
from dataactvalidator.health_check import create_app
logger = logging.getLogger(__name__)
if __name__ == '__main__':
configure_logging()
with create_app().app_context():
sess = GlobalDB.db().session
if not config_valid():
logger.error("No config for broker/fsrs/[service]/wsdl")
sys.exit(1)
else:
awards = ['Starting']
while len(awards) > 0:
procs = fetch_and_replace_batch(sess, PROCUREMENT)
grants = fetch_and_replace_batch(sess, GRANT)
awards = procs + grants
numSubAwards = sum(len(a.subawards) for a in awards)
logger.info("Inserted/Updated %s awards, %s subawards", len(awards), numSubAwards)
import logging
import sys
from dataactcore.interfaces.db import GlobalDB
from dataactcore.logging import configure_logging
from dataactbroker.fsrs import config_valid, fetch_and_replace_batch, GRANT, PROCUREMENT
from dataactvalidator.health_check import create_app
logger = logging.getLogger(__name__)
if __name__ == '__main__':
configure_logging()
with create_app().app_context():
sess = GlobalDB.db().session
if not config_valid():
logger.error("No config for broker/fsrs/[service]/wsdl")
sys.exit(1)
else:
awards = ['Starting']
while len(awards) > 0:
procs = fetch_and_replace_batch(sess, PROCUREMENT)
grants = fetch_and_replace_batch(sess, GRANT)
awards = procs + grants
numSubAwards = sum(len(a.subawards) for a in awards)
logger.info("Inserted/Updated %s awards, %s subawards",
len(awards), numSubAwards)
# Setups state mapping for deriving state name
config_state_mappings(sess, init=True)
if not config_valid():
logger.error("No config for broker/fsrs/[service]/wsdl")
sys.exit(1)
elif args.procurement and args.grants and args.ids:
logger.error(
"Cannot run both procurement and grant loads when specifying FSRS ids"
)
else:
# Regular FSRS data load, starts where last load left off
if len(sys.argv) <= 1:
awards = ['Starting']
while len(awards) > 0:
procs = fetch_and_replace_batch(sess, PROCUREMENT)
grants = fetch_and_replace_batch(sess, GRANT)
awards = procs + grants
log_fsrs_counts(awards)
elif args.procurement and args.ids:
for procurement_id in args.ids:
logger.info(
'Begin loading FSRS reports for procurement id {}'.
format(procurement_id))
procs = fetch_and_replace_batch(sess, PROCUREMENT,
procurement_id)
log_fsrs_counts(procs)
elif args.grants and args.ids:
for grant_id in args.ids:
