def test_account_balance_normal_sync(self, get_mock):
"""
Verifies that more than 2 years of balances are fetched on non-initial sync.
Args:
get_mock(Mock): mock of the api get function
"""
with open('tests/resources/trial_balance_report.json'
) as report_contents:
get_mock.return_value = json.load(report_contents)
# setup org and sync state
self.create_org(status=CONNECTED, changeset=1)
marker = (date(2010, 1, 1) -
timedelta(days=3 * 365)).strftime('%Y-%m-%d')
QboSyncData(id='test', account_balance_marker=marker).put()
# run sync
stage = AccountBalanceReportStage('test')
complete, _ = stage.next(payload={})
# stage should not be completed
self.assertFalse(complete)
# and marker should be incremented for the next sync
self.assertEqual(
QboSyncData.get_by_id('test').account_balance_marker, '2007-01-01')
def test_account_balance_initial_sync(self, get_mock):
"""
Verifies that only 2 years of balances are fetched on initial sync.
Args:
get_mock(Mock): mock of the api get function
"""
with open('tests/resources/trial_balance_report.json'
) as report_contents:
get_mock.return_value = json.load(report_contents)
# setup org and sync state (marker is 3 years ago)
self.create_org(status=CONNECTED, changeset=0)
marker = (date(2010, 1, 1) -
timedelta(days=3 * 365)).strftime('%Y-%m-%d')
QboSyncData(id='test', account_balance_marker=marker).put()
# run sync
stage = AccountBalanceReportStage('test')
complete, _ = stage.next(payload={})
# stage should be completed
self.assertTrue(complete)
# marker is cleared
self.assertIsNone(QboSyncData.get_by_id('test').account_balance_marker)
def test_company_info_deduplication(self, get_mock):
"""
Verifies that company info is not written to Item if it has not been changed since the last pull (qbo ignores
LastUpdatedTime filter on this endpoint).
Args:
get_mock(Mock): mock of the api get function
"""
get_mock.return_value = self.get_mock_api_response(
COMPANY_INFO_ENDPOINT_NAME)
self.create_org(status=CONNECTED)
# set sync state so that the next pull will be company info
QboSyncData(id='test',
stage_index=LIST_API_STAGE,
endpoint_index=COMPANY_INFO_ENDPOINT_INDEX).put()
# run the sync
stage = ListApiStage('test')
stage.next(payload={})
# one item should have been stored in Item
self.assertEqual(self.count_items(), 1)
# now sync again
QboSyncData(id='test',
stage_index=LIST_API_STAGE,
endpoint_index=COMPANY_INFO_ENDPOINT_INDEX).put()
stage = ListApiStage('test')
stage.next(payload={})
# and there should be no additional rows despite the api returning company info again (because it is the same)
self.assertEqual(self.count_items(), 1)
def test_account_balance_no_marker(self, get_mock):
"""
Verifies that balances progress tracking can handle the case where there is no account balance marker (brand new
file being connected, first sync)..
Args:
get_mock(Mock): mock of the api get function
"""
with open('tests/resources/trial_balance_report.json'
) as report_contents:
get_mock.return_value = json.load(report_contents)
# setup org and sync state
self.create_org(status=CONNECTED, changeset=1)
QboSyncData(id='test').put()
# run sync
stage = AccountBalanceReportStage('test')
complete, _ = stage.next(payload={})
# stage should be completed
self.assertFalse(complete)
# marker is set to the next day to fetch
self.assertEqual(
QboSyncData.get_by_id('test').account_balance_marker, '2009-12-31')
def test_multiple_pages(self, get_mock):
"""
Verifies that once an endpoint which returns 100 items they are saved and sync fetches the next page of the same
endpoint.
Args:
get_mock(Mock): mock of the api get function
"""
get_mock.return_value = self.get_mock_api_response(
INVOICE_ENDPOINT_NAME, 100)
self.create_org(status=CONNECTED)
# set sync state so that the next pull will be invoice
QboSyncData(id='test',
stage_index=LIST_API_STAGE,
endpoint_index=INVOICE_ENDPOINT_INDEX,
start_position=1).put()
# run the sync
stage = ListApiStage('test')
stage.next(payload={})
# all items should have been stored in Item
self.assertEqual(self.count_items(), 100)
# and start position should be shifted but the endpoint_index should stay the same
sync_data = QboSyncData.get_by_id('test')
self.assertEqual(sync_data.start_position, 101)
self.assertEqual(sync_data.endpoint_index, INVOICE_ENDPOINT_INDEX)
def test_endpoint_complete(self, get_mock):
"""
Verifies that once an endpoint returns less data than requested, the sync moves onto the next endpoint.
Args:
get_mock(Mock): mock of the api get function
"""
get_mock.return_value = self.get_mock_api_response(
INVOICE_ENDPOINT_NAME, 50)
self.create_org(status=CONNECTED)
# set sync state so that the next pull will be invoice
QboSyncData(id='test',
stage_index=LIST_API_STAGE,
endpoint_index=INVOICE_ENDPOINT_INDEX).put()
# run the sync
stage = ListApiStage('test')
stage.next(payload={})
# one item should have been stored in Item
self.assertEqual(self.count_items(), 50)
# and start position should be reset but the endpoint_index should be increased
sync_data = QboSyncData.get_by_id('test')
self.assertEqual(sync_data.start_position, 1)
self.assertEqual(sync_data.endpoint_index, INVOICE_ENDPOINT_INDEX + 1)
def test_account_balance_today(self, get_mock):
"""
Verifies that today's account balance is not fetched if the day hasn't ticked over.
Args:
get_mock(Mock): a mock of the qbo api call function
"""
get_mock.return_value = {'Rows': {'Row': []}}
# setup org and sync state
self.create_org(status=CONNECTED)
QboSyncData(id='test',
account_balance_initial_marker='2010-01-01').put()
# run sync
stage = AccountBalanceReportStage('test')
stage.next(payload={})
# balance has been pulled
get_mock.assert_not_called()
# marker should be left alone because it is org today
self.assertEqual(
QboSyncData.get_by_id('test').account_balance_initial_marker,
'2010-01-01')
def __init__(self, org_uid):
"""
Class initialiser. Retrieves stage of the sync which is in progress.
Args:
org_uid(str): org identifier
"""
self.sync_data = QboSyncData.get_by_id(org_uid) or QboSyncData(
id=org_uid, stage_index=0)
self.stage = STAGES[self.sync_data.stage_index](org_uid)
logging.info("running stage {}: {}".format(self.sync_data.stage_index,
type(self.stage).__name__))
def reset_endpoints_task(org_uid):
"""
Processes org reset task from the task queue (clears endpoint state to cause the next sync to fetch all the data,
and creates a task on the update queue to kick of the sync cycle for the org).
"""
org = Org.get_by_id(org_uid)
if (org.changeset_started_at
and not org.changeset_completed_at) or org.update_cycle_active:
logging.info("org syncing at the moment, will try again later")
return '', 423
endpoint_indexes = request.form.getlist('endpoint_index')
logging.info("resetting markers for org {} and endpoints {}".format(
org_uid, endpoint_indexes))
# TODO: this is a hack, this should be delegated to a qbo class, instantiated via a factory from the org provider
sync_data = QboSyncData.get_by_id(org_uid)
if not sync_data:
logging.warning("could not find sync data")
return '', 204
for endpoint_index in [int(_index) for _index in endpoint_indexes]:
sync_data.markers[endpoint_index] = START_OF_TIME
sync_data.put()
sync_utils.init_update(org_uid)
return '', 204
def __init__(self, org_uid):
"""
Initialises the class.
Args:
org_uid(str): org identifier
"""
self.org_uid = org_uid
self.org = Org.get_by_id(org_uid)
self.entity_id = self.org.entity_id
self.api_url = "{}company/{}/query?minorversion={}".format(
BASE_API_URI, self.entity_id, API_MINOR_VERSION)
self.sync_data = QboSyncData.get_by_id(org_uid) or QboSyncData(
id=org_uid, endpoint_index=0, start_position=1)
if self.sync_data.endpoint_index == 0:
logging.info(
"this is a start of a new changeset, starting to ingest all endpoints"
)
def __init__(self, org_uid):
"""
Initialises the class.
Args:
org_uid(str): org identifier
"""
self.org_uid = org_uid
self.sync_data = QboSyncData.get_by_id(org_uid)
self.org = Org.get_by_id(org_uid)
self.entity_id = self.org.entity_id
def test_account_balance_deduplication(self, get_mock):
"""
Verifies that an account balance is fetched, but hasn't changed since the last fetch, doesn't get saved to Item.
Args:
get_mock(Mock): mock of the api get function
"""
with open('tests/resources/trial_balance_report.json'
) as report_contents:
sample_report = json.load(report_contents)
get_mock.return_value = sample_report
# setup org and sync state
self.create_org(status=CONNECTED, changeset=0)
QboSyncData(id='test', account_balance_marker='2010-01-01').put()
# run sync
stage = AccountBalanceReportStage('test')
stage.next(payload={})
# there should be 30 Items saved
self.assertEqual(self.count_items(), 30)
# setup next changeset and reset the marker to the same day (simulate the same day fetching again, but on the
# next day update cycle)
org = Org.get_by_id('test')
org.changeset = 1
org.put()
QboSyncData(id='test', account_balance_marker='2010-01-01').put()
# patch response mock so that one balance has changed
sample_report['Rows']['Row'][0]['ColData'][1]['value'] = 100
get_mock.return_value = sample_report
# run sync
stage = AccountBalanceReportStage('test')
stage.next(payload={})
# there should be 31 Items saved (one new balance for changeset 1)
self.assertEqual(self.count_items(), 31)
def test_journal_sync(self, get_mock):
"""
Verifies that journals can be synthesised from the General Ledger report.
Args:
get_mock(Mock): mock of the api get function
"""
with open('tests/resources/general_ledger_report.json'
) as report_contents:
get_mock.return_value = json.load(report_contents)
self.create_org(status=CONNECTED)
QboSyncData(id='test', journal_dates=['2000-01-01']).put()
stage = JournalReportStage('test')
stage.next(payload={})
# 7 journals should be saved
self.assertEqual(self.count_items(), 7)
# the journal date ingested should be deleted
self.assertEqual(QboSyncData.get_by_id('test').journal_dates, [])
def test_account_balance_step(self, get_mock):
"""
Verifies account balance sync state management (that balances are fetched backwards).
Args:
get_mock(Mock): mock of the api get function
"""
with open('tests/resources/trial_balance_report.json'
) as report_contents:
get_mock.return_value = json.load(report_contents)
# setup org and sync state
self.create_org(status=CONNECTED)
QboSyncData(id='test', account_balance_marker='2000-01-01').put()
# run sync
stage = AccountBalanceReportStage('test')
stage.next(payload={})
# marker should be decremented for the next sync
self.assertEqual(
QboSyncData.get_by_id('test').account_balance_marker, '1999-12-31')
def test_account_balance_completion(self):
"""
Verifies that once no balances are found in the report stage is reported as complete.
"""
# setup org and sync state
self.create_org(status=CONNECTED, changeset=1)
QboSyncData(id='test', account_balance_marker='2000-01-01').put()
# run sync (report is empty)
stage = AccountBalanceReportStage('test')
complete, _ = stage.next(payload={})
# stage should be completed
self.assertTrue(complete)
def test_account_balance_parsing(self, get_mock):
"""
Verifies that account balances can be synthesised from the Trial Balance report.
Args:
get_mock(Mock): mock of the api get function
"""
with open('tests/resources/trial_balance_report.json'
) as report_contents:
get_mock.return_value = json.load(report_contents)
# setup org and sync state
self.create_org(status=CONNECTED, changeset=0)
QboSyncData(id='test', account_balance_marker='2000-01-01').put()
# run sync
stage = AccountBalanceReportStage('test')
stage.next(payload={})
# there should be 15 account balances saved, but each twice (once for -1 changeset)
self.assertEqual(self.count_items(), 30)
def test_country_extraction(self, get_mock):
"""
Verifies that the file country is saved from the company info endpoint.
Args:
get_mock(Mock): mock of the api get function
"""
get_mock.return_value = self.get_mock_api_response(
COMPANY_INFO_ENDPOINT_NAME)
self.create_org(status=CONNECTED)
# set sync state so that the next pull will be company info
QboSyncData(id='test',
stage_index=LIST_API_STAGE,
endpoint_index=COMPANY_INFO_ENDPOINT_INDEX).put()
# run the sync
stage = ListApiStage('test')
stage.next(payload={})
# the country should be saved
self.assertEqual(Org.get_by_id('test').country, 'AU')
