def create_items(org_uid, provider, changeset, endpoint, item_id, data):
"""
Creates items containing raw endpoint response ready to be saved into Item data kind (Item acts as raw endpoint
cache and is heavily used by the normalisation part of the data ingestion pipeline, dependent items needed for
normalisation are resolved from here as opposed to going back to the providers API).
Two instances of Item are created: one with the actual changeset during which the items was ingested, and one with
changeset of -1. This allows for the latest version of an item to be easily retrieved (an invoice might be ingested
as part of one changeset, but if it gets updated it will get ingested again as part of another changeset).
Args:
org_uid(str): org identifier
provider(str): data provider (eg. 'qbo', 'xerov2')
changeset(int): update cycle identifier
endpoint(str): endpoint which the item came from (eg. 'Invoice', 'Payment')
item_id(str): id of the item as is in the source system (provider)
data(object): item payload as provided by the provider's api (eg. output of the invoice endpoint)
Returns:
list(ndb.Model): a list of Item instances ready to be saved
"""
datastore_item_id = "{}_{}_{}_{}".format(org_uid, changeset, endpoint,
item_id)
changeset_item = Item(id=datastore_item_id,
org_uid=org_uid,
provider=provider,
changeset=changeset,
endpoint=endpoint,
item_id=item_id,
data=data)
latest_version_changeset = -1
datastore_item_id = "{}_{}_{}_{}".format(org_uid, latest_version_changeset,
endpoint, item_id)
latest_version = Item(id=datastore_item_id,
org_uid=org_uid,
provider=provider,
changeset=latest_version_changeset,
endpoint=endpoint,
item_id=item_id,
data=data)
return [changeset_item, latest_version]
def test_missing_found_in_item(self):
"""
Verifies that a missing item can be resolved from the Item.
"""
self.create_org(status=CONNECTED)
stage = MissingItemsStage('test')
MissingItem(org_uid='test',
missing_items=[{
'type': 'Account',
'id': '1'
}]).put()
Item(org_uid='test',
changeset=-1,
endpoint='Account',
item_id='1',
data={
'Id': '1'
}).put()
stage.next(payload={})
# the missing item should be deleted and resolved item should added Item ready for next publish
self.assertEqual(self.count_missing_items(), 0)
self.assertEqual(self.count_items(), 2)
def test_missing_without_payload_id(self):
"""
Verifies that a missing item without the Id field in the data can be processed.
"""
item_id = '1000_2010-01-01'
Item(org_uid='test',
changeset=-1,
endpoint='AccountBalance',
item_id=item_id,
data={
'Balance': '10'
}).put()
MissingItem(org_uid='test',
missing_items=[{
'type': 'AccountBalance',
'id': item_id
}]).put()
self.create_org(status=CONNECTED)
stage = MissingItemsStage('test')
stage.next(payload={})
# the missing item should be deleted and resolved item should added Item ready for next publish
self.assertEqual(self.count_missing_items(), 0)
self.assertEqual(self.count_items(), 2)