def do_test(self, conn_id):
"""Test we get a lot of data back based on the start date configured in base"""
# Select all streams and all fields within streams
found_catalogs = menagerie.get_catalogs(conn_id)
incremental_streams = {
key
for key, value in self.expected_replication_method().items()
if value == self.INCREMENTAL
}
# IF THERE ARE STREAMS THAT SHOULD NOT BE TESTED
# REPLACE THE EMPTY SET BELOW WITH THOSE STREAMS
untested_streams = self.child_streams().union(
{'disputes', 'events', 'transfers', 'payout_transactions'})
our_catalogs = get_catalogs(
conn_id, incremental_streams.difference(untested_streams))
self.select_all_streams_and_fields(conn_id,
our_catalogs,
select_all_fields=True)
# Create a record for each stream under test prior to the first sync
new_objects = {
stream: create_object(stream)
for stream in incremental_streams.difference(untested_streams)
}
# Run a sync job using orchestrator
first_sync_record_count = self.run_sync(conn_id)
first_total_records = reduce(lambda a, b: a + b,
first_sync_record_count.values())
# Count actual rows synced
first_sync_records = runner.get_records_from_target_output()
# set the start date for a new connection based off bookmarks largest value
first_max_bookmarks = self.max_bookmarks_by_stream(first_sync_records)
bookmark_list = [
next(iter(book.values()))
for stream, book in first_max_bookmarks.items()
]
bookmark_dates = []
for bookmark in bookmark_list:
try:
bookmark_dates.append(parse(bookmark))
except (ValueError, OverflowError, TypeError):
pass
if not bookmark_dates:
# THERE WERE NO BOOKMARKS THAT ARE DATES.
# REMOVE CODE TO FIND A START DATE AND ENTER ONE MANUALLY
raise ValueError
# largest_bookmark = reduce(lambda a, b: a if a > b else b, bookmark_dates)
# self.start_date = self.local_to_utc(largest_bookmark).strftime(self.START_DATE_FORMAT)
self.start_date = dt.strftime(dt.today() - timedelta(days=1),
self.START_DATE_FORMAT)
# create a new connection with the new start_date
conn_id = self.create_connection(original_properties=False)
# Select all streams and all fields within streams
found_catalogs = menagerie.get_catalogs(conn_id)
our_catalogs = [
catalog for catalog in found_catalogs
if catalog.get('tap_stream_id') in incremental_streams.difference(
untested_streams)
]
self.select_all_streams_and_fields(conn_id,
our_catalogs,
select_all_fields=True)
# TODO remove the updates, this is unnecessary. Verify with Harvest
# Update a record for each stream under test prior to the 2nd sync
first_sync_created, _ = self.split_records_into_created_and_updated(
first_sync_records)
updated = {} # holds id for updated objects in each stream
for stream in new_objects:
# There needs to be some test data for each stream, otherwise this will break
record = first_sync_created[stream]["messages"][0]["data"]
update_object(stream, record["id"])
updated[stream] = record["id"]
# Run a sync job using orchestrator
second_sync_record_count = self.run_sync(conn_id)
# tap-stripe uses events for updates, so these need filtered to validate bookmark
second_sync_records = runner.get_records_from_target_output()
second_sync_created, second_sync_updated = self.split_records_into_created_and_updated(
second_sync_records)
second_total_records = reduce(lambda a, b: a + b,
second_sync_record_count.values(), 0)
# Only examine bookmarks for "created" objects, not updates
second_min_bookmarks = self.min_bookmarks_by_stream(
second_sync_created)
# verify that at least one record synced and less records synced than the 1st connection
self.assertGreater(second_total_records, 0)
self.assertLess(first_total_records, second_total_records)
# validate that all newly created records are greater than the start_date
for stream in incremental_streams.difference(untested_streams):
with self.subTest(stream=stream):
# verify that each stream has less records in the first sync than the second
self.assertGreater(
second_sync_record_count.get(stream, 0),
first_sync_record_count.get(stream, 0),
msg="first had more records, start_date usage not verified"
)
# verify all data from 2nd sync >= start_date
target_mark = second_min_bookmarks.get(stream, {"mark": None})
target_value = next(iter(
target_mark.values())) # there should be only one
if target_value:
# it's okay if there isn't target data for a stream
try:
target_value = self.local_to_utc(parse(target_value))
expected_value = self.local_to_utc(
parse(self.start_date))
# verify that the minimum bookmark sent to the target for the second sync
# is greater than or equal to the start date
self.assertGreaterEqual(target_value, expected_value)
except (OverflowError, ValueError, TypeError):
print("bookmarks cannot be converted to dates, "
"can't test start_date for {}".format(stream))
if stream in updated:
delete_object(stream, updated[stream])
def do_test(self, conn_id):
"""
Verify that the sync only sent records to the target for selected streams
Update metadata[test] with a random number for each stream with event updates
Verify that the second sync includes at least one update for each stream
Verify that the second sync includes less records than the first sync
Verify that the updated metadata was picked up on the second sync
PREREQUISITE
For EACH stream that gets updates through events stream, there's at least 1 row
of data
"""
event_update_streams = {
# "balance_transactions" # Cannot be directly updated
"charges",
"coupons",
"customers",
# "disputes", # Cannot create directly with api
"invoice_items",
# "invoice_line_items", # Can't be updated via api
"invoices",
# "payout_transactions", # See bug in create_test
"payouts",
"plans",
"products",
# "subscription_items", # BUG https://stitchdata.atlassian.net/browse/SUP-1214
"subscriptions",
# "transfers", # Cannot be updated directly via api
}
our_catalogs = get_catalogs(conn_id, event_update_streams)
self.select_all_streams_and_fields(conn_id,
our_catalogs,
select_all_fields=True)
# Ensure each stream under test has data to start
new_objects = {
stream: create_object(stream)
for stream in event_update_streams
}
# Some streams will be updated implicitly
streams_to_update = event_update_streams.difference({
"invoice_line_items",
"subscription_items",
})
# Run a sync job using orchestrator
first_sync_record_count = self.run_sync(conn_id)
# verify that the sync only sent records to the target for selected streams (catalogs)
self.assertEqual(set(first_sync_record_count.keys()),
event_update_streams)
# Get the set of records from a first sync
first_sync_records = runner.get_records_from_target_output()
first_sync_created, _ = self.split_records_into_created_and_updated(
first_sync_records)
updated = {} # holds id for updated objects in each stream
for stream in streams_to_update:
# There needs to be some test data for each stream, otherwise this will break
self.assertGreater(len(first_sync_created[stream]["messages"]),
0,
msg='We did not get any new records from '
'the first sync for {}'.format(stream))
record = first_sync_created[stream]["messages"][0]["data"]
# We need to make sure the data actually changes, otherwise no event update
# will get created
update_object(stream, record["id"])
updated[stream] = record["id"]
# Run a second sync job using orchestrator
second_sync_record_count = self.run_sync(conn_id)
# Get the set of records from a second sync
second_sync_records = runner.get_records_from_target_output()
_, second_sync_updated = self.split_records_into_created_and_updated(
second_sync_records)
# # THIS MAKES AN ASSUMPTION THAT CHILD STREAMS DO NOT NEED TESTING.
# # ADJUST IF NECESSARY
for stream in event_update_streams.difference(self.child_streams()):
with self.subTest(stream=stream):
# verify that there is more than 1 record of data - setup necessary
self.assertGreater(
first_sync_record_count.get(stream, 0),
1,
msg="Data isn't set up to be able to test event updates",
)
# verify that you get at least one updated record on the second sync
self.assertGreaterEqual(
len(
second_sync_updated.get(stream,
{}).get("messages", [])),
1,
msg="second syc didn't have updates",
)
# verify that you get less data the 2nd time around since only updates
# should be picked up
self.assertLess(
second_sync_record_count.get(stream, 0),
first_sync_record_count.get(stream, 0),
msg="second syc had the same or more records",
)
# verify all the updated records in the 2nd sync are different from
# the first run
first_data = next(
record["data"] for record in first_sync_created.get(
stream, {}).get("messages", [])
if record.get("data", {}).get("id") == updated[stream])
second_data = next(
record["data"] for record in second_sync_updated.get(
stream, {}).get("messages", [])
if record.get("data", {}).get("id") == updated[stream])
# verify the updated timestamp is greater in the second sync
self.assertGreater(
second_data["updated"],
first_data["updated"],
"updated timestamp for second sync is not greater than first sync",
)
# verify the metadata[test] value actually changed
self.assertNotEqual(
second_data["metadata"].get("test_value", 0),
first_data["metadata"].get("test_value", 0),
"the test metadata should be different",
)
if stream in new_objects:
delete_object(stream, new_objects[stream]["id"])
def test_run(self):
"""
Verify that for each stream you can get multiple pages of data
and that when all fields are selected more than the automatic fields are replicated.
PREREQUISITE
For EACH stream add enough data that you surpass the limit of a single
fetch of data. For instance if you have a limit of 250 records ensure
that 251 (or more) records have been posted for that stream.
"""
conn_id = connections.ensure_connection(self)
incremental_streams = {key for key, value in self.expected_replication_method().items()
if value == self.INCREMENTAL}
untested_streams = self.child_streams().union({
'balance_transactions',
# 'charges',
# 'coupons',
# 'customers',
'disputes',
# 'invoice_items',
'invoice_line_items',
# 'invoices',
'payout_transactions',
# 'payouts',
# 'plans',
# 'products',
'subscription_items',
# 'subscriptions',
'transfers',
})
tested_streams = incremental_streams.difference(untested_streams)
# Select all streams and all fields within streams
found_catalogs = self.run_and_verify_check_mode(conn_id)
our_catalogs = get_catalogs(conn_id, tested_streams)
self.select_all_streams_and_fields(conn_id, our_catalogs, select_all_fields=True)
# Ensure tested streams have a record count which exceeds the API LIMIT
logging.info("Checking record counts for tested streams...")
streams_to_create = {}
for stream in tested_streams:
records = list_all_object(stream)
record_count = len(records)
streams_to_create[stream] = record_count
logging.info(" Stream {} has {} records created today".format(stream, record_count))
logging.info("Creating records for tested streams...")
new_objects = {stream: [] for stream in streams_to_create}
for stream in streams_to_create:
if stream != "events" and streams_to_create[stream] <= self.API_LIMIT:
while streams_to_create[stream] <= self.API_LIMIT:
logging.info("Creating a record for {} | {} records created today ".format(stream,
streams_to_create[stream]))
new_objects[stream].append(create_object(stream))
streams_to_create[stream] += 1
records = list_all_object(stream)
self.assertEqual(100, len(records))
logging.info(" Stream {} has at least {} records created today".format(stream, len(records) + 1))
# Run a sync job using orchestrator
record_count_by_stream = self.run_and_verify_sync(conn_id)
actual_fields_by_stream = runner.examine_target_output_for_fields()
for stream in incremental_streams.difference(untested_streams):
with self.subTest(stream=stream):
# verify that we can paginate with all fields selected
self.assertGreater(
record_count_by_stream.get(stream, -1),
self.expected_metadata().get(stream, {}).get(self.API_LIMIT, 0),
msg="The number of records is not over the stream max limit")
# verify that the automatic fields are sent to the target
actual = actual_fields_by_stream.get(stream) or set()
expected = self.expected_automatic_fields().get(stream, set())
self.assertTrue(actual.issuperset(expected),
msg="The fields sent to the target don't include all automatic fields. "
"Expected: {}, Actual: {}". format(expected, actual)
)
# verify we have more fields sent to the target than just automatic fields
# SKIP THIS ASSERTION IF ALL FIELDS ARE INTENTIONALLY AUTOMATIC FOR THIS STREAM
actual = actual_fields_by_stream.get(stream) or set()
expected = self.expected_automatic_fields().get(stream, set())
self.assertTrue(actual.symmetric_difference(expected),
msg="The fields sent to the target don't include any non-automatic fields"
)
if stream != "events":
actual = actual_fields_by_stream.get(stream, set())
expected = set(new_objects[stream][0].keys())
def do_test(self, conn_id):
"""
Verify that the sync only sent records to the target for selected streams
Create a new object for each stream
Verify that the second sync includes at least one create for each stream
Verify that the created record was picked up on the second sync
"""
streams_to_create = {
"balance_transactions", # should be created implicity with a create in the payouts or charges streams
"charges",
"coupons",
"customers",
"invoice_items",
"invoice_line_items", # this is created implicity by invoices, it just creates another invoice TODO get this outa here
"invoices", # this will create an invoice_item
"payouts",
"plans",
"products",
"subscription_items",
"subscriptions", # this will create a new plan and payment method
}
missing_streams_to_create = {
"disputes", # can be created by simulating a dispute transaction with a specific card number
# no way to create directly, see: https://stripe.com/docs/testing#disputes
"payout_transactions", # BUG (https://stitchdata.atlassian.net/browse/SUP-1294)
# depends on payouts and transactions
"transfers",
# needs an account that we can transfer to, not sure
# how to set up a test account we can use to create a transfer
}
our_catalogs = get_catalogs(conn_id, streams_to_create)
self.select_all_streams_and_fields(conn_id,
our_catalogs,
select_all_fields=True)
# Run a sync job using orchestrator
first_sync_record_count = self.run_sync(conn_id)
# verify that the sync only sent records to the target for selected streams (catalogs)
self.assertEqual(set(first_sync_record_count.keys()),
streams_to_create)
# Get the set of records from a first sync
first_sync_records = runner.get_records_from_target_output()
first_sync_created, _ = self.split_records_into_created_and_updated(
first_sync_records)
new_objects = {
stream: create_object(stream)
for stream in streams_to_create.difference(
{"balance_transactions"})
}
# Run a second sync job using orchestrator
second_sync_record_count = self.run_sync(conn_id)
# Get the set of records from a second sync
second_sync_records = runner.get_records_from_target_output()
second_sync_created, _ = self.split_records_into_created_and_updated(
second_sync_records)
# # THIS MAKES AN ASSUMPTION THAT CHILD STREAMS DO NOT NEED TESTING.
# # ADJUST IF NECESSARY
for stream in streams_to_create.difference(self.child_streams()):
with self.subTest(stream=stream):
second_sync_created_objects = second_sync_created.get(
stream, {}).get("messages", [])
# verify that you get at least one new record on the second sync
self.assertGreaterEqual(
len(second_sync_created_objects),
1,
msg="second sync didn't have created objects",
)
if stream == "balance_transactions":
sources = [
record.get("data", {}).get("source")
for record in second_sync_created_objects
]
self.assertTrue(new_objects['payouts']['id'] in sources)
self.assertTrue(new_objects['charges']['id'] in sources)
continue
# verify the new object is in the list of created objects
# from the second sync
self.assertTrue(
any(new_objects[stream]["id"] == record.get("data",
{}).get("id")
for record in second_sync_created_objects))
if stream in streams_to_create:
delete_object(stream, new_objects[stream]["id"])