def __init__(self, config):
super().__init__()
self.features = nn.Sequential(*(utils.create_object(
**layer) for layer in config['features']['layers']))
self.reshape = config['reshape']
self.classifier = nn.Sequential(*(utils.create_object(
**layer) for layer in config['classifier']['layers']))
self.loss = utils.create_object(**config['loss'])
def setUpClass(cls):
logging.info("Start Setup")
# Create data prior to first sync
cls.streams_to_test = {
"customers",
"charges",
"coupons",
"invoice_items",
"invoice_line_items",
"invoices",
"payouts",
"plans",
"products",
"subscription_items",
"subscriptions",
}
cls.expected_objects = {stream: [] for stream in cls.streams_to_test}
cls.existing_objects = {stream: [] for stream in cls.streams_to_test}
cls.new_objects = {stream: [] for stream in cls.streams_to_test}
for stream in cls.streams_to_test:
# create new records
stripe_obj = create_object(stream)
cls.new_objects[stream] = [stripe_obj_to_dict(stripe_obj)]
cls.expected_objects[stream] = cls.new_objects[stream]
def manage_addOAIToken(self, id=None, **kwargs):
""" """
if id is None:
id = u"%sr%s" % (int(kwargs['token_args'].get('cursor', 0)),
randint(10000, 99999))
ob = create_object(self, OAIToken, id)
if not kwargs['token_args'].has_key('id'):
kwargs['token_args']['id'] = id
process_form(ob, IOAIToken, kwargs)
ob.expiration = kwargs['token_args']['expirationDate']
return ob
def manage_addOAIServer(self, id='', REQUEST=None, **kwargs):
""" """
if REQUEST is not None:
form_data = dict(REQUEST.form)
else:
form_data = dict(kwargs)
try:
ob = create_object(self, OAIServer, id)
process_form(ob, IOAIServer, form_data)
ob.initialize()
except Exception, e:
if REQUEST is not None:
return REQUEST.RESPONSE.redirect(self.absolute_url() +
'/manage_main?manage_tabs_message=%s' % quote(str(e)))
else:
raise Exception(e)
def manage_addOAIHarvester(self, id='', REQUEST=None, **kwargs):
""" """
if REQUEST is not None:
form_data = dict(REQUEST.form)
else:
form_data = dict(kwargs)
try:
if id == '':
id = processId(form_data.get('title', 'zharvester')).lower()
ob = create_object(self, OAIHarvester, id)
process_form(ob, IOAIHarvester, form_data)
ob.update_sets()
except Exception, e:
transaction.abort()
if REQUEST is not None:
return REQUEST.RESPONSE.redirect(self.absolute_url() +
'/manage_main?manage_tabs_message=%s' % quote(str(e)))
else:
raise
def __init__(self):
self.queue = Queue()
with open('config.json') as config_file:
self.config = json.load(config_file)
self.model = NeuralNet(self.config)
with open('params.json') as f:
self.load_params(json.load(f))
self.params = self.config['optimizer']['parameters']
self.params['params'] = self.model.parameters()
self.parameter_lengths = list(
map(lambda x: x.data.nelement(), self.model.parameters()))
self.optimizer = utils.create_object(**self.config['optimizer'])
self.accuracy = None
self.lock = Lock()
self.changed = True
self.params_updated = False
self.loader = utils.get_data()
self.convert_params()
self.compute_accuracy()
Thread(target=self.update_model, daemon=True).start()
def test_run(self):
"""
Verify that for each stream you can get multiple pages of data
and that when all fields are selected more than the automatic fields are replicated.
PREREQUISITE
For EACH stream add enough data that you surpass the limit of a single
fetch of data. For instance if you have a limit of 250 records ensure
that 251 (or more) records have been posted for that stream.
"""
conn_id = connections.ensure_connection(self)
incremental_streams = {key for key, value in self.expected_replication_method().items()
if value == self.INCREMENTAL}
untested_streams = self.child_streams().union({
'balance_transactions',
# 'charges',
# 'coupons',
# 'customers',
'disputes',
# 'invoice_items',
'invoice_line_items',
# 'invoices',
'payout_transactions',
# 'payouts',
# 'plans',
# 'products',
'subscription_items',
# 'subscriptions',
'transfers',
})
tested_streams = incremental_streams.difference(untested_streams)
# Select all streams and all fields within streams
found_catalogs = self.run_and_verify_check_mode(conn_id)
our_catalogs = get_catalogs(conn_id, tested_streams)
self.select_all_streams_and_fields(conn_id, our_catalogs, select_all_fields=True)
# Ensure tested streams have a record count which exceeds the API LIMIT
logging.info("Checking record counts for tested streams...")
streams_to_create = {}
for stream in tested_streams:
records = list_all_object(stream)
record_count = len(records)
streams_to_create[stream] = record_count
logging.info(" Stream {} has {} records created today".format(stream, record_count))
logging.info("Creating records for tested streams...")
new_objects = {stream: [] for stream in streams_to_create}
for stream in streams_to_create:
if stream != "events" and streams_to_create[stream] <= self.API_LIMIT:
while streams_to_create[stream] <= self.API_LIMIT:
logging.info("Creating a record for {} | {} records created today ".format(stream,
streams_to_create[stream]))
new_objects[stream].append(create_object(stream))
streams_to_create[stream] += 1
records = list_all_object(stream)
self.assertEqual(100, len(records))
logging.info(" Stream {} has at least {} records created today".format(stream, len(records) + 1))
# Run a sync job using orchestrator
record_count_by_stream = self.run_and_verify_sync(conn_id)
actual_fields_by_stream = runner.examine_target_output_for_fields()
for stream in incremental_streams.difference(untested_streams):
with self.subTest(stream=stream):
# verify that we can paginate with all fields selected
self.assertGreater(
record_count_by_stream.get(stream, -1),
self.expected_metadata().get(stream, {}).get(self.API_LIMIT, 0),
msg="The number of records is not over the stream max limit")
# verify that the automatic fields are sent to the target
actual = actual_fields_by_stream.get(stream) or set()
expected = self.expected_automatic_fields().get(stream, set())
self.assertTrue(actual.issuperset(expected),
msg="The fields sent to the target don't include all automatic fields. "
"Expected: {}, Actual: {}". format(expected, actual)
)
# verify we have more fields sent to the target than just automatic fields
# SKIP THIS ASSERTION IF ALL FIELDS ARE INTENTIONALLY AUTOMATIC FOR THIS STREAM
actual = actual_fields_by_stream.get(stream) or set()
expected = self.expected_automatic_fields().get(stream, set())
self.assertTrue(actual.symmetric_difference(expected),
msg="The fields sent to the target don't include any non-automatic fields"
)
if stream != "events":
actual = actual_fields_by_stream.get(stream, set())
expected = set(new_objects[stream][0].keys())
def test_run(self):
"""
Verify that for each stream you can do a sync which records bookmarks.
That the bookmark is the maximum value sent to the target for the replication key.
That a second sync respects the bookmark
All data of the second sync is >= the bookmark from the first sync
The number of records in the 2nd sync is less then the first (This assumes that
new data added to the stream is done at a rate slow enough that you haven't
doubled the amount of data from the start date to the first sync between
the first sync and second sync run in this test)
Verify that only data for incremental streams is sent to the target
PREREQUISITE
For EACH stream that is incrementally replicated there are multiple rows of data with
different values for the replication key
"""
conn_id = self.create_connection()
expected_records = {stream: [] for stream in self.streams_to_create}
# Create 3 records for each stream total
for _ in range(2):
for stream in self.streams_to_create:
self.new_objects[stream].append(create_object(stream))
# Turn on tracking of hidden object creates in util_stripe
# so that we can properly assert on 2nd sync records and expectations
activate_tracking()
# Create the 3rd record now that we are recording all created objects
for stream in self.streams_to_create:
self.new_objects[stream].append(create_object(stream))
expected_records[stream].append(
{"id": self.new_objects[stream][-1]['id']})
# Select all streams and no fields within streams
found_catalogs = menagerie.get_catalogs(conn_id)
incremental_streams = {
key
for key, value in self.expected_replication_method().items()
if value == self.INCREMENTAL
}
# IF THERE ARE STREAMS THAT SHOULD NOT BE TESTED
# REPLACE THE EMPTY SET BELOW WITH THOSE STREAMS
untested_streams = self.child_streams().union({
'transfers',
'payout_transactions', # BUG see create test
'balance_transactions', # join stream, can't be updated
'disputes',
})
our_catalogs = [
catalog for catalog in found_catalogs
if catalog.get('tap_stream_id') in incremental_streams.difference(
untested_streams)
]
self.select_all_streams_and_fields(conn_id,
our_catalogs,
select_all_fields=True)
# Run a sync job using orchestrator and track the sync's time window
first_sync_start = self.local_to_utc(dt.utcnow())
first_sync_record_count = self.run_sync(conn_id)
first_sync_end = self.local_to_utc(dt.utcnow())
# verify that the sync only sent records to the target for selected streams (catalogs)
self.assertEqual(set(first_sync_record_count.keys()),
incremental_streams.difference(untested_streams))
first_sync_state = menagerie.get_state(conn_id)
# Get data about actual rows synced
first_sync_records = runner.get_records_from_target_output()
first_max_bookmarks = self.max_bookmarks_by_stream(first_sync_records)
first_max_events = self.max_events_bookmarks_by_stream(
first_sync_records)
first_min_bookmarks = self.min_bookmarks_by_stream(first_sync_records)
# Update one record from each stream prior to 2nd sync
first_sync_created, _ = self.split_records_into_created_and_updated(
first_sync_records)
updated_objects = {stream: []
for stream in self.streams_to_create
} # TODO Delete if never used
for stream in self.streams_to_create:
# There needs to be some test data for each stream, otherwise this will break
record = expected_records[stream][0]
# record = first_sync_created[stream]["messages"][0]["data"]
updated_objects[stream].append(update_object(stream, record["id"]))
expected_records[stream].append(
{"id": updated_objects[stream][-1]['id']})
# Ensure different times between udpates and inserts
sleep(2)
# Insert (create) one record for each stream prior to 2nd sync
for stream in self.streams_to_create:
self.new_objects[stream].append(create_object(stream))
expected_records[stream].append(
{"id": self.new_objects[stream][-1]['id']})
# Run a second sync job using orchestrator
second_sync_start = self.local_to_utc(dt.utcnow())
second_sync_record_count = self.run_sync(conn_id)
second_sync_end = self.local_to_utc(dt.utcnow())
second_sync_state = menagerie.get_state(conn_id)
# Get data about rows synced
second_sync_records = runner.get_records_from_target_output()
second_sync_created, second_sync_updated = self.split_records_into_created_and_updated(
second_sync_records)
second_min_bookmarks = self.min_bookmarks_by_stream(
second_sync_created)
second_max_bookmarks = self.max_bookmarks_by_stream(
second_sync_records)
second_max_events = self.max_events_bookmarks_by_stream(
second_sync_records)
self.assertTrue(second_max_events)
# Define bookmark keys
events_bookmark_key = "updates_created" # secondary bookmark from the events stream
target_events_bookmark_key = "updated" # the bookmark key for events
# Adjust expectations to account for invoices created by the invoice_line_items calls
for e in expected_records['invoice_line_items']:
expected_records['invoices'].append(e)
# THIS MAKES AN ASSUMPTION THAT CHILD STREAMS DO NOT HAVE BOOKMARKS.
# ADJUST IF NECESSARY
for stream in incremental_streams.difference(untested_streams):
with self.subTest(stream=stream):
# Get bookmark values from state and target data.
# Recall that there are two bookmarks for every object stream: an events-based
# bookmark for updates, and a standard repliction key-based bookmark for creates.
stream_bookmark_key = self.expected_replication_keys().get(
stream, set())
assert len(
stream_bookmark_key
) == 1 # There shouldn't be a compound replication key
stream_bookmark_key = stream_bookmark_key.pop()
state_value = first_sync_state.get("bookmarks", {}).get(
stream + "", {}).get(stream_bookmark_key)
target_value = first_max_bookmarks.get(
stream + "", {}).get(stream_bookmark_key)
target_min_value = first_min_bookmarks.get(
stream + "", {}).get(stream_bookmark_key)
# the events stream should get an empty set as events_events does not exist
state_value_events = first_sync_state.get("bookmarks", {}).get(
stream + "_events", {}).get(events_bookmark_key)
target_events_value = first_max_events.get(
stream + "_events", {}).get(target_events_bookmark_key)
try:
# attempt to parse the bookmark as a date
state_value = self.parse_bookmark_to_date(state_value)
state_value_events = self.parse_bookmark_to_date(
state_value_events)
target_value = self.parse_bookmark_to_date(target_value)
target_events_value = self.parse_bookmark_to_date(
target_events_value)
target_min_value = self.parse_bookmark_to_date(
target_min_value)
except (OverflowError, ValueError, TypeError):
print(
"bookmarks cannot be converted to dates, comparing values directly"
)
##########################################################################
### 1st Sync Tests
##########################################################################
if stream != 'payout_transactions':
# verify that there is data with different bookmark values - setup necessary
self.assertGreaterEqual(
target_value,
target_min_value,
msg="Data isn't set up to be able to test bookmarks")
# TODO run test with our time = current_time + 1 hour and see if the events stream is able to cover for any potential
# issues with our time window discrepancy capturing updates/creates
# verify that 'created' state is saved for the first sync in the appropriate
# time window, between (inclusive) the target value (max bookmark) for this stream and
# the first sync end time
self.assertGreaterEqual(
state_value,
target_value,
msg="The bookmark value isn't correct based on target data"
)
self.assertLessEqual(
state_value,
first_sync_end,
msg="The bookmark value isn't correct based on "
"the start sync")
if state_value_events != set() and stream != 'events':
if stream not in {
'invoices', 'customers'
}: # BUG https://stitchdata.atlassian.net/browse/SUP-1320
# verify that 'updates' state matches the target (max bookmark)
self.assertEqual(
state_value_events,
target_events_value,
msg="The bookmark value isn't correct "
"based on target data")
# NOTE: This assertion is no longer valid. Some streams will create records for other streams
# verify the last record has the max bookmark in the target (this should never fail)
# last_created = self.parse_bookmark_to_date(self.new_objects[stream][2]['created'])
# self.assertEqual(target_value, last_created,
# msg="The last created record for the first sync should "
# "be sthe max bookmark in the target and is not.")
else: # We need to ensure no assertions are missed for object streams
self.assertEqual(
stream,
'events',
msg=
"An object stream is missing it's object_events bookmark"
)
##########################################################################
### 2nd Sync Tests
##########################################################################
prev_target_value = target_value
prev_target_events_value = target_events_value
target_value = second_max_bookmarks.get(
stream + "", {}).get(stream_bookmark_key)
target_min_value = second_min_bookmarks.get(
stream + "", {}).get(stream_bookmark_key)
final_state_value = second_sync_state.get("bookmarks", {}).get(
stream + "", {}).get(stream_bookmark_key)
final_state_value_events = second_sync_state.get(
"bookmarks", {}).get(stream + "_events",
{}).get(events_bookmark_key)
target_events_value = second_max_events.get(
stream + "_events", {}).get(target_events_bookmark_key)
try:
final_state_value = self.parse_bookmark_to_date(
final_state_value)
final_state_value_events = self.parse_bookmark_to_date(
final_state_value_events)
target_value = self.parse_bookmark_to_date(target_value)
target_events_value = self.parse_bookmark_to_date(
target_events_value)
target_min_value = self.parse_bookmark_to_date(
target_min_value)
except (OverflowError, ValueError, TypeError):
print(
"bookmarks cannot be converted to dates, comparing values directly"
)
# verify that the 2nd sync gets more than zero records
self.assertGreater(second_sync_record_count.get(stream, 0),
0,
msg="second syc didn't have any records")
# verify that you get less data the 2nd time around
self.assertGreater(
first_sync_record_count.get(stream, 0),
second_sync_record_count.get(stream, 0),
msg=
"second syc didn't have less records, bookmark usage not verified"
)
# verify 'created' state is at or after the target (max bookmark) and before
# or equal to the second sync end
self.assertGreaterEqual(
final_state_value,
target_value,
msg="The bookmark value isn't correct based on target data"
)
self.assertLessEqual(
final_state_value,
second_sync_end,
msg="The bookmark value isn't correct based on "
"the end sync")
if final_state_value_events != set() and stream != 'events':
if stream not in {
'invoices', 'customers'
}: # BUG https://stitchdata.atlassian.net/browse/SUP-1320
# verify that 'updates' state matches the target (max bookmark)
self.assertEqual(
final_state_value_events,
target_events_value,
msg=
"The bookmark value isn't correct based on target data"
)
else: # We need to ensure no assertions are missed for object streams
self.assertEqual(
stream,
'events',
msg=
"An object stream is missing it's object_events bookmark"
)
# verify that the second sync state is greater than or equal to the minimum target value
self.assertGreaterEqual(target_value, target_min_value)
# Verify that the minimum bookmark in the second sync is greater than or equal to state value from the first sync.
self.assertGreaterEqual(
target_min_value,
prev_target_value,
msg=
"Minimum bookmark in 2nd sync does not match 1st sync state value"
)
if stream != 'events':
# verify that 2nd sync captures the expected records and nothing else
expected = expected_records[stream]
expected_set = set()
for record in expected: # creating a set of expected ids for current stream
for key, val in record.items():
if 'id' == key: expected_set.add(val)
actual = [
item["data"] for item in second_sync_records.get(
stream, {
'messages': []
}).get('messages')
]
actual_set = set()
for record in actual:
for key, val in record.items(
): # creating set of actual ids
if 'id' == key: actual_set.add(val)
if not expected_set.issubset(
actual_set
): # output with print in case next assertion fails
print(stream + "\nE: " + str(expected_set) + "\nA:" +
str(actual_set))
self.assertTrue(
expected_set.issubset(actual_set),
msg=
"We expected records to be sent to the target and they were not"
)
# To create an object, some streams require the creation of objects from different streams
# As a result we create more than what is obvious in the test to capture all streams
# The 'hidden_creates' set tracks those extra records, in order to verify we aren't
# getting more records than expected
hidden_creates = get_hidden_objects(stream)
if stream in {
'plans', # BUG https://stitchdata.atlassian.net/browse/SUP-1303
'subscriptions', # BUG https://stitchdata.atlassian.net/browse/SUP-1304
'invoice_items', # BUG https://stitchdata.atlassian.net/browse/SUP-1316
'customers'
}: # BUG https://stitchdata.atlassian.net/browse/SUP-1322
continue
# if stream == 'invoice_items': # UNCOMMENT WHEN SUP-1316 is addressed
# # verify everything we created was sent
# # NOTE: we annot account for all hidden records created in the bg for this stream
# self.assertEqual(expected_set.union(hidden_creates) - actual_set, set(),
# msg="Failed to capture all records that were explicitly created.")
# continue
# Many records are created implicitly and there are too many to track.
# For now we will just log
if not actual_set.difference(expected_set).issubset(
hidden_creates):
logging.warning(
"Some extra records sent to the target that were not in expected."
"Unexpected records: {}".format(
actual_set.difference(expected_set).difference(
hidden_creates)))
def test_run(self):
"""
Verify that for each stream you can get data when no fields are selected
and only the automatic fields are replicated.
"""
for stream in self.streams_to_create:
stripe_obj = create_object(stream)
stripe_json = json.dumps(stripe_obj, sort_keys=True, indent=2)
dict_obj = json.loads(stripe_json)
self.new_objects[stream].append(dict_obj)
# instantiate connection
conn_id = connections.ensure_connection(self)
# run check mode
found_catalogs = self.run_and_verify_check_mode(conn_id)
# table and field selection
streams_to_select = self.streams_to_create
self.perform_and_verify_table_and_field_selection(
conn_id, found_catalogs, streams_to_select, select_all_fields=True)
# run initial sync
first_record_count_by_stream = self.run_and_verify_sync(conn_id)
replicated_row_count = sum(first_record_count_by_stream.values())
synced_records = runner.get_records_from_target_output()
# Verify target has records for all synced streams
for stream, count in first_record_count_by_stream.items():
assert stream in self.expected_streams()
self.assertGreater(
count,
0,
msg="failed to replicate any data for: {}".format(stream))
print("total replicated row count: {}".format(replicated_row_count))
# Test by Stream
for stream in self.streams_to_create:
with self.subTest(stream=stream):
expected_records = self.records_data_type_conversions(
self.new_objects.get(stream))
data = synced_records.get(stream)
record_messages_keys = [
set(row['data'].keys()) for row in data['messages']
]
expected_keys = set()
for record in expected_records:
expected_keys.update(record.keys())
# Verify schema covers all fields
# BUG_1 | https://stitchdata.atlassian.net/browse/SRCE-4736
# to reproduce bug comment out the marked lines below
streams_to_skip_schema_assertion = { # BUG_1
'customers', 'subscriptions', 'products', 'invoice_items',
'payouts', 'charges', 'subscription_items', 'invoices',
'plans', 'invoice_line_items'
}
schema_keys = set(self.expected_schema_keys(stream))
if stream not in streams_to_skip_schema_assertion: # BUG_1
self.assertEqual(set(),
expected_keys.difference(schema_keys),
msg="\tFields missing from schema!")
# not a test, just logging the fields that are included in the schema but not in the expectations
if schema_keys.difference(expected_keys):
print(
"WARNING Fields missing from expectations: {}".format(
schema_keys.difference(expected_keys)))
# Verify that all fields sent to the target fall into the expected schema
for actual_keys in record_messages_keys:
self.assertTrue(
actual_keys.issubset(schema_keys),
msg=
"Expected all fields to be present, as defined by schemas/{}.json"
.format(stream) +
"EXPECTED (SCHEMA): {}\nACTUAL (REPLICATED KEYS): {}".
format(schema_keys, actual_keys))
actual_records = [row['data'] for row in data['messages']]
# Verify by pks, that we replicated the expected records and only the expected records
self.assertPKsEqual(stream, expected_records, actual_records)
expected_pks_to_record_dict = self.getPKsToRecordsDict(
stream, expected_records)
actual_pks_to_record_dict = self.getPKsToRecordsDict(
stream, actual_records)
if stream not in streams_to_skip_schema_assertion: # BUG_1
for pks_tuple, expected_record in expected_pks_to_record_dict.items(
):
if expected_record.get(
'updated') is None and expected_record.get(
'created'):
print(
"WARNING adding 'updated' to new {} record for comparison"
.format(stream))
expected_record['updated'] = expected_record[
'created']
actual_record = actual_pks_to_record_dict.get(
pks_tuple)
self.assertDictEqual(expected_record, actual_record)
def manage_addOAIRecord(self, id, **kwargs):
ob = create_object(self, OAIRecord, id)
process_form(ob, IOAIRecord, kwargs)
ob.initialize()
def test_run(self):
"""
Verify that the sync only sent records to the target for selected streams
Update metadata[test] with a random number for each stream with event updates
Verify that the second sync includes at least one update for each stream
Verify that the second sync includes less records than the first sync
Verify that the updated metadata was picked up on the second sync
PREREQUISITE
For EACH stream that gets updates through events stream, there's at least 1 row
of data
"""
conn_id = connections.ensure_connection(self)
event_update_streams = {
# "balance_transactions" # Cannot be directly updated
"charges",
"coupons",
"customers",
# "disputes", # Cannot create directly with api
"invoice_items",
# "invoice_line_items", # Can't be updated via api
"invoices",
# "payout_transactions", # See bug in create_test
"payouts",
"plans",
"products",
# "subscription_items", # BUG_9916 | https://jira.talendforge.org/browse/TDL-9916
"subscriptions",
# "transfers", # Cannot be updated directly via api
}
found_catalogs = self.run_and_verify_check_mode(conn_id)
our_catalogs = [catalog for catalog in found_catalogs
if catalog.get('tap_stream_id') in
event_update_streams]
self.select_all_streams_and_fields(
conn_id, our_catalogs, select_all_fields=True
)
# Ensure each stream under test has data to start
new_objects = {
stream: create_object(stream)
for stream in event_update_streams
}
# Some streams will be updated implicitly
streams_to_update = event_update_streams.difference({
"invoice_line_items",
"subscription_items",
})
# Run a sync job using orchestrator
first_sync_record_count = self.run_and_verify_sync(conn_id)
# verify that the sync only sent records to the target for selected streams (catalogs)
self.assertEqual(set(first_sync_record_count.keys()), event_update_streams)
# Get the set of records from a first sync
first_sync_records = runner.get_records_from_target_output()
first_sync_created, _ = self.split_records_into_created_and_updated(
first_sync_records
)
updated = {} # holds id for updated objects in each stream
for stream in streams_to_update:
# There needs to be some test data for each stream, otherwise this will break
self.assertGreater(len(first_sync_created[stream]["messages"]), 0,
msg='We did not get any new records from '
'the first sync for {}'.format(stream))
record = first_sync_created[stream]["messages"][0]["data"]
# We need to make sure the data actually changes, otherwise no event update
# will get created
update_object(stream, record["id"])
updated[stream] = record["id"]
# Run a second sync job using orchestrator
second_sync_record_count = self.run_and_verify_sync(conn_id)
# Get the set of records from a second sync
second_sync_records = runner.get_records_from_target_output()
_, second_sync_updated = self.split_records_into_created_and_updated(
second_sync_records
)
# # THIS MAKES AN ASSUMPTION THAT CHILD STREAMS DO NOT NEED TESTING.
# # ADJUST IF NECESSARY
for stream in event_update_streams.difference(self.child_streams()):
with self.subTest(stream=stream):
# verify that there is more than 1 record of data - setup necessary
self.assertGreater(
first_sync_record_count.get(stream, 0),
1,
msg="Data isn't set up to be able to test event updates",
)
# verify that you get at least one updated record on the second sync
self.assertGreaterEqual(
len(second_sync_updated.get(stream, {}).get("messages", [])),
1,
msg="second syc didn't have updates",
)
# verify that you get less data the 2nd time around since only updates
# should be picked up
self.assertLess(
second_sync_record_count.get(stream, 0),
first_sync_record_count.get(stream, 0),
msg="second syc had the same or more records",
)
# verify all the updated records in the 2nd sync are different from
# the first run
first_data = next(
record["data"]
for record in first_sync_created.get(stream, {}).get("messages", [])
if record.get("data", {}).get("id") == updated[stream]
)
second_data = next(
record["data"]
for record in second_sync_updated.get(stream, {}).get(
"messages", []
)
if record.get("data", {}).get("id") == updated[stream]
)
# verify the updated timestamp is greater in the second sync
self.assertGreater(
second_data["updated"],
first_data["updated"],
"updated timestamp for second sync is not greater than first sync",
)
# verify the metadata[test] value actually changed
self.assertNotEqual(
second_data["metadata"].get("test_value", 0),
first_data["metadata"].get("test_value", 0),
"the test metadata should be different",
)
if stream in new_objects:
delete_object(stream, new_objects[stream]["id"])
def do_test(self, conn_id):
"""
Verify that the sync only sent records to the target for selected streams
Create a new object for each stream
Verify that the second sync includes at least one create for each stream
Verify that the created record was picked up on the second sync
"""
streams_to_create = {
"balance_transactions", # should be created implicity with a create in the payouts or charges streams
"charges",
"coupons",
"customers",
"invoice_items",
"invoice_line_items", # this is created implicity by invoices, it just creates another invoice TODO get this outa here
"invoices", # this will create an invoice_item
"payouts",
"plans",
"products",
"subscription_items",
"subscriptions", # this will create a new plan and payment method
}
missing_streams_to_create = {
"disputes", # can be created by simulating a dispute transaction with a specific card number
# no way to create directly, see: https://stripe.com/docs/testing#disputes
"payout_transactions", # BUG (https://stitchdata.atlassian.net/browse/SUP-1294)
# depends on payouts and transactions
"transfers",
# needs an account that we can transfer to, not sure
# how to set up a test account we can use to create a transfer
}
our_catalogs = get_catalogs(conn_id, streams_to_create)
self.select_all_streams_and_fields(conn_id,
our_catalogs,
select_all_fields=True)
# Run a sync job using orchestrator
first_sync_record_count = self.run_sync(conn_id)
# verify that the sync only sent records to the target for selected streams (catalogs)
self.assertEqual(set(first_sync_record_count.keys()),
streams_to_create)
# Get the set of records from a first sync
first_sync_records = runner.get_records_from_target_output()
first_sync_created, _ = self.split_records_into_created_and_updated(
first_sync_records)
new_objects = {
stream: create_object(stream)
for stream in streams_to_create.difference(
{"balance_transactions"})
}
# Run a second sync job using orchestrator
second_sync_record_count = self.run_sync(conn_id)
# Get the set of records from a second sync
second_sync_records = runner.get_records_from_target_output()
second_sync_created, _ = self.split_records_into_created_and_updated(
second_sync_records)
# # THIS MAKES AN ASSUMPTION THAT CHILD STREAMS DO NOT NEED TESTING.
# # ADJUST IF NECESSARY
for stream in streams_to_create.difference(self.child_streams()):
with self.subTest(stream=stream):
second_sync_created_objects = second_sync_created.get(
stream, {}).get("messages", [])
# verify that you get at least one new record on the second sync
self.assertGreaterEqual(
len(second_sync_created_objects),
1,
msg="second sync didn't have created objects",
)
if stream == "balance_transactions":
sources = [
record.get("data", {}).get("source")
for record in second_sync_created_objects
]
self.assertTrue(new_objects['payouts']['id'] in sources)
self.assertTrue(new_objects['charges']['id'] in sources)
continue
# verify the new object is in the list of created objects
# from the second sync
self.assertTrue(
any(new_objects[stream]["id"] == record.get("data",
{}).get("id")
for record in second_sync_created_objects))
if stream in streams_to_create:
delete_object(stream, new_objects[stream]["id"])
async def create_mptcp_proxy(data_store, amqp, **kwargs):
ret, mptcp_proxy = utils.create_object(data_store, amqp,
Mptcp_proxy_schema, kwargs)
await send_create_proxy(data_store, amqp, mptcp_proxy)
raise web.HTTPAccepted(content_type="application/json", text=ret)
def test_run(self):
"""
Verify for each stream that you can do a sync which records bookmarks.
Verify that the bookmark is the max value sent to the target for the `date` PK field
Verify that the 2nd sync respects the bookmark
Verify that all data of the 2nd sync is >= the bookmark from the first sync
Verify that the number of records in the 2nd sync is less then the first
Verify inclusivivity of bookmarks
PREREQUISITE
For EACH stream that is incrementally replicated there are multiple rows of data with
different values for the replication key
"""
untested_streams = self.child_streams().union({
'transfers',
'payout_transactions', # BUG see create test
'balance_transactions', # join stream, can't be updated
'disputes',
})
cannot_update_streams = {
'invoice_line_items', # updates not available via api
}
# Ensure tested streams have existing records
expected_records_first_sync = {stream: [] for stream in self.streams_to_create}
for _ in range(2): # create 3 records for each stream but only expect the 3rd
for stream in self.streams_to_create:
self.new_objects[stream].append(create_object(stream))
for stream in self.streams_to_create:
self.new_objects[stream].append(create_object(stream))
expected_records_first_sync[stream].append({"id": self.new_objects[stream][-1]['id']})
self.START_DATE = self.get_properties().get('start_date')
# Instantiate connection with default start
conn_id = connections.ensure_connection(self)
# run in check mode
found_catalogs = self.run_and_verify_check_mode(conn_id)
# Select all testable streams and all fields within streams
streams_to_select = self.expected_incremental_streams().difference(untested_streams)
our_catalogs = [catalog for catalog in found_catalogs
if catalog.get('tap_stream_id') in
streams_to_select]
self.select_all_streams_and_fields(conn_id, our_catalogs, select_all_fields=True)
# Run a sync job using orchestrator
first_sync_start = self.local_to_utc(dt.utcnow())
first_sync_record_count = self.run_and_verify_sync(conn_id)
first_sync_end = self.local_to_utc(dt.utcnow())
# verify that the sync only sent records to the target for selected streams (catalogs)
self.assertEqual(
streams_to_select, set(first_sync_record_count.keys()),
msg="Expected only testable streams to be replicated: {}".format(first_sync_record_count)
)
first_sync_state = menagerie.get_state(conn_id)
# Get the set of records from a first sync
first_sync_records = runner.get_records_from_target_output()
# Add data before next sync via insert and update, and set expectations
created_records = {x: [] for x in self.expected_streams()}
updated_records = {x: [] for x in self.expected_streams()}
expected_records_second_sync = {x: [] for x in self.expected_streams()}
# Update one record from each stream prior to 2nd sync
first_sync_created, _ = self.split_records_into_created_and_updated(first_sync_records)
for stream in self.streams_to_create.difference(cannot_update_streams):
# There needs to be some test data for each stream, otherwise this will break
record = expected_records_first_sync[stream][0]
updated_record = update_object(stream, record["id"])
updated_records[stream].append(updated_record)
expected_records_second_sync[stream].append({"id": updated_record['id']})
# Ensure different times between udpates and inserts
sleep(2)
# Insert (create) one record for each stream prior to 2nd sync
for stream in self.streams_to_create:
created_record = create_object(stream)
self.new_objects[stream].append(created_record)
created_records[stream].append(created_record)
expected_records_second_sync[stream].append({"id": created_record['id']})
# ensure validity of expected_records_second_sync
for stream in self.streams_to_create:
if stream in self.expected_incremental_streams():
if stream in cannot_update_streams:
# Some streams will have only 1 record from the Insert
self.assertEqual(1, len(expected_records_second_sync.get(stream)),
msg="Expectations are invalid for incremental stream {}".format(stream)
)
continue
# Most streams will have 2 records from the Update and Insert
self.assertEqual(2, len(expected_records_second_sync.get(stream)),
msg="Expectations are invalid for incremental stream {}".format(stream)
)
elif stream in self.expected_full_table_streams():
self.assertEqual(
len(expected_records_second_sync.get(stream)),
len(expected_records_first_sync.get(stream)) + len(created_records[stream]),
msg="Expectations are invalid for full table stream {}".format(stream)
)
# created_records[stream] = self.records_data_type_conversions(created_records.get(stream))
# updated_records[stream] = self.records_data_type_conversions(updated_records.get(stream))
# Run a second sync job using orchestrator
second_sync_start = self.local_to_utc(dt.utcnow())
second_sync_record_count = self.run_and_verify_sync(conn_id)
second_sync_end = self.local_to_utc(dt.utcnow())
second_sync_state = menagerie.get_state(conn_id)
# Get the set of records from a second sync
second_sync_records = runner.get_records_from_target_output()
second_sync_created, second_sync_updated = self.split_records_into_created_and_updated(second_sync_records)
# Loop first_sync_records and compare against second_sync_records
for stream in self.streams_to_create.difference(untested_streams):
with self.subTest(stream=stream):
second_sync_data = [record.get("data") for record
in second_sync_records.get(stream, {}).get("messages", [])]
stream_replication_keys = self.expected_replication_keys()
stream_primary_keys = self.expected_primary_keys()
# TESTING INCREMENTAL STREAMS
if stream in self.expected_incremental_streams():
replication_keys = stream_replication_keys.get(stream)
# Verify both syncs write / keep the same bookmark keys
self.assertEqual(set(first_sync_state.get('bookmarks', {}).keys()),
set(second_sync_state.get('bookmarks', {}).keys()))
# verify that there is more than 1 record of data - setup necessary
self.assertGreater(first_sync_record_count.get(stream, 0), 1,
msg="Data isn't set up to be able to test full sync")
# verify that you get less data on the 2nd sync
self.assertGreater(
first_sync_record_count.get(stream, 0),
second_sync_record_count.get(stream, 0),
msg="first sync didn't have more records, bookmark usage not verified")
if stream in self.streams_to_create:
for replication_key in replication_keys:
updates_replication_key = "updates_created"
updates_stream = stream + "_events"
# Verify second sync's bookmarks move past the first sync's
self.assertGreater(
second_sync_state.get('bookmarks', {updates_stream: {}}).get(
updates_stream, {replication_key: -1}).get(updates_replication_key),
first_sync_state.get('bookmarks', {updates_stream: {}}).get(
updates_stream, {updates_replication_key: -1}).get(updates_replication_key)
)
# Verify that all data of the 2nd sync is >= the bookmark from the first sync
first_sync_bookmark = dt.fromtimestamp(
first_sync_state.get('bookmarks').get(updates_stream).get(updates_replication_key)
)
for record in second_sync_data:
date_value = record["updated"]
self.assertGreaterEqual(date_value,
dt.strftime(first_sync_bookmark, self.COMPARISON_FORMAT),
msg="A 2nd sync record has a replication-key that is less than or equal to the 1st sync bookmark.")
elif stream in self.expected_full_table_streams():
raise Exception("Expectations changed, but this test was not updated to reflect them.")
# TESTING APPLICABLE TO ALL STREAMS
# Verify that the expected records are replicated in the 2nd sync
# For incremental streams we should see at least 2 records (a new record and an updated record)
# but we may see more as the bookmmark is inclusive and there are hidden creates/updates due to
# dependencies between streams.
# For full table streams we should see 1 more record than the first sync
expected_records = expected_records_second_sync.get(stream)
primary_keys = stream_primary_keys.get(stream)
updated_pk_values = {tuple([record.get(pk) for pk in primary_keys])
for record in updated_records[stream]}
self.assertLessEqual(
len(expected_records), len(second_sync_data),
msg="Expected number of records are not less than or equal to actual for 2nd sync.\n" +
"Expected: {}\nActual: {}".format(len(expected_records), len(second_sync_data))
)
if (len(second_sync_data) - len(expected_records)) > 0:
logging.warn('Second sync replicated %s records more than our create and update for %s',
len(second_sync_data), stream)
if not primary_keys:
raise NotImplementedError("PKs are needed for comparing records")
# Verify that the inserted and updated records are replicated by the 2nd sync
for expected_record in expected_records:
expected_pk_value = expected_record.get('id')
sync_pk_values = [sync_record.get('id')
for sync_record in second_sync_data
if sync_record.get('id') == expected_pk_value]
self.assertTrue(
len(sync_pk_values) > 0,
msg="A record is missing from our sync: \nSTREAM: {}\tPK: {}".format(stream, expected_pk_value)
)
self.assertIn(expected_pk_value, sync_pk_values)
# Verify updated fields are replicated as expected
for updated_record in updated_records[stream]:
expected_updated_key = 'metadata'
expected_updated_value_substring = 'bob'
updated_pk_value = updated_record.get('id')
sync_records_metadata = [sync_record.get('metadata')
for sync_record in second_sync_data
if sync_record.get('id') == updated_pk_value]
self.assertTrue(len(sync_records_metadata) == 1)
self.assertIn(expected_updated_value_substring,
sync_records_metadata[0].get('test_value'))
async def create_expansion(data_store, amqp, **kwargs):
ret, expansion = utils.create_object(data_store, amqp, Expansion_schema,
kwargs)
await send_create_expansion(data_store, amqp, expansion)
raise web.HTTPAccepted(content_type="application/json", text=ret)
async def create_ipsec_policy(data_store, amqp, **kwargs):
ret, _ = utils.create_object(data_store, amqp, Ipsec_policy_schema, kwargs)
raise web.HTTPCreated(content_type="application/json", text=ret)
async def create_l2_tunnel(data_store, amqp, **kwargs):
ret, l2_tunnel = utils.create_object(data_store, amqp, L2_tunnel_schema,
kwargs)
await send_create_tunnel(data_store, amqp, l2_tunnel)
raise web.HTTPAccepted(content_type="application/json", text=ret)
def test_run(self):
"""
Verify that for each stream you can get multiple pages of data
when no fields are selected and only the automatic fields are replicated.
PREREQUISITE
For EACH stream add enough data that you surpass the limit of a single
fetch of data. For instance if you have a limit of 250 records ensure
that 251 (or more) records have been posted for that stream.
"""
conn_id = connections.ensure_connection(self)
streams_to_create = {
# "balance_transactions", # should be created implicity with a create in the payouts or charges streams
"charges",
"coupons",
"customers",
"invoice_items",
"invoice_line_items", # this is created implicity by invoices, it just creates another invoice
"invoices", # this will create an invoice_item
"payouts",
"plans",
"products",
"subscription_items",
"subscriptions", # this will create a new plan and payment method
}
untested_streams = {"disputes", "transfers", "payout_transactions"}
new_objects = {
stream: create_object(stream)
for stream in streams_to_create.difference()
}
# Select all streams and no fields within streams
# IF THERE ARE NO AUTOMATIC FIELDS FOR A STREAM
# WE WILL NEED TO UPDATE THE BELOW TO SELECT ONE
found_catalogs = self.run_and_verify_check_mode(conn_id)
self.select_all_streams_and_fields(conn_id,
found_catalogs,
select_all_fields=False)
# Run a sync job using orchestrator
record_count_by_stream = self.run_and_verify_sync(conn_id)
actual_fields_by_stream = runner.examine_target_output_for_fields()
for stream in self.expected_streams().difference(untested_streams):
with self.subTest(stream=stream):
# verify that you get some records for each stream
# SKIP THIS ASSERTION FOR STREAMS WHERE YOU CANNOT GET
# MORE THAN 1 PAGE OF DATA IN THE TEST ACCOUNT
self.assertGreater(
record_count_by_stream.get(stream, -1),
0,
msg="The number of records is not over the stream max limit"
)
# verify that only the automatic fields are sent to the target
actual = actual_fields_by_stream.get(stream) or set()
expected = self.expected_automatic_fields().get(stream, set())
self.assertEqual(
actual,
expected,
msg=
("The fields sent to the target are not the automatic fields. Expected: {}, Actual: {}"
.format(actual, expected)))
def do_test(self, conn_id):
"""Test we get a lot of data back based on the start date configured in base"""
# Select all streams and all fields within streams
found_catalogs = menagerie.get_catalogs(conn_id)
incremental_streams = {
key
for key, value in self.expected_replication_method().items()
if value == self.INCREMENTAL
}
# IF THERE ARE STREAMS THAT SHOULD NOT BE TESTED
# REPLACE THE EMPTY SET BELOW WITH THOSE STREAMS
untested_streams = self.child_streams().union(
{'disputes', 'events', 'transfers', 'payout_transactions'})
our_catalogs = get_catalogs(
conn_id, incremental_streams.difference(untested_streams))
self.select_all_streams_and_fields(conn_id,
our_catalogs,
select_all_fields=True)
# Create a record for each stream under test prior to the first sync
new_objects = {
stream: create_object(stream)
for stream in incremental_streams.difference(untested_streams)
}
# Run a sync job using orchestrator
first_sync_record_count = self.run_sync(conn_id)
first_total_records = reduce(lambda a, b: a + b,
first_sync_record_count.values())
# Count actual rows synced
first_sync_records = runner.get_records_from_target_output()
# set the start date for a new connection based off bookmarks largest value
first_max_bookmarks = self.max_bookmarks_by_stream(first_sync_records)
bookmark_list = [
next(iter(book.values()))
for stream, book in first_max_bookmarks.items()
]
bookmark_dates = []
for bookmark in bookmark_list:
try:
bookmark_dates.append(parse(bookmark))
except (ValueError, OverflowError, TypeError):
pass
if not bookmark_dates:
# THERE WERE NO BOOKMARKS THAT ARE DATES.
# REMOVE CODE TO FIND A START DATE AND ENTER ONE MANUALLY
raise ValueError
# largest_bookmark = reduce(lambda a, b: a if a > b else b, bookmark_dates)
# self.start_date = self.local_to_utc(largest_bookmark).strftime(self.START_DATE_FORMAT)
self.start_date = dt.strftime(dt.today() - timedelta(days=1),
self.START_DATE_FORMAT)
# create a new connection with the new start_date
conn_id = self.create_connection(original_properties=False)
# Select all streams and all fields within streams
found_catalogs = menagerie.get_catalogs(conn_id)
our_catalogs = [
catalog for catalog in found_catalogs
if catalog.get('tap_stream_id') in incremental_streams.difference(
untested_streams)
]
self.select_all_streams_and_fields(conn_id,
our_catalogs,
select_all_fields=True)
# TODO remove the updates, this is unnecessary. Verify with Harvest
# Update a record for each stream under test prior to the 2nd sync
first_sync_created, _ = self.split_records_into_created_and_updated(
first_sync_records)
updated = {} # holds id for updated objects in each stream
for stream in new_objects:
# There needs to be some test data for each stream, otherwise this will break
record = first_sync_created[stream]["messages"][0]["data"]
update_object(stream, record["id"])
updated[stream] = record["id"]
# Run a sync job using orchestrator
second_sync_record_count = self.run_sync(conn_id)
# tap-stripe uses events for updates, so these need filtered to validate bookmark
second_sync_records = runner.get_records_from_target_output()
second_sync_created, second_sync_updated = self.split_records_into_created_and_updated(
second_sync_records)
second_total_records = reduce(lambda a, b: a + b,
second_sync_record_count.values(), 0)
# Only examine bookmarks for "created" objects, not updates
second_min_bookmarks = self.min_bookmarks_by_stream(
second_sync_created)
# verify that at least one record synced and less records synced than the 1st connection
self.assertGreater(second_total_records, 0)
self.assertLess(first_total_records, second_total_records)
# validate that all newly created records are greater than the start_date
for stream in incremental_streams.difference(untested_streams):
with self.subTest(stream=stream):
# verify that each stream has less records in the first sync than the second
self.assertGreater(
second_sync_record_count.get(stream, 0),
first_sync_record_count.get(stream, 0),
msg="first had more records, start_date usage not verified"
)
# verify all data from 2nd sync >= start_date
target_mark = second_min_bookmarks.get(stream, {"mark": None})
target_value = next(iter(
target_mark.values())) # there should be only one
if target_value:
# it's okay if there isn't target data for a stream
try:
target_value = self.local_to_utc(parse(target_value))
expected_value = self.local_to_utc(
parse(self.start_date))
# verify that the minimum bookmark sent to the target for the second sync
# is greater than or equal to the start date
self.assertGreaterEqual(target_value, expected_value)
except (OverflowError, ValueError, TypeError):
print("bookmarks cannot be converted to dates, "
"can't test start_date for {}".format(stream))
if stream in updated:
delete_object(stream, updated[stream])
async def create_network(data_store, amqp, **kwargs):
ret, _ = utils.create_object(data_store, amqp, Network_schema, kwargs)
raise web.HTTPOk(content_type="application/json", text=ret)
