def __init__(self, config): super().__init__() self.features = nn.Sequential(*(utils.create_object( **layer) for layer in config['features']['layers'])) self.reshape = config['reshape'] self.classifier = nn.Sequential(*(utils.create_object( **layer) for layer in config['classifier']['layers'])) self.loss = utils.create_object(**config['loss'])
def setUpClass(cls): logging.info("Start Setup") # Create data prior to first sync cls.streams_to_test = { "customers", "charges", "coupons", "invoice_items", "invoice_line_items", "invoices", "payouts", "plans", "products", "subscription_items", "subscriptions", } cls.expected_objects = {stream: [] for stream in cls.streams_to_test} cls.existing_objects = {stream: [] for stream in cls.streams_to_test} cls.new_objects = {stream: [] for stream in cls.streams_to_test} for stream in cls.streams_to_test: # create new records stripe_obj = create_object(stream) cls.new_objects[stream] = [stripe_obj_to_dict(stripe_obj)] cls.expected_objects[stream] = cls.new_objects[stream]
def manage_addOAIToken(self, id=None, **kwargs): """ """ if id is None: id = u"%sr%s" % (int(kwargs['token_args'].get('cursor', 0)), randint(10000, 99999)) ob = create_object(self, OAIToken, id) if not kwargs['token_args'].has_key('id'): kwargs['token_args']['id'] = id process_form(ob, IOAIToken, kwargs) ob.expiration = kwargs['token_args']['expirationDate'] return ob
def manage_addOAIServer(self, id='', REQUEST=None, **kwargs): """ """ if REQUEST is not None: form_data = dict(REQUEST.form) else: form_data = dict(kwargs) try: ob = create_object(self, OAIServer, id) process_form(ob, IOAIServer, form_data) ob.initialize() except Exception, e: if REQUEST is not None: return REQUEST.RESPONSE.redirect(self.absolute_url() + '/manage_main?manage_tabs_message=%s' % quote(str(e))) else: raise Exception(e)
def manage_addOAIHarvester(self, id='', REQUEST=None, **kwargs): """ """ if REQUEST is not None: form_data = dict(REQUEST.form) else: form_data = dict(kwargs) try: if id == '': id = processId(form_data.get('title', 'zharvester')).lower() ob = create_object(self, OAIHarvester, id) process_form(ob, IOAIHarvester, form_data) ob.update_sets() except Exception, e: transaction.abort() if REQUEST is not None: return REQUEST.RESPONSE.redirect(self.absolute_url() + '/manage_main?manage_tabs_message=%s' % quote(str(e))) else: raise
def __init__(self): self.queue = Queue() with open('config.json') as config_file: self.config = json.load(config_file) self.model = NeuralNet(self.config) with open('params.json') as f: self.load_params(json.load(f)) self.params = self.config['optimizer']['parameters'] self.params['params'] = self.model.parameters() self.parameter_lengths = list( map(lambda x: x.data.nelement(), self.model.parameters())) self.optimizer = utils.create_object(**self.config['optimizer']) self.accuracy = None self.lock = Lock() self.changed = True self.params_updated = False self.loader = utils.get_data() self.convert_params() self.compute_accuracy() Thread(target=self.update_model, daemon=True).start()
def test_run(self): """ Verify that for each stream you can get multiple pages of data and that when all fields are selected more than the automatic fields are replicated. PREREQUISITE For EACH stream add enough data that you surpass the limit of a single fetch of data. For instance if you have a limit of 250 records ensure that 251 (or more) records have been posted for that stream. """ conn_id = connections.ensure_connection(self) incremental_streams = {key for key, value in self.expected_replication_method().items() if value == self.INCREMENTAL} untested_streams = self.child_streams().union({ 'balance_transactions', # 'charges', # 'coupons', # 'customers', 'disputes', # 'invoice_items', 'invoice_line_items', # 'invoices', 'payout_transactions', # 'payouts', # 'plans', # 'products', 'subscription_items', # 'subscriptions', 'transfers', }) tested_streams = incremental_streams.difference(untested_streams) # Select all streams and all fields within streams found_catalogs = self.run_and_verify_check_mode(conn_id) our_catalogs = get_catalogs(conn_id, tested_streams) self.select_all_streams_and_fields(conn_id, our_catalogs, select_all_fields=True) # Ensure tested streams have a record count which exceeds the API LIMIT logging.info("Checking record counts for tested streams...") streams_to_create = {} for stream in tested_streams: records = list_all_object(stream) record_count = len(records) streams_to_create[stream] = record_count logging.info(" Stream {} has {} records created today".format(stream, record_count)) logging.info("Creating records for tested streams...") new_objects = {stream: [] for stream in streams_to_create} for stream in streams_to_create: if stream != "events" and streams_to_create[stream] <= self.API_LIMIT: while streams_to_create[stream] <= self.API_LIMIT: logging.info("Creating a record for {} | {} records created today ".format(stream, streams_to_create[stream])) new_objects[stream].append(create_object(stream)) streams_to_create[stream] += 1 records = list_all_object(stream) self.assertEqual(100, len(records)) logging.info(" Stream {} has at least {} records created today".format(stream, len(records) + 1)) # Run a sync job using orchestrator record_count_by_stream = self.run_and_verify_sync(conn_id) actual_fields_by_stream = runner.examine_target_output_for_fields() for stream in incremental_streams.difference(untested_streams): with self.subTest(stream=stream): # verify that we can paginate with all fields selected self.assertGreater( record_count_by_stream.get(stream, -1), self.expected_metadata().get(stream, {}).get(self.API_LIMIT, 0), msg="The number of records is not over the stream max limit") # verify that the automatic fields are sent to the target actual = actual_fields_by_stream.get(stream) or set() expected = self.expected_automatic_fields().get(stream, set()) self.assertTrue(actual.issuperset(expected), msg="The fields sent to the target don't include all automatic fields. " "Expected: {}, Actual: {}". format(expected, actual) ) # verify we have more fields sent to the target than just automatic fields # SKIP THIS ASSERTION IF ALL FIELDS ARE INTENTIONALLY AUTOMATIC FOR THIS STREAM actual = actual_fields_by_stream.get(stream) or set() expected = self.expected_automatic_fields().get(stream, set()) self.assertTrue(actual.symmetric_difference(expected), msg="The fields sent to the target don't include any non-automatic fields" ) if stream != "events": actual = actual_fields_by_stream.get(stream, set()) expected = set(new_objects[stream][0].keys())
def test_run(self): """ Verify that for each stream you can do a sync which records bookmarks. That the bookmark is the maximum value sent to the target for the replication key. That a second sync respects the bookmark All data of the second sync is >= the bookmark from the first sync The number of records in the 2nd sync is less then the first (This assumes that new data added to the stream is done at a rate slow enough that you haven't doubled the amount of data from the start date to the first sync between the first sync and second sync run in this test) Verify that only data for incremental streams is sent to the target PREREQUISITE For EACH stream that is incrementally replicated there are multiple rows of data with different values for the replication key """ conn_id = self.create_connection() expected_records = {stream: [] for stream in self.streams_to_create} # Create 3 records for each stream total for _ in range(2): for stream in self.streams_to_create: self.new_objects[stream].append(create_object(stream)) # Turn on tracking of hidden object creates in util_stripe # so that we can properly assert on 2nd sync records and expectations activate_tracking() # Create the 3rd record now that we are recording all created objects for stream in self.streams_to_create: self.new_objects[stream].append(create_object(stream)) expected_records[stream].append( {"id": self.new_objects[stream][-1]['id']}) # Select all streams and no fields within streams found_catalogs = menagerie.get_catalogs(conn_id) incremental_streams = { key for key, value in self.expected_replication_method().items() if value == self.INCREMENTAL } # IF THERE ARE STREAMS THAT SHOULD NOT BE TESTED # REPLACE THE EMPTY SET BELOW WITH THOSE STREAMS untested_streams = self.child_streams().union({ 'transfers', 'payout_transactions', # BUG see create test 'balance_transactions', # join stream, can't be updated 'disputes', }) our_catalogs = [ catalog for catalog in found_catalogs if catalog.get('tap_stream_id') in incremental_streams.difference( untested_streams) ] self.select_all_streams_and_fields(conn_id, our_catalogs, select_all_fields=True) # Run a sync job using orchestrator and track the sync's time window first_sync_start = self.local_to_utc(dt.utcnow()) first_sync_record_count = self.run_sync(conn_id) first_sync_end = self.local_to_utc(dt.utcnow()) # verify that the sync only sent records to the target for selected streams (catalogs) self.assertEqual(set(first_sync_record_count.keys()), incremental_streams.difference(untested_streams)) first_sync_state = menagerie.get_state(conn_id) # Get data about actual rows synced first_sync_records = runner.get_records_from_target_output() first_max_bookmarks = self.max_bookmarks_by_stream(first_sync_records) first_max_events = self.max_events_bookmarks_by_stream( first_sync_records) first_min_bookmarks = self.min_bookmarks_by_stream(first_sync_records) # Update one record from each stream prior to 2nd sync first_sync_created, _ = self.split_records_into_created_and_updated( first_sync_records) updated_objects = {stream: [] for stream in self.streams_to_create } # TODO Delete if never used for stream in self.streams_to_create: # There needs to be some test data for each stream, otherwise this will break record = expected_records[stream][0] # record = first_sync_created[stream]["messages"][0]["data"] updated_objects[stream].append(update_object(stream, record["id"])) expected_records[stream].append( {"id": updated_objects[stream][-1]['id']}) # Ensure different times between udpates and inserts sleep(2) # Insert (create) one record for each stream prior to 2nd sync for stream in self.streams_to_create: self.new_objects[stream].append(create_object(stream)) expected_records[stream].append( {"id": self.new_objects[stream][-1]['id']}) # Run a second sync job using orchestrator second_sync_start = self.local_to_utc(dt.utcnow()) second_sync_record_count = self.run_sync(conn_id) second_sync_end = self.local_to_utc(dt.utcnow()) second_sync_state = menagerie.get_state(conn_id) # Get data about rows synced second_sync_records = runner.get_records_from_target_output() second_sync_created, second_sync_updated = self.split_records_into_created_and_updated( second_sync_records) second_min_bookmarks = self.min_bookmarks_by_stream( second_sync_created) second_max_bookmarks = self.max_bookmarks_by_stream( second_sync_records) second_max_events = self.max_events_bookmarks_by_stream( second_sync_records) self.assertTrue(second_max_events) # Define bookmark keys events_bookmark_key = "updates_created" # secondary bookmark from the events stream target_events_bookmark_key = "updated" # the bookmark key for events # Adjust expectations to account for invoices created by the invoice_line_items calls for e in expected_records['invoice_line_items']: expected_records['invoices'].append(e) # THIS MAKES AN ASSUMPTION THAT CHILD STREAMS DO NOT HAVE BOOKMARKS. # ADJUST IF NECESSARY for stream in incremental_streams.difference(untested_streams): with self.subTest(stream=stream): # Get bookmark values from state and target data. # Recall that there are two bookmarks for every object stream: an events-based # bookmark for updates, and a standard repliction key-based bookmark for creates. stream_bookmark_key = self.expected_replication_keys().get( stream, set()) assert len( stream_bookmark_key ) == 1 # There shouldn't be a compound replication key stream_bookmark_key = stream_bookmark_key.pop() state_value = first_sync_state.get("bookmarks", {}).get( stream + "", {}).get(stream_bookmark_key) target_value = first_max_bookmarks.get( stream + "", {}).get(stream_bookmark_key) target_min_value = first_min_bookmarks.get( stream + "", {}).get(stream_bookmark_key) # the events stream should get an empty set as events_events does not exist state_value_events = first_sync_state.get("bookmarks", {}).get( stream + "_events", {}).get(events_bookmark_key) target_events_value = first_max_events.get( stream + "_events", {}).get(target_events_bookmark_key) try: # attempt to parse the bookmark as a date state_value = self.parse_bookmark_to_date(state_value) state_value_events = self.parse_bookmark_to_date( state_value_events) target_value = self.parse_bookmark_to_date(target_value) target_events_value = self.parse_bookmark_to_date( target_events_value) target_min_value = self.parse_bookmark_to_date( target_min_value) except (OverflowError, ValueError, TypeError): print( "bookmarks cannot be converted to dates, comparing values directly" ) ########################################################################## ### 1st Sync Tests ########################################################################## if stream != 'payout_transactions': # verify that there is data with different bookmark values - setup necessary self.assertGreaterEqual( target_value, target_min_value, msg="Data isn't set up to be able to test bookmarks") # TODO run test with our time = current_time + 1 hour and see if the events stream is able to cover for any potential # issues with our time window discrepancy capturing updates/creates # verify that 'created' state is saved for the first sync in the appropriate # time window, between (inclusive) the target value (max bookmark) for this stream and # the first sync end time self.assertGreaterEqual( state_value, target_value, msg="The bookmark value isn't correct based on target data" ) self.assertLessEqual( state_value, first_sync_end, msg="The bookmark value isn't correct based on " "the start sync") if state_value_events != set() and stream != 'events': if stream not in { 'invoices', 'customers' }: # BUG https://stitchdata.atlassian.net/browse/SUP-1320 # verify that 'updates' state matches the target (max bookmark) self.assertEqual( state_value_events, target_events_value, msg="The bookmark value isn't correct " "based on target data") # NOTE: This assertion is no longer valid. Some streams will create records for other streams # verify the last record has the max bookmark in the target (this should never fail) # last_created = self.parse_bookmark_to_date(self.new_objects[stream][2]['created']) # self.assertEqual(target_value, last_created, # msg="The last created record for the first sync should " # "be sthe max bookmark in the target and is not.") else: # We need to ensure no assertions are missed for object streams self.assertEqual( stream, 'events', msg= "An object stream is missing it's object_events bookmark" ) ########################################################################## ### 2nd Sync Tests ########################################################################## prev_target_value = target_value prev_target_events_value = target_events_value target_value = second_max_bookmarks.get( stream + "", {}).get(stream_bookmark_key) target_min_value = second_min_bookmarks.get( stream + "", {}).get(stream_bookmark_key) final_state_value = second_sync_state.get("bookmarks", {}).get( stream + "", {}).get(stream_bookmark_key) final_state_value_events = second_sync_state.get( "bookmarks", {}).get(stream + "_events", {}).get(events_bookmark_key) target_events_value = second_max_events.get( stream + "_events", {}).get(target_events_bookmark_key) try: final_state_value = self.parse_bookmark_to_date( final_state_value) final_state_value_events = self.parse_bookmark_to_date( final_state_value_events) target_value = self.parse_bookmark_to_date(target_value) target_events_value = self.parse_bookmark_to_date( target_events_value) target_min_value = self.parse_bookmark_to_date( target_min_value) except (OverflowError, ValueError, TypeError): print( "bookmarks cannot be converted to dates, comparing values directly" ) # verify that the 2nd sync gets more than zero records self.assertGreater(second_sync_record_count.get(stream, 0), 0, msg="second syc didn't have any records") # verify that you get less data the 2nd time around self.assertGreater( first_sync_record_count.get(stream, 0), second_sync_record_count.get(stream, 0), msg= "second syc didn't have less records, bookmark usage not verified" ) # verify 'created' state is at or after the target (max bookmark) and before # or equal to the second sync end self.assertGreaterEqual( final_state_value, target_value, msg="The bookmark value isn't correct based on target data" ) self.assertLessEqual( final_state_value, second_sync_end, msg="The bookmark value isn't correct based on " "the end sync") if final_state_value_events != set() and stream != 'events': if stream not in { 'invoices', 'customers' }: # BUG https://stitchdata.atlassian.net/browse/SUP-1320 # verify that 'updates' state matches the target (max bookmark) self.assertEqual( final_state_value_events, target_events_value, msg= "The bookmark value isn't correct based on target data" ) else: # We need to ensure no assertions are missed for object streams self.assertEqual( stream, 'events', msg= "An object stream is missing it's object_events bookmark" ) # verify that the second sync state is greater than or equal to the minimum target value self.assertGreaterEqual(target_value, target_min_value) # Verify that the minimum bookmark in the second sync is greater than or equal to state value from the first sync. self.assertGreaterEqual( target_min_value, prev_target_value, msg= "Minimum bookmark in 2nd sync does not match 1st sync state value" ) if stream != 'events': # verify that 2nd sync captures the expected records and nothing else expected = expected_records[stream] expected_set = set() for record in expected: # creating a set of expected ids for current stream for key, val in record.items(): if 'id' == key: expected_set.add(val) actual = [ item["data"] for item in second_sync_records.get( stream, { 'messages': [] }).get('messages') ] actual_set = set() for record in actual: for key, val in record.items( ): # creating set of actual ids if 'id' == key: actual_set.add(val) if not expected_set.issubset( actual_set ): # output with print in case next assertion fails print(stream + "\nE: " + str(expected_set) + "\nA:" + str(actual_set)) self.assertTrue( expected_set.issubset(actual_set), msg= "We expected records to be sent to the target and they were not" ) # To create an object, some streams require the creation of objects from different streams # As a result we create more than what is obvious in the test to capture all streams # The 'hidden_creates' set tracks those extra records, in order to verify we aren't # getting more records than expected hidden_creates = get_hidden_objects(stream) if stream in { 'plans', # BUG https://stitchdata.atlassian.net/browse/SUP-1303 'subscriptions', # BUG https://stitchdata.atlassian.net/browse/SUP-1304 'invoice_items', # BUG https://stitchdata.atlassian.net/browse/SUP-1316 'customers' }: # BUG https://stitchdata.atlassian.net/browse/SUP-1322 continue # if stream == 'invoice_items': # UNCOMMENT WHEN SUP-1316 is addressed # # verify everything we created was sent # # NOTE: we annot account for all hidden records created in the bg for this stream # self.assertEqual(expected_set.union(hidden_creates) - actual_set, set(), # msg="Failed to capture all records that were explicitly created.") # continue # Many records are created implicitly and there are too many to track. # For now we will just log if not actual_set.difference(expected_set).issubset( hidden_creates): logging.warning( "Some extra records sent to the target that were not in expected." "Unexpected records: {}".format( actual_set.difference(expected_set).difference( hidden_creates)))
def test_run(self): """ Verify that for each stream you can get data when no fields are selected and only the automatic fields are replicated. """ for stream in self.streams_to_create: stripe_obj = create_object(stream) stripe_json = json.dumps(stripe_obj, sort_keys=True, indent=2) dict_obj = json.loads(stripe_json) self.new_objects[stream].append(dict_obj) # instantiate connection conn_id = connections.ensure_connection(self) # run check mode found_catalogs = self.run_and_verify_check_mode(conn_id) # table and field selection streams_to_select = self.streams_to_create self.perform_and_verify_table_and_field_selection( conn_id, found_catalogs, streams_to_select, select_all_fields=True) # run initial sync first_record_count_by_stream = self.run_and_verify_sync(conn_id) replicated_row_count = sum(first_record_count_by_stream.values()) synced_records = runner.get_records_from_target_output() # Verify target has records for all synced streams for stream, count in first_record_count_by_stream.items(): assert stream in self.expected_streams() self.assertGreater( count, 0, msg="failed to replicate any data for: {}".format(stream)) print("total replicated row count: {}".format(replicated_row_count)) # Test by Stream for stream in self.streams_to_create: with self.subTest(stream=stream): expected_records = self.records_data_type_conversions( self.new_objects.get(stream)) data = synced_records.get(stream) record_messages_keys = [ set(row['data'].keys()) for row in data['messages'] ] expected_keys = set() for record in expected_records: expected_keys.update(record.keys()) # Verify schema covers all fields # BUG_1 | https://stitchdata.atlassian.net/browse/SRCE-4736 # to reproduce bug comment out the marked lines below streams_to_skip_schema_assertion = { # BUG_1 'customers', 'subscriptions', 'products', 'invoice_items', 'payouts', 'charges', 'subscription_items', 'invoices', 'plans', 'invoice_line_items' } schema_keys = set(self.expected_schema_keys(stream)) if stream not in streams_to_skip_schema_assertion: # BUG_1 self.assertEqual(set(), expected_keys.difference(schema_keys), msg="\tFields missing from schema!") # not a test, just logging the fields that are included in the schema but not in the expectations if schema_keys.difference(expected_keys): print( "WARNING Fields missing from expectations: {}".format( schema_keys.difference(expected_keys))) # Verify that all fields sent to the target fall into the expected schema for actual_keys in record_messages_keys: self.assertTrue( actual_keys.issubset(schema_keys), msg= "Expected all fields to be present, as defined by schemas/{}.json" .format(stream) + "EXPECTED (SCHEMA): {}\nACTUAL (REPLICATED KEYS): {}". format(schema_keys, actual_keys)) actual_records = [row['data'] for row in data['messages']] # Verify by pks, that we replicated the expected records and only the expected records self.assertPKsEqual(stream, expected_records, actual_records) expected_pks_to_record_dict = self.getPKsToRecordsDict( stream, expected_records) actual_pks_to_record_dict = self.getPKsToRecordsDict( stream, actual_records) if stream not in streams_to_skip_schema_assertion: # BUG_1 for pks_tuple, expected_record in expected_pks_to_record_dict.items( ): if expected_record.get( 'updated') is None and expected_record.get( 'created'): print( "WARNING adding 'updated' to new {} record for comparison" .format(stream)) expected_record['updated'] = expected_record[ 'created'] actual_record = actual_pks_to_record_dict.get( pks_tuple) self.assertDictEqual(expected_record, actual_record)
def manage_addOAIRecord(self, id, **kwargs): ob = create_object(self, OAIRecord, id) process_form(ob, IOAIRecord, kwargs) ob.initialize()
def test_run(self): """ Verify that the sync only sent records to the target for selected streams Update metadata[test] with a random number for each stream with event updates Verify that the second sync includes at least one update for each stream Verify that the second sync includes less records than the first sync Verify that the updated metadata was picked up on the second sync PREREQUISITE For EACH stream that gets updates through events stream, there's at least 1 row of data """ conn_id = connections.ensure_connection(self) event_update_streams = { # "balance_transactions" # Cannot be directly updated "charges", "coupons", "customers", # "disputes", # Cannot create directly with api "invoice_items", # "invoice_line_items", # Can't be updated via api "invoices", # "payout_transactions", # See bug in create_test "payouts", "plans", "products", # "subscription_items", # BUG_9916 | https://jira.talendforge.org/browse/TDL-9916 "subscriptions", # "transfers", # Cannot be updated directly via api } found_catalogs = self.run_and_verify_check_mode(conn_id) our_catalogs = [catalog for catalog in found_catalogs if catalog.get('tap_stream_id') in event_update_streams] self.select_all_streams_and_fields( conn_id, our_catalogs, select_all_fields=True ) # Ensure each stream under test has data to start new_objects = { stream: create_object(stream) for stream in event_update_streams } # Some streams will be updated implicitly streams_to_update = event_update_streams.difference({ "invoice_line_items", "subscription_items", }) # Run a sync job using orchestrator first_sync_record_count = self.run_and_verify_sync(conn_id) # verify that the sync only sent records to the target for selected streams (catalogs) self.assertEqual(set(first_sync_record_count.keys()), event_update_streams) # Get the set of records from a first sync first_sync_records = runner.get_records_from_target_output() first_sync_created, _ = self.split_records_into_created_and_updated( first_sync_records ) updated = {} # holds id for updated objects in each stream for stream in streams_to_update: # There needs to be some test data for each stream, otherwise this will break self.assertGreater(len(first_sync_created[stream]["messages"]), 0, msg='We did not get any new records from ' 'the first sync for {}'.format(stream)) record = first_sync_created[stream]["messages"][0]["data"] # We need to make sure the data actually changes, otherwise no event update # will get created update_object(stream, record["id"]) updated[stream] = record["id"] # Run a second sync job using orchestrator second_sync_record_count = self.run_and_verify_sync(conn_id) # Get the set of records from a second sync second_sync_records = runner.get_records_from_target_output() _, second_sync_updated = self.split_records_into_created_and_updated( second_sync_records ) # # THIS MAKES AN ASSUMPTION THAT CHILD STREAMS DO NOT NEED TESTING. # # ADJUST IF NECESSARY for stream in event_update_streams.difference(self.child_streams()): with self.subTest(stream=stream): # verify that there is more than 1 record of data - setup necessary self.assertGreater( first_sync_record_count.get(stream, 0), 1, msg="Data isn't set up to be able to test event updates", ) # verify that you get at least one updated record on the second sync self.assertGreaterEqual( len(second_sync_updated.get(stream, {}).get("messages", [])), 1, msg="second syc didn't have updates", ) # verify that you get less data the 2nd time around since only updates # should be picked up self.assertLess( second_sync_record_count.get(stream, 0), first_sync_record_count.get(stream, 0), msg="second syc had the same or more records", ) # verify all the updated records in the 2nd sync are different from # the first run first_data = next( record["data"] for record in first_sync_created.get(stream, {}).get("messages", []) if record.get("data", {}).get("id") == updated[stream] ) second_data = next( record["data"] for record in second_sync_updated.get(stream, {}).get( "messages", [] ) if record.get("data", {}).get("id") == updated[stream] ) # verify the updated timestamp is greater in the second sync self.assertGreater( second_data["updated"], first_data["updated"], "updated timestamp for second sync is not greater than first sync", ) # verify the metadata[test] value actually changed self.assertNotEqual( second_data["metadata"].get("test_value", 0), first_data["metadata"].get("test_value", 0), "the test metadata should be different", ) if stream in new_objects: delete_object(stream, new_objects[stream]["id"])
def do_test(self, conn_id): """ Verify that the sync only sent records to the target for selected streams Create a new object for each stream Verify that the second sync includes at least one create for each stream Verify that the created record was picked up on the second sync """ streams_to_create = { "balance_transactions", # should be created implicity with a create in the payouts or charges streams "charges", "coupons", "customers", "invoice_items", "invoice_line_items", # this is created implicity by invoices, it just creates another invoice TODO get this outa here "invoices", # this will create an invoice_item "payouts", "plans", "products", "subscription_items", "subscriptions", # this will create a new plan and payment method } missing_streams_to_create = { "disputes", # can be created by simulating a dispute transaction with a specific card number # no way to create directly, see: https://stripe.com/docs/testing#disputes "payout_transactions", # BUG (https://stitchdata.atlassian.net/browse/SUP-1294) # depends on payouts and transactions "transfers", # needs an account that we can transfer to, not sure # how to set up a test account we can use to create a transfer } our_catalogs = get_catalogs(conn_id, streams_to_create) self.select_all_streams_and_fields(conn_id, our_catalogs, select_all_fields=True) # Run a sync job using orchestrator first_sync_record_count = self.run_sync(conn_id) # verify that the sync only sent records to the target for selected streams (catalogs) self.assertEqual(set(first_sync_record_count.keys()), streams_to_create) # Get the set of records from a first sync first_sync_records = runner.get_records_from_target_output() first_sync_created, _ = self.split_records_into_created_and_updated( first_sync_records) new_objects = { stream: create_object(stream) for stream in streams_to_create.difference( {"balance_transactions"}) } # Run a second sync job using orchestrator second_sync_record_count = self.run_sync(conn_id) # Get the set of records from a second sync second_sync_records = runner.get_records_from_target_output() second_sync_created, _ = self.split_records_into_created_and_updated( second_sync_records) # # THIS MAKES AN ASSUMPTION THAT CHILD STREAMS DO NOT NEED TESTING. # # ADJUST IF NECESSARY for stream in streams_to_create.difference(self.child_streams()): with self.subTest(stream=stream): second_sync_created_objects = second_sync_created.get( stream, {}).get("messages", []) # verify that you get at least one new record on the second sync self.assertGreaterEqual( len(second_sync_created_objects), 1, msg="second sync didn't have created objects", ) if stream == "balance_transactions": sources = [ record.get("data", {}).get("source") for record in second_sync_created_objects ] self.assertTrue(new_objects['payouts']['id'] in sources) self.assertTrue(new_objects['charges']['id'] in sources) continue # verify the new object is in the list of created objects # from the second sync self.assertTrue( any(new_objects[stream]["id"] == record.get("data", {}).get("id") for record in second_sync_created_objects)) if stream in streams_to_create: delete_object(stream, new_objects[stream]["id"])
async def create_mptcp_proxy(data_store, amqp, **kwargs): ret, mptcp_proxy = utils.create_object(data_store, amqp, Mptcp_proxy_schema, kwargs) await send_create_proxy(data_store, amqp, mptcp_proxy) raise web.HTTPAccepted(content_type="application/json", text=ret)
def test_run(self): """ Verify for each stream that you can do a sync which records bookmarks. Verify that the bookmark is the max value sent to the target for the `date` PK field Verify that the 2nd sync respects the bookmark Verify that all data of the 2nd sync is >= the bookmark from the first sync Verify that the number of records in the 2nd sync is less then the first Verify inclusivivity of bookmarks PREREQUISITE For EACH stream that is incrementally replicated there are multiple rows of data with different values for the replication key """ untested_streams = self.child_streams().union({ 'transfers', 'payout_transactions', # BUG see create test 'balance_transactions', # join stream, can't be updated 'disputes', }) cannot_update_streams = { 'invoice_line_items', # updates not available via api } # Ensure tested streams have existing records expected_records_first_sync = {stream: [] for stream in self.streams_to_create} for _ in range(2): # create 3 records for each stream but only expect the 3rd for stream in self.streams_to_create: self.new_objects[stream].append(create_object(stream)) for stream in self.streams_to_create: self.new_objects[stream].append(create_object(stream)) expected_records_first_sync[stream].append({"id": self.new_objects[stream][-1]['id']}) self.START_DATE = self.get_properties().get('start_date') # Instantiate connection with default start conn_id = connections.ensure_connection(self) # run in check mode found_catalogs = self.run_and_verify_check_mode(conn_id) # Select all testable streams and all fields within streams streams_to_select = self.expected_incremental_streams().difference(untested_streams) our_catalogs = [catalog for catalog in found_catalogs if catalog.get('tap_stream_id') in streams_to_select] self.select_all_streams_and_fields(conn_id, our_catalogs, select_all_fields=True) # Run a sync job using orchestrator first_sync_start = self.local_to_utc(dt.utcnow()) first_sync_record_count = self.run_and_verify_sync(conn_id) first_sync_end = self.local_to_utc(dt.utcnow()) # verify that the sync only sent records to the target for selected streams (catalogs) self.assertEqual( streams_to_select, set(first_sync_record_count.keys()), msg="Expected only testable streams to be replicated: {}".format(first_sync_record_count) ) first_sync_state = menagerie.get_state(conn_id) # Get the set of records from a first sync first_sync_records = runner.get_records_from_target_output() # Add data before next sync via insert and update, and set expectations created_records = {x: [] for x in self.expected_streams()} updated_records = {x: [] for x in self.expected_streams()} expected_records_second_sync = {x: [] for x in self.expected_streams()} # Update one record from each stream prior to 2nd sync first_sync_created, _ = self.split_records_into_created_and_updated(first_sync_records) for stream in self.streams_to_create.difference(cannot_update_streams): # There needs to be some test data for each stream, otherwise this will break record = expected_records_first_sync[stream][0] updated_record = update_object(stream, record["id"]) updated_records[stream].append(updated_record) expected_records_second_sync[stream].append({"id": updated_record['id']}) # Ensure different times between udpates and inserts sleep(2) # Insert (create) one record for each stream prior to 2nd sync for stream in self.streams_to_create: created_record = create_object(stream) self.new_objects[stream].append(created_record) created_records[stream].append(created_record) expected_records_second_sync[stream].append({"id": created_record['id']}) # ensure validity of expected_records_second_sync for stream in self.streams_to_create: if stream in self.expected_incremental_streams(): if stream in cannot_update_streams: # Some streams will have only 1 record from the Insert self.assertEqual(1, len(expected_records_second_sync.get(stream)), msg="Expectations are invalid for incremental stream {}".format(stream) ) continue # Most streams will have 2 records from the Update and Insert self.assertEqual(2, len(expected_records_second_sync.get(stream)), msg="Expectations are invalid for incremental stream {}".format(stream) ) elif stream in self.expected_full_table_streams(): self.assertEqual( len(expected_records_second_sync.get(stream)), len(expected_records_first_sync.get(stream)) + len(created_records[stream]), msg="Expectations are invalid for full table stream {}".format(stream) ) # created_records[stream] = self.records_data_type_conversions(created_records.get(stream)) # updated_records[stream] = self.records_data_type_conversions(updated_records.get(stream)) # Run a second sync job using orchestrator second_sync_start = self.local_to_utc(dt.utcnow()) second_sync_record_count = self.run_and_verify_sync(conn_id) second_sync_end = self.local_to_utc(dt.utcnow()) second_sync_state = menagerie.get_state(conn_id) # Get the set of records from a second sync second_sync_records = runner.get_records_from_target_output() second_sync_created, second_sync_updated = self.split_records_into_created_and_updated(second_sync_records) # Loop first_sync_records and compare against second_sync_records for stream in self.streams_to_create.difference(untested_streams): with self.subTest(stream=stream): second_sync_data = [record.get("data") for record in second_sync_records.get(stream, {}).get("messages", [])] stream_replication_keys = self.expected_replication_keys() stream_primary_keys = self.expected_primary_keys() # TESTING INCREMENTAL STREAMS if stream in self.expected_incremental_streams(): replication_keys = stream_replication_keys.get(stream) # Verify both syncs write / keep the same bookmark keys self.assertEqual(set(first_sync_state.get('bookmarks', {}).keys()), set(second_sync_state.get('bookmarks', {}).keys())) # verify that there is more than 1 record of data - setup necessary self.assertGreater(first_sync_record_count.get(stream, 0), 1, msg="Data isn't set up to be able to test full sync") # verify that you get less data on the 2nd sync self.assertGreater( first_sync_record_count.get(stream, 0), second_sync_record_count.get(stream, 0), msg="first sync didn't have more records, bookmark usage not verified") if stream in self.streams_to_create: for replication_key in replication_keys: updates_replication_key = "updates_created" updates_stream = stream + "_events" # Verify second sync's bookmarks move past the first sync's self.assertGreater( second_sync_state.get('bookmarks', {updates_stream: {}}).get( updates_stream, {replication_key: -1}).get(updates_replication_key), first_sync_state.get('bookmarks', {updates_stream: {}}).get( updates_stream, {updates_replication_key: -1}).get(updates_replication_key) ) # Verify that all data of the 2nd sync is >= the bookmark from the first sync first_sync_bookmark = dt.fromtimestamp( first_sync_state.get('bookmarks').get(updates_stream).get(updates_replication_key) ) for record in second_sync_data: date_value = record["updated"] self.assertGreaterEqual(date_value, dt.strftime(first_sync_bookmark, self.COMPARISON_FORMAT), msg="A 2nd sync record has a replication-key that is less than or equal to the 1st sync bookmark.") elif stream in self.expected_full_table_streams(): raise Exception("Expectations changed, but this test was not updated to reflect them.") # TESTING APPLICABLE TO ALL STREAMS # Verify that the expected records are replicated in the 2nd sync # For incremental streams we should see at least 2 records (a new record and an updated record) # but we may see more as the bookmmark is inclusive and there are hidden creates/updates due to # dependencies between streams. # For full table streams we should see 1 more record than the first sync expected_records = expected_records_second_sync.get(stream) primary_keys = stream_primary_keys.get(stream) updated_pk_values = {tuple([record.get(pk) for pk in primary_keys]) for record in updated_records[stream]} self.assertLessEqual( len(expected_records), len(second_sync_data), msg="Expected number of records are not less than or equal to actual for 2nd sync.\n" + "Expected: {}\nActual: {}".format(len(expected_records), len(second_sync_data)) ) if (len(second_sync_data) - len(expected_records)) > 0: logging.warn('Second sync replicated %s records more than our create and update for %s', len(second_sync_data), stream) if not primary_keys: raise NotImplementedError("PKs are needed for comparing records") # Verify that the inserted and updated records are replicated by the 2nd sync for expected_record in expected_records: expected_pk_value = expected_record.get('id') sync_pk_values = [sync_record.get('id') for sync_record in second_sync_data if sync_record.get('id') == expected_pk_value] self.assertTrue( len(sync_pk_values) > 0, msg="A record is missing from our sync: \nSTREAM: {}\tPK: {}".format(stream, expected_pk_value) ) self.assertIn(expected_pk_value, sync_pk_values) # Verify updated fields are replicated as expected for updated_record in updated_records[stream]: expected_updated_key = 'metadata' expected_updated_value_substring = 'bob' updated_pk_value = updated_record.get('id') sync_records_metadata = [sync_record.get('metadata') for sync_record in second_sync_data if sync_record.get('id') == updated_pk_value] self.assertTrue(len(sync_records_metadata) == 1) self.assertIn(expected_updated_value_substring, sync_records_metadata[0].get('test_value'))
async def create_expansion(data_store, amqp, **kwargs): ret, expansion = utils.create_object(data_store, amqp, Expansion_schema, kwargs) await send_create_expansion(data_store, amqp, expansion) raise web.HTTPAccepted(content_type="application/json", text=ret)
async def create_ipsec_policy(data_store, amqp, **kwargs): ret, _ = utils.create_object(data_store, amqp, Ipsec_policy_schema, kwargs) raise web.HTTPCreated(content_type="application/json", text=ret)
async def create_l2_tunnel(data_store, amqp, **kwargs): ret, l2_tunnel = utils.create_object(data_store, amqp, L2_tunnel_schema, kwargs) await send_create_tunnel(data_store, amqp, l2_tunnel) raise web.HTTPAccepted(content_type="application/json", text=ret)
def test_run(self): """ Verify that for each stream you can get multiple pages of data when no fields are selected and only the automatic fields are replicated. PREREQUISITE For EACH stream add enough data that you surpass the limit of a single fetch of data. For instance if you have a limit of 250 records ensure that 251 (or more) records have been posted for that stream. """ conn_id = connections.ensure_connection(self) streams_to_create = { # "balance_transactions", # should be created implicity with a create in the payouts or charges streams "charges", "coupons", "customers", "invoice_items", "invoice_line_items", # this is created implicity by invoices, it just creates another invoice "invoices", # this will create an invoice_item "payouts", "plans", "products", "subscription_items", "subscriptions", # this will create a new plan and payment method } untested_streams = {"disputes", "transfers", "payout_transactions"} new_objects = { stream: create_object(stream) for stream in streams_to_create.difference() } # Select all streams and no fields within streams # IF THERE ARE NO AUTOMATIC FIELDS FOR A STREAM # WE WILL NEED TO UPDATE THE BELOW TO SELECT ONE found_catalogs = self.run_and_verify_check_mode(conn_id) self.select_all_streams_and_fields(conn_id, found_catalogs, select_all_fields=False) # Run a sync job using orchestrator record_count_by_stream = self.run_and_verify_sync(conn_id) actual_fields_by_stream = runner.examine_target_output_for_fields() for stream in self.expected_streams().difference(untested_streams): with self.subTest(stream=stream): # verify that you get some records for each stream # SKIP THIS ASSERTION FOR STREAMS WHERE YOU CANNOT GET # MORE THAN 1 PAGE OF DATA IN THE TEST ACCOUNT self.assertGreater( record_count_by_stream.get(stream, -1), 0, msg="The number of records is not over the stream max limit" ) # verify that only the automatic fields are sent to the target actual = actual_fields_by_stream.get(stream) or set() expected = self.expected_automatic_fields().get(stream, set()) self.assertEqual( actual, expected, msg= ("The fields sent to the target are not the automatic fields. Expected: {}, Actual: {}" .format(actual, expected)))
def do_test(self, conn_id): """Test we get a lot of data back based on the start date configured in base""" # Select all streams and all fields within streams found_catalogs = menagerie.get_catalogs(conn_id) incremental_streams = { key for key, value in self.expected_replication_method().items() if value == self.INCREMENTAL } # IF THERE ARE STREAMS THAT SHOULD NOT BE TESTED # REPLACE THE EMPTY SET BELOW WITH THOSE STREAMS untested_streams = self.child_streams().union( {'disputes', 'events', 'transfers', 'payout_transactions'}) our_catalogs = get_catalogs( conn_id, incremental_streams.difference(untested_streams)) self.select_all_streams_and_fields(conn_id, our_catalogs, select_all_fields=True) # Create a record for each stream under test prior to the first sync new_objects = { stream: create_object(stream) for stream in incremental_streams.difference(untested_streams) } # Run a sync job using orchestrator first_sync_record_count = self.run_sync(conn_id) first_total_records = reduce(lambda a, b: a + b, first_sync_record_count.values()) # Count actual rows synced first_sync_records = runner.get_records_from_target_output() # set the start date for a new connection based off bookmarks largest value first_max_bookmarks = self.max_bookmarks_by_stream(first_sync_records) bookmark_list = [ next(iter(book.values())) for stream, book in first_max_bookmarks.items() ] bookmark_dates = [] for bookmark in bookmark_list: try: bookmark_dates.append(parse(bookmark)) except (ValueError, OverflowError, TypeError): pass if not bookmark_dates: # THERE WERE NO BOOKMARKS THAT ARE DATES. # REMOVE CODE TO FIND A START DATE AND ENTER ONE MANUALLY raise ValueError # largest_bookmark = reduce(lambda a, b: a if a > b else b, bookmark_dates) # self.start_date = self.local_to_utc(largest_bookmark).strftime(self.START_DATE_FORMAT) self.start_date = dt.strftime(dt.today() - timedelta(days=1), self.START_DATE_FORMAT) # create a new connection with the new start_date conn_id = self.create_connection(original_properties=False) # Select all streams and all fields within streams found_catalogs = menagerie.get_catalogs(conn_id) our_catalogs = [ catalog for catalog in found_catalogs if catalog.get('tap_stream_id') in incremental_streams.difference( untested_streams) ] self.select_all_streams_and_fields(conn_id, our_catalogs, select_all_fields=True) # TODO remove the updates, this is unnecessary. Verify with Harvest # Update a record for each stream under test prior to the 2nd sync first_sync_created, _ = self.split_records_into_created_and_updated( first_sync_records) updated = {} # holds id for updated objects in each stream for stream in new_objects: # There needs to be some test data for each stream, otherwise this will break record = first_sync_created[stream]["messages"][0]["data"] update_object(stream, record["id"]) updated[stream] = record["id"] # Run a sync job using orchestrator second_sync_record_count = self.run_sync(conn_id) # tap-stripe uses events for updates, so these need filtered to validate bookmark second_sync_records = runner.get_records_from_target_output() second_sync_created, second_sync_updated = self.split_records_into_created_and_updated( second_sync_records) second_total_records = reduce(lambda a, b: a + b, second_sync_record_count.values(), 0) # Only examine bookmarks for "created" objects, not updates second_min_bookmarks = self.min_bookmarks_by_stream( second_sync_created) # verify that at least one record synced and less records synced than the 1st connection self.assertGreater(second_total_records, 0) self.assertLess(first_total_records, second_total_records) # validate that all newly created records are greater than the start_date for stream in incremental_streams.difference(untested_streams): with self.subTest(stream=stream): # verify that each stream has less records in the first sync than the second self.assertGreater( second_sync_record_count.get(stream, 0), first_sync_record_count.get(stream, 0), msg="first had more records, start_date usage not verified" ) # verify all data from 2nd sync >= start_date target_mark = second_min_bookmarks.get(stream, {"mark": None}) target_value = next(iter( target_mark.values())) # there should be only one if target_value: # it's okay if there isn't target data for a stream try: target_value = self.local_to_utc(parse(target_value)) expected_value = self.local_to_utc( parse(self.start_date)) # verify that the minimum bookmark sent to the target for the second sync # is greater than or equal to the start date self.assertGreaterEqual(target_value, expected_value) except (OverflowError, ValueError, TypeError): print("bookmarks cannot be converted to dates, " "can't test start_date for {}".format(stream)) if stream in updated: delete_object(stream, updated[stream])
async def create_network(data_store, amqp, **kwargs): ret, _ = utils.create_object(data_store, amqp, Network_schema, kwargs) raise web.HTTPOk(content_type="application/json", text=ret)