def setUp(self): ensure_environment_variables_set() with get_test_connection() as client: ############# Drop all dbs/collections ############# drop_all_collections(client) ############# Add simple collections ############# # simple_coll_1 has 50 documents] client["simple_db"]["simple_coll_1"].insert_many(generate_simple_coll_docs(50)) # simple_coll_2 has 100 documents client["simple_db"]["simple_coll_2"].insert_many(generate_simple_coll_docs(100)) ############# Add Index on date_field ############ client["simple_db"]["simple_coll_1"].create_index([("date_field", pymongo.ASCENDING)]) client["simple_db"]["simple_coll_2"].create_index([("date_field", pymongo.ASCENDING)]) # Add simple_coll per key type for key_name in self.key_names(): client["simple_db"]["simple_coll_{}".format(key_name)].insert_many(generate_simple_coll_docs(50)) # add index on field client["simple_db"]["simple_coll_{}".format(key_name)].create_index([(key_name, pymongo.ASCENDING)])
def setUp(self): if not all([ x for x in [ os.getenv('TAP_MONGODB_HOST'), os.getenv('TAP_MONGODB_USER'), os.getenv('TAP_MONGODB_PASSWORD'), os.getenv('TAP_MONGODB_PORT'), os.getenv('TAP_MONGODB_DBNAME') ] ]): #pylint: disable=line-too-long raise Exception( "set TAP_MONGODB_HOST, TAP_MONGODB_USER, TAP_MONGODB_PASSWORD, TAP_MONGODB_PORT, TAP_MONGODB_DBNAME" ) with get_test_connection() as client: ############# Drop all dbs/collections ############# drop_all_collections(client) ############# Add simple collections ############# # simple_coll_1 has 50 documents client["simple_db"]["simple_coll_1"].insert_many( generate_simple_coll_docs(50)) # simple_coll_2 has 100 documents client["simple_db"]["simple_coll_2"].insert_many( generate_simple_coll_docs(100))
def setUp(self): if not all([x for x in [os.getenv('TAP_MONGODB_HOST'), os.getenv('TAP_MONGODB_USER'), os.getenv('TAP_MONGODB_PASSWORD'), os.getenv('TAP_MONGODB_PORT'), os.getenv('TAP_MONGODB_DBNAME')]]): #pylint: disable=line-too-long raise Exception("set TAP_MONGODB_HOST, TAP_MONGODB_USER, TAP_MONGODB_PASSWORD, TAP_MONGODB_PORT, TAP_MONGODB_DBNAME") with get_test_connection() as client: ############# Drop all dbs/collections ############# drop_all_collections(client) ############# Add datatype collections ############# pattern = re.compile('.*') regex = bson.Regex.from_native(pattern) regex.flags ^= re.UNICODE datatype_doc = { "double_field": 4.3, "string_field": "a sample string", "object_field" : { "obj_field_1_key": "obj_field_1_val", "obj_field_2_key": "obj_field_2_val" }, "array_field" : [ "array_item_1", "array_item_2", "array_item_3" ], "binary_data_field" : bson.Binary(b"a binary string"), "object_id_field": bson.objectid.ObjectId(b'123456789123'), "boolean_field" : True, "date_field" : datetime.datetime(2019, 8, 15, 19, 29, 14, 578000), "null_field": None, "regex_field" : regex, "32_bit_integer_field" : 32, "timestamp_field" : bson.timestamp.Timestamp(1565897157, 1), "64_bit_integer_field" : 34359738368, "decimal_field" : bson.Decimal128(decimal.Decimal('1.34')), "javaScript_field" : bson.code.Code("var x, y, z;"), "javaScript_with_scope_field" : bson.code.Code("function incrementX() { x++; }", scope={"x": 1}), "min_key_field" : bson.min_key.MinKey, "max_key_field" : bson.max_key.MaxKey, "uuid_field": uuid.UUID('3e139ff5-d622-45c6-bf9e-1dfec72820c4'), "dbref_field": bson.dbref.DBRef("some_collection", bson.objectid.ObjectId(b'123456789123'), database='some_database') } client["datatype_db"]["datatype_coll_1"].insert_one(datatype_doc) # NB: Insert an invalid datetime to confirm that works correctly run_mongodb_javascript("datatype_db", "db.invalid_datatype_coll.insert({ \"date_field\": new ISODate(\"0000-01-01T00:00:00.000Z\") });")
def setUp(self): ensure_environment_variables_set() with get_test_connection() as client: ############# Drop all dbs/collections ############# drop_all_collections(client) ############# Add simple collections ############ # simple_coll_1 has 50 documents client["simple_db"]["simple_coll_1"].insert_many( generate_simple_coll_docs(50))
def setUp(self): ensure_environment_variables_set() with get_test_connection() as client: # drop all dbs/collections drop_all_collections(client) # simple_coll_1 has 50 documents, id is an integer instead of ObjectId client["simple_db"]["simple_coll_1"].insert_many(generate_simple_coll_docs(50)) # simple_coll_2 has 100 documents, id is an integer instead of ObjectId client["simple_db"]["simple_coll_2"].insert_many(generate_simple_binary_coll_docs(50))
def setUp(self): ensure_environment_variables_set() with get_test_connection() as client: drop_all_collections(client) # simple_coll_1 has 50 documents client["simple_db"]["simple_coll_1"].insert_many( generate_simple_coll_docs(50)) # simple_coll_2 has 100 documents client["simple_db"]["simple_coll_2"].insert_many( generate_simple_coll_docs(100))
def setUp(self): if not all([ x for x in [ os.getenv('TAP_MONGODB_HOST'), os.getenv('TAP_MONGODB_USER'), os.getenv('TAP_MONGODB_PASSWORD'), os.getenv('TAP_MONGODB_PORT'), os.getenv('TAP_MONGODB_DBNAME') ] ]): #pylint: disable=line-too-long raise Exception( "set TAP_MONGODB_HOST, TAP_MONGODB_USER, TAP_MONGODB_PASSWORD, TAP_MONGODB_PORT, TAP_MONGODB_DBNAME" ) with get_test_connection() as client: ############# Drop all dbs/collections ############# drop_all_collections(client) ############# Add simple collections ############# # simple_coll_1 has 50 documents client["simple_db"]["simple_coll_1"].insert_many( generate_simple_coll_docs(50)) # simple_coll_2 has 100 documents client["simple_db"]["simple_coll_2"].insert_many( generate_simple_coll_docs(100)) ############# Add Index on date_field ############ client["simple_db"]["simple_coll_1"].create_index([ ("date_field", pymongo.ASCENDING) ]) client["simple_db"]["simple_coll_2"].create_index([ ("date_field", pymongo.ASCENDING) ]) # Add simple_coll per key type for key_name in self.key_names(): client["simple_db"]["simple_coll_{}".format( key_name)].insert_many(generate_simple_coll_docs(50)) # add index on field client["simple_db"]["simple_coll_{}".format( key_name)].create_index([(key_name, pymongo.ASCENDING)])
def setUp(self): if not all([x for x in [os.getenv('TAP_MONGODB_HOST'), os.getenv('TAP_MONGODB_USER'), os.getenv('TAP_MONGODB_PASSWORD'), os.getenv('TAP_MONGODB_PORT'), os.getenv('TAP_MONGODB_DBNAME')]]): #pylint: disable=line-too-long raise Exception("set TAP_MONGODB_HOST, TAP_MONGODB_USER, TAP_MONGODB_PASSWORD, TAP_MONGODB_PORT, TAP_MONGODB_DBNAME") with get_test_connection() as client: # drop all dbs/collections drop_all_collections(client) # simple_coll_1 has 50 documents client["simple_db"]["simple_coll_1"].insert_many(generate_simple_coll_docs(50)) # create view on simple_coll_1 client["simple_db"].command(bson.son.SON([("create", "simple_view_1"), ("viewOn", "simple_coll_1"), ("pipeline", [])])) # simple_coll_2 has 100 documents client["simple_db"]["simple_coll_2"].insert_many(generate_simple_coll_docs(100)) # admin_coll_1 has 50 documents client["admin"]["admin_coll_1"].insert_many(generate_simple_coll_docs(50)) # simple_coll_3 is an empty collection client["simple_db"].create_collection("simple_coll_3") # simple_coll_4 has documents with special chars and a lot of nesting client["simple_db"]["simple_coll_4"].insert_one({"hebrew_ישרא": "hebrew_ישרא"}) client["simple_db"]["simple_coll_4"].insert_one({"hebrew_ישרא": 2}) client["simple_db"]["simple_coll_4"].insert_one({"another_hebrew_ישראל": "another_hebrew_ישרא"}) nested_doc = {"field0": {}} current_doc = nested_doc for i in range(1, 101): current_doc["field{}".format(i-1)]["field{}".format(i)] = {} current_doc = current_doc["field{}".format(i-1)] current_doc["field100"] = "some_value" client["simple_db"]["simple_coll_4"].insert_one(nested_doc) max_col_doc = {} for x in range(1600): max_col_doc['col_{}'.format(x)] = x client["simple_db"]["simple_coll_4"].insert_one(max_col_doc)
def setUp(self): ensure_environment_variables_set() with get_test_connection() as client: # drop all dbs/collections drop_all_collections(client) # simple_coll_1 has 50 documents client["simple_db"]["simple_coll_1"].insert_many(generate_simple_coll_docs(50)) # create view on simple_coll_1 client["simple_db"].command(bson.son.SON([("create", "simple_view_1"), ("viewOn", "simple_coll_1"), ("pipeline", [])])) # simple_coll_2 has 100 documents client["simple_db"]["simple_coll_2"].insert_many(generate_simple_coll_docs(100)) # admin_coll_1 has 50 documents client["admin"]["admin_coll_1"].insert_many(generate_simple_coll_docs(50)) # simple_coll_3 is an empty collection client["simple_db"].create_collection("simple_coll_3") # simple_coll_4 has documents with special chars and a lot of nesting client["simple_db"]["simple_coll_4"].insert_one({"hebrew_ישרא": "hebrew_ישרא"}) client["simple_db"]["simple_coll_4"].insert_one({"hebrew_ישרא": 2}) client["simple_db"]["simple_coll_4"].insert_one({"another_hebrew_ישראל": "another_hebrew_ישרא"}) nested_doc = {"field0": {}} current_doc = nested_doc for i in range(1, 101): current_doc["field{}".format(i-1)]["field{}".format(i)] = {} current_doc = current_doc["field{}".format(i-1)] current_doc["field100"] = "some_value" client["simple_db"]["simple_coll_4"].insert_one(nested_doc) max_col_doc = {} for x in range(1600): max_col_doc['col_{}'.format(x)] = x client["simple_db"]["simple_coll_4"].insert_one(max_col_doc)
def setUp(self): if not all([ x for x in [ os.getenv('TAP_MONGODB_HOST'), os.getenv('TAP_MONGODB_USER'), os.getenv('TAP_MONGODB_PASSWORD'), os.getenv('TAP_MONGODB_PORT'), os.getenv('TAP_MONGODB_DBNAME') ] ]): #pylint: disable=line-too-long raise Exception( "set TAP_MONGODB_HOST, TAP_MONGODB_USER, TAP_MONGODB_PASSWORD, TAP_MONGODB_PORT, TAP_MONGODB_DBNAME" ) with get_test_connection() as client: # drop all dbs/collections drop_all_collections(client) # simple_coll_1 has 50 documents client["simple_db"]["simple_coll_1"].insert_many( generate_simple_coll_docs(50)) # simple_coll_2 has 100 documents client["simple_db"]["simple_coll_2"].insert_many( generate_simple_coll_docs(100)) # admin_coll_1 has 50 documents client["admin"]["admin_coll_1"].insert_many( generate_simple_coll_docs(50)) # create view on simple_coll_1 client["simple_db"].command( bson.son.SON([("create", "simple_view_1"), ("viewOn", "simple_coll_1"), ("pipeline", [])])) # collections with same names as others in different dbs client["simple_db_2"]["simple_coll_1"].insert_many( generate_simple_coll_docs(50)) client["simple_db_2"]["SIMPLE_COLL_1"].insert_many( generate_simple_coll_docs(50)) # collections with special characters in names client["special_db"]["hebrew_ישראל"].insert_many( generate_simple_coll_docs(50)) client['special_db']['hello!world?'].insert_many( generate_simple_coll_docs(50)) # Add datatype collections pattern = re.compile('.*') regex = bson.Regex.from_native(pattern) regex.flags ^= re.UNICODE datatype_doc = { "double_field": 4.3, "string_field": "a sample string", "object_field": { "obj_field_1_key": "obj_field_1_val", "obj_field_2_key": "obj_field_2_val" }, "array_field": ["array_item_1", "array_item_2", "array_item_3"], "binary_data_field": b"a binary string", "object_id_field": bson.objectid.ObjectId(b'123456789123'), "boolean_field": True, "date_field": datetime.datetime.now(), "null_field": None, "regex_field": regex, "32_bit_integer_field": 32, "timestamp_field": bson.timestamp.Timestamp(int(time.time()), 1), "64_bit_integer_field": 34359738368, "decimal_field": bson.Decimal128(decimal.Decimal('1.34')), "javaScript_field": bson.code.Code("var x, y, z;"), "javaScript_with_scope_field": bson.code.Code("function incrementX() { x++; }", scope={"x": 1}), "min_key_field": bson.min_key.MinKey, "max_key_field": bson.max_key.MaxKey } client["datatype_db"]["datatype_coll_1"].insert_one(datatype_doc)
def test_run(self): ensure_environment_variables_set() with get_test_connection() as client: # drop all dbs/collections drop_all_collections(client) # Create session 1 and insert docs to simple_coll_1 & simple_coll_2 ################# # Session 1 ################# session1 = client.start_session() session1.start_transaction() # simple_coll_1 has 10 documents client["simple_db"]["simple_coll_1"].insert_many( generate_simple_coll_docs(10)) # simple_coll_2 has 20 documents client["simple_db"]["simple_coll_2"].insert_many( generate_simple_coll_docs(20)) session1.commit_transaction() # Create session 2 ''' create empty collection update documents in simple_coll_1 & simple_coll_2 and tie to session 2 insert documents in simple_coll_3 and tie to session 2 execute the sync with uncommitted changes validate that the uncommitted changes are not replicated by the sync ''' ################ # Session 2 ################ session2 = client.start_session() session2.start_transaction() # simple_coll_3 is an empty collection client["simple_db"].create_collection("simple_coll_3") # update document from coll 1 and coll 2 client["simple_db"]["simple_coll_1"].update_one( {"int_field": 5}, {"$set": { "int_field": 11 }}, session=session2) client["simple_db"]["simple_coll_2"].update_one( {"int_field": 10}, {"$set": { "int_field": 21 }}, session=session2) # insert document to coll 3 client["simple_db"]["simple_coll_3"].insert_many( generate_simple_coll_docs(5), session=session2) # deletes do not matter in incremental replication, invalid scenario to test conn_id = connections.ensure_connection(self) # run in discovery mode check_job_name = runner.run_check_mode(self, conn_id) # verify check exit codes exit_status = menagerie.get_exit_status(conn_id, check_job_name) menagerie.verify_check_exit_status(self, exit_status, check_job_name) # verify the tap discovered the right streams found_catalogs = menagerie.get_catalogs(conn_id) # assert we find the correct streams which includes all collections which are part of session1 and session2 self.assertEqual(self.expected_check_streams_sync_1(), {c['tap_stream_id'] for c in found_catalogs}) # Select streams and add replication method metadata for stream_catalog in found_catalogs: annotated_schema = menagerie.get_annotated_schema( conn_id, stream_catalog['stream_id']) additional_md = [{ "breadcrumb": [], "metadata": { 'replication-method': 'INCREMENTAL', 'replication_key': 'int_field' } }] selected_metadata = connections.select_catalog_and_fields_via_metadata( conn_id, stream_catalog, annotated_schema, additional_md) # run full table sync sync_1 = runner.run_sync_mode(self, conn_id) # check exit status exit_status = menagerie.get_exit_status(conn_id, sync_1) menagerie.verify_sync_exit_status(self, exit_status, sync_1) # streams that we synced are the ones that we expect to see records_by_stream = runner.get_records_from_target_output() record_count_by_stream = runner.examine_target_output_file( self, conn_id, self.expected_sync_streams_1(), self.expected_pks_1()) # validate the record count in collections which are part of session1 and session2, should not read updates on coll 1 and coll 2 and insert on coll 3. Because the transaction is not committed self.assertEqual(self.expected_row_counts_sync_1(), record_count_by_stream) # validate there are no duplicates replicated as part of sync1 records_2 = {} pk_dict_2 = {} for stream in self.expected_sync_streams_1(): records_2[stream] = [ x for x in records_by_stream[stream]['messages'] if x.get('action') == 'upsert' ] pk_2 = [] for record in range(len(records_2[stream])): pk_2.append(records_2[stream][record]['data']['int_field']) pk_dict_2[stream] = pk_2 self.assertEqual(self.expected_pk_values_2(), pk_dict_2) session2.commit_transaction() # Create session 3 ''' Execute another sync Validate that the documents committed as part of session 2 should now be replicated in sync_2 ''' ################ # Session 3 ################ session3 = client.start_session() session3.start_transaction() # Run 2nd sync # run in discovery mode sync_2 = runner.run_sync_mode(self, conn_id) exit_status_2 = menagerie.get_exit_status(conn_id, sync_2) menagerie.verify_sync_exit_status(self, exit_status_2, sync_2) records_by_stream_2 = runner.get_records_from_target_output() record_count_by_stream_2 = runner.examine_target_output_file( self, conn_id, self.expected_sync_streams_2(), self.expected_pks_2()) # validate that we see the updates to coll 1 and coll 2 and insert to coll 3 in the 2nd sync # we see 2 records for coll 1 and coll 2, 1 record for update and the other record for the bookmarked record self.assertEqual(self.expected_row_counts_sync_2(), record_count_by_stream_2) # validate there are no duplicates replicated as part of sync1 records_3 = {} pk_dict_3 = {} for stream in self.expected_sync_streams_1(): records_3[stream] = [ x for x in records_by_stream_2[stream]['messages'] if x.get('action') == 'upsert' ] pk_3 = [] for record in range(len(records_3[stream])): pk_3.append(records_3[stream][record]['data']['int_field']) pk_dict_3[stream] = pk_3 self.assertEqual(self.expected_pk_values_3(), pk_dict_3) # Test case to validate tap behaviour when we delete bookmarked document and run sync state_2 = menagerie.get_state(conn_id) for stream in self.expected_check_streams_sync_1(): rep_key_value = state_2['bookmarks'][stream][ 'replication_key_value'] if stream == 'simple_db-simple_coll_1': collection = 'simple_coll_1' elif stream == 'simple_db-simple_coll_2': collection = 'simple_coll_2' elif stream == 'simple_db-simple_coll_3': collection = 'simple_coll_3' client["simple_db"][collection].delete_one( {"int_field": int(rep_key_value)}, session=session3) session3.commit_transaction() ''' Execute the sync, after the commit on session 3 Session 3 commits includes deleting the bookmarked value in each of the collection Validate the state does not change after deleting the bookmarked value Validate that the sync does not replicate any documents ''' state_3 = menagerie.get_state(conn_id) sync_3 = runner.run_sync_mode(self, conn_id) exit_status_3 = menagerie.get_exit_status(conn_id, sync_3) menagerie.verify_sync_exit_status(self, exit_status_3, sync_3) records_by_stream_3 = runner.get_records_from_target_output() record_count_by_stream_3 = runner.examine_target_output_file( self, conn_id, self.expected_sync_streams_2(), self.expected_pks_2()) # validate that we see 0 records being replicated because we deleted the bookmark value on each of the collection self.assertEqual(self.expected_row_counts_sync_3(), record_count_by_stream_3) # validate that the state value has not changed after deleting the bookmarked value in each collection self.assertEqual(state_2, state_3)