def test_delete_duplicates_multiple_accounts(event_producer, db_create_host, db_get_host, inventory_config): canonical_facts = { "insights_id": generate_uuid(), "subscription_manager_id": generate_uuid(), "bios_uuid": generate_uuid(), "satellite_id": generate_uuid(), "fqdn": generate_random_string(), } host1 = minimal_db_host(canonical_facts=canonical_facts, account="111111") created_host1 = db_create_host(host=host1).id host2 = minimal_db_host(canonical_facts=canonical_facts, account="222222") created_host2 = db_create_host(host=host2).id Session = _init_db(inventory_config) sessions = [Session() for _ in range(3)] with multi_session_guard(sessions): deleted_hosts_count = host_delete_duplicates_run( inventory_config, mock.Mock(), *sessions, event_producer, shutdown_handler=mock.Mock(**{"shut_down.return_value": False}), ) assert deleted_hosts_count == 0 assert db_get_host(created_host1) assert db_get_host(created_host2)
def test_delete_duplicates_elevated_ids_not_matching( event_producer, db_create_host, db_get_host, inventory_config, tested_id ): canonical_facts = { "provider_id": generate_uuid(), "insights_id": generate_uuid(), "subscription_manager_id": generate_uuid(), "bios_uuid": generate_uuid(), "satellite_id": generate_uuid(), "fqdn": generate_random_string(), } if tested_id == "provider_id": canonical_facts["provider_type"] = "aws" if tested_id in ("insights_id", "subscription_manager_id"): canonical_facts.pop("provider_id", None) if tested_id == "subscription_manager_id": canonical_facts.pop("insights_id", None) host_count = 10 created_hosts = [] # Hosts with the same amount of canonical facts for _ in range(host_count): canonical_facts[tested_id] = generate_uuid() host = minimal_db_host(canonical_facts=canonical_facts) created_hosts.append(db_create_host(host=host)) # Hosts with less canonical facts for _ in range(host_count): facts = {tested_id: generate_uuid()} host = minimal_db_host(canonical_facts=facts) created_hosts.append(db_create_host(host=host)) # Hosts with more canonical facts for _ in range(host_count): canonical_facts[tested_id] = generate_uuid() canonical_facts["ip_addresses"] = ["10.0.0.10"] host = minimal_db_host(canonical_facts=canonical_facts) created_hosts.append(db_create_host(host=host)) for host in created_hosts: assert db_get_host(host.id) Session = _init_db(inventory_config) sessions = [Session() for _ in range(3)] with multi_session_guard(sessions): deleted_hosts_count = host_delete_duplicates_run( inventory_config, mock.Mock(), *sessions, event_producer, shutdown_handler=mock.Mock(**{"shut_down.return_value": False}), ) assert deleted_hosts_count == 0 for host in created_hosts: assert db_get_host(host.id)
def test_delete_duplicates_without_elevated_matching(event_producer, db_create_host, db_get_host, inventory_config): canonical_facts = { "bios_uuid": generate_uuid(), "satellite_id": generate_uuid(), "fqdn": generate_random_string(), "ip_addresses": ["10.0.0.1"], "mac_addresses": ["aa:bb:cc:dd:ee:ff"], } host_count = 10 created_hosts = [] # Hosts with less canonical facts for fact in canonical_facts: facts = {fact: canonical_facts[fact]} host = minimal_db_host(canonical_facts=facts) created_hosts.append(db_create_host(host=host)) # # Hosts with more canonical facts # for fact in ELEVATED_IDS: # facts = deepcopy(canonical_facts) # facts[fact] = generate_uuid() # if fact == "provider_id": # facts["provider_type"] = "aws" # host = minimal_db_host(canonical_facts=facts) # created_hosts.append(db_create_host(host=host)) <-- Issue with missing elevated ID # Hosts with the same amount of canonical facts for _ in range(host_count): host = minimal_db_host(canonical_facts=canonical_facts) created_hosts.append(db_create_host(host=host)) for host in created_hosts: assert db_get_host(host.id) Session = _init_db(inventory_config) sessions = [Session() for _ in range(3)] with multi_session_guard(sessions): deleted_hosts_count = host_delete_duplicates_run( inventory_config, mock.Mock(), *sessions, event_producer, shutdown_handler=mock.Mock(**{"shut_down.return_value": False}), ) # Issue with missing elevated ID # assert deleted_hosts_count == host_count + len(canonical_facts) + len(ELEVATED_IDS) - 1 assert deleted_hosts_count == host_count + len(canonical_facts) - 1 for i in range(len(created_hosts) - 1): assert not db_get_host(created_hosts[i].id) assert db_get_host(created_hosts[-1].id)
def test_delete_duplicates_customer_scenario_2(event_producer, db_create_host, db_get_host, inventory_config): staleness_timestamps = get_staleness_timestamps() rhsm_id = generate_uuid() bios_uuid = generate_uuid() canonical_facts = { "insights_id": generate_uuid(), "subscription_manager_id": rhsm_id, "bios_uuid": bios_uuid, "satellite_id": rhsm_id, "fqdn": "rozrhjrad01.base.srvco.net", "ip_addresses": ["10.230.230.10", "10.230.230.13"], "mac_addresses": ["00:50:56:ac:56:45", "00:50:56:ac:48:61", "00:00:00:00:00:00"], } host_data = { "stale_timestamp": staleness_timestamps["stale_warning"], "reporter": "puptoo", "canonical_facts": canonical_facts, } host1 = minimal_db_host(**host_data) created_host1 = db_create_host(host=host1) host_data["canonical_facts"]["ip_addresses"] = ["10.230.230.3", "10.230.230.4"] host2 = minimal_db_host(**host_data) created_host2 = db_create_host(host=host2) host_data["canonical_facts"]["ip_addresses"] = ["10.230.230.1", "10.230.230.4"] host_data["stale_timestamp"] = staleness_timestamps["fresh"] host3 = minimal_db_host(**host_data) created_host3 = db_create_host(host=host3) assert db_get_host(created_host1.id) assert db_get_host(created_host2.id) assert db_get_host(created_host3.id) Session = _init_db(inventory_config) sessions = [Session() for _ in range(3)] with multi_session_guard(sessions): deleted_hosts_count = host_delete_duplicates_run( inventory_config, mock.Mock(), *sessions, event_producer, shutdown_handler=mock.Mock(**{"shut_down.return_value": False}), ) assert deleted_hosts_count == 2 assert not db_get_host(created_host1.id) assert not db_get_host(created_host2.id) assert db_get_host(created_host3.id)
def test_reaper_shutdown_handler(event_datetime_mock, db_create_host, db_get_hosts, inventory_config): staleness_timestamps = get_staleness_timestamps() created_host_ids = [] host_count = 3 for _ in range(host_count): host_data = minimal_db_host( stale_timestamp=staleness_timestamps["culled"].isoformat(), reporter="some reporter" ) created_host = db_create_host(host_data) created_host_ids.append(created_host.id) created_hosts = db_get_hosts(created_host_ids) assert created_hosts.count() == host_count event_producer_mock = mock.Mock() threadctx.request_id = UNKNOWN_REQUEST_ID_VALUE host_reaper_run( inventory_config, mock.Mock(), db.session, event_producer_mock, shutdown_handler=mock.Mock(**{"shut_down.side_effect": (False, True)}), ) remaining_hosts = db_get_hosts(created_host_ids) assert remaining_hosts.count() == 1 assert event_producer_mock.write_event.call_count == 2
def _db_create_host(identity=SYSTEM_IDENTITY, host=None, extra_data=None): extra_data = extra_data or {} host = host or minimal_db_host(account=identity["account_number"], **extra_data) db.session.add(host) db.session.commit() return host
def test_culled_host_is_removed(event_producer_mock, event_datetime_mock, db_create_host, db_get_host, inventory_config): staleness_timestamps = get_staleness_timestamps() host = minimal_db_host( stale_timestamp=staleness_timestamps["culled"].isoformat(), reporter="some reporter") created_host = db_create_host(host) assert db_get_host(created_host.id) threadctx.request_id = UNKNOWN_REQUEST_ID_VALUE host_reaper_run( inventory_config, mock.Mock(), db.session, event_producer_mock, shutdown_handler=mock.Mock(**{"shut_down.return_value": False}), ) assert not db_get_host(created_host.id) assert_delete_event_is_valid(event_producer=event_producer_mock, host=created_host, timestamp=event_datetime_mock)
def test_synchronize_host_event(event_producer_mock, event_datetime_mock, db_create_host, db_get_host, inventory_config): staleness_timestamps = get_staleness_timestamps() host = minimal_db_host( stale_timestamp=staleness_timestamps["culled"].isoformat(), reporter="some reporter") created_host = db_create_host(host) assert db_get_host(created_host.id) threadctx.request_id = UNKNOWN_REQUEST_ID_VALUE host_synchronizer_run( inventory_config, mock.Mock(), db.session, event_producer_mock, shutdown_handler=mock.Mock(**{"shut_down.return_value": False}), ) # check if host exist thought event synchronizer must find it to produce an update event. assert db_get_host(created_host.id) assert_synchronize_event_is_valid(event_producer=event_producer_mock, host=created_host, timestamp=event_datetime_mock)
def test_non_culled_host_is_not_removed( event_producer_mock, event_datetime_mock, db_create_host, db_get_hosts, inventory_config ): staleness_timestamps = get_staleness_timestamps() created_hosts = [] for stale_timestamp in ( staleness_timestamps["stale_warning"], staleness_timestamps["stale"], staleness_timestamps["fresh"], ): host = minimal_db_host(stale_timestamp=stale_timestamp.isoformat(), reporter="some reporter") created_host = db_create_host(host) created_hosts.append(created_host) created_host_ids = [host.id for host in created_hosts] retrieved_hosts = db_get_hosts(created_host_ids) assert created_host_ids == [host.id for host in retrieved_hosts] threadctx.request_id = UNKNOWN_REQUEST_ID_VALUE host_reaper_run( inventory_config, mock.Mock(), db.session, event_producer_mock, shutdown_handler=mock.Mock(**{"shut_down.return_value": False}), ) retrieved_hosts = db_get_hosts(created_host_ids) assert created_host_ids == [host.id for host in retrieved_hosts] assert event_producer_mock.event is None
def test_find_host_using_superset_canonical_fact_match(db_create_host): canonical_facts = {"fqdn": "fred", "bios_uuid": generate_uuid()} # Create the superset of canonical facts to search by superset_canonical_facts = canonical_facts.copy() superset_canonical_facts["satellite_id"] = generate_uuid() host = minimal_db_host(canonical_facts=canonical_facts) created_host = db_create_host(host=host) assert_host_exists_in_db(created_host.id, superset_canonical_facts)
def test_delete_duplicate_host(event_producer_mock, db_create_host, db_get_host, inventory_config): # make two hosts that are the same canonical_facts = { "provider_type": ProviderType.AWS, # Doesn't matter "provider_id": generate_uuid(), "insights_id": generate_uuid(), "subscription_manager_id": generate_uuid(), } old_host = minimal_db_host(canonical_facts=canonical_facts) new_host = minimal_db_host(canonical_facts=canonical_facts) created_old_host = db_create_host(host=old_host) created_new_host = db_create_host(host=new_host) assert created_old_host.id != created_new_host.id old_host_id = created_old_host.id assert created_old_host.canonical_facts["provider_id"] == created_new_host.canonical_facts["provider_id"] threadctx.request_id = UNKNOWN_REQUEST_ID_VALUE Session = _init_db(inventory_config) accounts_session = Session() hosts_session = Session() misc_session = Session() with multi_session_guard([accounts_session, hosts_session, misc_session]): num_deleted = host_delete_duplicates_run( inventory_config, mock.Mock(), accounts_session, hosts_session, misc_session, event_producer_mock, shutdown_handler=mock.Mock(**{"shut_down.return_value": False}), ) print("deleted this many hosts:") print(num_deleted) assert num_deleted == 1 assert db_get_host(created_new_host.id) assert not db_get_host(old_host_id)
def test_creates_delete_event_when_missing_from_db(mocker, db_create_host, inventory_config, num_existing, num_missing): event_producer_mock = mock.Mock() threadctx.request_id = UNKNOWN_REQUEST_ID_VALUE event_list = [] existing_hosts_created = 0 missing_hosts_id_list = [] # Set up DB and event queue while existing_hosts_created < num_existing or len( missing_hosts_id_list) < num_missing: if existing_hosts_created < num_existing: host = minimal_db_host() db_create_host(host=host) event_list.append(build_event(EventType.created, host)) existing_hosts_created += 1 if len(missing_hosts_id_list) < num_missing: missing_host = minimal_db_host() missing_host.id = generate_uuid() event_list.append(build_event(EventType.updated, missing_host)) missing_hosts_id_list.append(missing_host.id) consumer_mock = create_kafka_consumer_mock(mocker, inventory_config, 1, 0, 1, event_list) rebuild_events_run( inventory_config, mock.Mock(), db.session, consumer_mock, event_producer_mock, shutdown_handler=mock.Mock(**{"shut_down.return_value": False}), ) assert event_producer_mock.write_event.call_count == num_missing for i in range(num_missing): produced_event = json.loads( event_producer_mock.write_event.call_args_list[i][0][0]) assert produced_event["type"] == "delete" assert produced_event["id"] in missing_hosts_id_list
def test_find_host_using_subset_canonical_fact_match(db_create_host): fqdn = "fred.flintstone.com" canonical_facts = {"fqdn": fqdn, "bios_uuid": generate_uuid(), "rhel_machine_id": generate_uuid()} host = minimal_db_host(canonical_facts=canonical_facts) created_host = db_create_host(host) # Create the subset of canonical facts to search by subset_canonical_facts = {"fqdn": fqdn} assert_host_exists_in_db(created_host.id, subset_canonical_facts)
def test_find_correct_host_when_similar_canonical_facts(db_create_host): cf1 = { "fqdn": "fred", "bios_uuid": generate_uuid(), "insights_id": generate_uuid() } cf2 = { "fqdn": "george", "bios_uuid": generate_uuid(), "insights_id": generate_uuid() } cf3 = { "fqdn": cf1["fqdn"], "bios_uuid": cf1["bios_uuid"], "insights_id": cf2["insights_id"] } db_create_host(host=minimal_db_host(canonical_facts=cf1)) created_host_2 = db_create_host(host=minimal_db_host(canonical_facts=cf2)) assert_host_exists_in_db(created_host_2.id, cf3)
def test_find_host_using_subscription_manager_id_match(db_create_host): canonical_facts = {"fqdn": "fred", "bios_uuid": generate_uuid(), "subscription_manager_id": generate_uuid()} # Change the bios_uuid so that falling back to subset match will fail search_canonical_facts = { "bios_uuid": generate_uuid(), "subscription_manager_id": canonical_facts["subscription_manager_id"], } host = minimal_db_host(canonical_facts=canonical_facts) created_host = db_create_host(host) assert_host_exists_in_db(created_host.id, search_canonical_facts)
def test_find_host_using_elevated_ids_match(db_create_host, host_create_order, expected_host): hosts_canonical_facts = ({"subscription_manager_id": generate_uuid()}, {"insights_id": generate_uuid()}) created_hosts = [] for host_canonical_facts in host_create_order: host = minimal_db_host(canonical_facts=hosts_canonical_facts[host_canonical_facts]) created_host = db_create_host(host) created_hosts.append(created_host) search_canonical_facts = { key: value for host_canonical_facts in hosts_canonical_facts for key, value in host_canonical_facts.items() } assert_host_exists_in_db(created_hosts[expected_host].id, search_canonical_facts)
def test_find_host_using_insights_id_match(db_create_host): canonical_facts = {"fqdn": "fred", "bios_uuid": generate_uuid(), "insights_id": generate_uuid()} # Change the canonical facts except the insights_id...match on insights_id search_canonical_facts = { "fqdn": "barney", "bios_uuid": generate_uuid(), "insights_id": canonical_facts["insights_id"], } host = minimal_db_host(canonical_facts=canonical_facts) created_host = db_create_host(host) assert_host_exists_in_db(created_host.id, search_canonical_facts)
def test_elevated_id_priority_order_match(db_create_host, changing_id): base_canonical_facts = { "provider_id": generate_uuid(), "insights_id": generate_uuid(), "subscription_manager_id": generate_uuid(), } created_host = db_create_host(host=minimal_db_host( canonical_facts=base_canonical_facts)) match_host_canonical_facts = base_canonical_facts.copy() match_host_canonical_facts[changing_id] = generate_uuid() assert_host_exists_in_db(created_host.id, match_host_canonical_facts)
def test_no_merge_when_different_facts(db_create_host): cf1 = { "fqdn": "fred", "bios_uuid": generate_uuid(), "insights_id": generate_uuid() } cf2 = { "fqdn": "george", "bios_uuid": generate_uuid(), "subscription_manager_id": generate_uuid() } db_create_host(host=minimal_db_host(canonical_facts=cf1)) assert_host_missing_from_db(cf2)
def test_find_host_canonical_fact_subset_match_different_elevated_ids( db_create_host): base_canonical_facts = {"fqdn": "fred", "bios_uuid": generate_uuid()} created_host_canonical_facts = base_canonical_facts.copy() created_host_canonical_facts["insights_id"] = generate_uuid() # Create the subset of canonical facts to search by search_canonical_facts = {"fqdn": "fred"} search_canonical_facts["subscription_manager_id"] = generate_uuid() created_host = db_create_host(host=minimal_db_host( canonical_facts=created_host_canonical_facts)) assert_host_exists_in_db(created_host.id, search_canonical_facts)
def _db_create_multiple_hosts(hosts=None, how_many=10, extra_data=None): extra_data = extra_data or {} created_hosts = [] if type(hosts) == list: for host in hosts: db.session.add(host) created_hosts.append(host) else: for _ in range(how_many): host = minimal_db_host(**extra_data) db.session.add(host) created_hosts.append(host) db.session.commit() return created_hosts
def test_elevated_id_priority_order_nomatch(db_create_host, changing_id): base_canonical_facts = { "insights_id": generate_uuid(), "subscription_manager_id": generate_uuid() } created_host_canonical_facts = base_canonical_facts.copy() created_host_canonical_facts[changing_id] = generate_uuid() search_canonical_facts = base_canonical_facts.copy() search_canonical_facts[changing_id] = generate_uuid() created_host = db_create_host(host=minimal_db_host( canonical_facts=created_host_canonical_facts)) assert_host_exists_in_db(created_host.id, created_host_canonical_facts) assert_host_missing_from_db(search_canonical_facts)
def _db_create_multiple_hosts(identity=SYSTEM_IDENTITY, hosts=None, how_many=10, extra_data=None): extra_data = extra_data or {} created_hosts = [] if type(hosts) == list: for host in hosts: db.session.add(host) created_hosts.append(host) else: for _ in range(how_many): host = minimal_db_host(account=identity["account_number"], **extra_data) db.session.add(host) created_hosts.append(host) db.session.commit() return created_hosts
def test_culled_edge_host_is_not_removed(event_producer_mock, db_create_host, db_get_host, inventory_config): staleness_timestamps = get_staleness_timestamps() host = minimal_db_host( stale_timestamp=staleness_timestamps["culled"], reporter="some reporter", system_profile_facts={"host_type": "edge"}, ) created_host = db_create_host(host=host) assert db_get_host(created_host.id) threadctx.request_id = UNKNOWN_REQUEST_ID_VALUE host_reaper_run( inventory_config, mock.Mock(), db.session, event_producer_mock, shutdown_handler=mock.Mock(**{"shut_down.return_value": False}), ) assert db_get_host(created_host.id)
def test_no_delete_when_hosts_present(mocker, db_create_host, inventory_config): event_producer_mock = mock.Mock() threadctx.request_id = UNKNOWN_REQUEST_ID_VALUE event_list = [] for _ in range(4): host = minimal_db_host() db_create_host(host=host) event_list.append(build_event(EventType.created, host)) consumer_mock = create_kafka_consumer_mock(mocker, inventory_config, 1, 0, 1, event_list) rebuild_events_run( inventory_config, mock.Mock(), db.session, consumer_mock, event_producer_mock, shutdown_handler=mock.Mock(**{"shut_down.return_value": False}), ) assert event_producer_mock.write_event.call_count == 0
def test_delete_duplicates_last_modified(event_producer, db_create_multiple_hosts, db_get_host, inventory_config): """Test that the deletion script always keeps host with the latest 'modified_on' date""" canonical_facts = { "provider_id": generate_uuid(), "insights_id": generate_uuid(), "subscription_manager_id": generate_uuid(), "bios_uuid": generate_uuid(), "satellite_id": generate_uuid(), "fqdn": generate_random_string(), "provider_type": "aws", } host_count = 100 hosts = [minimal_db_host(canonical_facts=canonical_facts) for _ in range(host_count)] created_host_ids = [host.id for host in db_create_multiple_hosts(hosts=hosts)] updated_host = update_host_in_db(choice(created_host_ids), display_name="new-display-name") for host_id in created_host_ids: assert db_get_host(host_id) Session = _init_db(inventory_config) sessions = [Session() for _ in range(3)] with multi_session_guard(sessions): deleted_hosts_count = host_delete_duplicates_run( inventory_config, mock.Mock(), *sessions, event_producer, shutdown_handler=mock.Mock(**{"shut_down.return_value": False}), ) assert deleted_hosts_count == host_count - 1 for host_id in created_host_ids: if host_id != updated_host.id: assert not db_get_host(host_id) else: assert db_get_host(host_id)
def test_delete_duplicates_customer_scenario_1(event_producer, db_create_host, db_get_host, inventory_config): staleness_timestamps = get_staleness_timestamps() rhsm_id = generate_uuid() bios_uuid = generate_uuid() canonical_facts = { "insights_id": generate_uuid(), "subscription_manager_id": rhsm_id, "bios_uuid": bios_uuid, "satellite_id": rhsm_id, "fqdn": "rn001018", "ip_addresses": ["10.230.230.3"], "mac_addresses": ["00:50:56:ab:5a:22", "00:00:00:00:00:00"], } host_data = { "stale_timestamp": staleness_timestamps["stale_warning"], "reporter": "puptoo", "canonical_facts": canonical_facts, } host1 = minimal_db_host(**host_data) created_host1 = db_create_host(host=host1) host_data["canonical_facts"]["ip_addresses"] = ["10.230.230.30"] host_data["canonical_facts"].pop("bios_uuid") host_data["stale_timestamp"] = staleness_timestamps["stale"] host2 = minimal_db_host(**host_data) created_host2 = db_create_host(host=host2) host_data["canonical_facts"]["ip_addresses"] = ["10.230.230.3"] host3 = minimal_db_host(**host_data) created_host3 = db_create_host(host=host3) host_data["reporter"] = "yupana" host_data["canonical_facts"]["ip_addresses"] = ["10.230.230.1"] host_data["canonical_facts"]["mac_addresses"] = ["00:50:56:ab:5a:22"] host_data["canonical_facts"]["bios_uuid"] = bios_uuid host_data["canonical_facts"]["fqdn"] = "rn001018.bcbst.com" host_data["stale_timestamp"] = staleness_timestamps["fresh"] host4 = minimal_db_host(**host_data) created_host4 = db_create_host(host=host4) host_data["reporter"] = "puptoo" host_data["canonical_facts"]["ip_addresses"] = ["10.230.230.15"] host_data["canonical_facts"]["mac_addresses"] = ["00:50:56:ab:5a:22", "00:00:00:00:00:00"] host_data["canonical_facts"].pop("bios_uuid") host_data["canonical_facts"]["fqdn"] = "rn001018" host5 = minimal_db_host(**host_data) created_host5 = db_create_host(host=host5) assert db_get_host(created_host1.id) assert db_get_host(created_host2.id) assert db_get_host(created_host3.id) assert db_get_host(created_host4.id) assert db_get_host(created_host5.id) Session = _init_db(inventory_config) sessions = [Session() for _ in range(3)] with multi_session_guard(sessions): deleted_hosts_count = host_delete_duplicates_run( inventory_config, mock.Mock(), *sessions, event_producer, shutdown_handler=mock.Mock(**{"shut_down.return_value": False}), ) assert deleted_hosts_count == 4 assert not db_get_host(created_host1.id) assert not db_get_host(created_host2.id) assert not db_get_host(created_host3.id) assert not db_get_host(created_host4.id) assert db_get_host(created_host5.id)
def test_delete_dupe_more_hosts_than_chunk_size( event_producer_mock, db_get_host, db_create_multiple_hosts, db_create_host, inventory_config ): canonical_facts_1 = { "provider_id": generate_uuid(), "insights_id": generate_uuid(), "subscription_manager_id": generate_uuid(), } canonical_facts_2 = { "provider_id": generate_uuid(), "insights_id": generate_uuid(), "subscription_manager_id": generate_uuid(), } chunk_size = inventory_config.script_chunk_size num_hosts = chunk_size * 3 + 15 # create host before big chunk. Hosts are ordered by modified date so creation # order is important old_host_1 = minimal_db_host(canonical_facts=canonical_facts_1) new_host_1 = minimal_db_host(canonical_facts=canonical_facts_1) created_old_host_1 = db_create_host(host=old_host_1) created_new_host_1 = db_create_host(host=new_host_1) # create big chunk of hosts db_create_multiple_hosts(how_many=num_hosts) # create another host after old_host_2 = minimal_db_host(canonical_facts=canonical_facts_2) new_host_2 = minimal_db_host(canonical_facts=canonical_facts_2) created_old_host_2 = db_create_host(host=old_host_2) created_new_host_2 = db_create_host(host=new_host_2) assert created_old_host_1.id != created_new_host_1.id assert created_old_host_2.id != created_new_host_2.id threadctx.request_id = UNKNOWN_REQUEST_ID_VALUE Session = _init_db(inventory_config) accounts_session = Session() hosts_session = Session() misc_session = Session() with multi_session_guard([accounts_session, hosts_session, misc_session]): num_deleted = host_delete_duplicates_run( inventory_config, mock.Mock(), accounts_session, hosts_session, misc_session, event_producer_mock, shutdown_handler=mock.Mock(**{"shut_down.return_value": False}), ) assert num_deleted == 2 assert db_get_host(created_new_host_1.id) assert not db_get_host(created_old_host_1.id) assert db_get_host(created_new_host_2.id) assert not db_get_host(created_old_host_2.id)
def test_delete_duplicates_multiple_scenarios( event_producer, db_create_host, db_create_multiple_hosts, db_get_host, inventory_config, script_function ): chunk_size = inventory_config.script_chunk_size # Customer scenario staleness_timestamps = get_staleness_timestamps() rhsm_id = generate_uuid() bios_uuid = generate_uuid() canonical_facts = { "insights_id": generate_uuid(), "subscription_manager_id": rhsm_id, "bios_uuid": bios_uuid, "satellite_id": rhsm_id, "fqdn": "rozrhjrad01.base.srvco.net", "ip_addresses": ["10.230.230.10", "10.230.230.13"], "mac_addresses": ["00:50:56:ac:56:45", "00:50:56:ac:48:61", "00:00:00:00:00:00"], } host_data = { "stale_timestamp": staleness_timestamps["stale_warning"], "reporter": "puptoo", "canonical_facts": canonical_facts, } customer_host1 = minimal_db_host(**host_data) customer_created_host1 = db_create_host(host=customer_host1).id host_data["canonical_facts"]["ip_addresses"] = ["10.230.230.3", "10.230.230.4"] customer_host2 = minimal_db_host(**host_data) customer_created_host2 = db_create_host(host=customer_host2).id host_data["canonical_facts"]["ip_addresses"] = ["10.230.230.1", "10.230.230.4"] host_data["stale_timestamp"] = staleness_timestamps["fresh"] customer_host3 = minimal_db_host(**host_data) customer_created_host3 = db_create_host(host=customer_host3).id assert db_get_host(customer_created_host1) assert db_get_host(customer_created_host2) assert db_get_host(customer_created_host3) # Matching elevated ID def _gen_canonical_facts(): return { "insights_id": generate_uuid(), "subscription_manager_id": generate_uuid(), "bios_uuid": generate_uuid(), "satellite_id": generate_uuid(), "fqdn": generate_random_string(), } elevated_matching_host_count = 10 elevated_id = generate_uuid() elevated_matching_created_hosts = [] # Hosts with the same amount of canonical facts for _ in range(elevated_matching_host_count): canonical_facts = _gen_canonical_facts() canonical_facts["insights_id"] = elevated_id host = minimal_db_host(canonical_facts=canonical_facts) elevated_matching_created_hosts.append(db_create_host(host=host).id) # Hosts with less canonical facts for _ in range(elevated_matching_host_count): canonical_facts = {"insights_id": elevated_id} host = minimal_db_host(canonical_facts=canonical_facts) elevated_matching_created_hosts.append(db_create_host(host=host).id) # Create a lot of hosts to test that the script deletes duplicates in multiple chunks db_create_multiple_hosts(how_many=chunk_size) # Hosts with more canonical facts for _ in range(elevated_matching_host_count): canonical_facts = _gen_canonical_facts() canonical_facts["insights_id"] = elevated_id canonical_facts["ip_addresses"] = [f"10.0.0.{randint(1, 255)}"] host = minimal_db_host(canonical_facts=canonical_facts) elevated_matching_created_hosts.append(db_create_host(host=host).id) for host in elevated_matching_created_hosts: assert db_get_host(host) # Elevated IDs not matching elevated_not_matching_canonical_facts = _gen_canonical_facts() elevated_not_matching_host_count = 10 elevated_not_matching_created_hosts = [] # Hosts with the same amount of canonical facts for _ in range(elevated_not_matching_host_count): elevated_not_matching_canonical_facts["insights_id"] = generate_uuid() host = minimal_db_host(canonical_facts=elevated_not_matching_canonical_facts) elevated_not_matching_created_hosts.append(db_create_host(host=host).id) # Hosts with less canonical facts for _ in range(elevated_not_matching_host_count): facts = {"insights_id": generate_uuid()} host = minimal_db_host(canonical_facts=facts) elevated_not_matching_created_hosts.append(db_create_host(host=host).id) # Hosts with more canonical facts for _ in range(elevated_not_matching_host_count): elevated_not_matching_canonical_facts["insights_id"] = generate_uuid() elevated_not_matching_canonical_facts["ip_addresses"] = ["10.0.0.10"] host = minimal_db_host(canonical_facts=elevated_not_matching_canonical_facts) elevated_not_matching_created_hosts.append(db_create_host(host=host).id) for host in elevated_not_matching_created_hosts: assert db_get_host(host) # Without elevated IDs - canonical facts matching without_elevated_matching_canonical_facts = { "bios_uuid": generate_uuid(), "satellite_id": generate_uuid(), "fqdn": generate_random_string(), "ip_addresses": ["10.0.0.1"], "mac_addresses": ["aa:bb:cc:dd:ee:ff"], } without_elevated_matching_host_count = 10 without_elevated_matching_created_hosts = [] # Hosts with less canonical facts for fact in without_elevated_matching_canonical_facts: facts = {fact: without_elevated_matching_canonical_facts[fact]} host = minimal_db_host(canonical_facts=facts) without_elevated_matching_created_hosts.append(db_create_host(host=host).id) # Create a lot of hosts to test that the script deletes duplicates in multiple chunks db_create_multiple_hosts(how_many=chunk_size) # Hosts with the same amount of canonical facts for _ in range(without_elevated_matching_host_count): host = minimal_db_host(canonical_facts=without_elevated_matching_canonical_facts) without_elevated_matching_created_hosts.append(db_create_host(host=host).id) for host in without_elevated_matching_created_hosts: assert db_get_host(host) # Without elevated IDs - canonical facts not matching without_elevated_not_matching_canonical_facts = { "bios_uuid": generate_uuid(), "satellite_id": generate_uuid(), "fqdn": generate_random_string(), "ip_addresses": ["0.0.0.0"], "mac_addresses": ["aa:bb:cc:dd:ee:ff"], } without_elevated_not_matching_host_count = 10 without_elevated_not_matching_created_hosts = [] # Hosts with the same amount of canonical facts for _ in range(without_elevated_not_matching_host_count): facts = deepcopy(without_elevated_not_matching_canonical_facts) facts["fqdn"] = generate_random_string() host = minimal_db_host(canonical_facts=facts) without_elevated_not_matching_created_hosts.append(db_create_host(host=host).id) # Hosts with less canonical facts for _ in range(without_elevated_not_matching_host_count): facts = {"fqdn": generate_random_string()} host = minimal_db_host(canonical_facts=facts) without_elevated_not_matching_created_hosts.append(db_create_host(host=host).id) # Hosts with more canonical facts for fact in ELEVATED_IDS: facts = deepcopy(without_elevated_not_matching_canonical_facts) facts["fqdn"] = generate_random_string() facts[fact] = generate_uuid() if fact == "provider_id": facts["provider_type"] = "aws" host = minimal_db_host(canonical_facts=facts) without_elevated_not_matching_created_hosts.append(db_create_host(host=host).id) for host in without_elevated_not_matching_created_hosts: assert db_get_host(host) if script_function == "run": Session = _init_db(inventory_config) sessions = [Session() for _ in range(3)] with multi_session_guard(sessions): deleted_hosts_count = host_delete_duplicates_run( inventory_config, mock.Mock(), *sessions, event_producer, shutdown_handler=mock.Mock(**{"shut_down.return_value": False}), ) assert deleted_hosts_count == elevated_matching_host_count * 3 + without_elevated_matching_host_count + len( without_elevated_matching_canonical_facts ) else: host_delete_duplicates_main(mock.Mock()) assert not db_get_host(customer_created_host1) assert not db_get_host(customer_created_host2) assert db_get_host(customer_created_host3) for i in range(len(elevated_matching_created_hosts) - 1): assert not db_get_host(elevated_matching_created_hosts[i]) assert db_get_host(elevated_matching_created_hosts[-1]) for host in elevated_not_matching_created_hosts: assert db_get_host(host) for i in range(len(without_elevated_matching_created_hosts) - 1): assert not db_get_host(without_elevated_matching_created_hosts[i]) assert db_get_host(without_elevated_matching_created_hosts[-1]) for host in without_elevated_not_matching_created_hosts: assert db_get_host(host)
def test_delete_duplicates_without_elevated_not_matching( event_producer, db_create_host, db_get_host, inventory_config, tested_fact ): def _generate_fact(fact_name): if fact_name == "fqdn": return generate_random_string() if fact_name == "ip_addresses": return [f"{randint(1, 255)}.{randint(0, 255)}.{randint(0, 255)}.{randint(1, 255)}"] if fact_name == "mac_addresses": hex_chars = "0123456789abcdef" addr = ":".join([f"{choice(hex_chars)}{choice(hex_chars)}" for _ in range(6)]) return [addr] return generate_uuid() canonical_facts = { "bios_uuid": generate_uuid(), "satellite_id": generate_uuid(), "fqdn": generate_random_string(), "ip_addresses": ["0.0.0.0"], "mac_addresses": ["aa:bb:cc:dd:ee:ff"], } host_count = 10 created_hosts = [] # Hosts with the same amount of canonical facts for _ in range(host_count): facts = deepcopy(canonical_facts) facts[tested_fact] = _generate_fact(tested_fact) host = minimal_db_host(canonical_facts=facts) created_hosts.append(db_create_host(host=host)) # Hosts with less canonical facts for _ in range(host_count): facts = {tested_fact: _generate_fact(tested_fact)} host = minimal_db_host(canonical_facts=facts) created_hosts.append(db_create_host(host=host)) # Hosts with more canonical facts for fact in ELEVATED_IDS: facts = deepcopy(canonical_facts) facts[tested_fact] = _generate_fact(tested_fact) facts[fact] = generate_uuid() if fact == "provider_id": facts["provider_type"] = "aws" host = minimal_db_host(canonical_facts=facts) created_hosts.append(db_create_host(host=host)) for host in created_hosts: assert db_get_host(host.id) Session = _init_db(inventory_config) sessions = [Session() for _ in range(3)] with multi_session_guard(sessions): deleted_hosts_count = host_delete_duplicates_run( inventory_config, mock.Mock(), *sessions, event_producer, shutdown_handler=mock.Mock(**{"shut_down.return_value": False}), ) assert deleted_hosts_count == 0 for host in created_hosts: assert db_get_host(host.id)