def update_dataset(request: UpdateDatasetRequest) -> Dataset:
"""
Update dataset values: fetcher_active, write_back, and matcher_active.
"""
config = ThreatExchangeConfig.getx(str(request.privacy_group_id))
config.fetcher_active = request.fetcher_active
config.write_back = request.write_back
config.matcher_active = request.matcher_active
updated_config = hmaconfig.update_config(config).__dict__
updated_config["privacy_group_id"] = updated_config["name"]
additional_config = AdditionalMatchSettingsConfig.get(
str(request.privacy_group_id))
if request.pdq_match_threshold:
if additional_config:
additional_config.pdq_match_threshold = int(
request.pdq_match_threshold)
hmaconfig.update_config(additional_config)
else:
additional_config = AdditionalMatchSettingsConfig(
str(request.privacy_group_id),
int(request.pdq_match_threshold))
hmaconfig.create_config(additional_config)
elif additional_config: # pdq_match_threshold was set and now should be removed
hmaconfig.delete_config(additional_config)
return Dataset.from_dict(updated_config)
def delete_match_settings(key=None, ) -> MatchSettingsUpdateResponse:
"""
Delete a match settings config for a given privacy_group_id
"""
if config := AdditionalMatchSettingsConfig.get(str(key)):
hmaconfig.delete_config(config)
return MatchSettingsUpdateResponse(
f"match_settings deleted for pg_id {key}")
def create_or_update_match_settings(
request: MatchSettingsUpdateRequest, ) -> MatchSettingsUpdateResponse:
"""
Create or update a match settings config for a given privacy_group_id
"""
if config := AdditionalMatchSettingsConfig.get(
request.privacy_group_id):
config.pdq_match_threshold = request.pdq_match_threshold
hmaconfig.update_config(config)
return MatchSettingsUpdateResponse(
f"match_settings updated for pg_id {request.privacy_group_id} with pdq_match_threshold={request.pdq_match_threshold}"
)
def get_all_match_settings() -> MatchSettingsResponse:
"""
Return all match settings configs
"""
return MatchSettingsResponse(match_settings=[
MatchSettingsResponseBody(c)
for c in AdditionalMatchSettingsConfig.get_all()
])
def _get_privacy_group_matcher_pdq_threshold(privacy_group_id: str,
cache_buster) -> int:
config = AdditionalMatchSettingsConfig.get(privacy_group_id)
if not config:
logger.debug(
"Privacy group %s does not have custom pdq_match_threshold. Using default defined in PDQ_CONFIDENT_MATCH_THRESHOLD",
privacy_group_id,
)
return PdqSignal.PDQ_CONFIDENT_MATCH_THRESHOLD
logger.debug("pdq_match_threshold for %s is %s", privacy_group_id,
config.pdq_match_threshold)
return config.pdq_match_threshold
def _get_threat_exchange_datasets(
table: Table,
threat_exchange_data_bucket_name: str,
threat_exchange_data_folder: str,
) -> t.List[ThreatExchangeDatasetSummary]:
collaborations = ThreatExchangeConfig.get_all()
hash_counts: t.Dict[str, t.Tuple[
int, str]] = _get_signal_hash_count_and_last_modified(
threat_exchange_data_bucket_name,
threat_exchange_data_folder,
)
summaries = []
for collab in collaborations:
if additional_config := AdditionalMatchSettingsConfig.get(
str(collab.privacy_group_id)):
pdq_match_threshold = str(additional_config.pdq_match_threshold)
else:
pdq_match_threshold = ""
summaries.append(
ThreatExchangeDatasetSummary(
collab.privacy_group_id,
collab.privacy_group_name,
collab.description,
collab.fetcher_active,
collab.matcher_active,
collab.write_back,
collab.in_use,
hash_count=t.cast(
int,
hash_counts.get(
collab.privacy_group_id,
[-1, ""],
)[0],
),
match_count=-1, # fix will be based on new count system
pdq_match_threshold=pdq_match_threshold,
))
def _get_max_pdq_threshold_for_active_matcher_privacy_groups(
cache_buster, ) -> int:
active_pg_names = _get_all_matcher_active_privacy_groups(cache_buster)
if not active_pg_names:
return 0
active_pdq_thresholds = [
config.pdq_match_threshold
for config in AdditionalMatchSettingsConfig.get_all()
if config.name in active_pg_names
]
if active_pdq_thresholds:
return max(active_pdq_thresholds)
# no custom threshold set for active privacy_groups
return 0
def _create_privacy_groups(self):
# Since we already have a mock_dynamodb2 courtesy BanksTableTestBase,
# re-use it for initing configs. Requires some clever hot-wiring.
config_test_mock = config_test.ConfigTest()
config_test_mock.mock_dynamodb2 = self.__class__.mock_dynamodb2
config_test_mock.create_mocked_table()
HMAConfig.initialize(config_test_mock.TABLE_NAME)
# Hot wiring ends...
self.active_pg = ThreatExchangeConfig(
"ACTIVE_PG", True, "", True, True, True, "ACTIVE_PG"
)
create_config(self.active_pg)
# Active PG has a distance threshold of 31.
create_config(AdditionalMatchSettingsConfig("ACTIVE_PG", 31))
self.inactive_pg = ThreatExchangeConfig(
"INACTIVE_PG", True, "", True, True, False, "INACTIVE_PG"
)
create_config(self.inactive_pg)
def get_match_settings(key=None, ) -> MatchSettingsResponseBody:
"""
Return a match settings config for a given privacy_group_id
"""
if config := AdditionalMatchSettingsConfig.get(str(key)):
return MatchSettingsResponseBody(config)
apply=[jsoninator(MatchSettingsUpdateRequest)])
def create_or_update_match_settings(
request: MatchSettingsUpdateRequest, ) -> MatchSettingsUpdateResponse:
"""
Create or update a match settings config for a given privacy_group_id
"""
if config := AdditionalMatchSettingsConfig.get(
request.privacy_group_id):
config.pdq_match_threshold = request.pdq_match_threshold
hmaconfig.update_config(config)
return MatchSettingsUpdateResponse(
f"match_settings updated for pg_id {request.privacy_group_id} with pdq_match_threshold={request.pdq_match_threshold}"
)
config = AdditionalMatchSettingsConfig(request.privacy_group_id,
request.pdq_match_threshold)
hmaconfig.create_config(config)
return MatchSettingsUpdateResponse(
f"match_settings created for pg_id {request.privacy_group_id} with pdq_match_threshold={request.pdq_match_threshold}"
)
@datasets_api.delete("/match-settings/<key>", apply=[jsoninator])
def delete_match_settings(key=None, ) -> MatchSettingsUpdateResponse:
"""
Delete a match settings config for a given privacy_group_id
"""
if config := AdditionalMatchSettingsConfig.get(str(key)):
hmaconfig.delete_config(config)
return MatchSettingsUpdateResponse(
f"match_settings deleted for pg_id {key}")
return bottle.abort(400, f"No match_settings for pg_id {key} found")