def test_delete(self):
a_config = config.HMAConfig("a")
config.create_config(a_config)
self.assertEqual({c.name for c in config.HMAConfig.get_all()}, {"a"})
config.delete_config(a_config)
self.assertEqual({c.name for c in config.HMAConfig.get_all()}, set())
self.assertEqual(None, config.HMAConfig.get("a"))
def update_action_rule(
request: ActionRulesRequest,
old_name: str,
) -> ActionRulesResponse:
"""
Update the action rule with name=<oldname>.
"""
logger.info("old_name: %s", old_name)
logger.info("request: %s", request)
error_message = ""
if ActionRule.exists(request.action_rule.name):
try:
hmaconfig.update_config(request.action_rule)
except Exception as e:
error_message = "Unexpected error."
handle_unexpected_error(e)
elif ActionRule.exists(old_name):
try:
hmaconfig.create_config(request.action_rule)
hmaconfig.delete_config_by_type_and_name("ActionRule", old_name)
except Exception as e:
error_message = "Unexpected error."
handle_unexpected_error(e)
else:
error_message = f"An action rule named '{request.action_rule.name}' or '{old_name}' does not exist."
logger.warning(
"An attempt was made to update an action rule named either '%s' or '%s' but neither exist.",
request.action_rule.name,
old_name,
)
response.status = 500
return ActionRulesResponse(error_message)
def create_action_rule(
request: ActionRulesRequest,
) -> ActionRulesResponse:
"""
Create an action rule.
"""
logger.info("request: %s", request)
error_message = ""
try:
hmaconfig.create_config(request.action_rule)
except ClientError as e:
# TODO this test for "already exists" should be moved to a common place
if e.response["Error"]["Code"] == "ConditionalCheckFailedException":
error_message = f"An action rule with the name '{request.action_rule.name}' already exists."
logger.warning(
"Duplicate action rule creation attempted: %s",
e.response["Error"]["Message"],
)
else:
error_message = "Unexpected error."
logger.error(
"Unexpected client error: %s", e.response["Error"]["Message"]
)
logger.exception(e)
response.status = 500
except Exception as e:
error_message = "Unexpected error."
handle_unexpected_error(e)
return ActionRulesResponse(error_message)
def update_dataset(request: UpdateDatasetRequest) -> Dataset:
"""
Update dataset values: fetcher_active, write_back, and matcher_active.
"""
config = ThreatExchangeConfig.getx(str(request.privacy_group_id))
config.fetcher_active = request.fetcher_active
config.write_back = request.write_back
config.matcher_active = request.matcher_active
updated_config = hmaconfig.update_config(config).__dict__
updated_config["privacy_group_id"] = updated_config["name"]
additional_config = AdditionalMatchSettingsConfig.get(
str(request.privacy_group_id))
if request.pdq_match_threshold:
if additional_config:
additional_config.pdq_match_threshold = int(
request.pdq_match_threshold)
hmaconfig.update_config(additional_config)
else:
additional_config = AdditionalMatchSettingsConfig(
str(request.privacy_group_id),
int(request.pdq_match_threshold))
hmaconfig.create_config(additional_config)
elif additional_config: # pdq_match_threshold was set and now should be removed
hmaconfig.delete_config(additional_config)
return Dataset.from_dict(updated_config)
def sync_privacy_groups():
api_key = AWSSecrets.te_api_key()
api = ThreatExchangeAPI(api_key)
privacy_group_member_list = api.get_threat_privacy_groups_member()
privacy_group_owner_list = api.get_threat_privacy_groups_owner()
unique_privacy_groups = set(privacy_group_member_list + privacy_group_owner_list)
priavcy_group_id_in_use = set()
for privacy_group in unique_privacy_groups:
if privacy_group.threat_updates_enabled:
# HMA can only read from privacy groups that have threat_updates enabled.
# # See here for more details:
# https://developers.facebook.com/docs/threat-exchange/reference/apis/threat-updates/v9.0
logger.info("Adding collaboration name %s", privacy_group.name)
priavcy_group_id_in_use.add(privacy_group.id)
config = ThreatExchangeConfig(
privacy_group.id,
# TODO Currently default to True for testing purpose,
# need to switch it to False before v0 launch
fetcher_active=FETCHER_ACTIVE,
privacy_group_name=privacy_group.name,
in_use=True,
write_back=WRITE_BACK,
)
try:
hmaconfig.create_config(config)
except ClientError as e:
if e.response["Error"]["Code"] == "ConditionalCheckFailedException":
logger.warning(
"Can't insert duplicated config, %s",
e.response["Error"]["Message"],
)
else:
raise
update_privacy_group_in_use(priavcy_group_id_in_use)
def create_privacy_group_if_not_exists(
privacy_group_id: str,
privacy_group_name: str,
description: str = "",
in_use: bool = True,
fetcher_active: bool = FETCHER_ACTIVE_DEFAULT,
matcher_active: bool = MATCHER_ACTIVE_DEFAULT,
write_back: bool = WRITE_BACK_DEFAULT,
):
logger.info("Adding collaboration name %s", privacy_group_name)
config = ThreatExchangeConfig(
privacy_group_id,
fetcher_active=fetcher_active,
privacy_group_name=privacy_group_name,
in_use=in_use,
description=description,
matcher_active=matcher_active,
write_back=write_back,
)
try:
hmaconfig.create_config(config)
except ClientError as e:
if e.response["Error"]["Code"] == "ConditionalCheckFailedException":
logger.warning(
"Can't insert duplicated config, %s",
e.response["Error"]["Message"],
)
if description:
update_privacy_group_description(privacy_group_id, description)
else:
raise
def test_get(self):
a_config = config.HMAConfig("a")
config.create_config(a_config)
self.assertEqual(a_config, config.HMAConfig.get("a"))
self.assertEqual(a_config, config.HMAConfig.getx("a"))
self.assertIsNone(config.HMAConfig.get("b"))
with self.assertRaisesRegex(ValueError, "No HMAConfig named b"):
self.assertIsNone(config.HMAConfig.getx("b"))
def create_action(request: CreateUpdateActionRequest) -> CreateActionResponse:
"""
Create an action.
"""
config = ActionPerformer._get_subtypes_by_name()[request.config_subtype](
**{"name": request.name, **request.fields}
)
hmaconfig.create_config(config)
return CreateActionResponse(response="The action config is created.")
def test_rename_behavior(self):
"""Rename is not fully supported and so generates new records atm"""
a_config = config.HMAConfig("First")
config.create_config(a_config)
a_config.name = "Second"
config.create_config(a_config)
all_configs = config.HMAConfig.get_all()
self.assertEqual({c.name for c in all_configs}, {"First", "Second"})
def assertEqualsAfterDynamodb(self, config_instance):
"""Asserts configs are still equal after writing to DB"""
try:
config.create_config(config_instance)
except ClientError as e:
if e.response["Error"][
"Code"] == "ConditionalCheckFailedException":
print("can't insert duplicated item")
else:
raise
from_db = config_instance.get(config_instance.name)
self.assertEqual(config_instance, from_db)
def do_migrate(self):
create_config(
ToggleableContentTypeConfig(
name=ToggleableContentTypeConfig.get_name_from_type(
VideoContent),
content_type_class=full_class_name(VideoContent),
enabled=True,
))
create_config(
ToggleableContentTypeConfig(
name=ToggleableContentTypeConfig.get_name_from_type(
PhotoContent),
content_type_class=full_class_name(PhotoContent),
enabled=True,
))
create_config(
ToggleableSignalTypeConfig(
name=ToggleableSignalTypeConfig.get_name_from_type(
VideoMD5Signal),
signal_type_class=full_class_name(VideoMD5Signal),
enabled=True,
))
create_config(
ToggleableSignalTypeConfig(
name=ToggleableSignalTypeConfig.get_name_from_type(PdqSignal),
signal_type_class=full_class_name(PdqSignal),
enabled=True,
))
def test_get_all(self):
@dataclass
class GetAllConfig(config.HMAConfig):
a: int = 1
b: str = "a"
c: t.List[str] = field(default_factory=list)
configs_to_make = [
GetAllConfig("a", a=2),
GetAllConfig("b", b="b"),
GetAllConfig("c", c=["a", "b", "c"]),
]
made_configs = []
self.assertCountEqual(made_configs, GetAllConfig.get_all())
for c in configs_to_make:
config.create_config(c)
made_configs.append(c)
self.assertCountEqual(made_configs, GetAllConfig.get_all())
config.delete_config(made_configs[-1])
made_configs = made_configs[:-1]
self.assertCountEqual(made_configs, GetAllConfig.get_all())
def _create_privacy_groups(self):
# Since we already have a mock_dynamodb2 courtesy BanksTableTestBase,
# re-use it for initing configs. Requires some clever hot-wiring.
config_test_mock = config_test.ConfigTest()
config_test_mock.mock_dynamodb2 = self.__class__.mock_dynamodb2
config_test_mock.create_mocked_table()
HMAConfig.initialize(config_test_mock.TABLE_NAME)
# Hot wiring ends...
self.active_pg = ThreatExchangeConfig(
"ACTIVE_PG", True, "", True, True, True, "ACTIVE_PG"
)
create_config(self.active_pg)
# Active PG has a distance threshold of 31.
create_config(AdditionalMatchSettingsConfig("ACTIVE_PG", 31))
self.inactive_pg = ThreatExchangeConfig(
"INACTIVE_PG", True, "", True, True, False, "INACTIVE_PG"
)
create_config(self.inactive_pg)
request: MatchSettingsUpdateRequest, ) -> MatchSettingsUpdateResponse:
"""
Create or update a match settings config for a given privacy_group_id
"""
if config := AdditionalMatchSettingsConfig.get(
request.privacy_group_id):
config.pdq_match_threshold = request.pdq_match_threshold
hmaconfig.update_config(config)
return MatchSettingsUpdateResponse(
f"match_settings updated for pg_id {request.privacy_group_id} with pdq_match_threshold={request.pdq_match_threshold}"
)
config = AdditionalMatchSettingsConfig(request.privacy_group_id,
request.pdq_match_threshold)
hmaconfig.create_config(config)
return MatchSettingsUpdateResponse(
f"match_settings created for pg_id {request.privacy_group_id} with pdq_match_threshold={request.pdq_match_threshold}"
)
@datasets_api.delete("/match-settings/<key>", apply=[jsoninator])
def delete_match_settings(key=None, ) -> MatchSettingsUpdateResponse:
"""
Delete a match settings config for a given privacy_group_id
"""
if config := AdditionalMatchSettingsConfig.get(str(key)):
hmaconfig.delete_config(config)
return MatchSettingsUpdateResponse(
f"match_settings deleted for pg_id {key}")
return bottle.abort(400, f"No match_settings for pg_id {key} found")
def test_subconfigs(self):
class MultiConfig(config.HMAConfigWithSubtypes):
@staticmethod
def get_subtype_classes():
return [
SubtypeOne,
SubtypeTwo,
SubtypeThree,
]
@dataclass
class SubtypeOne(MultiConfig):
a: int
@dataclass
class SubtypeAbstractParentClass(MultiConfig):
a: bool
@dataclass
class SubtypeTwo(MultiConfig):
b: str
@dataclass
class SubtypeThree(SubtypeAbstractParentClass):
a: t.List[float] # type: ignore
one = SubtypeOne("One", 5)
two = SubtypeTwo("Two", "five")
three = SubtypeThree("Three", [5.0, 0.00001]) # ah ah ah
config.create_config(one)
config.create_config(two)
config.create_config(three)
self.assertEqualsAfterDynamodb(one)
self.assertEqualsAfterDynamodb(two)
self.assertEqualsAfterDynamodb(three)
# Getting by the superclass gets you all of them
self.assertCountEqual([one, two, three], MultiConfig.get_all())
self.assertEqual(one, MultiConfig.get("One"))
self.assertEqual(three, MultiConfig.get("Three"))
# Getting by the subclass gets you one of them
self.assertEqual(three, SubtypeThree.get("Three"))
self.assertIsNone(SubtypeOne.get("Three"))
# Renaming behavior stomps on the old one
one_replaced = SubtypeTwo("One", "replaces two")
config.update_config(one_replaced)
self.assertIsNone(SubtypeOne.get("One"))
self.assertEqual(one_replaced, MultiConfig.get("One"))
self.assertEqual(one_replaced, SubtypeTwo.get("One"))
# Writing the superclass gives you an error
with self.assertRaisesRegex(
ValueError,
"Tried to write MultiConfig instead of its subtypes"):
config.create_config(MultiConfig("Foo"))
# Writing the "abstract" config gives you an error
with self.assertRaisesRegex(
ValueError,
"Tried to write subtype SubtypeAbstractParentClass"
" but it's not in get_subtype_classes",
):
config.create_config(SubtypeAbstractParentClass("Foo", False))
return {"action_performed": "true"}
if __name__ == "__main__":
lambda_init_once()
banked_signals = [
BankedSignal("2862392437204724", "bank 4", "te"),
BankedSignal("4194946153908639", "bank 4", "te"),
]
match_message = MatchMessage("key", "hash", banked_signals)
configs: t.List[ActionPerformer] = [
WebhookPostActionPerformer( # type: ignore
"EnqueueForReview",
"https://webhook.site/ff7ebc37-514a-439e-9a03-46f86989e195",
),
]
for performer_config in configs:
config.create_config(performer_config)
action_message = ActionMessage(
"key",
"hash",
matching_banked_signals=banked_signals,
action_label=ActionLabel("EnqueForReview"),
)
event = {"Records": [{"body": action_message.to_aws()}]}
lambda_handler(event, None)
def load_defaults(_args):
"""
Load a hardcoded set of defaults which are useful in testing
"""
# Could also put the default on the class, but seems too fancy
configs = [
ThreatExchangeConfig(
name="303636684709969",
fetcher_active=True,
privacy_group_name="Test Config 1",
write_back=True,
in_use=True,
description="test description",
matcher_active=True,
),
ThreatExchangeConfig(
name="258601789084078",
fetcher_active=True,
privacy_group_name="Test Config 2",
write_back=True,
in_use=True,
description="test description",
matcher_active=True,
),
WebhookPostActionPerformer(
name="EnqueueForReview",
url="https://webhook.site/ff7ebc37-514a-439e-9a03-46f86989e195",
headers='{"Connection":"keep-alive"}',
# monitoring page:
# https://webhook.site/#!/ff7ebc37-514a-439e-9a03-46f86989e195
),
WebhookPostActionPerformer(
name="EnqueueMiniCastleForReview",
url="https://webhook.site/01cef721-bdcc-4681-8430-679c75659867",
headers='{"Connection":"keep-alive"}',
# monitoring page:
# https://webhook.site/#!/01cef721-bdcc-4681-8430-679c75659867
),
WebhookPostActionPerformer(
name="EnqueueSailboatForReview",
url="https://webhook.site/fa5c5ad5-f5cc-4692-bf03-a03a4ae3f714",
headers='{"Connection":"keep-alive"}',
# monitoring page:
# https://webhook.site/#!/fa5c5ad5-f5cc-4692-bf03-a03a4ae3f714
),
ActionRule(
name="Enqueue Mini-Castle for Review",
action_label=ActionLabel("EnqueueMiniCastleForReview"),
must_have_labels=set([
BankIDClassificationLabel("303636684709969"),
ClassificationLabel("true_positive"),
]),
must_not_have_labels=set(
[BankedContentIDClassificationLabel("3364504410306721")]),
),
ActionRule(
name="Enqueue Sailboat for Review",
action_label=ActionLabel("EnqueueSailboatForReview"),
must_have_labels=set([
BankIDClassificationLabel("303636684709969"),
ClassificationLabel("true_positive"),
BankedContentIDClassificationLabel("3364504410306721"),
]),
must_not_have_labels=set(),
),
]
for config in configs:
# Someday maybe can do filtering or something, I dunno
# Add try catch block to avoid test failure
try:
hmaconfig.create_config(config)
except ClientError as e:
if e.response["Error"][
"Code"] == "ConditionalCheckFailedException":
print(
"Can't insert duplicated config, " +
e.response["Error"]["Message"], )
else:
raise
print(config)
