def test_delete(self): a_config = config.HMAConfig("a") config.create_config(a_config) self.assertEqual({c.name for c in config.HMAConfig.get_all()}, {"a"}) config.delete_config(a_config) self.assertEqual({c.name for c in config.HMAConfig.get_all()}, set()) self.assertEqual(None, config.HMAConfig.get("a"))
def update_action_rule( request: ActionRulesRequest, old_name: str, ) -> ActionRulesResponse: """ Update the action rule with name=<oldname>. """ logger.info("old_name: %s", old_name) logger.info("request: %s", request) error_message = "" if ActionRule.exists(request.action_rule.name): try: hmaconfig.update_config(request.action_rule) except Exception as e: error_message = "Unexpected error." handle_unexpected_error(e) elif ActionRule.exists(old_name): try: hmaconfig.create_config(request.action_rule) hmaconfig.delete_config_by_type_and_name("ActionRule", old_name) except Exception as e: error_message = "Unexpected error." handle_unexpected_error(e) else: error_message = f"An action rule named '{request.action_rule.name}' or '{old_name}' does not exist." logger.warning( "An attempt was made to update an action rule named either '%s' or '%s' but neither exist.", request.action_rule.name, old_name, ) response.status = 500 return ActionRulesResponse(error_message)
def create_action_rule( request: ActionRulesRequest, ) -> ActionRulesResponse: """ Create an action rule. """ logger.info("request: %s", request) error_message = "" try: hmaconfig.create_config(request.action_rule) except ClientError as e: # TODO this test for "already exists" should be moved to a common place if e.response["Error"]["Code"] == "ConditionalCheckFailedException": error_message = f"An action rule with the name '{request.action_rule.name}' already exists." logger.warning( "Duplicate action rule creation attempted: %s", e.response["Error"]["Message"], ) else: error_message = "Unexpected error." logger.error( "Unexpected client error: %s", e.response["Error"]["Message"] ) logger.exception(e) response.status = 500 except Exception as e: error_message = "Unexpected error." handle_unexpected_error(e) return ActionRulesResponse(error_message)
def update_dataset(request: UpdateDatasetRequest) -> Dataset: """ Update dataset values: fetcher_active, write_back, and matcher_active. """ config = ThreatExchangeConfig.getx(str(request.privacy_group_id)) config.fetcher_active = request.fetcher_active config.write_back = request.write_back config.matcher_active = request.matcher_active updated_config = hmaconfig.update_config(config).__dict__ updated_config["privacy_group_id"] = updated_config["name"] additional_config = AdditionalMatchSettingsConfig.get( str(request.privacy_group_id)) if request.pdq_match_threshold: if additional_config: additional_config.pdq_match_threshold = int( request.pdq_match_threshold) hmaconfig.update_config(additional_config) else: additional_config = AdditionalMatchSettingsConfig( str(request.privacy_group_id), int(request.pdq_match_threshold)) hmaconfig.create_config(additional_config) elif additional_config: # pdq_match_threshold was set and now should be removed hmaconfig.delete_config(additional_config) return Dataset.from_dict(updated_config)
def sync_privacy_groups(): api_key = AWSSecrets.te_api_key() api = ThreatExchangeAPI(api_key) privacy_group_member_list = api.get_threat_privacy_groups_member() privacy_group_owner_list = api.get_threat_privacy_groups_owner() unique_privacy_groups = set(privacy_group_member_list + privacy_group_owner_list) priavcy_group_id_in_use = set() for privacy_group in unique_privacy_groups: if privacy_group.threat_updates_enabled: # HMA can only read from privacy groups that have threat_updates enabled. # # See here for more details: # https://developers.facebook.com/docs/threat-exchange/reference/apis/threat-updates/v9.0 logger.info("Adding collaboration name %s", privacy_group.name) priavcy_group_id_in_use.add(privacy_group.id) config = ThreatExchangeConfig( privacy_group.id, # TODO Currently default to True for testing purpose, # need to switch it to False before v0 launch fetcher_active=FETCHER_ACTIVE, privacy_group_name=privacy_group.name, in_use=True, write_back=WRITE_BACK, ) try: hmaconfig.create_config(config) except ClientError as e: if e.response["Error"]["Code"] == "ConditionalCheckFailedException": logger.warning( "Can't insert duplicated config, %s", e.response["Error"]["Message"], ) else: raise update_privacy_group_in_use(priavcy_group_id_in_use)
def create_privacy_group_if_not_exists( privacy_group_id: str, privacy_group_name: str, description: str = "", in_use: bool = True, fetcher_active: bool = FETCHER_ACTIVE_DEFAULT, matcher_active: bool = MATCHER_ACTIVE_DEFAULT, write_back: bool = WRITE_BACK_DEFAULT, ): logger.info("Adding collaboration name %s", privacy_group_name) config = ThreatExchangeConfig( privacy_group_id, fetcher_active=fetcher_active, privacy_group_name=privacy_group_name, in_use=in_use, description=description, matcher_active=matcher_active, write_back=write_back, ) try: hmaconfig.create_config(config) except ClientError as e: if e.response["Error"]["Code"] == "ConditionalCheckFailedException": logger.warning( "Can't insert duplicated config, %s", e.response["Error"]["Message"], ) if description: update_privacy_group_description(privacy_group_id, description) else: raise
def test_get(self): a_config = config.HMAConfig("a") config.create_config(a_config) self.assertEqual(a_config, config.HMAConfig.get("a")) self.assertEqual(a_config, config.HMAConfig.getx("a")) self.assertIsNone(config.HMAConfig.get("b")) with self.assertRaisesRegex(ValueError, "No HMAConfig named b"): self.assertIsNone(config.HMAConfig.getx("b"))
def create_action(request: CreateUpdateActionRequest) -> CreateActionResponse: """ Create an action. """ config = ActionPerformer._get_subtypes_by_name()[request.config_subtype]( **{"name": request.name, **request.fields} ) hmaconfig.create_config(config) return CreateActionResponse(response="The action config is created.")
def test_rename_behavior(self): """Rename is not fully supported and so generates new records atm""" a_config = config.HMAConfig("First") config.create_config(a_config) a_config.name = "Second" config.create_config(a_config) all_configs = config.HMAConfig.get_all() self.assertEqual({c.name for c in all_configs}, {"First", "Second"})
def assertEqualsAfterDynamodb(self, config_instance): """Asserts configs are still equal after writing to DB""" try: config.create_config(config_instance) except ClientError as e: if e.response["Error"][ "Code"] == "ConditionalCheckFailedException": print("can't insert duplicated item") else: raise from_db = config_instance.get(config_instance.name) self.assertEqual(config_instance, from_db)
def do_migrate(self): create_config( ToggleableContentTypeConfig( name=ToggleableContentTypeConfig.get_name_from_type( VideoContent), content_type_class=full_class_name(VideoContent), enabled=True, )) create_config( ToggleableContentTypeConfig( name=ToggleableContentTypeConfig.get_name_from_type( PhotoContent), content_type_class=full_class_name(PhotoContent), enabled=True, )) create_config( ToggleableSignalTypeConfig( name=ToggleableSignalTypeConfig.get_name_from_type( VideoMD5Signal), signal_type_class=full_class_name(VideoMD5Signal), enabled=True, )) create_config( ToggleableSignalTypeConfig( name=ToggleableSignalTypeConfig.get_name_from_type(PdqSignal), signal_type_class=full_class_name(PdqSignal), enabled=True, ))
def test_get_all(self): @dataclass class GetAllConfig(config.HMAConfig): a: int = 1 b: str = "a" c: t.List[str] = field(default_factory=list) configs_to_make = [ GetAllConfig("a", a=2), GetAllConfig("b", b="b"), GetAllConfig("c", c=["a", "b", "c"]), ] made_configs = [] self.assertCountEqual(made_configs, GetAllConfig.get_all()) for c in configs_to_make: config.create_config(c) made_configs.append(c) self.assertCountEqual(made_configs, GetAllConfig.get_all()) config.delete_config(made_configs[-1]) made_configs = made_configs[:-1] self.assertCountEqual(made_configs, GetAllConfig.get_all())
def _create_privacy_groups(self): # Since we already have a mock_dynamodb2 courtesy BanksTableTestBase, # re-use it for initing configs. Requires some clever hot-wiring. config_test_mock = config_test.ConfigTest() config_test_mock.mock_dynamodb2 = self.__class__.mock_dynamodb2 config_test_mock.create_mocked_table() HMAConfig.initialize(config_test_mock.TABLE_NAME) # Hot wiring ends... self.active_pg = ThreatExchangeConfig( "ACTIVE_PG", True, "", True, True, True, "ACTIVE_PG" ) create_config(self.active_pg) # Active PG has a distance threshold of 31. create_config(AdditionalMatchSettingsConfig("ACTIVE_PG", 31)) self.inactive_pg = ThreatExchangeConfig( "INACTIVE_PG", True, "", True, True, False, "INACTIVE_PG" ) create_config(self.inactive_pg)
request: MatchSettingsUpdateRequest, ) -> MatchSettingsUpdateResponse: """ Create or update a match settings config for a given privacy_group_id """ if config := AdditionalMatchSettingsConfig.get( request.privacy_group_id): config.pdq_match_threshold = request.pdq_match_threshold hmaconfig.update_config(config) return MatchSettingsUpdateResponse( f"match_settings updated for pg_id {request.privacy_group_id} with pdq_match_threshold={request.pdq_match_threshold}" ) config = AdditionalMatchSettingsConfig(request.privacy_group_id, request.pdq_match_threshold) hmaconfig.create_config(config) return MatchSettingsUpdateResponse( f"match_settings created for pg_id {request.privacy_group_id} with pdq_match_threshold={request.pdq_match_threshold}" ) @datasets_api.delete("/match-settings/<key>", apply=[jsoninator]) def delete_match_settings(key=None, ) -> MatchSettingsUpdateResponse: """ Delete a match settings config for a given privacy_group_id """ if config := AdditionalMatchSettingsConfig.get(str(key)): hmaconfig.delete_config(config) return MatchSettingsUpdateResponse( f"match_settings deleted for pg_id {key}") return bottle.abort(400, f"No match_settings for pg_id {key} found")
def test_subconfigs(self): class MultiConfig(config.HMAConfigWithSubtypes): @staticmethod def get_subtype_classes(): return [ SubtypeOne, SubtypeTwo, SubtypeThree, ] @dataclass class SubtypeOne(MultiConfig): a: int @dataclass class SubtypeAbstractParentClass(MultiConfig): a: bool @dataclass class SubtypeTwo(MultiConfig): b: str @dataclass class SubtypeThree(SubtypeAbstractParentClass): a: t.List[float] # type: ignore one = SubtypeOne("One", 5) two = SubtypeTwo("Two", "five") three = SubtypeThree("Three", [5.0, 0.00001]) # ah ah ah config.create_config(one) config.create_config(two) config.create_config(three) self.assertEqualsAfterDynamodb(one) self.assertEqualsAfterDynamodb(two) self.assertEqualsAfterDynamodb(three) # Getting by the superclass gets you all of them self.assertCountEqual([one, two, three], MultiConfig.get_all()) self.assertEqual(one, MultiConfig.get("One")) self.assertEqual(three, MultiConfig.get("Three")) # Getting by the subclass gets you one of them self.assertEqual(three, SubtypeThree.get("Three")) self.assertIsNone(SubtypeOne.get("Three")) # Renaming behavior stomps on the old one one_replaced = SubtypeTwo("One", "replaces two") config.update_config(one_replaced) self.assertIsNone(SubtypeOne.get("One")) self.assertEqual(one_replaced, MultiConfig.get("One")) self.assertEqual(one_replaced, SubtypeTwo.get("One")) # Writing the superclass gives you an error with self.assertRaisesRegex( ValueError, "Tried to write MultiConfig instead of its subtypes"): config.create_config(MultiConfig("Foo")) # Writing the "abstract" config gives you an error with self.assertRaisesRegex( ValueError, "Tried to write subtype SubtypeAbstractParentClass" " but it's not in get_subtype_classes", ): config.create_config(SubtypeAbstractParentClass("Foo", False))
return {"action_performed": "true"} if __name__ == "__main__": lambda_init_once() banked_signals = [ BankedSignal("2862392437204724", "bank 4", "te"), BankedSignal("4194946153908639", "bank 4", "te"), ] match_message = MatchMessage("key", "hash", banked_signals) configs: t.List[ActionPerformer] = [ WebhookPostActionPerformer( # type: ignore "EnqueueForReview", "https://webhook.site/ff7ebc37-514a-439e-9a03-46f86989e195", ), ] for performer_config in configs: config.create_config(performer_config) action_message = ActionMessage( "key", "hash", matching_banked_signals=banked_signals, action_label=ActionLabel("EnqueForReview"), ) event = {"Records": [{"body": action_message.to_aws()}]} lambda_handler(event, None)
def load_defaults(_args): """ Load a hardcoded set of defaults which are useful in testing """ # Could also put the default on the class, but seems too fancy configs = [ ThreatExchangeConfig( name="303636684709969", fetcher_active=True, privacy_group_name="Test Config 1", write_back=True, in_use=True, description="test description", matcher_active=True, ), ThreatExchangeConfig( name="258601789084078", fetcher_active=True, privacy_group_name="Test Config 2", write_back=True, in_use=True, description="test description", matcher_active=True, ), WebhookPostActionPerformer( name="EnqueueForReview", url="https://webhook.site/ff7ebc37-514a-439e-9a03-46f86989e195", headers='{"Connection":"keep-alive"}', # monitoring page: # https://webhook.site/#!/ff7ebc37-514a-439e-9a03-46f86989e195 ), WebhookPostActionPerformer( name="EnqueueMiniCastleForReview", url="https://webhook.site/01cef721-bdcc-4681-8430-679c75659867", headers='{"Connection":"keep-alive"}', # monitoring page: # https://webhook.site/#!/01cef721-bdcc-4681-8430-679c75659867 ), WebhookPostActionPerformer( name="EnqueueSailboatForReview", url="https://webhook.site/fa5c5ad5-f5cc-4692-bf03-a03a4ae3f714", headers='{"Connection":"keep-alive"}', # monitoring page: # https://webhook.site/#!/fa5c5ad5-f5cc-4692-bf03-a03a4ae3f714 ), ActionRule( name="Enqueue Mini-Castle for Review", action_label=ActionLabel("EnqueueMiniCastleForReview"), must_have_labels=set([ BankIDClassificationLabel("303636684709969"), ClassificationLabel("true_positive"), ]), must_not_have_labels=set( [BankedContentIDClassificationLabel("3364504410306721")]), ), ActionRule( name="Enqueue Sailboat for Review", action_label=ActionLabel("EnqueueSailboatForReview"), must_have_labels=set([ BankIDClassificationLabel("303636684709969"), ClassificationLabel("true_positive"), BankedContentIDClassificationLabel("3364504410306721"), ]), must_not_have_labels=set(), ), ] for config in configs: # Someday maybe can do filtering or something, I dunno # Add try catch block to avoid test failure try: hmaconfig.create_config(config) except ClientError as e: if e.response["Error"][ "Code"] == "ConditionalCheckFailedException": print( "Can't insert duplicated config, " + e.response["Error"]["Message"], ) else: raise print(config)