def sync_privacy_groups():
api_key = AWSSecrets.te_api_key()
api = ThreatExchangeAPI(api_key)
privacy_group_member_list = api.get_threat_privacy_groups_member()
privacy_group_owner_list = api.get_threat_privacy_groups_owner()
unique_privacy_groups = set(privacy_group_member_list + privacy_group_owner_list)
priavcy_group_id_in_use = set()
for privacy_group in unique_privacy_groups:
if privacy_group.threat_updates_enabled:
# HMA can only read from privacy groups that have threat_updates enabled.
# # See here for more details:
# https://developers.facebook.com/docs/threat-exchange/reference/apis/threat-updates/v9.0
logger.info("Adding collaboration name %s", privacy_group.name)
priavcy_group_id_in_use.add(privacy_group.id)
config = ThreatExchangeConfig(
privacy_group.id,
# TODO Currently default to True for testing purpose,
# need to switch it to False before v0 launch
fetcher_active=FETCHER_ACTIVE,
privacy_group_name=privacy_group.name,
in_use=True,
write_back=WRITE_BACK,
)
try:
hmaconfig.create_config(config)
except ClientError as e:
if e.response["Error"]["Code"] == "ConditionalCheckFailedException":
logger.warning(
"Can't insert duplicated config, %s",
e.response["Error"]["Message"],
)
else:
raise
update_privacy_group_in_use(priavcy_group_id_in_use)
def validate_access_token(token: str) -> bool:
access_tokens = AWSSecrets().hma_api_tokens()
if not access_tokens or not token:
logger.debug("No access tokens found")
return False
if token in access_tokens:
return True
return False
def get_pytx_functionality_mapping() -> HMAFunctionalityMapping:
"""
Call from HMA entrypoints. Ensure HMAConfig.initialize() has already been
called.
"""
fetchers: t.List[SignalExchangeAPI] = []
threatexchange_api_token = AWSSecrets().te_api_token()
if threatexchange_api_token not in (None, ""):
fetchers.append(
FBThreatExchangeSignalExchangeAPI(threatexchange_api_token),
)
return HMAFunctionalityMapping(
HMASignalTypeMapping.get_from_config(),
FetcherMapping(fetchers=fetchers),
None,
)
def perform_action(self, match_message: MatchMessage) -> None:
api_key = AWSSecrets.te_api_key()
api = ThreatExchangeAPI(api_key)
indicator_ids = {
dataset_match_details.banked_content_id
for dataset_match_details in match_message.matching_banked_signals
if dataset_match_details.bank_source == "te"
}
descriptor_ids = {
descriptor_id["id"]
for indicator_id in indicator_ids
for descriptor_id in api.get_threat_descriptors_from_indicator(indicator_id)
}
for id in descriptor_ids:
api.react_to_threat_descriptor(id, self.reaction)
logger.info("reacted %s to descriptor %s", self.reaction, id)
def update_threatexchange_token() -> t.Dict:
"""
Given a new threatexchange token as part of the request, makes dummy
call to threatexchange to see if it is a valid token. If found valid,
updates the secret.
Returns 200 if successful, 400 if the token is invalid. Response body is
always and empty JSON object.
Expected JSON Keys:
* `token`: the threatexchange token
"""
token = bottle.request.json["token"]
is_valid_token = try_api_token(token)
if is_valid_token:
AWSSecrets().update_te_api_token(token)
return {}
bottle.response.status_code = 400
return {}
def sync_privacy_groups():
api_key = AWSSecrets().te_api_key()
api = ThreatExchangeAPI(api_key)
privacy_group_member_list = api.get_threat_privacy_groups_member()
privacy_group_owner_list = api.get_threat_privacy_groups_owner()
unique_privacy_groups = set(privacy_group_member_list +
privacy_group_owner_list)
priavcy_group_id_in_use = set()
for privacy_group in unique_privacy_groups:
if privacy_group.threat_updates_enabled:
# HMA can only read from privacy groups that have threat_updates enabled.
# # See here for more details:
# https://developers.facebook.com/docs/threat-exchange/reference/apis/threat-updates/v9.0
priavcy_group_id_in_use.add(privacy_group.id)
create_privacy_group_if_not_exists(
str(privacy_group.id),
privacy_group_name=privacy_group.name,
description=privacy_group.description,
)
update_privacy_groups_in_use(priavcy_group_id_in_use)
def sync_privacy_groups():
api_token = AWSSecrets().te_api_token()
api = ThreatExchangeAPI(api_token)
privacy_group_member_list = api.get_threat_privacy_groups_member()
privacy_group_owner_list = api.get_threat_privacy_groups_owner()
unique_privacy_groups = set(privacy_group_member_list + privacy_group_owner_list)
priavcy_group_id_in_use = set(
SAMPLE_DATASET_PRIVACY_GROUP_ID
) # add sample test dataset id to avoid disable it when syncing from HMA UI
for privacy_group in unique_privacy_groups:
if privacy_group.threat_updates_enabled:
# HMA can only read from privacy groups that have threat_updates enabled.
# # See here for more details:
# https://developers.facebook.com/docs/threat-exchange/reference/apis/threat-updates/v9.0
priavcy_group_id_in_use.add(str(privacy_group.id))
create_privacy_group_if_not_exists(
str(privacy_group.id),
privacy_group_name=privacy_group.name,
description=privacy_group.description,
)
update_privacy_groups_in_use(priavcy_group_id_in_use)
def te_api(self) -> ThreatExchangeAPI:
mock_te_api = os.environ.get("MOCK_TE_API")
if mock_te_api == "True":
return MockedThreatExchangeAPI()
api_token = AWSSecrets().te_api_token()
return ThreatExchangeAPI(api_token)
def lambda_handler(event, context):
config = FetcherConfig.get()
paginator = dynamodb.meta.client.get_paginator("scan")
response_iterator = paginator.paginate(
TableName=config.collab_config_table,
ProjectionExpression=",".join(("#Name", "privacy_group", "tags")),
ExpressionAttributeNames={"#Name": "Name"},
)
collabs = []
for page in response_iterator:
for item in page["Items"]:
collabs.append((item["Name"], item["privacy_group"]))
now = datetime.now()
current_time = now.strftime("%H:%M:%S")
names = [collab[0] for collab in collabs[:5]]
if len(names) < len(collabs):
names[-1] = "..."
data = f"Triggered at time {current_time}, found {len(collabs)} collabs: {', '.join(names)}"
logger.info(data)
api_key = AWSSecrets.te_api_key()
api = ThreatExchangeAPI(api_key)
te_data_bucket = s3.Bucket(config.s3_bucket)
stores = []
for name, privacy_group in collabs:
logger.info("Processing updates for collaboration %s", name)
indicator_store = ThreatUpdateS3PDQStore(
privacy_group,
api.app_id,
te_data_bucket,
config.s3_te_data_folder,
)
stores.append(indicator_store)
indicator_store.load_checkpoint()
if indicator_store.stale:
logger.warning("Store for %s - %d stale! Resetting.", name,
privacy_group)
indicator_store.reset()
if indicator_store.fetch_checkpoint >= now.timestamp():
continue
delta = indicator_store.next_delta
try:
delta.incremental_sync_from_threatexchange(api, )
except:
# Don't need to call .exception() here because we're just re-raising
logger.error("Exception occurred! Attempting to save...")
# Force delta to show finished
delta.end = delta.current
raise
finally:
if delta:
logging.info("Fetch complete, applying %d updates",
len(delta.updates))
indicator_store.apply_updates(delta)
else:
logging.error("Failed before fetching any records")
# TODO add TE data to indexer
return {"statusCode": 200, "body": "Sure Yeah why not"}
def lambda_handler(event, context):
lambda_init_once()
config = FetcherConfig.get()
collabs = ThreatExchangeConfig.get_all()
now = datetime.now()
current_time = now.strftime("%H:%M:%S")
names = [collab.privacy_group_name for collab in collabs[:5]]
if len(names) < len(collabs):
names[-1] = "..."
data = f"Triggered at time {current_time}, found {len(collabs)} collabs: {', '.join(names)}"
logger.info(data)
api_key = AWSSecrets().te_api_key()
api = ThreatExchangeAPI(api_key)
for collab in collabs:
logger.info(
"Processing updates for collaboration %s", collab.privacy_group_name
)
if not is_int(collab.privacy_group_id):
logger.info(
f"Fetch skipped because privacy_group_id({collab.privacy_group_id}) is not an int"
)
continue
indicator_store = ThreatUpdateS3Store(
int(collab.privacy_group_id),
api.app_id,
s3_client=get_s3_client(),
s3_bucket_name=config.s3_bucket,
s3_te_data_folder=config.s3_te_data_folder,
data_store_table=config.data_store_table,
supported_signal_types=[VideoMD5Signal, PdqSignal],
)
indicator_store.load_checkpoint()
if indicator_store.stale:
logger.warning(
"Store for %s - %d stale! Resetting.",
collab.privacy_group_name,
int(collab.privacy_group_id),
)
indicator_store.reset()
if indicator_store.fetch_checkpoint >= now.timestamp():
continue
delta = indicator_store.next_delta
try:
delta.incremental_sync_from_threatexchange(
api,
)
except:
# Don't need to call .exception() here because we're just re-raising
logger.error("Exception occurred! Attempting to save...")
# Force delta to show finished
delta.end = delta.current
raise
finally:
if delta:
logging.info("Fetch complete, applying %d updates", len(delta.updates))
indicator_store.apply_updates(
delta, post_apply_fn=indicator_store.post_apply
)
else:
logging.error("Failed before fetching any records")
def lambda_handler(_event, _context):
"""
Run through threatexchange privacy groups and fetch updates to them. If this
is the first time for a privacy group, will fetch from the start, else only
updates since the last time.
Note: since this is a scheduled job, we swallow all exceptions. We only log
exceptions and move on.
"""
lambda_init_once()
config = FetcherConfig.get()
collabs = ThreatExchangeConfig.get_all()
now = datetime.now()
current_time = now.strftime("%H:%M:%S")
names = [collab.privacy_group_name for collab in collabs[:5]]
if len(names) < len(collabs):
names[-1] = "..."
data = f"Triggered at time {current_time}, found {len(collabs)} collabs: {', '.join(names)}"
logger.info(data)
api_token = AWSSecrets().te_api_token()
api = ThreatExchangeAPI(api_token)
for collab in collabs:
logger.info(
"Processing updates for collaboration %s", collab.privacy_group_name
)
if not is_int(collab.privacy_group_id):
logger.info(
f"Fetch skipped because privacy_group_id({collab.privacy_group_id}) is not an int"
)
continue
if not collab.fetcher_active:
logger.info(
f"Fetch skipped because configs has `fetcher_active` set to false for privacy_group_id({collab.privacy_group_id})"
)
continue
indicator_store = ThreatUpdateS3Store(
int(collab.privacy_group_id),
api.app_id,
s3_client=get_s3_client(),
s3_bucket_name=config.s3_bucket,
s3_te_data_folder=config.s3_te_data_folder,
data_store_table=config.data_store_table,
supported_signal_types=[VideoMD5Signal, PdqSignal],
)
try:
indicator_store.load_checkpoint()
if indicator_store.stale:
logger.warning(
"Store for %s - %d stale! Resetting.",
collab.privacy_group_name,
int(collab.privacy_group_id),
)
indicator_store.reset()
if indicator_store.fetch_checkpoint >= now.timestamp():
continue
delta = indicator_store.next_delta
delta.incremental_sync_from_threatexchange(
api, limit=MAX_DESCRIPTORS_UPDATED, progress_fn=ProgressLogger()
)
except Exception: # pylint: disable=broad-except
logger.exception(
"Encountered exception while getting updates. Will attempt saving.."
)
# Force delta to show finished
delta.end = delta.current
finally:
if delta:
logging.info("Fetch complete, applying %d updates", len(delta.updates))
indicator_store.apply_updates(
delta, post_apply_fn=indicator_store.post_apply
)
else:
logging.error("Failed before fetching any records")
def lambda_handler(event, context):
lambda_init_once()
config = FetcherConfig.get()
collabs = ThreatExchangeConfig.get_all()
now = datetime.now()
current_time = now.strftime("%H:%M:%S")
names = [collab.privacy_group_name for collab in collabs[:5]]
if len(names) < len(collabs):
names[-1] = "..."
data = f"Triggered at time {current_time}, found {len(collabs)} collabs: {', '.join(names)}"
logger.info(data)
api_key = AWSSecrets.te_api_key()
api = ThreatExchangeAPI(api_key)
te_data_bucket = s3.Bucket(config.s3_bucket)
stores = []
for collab in collabs:
logger.info("Processing updates for collaboration %s",
collab.privacy_group_name)
indicator_store = ThreatUpdateS3PDQStore(
collab.privacy_group_id,
api.app_id,
te_data_bucket,
config.s3_te_data_folder,
config.data_store_table,
)
stores.append(indicator_store)
indicator_store.load_checkpoint()
if indicator_store.stale:
logger.warning(
"Store for %s - %d stale! Resetting.",
collab.privacy_group_name,
collab.privacy_group_id,
)
indicator_store.reset()
if indicator_store.fetch_checkpoint >= now.timestamp():
continue
delta = indicator_store.next_delta
try:
delta.incremental_sync_from_threatexchange(api, )
except:
# Don't need to call .exception() here because we're just re-raising
logger.error("Exception occurred! Attempting to save...")
# Force delta to show finished
delta.end = delta.current
raise
finally:
if delta:
logging.info("Fetch complete, applying %d updates",
len(delta.updates))
indicator_store.apply_updates(
delta, post_apply_fn=indicator_store.post_apply)
else:
logging.error("Failed before fetching any records")
# TODO add TE data to indexer
return {"statusCode": 200, "body": "Sure Yeah why not"}