def get_recent_items_page(
cls,
table: Table,
exclusive_start_key: t.Optional[DynamoDBCursorKey] = None
) -> RecentItems:
"""
Get a paginated list of recent match records. Subsequent calls must use
`return_value.last_evaluated_key`.
"""
if not exclusive_start_key:
# Evidently, https://github.com/boto/boto3/issues/2813 boto is able
# to distinguish fun(Parameter=None) from fun(). So, we can't use
# exclusive_start_key's optionality. We have to do an if clause!
# Fun!
result = table.query(
IndexName="GSI-2",
Limit=100,
ScanIndexForward=False,
KeyConditionExpression=Key("GSI2-PK").eq(
DynamoDBItem.get_dynamodb_type_key(cls.__name__)),
)
else:
result = table.query(
IndexName="GSI-2",
Limit=100,
ExclusiveStartKey=exclusive_start_key,
ScanIndexForward=False,
KeyConditionExpression=Key("GSI2-PK").eq(
DynamoDBItem.get_dynamodb_type_key(cls.__name__)),
)
return RecentItems(
t.cast(DynamoDBCursorKey, result.get("LastEvaluatedKey", None)),
cls._result_items_to_records(result["Items"]),
)
def get_from_content_id(
cls,
table: Table,
content_id: str,
signal_type: t.Optional[t.Type[SignalType]] = None,
) -> t.List["PipelineHashRecord"]:
"""
Returns all available PipelineHashRecords for a content_id.
"""
expected_pk = cls.get_dynamodb_content_key(content_id)
if signal_type is None:
condition_expression = Key("PK").eq(expected_pk) & Key("SK").begins_with(
DynamoDBItem.TYPE_PREFIX
)
else:
condition_expression = Key("PK").eq(expected_pk) & Key("SK").eq(
DynamoDBItem.get_dynamodb_type_key(signal_type.get_name())
)
return cls._result_items_to_records(
table.query(
KeyConditionExpression=condition_expression,
).get("Items", [])
)
def get_from_content_id(cls, table: Table,
content_id: str) -> t.List["MatchRecord"]:
"""
Return all matches for a content_id.
"""
content_key = DynamoDBItem.get_dynamodb_content_key(content_id)
source_prefix = DynamoDBItem.SIGNAL_KEY_PREFIX
return cls._result_items_to_records(
table.query(KeyConditionExpression=Key("PK").eq(content_key)
& Key("SK").begins_with(source_prefix), ).get(
"Items", []))
def get_from_content_id(
cls,
table: Table,
content_id: str,
) -> t.List["ActionEvent"]:
items = table.query(
KeyConditionExpression=Key("PK").eq(
cls.get_dynamodb_content_key(content_id))
& Key("SK").begins_with(cls.ACTION_TIME_PREFIX),
ProjectionExpression=cls.DEFAULT_PROJ_EXP,
).get("Items", [])
return cls._result_item_to_action_event(items)
def get_from_signal(cls, table: Table, signal_id: t.Union[str, int],
signal_source: str) -> t.List["MatchRecord"]:
"""
Return all matches for a signal. Needs source and id to uniquely
identify a signal.
"""
signal_key = DynamoDBItem.get_dynamodb_signal_key(
signal_source, signal_id)
return cls._result_items_to_records(
table.query(
IndexName="GSI-1",
KeyConditionExpression=Key("GSI1-PK").eq(signal_key),
).get("Items", []))
def get_recent_items_page(
cls,
table: Table,
signal_type_mapping: HMASignalTypeMapping,
exclusive_start_key: t.Optional[DynamoDBCursorKey] = None,
) -> PaginatedResponse["PipelineHashRecord"]:
"""
Get a paginated list of recent items.
"""
if not exclusive_start_key:
# Evidently, https://github.com/boto/boto3/issues/2813 boto is able
# to distinguish fun(Parameter=None) from fun(). So, we can't use
# exclusive_start_key's optionality. We have to do an if clause!
# Fun!
result = table.query(
IndexName="GSI-2",
ScanIndexForward=False,
Limit=100,
KeyConditionExpression=Key("GSI2-PK").eq(
DynamoDBItem.get_dynamodb_type_key(cls.__name__)),
)
else:
result = table.query(
IndexName="GSI-2",
ExclusiveStartKey=exclusive_start_key,
ScanIndexForward=False,
Limit=100,
KeyConditionExpression=Key("GSI2-PK").eq(
DynamoDBItem.get_dynamodb_type_key(cls.__name__)),
)
return PaginatedResponse(
t.cast(DynamoDBCursorKey, result.get("LastEvaluatedKey", None)),
cls._result_items_to_records(
result["Items"], signal_type_mapping=signal_type_mapping),
)
def get_from_signal(
cls,
table: Table,
signal_id: t.Union[str, int],
signal_source: str,
) -> t.List["PDQSignalMetadata"]:
items = table.query(
KeyConditionExpression=Key("PK").eq(
cls.get_dynamodb_signal_key(signal_source, signal_id))
& Key("SK").begins_with(cls.DATASET_PREFIX),
ProjectionExpression=
"PK, ContentHash, UpdatedAt, SK, SignalSource, SignalHash, Tags, PendingOpinionChange",
FilterExpression=Attr("HashType").eq(cls.SIGNAL_TYPE),
).get("Items", [])
return cls._result_items_to_metadata(items)
def get_all(cls, of: str, by: str, table: Table) -> t.List["ParameterizedCount"]:
return [
cls(
of,
by,
value=t.cast(str, item.get("SK"))[
cls.SKEY_PREFIX_LENGTH :
], # strip the "val#" portion
cached_value=t.cast(int, item.get("CurrentCount", 0)),
)
for item in table.query(
ScanIndexForward=True,
KeyConditionExpression=Key("PK").eq(
cls._get_pkey_for_parameterized(of, by)
),
)["Items"]
]
def from_time_range(cls,
table: Table,
hash_type: str,
start_time: str = None,
end_time: str = None) -> t.List[t.Dict]:
"""
Given a hash type and time range, give me all the matches found for that type and time range
"""
if start_time is None:
start_time = datetime.datetime.min.isoformat()
if end_time is None:
end_time = datetime.datetime.max.isoformat()
return table.query(
IndexName="GSI-2",
KeyConditionExpression=Key("GSI2-PK").eq(hash_type)
& Key("UpdatedAt").between(start_time, end_time),
ProjectionExpression=cls.DEFAULT_PROJ_EXP,
).get("Items", [])
def from_content_key(table: Table,
content_key: str,
hash_type_key: str = None) -> t.List[t.Dict]:
"""
Given a content key (and optional hash type), return its content hash (for that type).
Written to be agnostic to hash type so it can be reused by other types of 'HashRecord's.
"""
if hash_type_key is None:
key_con_exp = Key("PK").eq(content_key) & Key("SK").begins_with(
DynamoDBItem.SIGNAL_KEY_PREFIX)
else:
key_con_exp = Key("PK").eq(content_key) & Key("SK").eq(
hash_type_key)
return table.query(
KeyConditionExpression=key_con_exp,
ProjectionExpression="PK, ContentHash, UpdatedAt, Quality",
).get("Items", [])
def get_from_signal(
cls,
table: Table,
signal_id: t.Union[str, int],
) -> t.List["ThreatExchangeSignalMetadata"]:
"""
Load objects for this signal across all privacy groups. A signal_id,
which maps to indicator_id on threatexchange, can be part of multiple
privacy groups. Opinions are registered on a (privacy_group,
indicator_id) tuple. Not exactly, but kind of.
"""
pk = cls.get_dynamodb_signal_key(cls.SIGNAL_SOURCE_SHORTCODE, signal_id)
items = table.query(
KeyConditionExpression=Key("PK").eq(pk)
& Key("SK").begins_with(cls.PRIVACY_GROUP_PREFIX),
ProjectionExpression=cls.PROJECTION_EXPRESSION,
).get("Items")
return cls._result_items_to_metadata(items or [])
def from_signal_key(
cls,
table: Table,
signal_key: str,
hash_type: str = None,
) -> t.List[t.Dict]:
"""
Given a Signal ID/Key (and optional hash type), give me any content matches found
"""
filter_exp = None
if not hash_type is None:
filter_exp = Attr("HashType").eq(hash_type)
return table.query(
IndexName="GSI-1",
KeyConditionExpression=Key("GSI1-PK").eq(signal_key),
ProjectionExpression=cls.DEFAULT_PROJ_EXP,
FilterExpression=filter_exp,
).get("Items", [])
def from_content_key(
cls,
table: Table,
content_key: str,
source_prefix: str = DynamoDBItem.SIGNAL_KEY_PREFIX,
hash_type: str = None,
) -> t.List[t.Dict]:
"""
Given a content key (and optional hash type), give me its content hash (for that type).
"""
filter_exp = None
if not hash_type is None:
filter_exp = Attr("HashType").eq(hash_type)
return table.query(
KeyConditionExpression=Key("PK").eq(content_key)
& Key("SK").begins_with(source_prefix),
ProjectionExpression=cls.DEFAULT_PROJ_EXP,
FilterExpression=filter_exp,
).get("Items", [])