def from_dict(cls, d, signal_type_mapping: HMASignalTypeMapping):
base = cls(
**{f.name: d.get(f.name, None)
for f in dataclasses.fields(cls)})
base.content_type = signal_type_mapping.get_content_type_enforce(
t.cast(str, base.content_type))
return t.cast("MatchesForMediaRequest", base)
def from_dict(cls, d, signal_type_mapping: HMASignalTypeMapping):
base = cls(
**{f.name: d.get(f.name, None)
for f in dataclasses.fields(cls)})
base.content_type = signal_type_mapping.get_content_type_enforce(
base.content_type # type:ignore
)
return base
def from_dict(
cls, d: t.Dict, signal_type_mapping: HMASignalTypeMapping
) -> "MatchesForHashRequest":
base = cls(
**{f.name: d.get(f.name, None)
for f in dataclasses.fields(cls)})
base.signal_type = signal_type_mapping.get_signal_type_enforce(
t.cast(str, base.signal_type))
return t.cast("MatchesForHashRequest", base)
def from_sqs_message(
cls, d: dict, signal_type_mapping: HMASignalTypeMapping
) -> "URLSubmissionMessage":
return cls(
content_type=signal_type_mapping.get_content_type_enforce(
d["ContentType"]),
content_id=d["ContentId"],
url=d["URL"],
event_type=d["EventType"],
)
def from_sqs_message(
cls, d: dict,
signal_type_mapping: HMASignalTypeMapping) -> "PipelineHashRecord":
return cls(
content_id=d["ContentId"],
signal_type=signal_type_mapping.get_signal_type_enforce(
d["SignalType"]),
content_hash=d["ContentHash"],
signal_specific_attributes=d["SignalSpecificAttributes"],
updated_at=datetime.datetime.fromisoformat(d["UpdatedAt"]),
)
def from_dynamodb_item(
cls, item: t.Dict,
signal_type_mapping: HMASignalTypeMapping) -> "BankMemberSignal":
return cls(
bank_id=item["BankId"],
bank_member_id=item["BankMemberId"],
signal_id=item["SignalId"],
signal_type=signal_type_mapping.get_signal_type_enforce(
item["SignalType"]),
signal_value=item["SignalValue"],
updated_at=datetime.fromisoformat(item["UpdatedAt"]),
)
def _result_item_to_metadata(
cls,
item: t.Dict,
signal_type_mapping: HMASignalTypeMapping,
) -> "ThreatExchangeSignalMetadata":
return ThreatExchangeSignalMetadata(
signal_id=cls.remove_signal_key_prefix(item["PK"],
item["SignalSource"]),
privacy_group_id=item["PrivacyGroup"],
updated_at=datetime.datetime.fromisoformat(item["UpdatedAt"]),
signal_type=signal_type_mapping.get_signal_type_enforce(
item["SignalType"]),
signal_hash=item["SignalHash"],
tags=item["Tags"],
pending_opinion_change=PendingThreatExchangeOpinionChange(
item.get(
"PendingThreatExchangeOpinionChange",
PendingThreatExchangeOpinionChange.NONE.value,
)),
)
def _result_items_to_records(
cls,
items: t.List[t.Dict],
signal_type_mapping: HMASignalTypeMapping,
) -> t.List["PipelineHashRecord"]:
"""
Get a paginated list of recent hash records. Subsequent calls must use
`return_value.last_evaluated_key`.
"""
return [
PipelineHashRecord(
content_id=item["PK"][len(cls.CONTENT_KEY_PREFIX):],
signal_type=signal_type_mapping.get_signal_type_enforce(
item["SignalType"]),
content_hash=item["ContentHash"],
updated_at=datetime.datetime.fromisoformat(item["UpdatedAt"]),
signal_specific_attributes=cls.
deserialize_signal_specific_attributes(item),
) for item in items
]
def from_dynamodb_item(
cls, item: t.Dict,
signal_type_mapping: HMASignalTypeMapping) -> "BankMember":
content_type = signal_type_mapping.get_content_type_enforce(
item["ContentType"])
return cls(
bank_id=item["BankId"],
bank_member_id=item["BankMemberId"],
content_type=content_type,
storage_bucket=item["StorageBucket"],
storage_key=item["StorageKey"],
raw_content=item["RawContent"],
notes=item["Notes"],
created_at=datetime.fromisoformat(item["CreatedAt"]),
updated_at=datetime.fromisoformat(item["UpdatedAt"]),
is_removed=item["IsRemoved"],
is_media_unavailable=item["IsMediaUnavailable"],
# tags default to empty set
bank_member_tags=cls.dynamodb_attribute_to_set(
item.get("BankMemberTags", set())),
)
def _result_items_to_records(
cls,
items: t.List[t.Dict],
signal_type_mapping: HMASignalTypeMapping,
) -> t.List["MatchRecord"]:
return [
MatchRecord(
content_id=cls.remove_content_key_prefix(item["PK"]),
content_hash=item["ContentHash"],
updated_at=datetime.datetime.fromisoformat(item["UpdatedAt"]),
signal_type=signal_type_mapping.get_signal_type_enforce(
item["SignalType"]),
signal_id=cls.remove_signal_key_prefix(item["SK"],
item["SignalSource"]),
signal_source=item["SignalSource"],
signal_hash=item["SignalHash"],
signal_specific_attributes=cls.
deserialize_signal_specific_attributes(item),
match_distance=item.get("MatchDistance"),
) for item in items
]
def _result_item_to_object(
cls, item: t.Dict,
signal_type_mapping: HMASignalTypeMapping) -> "ContentObject":
content_ref_type = ContentRefType(item["ContentRefType"])
content_type = signal_type_mapping.get_content_type_enforce(
item["ContentType"])
# This value is added in the case that no additional fields
# were provided and can be safely discarded.
item["AdditionalFields"].discard(cls.ADDITIONAL_FIELDS_PLACE_HOLDER)
return ContentObject(
content_id=cls.remove_content_key_prefix(item["PK"], ),
content_type=content_type,
content_ref=item["ContentRef"],
content_ref_type=content_ref_type,
additional_fields=item["AdditionalFields"],
# Notes careful not using this version to write back to the table...
# Will dup previous submissions...
submission_times=[
datetime.datetime.fromisoformat(s)
for s in item["SubmissionTimes"]
],
created_at=datetime.datetime.fromisoformat(item["CreatedAt"]),
updated_at=datetime.datetime.fromisoformat(item["UpdatedAt"]),
)
def from_dict(cls, d, signal_type_mapping: HMASignalTypeMapping):
base = super().from_dict(d, signal_type_mapping)
base.signal_type = signal_type_mapping.get_signal_type_enforce(
base.signal_type)
return base
def get_default_signal_type_mapping() -> HMASignalTypeMapping:
return HMASignalTypeMapping(
content_types=[PhotoContent, VideoContent],
signal_types=[PdqSignal, VideoMD5Signal],
)