def execute(self, settings: CLISettings) -> None:
existing = settings.get_collab(self.collab_name)
if existing:
if self.create:
raise CommandError(
f'there\'s an existing collaboration named "{self.collab_name}"',
2)
if existing.api != self._API_CLS.get_name():
raise CommandError(
f"the existing collab is for the {existing.api} api, delete that one first",
2,
)
assert (existing.__class__ == self._API_CLS.get_config_class()
), "api name the same, but class different?"
for name, val in self.edit_kwargs.items():
setattr(existing, name, val)
settings._state.update_collab(existing)
elif self.create:
logging.debug("Creating config with args: %s", self.edit_kwargs)
to_create = self._API_CLS.get_config_class()(**self.edit_kwargs)
settings._state.update_collab(to_create)
else:
raise CommandError("no such config! Did you mean to use --create?",
2)
def execute_remove(self, settings: CLISettings) -> None:
if not self.module:
raise CommandError("Which module you are remove is required", 2)
config = settings.get_persistent_config()
if self.module not in config.extensions:
raise CommandError(f"You haven't added {self.module}", 2)
config.extensions.remove(self.module)
settings.set_persistent_config(config)
def execute(self, settings: CLISettings) -> None:
if not settings.index.list():
if not settings.in_demo_mode:
raise CommandError(
"No indices available. Do you need to fetch?")
self.stderr(
"You haven't built any indices, so we'll call `fetch` for you!"
)
FetchCommand().execute(settings)
signal_types = settings.get_signal_types_for_content(self.content_type)
if self.only_signal:
signal_types = [self.only_signal]
types: t.Tuple[type, ...] = (FileHasher, MatchesStr)
if self.as_hashes:
types = (BytesHasher, TextHasher, FileHasher)
signal_types = [s for s in signal_types if issubclass(s, types)]
if self.as_hashes and len(signal_types) > 1:
raise CommandError(
"Too many SignalTypes for --as-hashes. Use also --only-signal",
2)
logging.info(
"Signal types that apply: %s",
", ".join(s.get_name() for s in signal_types) or "None!",
)
indices: t.List[t.Tuple[t.Type[SignalType], SignalTypeIndex]] = []
for s_type in signal_types:
index = settings.index.load(s_type)
if index is None:
logging.info("No index for %s, skipping", s_type.get_name())
continue
indices.append((s_type, index))
if not indices:
self.stderr("No data to match against")
return
for path in self.files:
for s_type, index in indices:
seen = set() # TODO - maybe take the highest certainty?
results = []
if self.as_hashes:
results = _match_hashes(path, s_type, index)
else:
results = _match_file(path, s_type, index)
for r in results:
metadatas: t.List[t.Tuple[
str, FetchedSignalMetadata]] = r.metadata
for collab, fetched_data in metadatas:
if collab in seen:
continue
seen.add(collab)
print(s_type.get_name(), f"- ({collab})", fetched_data)
def get_manifest(self,
module_name: str) -> ThreatExchangeExtensionManifest:
try:
return ThreatExchangeExtensionManifest.load_from_module_name(
module_name)
except ValueError as ve:
raise CommandError(str(ve), 2)
def _read_state(
self,
collab_name: str,
) -> t.Optional[FetchDeltaTyped]:
file = self.collab_file(collab_name)
if not file.is_file():
return None
try:
with file.open("rb") as f:
delta = pickle.load(f)
assert isinstance(delta, FetchDelta), "Unexpected class type?"
delta = t.cast(FetchDeltaTyped, delta)
assert (delta.checkpoint.__class__ == self.api_cls.
get_checkpoint_cls()), "wrong checkpoint class?"
logging.debug("Loaded %s with %d records", collab_name,
len(delta.updates))
return delta
except Exception:
logging.exception("Failed to read state for %s", collab_name)
raise CommandError(
f"Failed to read state for {collab_name}. "
"You might have to delete it with `threatexchange fetch --clear`"
)
def execute_import(self, settings: CLISettings,
privacy_group_id: int) -> None:
api = self.get_te_api(settings)
pg = api.get_privacy_group(privacy_group_id)
if settings.get_collab(pg.name) is not None:
raise CommandError(
f"A collaboration already exists with the name {pg.name}", 2)
settings._state.update_collab(
FBThreatExchangeCollabConfig(name=pg.name,
privacy_group=privacy_group_id))
def execute_add(self, settings: CLISettings) -> None:
if not self.module:
raise CommandError("module is required", 2)
if self.module in settings.get_persistent_config().extensions:
self.execute_list(settings)
return
manifest = self.get_manifest(self.module)
# Validate our new setups by pretending to create a new mapping with the new classes
content_and_settings = tx_meta.SignalTypeMapping(
list(
itertools.chain(settings.get_all_content_types(),
manifest.content_types)),
list(
itertools.chain(settings.get_all_signal_types(),
manifest.signal_types)),
)
# For APIs, we also need to make sure they can be instanciated without args for the CLI
apis = []
for new_api in manifest.apis:
try:
instance = new_api()
except Exception as e:
logging.exception(
f"Failed to instanciante API {new_api.get_name()}")
raise CommandError(
f"Not able to instanciate API {new_api.get_name()} - throws {e}"
)
apis.append(instance)
apis.extend(settings.apis.get_all())
tx_meta.FetcherMapping(apis)
self.execute_list(settings)
config = settings.get_persistent_config()
config.extensions.add(self.module)
settings.set_persistent_config(config)
def execute(self, settings: CLISettings) -> None:
hashers = [
s
for s in settings.get_signal_types_for_content(self.content_type)
if issubclass(s, FileHasher)
]
if self.signal_type is not None:
if self.signal_type not in hashers:
raise CommandError.user(
f"{self.signal_type.get_name()} "
f"does not apply to {self.content_type.get_name()}"
)
hashers = [self.signal_type] # type: ignore # can't detect intersection types
for file in self.files:
for hasher in hashers:
hash_str = hasher.hash_from_file(file)
if hash_str:
print(hasher.get_name(), hash_str)
def _match_hashes(path: pathlib.Path, s_type: t.Type[SignalType],
index: SignalTypeIndex) -> t.List[IndexMatch]:
ret = []
for hash in path.read_text().splitlines():
hash = hash.strip()
if not hash:
continue
try:
hash = s_type.validate_signal_str(hash)
except Exception:
logging.exception("%s failed verification on %s",
s_type.get_name(), hash)
hash_repr = repr(hash)
if len(hash_repr) > 50:
hash_repr = hash_repr[:47] + "..."
raise CommandError(
f"{hash_repr} from {path} is not a valid hash for {s_type.get_name()}",
2,
)
ret.extend(index.query(hash))
return ret
def execute(self, settings: CLISettings) -> None:
if settings.fetched_state.empty():
if not settings.in_demo_mode:
raise CommandError(
"No stored state available. Do you need to fetch?")
# FetchCommand currently imports dataset for index build, so inline for circular import
from threatexchange.cli.fetch_cmd import FetchCommand
self.stderr(
"You haven't fetched any state, so we'll call `fetch` for you!"
)
FetchCommand().execute(settings)
# Maybe consider subcommands?
if self.clear_indices:
self.execute_clear_indices(settings)
elif self.rebuild_indices:
self.execute_generate_indices(settings)
elif self.print_signals:
self.execute_print_signals(settings)
else:
assert self.signal_summary
self.execute_print_summary(settings)
def __init__(
self,
content_type: t.Type[ContentType],
only_signal: t.Optional[t.Type[SignalType]],
hashes: bool,
files: t.List[pathlib.Path],
show_false_positives: bool,
hide_disputed: bool,
) -> None:
self.content_type = content_type
self.only_signal = only_signal
self.as_hashes = hashes
self.show_false_positives = show_false_positives
self.hide_disputed = hide_disputed
self.files = files
if only_signal and content_type not in only_signal.get_content_types():
raise CommandError(
f"{only_signal.get_name()} does not "
f"apply to {content_type.get_name()}",
2,
)
def execute(self, settings: CLISettings) -> None:
collab = settings.get_collab(self.collab_name)
if collab is None:
raise CommandError("No such collab", 2)
settings._state.delete_collab(
collab) # TODO clean private member access
def execute(self, settings: CLISettings) -> None:
raise CommandError("subcommand is required", 2)