def execute(self, settings: CLISettings) -> None: existing = settings.get_collab(self.collab_name) if existing: if self.create: raise CommandError( f'there\'s an existing collaboration named "{self.collab_name}"', 2) if existing.api != self._API_CLS.get_name(): raise CommandError( f"the existing collab is for the {existing.api} api, delete that one first", 2, ) assert (existing.__class__ == self._API_CLS.get_config_class() ), "api name the same, but class different?" for name, val in self.edit_kwargs.items(): setattr(existing, name, val) settings._state.update_collab(existing) elif self.create: logging.debug("Creating config with args: %s", self.edit_kwargs) to_create = self._API_CLS.get_config_class()(**self.edit_kwargs) settings._state.update_collab(to_create) else: raise CommandError("no such config! Did you mean to use --create?", 2)
def execute_remove(self, settings: CLISettings) -> None: if not self.module: raise CommandError("Which module you are remove is required", 2) config = settings.get_persistent_config() if self.module not in config.extensions: raise CommandError(f"You haven't added {self.module}", 2) config.extensions.remove(self.module) settings.set_persistent_config(config)
def execute(self, settings: CLISettings) -> None: if not settings.index.list(): if not settings.in_demo_mode: raise CommandError( "No indices available. Do you need to fetch?") self.stderr( "You haven't built any indices, so we'll call `fetch` for you!" ) FetchCommand().execute(settings) signal_types = settings.get_signal_types_for_content(self.content_type) if self.only_signal: signal_types = [self.only_signal] types: t.Tuple[type, ...] = (FileHasher, MatchesStr) if self.as_hashes: types = (BytesHasher, TextHasher, FileHasher) signal_types = [s for s in signal_types if issubclass(s, types)] if self.as_hashes and len(signal_types) > 1: raise CommandError( "Too many SignalTypes for --as-hashes. Use also --only-signal", 2) logging.info( "Signal types that apply: %s", ", ".join(s.get_name() for s in signal_types) or "None!", ) indices: t.List[t.Tuple[t.Type[SignalType], SignalTypeIndex]] = [] for s_type in signal_types: index = settings.index.load(s_type) if index is None: logging.info("No index for %s, skipping", s_type.get_name()) continue indices.append((s_type, index)) if not indices: self.stderr("No data to match against") return for path in self.files: for s_type, index in indices: seen = set() # TODO - maybe take the highest certainty? results = [] if self.as_hashes: results = _match_hashes(path, s_type, index) else: results = _match_file(path, s_type, index) for r in results: metadatas: t.List[t.Tuple[ str, FetchedSignalMetadata]] = r.metadata for collab, fetched_data in metadatas: if collab in seen: continue seen.add(collab) print(s_type.get_name(), f"- ({collab})", fetched_data)
def get_manifest(self, module_name: str) -> ThreatExchangeExtensionManifest: try: return ThreatExchangeExtensionManifest.load_from_module_name( module_name) except ValueError as ve: raise CommandError(str(ve), 2)
def _read_state( self, collab_name: str, ) -> t.Optional[FetchDeltaTyped]: file = self.collab_file(collab_name) if not file.is_file(): return None try: with file.open("rb") as f: delta = pickle.load(f) assert isinstance(delta, FetchDelta), "Unexpected class type?" delta = t.cast(FetchDeltaTyped, delta) assert (delta.checkpoint.__class__ == self.api_cls. get_checkpoint_cls()), "wrong checkpoint class?" logging.debug("Loaded %s with %d records", collab_name, len(delta.updates)) return delta except Exception: logging.exception("Failed to read state for %s", collab_name) raise CommandError( f"Failed to read state for {collab_name}. " "You might have to delete it with `threatexchange fetch --clear`" )
def execute_import(self, settings: CLISettings, privacy_group_id: int) -> None: api = self.get_te_api(settings) pg = api.get_privacy_group(privacy_group_id) if settings.get_collab(pg.name) is not None: raise CommandError( f"A collaboration already exists with the name {pg.name}", 2) settings._state.update_collab( FBThreatExchangeCollabConfig(name=pg.name, privacy_group=privacy_group_id))
def execute_add(self, settings: CLISettings) -> None: if not self.module: raise CommandError("module is required", 2) if self.module in settings.get_persistent_config().extensions: self.execute_list(settings) return manifest = self.get_manifest(self.module) # Validate our new setups by pretending to create a new mapping with the new classes content_and_settings = tx_meta.SignalTypeMapping( list( itertools.chain(settings.get_all_content_types(), manifest.content_types)), list( itertools.chain(settings.get_all_signal_types(), manifest.signal_types)), ) # For APIs, we also need to make sure they can be instanciated without args for the CLI apis = [] for new_api in manifest.apis: try: instance = new_api() except Exception as e: logging.exception( f"Failed to instanciante API {new_api.get_name()}") raise CommandError( f"Not able to instanciate API {new_api.get_name()} - throws {e}" ) apis.append(instance) apis.extend(settings.apis.get_all()) tx_meta.FetcherMapping(apis) self.execute_list(settings) config = settings.get_persistent_config() config.extensions.add(self.module) settings.set_persistent_config(config)
def execute(self, settings: CLISettings) -> None: hashers = [ s for s in settings.get_signal_types_for_content(self.content_type) if issubclass(s, FileHasher) ] if self.signal_type is not None: if self.signal_type not in hashers: raise CommandError.user( f"{self.signal_type.get_name()} " f"does not apply to {self.content_type.get_name()}" ) hashers = [self.signal_type] # type: ignore # can't detect intersection types for file in self.files: for hasher in hashers: hash_str = hasher.hash_from_file(file) if hash_str: print(hasher.get_name(), hash_str)
def _match_hashes(path: pathlib.Path, s_type: t.Type[SignalType], index: SignalTypeIndex) -> t.List[IndexMatch]: ret = [] for hash in path.read_text().splitlines(): hash = hash.strip() if not hash: continue try: hash = s_type.validate_signal_str(hash) except Exception: logging.exception("%s failed verification on %s", s_type.get_name(), hash) hash_repr = repr(hash) if len(hash_repr) > 50: hash_repr = hash_repr[:47] + "..." raise CommandError( f"{hash_repr} from {path} is not a valid hash for {s_type.get_name()}", 2, ) ret.extend(index.query(hash)) return ret
def execute(self, settings: CLISettings) -> None: if settings.fetched_state.empty(): if not settings.in_demo_mode: raise CommandError( "No stored state available. Do you need to fetch?") # FetchCommand currently imports dataset for index build, so inline for circular import from threatexchange.cli.fetch_cmd import FetchCommand self.stderr( "You haven't fetched any state, so we'll call `fetch` for you!" ) FetchCommand().execute(settings) # Maybe consider subcommands? if self.clear_indices: self.execute_clear_indices(settings) elif self.rebuild_indices: self.execute_generate_indices(settings) elif self.print_signals: self.execute_print_signals(settings) else: assert self.signal_summary self.execute_print_summary(settings)
def __init__( self, content_type: t.Type[ContentType], only_signal: t.Optional[t.Type[SignalType]], hashes: bool, files: t.List[pathlib.Path], show_false_positives: bool, hide_disputed: bool, ) -> None: self.content_type = content_type self.only_signal = only_signal self.as_hashes = hashes self.show_false_positives = show_false_positives self.hide_disputed = hide_disputed self.files = files if only_signal and content_type not in only_signal.get_content_types(): raise CommandError( f"{only_signal.get_name()} does not " f"apply to {content_type.get_name()}", 2, )
def execute(self, settings: CLISettings) -> None: collab = settings.get_collab(self.collab_name) if collab is None: raise CommandError("No such collab", 2) settings._state.delete_collab( collab) # TODO clean private member access
def execute(self, settings: CLISettings) -> None: raise CommandError("subcommand is required", 2)