def backend_status() -> BackendStatusSchema: """ Returns the current status of backend services, and returns it. Intended to be used by the webpage. This endpoint is not stable and may be subject to change in the future. """ agents = [] components = { "mquery": mquery_version(), } for name, agent_spec in db.get_active_agents().items(): try: ursa = UrsaDb(agent_spec.ursadb_url) status = ursa.status() tasks = status["result"]["tasks"] ursadb_version = status["result"]["ursadb_version"] agents.append( AgentSchema( name=name, alive=True, tasks=tasks, spec=agent_spec ) ) components[f"ursadb ({name})"] = ursadb_version except Again: agents.append( AgentSchema(name=name, alive=False, tasks=[], spec=agent_spec) ) components[f"ursadb ({name})"] = "unknown" return BackendStatusSchema(agents=agents, components=components,)
def backend_status() -> BackendStatusSchema: agents = [] components = { "mquery": mquery_version(), } for name, agent_spec in db.get_active_agents().items(): try: ursa = UrsaDb(agent_spec.ursadb_url) status = ursa.status() tasks = status["result"]["tasks"] ursadb_version = status["result"]["ursadb_version"] agents.append( AgentSchema( name=name, alive=True, tasks=tasks, spec=agent_spec ) ) components[f"ursadb ({name})"] = ursadb_version except Again: agents.append( AgentSchema(name=name, alive=False, tasks=[], spec=agent_spec) ) components[f"ursadb ({name})"] = "unknown" return BackendStatusSchema(agents=agents, components=components,)
def main() -> None: logging.basicConfig(level=logging.INFO) parser = argparse.ArgumentParser(description="Reindex local files.") parser.add_argument( "--mode", help="Mode of operation. Only prepare batches, index them, or both.", default="prepare-and-index", choices=["prepare", "index", "prepare-and-index"], ) # switches relevant for both "prepare" and "index" modes parser.add_argument( "--ursadb", help="URL of the ursadb instance.", default="tcp://localhost:9281", ) parser.add_argument("--workdir", help="Path to a working directory.", default=None) # switches relevant only for "prepare" mode parser.add_argument("--batch", help="Size of indexing batch.", type=int, default=1000) parser.add_argument("--path", help="Path of samples to be indexed.", default=None) parser.add_argument( "--path-mount", help= "Path to the samples to be indexed, as seen by ursadb (if different).", default=None, ) parser.add_argument( "--max-file-size-mb", type=int, help="Maximum file size, in MB, to index. 128 By default.", default=128, ) # switches relevant only for "index" mode parser.add_argument( "--type", dest="types", help="Index types. By default [gram3, text4, wide8, hash4]", action="append", default=[], choices=["gram3", "text4", "hash4", "wide8"], ) parser.add_argument( "--tag", dest="tags", help="Additional tags for indexed datasets.", action="append", default=[], ) parser.add_argument( "--workers", help="Number of parallel indexing jobs.", type=int, default=2, ) parser.add_argument( "--working-datasets", help="Numer of working datasets (uses sane value by default).", type=int, default=None, ) args = parser.parse_args() types = list(set(args.types)) if args.workdir is None: logging.error("--workdir is a required parameter") return try: ursa = UrsaDb(args.ursadb) ursa.status() except Exception: logging.error("Can't connect to ursadb instance at %s", args.ursadb) if args.mode == "prepare" or args.mode == "prepare-and-index": # Path must exist if args.path is None: logging.error("Path (--path) is a required parameter.") return if args.path_mount is not None: path_mount = args.path_mount else: path_mount = args.path path = Path(args.path) if not path.exists: logging.error("Path (--path) %s does not exist.", args.path) return # We're starting a new indexing operation. Workdir must not exist. workdir = Path(args.workdir) if workdir.exists() and list(workdir.iterdir()): logging.error( "Workdir %s already exists and is not empty. Remove it or choose another one.", args.workdir, ) return max_file_size = args.max_file_size_mb * 1024 * 1024 prepare(args.ursadb, workdir, path, args.batch, max_file_size, path_mount) if args.mode == "index" or args.mode == "prepare-and-index": # By default use only all index types. if not args.types: types = ["gram3", "text4", "wide8", "hash4"] # We're continuing an existing operation. Workdir must exist. workdir = Path(args.workdir) if not workdir.exists(): logging.error( "Running with mode=index, but workdir %s doesn't exist", args.workdir, ) return index( args.ursadb, workdir, types, args.tags, args.workers, args.working_datasets, ) logging.info("Indexing finished. Consider compacting the database now")