def backend_status() -> BackendStatusSchema:
"""
Returns the current status of backend services, and returns it. Intended to
be used by the webpage.
This endpoint is not stable and may be subject to change in the future.
"""
agents = []
components = {
"mquery": mquery_version(),
}
for name, agent_spec in db.get_active_agents().items():
try:
ursa = UrsaDb(agent_spec.ursadb_url)
status = ursa.status()
tasks = status["result"]["tasks"]
ursadb_version = status["result"]["ursadb_version"]
agents.append(
AgentSchema(
name=name, alive=True, tasks=tasks, spec=agent_spec
)
)
components[f"ursadb ({name})"] = ursadb_version
except Again:
agents.append(
AgentSchema(name=name, alive=False, tasks=[], spec=agent_spec)
)
components[f"ursadb ({name})"] = "unknown"
return BackendStatusSchema(agents=agents, components=components,)
def backend_status() -> BackendStatusSchema:
agents = []
components = {
"mquery": mquery_version(),
}
for name, agent_spec in db.get_active_agents().items():
try:
ursa = UrsaDb(agent_spec.ursadb_url)
status = ursa.status()
tasks = status["result"]["tasks"]
ursadb_version = status["result"]["ursadb_version"]
agents.append(
AgentSchema(
name=name, alive=True, tasks=tasks, spec=agent_spec
)
)
components[f"ursadb ({name})"] = ursadb_version
except Again:
agents.append(
AgentSchema(name=name, alive=False, tasks=[], spec=agent_spec)
)
components[f"ursadb ({name})"] = "unknown"
return BackendStatusSchema(agents=agents, components=components,)
def main() -> None:
logging.basicConfig(level=logging.INFO)
parser = argparse.ArgumentParser(description="Reindex local files.")
parser.add_argument(
"--mode",
help="Mode of operation. Only prepare batches, index them, or both.",
default="prepare-and-index",
choices=["prepare", "index", "prepare-and-index"],
)
# switches relevant for both "prepare" and "index" modes
parser.add_argument(
"--ursadb",
help="URL of the ursadb instance.",
default="tcp://localhost:9281",
)
parser.add_argument("--workdir",
help="Path to a working directory.",
default=None)
# switches relevant only for "prepare" mode
parser.add_argument("--batch",
help="Size of indexing batch.",
type=int,
default=1000)
parser.add_argument("--path",
help="Path of samples to be indexed.",
default=None)
parser.add_argument(
"--path-mount",
help=
"Path to the samples to be indexed, as seen by ursadb (if different).",
default=None,
)
parser.add_argument(
"--max-file-size-mb",
type=int,
help="Maximum file size, in MB, to index. 128 By default.",
default=128,
)
# switches relevant only for "index" mode
parser.add_argument(
"--type",
dest="types",
help="Index types. By default [gram3, text4, wide8, hash4]",
action="append",
default=[],
choices=["gram3", "text4", "hash4", "wide8"],
)
parser.add_argument(
"--tag",
dest="tags",
help="Additional tags for indexed datasets.",
action="append",
default=[],
)
parser.add_argument(
"--workers",
help="Number of parallel indexing jobs.",
type=int,
default=2,
)
parser.add_argument(
"--working-datasets",
help="Numer of working datasets (uses sane value by default).",
type=int,
default=None,
)
args = parser.parse_args()
types = list(set(args.types))
if args.workdir is None:
logging.error("--workdir is a required parameter")
return
try:
ursa = UrsaDb(args.ursadb)
ursa.status()
except Exception:
logging.error("Can't connect to ursadb instance at %s", args.ursadb)
if args.mode == "prepare" or args.mode == "prepare-and-index":
# Path must exist
if args.path is None:
logging.error("Path (--path) is a required parameter.")
return
if args.path_mount is not None:
path_mount = args.path_mount
else:
path_mount = args.path
path = Path(args.path)
if not path.exists:
logging.error("Path (--path) %s does not exist.", args.path)
return
# We're starting a new indexing operation. Workdir must not exist.
workdir = Path(args.workdir)
if workdir.exists() and list(workdir.iterdir()):
logging.error(
"Workdir %s already exists and is not empty. Remove it or choose another one.",
args.workdir,
)
return
max_file_size = args.max_file_size_mb * 1024 * 1024
prepare(args.ursadb, workdir, path, args.batch, max_file_size,
path_mount)
if args.mode == "index" or args.mode == "prepare-and-index":
# By default use only all index types.
if not args.types:
types = ["gram3", "text4", "wide8", "hash4"]
# We're continuing an existing operation. Workdir must exist.
workdir = Path(args.workdir)
if not workdir.exists():
logging.error(
"Running with mode=index, but workdir %s doesn't exist",
args.workdir,
)
return
index(
args.ursadb,
workdir,
types,
args.tags,
args.workers,
args.working_datasets,
)
logging.info("Indexing finished. Consider compacting the database now")
