def __main__():
parser = argparse.ArgumentParser()
parser.add_argument("query", type=str, help="S-exp query to execute")
parser.add_argument("-v",
"--verbose",
help="Increase output verbosity",
action="store_true")
parser.add_argument("--node-text",
help="Show the text content of the matched nodes",
action="store_true")
args = parser.parse_args()
if args.verbose:
log.setLevel(log.DEBUG)
log.debug("Verbose logging enabled.")
parsed_s_query = sexp.parseString(args.query)
log.debug(parsed_s_query)
r = find_nodes_by_query(parsed_s_query)
rl = []
for n in r:
log.info(f"{n} in {n.file.fetch()}")
n_sexp = node_as_sexp(n, maxdepth=3, indent=2, show_start_coords=True)
log.info(f"{n_sexp}")
if args.node_text:
log.info(f"{n.text.fetch()}")
rl.append(n)
log.info(f"{len(rl)} results returned")
def __main__():
parser = argparse.ArgumentParser()
parser.add_argument("--nworkers",
type=int,
help="number of workers",
default=os.cpu_count())
parser.add_argument("--jobitems",
type=int,
help="items in a single job",
default=200)
parser.add_argument("--njobs",
type=int,
help="number of total jobs to complete",
default=200)
parser.add_argument("--itemtime",
type=float,
help="time taken per item in a job",
default=0.1)
parser.add_argument("-v",
"--verbose",
help="Increase output verbosity",
action="store_true")
args = parser.parse_args()
if args.verbose:
log.setLevel(log.DEBUG)
log.debug("Verbose logging enabled.")
with ProcessPool(max_workers=args.nworkers) as executor:
multiprogress.main_proc_setup()
multiprogress.start_server_thread()
en_manager_proxy = multiprogress.get_manager_proxy()
en_manager = multiprogress.get_manager()
ret_futures = []
# log.debug(f"counter_generator: {repr(counter_generator)}")
log.info(f"Starting jobs...")
for i in range(args.njobs):
ret_futures.append(
executor.schedule(
slow_worker,
(i, args.jobitems, args.itemtime, en_manager_proxy)))
log.info(f"Waiting for jobs to complete...")
cntr_all_jobs = en_manager.counter(desc="all jobs",
total=args.njobs,
color='blue')
log.debug(f"cntr_all_jobs: {repr(cntr_all_jobs)}")
for f in futures.as_completed(ret_futures):
f.result()
log.debug(f"finished a job!")
cntr_all_jobs.update()
log.info(f"All jobs completed!")
def __main__():
parser = argparse.ArgumentParser()
parser.add_argument("--db",
"--database",
type=str,
help="Database connection string",
default=os.environ.get(
'WST_DB_URI', "http://*****:*****@localhost:8529/wst"))
parser.add_argument("-v",
"--verbose",
help="Increase output verbosity",
action="store_true")
parser.set_defaults(en_manager=enlighten.get_manager())
subcmds = parser.add_subparsers(title="Collector commands")
# analysis
cmd_analyze = subcmds.add_parser('analyze',
aliases=['add', 'a'],
help="Analyze repositories")
cmd_analyze.set_defaults(func=analyze)
cmd_analyze.add_argument("repo_url",
type=str,
help="URI for cloning the repository")
cmd_analyze.add_argument(
"-w",
"--workers",
type=int,
help=
"Number of workers to use for processing files, default: os.cpu_count()",
default=None)
cmd_analyze.add_argument(
"--skip-exists",
"--skip-existing",
action="store_true",
help=
"Skip the analysis if the repo document already exists in the database"
)
cmd_analyze.add_argument(
"--interactive-debug",
action="store_true",
help="Start the interactive debugger after repo setup")
cmd_analyze.add_argument(
"--overwrite-incomplete",
action="store_true",
help="Overwrite existing but incomplete / unfinished data in the DB")
cmd_analyze.add_argument(
"-t",
"--target-commit",
type=str,
help="Checkout and analyze a specific commit from the repo",
default=None)
# batch analysis
cmd_batch = subcmds.add_parser(
'batch',
aliases=['addbatch', 'addmulti'],
help="Analyze multiple repos from a JSON specification list")
set_batch_analyze_args(cmd_batch)
# delete data selectively
cmd_delete = subcmds.add_parser('delete',
aliases=['del'],
help="Delete tree data selectively")
cmd_delete.set_defaults(func=delete)
cmd_delete.add_argument(
"which_repo",
type=str,
help="URI or commit SHA for which repo's data to delete")
# db setup
cmd_db = subcmds.add_parser('db',
aliases=['database'],
help="Manage the database")
subcmds_db = cmd_db.add_subparsers(title="Manage the database")
cmd_db_init = subcmds_db.add_parser('initialize',
aliases=['init', 'setup'],
help="Set up the database")
cmd_db_init.set_defaults(func=database_init)
cmd_db_init.add_argument(
"-d",
"--delete",
help="Delete any existing data in the database",
action="store_true",
)
args = parser.parse_args()
if args.verbose:
log.setLevel(log.DEBUG)
log.debug("Verbose logging enabled.")
log.info(f"DB connection: {desensitize_url(args.db)}")
if 'func' not in args:
log.warn(f"Please supply a valid subcommand!")
return
try:
args.func(args)
except KeyboardInterrupt as e:
log.warn(f"Stopping all child processes...")
cur_proc = psutil.Process()
children = cur_proc.children(recursive=True)
for c in children:
os.kill(c.pid, signal.SIGINT)
psutil.wait_procs(children, timeout=5)
children = cur_proc.children(recursive=True)
for c in children:
c.terminate()
raise e
parser.add_argument("file_path", type=str, help="File to parse")
parser.add_argument(
"-v",
"--verbose",
action="store_true",
)
parser.add_argument("--db",
"--database",
type=str,
help="Database connection string",
default=os.environ.get(
'WST_DB_URI', "http://*****:*****@localhost:8529/wst"))
args = parser.parse_args()
if args.verbose:
log.setLevel(log.DEBUG)
client = ArangoClient(hosts=strip_url(args.db))
p = urlparse(args.db)
db = client.db(p.path[1:], username=p.username, password=p.password)
lang = TreeSitterAutoBuiltLanguage(args.language)
tree = lang.parse_file(args.file_path)
cur = tree.walk()
cur = TreeSitterCursorIterator(cur, nodefilter=lambda x: True)
log.debug(cur)
root = cur.peek()