def test_open(w):
mask = IN_OPEN | IN_CLOSE
w.add('testfile', mask)
watch = w._paths[P('testfile')]
assert len(watch.links) == 2
assert watch.path == P('testfile')
assert watch.watcher == w
assert watch.mask == mask
link1 = watch.links[0]
assert link1.idx == 0
assert link1.path == str(P.cwd())
assert link1.rest == 'testfile'
assert link1.mask == IN_UNMOUNT | IN_ONLYDIR | IN_EXCL_UNLINK | IN_IGNORED | IN_MOVE | IN_DELETE | IN_CREATE
assert link1.watch == watch
wd = link1.wd
assert wd.callbacks['testfile'] == [link1]
assert wd.mask == link1.mask
assert wd.watcher == w
watchdesc = wd.wd
assert w._watchdescriptors[watchdesc] == wd
link2 = watch.links[1]
assert link2.idx == 1
assert link2.path == str(P.cwd()['testfile'])
assert link2.rest == '.'
assert link2.mask == IN_OPEN | IN_CLOSE
assert link2.watch == watch
wd = link2.wd
assert wd.callbacks[None] == [link2]
assert wd.mask == link2.mask
assert wd.watcher == w
watchdesc = wd.wd
assert w._watchdescriptors[watchdesc] == wd
open('testfile').close()
evts = w.read(block=False)
ev1, ev2 = evts
assert ev1.open
assert ev2.close
assert ev2.close_nowrite
os.remove('testfile')
ev3 = w.read(block=False)[0]
assert ev3.path_delete and ev3.path_changed
assert ev3.path == 'testfile'
assert P(ev3.name).parts[-1] == 'testfile'
w.close()
def test_open(w):
mask = IN_OPEN | IN_CLOSE
w.add('testfile', mask)
watch = w._paths[P('testfile')]
assert len(watch.links) == 2
assert watch.path == P('testfile')
assert watch.watcher == w
assert watch.mask == mask
link1 = watch.links[0]
assert link1.idx == 0
assert link1.path == str(P.cwd())
assert link1.rest == 'testfile'
assert link1.mask == IN_UNMOUNT | IN_ONLYDIR | IN_EXCL_UNLINK | IN_IGNORED | IN_MOVE | IN_DELETE | IN_CREATE
assert link1.watch == watch
wd = link1.wd
assert wd.callbacks['testfile'] == [link1]
assert wd.mask == link1.mask
assert wd.watcher == w
watchdesc = wd.wd
assert w._watchdescriptors[watchdesc] == wd
link2 = watch.links[1]
assert link2.idx == 1
assert link2.path == str(P.cwd()['testfile'])
assert link2.rest == '.'
assert link2.mask == IN_OPEN | IN_CLOSE
assert link2.watch == watch
wd = link2.wd
assert wd.callbacks[None] == [link2]
assert wd.mask == link2.mask
assert wd.watcher == w
watchdesc = wd.wd
assert w._watchdescriptors[watchdesc] == wd
open('testfile').close()
evts = w.read(block=False)
ev1, ev2 = evts
assert ev1.open
assert ev2.close
assert ev2.close_nowrite
os.remove('testfile')
ev3 = w.read(block=False)[0]
assert ev3.path_delete and ev3.path_changed
assert ev3.path == 'testfile'
assert P(ev3.name).parts[-1] == 'testfile'
w.close()
def test_multi(w):
open('file2', 'w').close()
os.symlink('file2', 'link2')
os.symlink(str(P.cwd()['link2']), 'link3')
os.symlink('testfile', 'link4')
w.add('link3', IN_OPEN)
w.add('link4', IN_OPEN)
open('file2').close()
evts = w.read()
assert len(evts) == 1
e = evts[0]
assert e.open
assert e.path == 'link3'
open('testfile').close()
evts = w.read()
assert len(evts) == 1
e = evts[0]
assert e.open
assert e.path == 'link4'
assert len(w._watchdescriptors) == 5
os.remove('link3')
os.symlink('link4', 'link3')
evts = w.read()
open('testfile').close()
evts.extend(w.read())
assert len(evts) == 4
def test_db_can_return_type_of_callers_globals(db_path: Path) -> None:
@memoize(db_path=db_path)
def foo():
return PosixPath.cwd()
assert foo() == PosixPath.cwd()
assert isinstance(foo(), PosixPath)
def test_multi(w):
open('file2', 'w').close()
os.symlink('file2', 'link2')
os.symlink(str(P.cwd()['link2']), 'link3')
os.symlink('testfile', 'link4')
w.add('link3', IN_OPEN)
w.add('link4', IN_OPEN)
open('file2').close()
evts = w.read()
assert len(evts) == 1
e = evts[0]
assert e.open
assert e.path == 'link3'
open('testfile').close()
evts = w.read()
assert len(evts) == 1
e = evts[0]
assert e.open
assert e.path == 'link4'
assert len(w._watchdescriptors) == 5
os.remove('link3')
os.symlink('link4', 'link3')
evts = w.read()
open('testfile').close()
evts.extend(w.read())
assert len(evts) == 4
def misc_drdscomps(datadir, outdir, thesis):
"""Generate plots comparing DR and DS (with BDT cuts shown)."""
import tdub.internal.drds as tdid
curdir = PosixPath.cwd().resolve()
if outdir is not None:
outdir = PosixPath(outdir).resolve()
else:
outdir = curdir
outdir.mkdir(exist_ok=True, parents=True)
os.chdir(outdir)
tdid.bdt_cut_plots(datadir, thesis=thesis)
os.chdir(curdir)
def __gen_filename(self, key, version_id=None, filename=None): if filename is not None: filename = PosixPath(filename).resolve() if filename.is_file(): return filename.as_posix() elif filename.is_dir() and filename.exists(): basepath = filename else: return None else: basepath = PosixPath.cwd() leaf = key if version_id is not None: leaf = '%s-%s'%(leaf, version_id) return basepath.joinpath(leaf).as_posix()
def train_shapes(datadir, outdir):
"""Generate shape comparison plots."""
if outdir is None:
outdir = PosixPath.cwd()
else:
outdir = PosixPath(outdir)
outdir.mkdir(exist_ok=True, parents=True)
import tdub.internal.shapecomps as tdisc
import tdub.ml_train as tdmlt
import tdub.config
tdub.config.init_meta_table()
meta_table = tdub.config.PLOTTING_META_TABLE
datadir = PosixPath(datadir)
region = (datadir / "region.txt").read_text().strip()
df, y, w = tdmlt.load_prepped(datadir)
var_region_binning = tdmlt.var_and_binning_for_region(df, region, meta_table)
for v, r, b in var_region_binning:
tdisc.dist_comparison_plot(v, r, b, df, y, w, meta_table, outdir=outdir)
def _update_curdir(self, remember_curdir):
if remember_curdir is True:
self.cwd = PosixPath.cwd()
elif remember_curdir is False:
self.cwd = _PathWatch.curdir
def __init__(self):
self.verbose = True
self.home = PosixPath.cwd()
self.home = PosixPath.cwd()
def log(self, msg, *args):
"""Logs a message to stderr."""
if args:
msg %= args
click.echo(msg, file=sys.stderr)
def vlog(self, msg, *args):
"""Logs a message to stderr only if verbose is enabled."""
if self.verbose:
self.log(msg, *args)
pass_environment = click.make_pass_decorator(Environment, ensure=True)
cmd_folder = PosixPath(PosixPath.cwd()) / 'jeeves' / 'commands'
cmd_folder = cmd_folder.expanduser()
class ComplexCLI(click.MultiCommand):
def list_commands(self, ctx):
rv = []
for filename in cmd_folder.glob('*.py'):
filename = str(filename)
if '__init__.py' in filename:
continue
cmd = filename.partition('/commands/')[2][:-3]
rv.append(cmd)
rv.sort()
return rv
def main():
logger.info('Starting henry')
HELP_PATH = os.path.join(os.path.dirname(henry.__file__),
'.support_files/help.rtf')
with open(HELP_PATH, 'r', encoding='unicode_escape') as myfile:
descStr = myfile.read()
# load custom config settings if defined in ~/.henry/henry.json
settings_file = PosixPath(os.path.join(METADATA_PATH,
'settings.json')).expanduser()
timeout = 120
config_path = PosixPath.cwd().joinpath('config.yml')
if settings_file.is_file():
with open(settings_file, 'r') as f:
settings = json.load(f)
timeout = settings.get('api_conn_timeout', timeout)
if type(timeout) is list:
timeout = tuple(timeout)
config_path = settings.get('config_path', config_path)
logger.info(
f'Loaded config settings from ~/.henry/settings.json, {settings}')
else:
logger.info('No custom config file found. Using defaults.')
parser = argparse.ArgumentParser(
description=descStr,
formatter_class=argparse.RawDescriptionHelpFormatter,
prog='henry',
usage='henry command subcommand '
'[subcommand options] [global '
'options]',
allow_abbrev=False,
add_help=False)
subparsers = parser.add_subparsers(dest='command', help=argparse.SUPPRESS)
parser.add_argument("-h", "--help", action="help", help=argparse.SUPPRESS)
# subparsers.required = True # works, but might do without for now.
pulse = subparsers.add_parser('pulse', help='pulse help')
analyze_parser = subparsers.add_parser('analyze',
help='analyze help',
usage='henry analyze')
analyze_parser.set_defaults(which=None)
analyze_subparsers = analyze_parser.add_subparsers()
analyze_projects = analyze_subparsers.add_parser('projects')
analyze_models = analyze_subparsers.add_parser('models')
analyze_explores = analyze_subparsers.add_parser('explores')
# project subcommand
analyze_projects.set_defaults(which='projects')
analyze_projects.add_argument('-p',
'--project',
type=str,
default=None,
help='Filter on a project')
analyze_projects.add_argument('--order_by',
nargs=2,
metavar=('ORDER_FIELD', 'ASC/DESC'),
dest='sortkey',
help='Sort results by a field')
analyze_projects.add_argument('--limit',
type=int,
default=None,
nargs=1,
help='Limit results. No limit by default')
# models subcommand
analyze_models.set_defaults(which='models')
models_group = analyze_models.add_mutually_exclusive_group()
models_group.add_argument('-p',
'--project',
type=str,
default=None,
help='Filter on project')
models_group.add_argument('-model',
'--model',
type=str,
default=None,
help='Filter on model')
analyze_models.add_argument('--timeframe',
type=int,
default=90,
help='Timeframe (between 0 and 90)')
analyze_models.add_argument('--min_queries',
type=int,
default=0,
help='Query threshold')
analyze_models.add_argument('--order_by',
nargs=2,
metavar=('ORDER_FIELD', 'ASC/DESC'),
dest='sortkey',
help='Sort results by a field')
analyze_models.add_argument('--limit',
type=int,
default=None,
nargs=1,
help='Limit results. No limit by default')
# explores subcommand
analyze_explores.set_defaults(which='explores')
analyze_explores.add_argument('-model',
'--model',
type=str,
default=None,
required=('--explore') in sys.argv,
help='Filter on model')
analyze_explores.add_argument('-e',
'--explore',
default=None,
help='Filter on model')
analyze_explores.add_argument('--timeframe',
type=int,
default=90,
help='Timeframe (between 0 and 90)')
analyze_explores.add_argument('--min_queries',
type=int,
default=0,
help='Query threshold')
analyze_explores.add_argument('--order_by',
nargs=2,
metavar=('ORDER_FIELD', 'ASC/DESC'),
dest='sortkey',
help='Sort results by a field')
analyze_explores.add_argument('--limit',
type=int,
default=None,
nargs=1,
help='Limit results. No limit by default')
# VACUUM Subcommand
vacuum_parser = subparsers.add_parser('vacuum',
help='vacuum help',
usage='henry vacuum')
vacuum_parser.set_defaults(which=None)
vacuum_subparsers = vacuum_parser.add_subparsers()
vacuum_models = vacuum_subparsers.add_parser('models')
vacuum_explores = vacuum_subparsers.add_parser('explores')
vacuum_models.set_defaults(which='models')
vm_group = vacuum_models.add_mutually_exclusive_group()
vm_group.add_argument('-p',
'--project',
type=str,
default=None,
help='Filter on Project')
vm_group.add_argument('-m',
'--model',
type=str,
default=None,
help='Filter on model')
vacuum_models.add_argument('--timeframe',
type=int,
default=90,
help='Usage period to examine (in the range of '
'0-90 days). Default: 90 days.')
vacuum_models.add_argument('--min_queries',
type=int,
default=0,
help='Vacuum threshold. Explores with less '
'queries in the given usage period will '
'be vacuumed. Default: 0 queries.')
vacuum_explores.set_defaults(which='explores')
vacuum_explores.add_argument('-m',
'--model',
type=str,
default=None,
required=('--explore') in sys.argv,
help='Filter on model')
vacuum_explores.add_argument('-e',
'--explore',
type=str,
default=None,
help='Filter on explore')
vacuum_explores.add_argument('--timeframe',
type=int,
default=90,
help='Timeframe (between 0 and 90)')
vacuum_explores.add_argument('--min_queries',
type=int,
default=0,
help='Query threshold')
for subparser in [
analyze_projects, analyze_models, analyze_explores, vacuum_models,
vacuum_explores, pulse
]:
subparser.add_argument('--output',
type=str,
default=None,
help='Path to file for saving the output')
subparser.add_argument('-q',
'--quiet',
action='store_true',
help='Silence output')
subparser.add_argument('--plain',
default=None,
action='store_true',
help='Show results in a table format '
'without the gridlines')
subparser.add_argument_group("Authentication")
subparser.add_argument(
'--host',
type=str,
default='looker',
required=any(
k in sys.argv
for k in ['--client_id', '--cliet_secret', '--alias']),
help=argparse.SUPPRESS)
subparser.add_argument('--port',
type=int,
default=19999,
help=argparse.SUPPRESS)
subparser.add_argument('--client_id',
type=str,
required=any(
k in sys.argv
for k in ['--client_secret', '--alias']),
help=argparse.SUPPRESS)
subparser.add_argument(
'--client_secret',
type=str,
required=any(k in sys.argv for k in ['--client_id', '--alias']),
help=argparse.SUPPRESS)
subparser.add_argument('--persist',
action='store_true',
help=argparse.SUPPRESS)
subparser.add_argument('--alias', type=str, help=argparse.SUPPRESS)
subparser.add_argument('--path',
type=str,
default='',
help=argparse.SUPPRESS)
args = vars(parser.parse_args())
_args = {}
for key, value in args.items():
if key == 'client_secret':
_args[key] = '[FILTERED]'
else:
_args[key] = value
logger.info('Parsing args, %s', _args)
if not args['command']:
print('usage:', parser.usage)
print('\nNo command specified. Try `henry --help` for help.')
sys.exit(1)
auth_params = ('host', 'port', 'client_id', 'client_secret', 'persist',
'alias', 'path')
auth_args = {k: args[k] for k in auth_params}
# authenticate
if args['command'] != 'pulse':
cmd = args['command'] + ' ' + args['which']
else:
cmd = args['command']
session_info = f'Henry v{pkg.__version__}: cmd={cmd}' \
f', sid=#{uuid.uuid1()}'
looker = authenticate(timeout, session_info, config_path, **auth_args)
# map subcommand to function
if args['command'] in ('analyze', 'vacuum'):
if args['which'] is None:
parser.error("No command")
else:
with Spinner():
if args['command'] == 'analyze':
analyze = Analyze(looker)
result = analyze.analyze(**args)
else:
vacuum = Vacuum(looker)
result = vacuum.vacuum(**args)
# silence outout if --silence flag is used
if not args['quiet']:
print(result)
elif args['command'] == 'pulse':
pulse = Pulse(looker)
result = pulse.run_all()
else:
print('No command passed')
# save to file if --output flag is used
if args['output']:
logger.info('Saving results to file: %s', args['output'])
if os.path.isdir(args['output']):
error = IsADirectoryError(errno.EISDIR, os.strerror(errno.EISDIR),
args['output'])
logger.error(error)
raise error
elif not args['output'].endswith('.txt'):
error = ValueError('Output file must be a .txt file')
logger.exception(error)
raise error
elif os.path.isfile(args['output']):
error = FileExistsError(errno.EEXIST, os.strerror(errno.EEXIST),
args['output'])
logger.error(error)
raise error
else:
try:
f = open(args['output'], 'w+')
f.write(result + '\n')
f.close()
logger.info('Results succesfully saved.')
except Exception as e:
logger.error(e)
raise (e)
def foo():
return PosixPath.cwd()
