def __init__(self, limit=None, **kwargs):
super().__init__(**kwargs)
try:
config = PyFileConfigLoader(os.path.expanduser(
self.config_file)).load_config()
config.merge(self.config)
self.update_config(config)
except ConfigFileNotFound:
pass
self.paths = [p for p in self.paths if os.path.isdir(p)]
dirs = []
for p in self.paths:
for d in glob(os.path.join(p, self.directory_pattern)):
if (os.path.isdir(d) and not os.path.islink(d)) \
and len(glob(os.path.join(d, self.file_glob))) > 0:
dirs.append(d)
if limit is not None and len(dirs) == limit:
break
if self.hour != -1:
dirs = [d for d in dirs if d[-2:] == '{:02}'.format(self.hour)]
if self.from_date != '':
dirs = [d for d in dirs if d[-10:-2] >= self.from_date]
if self.to_date != '':
dirs = [d for d in dirs if d[-10:-2] <= self.to_date]
self.dirs = sorted(dirs, key=lambda s: s[-10:])
assert len(self.dirs) > 0, "no directories added"
self.log.info("WuRFiles initialized with %s directories.",
len(self.dirs))
def load_config():
loader = PyFileConfigLoader(CONFIG_FILE_NAME, CONFIG_SEARCH_PATH)
try:
config = loader.load_config()
except ConfigFileNotFound:
config = Config()
return config
def __init__(self, config={}, **kwargs):
try:
cfg = PyFileConfigLoader(self.config_file).load_config()
cfg.merge(config)
super().__init__(config=cfg, **kwargs)
except ConfigFileNotFound:
print('not found')
super().__init__(config=Config(config), **kwargs)
def __init__(self, *args, config={}, **kwargs):
try:
cfg = PyFileConfigLoader(os.path.expanduser(
self.config_file)).load_config()
cfg.merge(config)
except ConfigFileNotFound:
cfg = Config(config)
super().__init__(config=cfg, **kwargs)
def test_python(self):
fd, fname = mkstemp('.py', prefix=u'μnïcø∂e')
f = os.fdopen(fd, 'w')
f.write(pyfile)
f.close()
# Unlink the file
cl = PyFileConfigLoader(fname, log=log)
config = cl.load_config()
self._check_conf(config)
def load_config(self) -> None:
"""設定ファイルを読み込みパラメータを更新する."""
p = Path(CONFIG_FILE)
if p.exists():
# 設定ファイルが存在すれば読み込む.
loader = PyFileConfigLoader(str(p))
c = loader.load_config()
# 自身のパラメータを更新する.
self.update_config(c)
def test_python(self):
fd, fname = mkstemp(".py")
f = os.fdopen(fd, "w")
f.write(pyfile)
f.close()
# Unlink the file
cl = PyFileConfigLoader(fname, log=log)
config = cl.load_config()
self._check_conf(config)
def test_python(self):
fd, fname = mkstemp('.py', prefix=u'μnïcø∂e')
f = os.fdopen(fd, 'w')
f.write(pyfile)
f.close()
# Unlink the file
cl = PyFileConfigLoader(fname, log=log)
config = cl.load_config()
self._check_conf(config)
def get_config_from_file(path, content):
# Write config file
filename = 'config.py'
config_file = path / filename
config_file.write_text(content)
# Load written file.
loader = PyFileConfigLoader(filename, path=str(path))
cfg = loader.load_config()
return cfg
def config_ipw(ipw):
"""Apply and then modify default settings of IPython Qt widget"""
ipython_dir = get_ipython_dir()
profile_dir = os.path.join(ipython_dir, 'profile_default')
cl = PyFileConfigLoader('ipython_qtconsole_config.py', profile_dir)
config = cl.load_config()
ipw.config = config
ipw.set_default_style(colors='Linux')
ipw.font = QtGui.QFont('Lucida Console', 11) # 3rd arg can be e.g. QFont.Bold
ipw.font.setFixedPitch(True)
def config_ipw(ipw):
"""Apply and then modify default settings of IPython Qt widget"""
ipython_dir = get_ipython_dir()
profile_dir = os.path.join(ipython_dir, 'profile_default')
cl = PyFileConfigLoader('ipython_config.py', profile_dir)
config = cl.load_config()
ipw.config = config
ipw.set_default_style(colors='Linux')
ipw.font = QtGui.QFont('Lucida Console', 11) # 3rd arg can be e.g. QFont.Bold
ipw.font.setFixedPitch(True)
ipw.buffer_size = 100000 # number of scrollback lines to keep before truncation
def __init__(self) -> None:
"""コンストラクタ.
- 外部設定ファイルが存在する場合は読込み、このクラスのプロパティを更新します.
"""
p = Path.home() / ".myclock" / "config.py"
if p.exists():
# 設定ファイルが存在すれば読み込む.
loader = PyFileConfigLoader(str(p))
c = loader.load_config()
# 自身のパラメータを更新する.
self.update_config(c)
def config_ipw(ipw):
"""Apply and then modify default settings of IPython Qt widget"""
ipython_dir = get_ipython_dir()
profile_dir = os.path.join(ipython_dir, 'profile_default')
cl = PyFileConfigLoader('ipython_config.py', profile_dir)
config = cl.load_config()
ipw.config = config
ipw.set_default_style(colors='Linux')
ipw.font = QtGui.QFont('Lucida Console',
11) # 3rd arg can be e.g. QFont.Bold
ipw.font.setFixedPitch(True)
ipw.buffer_size = 100000 # number of scrollback lines to keep before truncation
def _binderhub_config():
"""Load the binderhub configuration
Currently separate from the app fixture
so that it can have a different scope (only once per session).
"""
cfg = PyFileConfigLoader(binderhub_config_path).load_config()
global K8S_AVAILABLE
try:
kubernetes.config.load_kube_config()
except Exception:
cfg.BinderHub.builder_required = False
K8S_AVAILABLE = False
if ON_TRAVIS:
pytest.fail("Kubernetes should be available on Travis")
else:
K8S_AVAILABLE = True
if REMOTE_BINDER:
return
# check if Hub is running and ready
try:
requests.get(cfg.BinderHub.hub_url, timeout=5, allow_redirects=False)
except Exception as e:
print(f"JupyterHub not available at {cfg.BinderHub.hub_url}: {e}")
if ON_TRAVIS:
pytest.fail("JupyterHub should be available on Travis")
cfg.BinderHub.hub_url = ''
else:
print(f"JupyterHub available at {cfg.BinderHub.hub_url}")
return cfg
def load_jupytext_configuration_file(jupytext_config_file, stream=None):
"""Read a Jupytext config file, and return a dict"""
if not jupytext_config_file.endswith(".py") and stream is None:
with open(jupytext_config_file) as stream:
return load_jupytext_configuration_file(jupytext_config_file, stream.read())
try:
if jupytext_config_file.endswith((".toml", "jupytext")):
import toml
return toml.loads(stream)
if jupytext_config_file.endswith((".yml", ".yaml")):
return yaml.safe_load(stream)
if jupytext_config_file.endswith(".json"):
return json.loads(stream)
return PyFileConfigLoader(jupytext_config_file).load_config()
except (ValueError, NameError) as err:
raise JupytextConfigurationError(
"The Jupytext configuration file {} is incorrect: {}".format(
jupytext_config_file, err
)
)
def _nbexchange_config():
"""Load the nbexchange configuration
Currently separate from the app fixture
so that it can have a different scope (only once per session).
"""
cfg = PyFileConfigLoader(testing_config).load_config()
return cfg
def load_config(self, config_name):
cfg = Config()
for path in jupyter_config_path():
file = os.path.join(path, config_name)
if os.path.isfile(file):
new_cfg = PyFileConfigLoader(file).load_config()
cfg.merge(new_cfg)
return cfg
def get_config(self):
cfg = Config()
for config_path in jupyter_config_path():
path = os.path.join(config_path, '{}.py'.format(self.config_name))
if os.path.isfile(path):
self.recursive_update(cfg,
PyFileConfigLoader(path).load_config())
return cfg
def __init__(self, filename, *args, config={}, **kwargs):
try:
cfg = PyFileConfigLoader(os.path.expanduser(
self.config_file)).load_config()
cfg.merge(config)
except ConfigFileNotFound:
pass
super().__init__(config=cfg, **kwargs)
z = ZipFile(self.path)
self._reader = shapereader(**{
k: z.open('{}.{}'.format(filename, k))
for k in ['shp', 'shx', 'dbf']
})
self._geometry_factory = GEOMETRY_FACTORIES.get(self._reader.shapeType)
self._fields = self._reader.fields
z.close()
def app(request, io_loop, _binderhub_config):
"""Launch the BinderHub app
Currently reads minikube test config from the repo.
TODO: support input of test config files.
Detects whether kubernetes is available, and if not disables build.
Detects whether jupyterhub is available, and if not disables launch.
app.url will contain the base URL of binderhub.
"""
if REMOTE_BINDER:
app = RemoteBinderHub()
# wait for the remote binder to be up
remaining = 30
deadline = time.monotonic() + remaining
success = False
last_error = None
while remaining:
try:
requests.get(BINDER_URL, timeout=remaining)
except Exception as e:
print(f"Waiting for binder: {e}")
last_error = e
time.sleep(1)
remaining = deadline - time.monotonic()
else:
success = True
break
if not success:
raise last_error
app.url = BINDER_URL
app._configured_bhub = BinderHub(config=_binderhub_config)
return app
if hasattr(request, 'param') and request.param is True:
# load conf for auth test
cfg = PyFileConfigLoader(
binderhub_config_auth_additions_path).load_config()
_binderhub_config.merge(cfg)
bhub = BinderHub.instance(config=_binderhub_config)
bhub.initialize([])
bhub.start(run_loop=False)
# instantiating binderhub configures this
# override again
AsyncHTTPClient.configure(MockAsyncHTTPClient)
def cleanup():
bhub.stop()
BinderHub.clear_instance()
request.addfinalizer(cleanup)
# convenience for accessing binder in tests
bhub.url = f'http://127.0.0.1:{bhub.port}{bhub.base_url}'.rstrip('/')
return bhub
def load_multi_configs(file_args, cmd_args):
"""This method try to mimics the behavior of the sub_config. It currently
only take one base and one main.
Args:
file_args:
cmd_args:
Returns:
"""
cl_conf = load_command_line_config(cmd_args)
if len(file_args) > 0:
base_conf = file_args[0]
loader = PyFileConfigLoader(base_conf)
loader.load_config()
for conf in file_args[1:]:
# Since subconfig will be merged to and override the base.
loader.load_subconfig(conf)
all_conf = loader.config
all_conf.merge(cl_conf)
return all_conf
else:
return cl_conf
def _binderhub_config():
cfg = PyFileConfigLoader(minikube_testing_config).load_config()
cfg.BinderHub.build_namespace = 'binder-test'
try:
kubernetes.config.load_kube_config()
except Exception:
cfg.BinderHub.builder_required = False
# check if Hub is running and ready
try:
urlopen(cfg.BinderHub.hub_url, timeout=5).close()
except Exception as e:
print(f"Hub not available at {cfg.BinderHub.hub_url}: {e}")
cfg.BinderHub.hub_url = ''
return cfg
def update(self, cpath, source, path):
self.log.info('updating indices for {}, {}({})'.format(
source, path, cpath))
self.config.merge(PyFileConfigLoader(cpath).load_config())
db = NBSearchDB(config=self.config)
gfs = GridFS(db.get_database())
collection = db.get_database()[db.collection]
source = get_source(source, self.config)
updated = 0
failed = []
for file in source.get_files():
if path is not None and os.path.split(
file['path'])[-1] != os.path.split(path)[-1]:
continue
try:
self.update_notebook(source, file, collection, gfs)
updated += 1
except:
self.log.exception('failed to update index for {}'.format(
file['path']))
failed.append(file)
indices = collection.index_information()
self.log.info('creating indices...')
if not any([('_fts', 'text') in i['key'] for i in indices.values()]):
collection.create_index([('cells.outputs.text', 'text'),
('cells.source', 'text')])
self.log.info('index created: {}'.format('text'))
else:
self.log.info('already exists: {}'.format('text'))
for field in [
'atime',
'mtime',
'path',
'server',
'cells.metadata.lc_cell_meme.current',
'cells.metadata.lc_cell_meme.previous',
'cells.metadata.lc_cell_meme.next',
]:
if not any([(field, 1) in i['key'] for i in indices.values()]):
collection.create_index([(field, 1)])
self.log.info('index created: {}'.format((field, 1)))
else:
self.log.info('already exists: {}'.format((field, 1)))
self.log.info('finished: {} updates, {} fails'.format(
updated, len(failed)))
if len(failed) > 0:
raise RuntimeError('Failed to update: {}'.format(','.join(
[f['path'] for f in failed])))
def _get_console_config(self):
# ===== THIS ======
try:
from traitlets.config.loader import PyFileConfigLoader
except ImportError:
from IPython.config.loader import PyFileConfigLoader
ipcp = os.path.sep.join((os.path.dirname(__file__), "ipython_profile",
"ipython_embedded_config.py"))
c = PyFileConfigLoader(ipcp).load_config()
# ===== OR THIS =====
# import hyperspy.Release
# try:
# from traitlets.config import Config
# except ImportError:
# from IPython.config import Config
# c = Config()
# c.FrontendWidget.banner = hyperspy.Release.info
# ===== END =====
return c
def _binderhub_config():
"""Load the binderhub configuration
Currently separate from the app fixture
so that it can have a different scope (only once per session).
"""
cfg = PyFileConfigLoader(binderhub_config_path).load_config()
global K8S_AVAILABLE
global K8S_NAMESPACE
try:
kubernetes.config.load_kube_config()
except Exception:
cfg.BinderHub.builder_required = False
K8S_AVAILABLE = False
K8S_NAMESPACE = None
else:
K8S_AVAILABLE = True
# get the current context's namespace or assume it is "default"
K8S_NAMESPACE = (subprocess.check_output(
[
"kubectl",
"config",
"view",
"--minify",
"--output",
"jsonpath={..namespace}",
],
text=True,
).strip() or "default")
if REMOTE_BINDER:
return
# check if Hub is running and ready
try:
requests.get(cfg.BinderHub.hub_url, timeout=5, allow_redirects=False)
except Exception as e:
print(f"JupyterHub not available at {cfg.BinderHub.hub_url}: {e}")
cfg.BinderHub.hub_url = ""
else:
print(f"JupyterHub available at {cfg.BinderHub.hub_url}")
return cfg
def _load_dict(directory):
#check the config file exists
if not os.path.exists(os.path.join(directory, 'jupyterhub_config.py')):
sys.exit(
f"""
There is no jupyterhub_config.py in the specified directory ({directory}).
Please specify a directory with a valid jupyterhub_config.py file.
"""
)
#load the config
config = Config()
config.merge(PyFileConfigLoader('jupyterhub_config.py', path=directory).load_config())
try:
epwrds = config.DictionaryAuthenticator.encrypted_passwords
if not isinstance(epwrds, dict):
epwrds = epwrds.to_dict()
except KeyError as e:
print('jupyterhub_config.py does not have an entry for c.DictionaryAuthenticator.encrypted_passwords; continuing with empty dict')
epwrds = {}
return epwrds
def load_jupytext_configuration_file(jupytext_config_file):
"""Read a Jupytext config file, and return a JupytextConfig object"""
if jupytext_config_file.endswith((".toml", "jupytext")):
import toml
config = toml.load(jupytext_config_file)
return JupytextConfiguration(**config)
if jupytext_config_file.endswith((".yml", ".yaml")):
with open(jupytext_config_file) as stream:
config = yaml.safe_load(stream)
return JupytextConfiguration(**config)
if jupytext_config_file.endswith(".py"):
return JupytextConfiguration(
**PyFileConfigLoader(jupytext_config_file).load_config())
return JupytextConfiguration(
**JSONFileConfigLoader(jupytext_config_file).load_config())
extent_based_provenance = Bool(
help=
'disable extent based provenance for event mentions to use trigger based provenance',
default_value=True).tag(config=True)
evt_merge_level = Integer(
help='How much level of type level to consider when merge events',
default_value=0).tag(config=True)
if __name__ == '__main__':
from event import util
import sys
conf = PyFileConfigLoader(sys.argv[1]).load_config()
cl_conf = util.load_command_line_config(sys.argv[2:])
conf.merge(cl_conf)
params = CombineParams(config=conf)
log_file = os.path.join(params.output_folder, 'combiner.log')
if os.path.exists(log_file):
os.remove(log_file)
print("Logs will be output at {}".format(log_file))
util.set_file_log(log_file)
# util.set_basic_log()
def load_file_config(config_path):
loader = PyFileConfigLoader(config_path)
conf = loader.load_config()
return conf
def get_config(filename='nbx_config.py'):
config_dir = jupyter_config_dir()
pyloader = PyFileConfigLoader(filename, path=config_dir)
config = pyloader.load_config()
return config
import os
from dotenv import load_dotenv
load_dotenv(os.getenv('IGIT_ENV_FILE'))
# from igit_debug.loggr import Loggr
from traitlets.config import Unicode, Configurable
from traitlets.config.loader import PyFileConfigLoader
class MyClass(Configurable):
name = Unicode(u'defaultname', help="the name of the object").tag(config=True)
raise_config_file_errors = True
IGIT_DIR = os.environ.get('IGIT_DIR')
cl = PyFileConfigLoader('igitconfig.py',
path=IGIT_DIR,
# log=Loggr()
)
cfg = cl.load_config()
myinst = MyClass(config=cfg)
print(f"{myinst = }")
print(f"{myinst.name = }")
def load_all_config(args):
cl_conf = load_command_line_config(args[2:])
conf = PyFileConfigLoader(args[1]).load_config()
conf.merge(cl_conf)
return conf
api_cnt += 1
if not (api_cnt % 5):
if self.linker_type == 'tagme':
time.sleep(1)
d_info[self.linker_type][field] = l_ana
print >> out, d_id + '\t' + json.dumps(d_info)
if not line_cnt % 100:
logging.info('annotated [%d] data', line_cnt)
out.close()
logging.info('finished [%d] data annotation', line_cnt + 1)
if __name__ == '__main__':
from knowledge4ir.utils import set_basic_log
from traitlets.config.loader import PyFileConfigLoader
set_basic_log()
if 2 != len(sys.argv):
print "1 para: config"
Annotator.class_print_help()
sys.exit(-1)
conf = PyFileConfigLoader(sys.argv[1]).load_config()
annotator = Annotator(config=conf)
annotator.annotate()
type=int,
help="number of train samples")
parser.add_argument("--load_model", '-l', action='store_true')
parser.add_argument("--test", action="store_true")
parser.add_argument("--test_file")
parser.add_argument("--test_size", type=int, default=0)
parser.add_argument("--output_score_file", '-o')
parser.add_argument("--emb_file_path", '-e')
parser.add_argument("--checkpoint_dir", '-s', help="store data to here")
cmd_line = "sample.config --train --train_file /home/hujun/data/MM2017/kernel/train.txt --validation_file /home/hujun/data/MM2017/kernel/test.txt --train_size 20 --checkpoint_dir /home/hujun/data/MM2017/log"
sys.argv = sys.argv[:1] + cmd_line.split()
args = parser.parse_args()
conf = PyFileConfigLoader(args.config_file_path).load_config()
if args.train:
nn = Knrm(config=conf)
nn.train(train_pair_file_path=args.train_file,
val_pair_file_path=args.validation_file,
train_size=args.train_size,
checkpoint_dir=args.checkpoint_dir,
load_model=args.load_model)
else:
nn = Knrm(config=conf)
nn.test(test_point_file_path=args.test_file,
test_size=args.test_size,
output_file_path=args.output_score_file,
load_model=True,
checkpoint_dir=args.checkpoint_dir)
def plugin_config():
cfg = PyFileConfigLoader(testing_plugin_config).load_config()
return cfg
