def __init__(self, limit=None, **kwargs): super().__init__(**kwargs) try: config = PyFileConfigLoader(os.path.expanduser( self.config_file)).load_config() config.merge(self.config) self.update_config(config) except ConfigFileNotFound: pass self.paths = [p for p in self.paths if os.path.isdir(p)] dirs = [] for p in self.paths: for d in glob(os.path.join(p, self.directory_pattern)): if (os.path.isdir(d) and not os.path.islink(d)) \ and len(glob(os.path.join(d, self.file_glob))) > 0: dirs.append(d) if limit is not None and len(dirs) == limit: break if self.hour != -1: dirs = [d for d in dirs if d[-2:] == '{:02}'.format(self.hour)] if self.from_date != '': dirs = [d for d in dirs if d[-10:-2] >= self.from_date] if self.to_date != '': dirs = [d for d in dirs if d[-10:-2] <= self.to_date] self.dirs = sorted(dirs, key=lambda s: s[-10:]) assert len(self.dirs) > 0, "no directories added" self.log.info("WuRFiles initialized with %s directories.", len(self.dirs))
def load_config(): loader = PyFileConfigLoader(CONFIG_FILE_NAME, CONFIG_SEARCH_PATH) try: config = loader.load_config() except ConfigFileNotFound: config = Config() return config
def __init__(self, config={}, **kwargs): try: cfg = PyFileConfigLoader(self.config_file).load_config() cfg.merge(config) super().__init__(config=cfg, **kwargs) except ConfigFileNotFound: print('not found') super().__init__(config=Config(config), **kwargs)
def __init__(self, *args, config={}, **kwargs): try: cfg = PyFileConfigLoader(os.path.expanduser( self.config_file)).load_config() cfg.merge(config) except ConfigFileNotFound: cfg = Config(config) super().__init__(config=cfg, **kwargs)
def test_python(self): fd, fname = mkstemp('.py', prefix=u'μnïcø∂e') f = os.fdopen(fd, 'w') f.write(pyfile) f.close() # Unlink the file cl = PyFileConfigLoader(fname, log=log) config = cl.load_config() self._check_conf(config)
def load_config(self) -> None: """設定ファイルを読み込みパラメータを更新する.""" p = Path(CONFIG_FILE) if p.exists(): # 設定ファイルが存在すれば読み込む. loader = PyFileConfigLoader(str(p)) c = loader.load_config() # 自身のパラメータを更新する. self.update_config(c)
def test_python(self): fd, fname = mkstemp(".py") f = os.fdopen(fd, "w") f.write(pyfile) f.close() # Unlink the file cl = PyFileConfigLoader(fname, log=log) config = cl.load_config() self._check_conf(config)
def get_config_from_file(path, content): # Write config file filename = 'config.py' config_file = path / filename config_file.write_text(content) # Load written file. loader = PyFileConfigLoader(filename, path=str(path)) cfg = loader.load_config() return cfg
def config_ipw(ipw): """Apply and then modify default settings of IPython Qt widget""" ipython_dir = get_ipython_dir() profile_dir = os.path.join(ipython_dir, 'profile_default') cl = PyFileConfigLoader('ipython_qtconsole_config.py', profile_dir) config = cl.load_config() ipw.config = config ipw.set_default_style(colors='Linux') ipw.font = QtGui.QFont('Lucida Console', 11) # 3rd arg can be e.g. QFont.Bold ipw.font.setFixedPitch(True)
def config_ipw(ipw): """Apply and then modify default settings of IPython Qt widget""" ipython_dir = get_ipython_dir() profile_dir = os.path.join(ipython_dir, 'profile_default') cl = PyFileConfigLoader('ipython_config.py', profile_dir) config = cl.load_config() ipw.config = config ipw.set_default_style(colors='Linux') ipw.font = QtGui.QFont('Lucida Console', 11) # 3rd arg can be e.g. QFont.Bold ipw.font.setFixedPitch(True) ipw.buffer_size = 100000 # number of scrollback lines to keep before truncation
def __init__(self) -> None: """コンストラクタ. - 外部設定ファイルが存在する場合は読込み、このクラスのプロパティを更新します. """ p = Path.home() / ".myclock" / "config.py" if p.exists(): # 設定ファイルが存在すれば読み込む. loader = PyFileConfigLoader(str(p)) c = loader.load_config() # 自身のパラメータを更新する. self.update_config(c)
def _binderhub_config(): """Load the binderhub configuration Currently separate from the app fixture so that it can have a different scope (only once per session). """ cfg = PyFileConfigLoader(binderhub_config_path).load_config() global K8S_AVAILABLE try: kubernetes.config.load_kube_config() except Exception: cfg.BinderHub.builder_required = False K8S_AVAILABLE = False if ON_TRAVIS: pytest.fail("Kubernetes should be available on Travis") else: K8S_AVAILABLE = True if REMOTE_BINDER: return # check if Hub is running and ready try: requests.get(cfg.BinderHub.hub_url, timeout=5, allow_redirects=False) except Exception as e: print(f"JupyterHub not available at {cfg.BinderHub.hub_url}: {e}") if ON_TRAVIS: pytest.fail("JupyterHub should be available on Travis") cfg.BinderHub.hub_url = '' else: print(f"JupyterHub available at {cfg.BinderHub.hub_url}") return cfg
def load_jupytext_configuration_file(jupytext_config_file, stream=None): """Read a Jupytext config file, and return a dict""" if not jupytext_config_file.endswith(".py") and stream is None: with open(jupytext_config_file) as stream: return load_jupytext_configuration_file(jupytext_config_file, stream.read()) try: if jupytext_config_file.endswith((".toml", "jupytext")): import toml return toml.loads(stream) if jupytext_config_file.endswith((".yml", ".yaml")): return yaml.safe_load(stream) if jupytext_config_file.endswith(".json"): return json.loads(stream) return PyFileConfigLoader(jupytext_config_file).load_config() except (ValueError, NameError) as err: raise JupytextConfigurationError( "The Jupytext configuration file {} is incorrect: {}".format( jupytext_config_file, err ) )
def _nbexchange_config(): """Load the nbexchange configuration Currently separate from the app fixture so that it can have a different scope (only once per session). """ cfg = PyFileConfigLoader(testing_config).load_config() return cfg
def load_config(self, config_name): cfg = Config() for path in jupyter_config_path(): file = os.path.join(path, config_name) if os.path.isfile(file): new_cfg = PyFileConfigLoader(file).load_config() cfg.merge(new_cfg) return cfg
def get_config(self): cfg = Config() for config_path in jupyter_config_path(): path = os.path.join(config_path, '{}.py'.format(self.config_name)) if os.path.isfile(path): self.recursive_update(cfg, PyFileConfigLoader(path).load_config()) return cfg
def __init__(self, filename, *args, config={}, **kwargs): try: cfg = PyFileConfigLoader(os.path.expanduser( self.config_file)).load_config() cfg.merge(config) except ConfigFileNotFound: pass super().__init__(config=cfg, **kwargs) z = ZipFile(self.path) self._reader = shapereader(**{ k: z.open('{}.{}'.format(filename, k)) for k in ['shp', 'shx', 'dbf'] }) self._geometry_factory = GEOMETRY_FACTORIES.get(self._reader.shapeType) self._fields = self._reader.fields z.close()
def app(request, io_loop, _binderhub_config): """Launch the BinderHub app Currently reads minikube test config from the repo. TODO: support input of test config files. Detects whether kubernetes is available, and if not disables build. Detects whether jupyterhub is available, and if not disables launch. app.url will contain the base URL of binderhub. """ if REMOTE_BINDER: app = RemoteBinderHub() # wait for the remote binder to be up remaining = 30 deadline = time.monotonic() + remaining success = False last_error = None while remaining: try: requests.get(BINDER_URL, timeout=remaining) except Exception as e: print(f"Waiting for binder: {e}") last_error = e time.sleep(1) remaining = deadline - time.monotonic() else: success = True break if not success: raise last_error app.url = BINDER_URL app._configured_bhub = BinderHub(config=_binderhub_config) return app if hasattr(request, 'param') and request.param is True: # load conf for auth test cfg = PyFileConfigLoader( binderhub_config_auth_additions_path).load_config() _binderhub_config.merge(cfg) bhub = BinderHub.instance(config=_binderhub_config) bhub.initialize([]) bhub.start(run_loop=False) # instantiating binderhub configures this # override again AsyncHTTPClient.configure(MockAsyncHTTPClient) def cleanup(): bhub.stop() BinderHub.clear_instance() request.addfinalizer(cleanup) # convenience for accessing binder in tests bhub.url = f'http://127.0.0.1:{bhub.port}{bhub.base_url}'.rstrip('/') return bhub
def load_multi_configs(file_args, cmd_args): """This method try to mimics the behavior of the sub_config. It currently only take one base and one main. Args: file_args: cmd_args: Returns: """ cl_conf = load_command_line_config(cmd_args) if len(file_args) > 0: base_conf = file_args[0] loader = PyFileConfigLoader(base_conf) loader.load_config() for conf in file_args[1:]: # Since subconfig will be merged to and override the base. loader.load_subconfig(conf) all_conf = loader.config all_conf.merge(cl_conf) return all_conf else: return cl_conf
def _binderhub_config(): cfg = PyFileConfigLoader(minikube_testing_config).load_config() cfg.BinderHub.build_namespace = 'binder-test' try: kubernetes.config.load_kube_config() except Exception: cfg.BinderHub.builder_required = False # check if Hub is running and ready try: urlopen(cfg.BinderHub.hub_url, timeout=5).close() except Exception as e: print(f"Hub not available at {cfg.BinderHub.hub_url}: {e}") cfg.BinderHub.hub_url = '' return cfg
def update(self, cpath, source, path): self.log.info('updating indices for {}, {}({})'.format( source, path, cpath)) self.config.merge(PyFileConfigLoader(cpath).load_config()) db = NBSearchDB(config=self.config) gfs = GridFS(db.get_database()) collection = db.get_database()[db.collection] source = get_source(source, self.config) updated = 0 failed = [] for file in source.get_files(): if path is not None and os.path.split( file['path'])[-1] != os.path.split(path)[-1]: continue try: self.update_notebook(source, file, collection, gfs) updated += 1 except: self.log.exception('failed to update index for {}'.format( file['path'])) failed.append(file) indices = collection.index_information() self.log.info('creating indices...') if not any([('_fts', 'text') in i['key'] for i in indices.values()]): collection.create_index([('cells.outputs.text', 'text'), ('cells.source', 'text')]) self.log.info('index created: {}'.format('text')) else: self.log.info('already exists: {}'.format('text')) for field in [ 'atime', 'mtime', 'path', 'server', 'cells.metadata.lc_cell_meme.current', 'cells.metadata.lc_cell_meme.previous', 'cells.metadata.lc_cell_meme.next', ]: if not any([(field, 1) in i['key'] for i in indices.values()]): collection.create_index([(field, 1)]) self.log.info('index created: {}'.format((field, 1))) else: self.log.info('already exists: {}'.format((field, 1))) self.log.info('finished: {} updates, {} fails'.format( updated, len(failed))) if len(failed) > 0: raise RuntimeError('Failed to update: {}'.format(','.join( [f['path'] for f in failed])))
def _get_console_config(self): # ===== THIS ====== try: from traitlets.config.loader import PyFileConfigLoader except ImportError: from IPython.config.loader import PyFileConfigLoader ipcp = os.path.sep.join((os.path.dirname(__file__), "ipython_profile", "ipython_embedded_config.py")) c = PyFileConfigLoader(ipcp).load_config() # ===== OR THIS ===== # import hyperspy.Release # try: # from traitlets.config import Config # except ImportError: # from IPython.config import Config # c = Config() # c.FrontendWidget.banner = hyperspy.Release.info # ===== END ===== return c
def _binderhub_config(): """Load the binderhub configuration Currently separate from the app fixture so that it can have a different scope (only once per session). """ cfg = PyFileConfigLoader(binderhub_config_path).load_config() global K8S_AVAILABLE global K8S_NAMESPACE try: kubernetes.config.load_kube_config() except Exception: cfg.BinderHub.builder_required = False K8S_AVAILABLE = False K8S_NAMESPACE = None else: K8S_AVAILABLE = True # get the current context's namespace or assume it is "default" K8S_NAMESPACE = (subprocess.check_output( [ "kubectl", "config", "view", "--minify", "--output", "jsonpath={..namespace}", ], text=True, ).strip() or "default") if REMOTE_BINDER: return # check if Hub is running and ready try: requests.get(cfg.BinderHub.hub_url, timeout=5, allow_redirects=False) except Exception as e: print(f"JupyterHub not available at {cfg.BinderHub.hub_url}: {e}") cfg.BinderHub.hub_url = "" else: print(f"JupyterHub available at {cfg.BinderHub.hub_url}") return cfg
def _load_dict(directory): #check the config file exists if not os.path.exists(os.path.join(directory, 'jupyterhub_config.py')): sys.exit( f""" There is no jupyterhub_config.py in the specified directory ({directory}). Please specify a directory with a valid jupyterhub_config.py file. """ ) #load the config config = Config() config.merge(PyFileConfigLoader('jupyterhub_config.py', path=directory).load_config()) try: epwrds = config.DictionaryAuthenticator.encrypted_passwords if not isinstance(epwrds, dict): epwrds = epwrds.to_dict() except KeyError as e: print('jupyterhub_config.py does not have an entry for c.DictionaryAuthenticator.encrypted_passwords; continuing with empty dict') epwrds = {} return epwrds
def load_jupytext_configuration_file(jupytext_config_file): """Read a Jupytext config file, and return a JupytextConfig object""" if jupytext_config_file.endswith((".toml", "jupytext")): import toml config = toml.load(jupytext_config_file) return JupytextConfiguration(**config) if jupytext_config_file.endswith((".yml", ".yaml")): with open(jupytext_config_file) as stream: config = yaml.safe_load(stream) return JupytextConfiguration(**config) if jupytext_config_file.endswith(".py"): return JupytextConfiguration( **PyFileConfigLoader(jupytext_config_file).load_config()) return JupytextConfiguration( **JSONFileConfigLoader(jupytext_config_file).load_config())
extent_based_provenance = Bool( help= 'disable extent based provenance for event mentions to use trigger based provenance', default_value=True).tag(config=True) evt_merge_level = Integer( help='How much level of type level to consider when merge events', default_value=0).tag(config=True) if __name__ == '__main__': from event import util import sys conf = PyFileConfigLoader(sys.argv[1]).load_config() cl_conf = util.load_command_line_config(sys.argv[2:]) conf.merge(cl_conf) params = CombineParams(config=conf) log_file = os.path.join(params.output_folder, 'combiner.log') if os.path.exists(log_file): os.remove(log_file) print("Logs will be output at {}".format(log_file)) util.set_file_log(log_file) # util.set_basic_log()
def load_file_config(config_path): loader = PyFileConfigLoader(config_path) conf = loader.load_config() return conf
def get_config(filename='nbx_config.py'): config_dir = jupyter_config_dir() pyloader = PyFileConfigLoader(filename, path=config_dir) config = pyloader.load_config() return config
import os from dotenv import load_dotenv load_dotenv(os.getenv('IGIT_ENV_FILE')) # from igit_debug.loggr import Loggr from traitlets.config import Unicode, Configurable from traitlets.config.loader import PyFileConfigLoader class MyClass(Configurable): name = Unicode(u'defaultname', help="the name of the object").tag(config=True) raise_config_file_errors = True IGIT_DIR = os.environ.get('IGIT_DIR') cl = PyFileConfigLoader('igitconfig.py', path=IGIT_DIR, # log=Loggr() ) cfg = cl.load_config() myinst = MyClass(config=cfg) print(f"{myinst = }") print(f"{myinst.name = }")
def load_all_config(args): cl_conf = load_command_line_config(args[2:]) conf = PyFileConfigLoader(args[1]).load_config() conf.merge(cl_conf) return conf
api_cnt += 1 if not (api_cnt % 5): if self.linker_type == 'tagme': time.sleep(1) d_info[self.linker_type][field] = l_ana print >> out, d_id + '\t' + json.dumps(d_info) if not line_cnt % 100: logging.info('annotated [%d] data', line_cnt) out.close() logging.info('finished [%d] data annotation', line_cnt + 1) if __name__ == '__main__': from knowledge4ir.utils import set_basic_log from traitlets.config.loader import PyFileConfigLoader set_basic_log() if 2 != len(sys.argv): print "1 para: config" Annotator.class_print_help() sys.exit(-1) conf = PyFileConfigLoader(sys.argv[1]).load_config() annotator = Annotator(config=conf) annotator.annotate()
type=int, help="number of train samples") parser.add_argument("--load_model", '-l', action='store_true') parser.add_argument("--test", action="store_true") parser.add_argument("--test_file") parser.add_argument("--test_size", type=int, default=0) parser.add_argument("--output_score_file", '-o') parser.add_argument("--emb_file_path", '-e') parser.add_argument("--checkpoint_dir", '-s', help="store data to here") cmd_line = "sample.config --train --train_file /home/hujun/data/MM2017/kernel/train.txt --validation_file /home/hujun/data/MM2017/kernel/test.txt --train_size 20 --checkpoint_dir /home/hujun/data/MM2017/log" sys.argv = sys.argv[:1] + cmd_line.split() args = parser.parse_args() conf = PyFileConfigLoader(args.config_file_path).load_config() if args.train: nn = Knrm(config=conf) nn.train(train_pair_file_path=args.train_file, val_pair_file_path=args.validation_file, train_size=args.train_size, checkpoint_dir=args.checkpoint_dir, load_model=args.load_model) else: nn = Knrm(config=conf) nn.test(test_point_file_path=args.test_file, test_size=args.test_size, output_file_path=args.output_score_file, load_model=True, checkpoint_dir=args.checkpoint_dir)
def plugin_config(): cfg = PyFileConfigLoader(testing_plugin_config).load_config() return cfg