def __init__(self, _dict=dict, preserve=False): """Create the encoder but override some default functions.""" super(RuleTomlEncoder, self).__init__(_dict, preserve) self._old_dump_str = toml.TomlEncoder().dump_funcs[str] self._old_dump_list = toml.TomlEncoder().dump_funcs[list] self.dump_funcs[str] = self.dump_str self.dump_funcs[type(u"")] = self.dump_str self.dump_funcs[list] = self.dump_list self.dump_funcs[NonformattedField] = self.dump_str
def main(toml_path, toml_template_path, yaml_path, fill_see3cam_id): decoder = toml.TomlDecoder(_dict=OrderedDict) encoder = toml.TomlEncoder(_dict=OrderedDict) f_yaml = open(yaml_path, "r") data = yaml.load(f_yaml) toml.TomlEncoder = encoder dict_toml = toml.load(open(toml_template_path), _dict=OrderedDict, decoder=decoder) if fill_see3cam_id: dict_toml["Rgb"]["device_id"] = get_see3cam_device_id() dict_toml["Rgb"]["width"] = data["cam0"]["resolution"][0] dict_toml["Rgb"]["height"] = data["cam0"]["resolution"][1] dict_toml["Rgb"]["fx"] = data["cam0"]["intrinsics"][0] dict_toml["Rgb"]["fy"] = data["cam0"]["intrinsics"][1] dict_toml["Rgb"]["cx"] = data["cam0"]["intrinsics"][2] dict_toml["Rgb"]["cy"] = data["cam0"]["intrinsics"][3] dict_toml["Rgb"]["k1"] = data["cam0"]["distortion_coeffs"][0] dict_toml["Rgb"]["k2"] = data["cam0"]["distortion_coeffs"][1] dict_toml["Rgb"]["k3"] = data["cam0"]["distortion_coeffs"][2] dict_toml["Rgb"]["k4"] = data["cam0"]["distortion_coeffs"][3] dict_toml["Rgb"]["k5"] = 0 dict_toml["Rgb"]["k6"] = 0 with open(toml_path, "w") as f: toml.encoder.dump(dict_toml, f) print("generated")
def dumps( data: Mapping[str, Any], encoder: Union[Type[toml.TomlEncoder], toml.TomlEncoder] = toml.TomlEncoder, ) -> str: r""" Convert ``data`` to a TOML string. :param data: :param encoder: The :class:`toml.TomlEncoder` to use for constructing the output string. :returns: A string containing the ``TOML`` corresponding to ``data``. .. versionchanged:: 0.5.0 The default value for ``encoder`` changed from :py:obj:`None` to :class:`toml.TomlEncoder` Explicitly passing ``encoder=None`` is deprecated and support will be removed in 1.0.0 """ if isinstance(encoder, type): encoder = encoder() elif encoder is None: warnings.warn( "Passing encoder=None to 'dom_toml.dumps' is deprecated since 0.5.0 and support will be removed in 1.0.0", DeprecationWarning, ) encoder = toml.TomlEncoder() return toml.dumps(data, encoder=encoder)
def all_systems(ctx): systems = [ 'H2', 'B', 'Be', 'LiH', *({'name': 'Hn', 'n': 10, 'dist': d} for d in [1.2, 1.8, 3.6]), ] cass = {'H2': [2, 2], 'B': [4, 3], 'LiH': [4, 2], 'Hn': [6, 4], 'Be': [4, 2]} param_sets = ['SD-SJ', 'SD-SJBF', 'MD-SJ', 'MD-SJBF'] for system, param_set in product(systems, param_sets): sys_name = system if isinstance(system, str) else system['name'] sys_label = sys_name if sys_name == 'Hn': sys_label += f'-{system["dist"]}' path = ctx.obj['basedir'] / sys_label / param_set if path.exists(): continue print(path) params = NestedDict() params['system'] = system if 'MD' in param_set: params['model_kwargs.cas'] = cass[sys_name] if 'BF' not in param_set: params['model_kwargs.omni_kwargs.with_backflow'] = False path.mkdir(parents=True) (path / 'param.toml').write_text(toml.dumps(params, encoder=toml.TomlEncoder()))
def main(out, template): zense = PyPicoZenseManager(0) decoder = toml.TomlDecoder(_dict=OrderedDict) encoder = toml.TomlEncoder(_dict=OrderedDict) toml.TomlEncoder = encoder dict_toml = toml.load(open(template), _dict=OrderedDict, decoder=decoder) dict_toml["Camera0"]["serial_no"] = zense.getSerialNumber().decode() params = zense.getCameraParameter() dict_toml["Camera0"]["fx"] = params[0] dict_toml["Camera0"]["fy"] = params[1] dict_toml["Camera0"]["cx"] = params[2] dict_toml["Camera0"]["cy"] = params[3] dict_toml["Camera0_Factory"]["fx"] = params[0] dict_toml["Camera0_Factory"]["fy"] = params[1] dict_toml["Camera0_Factory"]["cx"] = params[2] dict_toml["Camera0_Factory"]["cy"] = params[3] dict_toml["Camera0_Factory"]["p1"] = params[4] dict_toml["Camera0_Factory"]["p2"] = params[5] dict_toml["Camera0_Factory"]["k1"] = params[6] dict_toml["Camera0_Factory"]["k2"] = params[7] dict_toml["Camera0_Factory"]["k3"] = params[8] dict_toml["Camera0_Factory"]["k4"] = params[9] dict_toml["Camera0_Factory"]["k5"] = params[10] dict_toml["Camera0_Factory"]["k6"] = params[11] with open(out, "w") as f: toml.encoder.dump(dict_toml, f) print("generated") del zense
def _parse_pipfile(self, contents): # If any outline tables are present... if ("[packages." in contents) or ("[dev-packages." in contents): data = toml.loads(contents) # Convert all outline tables to inline tables. for section in ("packages", "dev-packages"): for package in data.get(section, {}): # Convert things to inline tables — fancy :) if hasattr(data[section][package], "keys"): _data = data[section][package] data[section][package] = toml.TomlDecoder( ).get_empty_inline_table() data[section][package].update(_data) toml_encoder = toml.TomlEncoder(preserve=True) # We lose comments here, but it's for the best.) try: return contoml.loads(toml.dumps(data, encoder=toml_encoder)) except RuntimeError: return toml.loads(toml.dumps(data, encoder=toml_encoder)) else: # Fallback to toml parser, for large files. try: return contoml.loads(contents) except Exception: return toml.loads(contents)
def leanpkg_upgrade_proc(): with open('leanpkg.toml', 'r') as lean_toml: local_toml = toml.loads(lean_toml.read()) local_lean_version = local_toml['package']['lean_version'] urllib.request.urlretrieve( 'https://raw.githubusercontent.com/leanprover-community/mathlib/master/leanpkg.toml', 'mathlib_leanpkg.toml') with open('mathlib_leanpkg.toml', 'r') as lean_toml: mathlib_toml = toml.loads(lean_toml.read()) mathlib_lean_version = mathlib_toml['package']['lean_version'] lean_version_prefix = 'leanprover-community/lean:' if local_lean_version.startswith( lean_version_prefix) and mathlib_lean_version.startswith( lean_version_prefix): local_lean_version_int = [ int(i) for i in local_lean_version[len(lean_version_prefix):].split('.') ] mathlib_lean_version_int = [ int(i) for i in mathlib_lean_version[len(lean_version_prefix):].split('.') ] print(mathlib_lean_version_int, local_lean_version_int) if mathlib_lean_version_int > local_lean_version_int: local_toml['package']['lean_version'] = mathlib_lean_version with open('leanpkg.toml', 'w') as lean_toml: # `preserve = True` seems to be an undocumented feature of `toml`. # without it, when the project has exactly one dependency, # the resulting `leanpkg.toml` is malformed: it compresses the `dependencies` section # into `[dependencies.repo_name]`. toml.dump(local_toml, lean_toml, encoder=toml.TomlEncoder(preserve=True)) return subprocess.Popen(['leanpkg', 'upgrade'])
def boron(ctx): payload = chain( product( ['never', 'training', 'always'], [(None, 1), ([4, 3], 3), ([8, 3], 16), ([8, 3], 100)], [100, 10], [(None, {})], ), product( ['training'], [(None, 1), ([8, 3], 16)], [100, 10], [('lrx2', {'train_kwargs.lr_scheduler_kwargs.CyclicLR.max_lr': 0.02})], ), product( ['training'], [(None, 1), ([8, 3], 16)], [100, 10], [('lrx3', {'train_kwargs.lr_scheduler_kwargs.CyclicLR.max_lr': 0.03})], ), product( ['training'], [(None, 1), ([8, 3], 16)], [100, 10], [('qx2', {'train_kwargs.fit_kwargs.q': 10})], ), ) for use_slgld, (cas, conf_lim), epoch_size, (extra_lbl, extra) in payload: label = f'{use_slgld}_ndet-{conf_lim}_epoch-{epoch_size}' if extra: label += f'_{extra_lbl}' path = ctx.obj['basedir'] / label if path.exists(): continue print(path) param = NestedDict() param['system'] = 'B' param['train_kwargs.n_steps'] = 10_000 param['train_kwargs.batch_size'] = 10_000 param['train_kwargs.fit_kwargs.subbatch_size'] = 5_000 param['train_kwargs.sampler_kwargs.n_discard'] = 10 param['model_kwargs.pauli_kwargs.conf_cutoff'] = 1e-8 param['model_kwargs.pauli_kwargs.rc_scaling'] = 3.0 param['model_kwargs.pauli_kwargs.cusp_alpha'] = 3.0 param['model_kwargs.omni_kwargs.subnet_kwargs.n_layers_h'] = 2 param['model_kwargs.pauli_kwargs.conf_limit'] = conf_lim param['model_kwargs.pauli_kwargs.use_sloglindet'] = use_slgld param['model_kwargs.cas'] = cas param['train_kwargs.epoch_size'] = epoch_size for k, v in extra.items(): param[k] = v path.mkdir(parents=True) (path / 'param.toml').write_text(toml.dumps(param, encoder=toml.TomlEncoder()))
def sampling(ctx, training): for param_path in Path(training).glob('**/param.toml'): train_path = param_path.parent label = train_path.relative_to(training) chkpts = [ p.stem.split('-')[1] for p in (train_path / 'chkpts').glob('state-*.pt') ] if not chkpts: continue step = max(chkpts) path = ctx.obj['basedir'] / label print(path) params = toml.loads((train_path / 'param.toml').read_text()) train_path = Path(os.path.relpath(train_path.resolve(), path.resolve())) path.mkdir(parents=True) (path / 'param.toml').write_text(toml.dumps(params, encoder=toml.TomlEncoder())) (path / 'state.pt').symlink_to(train_path / f'chkpts/state-{step}.pt')
def sampling_states(ctx, state, param): base = os.path.commonpath(state) if param: param = toml.loads(Path(param).read_text()) for state_path in sorted(state): label = os.path.splitext(os.path.relpath(state_path, base))[0] train_path = Path(state_path).parents[1] params = NestedDict() path = ctx.obj['basedir'] / label print(path) params_train = toml.loads((train_path / 'param.toml').read_text()) params.update(params_train) if param: params.update(param) path.mkdir(parents=True) (path / 'param.toml').write_text(toml.dumps(params, encoder=toml.TomlEncoder())) state_path = Path(os.path.relpath(Path(state_path).resolve(), path.resolve())) (path / 'state.pt').symlink_to(state_path)
def main(out): zense = PyPicoZenseManager(0) decoder = toml.TomlDecoder(_dict=OrderedDict) encoder = toml.TomlEncoder(_dict=OrderedDict) toml.TomlEncoder = encoder dict_toml = toml.load(open('template.toml'), _dict=OrderedDict, decoder=decoder) dict_toml["Camera0"]["serial_no"] = zense.getSerialNumber() intrinsic_depth_params = zense.getCameraParameter() intrinsic_rgb_params = zense.getRGBCameraParameter() intrinsic_elems = [ "fx", "fy", "cx", "cy", "p1", "p2", "k1", "k2", "k3", "k4", "k5", "k6" ] for i, _elem in enumerate(intrinsic_elems): dict_toml["Camera0_Factory"][_elem] = intrinsic_depth_params[i] for i, _elem in enumerate(intrinsic_elems): dict_toml["Camera0_RGB_Factory"][_elem] = intrinsic_rgb_params[i] ext_params = zense.getExtrinsicParameter() _rotation_ext = ext_params[0] _translation_ext = ext_params[1] rotation_extrinsic_elems = [ "r11", "r12", "r13", "r21", "r22", "r23", "r31", "r32", "r33" ] translation_extrinsic_elems = ["tx", "ty", "tz"] for i, _elem in enumerate(rotation_extrinsic_elems): dict_toml["Camera0_Extrinsic_Factory"][_elem] = _rotation_ext[i] for i, _elem in enumerate(translation_extrinsic_elems): dict_toml["Camera0_Extrinsic_Factory"][_elem] = _translation_ext[i] with open(out, "w") as f: toml.encoder.dump(dict_toml, f) print("generated") del zense
def cyclobutadiene(ctx): for label in ['ground', 'transition']: path = ctx.obj['basedir'] / label print(path) param = NestedDict() param['system'] = f'dlqmc.systems:cyclobutadiene_{label}' param['model_kwargs.cas'] = [8, 4] param['model_kwargs.pauli_kwargs.conf_cutoff'] = 1e-8 param['model_kwargs.pauli_kwargs.conf_limit'] = 10 param['model_kwargs.pauli_kwargs.rc_scaling'] = 3.0 param['model_kwargs.pauli_kwargs.cusp_alpha'] = 3.0 param['model_kwargs.pauli_kwargs.use_sloglindet'] = 'training' param['model_kwargs.omni_kwargs.subnet_kwargs.n_layers_h'] = 2 param['train_kwargs.n_steps'] = 5_000 param['train_kwargs.batch_size'] = 1_000 param['train_kwargs.epoch_size'] = 5 param['train_kwargs.fit_kwargs.subbatch_size'] = 500 param['train_kwargs.sampler_kwargs.n_decorrelate'] = 20 param['train_kwargs.lr_scheduler_kwargs.CyclicLR.step_size_up'] = 375 path.mkdir(parents=True) (path / 'param.toml').write_text(toml.dumps(param, encoder=toml.TomlEncoder()))
def hyperparam_scan_co2(ctx): learning_rates = [0.3e-3, 1e-3, 3e-3] batch_sizes = [1000, 2000, 4000] epoch_sizes = [3, 5, 8] ns_decorrelate = [5, 10, 20] payload = product(learning_rates, batch_sizes, epoch_sizes, ns_decorrelate) for lr, bs, es, n_decorr in payload: label = f'lr-{lr}_bs-{bs}_es-{es}_decorr-{n_decorr}' path = ctx.obj['basedir'] / label if path.exists(): continue print(path) params = NestedDict() params['system'] = 'CO2' params['train_kwargs.n_steps'] = 2000 params['train_kwargs.learning_rate'] = lr params['train_kwargs.batch_size'] = bs params['train_kwargs.epoch_size'] = es params['train_kwargs.sampler_kwargs.n_decorrelate'] = n_decorr path.mkdir(parents=True) (path / 'param.toml').write_text(toml.dumps(params, encoder=toml.TomlEncoder()))
def main(): cfg_template_path = str(Path(CFG_DIR_PATH, "realsense_toml_template.toml")) cfg_output_path = str(Path(CFG_DIR_PATH, "realsense.toml")) decoder = toml.TomlDecoder(_dict=OrderedDict) encoder = toml.TomlEncoder(_dict=OrderedDict) toml.TomlEncoder = encoder dict_toml = toml.load(open(cfg_template_path), _dict=OrderedDict, decoder=decoder) rs_mng = RealSenseManager() intrinsic_depth = rs_mng.intrinsic_depth intrinsic_color = rs_mng.intrinsic_color intrinsic_ir_left = rs_mng.intrinsic_ir_left intrinsic_ir_right = rs_mng.intrinsic_ir_right translation_l2r = rs_mng.translation_ir_left2right translation_l2c = rs_mng.translation_ir_left2color rotation_dcm_l2r = rs_mng.rotation_dcm_ir_left2right rotation_dcm_l2c = rs_mng.rotation_dcm_ir_left2color set_intrinsics(dict_toml, "RGB_Intrinsics", intrinsic_color) set_intrinsics(dict_toml, "Depth_Intrinsics", intrinsic_depth) set_intrinsics(dict_toml, "IR_Left_Intrinsics", intrinsic_ir_left) set_intrinsics(dict_toml, "IR_Right_Intrinsics", intrinsic_ir_right) set_translation(dict_toml, "IR_L2R_Translation", translation_l2r) set_translation(dict_toml, "IR_L2C_Translation", translation_l2c) set_rotation(dict_toml, "IR_L2R_Rotation", rotation_dcm_l2r) set_rotation(dict_toml, "IR_L2C_Rotation", rotation_dcm_l2c) with open(cfg_output_path, "w") as f: toml.encoder.dump(dict_toml, f) print("generated") del rs_mng
def main(out): zense = PyPicoZenseManager(0) decoder = toml.TomlDecoder(_dict=OrderedDict) encoder = toml.TomlEncoder(_dict=OrderedDict) toml.TomlEncoder = encoder dict_toml = toml.load(open('template.toml'), _dict=OrderedDict, decoder=decoder) dict_toml["Camera0"]["serial_no"] = zense.getSerialNumber() params = zense.getCameraParameter() intrinsic_elems = ["fx", "fy", "cx", "cy", \ "p1", "p2", "k1", "k2", \ "k3", "k4", "k5", "k6"] for i, _elem in enumerate(intrinsic_elems): dict_toml["Camera0_Factory"][_elem] = params[i] with open(out, "w") as f: toml.encoder.dump(dict_toml, f) print("generated") del zense
def encode_toml_value(s): return toml.TomlEncoder().dump_value(s)
def dumps(o): # strip the result, because `toml.dumps` adds a lot of newlines return toml.dumps(o, encoder=toml.TomlEncoder(dict)).strip()
# encoding: utf-8 from ..engine.osc import osc_method import glob import traceback from threading import Thread import toml tomlencoder = toml.TomlEncoder() def dump_float(f): s = "%g" % f if '.' not in s: s += '.0' return s tomlencoder.dump_funcs[float] = dump_float import logging LOGGER = logging.getLogger(__name__) class Scenes(object): def __init__(self, *args, **kwargs): super(Scenes, self).__init__(*args, **kwargs) self.scenes = {}
def __init__(self, *, commented=False): self._stream = None self._commented = commented self._encoder = toml.TomlEncoder() self._at_start = True