def __init__(self, _dict=dict, preserve=False):
"""Create the encoder but override some default functions."""
super(RuleTomlEncoder, self).__init__(_dict, preserve)
self._old_dump_str = toml.TomlEncoder().dump_funcs[str]
self._old_dump_list = toml.TomlEncoder().dump_funcs[list]
self.dump_funcs[str] = self.dump_str
self.dump_funcs[type(u"")] = self.dump_str
self.dump_funcs[list] = self.dump_list
self.dump_funcs[NonformattedField] = self.dump_str
def main(toml_path, toml_template_path, yaml_path, fill_see3cam_id):
decoder = toml.TomlDecoder(_dict=OrderedDict)
encoder = toml.TomlEncoder(_dict=OrderedDict)
f_yaml = open(yaml_path, "r")
data = yaml.load(f_yaml)
toml.TomlEncoder = encoder
dict_toml = toml.load(open(toml_template_path),
_dict=OrderedDict,
decoder=decoder)
if fill_see3cam_id:
dict_toml["Rgb"]["device_id"] = get_see3cam_device_id()
dict_toml["Rgb"]["width"] = data["cam0"]["resolution"][0]
dict_toml["Rgb"]["height"] = data["cam0"]["resolution"][1]
dict_toml["Rgb"]["fx"] = data["cam0"]["intrinsics"][0]
dict_toml["Rgb"]["fy"] = data["cam0"]["intrinsics"][1]
dict_toml["Rgb"]["cx"] = data["cam0"]["intrinsics"][2]
dict_toml["Rgb"]["cy"] = data["cam0"]["intrinsics"][3]
dict_toml["Rgb"]["k1"] = data["cam0"]["distortion_coeffs"][0]
dict_toml["Rgb"]["k2"] = data["cam0"]["distortion_coeffs"][1]
dict_toml["Rgb"]["k3"] = data["cam0"]["distortion_coeffs"][2]
dict_toml["Rgb"]["k4"] = data["cam0"]["distortion_coeffs"][3]
dict_toml["Rgb"]["k5"] = 0
dict_toml["Rgb"]["k6"] = 0
with open(toml_path, "w") as f:
toml.encoder.dump(dict_toml, f)
print("generated")
def dumps(
data: Mapping[str, Any],
encoder: Union[Type[toml.TomlEncoder],
toml.TomlEncoder] = toml.TomlEncoder,
) -> str:
r"""
Convert ``data`` to a TOML string.
:param data:
:param encoder: The :class:`toml.TomlEncoder` to use for constructing the output string.
:returns: A string containing the ``TOML`` corresponding to ``data``.
.. versionchanged:: 0.5.0
The default value for ``encoder`` changed from :py:obj:`None` to :class:`toml.TomlEncoder`
Explicitly passing ``encoder=None`` is deprecated and support will be removed in 1.0.0
"""
if isinstance(encoder, type):
encoder = encoder()
elif encoder is None:
warnings.warn(
"Passing encoder=None to 'dom_toml.dumps' is deprecated since 0.5.0 and support will be removed in 1.0.0",
DeprecationWarning,
)
encoder = toml.TomlEncoder()
return toml.dumps(data, encoder=encoder)
def all_systems(ctx):
systems = [
'H2',
'B',
'Be',
'LiH',
*({'name': 'Hn', 'n': 10, 'dist': d} for d in [1.2, 1.8, 3.6]),
]
cass = {'H2': [2, 2], 'B': [4, 3], 'LiH': [4, 2], 'Hn': [6, 4], 'Be': [4, 2]}
param_sets = ['SD-SJ', 'SD-SJBF', 'MD-SJ', 'MD-SJBF']
for system, param_set in product(systems, param_sets):
sys_name = system if isinstance(system, str) else system['name']
sys_label = sys_name
if sys_name == 'Hn':
sys_label += f'-{system["dist"]}'
path = ctx.obj['basedir'] / sys_label / param_set
if path.exists():
continue
print(path)
params = NestedDict()
params['system'] = system
if 'MD' in param_set:
params['model_kwargs.cas'] = cass[sys_name]
if 'BF' not in param_set:
params['model_kwargs.omni_kwargs.with_backflow'] = False
path.mkdir(parents=True)
(path / 'param.toml').write_text(toml.dumps(params, encoder=toml.TomlEncoder()))
def main(out, template):
zense = PyPicoZenseManager(0)
decoder = toml.TomlDecoder(_dict=OrderedDict)
encoder = toml.TomlEncoder(_dict=OrderedDict)
toml.TomlEncoder = encoder
dict_toml = toml.load(open(template), _dict=OrderedDict, decoder=decoder)
dict_toml["Camera0"]["serial_no"] = zense.getSerialNumber().decode()
params = zense.getCameraParameter()
dict_toml["Camera0"]["fx"] = params[0]
dict_toml["Camera0"]["fy"] = params[1]
dict_toml["Camera0"]["cx"] = params[2]
dict_toml["Camera0"]["cy"] = params[3]
dict_toml["Camera0_Factory"]["fx"] = params[0]
dict_toml["Camera0_Factory"]["fy"] = params[1]
dict_toml["Camera0_Factory"]["cx"] = params[2]
dict_toml["Camera0_Factory"]["cy"] = params[3]
dict_toml["Camera0_Factory"]["p1"] = params[4]
dict_toml["Camera0_Factory"]["p2"] = params[5]
dict_toml["Camera0_Factory"]["k1"] = params[6]
dict_toml["Camera0_Factory"]["k2"] = params[7]
dict_toml["Camera0_Factory"]["k3"] = params[8]
dict_toml["Camera0_Factory"]["k4"] = params[9]
dict_toml["Camera0_Factory"]["k5"] = params[10]
dict_toml["Camera0_Factory"]["k6"] = params[11]
with open(out, "w") as f:
toml.encoder.dump(dict_toml, f)
print("generated")
del zense
def _parse_pipfile(self, contents):
# If any outline tables are present...
if ("[packages." in contents) or ("[dev-packages." in contents):
data = toml.loads(contents)
# Convert all outline tables to inline tables.
for section in ("packages", "dev-packages"):
for package in data.get(section, {}):
# Convert things to inline tables — fancy :)
if hasattr(data[section][package], "keys"):
_data = data[section][package]
data[section][package] = toml.TomlDecoder(
).get_empty_inline_table()
data[section][package].update(_data)
toml_encoder = toml.TomlEncoder(preserve=True)
# We lose comments here, but it's for the best.)
try:
return contoml.loads(toml.dumps(data, encoder=toml_encoder))
except RuntimeError:
return toml.loads(toml.dumps(data, encoder=toml_encoder))
else:
# Fallback to toml parser, for large files.
try:
return contoml.loads(contents)
except Exception:
return toml.loads(contents)
def leanpkg_upgrade_proc():
with open('leanpkg.toml', 'r') as lean_toml:
local_toml = toml.loads(lean_toml.read())
local_lean_version = local_toml['package']['lean_version']
urllib.request.urlretrieve(
'https://raw.githubusercontent.com/leanprover-community/mathlib/master/leanpkg.toml',
'mathlib_leanpkg.toml')
with open('mathlib_leanpkg.toml', 'r') as lean_toml:
mathlib_toml = toml.loads(lean_toml.read())
mathlib_lean_version = mathlib_toml['package']['lean_version']
lean_version_prefix = 'leanprover-community/lean:'
if local_lean_version.startswith(
lean_version_prefix) and mathlib_lean_version.startswith(
lean_version_prefix):
local_lean_version_int = [
int(i)
for i in local_lean_version[len(lean_version_prefix):].split('.')
]
mathlib_lean_version_int = [
int(i)
for i in mathlib_lean_version[len(lean_version_prefix):].split('.')
]
print(mathlib_lean_version_int, local_lean_version_int)
if mathlib_lean_version_int > local_lean_version_int:
local_toml['package']['lean_version'] = mathlib_lean_version
with open('leanpkg.toml', 'w') as lean_toml:
# `preserve = True` seems to be an undocumented feature of `toml`.
# without it, when the project has exactly one dependency,
# the resulting `leanpkg.toml` is malformed: it compresses the `dependencies` section
# into `[dependencies.repo_name]`.
toml.dump(local_toml,
lean_toml,
encoder=toml.TomlEncoder(preserve=True))
return subprocess.Popen(['leanpkg', 'upgrade'])
def boron(ctx):
payload = chain(
product(
['never', 'training', 'always'],
[(None, 1), ([4, 3], 3), ([8, 3], 16), ([8, 3], 100)],
[100, 10],
[(None, {})],
),
product(
['training'],
[(None, 1), ([8, 3], 16)],
[100, 10],
[('lrx2', {'train_kwargs.lr_scheduler_kwargs.CyclicLR.max_lr': 0.02})],
),
product(
['training'],
[(None, 1), ([8, 3], 16)],
[100, 10],
[('lrx3', {'train_kwargs.lr_scheduler_kwargs.CyclicLR.max_lr': 0.03})],
),
product(
['training'],
[(None, 1), ([8, 3], 16)],
[100, 10],
[('qx2', {'train_kwargs.fit_kwargs.q': 10})],
),
)
for use_slgld, (cas, conf_lim), epoch_size, (extra_lbl, extra) in payload:
label = f'{use_slgld}_ndet-{conf_lim}_epoch-{epoch_size}'
if extra:
label += f'_{extra_lbl}'
path = ctx.obj['basedir'] / label
if path.exists():
continue
print(path)
param = NestedDict()
param['system'] = 'B'
param['train_kwargs.n_steps'] = 10_000
param['train_kwargs.batch_size'] = 10_000
param['train_kwargs.fit_kwargs.subbatch_size'] = 5_000
param['train_kwargs.sampler_kwargs.n_discard'] = 10
param['model_kwargs.pauli_kwargs.conf_cutoff'] = 1e-8
param['model_kwargs.pauli_kwargs.rc_scaling'] = 3.0
param['model_kwargs.pauli_kwargs.cusp_alpha'] = 3.0
param['model_kwargs.omni_kwargs.subnet_kwargs.n_layers_h'] = 2
param['model_kwargs.pauli_kwargs.conf_limit'] = conf_lim
param['model_kwargs.pauli_kwargs.use_sloglindet'] = use_slgld
param['model_kwargs.cas'] = cas
param['train_kwargs.epoch_size'] = epoch_size
for k, v in extra.items():
param[k] = v
path.mkdir(parents=True)
(path / 'param.toml').write_text(toml.dumps(param, encoder=toml.TomlEncoder()))
def sampling(ctx, training):
for param_path in Path(training).glob('**/param.toml'):
train_path = param_path.parent
label = train_path.relative_to(training)
chkpts = [
p.stem.split('-')[1] for p in (train_path / 'chkpts').glob('state-*.pt')
]
if not chkpts:
continue
step = max(chkpts)
path = ctx.obj['basedir'] / label
print(path)
params = toml.loads((train_path / 'param.toml').read_text())
train_path = Path(os.path.relpath(train_path.resolve(), path.resolve()))
path.mkdir(parents=True)
(path / 'param.toml').write_text(toml.dumps(params, encoder=toml.TomlEncoder()))
(path / 'state.pt').symlink_to(train_path / f'chkpts/state-{step}.pt')
def sampling_states(ctx, state, param):
base = os.path.commonpath(state)
if param:
param = toml.loads(Path(param).read_text())
for state_path in sorted(state):
label = os.path.splitext(os.path.relpath(state_path, base))[0]
train_path = Path(state_path).parents[1]
params = NestedDict()
path = ctx.obj['basedir'] / label
print(path)
params_train = toml.loads((train_path / 'param.toml').read_text())
params.update(params_train)
if param:
params.update(param)
path.mkdir(parents=True)
(path / 'param.toml').write_text(toml.dumps(params, encoder=toml.TomlEncoder()))
state_path = Path(os.path.relpath(Path(state_path).resolve(), path.resolve()))
(path / 'state.pt').symlink_to(state_path)
def main(out):
zense = PyPicoZenseManager(0)
decoder = toml.TomlDecoder(_dict=OrderedDict)
encoder = toml.TomlEncoder(_dict=OrderedDict)
toml.TomlEncoder = encoder
dict_toml = toml.load(open('template.toml'),
_dict=OrderedDict,
decoder=decoder)
dict_toml["Camera0"]["serial_no"] = zense.getSerialNumber()
intrinsic_depth_params = zense.getCameraParameter()
intrinsic_rgb_params = zense.getRGBCameraParameter()
intrinsic_elems = [
"fx", "fy", "cx", "cy", "p1", "p2", "k1", "k2", "k3", "k4", "k5", "k6"
]
for i, _elem in enumerate(intrinsic_elems):
dict_toml["Camera0_Factory"][_elem] = intrinsic_depth_params[i]
for i, _elem in enumerate(intrinsic_elems):
dict_toml["Camera0_RGB_Factory"][_elem] = intrinsic_rgb_params[i]
ext_params = zense.getExtrinsicParameter()
_rotation_ext = ext_params[0]
_translation_ext = ext_params[1]
rotation_extrinsic_elems = [
"r11", "r12", "r13", "r21", "r22", "r23", "r31", "r32", "r33"
]
translation_extrinsic_elems = ["tx", "ty", "tz"]
for i, _elem in enumerate(rotation_extrinsic_elems):
dict_toml["Camera0_Extrinsic_Factory"][_elem] = _rotation_ext[i]
for i, _elem in enumerate(translation_extrinsic_elems):
dict_toml["Camera0_Extrinsic_Factory"][_elem] = _translation_ext[i]
with open(out, "w") as f:
toml.encoder.dump(dict_toml, f)
print("generated")
del zense
def cyclobutadiene(ctx):
for label in ['ground', 'transition']:
path = ctx.obj['basedir'] / label
print(path)
param = NestedDict()
param['system'] = f'dlqmc.systems:cyclobutadiene_{label}'
param['model_kwargs.cas'] = [8, 4]
param['model_kwargs.pauli_kwargs.conf_cutoff'] = 1e-8
param['model_kwargs.pauli_kwargs.conf_limit'] = 10
param['model_kwargs.pauli_kwargs.rc_scaling'] = 3.0
param['model_kwargs.pauli_kwargs.cusp_alpha'] = 3.0
param['model_kwargs.pauli_kwargs.use_sloglindet'] = 'training'
param['model_kwargs.omni_kwargs.subnet_kwargs.n_layers_h'] = 2
param['train_kwargs.n_steps'] = 5_000
param['train_kwargs.batch_size'] = 1_000
param['train_kwargs.epoch_size'] = 5
param['train_kwargs.fit_kwargs.subbatch_size'] = 500
param['train_kwargs.sampler_kwargs.n_decorrelate'] = 20
param['train_kwargs.lr_scheduler_kwargs.CyclicLR.step_size_up'] = 375
path.mkdir(parents=True)
(path / 'param.toml').write_text(toml.dumps(param, encoder=toml.TomlEncoder()))
def hyperparam_scan_co2(ctx):
learning_rates = [0.3e-3, 1e-3, 3e-3]
batch_sizes = [1000, 2000, 4000]
epoch_sizes = [3, 5, 8]
ns_decorrelate = [5, 10, 20]
payload = product(learning_rates, batch_sizes, epoch_sizes, ns_decorrelate)
for lr, bs, es, n_decorr in payload:
label = f'lr-{lr}_bs-{bs}_es-{es}_decorr-{n_decorr}'
path = ctx.obj['basedir'] / label
if path.exists():
continue
print(path)
params = NestedDict()
params['system'] = 'CO2'
params['train_kwargs.n_steps'] = 2000
params['train_kwargs.learning_rate'] = lr
params['train_kwargs.batch_size'] = bs
params['train_kwargs.epoch_size'] = es
params['train_kwargs.sampler_kwargs.n_decorrelate'] = n_decorr
path.mkdir(parents=True)
(path / 'param.toml').write_text(toml.dumps(params, encoder=toml.TomlEncoder()))
def main():
cfg_template_path = str(Path(CFG_DIR_PATH, "realsense_toml_template.toml"))
cfg_output_path = str(Path(CFG_DIR_PATH, "realsense.toml"))
decoder = toml.TomlDecoder(_dict=OrderedDict)
encoder = toml.TomlEncoder(_dict=OrderedDict)
toml.TomlEncoder = encoder
dict_toml = toml.load(open(cfg_template_path),
_dict=OrderedDict,
decoder=decoder)
rs_mng = RealSenseManager()
intrinsic_depth = rs_mng.intrinsic_depth
intrinsic_color = rs_mng.intrinsic_color
intrinsic_ir_left = rs_mng.intrinsic_ir_left
intrinsic_ir_right = rs_mng.intrinsic_ir_right
translation_l2r = rs_mng.translation_ir_left2right
translation_l2c = rs_mng.translation_ir_left2color
rotation_dcm_l2r = rs_mng.rotation_dcm_ir_left2right
rotation_dcm_l2c = rs_mng.rotation_dcm_ir_left2color
set_intrinsics(dict_toml, "RGB_Intrinsics", intrinsic_color)
set_intrinsics(dict_toml, "Depth_Intrinsics", intrinsic_depth)
set_intrinsics(dict_toml, "IR_Left_Intrinsics", intrinsic_ir_left)
set_intrinsics(dict_toml, "IR_Right_Intrinsics", intrinsic_ir_right)
set_translation(dict_toml, "IR_L2R_Translation", translation_l2r)
set_translation(dict_toml, "IR_L2C_Translation", translation_l2c)
set_rotation(dict_toml, "IR_L2R_Rotation", rotation_dcm_l2r)
set_rotation(dict_toml, "IR_L2C_Rotation", rotation_dcm_l2c)
with open(cfg_output_path, "w") as f:
toml.encoder.dump(dict_toml, f)
print("generated")
del rs_mng
def main(out):
zense = PyPicoZenseManager(0)
decoder = toml.TomlDecoder(_dict=OrderedDict)
encoder = toml.TomlEncoder(_dict=OrderedDict)
toml.TomlEncoder = encoder
dict_toml = toml.load(open('template.toml'),
_dict=OrderedDict,
decoder=decoder)
dict_toml["Camera0"]["serial_no"] = zense.getSerialNumber()
params = zense.getCameraParameter()
intrinsic_elems = ["fx", "fy", "cx", "cy", \
"p1", "p2", "k1", "k2", \
"k3", "k4", "k5", "k6"]
for i, _elem in enumerate(intrinsic_elems):
dict_toml["Camera0_Factory"][_elem] = params[i]
with open(out, "w") as f:
toml.encoder.dump(dict_toml, f)
print("generated")
del zense
def encode_toml_value(s):
return toml.TomlEncoder().dump_value(s)
def dumps(o):
# strip the result, because `toml.dumps` adds a lot of newlines
return toml.dumps(o, encoder=toml.TomlEncoder(dict)).strip()
# encoding: utf-8
from ..engine.osc import osc_method
import glob
import traceback
from threading import Thread
import toml
tomlencoder = toml.TomlEncoder()
def dump_float(f):
s = "%g" % f
if '.' not in s:
s += '.0'
return s
tomlencoder.dump_funcs[float] = dump_float
import logging
LOGGER = logging.getLogger(__name__)
class Scenes(object):
def __init__(self, *args, **kwargs):
super(Scenes, self).__init__(*args, **kwargs)
self.scenes = {}
def __init__(self, *, commented=False):
self._stream = None
self._commented = commented
self._encoder = toml.TomlEncoder()
self._at_start = True
