def test_setattr_default(): b = DefaultMunch(foo='bar', this_is='useful when subclassing') assert hasattr(b.values, '__call__') b.values = 'uh oh' assert b.values == 'uh oh' assert b['values'] is None assert b.__default__ is None assert '__default__' not in b
def test_copy_default(): undefined = object() m = DefaultMunch.fromDict({'urmom': {'sez': {'what': 'what'}}}, undefined) c = m.copy() assert c is not m assert c.urmom is not m.urmom assert c.urmom.sez is not m.urmom.sez assert c.urmom.sez.what == 'what' assert c == m assert c.urmom.sez.foo is undefined assert c.urmom.sez.__undefined__ is undefined
seed=seed, no_graphics=True) brain = env.brains[env.brain_names[0]] env_info = env.reset(train_mode=True)[env.brain_names[0]] n_agents = len(env_info.agents) print('Number of agents:', n_agents) action_size = brain.vector_action_space_size state_size = brain.vector_observation_space_size state_multiplier = brain.num_stacked_vector_observations action_type = brain.vector_action_space_type comment = f"MADDPG Unity Tennis" log_dir = os.path.join('./runs', current_time + '_' + comment) os.mkdir(log_dir) print(f"logging to {log_dir}") writer = SummaryWriter(log_dir=log_dir) config = DefaultMunch() config.seed = seed config.n_episodes = 40000 config.max_t = 1000 config.buffer_size = 100000 config.batch_size = 200 config.gamma = 0.99 config.tau = 0.001 config.lr_actor = 0.0001 config.lr_critic = 0.0001 config.n_agents = n_agents config.state_size = state_size * state_multiplier config.action_size = action_size config.learn_start = 10000 config.max_action = 1 # maximum value allowed for each action config.memory = ExperienceReplayMemory(config.buffer_size, seed)
def __init__(self, values): self.values = DefaultMunch.fromDict(values)
class Experiment(object): def __init__(self, conf_path=None, experiments_dir=None, hp=None): self.conf_path = conf_path self.conf = None self.log = None self.trainer = None self.tee = None self.current_run = 0 self.fast_text_model = None if self.conf_path: self.set_conf(self.conf_path) else: self.conf = DefaultMunch(None).fromDict(default_conf) if experiments_dir: self.conf.experiments_dir = experiments_dir assert self.conf.experiments_dir is not None self.conf.experiments_dir = Path(self.conf.experiments_dir).resolve() if not self.conf.experiments_dir.exists(): print("Creating %s" % str(self.conf.experiments_dir)) self.conf.experiments_dir.mkdir(parents=True) if not self.conf.exp_id: self.conf.exp_id = str(datetime.datetime.now())[:10] self.dir = get_new_dir(self.conf.experiments_dir, self.conf.exp_id) self.summary = { "params": {p: [] for p in self.conf.randomizable_params}, "other": {}, "metrics": {m: [] for m in metrics}, } def set_conf(self, path): with open(path, "r") as f: conf = yaml.safe_load(f) self.conf = DefaultMunch(None).fromDict(conf) def randomize(self, conf=None, verbose=0): conf = conf or self.conf params = conf.randomizable_params for p_name, p in params.items(): if self.conf.trainer_type == "FT_DST" and p_name == "embedding_dim": self.summary["params"][p_name].append(300) continue if p.type == "range": values = np.arange(p.min, p.max, p.step) elif p.type == "list": values = np.array(p.vals) elif p.type == "fixed": value = np.array(p.value) else: raise ValueError("Unkonw type {} for {}".format( p.type, p_name)) if p.type != "fixed": if p.distribution == "normal": value = normal_choice(values) elif p.distribution == "uniform": value = uniform_choice(values) else: # p.distribution == "deterministic" value = values[self.current_run % len(values)] setattr(self.trainer.hp, p_name, value.tolist()) self.summary["params"][p_name].append(value) if verbose > 0: print("{:20}: {:10}".format(p_name, value)) if verbose > 0: print("{:20}: {:10}".format("trainer.hp.dir", self.trainer.hp.dir)) def dump_conf(self, path): stringified = [] for attr, val in self.conf.items(): if isinstance(val, Path): self.conf[attr] = str(val) stringified.append(attr) with open(path, "w") as f: yaml.safe_dump(self.conf, f, default_flow_style=False) for attr in stringified: self.conf[attr] = Path(self.conf[attr]) def summarize(self): metrics = pd.DataFrame(self.summary["metrics"]) params = pd.DataFrame(self.summary["params"]) other = "\n".join("{:20}: {}".format(k, v) for k, v in self.summary["other"].items()) summary = "{}\n\n{}".format( other, pd.concat([metrics, params], axis=1).to_string()) with open(self.dir / "summary.txt", "a") as f: f.write(summary) metrics.to_csv(self.dir / "metrics.csv") params.to_csv(self.dir / "params.csv") self.dump_conf(self.dir / "conf.yaml") def setup(self, log=True): hp = HP(base_dir=self.dir) if self.conf.trainer_type == "DST": for attr, val in self.conf.hyperparameter.items(): if val is not None: setattr(hp, attr, val) self.trainer = DST(hp=hp) elif self.conf.trainer_type == "CDST": for attr, val in self.conf.hyperparameter.items(): if val is not None: setattr(hp, attr, val) self.trainer = CDST(hp=hp) elif self.conf.trainer_type == "FT_DST": for attr, val in self.conf.hyperparameter.items(): if attr != "embedding_dim": if val is not None: setattr(hp, attr, val) if not self.fast_text_model: print("Setting fast_text_model...", end="") self.fast_text_model = FastText.load_fasttext_format( hp.fast_text_model_file) print("Ok.") self.trainer = FT_DST(fast_text_model=self.fast_text_model, hp=hp) else: raise ValueError("Unknown Trainer") self.log = log if log: self.tee = Tee(str(self.trainer.hp.dir / "log.txt")) sys.stdout = self.tee def reset(self, verbose=0): self.setup() self.randomize(verbose) def delete(self): shutil.rmtree(self.dir, ignore_errors=True) def update_metrics(self, metrics): if metrics is None: metrics = None, None, None, None acc, mic, mac, wei = metrics self.summary["metrics"]["micro_f1"].append(mic) self.summary["metrics"]["macro_f1"].append(mac) self.summary["metrics"]["weighted_f1"].append(wei) self.summary["metrics"]["accuracy"].append(acc) def get_samples(self, samples, sample_size, is_val=False): preds, ys = None, None for _ in range(samples): x, y = self.trainer.get_input_pair(is_val, sample_size) pred = self.trainer.infer(x) if preds is None: preds, ys = pred, y else: preds = np.concatenate((preds, pred), axis=0) ys = np.concatenate((ys, y), axis=0) return expit(preds), ys def eval(self, thresholds, samples, sample_size, is_val=False): preds, ys = self.get_samples(samples, sample_size, is_val) averages = [None, "micro", "macro", "weighted"] metrics = {str(av): [] for av in averages} for av in averages: for threshold in thresholds: metrics[str(av)].append( f1_score(ys, preds > threshold, average=av)) return metrics def run(self, n_runs=None, randomize=True, log=True, verbose=0): n_runs = n_runs or self.conf.n_runs if n_runs is None: n_runs = np.iinfo(int).max print("\n= = > Run", self.current_run) self.setup(log) if randomize: self.randomize(verbose=verbose) while self.current_run < n_runs: if self.current_run > 0: print("\n= = > Run", self.current_run) try: metrics = self.trainer.train() self.trainer.dump_logits() self.update_metrics(metrics) self.summarize() except EndOfExperiment: print("\nStopping experiment. Delete?") answer = input("y/n") if "y" in answer: self.delete() return else: self.summary["other"][ "interrupting"] = "Keyboard interrupted" self.summarize() break self.current_run += 1 if self.current_run < n_runs: self.reset(verbose) # End of run # End of all runs self.summary["other"]["interrupting"] = "Done: all runs performed." self.summarize()
def test_delattr_default(): b = DefaultMunch(lol=42) del b.lol assert b.lol is None assert b['lol'] is None
def test_fromDict_default(): undefined = object() b = DefaultMunch.fromDict({'urmom': {'sez': {'what': 'what'}}}, undefined) assert b.urmom.sez.what == 'what' assert b.urmom.sez.foo is undefined
def __call__(self, value, path=""): log.debug('Visiting %s', path) if value in self.visited: raise UtilException("Serialisation failed: cycle detected: %s" % path) self.visited.append(value) log.debug('Type is %s', value.__class__) if isinstance(value, model.Base): names = dir(value) son = DefaultMunch(undefined) for name in names: if not self.can_emit(name, path): continue v = getattr(value, name) if self.omit and v is None: continue if hasattr(v, '__call__'): continue son[to_camel_case(name)] = self(v, "%s/%s" % (path, name)) elif isinstance(value, str): son = value elif isinstance(value, datetime.datetime): son = value.timestamp() elif isinstance(value, datetime.date): son = datetime.datetime.combine( value, datetime.datetime.min.time()).timestamp() elif isinstance(value, uuid.UUID): son = str(value) elif (hasattr(value, '__getitem__') and hasattr(value, 'keys') and hasattr(value, 'values') and not isinstance(value, RowProxy)): # Dictionaries son = DefaultMunch(undefined) for name in value.keys(): if not self.can_emit(name, path): continue v = value[name] if self.omit and v is None: continue son[to_camel_case(name)] = self(v, "%s/%s" % (path, name)) elif hasattr(value, '__iter__'): # Lists son = [] for i, v in enumerate(value): if not self.can_emit(i, path): continue if self.omit and v is None: continue son.append(self(v, "%s/%d" % (path, i))) else: son = value if (isinstance(son, str) and any(s.search(path) for s in self._sanitise)): son = bleach.clean(son, strip=True) self.visited.pop() return son
def open(self): with open(self.__path, "r") as ymlfile: self.__config = safe_load(ymlfile) self.__config = DefaultMunch.fromDict(self.__config)
print(f'Worker_id={worker_id}') env = UnityEnvironment("./environment/Tennis_Linux/Tennis.x86_64", worker_id=worker_id, seed=seed, no_graphics=False) brain = env.brains[env.brain_names[0]] env_info = env.reset(train_mode=False)[env.brain_names[0]] n_agents = len(env_info.agents) print('Number of agents:', n_agents) action_size = brain.vector_action_space_size state_size = brain.vector_observation_space_size state_multiplier = brain.num_stacked_vector_observations action_type = brain.vector_action_space_type comment = f"MADDPG Unity Tennis" rand_seed = 0 config = DefaultMunch() config.seed = seed config.n_episodes = 10 config.max_t = 1000 config.buffer_size = 100000 config.batch_size = 200 config.gamma = 0.99 config.tau = 0.001 config.lr_actor = 0.0001 config.lr_critic = 0.001 config.n_agents = n_agents config.state_size = state_size * state_multiplier config.action_size = action_size config.learn_start = 3000 config.max_action = 1 config.memory = ExperienceReplayMemory(config.buffer_size, rand_seed)
# missing variable mapping schema.csl.fields.text['volume-title'] = [ 'volumeTitle' ] # status is publication status, not legal status schema.csl.fields.text.status = [ 'status '] with open(os.path.join(root, 'setup/csl-vars.json')) as f: csl_vars = set(json.load(f).keys()) fix_csl_vars(schema.csl.fields.text, 'jurism', csl_vars) fix_csl_vars(schema.csl.fields.date, 'jurism', csl_vars) fix_csl_vars(schema.csl.names, 'jurism', csl_vars) return Munch.toDict(schema) data = DefaultMunch.fromDict({ 'zotero': fix_zotero_schema(load('https://api.zotero.org/schema', 'zotero.json')), 'jurism': fix_jurism_schema(load('https://raw.githubusercontent.com/Juris-M/zotero-schema/master/schema-jurism.json', 'juris-m.json')), }, None) class ExtraFields: @staticmethod def to_json(obj): if isinstance(obj, Munch): return { k: v for k, v in Munch.toDict(obj).items() if not v is None and not (type(v) == list and len(v) == 0) } else: return obj def __init__(self):
def _main(cli_args, chosen_deployment_name): """Deployment entry point. :param cli_args: The command-line arguments :type cli_args: ``list`` :param chosen_deployment_name: The deployment file :type chosen_deployment_name: ``str`` :returns: True on success :rtype: ``bool`` """ config_file = os.path.join( OKD_DEPLOYMENTS_DIRECTORY, chosen_deployment_name, io.get_deployment_config_filename(chosen_deployment_name)) if not os.path.isfile(config_file): print('Config file does not exist ({})'.format(chosen_deployment_name)) return False with codecs.open(config_file, 'r', 'utf8') as stream: deployment = DefaultMunch.fromDict(yaml.load(stream)) # First check: # is the version present # and do we support it? if 'version' not in deployment: print('The deployment configuration has no version.') return False if deployment.version not in SUPPORTED_DEPLOYMENT_VERSIONS: supported_versions = str(SUPPORTED_DEPLOYMENT_VERSIONS[0]) for version in SUPPORTED_DEPLOYMENT_VERSIONS[1:]: supported_versions += ', {}'.format(version) print('The deployment configuration file version ({})' ' is not supported.'.format(deployment.version)) print('Supported versions are: {}'.format(supported_versions)) return False # There must be an okd/inventories directory inventory_dir = deployment.okd.inventory_dir if not os.path.isdir('okd/inventories/{}'.format(inventory_dir)): print('Missing "okd/inventories" directory') print('Expected to find the inventory directory "{}"' ' but it was not there.'.format(inventory_dir)) print('Every deployment must have an "inventories" directory') return False # If the cluster SSH user is not defined, # insert it. if 'ssh_user' not in deployment.cluster: print('Setting default SSH user "{}"'.format( OKD_DEFAULT_CLUSTER_SSH_USER)) deployment.cluster.ssh_user = OKD_DEFAULT_CLUSTER_SSH_USER # ----- # Hello # ----- io.banner(deployment.name, full_heading=True, quiet=False) if not cli_args.auto_acknowledge and not cli_args.just_plan: # Display the orchestration description # (f there is one) if deployment.description: io.description(deployment.description) confirmation_word = io.get_confirmation_word() target = 'CREATE the Cluster' \ if cli_args.cluster else 'INSTALL OpenShift/OKD' confirmation = input('Enter "{}" to {}: '.format( confirmation_word, target)) if confirmation != confirmation_word: print('Phew! That was close!') return True # Some key information... okd_admin_password = os.environ.get(OKD_ADMIN_PASSWORD_ENV) if not okd_admin_password: io.error('You must define {}'.format(OKD_ADMIN_PASSWORD_ENV)) okd_api_hostname = deployment.cluster.public_hostname okd_api_port = deployment.cluster.api_port # ------- # Ansible (A specific version) # ------- # Install the ansible version name in the deployment file cmd = 'pip install --upgrade pip setuptools --user' rv, _ = io.run(cmd, '.', cli_args.quiet) if not rv: return False cmd = 'pip install ansible=={} --user'. \ format(deployment.okd.ansible_version) rv, _ = io.run(cmd, '.', cli_args.quiet) if not rv: return False t_dir = deployment.cluster.terraform_dir if cli_args.cluster: # ------ # Render (jinja2 files) # ------ # Translate content of Jinja2 template files # using the deployment configuration's YAML file content. if not cli_args.skip_rendering: cmd = './render.py {} --ssh-user {}'.\ format(chosen_deployment_name, deployment.cluster.ssh_user) cwd = '.' rv, _ = io.run(cmd, cwd, cli_args.quiet) if not rv: return False # If the deployment file has a 'my_machines' section # then we assume the user's provided their own cluster # and the Terraform step is not needed. if 'my_machines' in deployment: # ----------------- # Manual Templating # ----------------- # The user has provided their own cluster # and defined it in the my_machines section # of their deployment configuration. # # Here we process the rendered inventory files # just as Terraform would do. io.banner('Templating ...') print('inventory') if not templater.render(deployment): return False print('bastion/inventory') file_name = 'ansible/bastion/inventory.yaml.tpl' if not templater.\ render(deployment, template_file_name=file_name): return False print('post-okd/inventory') file_name = 'ansible/post-okd/inventory.yaml.tpl' if not templater. \ render(deployment, template_file_name=file_name, admin_password=okd_admin_password): return False else: # --------- # Terraform # --------- # Create compute instances for the cluster. cmd = 'terraform init' cwd = 'terraform/{}'.format(t_dir) rv, _ = io.run(cmd, cwd, cli_args.quiet) if not rv: return False # Plan or Apply? action = 'plan' if cli_args.just_plan else 'apply -auto-approve' cmd = 'terraform {}' \ ' -state=.terraform.{}'.format(action, chosen_deployment_name) cwd = 'terraform/{}'.format(t_dir) rv, _ = io.run(cmd, cwd, cli_args.quiet) if not rv: return False if cli_args.just_plan: # Just plan means just that... return True # ------- # Ansible # ------- # Run the bastion site file. if not cli_args.skip_pre_okd: extra_env = '' if deployment.okd.certificates: if deployment.okd.certificates.generate_api_cert: certbot_email = os.environ.get(OKD_CERTBOT_EMAIL_ENV) if not certbot_email: io.error( 'You must define {}'.format(OKD_CERTBOT_EMAIL_ENV)) return False extra_env += ' -e master_cert_email="{}"'.\ format(certbot_email) extra_env += ' -e public_hostname="{}"'. \ format(deployment.cluster.public_hostname) elif (deployment.okd.certificates.wildcard_cert or deployment.okd.certificates.master_api_cert): # User-supplied certificates - # expect a vault password file # in the deployment directory extra_env += ' --vault-password-file' \ ' {}/{}/vault-pass.txt'.\ format(OKD_DEPLOYMENTS_DIRECTORY, chosen_deployment_name) if OKD_DEPLOYMENTS_DIRECTORY != 'deployments': extra_env += ' -e deployments_directory="{}"'.\ format(OKD_DEPLOYMENTS_DIRECTORY) else: extra_env += ' -e deployments_directory="../../deployments"' keypair_name = os.environ.get(OKD_KEYPAIR_NAME_ENV) if not keypair_name: io.error('You must define {}'.format(OKD_KEYPAIR_NAME_ENV)) return False cmd = 'ansible-playbook site.yaml' \ ' {}' \ ' -e keypair_name={}' \ ' -e inventory_dir={}' \ ' -e cluster_ssh_user={}' \ ' -e deployment_name={}'.format(extra_env, keypair_name, deployment.okd.inventory_dir, deployment.cluster.ssh_user, chosen_deployment_name) cwd = 'ansible/bastion' rv, _ = io.run(cmd, cwd, cli_args.quiet) if not rv: return False # Now expose the Bastion's IP... if 'my_machines' in deployment: # Simulate the final step in Terraform, # i.e. exposing the bastion. # Doing this simplifies things for the user # i.e. "it looks and feels the same" io.banner('terraform output ...') print('bastion_ip = {}'.format(deployment.my_machines.bastion)) else: cmd = 'terraform output' \ ' -state=.terraform.{}'.format(chosen_deployment_name) cwd = 'terraform/{}'.format(t_dir) rv, _ = io.run(cmd, cwd, cli_args.quiet) if not rv: return False # Leave. return True # If we get here we're installing OpenShift/OKD # (on a cluster that is assumed to exist)... # ----- # Clone (OpenShift Ansible Repo) # ----- # ...and checkout the revision defined by the deployment tag. if not cli_args.skip_okd: # If the expected clone directory does not exist # then clone OpenShift Ansible. if not os.path.exists('openshift-ansible'): cmd = 'git clone' \ ' https://github.com/openshift/openshift-ansible.git' \ ' --no-checkout' cwd = '.' rv, _ = io.run(cmd, cwd, cli_args.quiet) if not rv: return False # Checkout the required OpenShift Ansible TAG cmd = 'git checkout tags/{}'. \ format(deployment.okd.ansible_tag) cwd = 'openshift-ansible' rv, _ = io.run(cmd, cwd, cli_args.quiet) if not rv: return False # ------- # Ansible (Pre-OKD) # ------- if not cli_args.skip_pre_okd: extra_env = '' if deployment.okd.certificates and\ deployment.okd.certificates.generate_api_cert: extra_env += ' -e public_hostname={}'. \ format(deployment.cluster.public_hostname) cmd = 'ansible-playbook site.yaml' \ ' {}' \ ' -i ../../okd/inventories/{}/inventory.yaml'.\ format(extra_env, inventory_dir) cwd = 'ansible/pre-okd' rv, _ = io.run(cmd, cwd, cli_args.quiet) if not rv: return False # ------- # Ansible (OKD) # ------- # Deploy using the playbooks named in the deployment # (from the checked-out version). if not cli_args.skip_okd: for play in deployment.okd.play: cmd = 'ansible-playbook ../openshift-ansible/playbooks/{}.yml' \ ' -i inventories/{}/inventory.yaml'.\ format(play, inventory_dir) cwd = 'okd' rv, _ = io.run(cmd, cwd, cli_args.quiet) if not rv: return False # ------- # Ansible (Post-OKD) # ------- if not cli_args.skip_post_okd: # Always run the 'site' playbook. # This adds the OKD admin and (optional) developer user accounts # and other common things like template deployment. # # The following variables are made available to all the playbooks: - # # - okd_api_hostname # - okd_admin # - okd_admin_password extra_env = '' dev_password = os.environ.get(OKD_DEVELOPER_PASSWORD_ENV) if dev_password: extra_env += ' -e okd_developer_password={}'.format(dev_password) # The template namespace # (optionally defined in the configuration) if deployment.okd.template and deployment.okd.template.namespace: template_namespace = deployment.okd.template.namespace extra_env += ' -e template_namespace={}'.format(template_namespace) cmd = 'ansible-playbook site.yaml' \ '{}' \ ' -e okd_api_hostname=https://{}:{}' \ ' -e okd_admin=admin' \ ' -e okd_admin_password={}' \ ' -e okd_deployment={}'. \ format(extra_env, okd_api_hostname, okd_api_port, okd_admin_password, chosen_deployment_name) cwd = 'ansible/post-okd' rv, _ = io.run(cmd, cwd, cli_args.quiet) if not rv: return False # Now iterate through the plays listed in the cluster's # 'post_okd' list... if deployment.okd.post_okd: for play in deployment.okd.post_okd: # Any user-defined 'extra' variables? play_vars = '' if play.vars: for var in play.vars: play_vars += '-e {} '.format(var) play_vars = play_vars[:-1] # Run the user playbook... cmd = 'ansible-playbook playbooks/{}/deploy.yaml' \ ' -e okd_api_hostname=https://{}:{}' \ ' -e okd_admin=admin' \ ' -e okd_admin_password={}' \ ' -e okd_deployment={}' \ ' {}'.\ format(play.play, okd_api_hostname, okd_api_port, okd_admin_password, chosen_deployment_name, play_vars) cwd = 'ansible/post-okd' rv, _ = io.run(cmd, cwd, cli_args.quiet) if not rv: return False # ------- # Success # ------- # OK if we get here. # Cluster created and OKD installed. return True
True))) @cli.command() @click.pass_context def link(ctx): final_projects = ctx.obj.manager.get_projects() for proj in final_projects: result = proj.link_global() click.echo(repr(result)) @cli.command() @click.option("-t", "--version_type", type=click.Choice(["patch", "minor", "major"]), default="patch") @click.option("-m", "--msg", type=str) @click.pass_context def deploy(ctx, version_type, msg): final_projects = ctx.obj.manager.get_projects() for proj in final_projects: result = proj.deploy(msg=msg, version_type=version_type) click.echo(repr(result)) if __name__ == "__main__": cli(obj=DefaultMunch(None, {}))
def set_conf(self, path): with open(path, "r") as f: conf = yaml.safe_load(f) self.conf = DefaultMunch(None).fromDict(conf)
from pprint import pformat from re import sub from flask_babel import lazy_gettext, lazy_ngettext from munch import DefaultMunch as Munch from toml import dumps, loads from . import dt Munch.__name__ = 'X' Munch.__repr__ = lambda self: f'X{pformat(self.__dict__)}' Munch.toTOML = lambda self: dumps(self) Munch.fromTOML = lambda data: Munch.fromDict(loads(data)) def X(_dict={}, **params): ''' :class:`X` is a `Munch <https://pypi.org/project/munch/>`_. :meth:`X` is also a function that returns a new :class:`X` from a ``dict`` or a set of parameters. :class:`Munch` is "`a dictionary that supports attribute-style access`". :meth:`X` offers us a few options:: user = X(id=1, name='sheila', email='*****@*****.**') user # we can create a new X from parameters X{'email': '*****@*****.**', 'id': 1, 'name': 'sheila'} user['id'] # we can index by the usual brace attribute syntax
def updateMethod(self, method): """Update the method for detecting clap :param dict method: A dict type parameter which defines a clap detection method """ self.method = Objectify.fromDict(method, False)
def copy(self): return Policy( DefaultMunch(undefined, self.context), self.rules.copy(), self.error_factory, self.aspect)
def cfg_strategy_fixture(): b = DefaultMunch() b.url = "test_url" b.api_key = "test_key" b.api_secret = "test_secret" b.name = "market_maker" b.instrument_name = "TEST-PERP" b.mid_price_based_calculation = False b.tick_size = 1 b.price_rounding = 2 b.cancel_orders_on_start = False b.stop_strategy_on_error = True b.cancel_orders_on_reconnection = True b.orders = DefaultMunch() b.orders.asks = [[0, 1]] b.orders.bids = [[0, 1]] return b
def test_pickle_default(): b = DefaultMunch.fromDict({"a": "b"}) assert pickle.loads(pickle.dumps(b)) == b
"frame_selection_method": "random", # frame selection method [choose from: 'random'] "img_resize": (256, 256), # image resize # Caching Options (Text Embeddings and Image Feature Extraction) "use_cache": True, # If True, preprocess text and store as cache, else, process text during runtime "text_embedding_name": "bert_pretrained", # text embedder [choose from: 'bert_pretrained'] "image_feature_extraction": "None" # image feature extraction method [choose from: 'None'] } }, "model": { "storygan": { "img_size": 256, "text_dim": 768, "text_hidden_dim": 75, "noise_dim": 75, "gen_channel": 192 * 8, "dis_channel": 96, "latent_img_dim": 15, "label_num": 2 } } } config = DefaultMunch.fromDict( config, None ) # for attribute style access of dictionary type ex) config.dataset ..etc
def test_repr_default(): b = DefaultMunch(foo=DefaultMunch(lol=True), ponies='are pretty!') assert repr(b).startswith("DefaultMunch(None, {'") assert "'ponies': 'are pretty!'" in repr(b)
def munchify_transform(config): ''' Convert a nested dictionary into a JavaScript-style object (Munch). ''' from munch import DefaultMunch return DefaultMunch.fromDict(config)
from collections import deque from FireflyEnv.env_utils import range_angle from munch import DefaultMunch from Config import Config arg = Config() rewards = deque(maxlen=100) # read configuration parameters #from Config import Config #arg = Config() filename = '20200301-121927' # agent information argument = torch.load(arg.data_path + 'data/' + '20200301-121927_arg.pkl') arg = DefaultMunch.fromDict(argument) # fix random seed import random random.seed(arg.SEED_NUMBER) import torch torch.manual_seed(arg.SEED_NUMBER) if torch.cuda.is_available(): torch.cuda.manual_seed(arg.SEED_NUMBER) import numpy as np np.random.seed(arg.SEED_NUMBER) import time torch.backends.cudnn.deterministic = True torch.backends.cudnn.benchmark = False
def test_pickle(): b = DefaultMunch.fromDict({"a": "b"}) assert pickle.loads(pickle.dumps(b)) == b
def extract_data_as_separate_csv( file_path: Path = None, result_folder: Path = Path(os.getcwd()), filter_columns: List[str] = None, sort_by_columns: List[str] = None, transform_function: Callable[[Union[int, str]], str] = None) -> Dict[str, Any]: """Read a CSV file from specified path, and extract the unique values from specified `filter_columns` in separate CSV files. Example: If we want to extract data for unique MAC addresses appeared in source, destination MAC address fields to individual files, we can call the function as split_csv(input_file, results_folder, data_columns=[ 'src_mac', 'dst_mac']). This will look for all unique MAC addresses appeared in 'src_mac' or 'dst_mac' columns. For each of these MAC addresses, it will filter all rows where that mac address appeared in 'src_mac' or 'dst_mac' columns, and output the result to separate CSV file (in specified results_folder). At the end, the function will provide summary object where keys are unique MAC addresses, and values contain path to output file, and number of rows in the output file for specific MAC address. Parameters ----------- file_path: str Path to input data file :type: string result_folder: str Path to folder where output CSV files should be written. default: current working directory filter_columns: List[str] columns from where unique values are extracted transform_function: function This function is used to transform unique values to human-understandable format. For example, data file contains MAC address as integer. If filtering is done over mac address, you can pass a transform function which\ converts integer representation to MAC to be used in file-names, and summary data Returns ------- summary: dictionary JSON (dictionary object) containing Raises ------ GenericError: exception Error raised if - Input file does not exist, or specified path is directory - Output folder path does not exist or specified path is not a directory - Data can not be loaded to DataFrame from a CSV file - Data can not be written from DataFrame to a CSV file """ output_folder = Path(result_folder) if output_folder.exists() is True and output_folder.is_dir() is False: raise GenericError( 'Specified output folder `{}` is not a directory'.format( result_folder)) if output_folder.exists() is False: os.makedirs(output_folder, exist_ok=True) data = load_csv_to_dataframe(file_path, fill_empty_values=True, verify_columns=filter_columns) base_filename_id = file_path.name.strip().split('.')[0] summary_data = DefaultMunch() unique_values = [] for col in filter_columns: unique_values.extend(list(data[col].unique())) unique_values = set(unique_values) logging.debug('`%s` unique values', len(unique_values)) for val in unique_values: st = time.time() if transform_function is not None: str_val = transform_function(val) else: str_val = val output_file_path = '{folder}/{prefix}-{value}.csv'.format( folder=output_folder, prefix=base_filename_id, value=str_val) filtered_data = get_filtered_data_from_data_frame( data, filter_columns, val, sort_by_columns) write_dataframe_to_csv_file(filtered_data, output_file_path, True, header=True, index=False) summary_data[str_val] = DefaultMunch(file=output_file_path, row_count=filtered_data.shape[0], time_taken=time.time() - st) logging.debug( 'Successfully filtered data for value = `%s` to `%s` in `%s` seconds', str_val, summary_data[str_val]['file'], summary_data[str_val]['time_taken']) return summary_data