def __init__(self, slinky, stepfile): self.slinky = slinky self.path = os.path.abspath(stepfile) self.module = os.path.split(stepfile)[1] self.steps = {} with open(self.path) as inf: firstline = inf.readline() if firstline.startswith('#!') and "python" in firstline: out = self._evil_fast_python_help_reader(self.path) else: out = self._general_reasonable_help_reader(self.path) try: yaml_for_out = yaml.safe_load(out) if yaml_for_out: for name, info in yaml.safe_load(out).items(): if name in self.steps: raise Exception, "stepfile %s has duplicate definitions for '%s'" % (self.path, name) self.steps[name] = Step(self, name, info) else: raise ValueError("There was no yaml.safe_load(out) for self.path `{}`:\n{}".format(self.path, out)) except yaml.scanner.ScannerError as e: raise ValueError("{e} thrown while looking for step {sn} {ss} and out {out}".format( e=str(e), sn=self.path, ss=ss, out=out))
def __read_over(self, overstate): ''' Read in the overstate file ''' if overstate: with salt.utils.fopen(overstate) as fp_: try: # TODO Use render system return self.__sort_stages(yaml.safe_load(fp_)) except Exception: return {} if self.saltenv not in self.opts['file_roots']: return {} for root in self.opts['file_roots'][self.saltenv]: fn_ = os.path.join( root, self.opts.get('overstate', 'overstate.sls') ) if not os.path.isfile(fn_): continue with salt.utils.fopen(fn_) as fp_: try: # TODO Use render system return self.__sort_stages(yaml.safe_load(fp_)) except Exception: return {} return {}
def load_config_file(path): """Load configuration settings from file. Return tuple of configuration dictionary and base path. """ if os.path.isdir(path): for filename in ('.s3_website.yaml', '.s3_website.yml'): filepath = os.path.join(path, filename) try: with open(filepath, 'r') as f: config = yaml.safe_load(f) except: logger.debug('Unable to load config from {}'.format(filepath), exc_info=True) else: base_path = path break else: raise ValueError( 'Unable to find .s3_website.yaml in {}'.format(path)) else: with open(path, 'r') as f: config = yaml.safe_load(f) base_path = os.path.dirname(path) return config, base_path
def process_remote_sources(raw_config, environment=None): """Stage remote package sources and merge in remote configs. Args: raw_config (str): the raw stacker configuration string. environment (dict, optional): any environment values that should be passed to the config Returns: str: the raw stacker configuration string """ config = yaml.safe_load(raw_config) if config and config.get('package_sources'): processor = SourceProcessor( sources=config['package_sources'], stacker_cache_dir=config.get('stacker_cache_dir') ) processor.get_package_sources() if processor.configs_to_merge: for i in processor.configs_to_merge: logger.debug("Merging in remote config \"%s\"", i) remote_config = yaml.safe_load(open(i)) config = merge_map(remote_config, config) # Call the render again as the package_sources may have merged in # additional environment lookups if not environment: environment = {} return render(str(config), environment) return raw_config
def _validate_output(serial_str): try: yaml.safe_load(StringIO(serial_str)) except Exception: return False else: return True
def get_playbook(self): """ If necessary, fetch and read the playbook file """ playbook = self.config['playbook'] if isinstance(playbook, list): # Multiple plays in a list self.playbook = playbook elif isinstance(playbook, str) and playbook.startswith(('http://', 'https://')): response = requests.get(playbook) response.raise_for_status() self.playbook = yaml.safe_load(response.text) elif isinstance(playbook, str): try: playbook_path = os.path.expanduser(playbook) if not playbook_path.startswith('/'): # If the path is not absolute at this point, look for the # playbook in the repo dir. If it's not there, we assume # the path is relative to the working directory pb_in_repo = os.path.join(self.repo_path, playbook_path) if os.path.exists(pb_in_repo): playbook_path = pb_in_repo self.playbook_file = file(playbook_path) playbook_yaml = yaml.safe_load(self.playbook_file) self.playbook = playbook_yaml except Exception: log.error("Unable to read playbook file %s", playbook) raise else: raise TypeError( "playbook value must either be a list, URL or a filename") log.info("Playbook: %s", self.playbook)
def configure(self): if os.path.isfile(installation_path + "/conf/user_data.yaml.tmpl"): userdatayaml = installation_path + "/user-data/" + self.username if os.path.isfile(userdatayaml): userdatayaml_data_stream = open(userdatayaml, "r") yaml_parsed_userdata = yaml.safe_load(userdatayaml_data_stream) userdatayaml_data_stream.close() myversion = yaml_parsed_userdata.get("PHP") backend_config_file = installation_path + "/conf/backends.yaml" backend_data_yaml = open(backend_config_file, "r") backend_data_yaml_parsed = yaml.safe_load(backend_data_yaml) backend_data_yaml.close() if "PHP" in backend_data_yaml_parsed: php_backends_dict = backend_data_yaml_parsed["PHP"] php_path = php_backends_dict.get(myversion) php_profile_set(self.username, php_path) path_to_socket = php_path + "/var/run/" + self.username + ".sock" if os.path.islink("/opt/fpmsockets/" + self.username + ".sock"): os.remove("/opt/fpmsockets/" + self.username + ".sock") os.symlink(path_to_socket, "/opt/fpmsockets/" + self.username + ".sock") else: os.symlink(path_to_socket, "/opt/fpmsockets/" + self.username + ".sock") else: print("ERROR:: PHP Backends missing") else: subprocess.call("cp " + installation_path + "/conf/user_data.yaml.tmpl " + userdatayaml, shell=True) cpuser_uid = pwd.getpwnam(self.username).pw_uid cpuser_gid = grp.getgrnam(self.username).gr_gid os.chown(userdatayaml, cpuser_uid, cpuser_gid) os.chmod(userdatayaml, 0660) self.configure() else: sys.exit(0)
def _subp(self, *args, **kwargs): # supports subp calling with cmd as args or kwargs if 'args' not in kwargs: kwargs['args'] = args[0] self.subp_called.append(kwargs) snap_cmds = [] args = kwargs['args'] # here we basically parse the snappy command invoked # and append to snapcmds a list of (mode, pkg, config) if args[0:2] == ['snappy', 'config']: if args[3] == "-": config = kwargs.get('data', '') else: with open(args[3], "rb") as fp: config = yaml.safe_load(fp.read()) self.snapcmds.append(['config', args[2], config]) elif args[0:2] == ['snappy', 'install']: config = None pkg = None for arg in args[2:]: if arg.startswith("-"): continue if not pkg: pkg = arg elif not config: cfgfile = arg if cfgfile == "-": config = kwargs.get('data', '') elif cfgfile: with open(cfgfile, "rb") as fp: config = yaml.safe_load(fp.read()) self.snapcmds.append(['install', pkg, config])
def __init__(self, filename=None, data=None, fd=None): super(YamlParser, self).__init__() if fd: self.parsed = yaml.safe_load(fd) self.fd = fd try: self.filename = fd.name except AttributeError: self.filename = None elif filename: try: self.parsed = yaml.safe_load(open(filename, "rb")) or OrderedYamlDict() except IOError as e: if e.errno == errno.EACCES: # Specifically catch access denied errors, this usually indicates the # user wanted to read the file, and it existed, but they lacked the # permissions. raise IOError(e) else: self.parsed = OrderedYamlDict() except OSError: self.parsed = OrderedYamlDict() self.filename = filename elif data is not None: fd = StringIO.StringIO(data) self.parsed = yaml.safe_load(fd) self.filename = filename else: raise Error("Filename not specified.")
def get_disks(vm_): ''' Return the disks of a named vm CLI Example:: salt '*' virt.get_disks <vm name> ''' disks = {} doc = minidom.parse(StringIO(get_xml(vm_))) for elem in doc.getElementsByTagName('disk'): sources = elem.getElementsByTagName('source') targets = elem.getElementsByTagName('target') if len(sources) > 0: source = sources[0] else: continue if len(targets) > 0: target = targets[0] else: continue if 'dev' in list(target.attributes) and 'file' in list(source.attributes): disks[target.getAttribute('dev')] = { 'file': source.getAttribute('file')} for dev in disks: try: disks[dev].update(yaml.safe_load(subprocess.Popen('qemu-img info ' + disks[dev]['file'], shell=True, stdout=subprocess.PIPE).communicate()[0])) except TypeError: disks[dev].update(yaml.safe_load('image: Does not exist')) return disks
def test_configuration_with_binary_strings(): """ Regression test: serialization was failing on binary strings """ import yaml obj = '\xaa\xbb\x00\xff\xff\x00ABC' assert yaml.load(yaml.dump(obj)) == obj assert yaml.safe_load(yaml.safe_dump(obj)) == obj obj = {'blob': '\xaa\xbb\x00\xff\xff\x00ABC'} assert yaml.load(yaml.dump(obj)) == obj assert yaml.safe_load(yaml.safe_dump(obj)) == obj obj = { 'function': 'jobcontrol.utils.testing:job_simple_echo', 'title': None, 'notes': None, # 'args': ('\xaa\xbb\x00\xff\xff\x00ABC',), 'args': '\xaa\xbb\x00\xff\xff\x00ABC', 'dependencies': [], 'kwargs': {}, 'id': 'f974e89f-4ae3-40cc-8316-b78e42bd5cc8', } dump(obj)
def wait_until_started(self, wait_load=True): """ Wait until server is started. Server consists of two parts: 1) wait until server is listening on sockets 2) wait until server tells us his status """ if wait_load: msg = 'entering the event loop|will retry binding|hot standby mode' p = self.process if not self.gdb and not self.lldb else None self.logfile_pos.seek_wait(msg, p, self.name) while True: try: temp = AdminConnection('localhost', self.admin.port) if not wait_load: ans = yaml.safe_load(temp.execute("2 + 2")) return True ans = yaml.safe_load(temp.execute('box.info.status'))[0] if ans in ('running', 'hot_standby', 'orphan'): return True elif ans in ('loading'): continue else: raise Exception( "Strange output for `box.info.status`: %s" % (ans) ) except socket.error as e: if e.errno == errno.ECONNREFUSED: time.sleep(0.1) continue raise
def render_template(path, week=None, **kwargs): with open('out/report.yml') as r: report = yaml.safe_load(r) with open('bloggers.yml') as f: users = yaml.safe_load(f) if week: week = parse(week, default=START) else: week = START week = (week - START).days / 7 week_start = START + (week * datetime.timedelta(7)) week_end = START + ((week + 1) * datetime.timedelta(7)) good = [] lame = [] skip = [] userlist = [] class User(object): pass for (un, rec) in users.items(): u = User() u.username = un u.links = rec['links'] u.start = rec['start'] u.end = rec.get('end') u.skip = parse_skip(rec) u.weeks = report.get(un, []) userlist.append(u) def user_key(u): return (u.start, u.username) userlist.sort(key=user_key) for u in userlist: user_start = parse(u.start, default=START) if u.end and parse(u.end, default=START) <= week_start: continue if should_skip(u.skip, week): pass elif user_start > week_start: skip.append(u) elif len(u.weeks) <= week or not u.weeks[week]: lame.append(u) else: good.append(u) debts = get_debts() return Template(filename=path, output_encoding='utf-8').render( week=week, week_start=week_start,week_end=week_end, good=good, lame=lame, skip=skip, userlist=userlist, pool=get_balance('Pool'), paid=get_balance('Pool:Paid'), debts=debts, **kwargs)
def get_disks(vm_): """ Return the disks of a named vm CLI Example:: salt '*' virt.get_disks <vm name> """ disks = {} doc = minidom.parse(StringIO.StringIO(get_xml(vm_))) for elem in doc.getElementsByTagName("disk"): sources = elem.getElementsByTagName("source") targets = elem.getElementsByTagName("target") if len(sources) > 0: source = sources[0] else: continue if len(targets) > 0: target = targets[0] else: continue if "dev" in target.attributes.keys() and "file" in source.attributes.keys(): disks[target.getAttribute("dev")] = {"file": source.getAttribute("file")} for dev in disks: try: disks[dev].update( yaml.safe_load( subprocess.Popen( "qemu-img info " + disks[dev]["file"], shell=True, stdout=subprocess.PIPE ).communicate()[0] ) ) except TypeError: disks[dev].update(yaml.safe_load("image: Does not exist")) return disks
def test_deprecation(spec_fixture, workspace_manager_fixture, # noqa test_workspace, tmpdir): """Verify execution runs with deprecated option """ my_temp_dir = tmpdir.mkdir("tmp") deprecated_output = my_temp_dir.join("deprecated_output.yml") deprecated_input_string = \ ['example', '--deprecated-way', 'TestingValue', '--dry-run', '-o', str(deprecated_output)] output = my_temp_dir.join("output.yml") input_string = \ ['example', '--new-way', 'TestingValue', '--dry-run', '-o', str(output)] spec_manager = api.SpecManager() spec_manager.register_spec(spec_fixture) workspace_manager_fixture.activate(test_workspace.name) spec_manager.run_specs(args=deprecated_input_string) spec_manager.run_specs(args=input_string) with open(deprecated_output.strpath) as fp: deprecated_yml = yaml.safe_load(fp)["provision"] with open(output.strpath) as fp: new_yml = yaml.safe_load(fp)["provision"] assert deprecated_yml.get('new', None).get('way', None) == 'TestingValue' assert new_yml.get('new', None).get('way', None) == 'TestingValue'
def _read_config(config_file_path): if not config_file_path: config_file_path = CONFIG_FILE_NAME defaults_config_file_path = os.path.join( os.path.dirname(os.path.realpath(__file__)), DEFAULTS_CONFIG_FILE_NAME) if not os.path.exists(config_file_path) or not os.path.exists( defaults_config_file_path): if not os.path.exists(defaults_config_file_path): raise ValueError('Missing the defaults configuration file; ' 'expected to find it at {0}'.format( defaults_config_file_path)) raise ValueError('Missing the configuration file; expected to find ' 'it at {0}'.format(config_file_path)) lgr.debug('reading provider config files') with open(config_file_path, 'r') as config_file, \ open(defaults_config_file_path, 'r') as defaults_config_file: lgr.debug('safe loading user config') user_config = yaml.safe_load(config_file.read()) lgr.debug('safe loading default config') defaults_config = yaml.safe_load(defaults_config_file.read()) lgr.debug('merging configs') merged_config = _deep_merge_dictionaries(user_config, defaults_config) \ if user_config else defaults_config return merged_config
def validate_config(self, template_data, yaml_file): try: yaml.safe_load(template_data) except (yaml.scanner.ScannerError, yaml.YAMLError) as e: self.log.error("Config for file {} contains invalid yaml, got " "error {}".format(yaml_file, e)) raise e
def get_bundle(bundle, to_file=False): """ Attempts to grab the bundle.yaml Arguments: bundle: URL of bundle or absolute path to local bundle to_file: store to a temporary file Returns: Dictionary of bundle's yaml unless to_file is True, then returns the path to the downloaded bundle """ if path.isfile(bundle): if to_file: with NamedTemporaryFile(mode="w", encoding="utf-8", delete=False) as tempf: shutil.copyfile(bundle, tempf.name) return tempf.name else: with open(bundle) as f: return yaml.safe_load(f.read()) req = requests.get(bundle) if not req.ok: raise Exception("Problem getting bundle: {}".format(req)) if to_file: with NamedTemporaryFile(mode="w", encoding="utf-8", delete=False) as tempf: tempf.write(req.text) return tempf.name else: return yaml.safe_load(req.text)
def init(config, use, debug, verbose, accounts, tags, policies, resource=None, policy_tags=()): level = verbose and logging.DEBUG or logging.INFO logging.basicConfig( level=level, format="%(asctime)s: %(name)s:%(levelname)s %(message)s") logging.getLogger('botocore').setLevel(logging.ERROR) logging.getLogger('custodian').setLevel(logging.WARNING) logging.getLogger('custodian.s3').setLevel(logging.ERROR) logging.getLogger('urllib3').setLevel(logging.WARNING) with open(config) as fh: accounts_config = yaml.safe_load(fh.read()) jsonschema.validate(accounts_config, CONFIG_SCHEMA) if use: with open(use) as fh: custodian_config = yaml.safe_load(fh.read()) else: custodian_config = {} accounts_config['accounts'] = list(accounts_iterator(accounts_config)) filter_policies(custodian_config, policy_tags, policies, resource) filter_accounts(accounts_config, tags, accounts) load_resources() MainThreadExecutor.c7n_async = False executor = debug and MainThreadExecutor or ProcessPoolExecutor return accounts_config, custodian_config, executor
def main(): parser = argparse.ArgumentParser() parser.add_argument('--old', required=True, help='Current policy file') parser.add_argument('--new', required=True, help='New policy file') parser.add_argument('--mode', choices=['add', 'remove'], default='remove', help='Diffs to be shown') parsed_args = parser.parse_args() with open(parsed_args.old) as f: old_data = yaml.safe_load(f) with open(parsed_args.new) as f: new_data = yaml.safe_load(f) added = set(new_data.keys()) - set(old_data.keys()) removed = set(old_data.keys()) - set(new_data.keys()) if parsed_args.mode == 'remove': for key in sorted(removed): print(key) if parsed_args.mode == 'add': for key in sorted(added): print(key)
def __init__(self): try: self.name=m3tt.get_m3_config()['sys_test'] except KeyError: print 'Key sys_test not defined in m3_config.yml. Exiting...' exit() path=m3tt.get_m3_config_path() filename= path+'sys/m3_sys_test_'+self.name+'.yml' try: f=file(filename,'r') self.config= yaml.safe_load(f.read()) f.close() except IOError: print 'File ',filename,'not found. Exiting...' exit() self.log=[] #load included self.config_include={'not_val':[],'in_range':[]} if self.config.has_key('include'): for fi in self.config['include']: filename= path+'sys/m3_sys_test_'+fi+'.yml' try: f=file(filename,'r') print 'Including: ',filename c= yaml.safe_load(f.read()) f.close() if c.has_key('not_val'): self.config_include['not_val'].append(c['not_val']) if c.has_key('in_range'): self.config_include['in_range'].append(c['in_range']) except IOError: print 'File ',filename,'not found. Exiting...' exit()
def test_save_shortcut_updated(self): OLD_YML = """\ projects: default: 12345 prod: 33333 """ runner = CliRunner() with runner.isolated_filesystem(): with open('scrapinghub.yml', 'w') as f: f.write(textwrap.dedent(OLD_YML)) conf = ShubConfig() conf.load_file('scrapinghub.yml') del conf.projects['prod'] print(conf.projects) conf.save('scrapinghub.yml') with open('scrapinghub.yml', 'r') as f: new_yml = yaml.safe_load(f) # Should not contain 'projects' self.assertEqual(new_yml, {'project': 12345}) conf = ShubConfig() conf.load_file('scrapinghub.yml') # Should also work in reverse conf.projects['prod'] = 33333 conf.save('scrapinghub.yml') with open('scrapinghub.yml', 'r') as f: new_yml = yaml.safe_load(f) # Should not contain 'project' singleton self.assertEqual( new_yml, {'projects': {'default': 12345, 'prod': 33333}}, ) # Make sure it is readable again ShubConfig().load_file('scrapinghub.yml')
def settings(self): core_path = os.path.join(get_root_path('indico'), 'modules', 'events', 'themes.yaml') with open(core_path) as f: core_data = f.read() core_settings = yaml.safe_load(core_data) # YAML doesn't give us access to anchors so we need to include the base yaml. # Since duplicate keys are invalid (and may start failing in the future) we # rename them - this also makes it easy to throw them away after parsing the # file provided by a plugin. core_data = re.sub(r'^(\S+:)$', r'__core_\1', core_data, flags=re.MULTILINE) for plugin, path in values_from_signal(signals.plugin.get_event_themes_files.send(), return_plugins=True): with open(path) as f: data = f.read() settings = {k: v for k, v in yaml.safe_load(core_data + '\n' + data).viewitems() if not k.startswith('__core_')} # We assume there's no more than one theme plugin that provides defaults. # If that's not the case the last one "wins". We could reject this but it # is quite unlikely that people have multiple theme plugins in the first # place, even more so theme plugins that specify defaults. core_settings['defaults'].update(settings.get('defaults', {})) # Same for definitions - we assume plugin authors are responsible enough # to avoid using definition names that are likely to cause collisions. # Either way, if someone does this on purpose chances are good they want # to override a default style so let them do so... for name, definition in settings.get('definitions', {}).viewitems(): definition['plugin'] = plugin definition.setdefault('user_visible', False) core_settings['definitions'][name] = definition return core_settings
def generate_confs(tileset, ignore_warnings=True, renderd=False): """ Takes a Tileset object and returns mapproxy and seed config files """ # Start with a sane configuration using MapProxy's defaults mapproxy_config = load_default_config() tileset_conf_json = get_mapproxy_conf(tileset) tileset_conf = yaml.safe_load(tileset_conf_json) # merge our config load_config(mapproxy_config, config_dict=tileset_conf) seed_conf_json = get_seed_conf(tileset) seed_conf = yaml.safe_load(seed_conf_json) errors, informal_only = validate_options(mapproxy_config) if not informal_only or (errors and not ignore_warnings): raise ConfigurationError('invalid configuration - {}'.format(', '.join(errors))) mapproxy_cf = ProxyConfiguration(mapproxy_config, seed=seed, renderd=renderd) errors, informal_only = validate_seed_conf(seed_conf) if not informal_only: raise SeedConfigurationError('invalid seed configuration - {}'.format(', '.join(errors))) seed_cf = SeedingConfiguration(seed_conf, mapproxy_conf=mapproxy_cf) return mapproxy_cf, seed_cf
def configure_sources(update=False, sources_var='install_sources', keys_var='install_keys'): """ Configure multiple sources from charm configuration Example config: install_sources: - "ppa:foo" - "http://example.com/repo precise main" install_keys: - null - "a1b2c3d4" Note that 'null' (a.k.a. None) should not be quoted. """ sources = safe_load(config(sources_var)) keys = config(keys_var) if keys is not None: keys = safe_load(keys) if isinstance(sources, basestring) and ( keys is None or isinstance(keys, basestring)): add_source(sources, keys) else: if not len(sources) == len(keys): msg = 'Install sources and keys lists are different lengths' raise SourceConfigError(msg) for src_num in range(len(sources)): add_source(sources[src_num], keys[src_num]) if update: apt_update(fatal=True)
def get_playbook(self): """ If necessary, fetch and read the playbook file """ playbook = self.config['playbook'] if isinstance(playbook, list): # Multiple plays in a list self.playbook = playbook elif isinstance(playbook, str) and playbook.startswith(('http://', 'https://')): response = requests.get(playbook) response.raise_for_status() self.playbook = yaml.safe_load(response.text) elif isinstance(playbook, str): try: playbook_path = os.path.expanduser(playbook) self.playbook_file = file(playbook_path) playbook_yaml = yaml.safe_load(self.playbook_file) self.playbook = playbook_yaml except Exception: log.error("Unable to read playbook file %s", playbook) raise else: raise TypeError( "playbook value must either be a list, URL or a filename") log.info("Playbook: %s", self.playbook)
def test_config_yaml(self): import yaml cli.config(Args(), ["--format=yaml"]) data1 = yaml.safe_load(self.stdout.getvalue().strip().replace(helper.ROOT_PATH, "<ROOT>")) data2 = yaml.safe_load(config_file("data.json")) self.assertEqual(data1, data2)
def test_for_consistency(tests_path): """ Ensure that there is consistency between environment.yml dependencies and conda.recipe/meta.yaml requirements. """ dev_pkgs = set([ 'pytest', 'pytest-pep8', 'pytest-xdist', 'pycodestyle', 'pylint', 'coverage' ]) # read conda.recipe/meta.yaml requirements meta_file = os.path.join(tests_path, '..', '..', 'conda.recipe', 'meta.yaml') with open(meta_file, 'r') as stream: meta = yaml.safe_load(stream) bld = set(meta['requirements']['build']) run = set(meta['requirements']['run']) # confirm conda.recipe/meta.yaml build and run requirements are the same assert bld == run # read environment.yml dependencies envr_file = os.path.join(tests_path, '..', '..', 'environment.yml') with open(envr_file, 'r') as stream: envr = yaml.safe_load(stream) env = set(envr['dependencies']) # confirm that extras in env (relative to run) equal the dev_pkgs set extras = env - run assert extras == dev_pkgs
def __init__(self, connections_amount=2, connection_propability=0.2): self.__fire = 1000 with open("items.yaml") as data_file: items_data = safe_load(data_file) self.items = [ Item(elem['name'], elem['description'], elem['capacity']) for elem in items_data ] with open("rooms.yaml") as data_file: rooms_data = safe_load(data_file) self.rooms = [ Room( elem['name'], elem['description'], random.sample(self.items, random.randint(0,2)), bool(elem.get('water_source', 0)) ) for elem in rooms_data ] self.graph = graph_gen(len(self.rooms), connections_amount,\ connection_propability) random.shuffle(self.rooms) for no, room in enumerate(self.rooms): room.neighbors = [self.rooms[index].name for index in \ self.graph.neighbors(no)] #we start at 0 self.fired_room = self.rooms[random.randint(1, len(self.rooms))].name self.spread_fire()
def generate_ui(path, modules=[]): """Takes a path to a YAML UI definition, and generates a UI tree for it. :param definition: A UI definition representing the UI to be created. :param modules: (Optional) A list of module names which need to be imported in order to generate the UI tree. This should include all module names which define custom widgets or callbacks using decorators that are used in the definition. """ for module in modules: importlib.import_module(module) with open(path, 'r') as ui_file: ui = yaml.safe_load(ui_file) full_style = {} for style_path in ui.get('include', []): with open(style_path, 'r') as style_file: style = yaml.safe_load(style_file) for definition in style: full_style[definition['name']] = definition['properties'] definition = ui['definition'] root_class = yamlui.class_mapping.get(definition['object']) if root_class is None: raise Exception('ERROR: Root class is an unrecognised widget type.') root = root_class(definition, style=full_style) ui_name = os.path.basename(path) yamlui.trees[ui_name] = build_dictionary(root, ui_name) return root
def create_yaml_files(self, datafile): # Load existing Yaml file (If any) # Load main verifications datafiles #main_file = OrderedDict() main_file = {} nxos_file = { 'extends': '%ENV{VIRTUAL_ENV}/genie_yamls/verification_datafile.yaml' } iosxe_file = { 'extends': '%ENV{VIRTUAL_ENV}/genie_yamls/verification_datafile.yaml' } iosxr_file = { 'extends': '%ENV{VIRTUAL_ENV}/genie_yamls/verification_datafile.yaml' } nxos = [] iosxe = [] iosxr = [] # Load the file with open(datafile, 'r') as f: parser_yaml = yaml.safe_load(f) main_yaml = os.path.join(YAMLS, 'verification_datafile.yaml') with open(main_yaml, 'r') as f: content = yaml.safe_load(f) nxos_yaml = os.path.join(YAMLS, 'nxos', 'verification_datafile_nxos.yaml') with open(nxos_yaml, 'r') as f: nxos_content = yaml.safe_load(f) iosxe_yaml = os.path.join(YAMLS, 'iosxe', 'verification_datafile_iosxe.yaml') with open(iosxe_yaml, 'r') as f: iosxe_content = yaml.safe_load(f) iosxr_yaml = os.path.join(YAMLS, 'iosxr', 'verification_datafile_xr.yaml') with open(iosxr_yaml, 'r') as f: iosxr_content = yaml.safe_load(f) # All parser should be in this verification datafile for osx in self.parsers: if osx == 'nxos': os_yaml = nxos_content os_file = nxos_file triggers = nxos elif osx == 'iosxe': os_yaml = iosxe_content os_file = iosxe_file triggers = iosxe elif osx == 'iosxr': os_yaml = iosxr_content os_file = iosxr_file triggers = iosxr for file in self.parsers[osx]: for parser in self.parsers[osx][file]: # Check which context exists for context in CONTEXTS: if not hasattr(parser, context): continue parser_name = parser.__name__ # Verification name verification_name = 'Verify_{p}'.format( p=parser_name.replace('Show', '')) if context != 'cli': verification_name = '{v}_{c}'.format( v=verification_name, c=context) values = [] if parser_name in parser_yaml: # initial index number index_num = None # For all of the combination, add it # Make the lists ready to go for key, items in sorted( parser_yaml[parser_name].items(), reverse=True): if isinstance(items, dict): if key not in parser.__module__: continue for ky, val in sorted(items.items(), reverse=True): count = 0 if ky == 'zos': try: index_num = val.index(osx) except: values.append(None) break continue if index_num is not None: val = val[index_num] for item in val: if item == '' or item is None: count += 1 continue try: values[count].extend( [ky, val[count]]) except IndexError: values.append([ky, val[count]]) count += 1 else: count = 0 if key == 'zos': try: index_num = items.index(osx) except: values.append(None) break continue if index_num is not None: items = items[index_num] for item in items: if item == '' or item is None: count += 1 continue try: values[count].extend( [key, items[count]]) except IndexError: values.append([key, items[count]]) count += 1 else: values.append(None) for value in values: if value is not None: veri_name = '{v}_{e}'.format( v=verification_name, e='_'.join(value).replace(' ', '_')) else: veri_name = verification_name main_file[veri_name] = {} main_file[veri_name]['source'] = { 'class': 'genie.harness.base.Template' } main_file[veri_name]['context'] = context main_file[veri_name]['cmd'] = {} main_file[veri_name]['cmd'][ 'pkg'] = 'genie.libs.parser' main_file[veri_name]['cmd'][ 'class'] = '{f}.{p}'.format(f=file, p=parser.__name__) os_file[veri_name] = {} if veri_name not in EXCLUDE_DEVICES: os_file[veri_name]['devices'] = {'uut': 'None'} if value is not None: for i in range(0, len(value), 2): if value[i + 1] != 'default': if 'parameters' not in os_file[ veri_name]: os_file[veri_name][ 'parameters'] = {} os_file[veri_name]['parameters'][ value[i]] = value[i + 1] if veri_name in content: # Good already exists # Do not copy source and cmd # But keep the rest try: del content[veri_name]['source'] except: pass try: del content[veri_name]['cmd'] except: pass merge_dict(main_file[veri_name], content[veri_name]) if veri_name in os_yaml: merge_dict(os_file[veri_name], os_yaml[veri_name]) triggers.append(veri_name) # Create the files with open('verification_datafile.yaml', 'w') as f: yaml.dump(main_file, f, default_flow_style=False) self.clean_up('nxos') with open('nxos/verification_datafile_nxos.yaml', 'w') as f: yaml.dump(nxos_file, f, default_flow_style=False) self.clean_up('iosxe') with open('iosxe/verification_datafile_iosxe.yaml', 'w') as f: yaml.dump(iosxe_file, f, default_flow_style=False) self.clean_up('iosxr') with open('iosxr/verification_datafile_xr.yaml', 'w') as f: yaml.dump(iosxr_file, f, default_flow_style=False) log.info(banner('nxos')) log.info('\n'.join(nxos)) log.info(banner('iosxe')) log.info('\n'.join(iosxe)) log.info(banner('iosxr')) log.info('\n'.join(iosxr)) return main_file
# This script is to contain all the parameters within the simulation # from parameters.config import read_config import yaml with open("params.yml", 'r') as stream: params_dict = yaml.safe_load(stream) print (params_dict) # Cahn Hilliard Model type: MOBILITY_MODEL = params_dict["MOBILITY_MODEL"] # Compositon parameters A_RAW = params_dict["A_RAW"] NOISE_MAGNITUDE = params_dict["NOISE_MAGNITUDE"] # Material parameters (either PS_PMMA / PS_PB) MATERIAL_CHOICE = params_dict["MATERIAL_CHOICE"] SIZE_DISPARITY = params_dict["SIZE_DISPARITY"] # Homogenous Free energy function (PC_SAFT, FH, etc) GIBBS = params_dict["GIBBS"] # Numerics DT = params_dict["DT"] TIME_MAX = params_dict["TIME_MAX"] N_CELLS = params_dict["N_CELLS"] DOMAIN_LENGTH = params_dict["DOMAIN_LENGTH"] theta_ch = params_dict["theta_ch"] MESH_TYPE = params_dict["MESH_TYPE"]
def update_nrpe_checks(): if os.path.isdir(NAGIOS_PLUGINS): rsync(os.path.join(charm_dir(), 'scripts', 'check_rabbitmq.py'), os.path.join(NAGIOS_PLUGINS, 'check_rabbitmq.py')) rsync(os.path.join(charm_dir(), 'scripts', 'check_rabbitmq_queues.py'), os.path.join(NAGIOS_PLUGINS, 'check_rabbitmq_queues.py')) if config('stats_cron_schedule'): script = os.path.join(SCRIPTS_DIR, 'collect_rabbitmq_stats.sh') cronjob = CRONJOB_CMD.format(schedule=config('stats_cron_schedule'), timeout=config('cron-timeout'), command=script) rsync(os.path.join(charm_dir(), 'scripts', 'collect_rabbitmq_stats.sh'), script) write_file(STATS_CRONFILE, cronjob) if config('management_plugin'): rsync(os.path.join(charm_dir(), 'scripts', 'check_rabbitmq_cluster.py'), os.path.join(NAGIOS_PLUGINS, 'check_rabbitmq_cluster.py')) elif os.path.isfile(STATS_CRONFILE): os.remove(STATS_CRONFILE) # Find out if nrpe set nagios_hostname hostname = nrpe.get_nagios_hostname() myunit = nrpe.get_nagios_unit_name() # create unique user and vhost for each unit current_unit = local_unit().replace('/', '-') user = '******' % current_unit vhost = 'nagios-%s' % current_unit password = rabbit.get_rabbit_password(user, local=True) rabbit.create_vhost(vhost) rabbit.create_user(user, password, ['monitoring']) rabbit.grant_permissions(user, vhost) nrpe_compat = nrpe.NRPE(hostname=hostname) if config('ssl') in ['off', 'on']: cmd = ('{plugins_dir}/check_rabbitmq.py --user {user} ' '--password {password} --vhost {vhost}') cmd = cmd.format(plugins_dir=NAGIOS_PLUGINS, user=user, password=password, vhost=vhost) nrpe_compat.add_check( shortname=rabbit.RABBIT_USER, description='Check RabbitMQ {%s}' % myunit, check_cmd=cmd ) if config('ssl') in ['only', 'on']: log('Adding rabbitmq SSL check', level=DEBUG) cmd = ('{plugins_dir}/check_rabbitmq.py --user {user} ' '--password {password} --vhost {vhost} ' '--ssl --ssl-ca {ssl_ca} --port {port}') cmd = cmd.format(plugins_dir=NAGIOS_PLUGINS, user=user, password=password, port=int(config('ssl_port')), vhost=vhost, ssl_ca=SSL_CA_FILE) nrpe_compat.add_check( shortname=rabbit.RABBIT_USER + "_ssl", description='Check RabbitMQ (SSL) {%s}' % myunit, check_cmd=cmd ) if config('queue_thresholds'): cmd = "" # If value of queue_thresholds is incorrect we want the hook to fail for item in yaml.safe_load(config('queue_thresholds')): cmd += ' -c "{}" "{}" {} {}'.format(*item) nrpe_compat.add_check( shortname=rabbit.RABBIT_USER + '_queue', description='Check RabbitMQ Queues', check_cmd='{}/check_rabbitmq_queues.py{} {}'.format( NAGIOS_PLUGINS, cmd, STATS_DATAFILE) ) if config('management_plugin'): # add NRPE check nrpe_compat.add_check( shortname=rabbit.RABBIT_USER + '_cluster', description='Check RabbitMQ Cluster', check_cmd='{}/check_rabbitmq_cluster.py --port {} --user {} --password {}'.format( NAGIOS_PLUGINS, rabbit.get_managment_port(), user, password ) ) nrpe_compat.write()
def from_yaml(data): if isinstance(data, string_types): return yaml.safe_load(data) return data
comp = zlib.compress(weights_np, level=9) print(len(comp)) # sim.export(path=os.path.join( # os.path.join(exp_folder, # image_name, 'checkpoints')), filename_prefix='model_aimet_quantized_retrained', dummy_input=dummy_in, set_onnx_layer_names=False) print(res) return quantized_model, res, len(comp), state_dict if __name__ == '__main__': device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') exp_folder = 'siren/exp/KODAK21_epochs10000_lr0.0001_hdims32_hlayer4_nerf_sine_l1_reg5e-05_enc_scale10.0/' #exp_folder = 'siren/exp/KODAK21_epochs10000_lr0.0001_hdims64_hlayer4_nerf_sine_enc_scale10.0/' exp_folder = '/home/yannick/PycharmProjects/INR/exp/KODAK21_epochs10000_lr0.0001_mlp_[8]_hdims64_hlayer2_nerf_sine_enc_scale4.0' TRAINING_FLAGS = yaml.safe_load(open(os.path.join(exp_folder, 'FLAGS.yml'), 'r')) image_name = 'kodim21' imglob = glob.glob('/home/yannick/KODAK/kodim21.png') for im in imglob: image_name = im.split('/')[-1].split('.')[0] img_dataset = dataio.ImageFile(im) img = PIL.Image.open(im) scale = TRAINING_FLAGS['downscaling_factor'] image_resolution = (img.size[1] // scale, img.size[0] // scale) coord_dataset = dataio.Implicit2DWrapper(img_dataset, sidelength=image_resolution) dataloader = DataLoader(coord_dataset, shuffle=True, batch_size=1, pin_memory=True, num_workers=0) input_shape = (1, coord_dataset.mgrid.shape[0], coord_dataset.mgrid.shape[1]) print(input_shape)
def __init__(self, config_file): with open(config_file, 'r') as file: try: self.config = yaml.safe_load(file) except yaml.YAMLError as err: print(err) self.floats = [] if 'floats' in self.config: for floatOpt in self.config['floats']: for opt, val in floatOpt.items(): if len(val) > 0 and val != 'default': self.floats += [LatexLengthOption(opt, val)] self.compact_items = False self.compact_enums = False if 'lists' in self.config: list_config = self.config['lists'] if 'compact-items' in list_config: self.compact_items = list_config['compact-items'] if 'compact-enums' in list_config: self.compact_enums = list_config['compact-enums'] self.headings = [] if 'headings' in self.config: heads_config = self.config['headings'] for head in heads_config: head_config = heads_config[head] left = head_config['left'] or "0pt" above = head_config['above'] or "0pt" below = head_config['below'] or "0pt" self.headings += [ LatexCommand('titlespacing', None, [f'\\{head}', left, above, below]) ] self.paragraphs = [] if 'paragraphs' in self.config: for opt, val in self.config['paragraphs'].items(): if len(val) > 0 and val != 'default': self.paragraphs += [LatexLengthOption(opt, val)] self.equations = [] if 'equations' in self.config: for opt, val in self.config['equations'].items(): if len(val) > 0 and val != 'default': self.equations += [LatexLengthOption(opt, val)] spread_factor = 1.0 if 'line-spread' in self.config: spread_factor = self.config['line-spread'] self.line_spread = [LatexCommand('linespread', [], str(spread_factor))] self.caption_opts = [] if 'captions' in self.config: if 'above-skip' in self.config['captions']: aboveskip = self.config['captions']['above-skip'] if aboveskip != 'default': self.caption_opts += [f'above-skip={aboveskip}'] if 'below-skip' in self.config['captions']: belowskip = self.config['captions']['below-skip'] if belowskip != 'default': self.caption_opts += [f'below-skip={belowskip}'] if 'skip' in self.config['captions']: skip = self.config['captions']['skip'] if skip != 'default': self.caption_opts += [f'skip={skip}'] if 'font' in self.config['captions']: font = self.config['captions']['font'] if font != 'default': self.caption_opts += [f'font={font}']
# See https://docs.djangoproject.com/en/3.2/howto/deployment/checklist/ # SECURITY WARNING: keep the secret key used in production secret! # SECURITY WARNING: don't run with debug turned on in production! DEBUG = False if os.getenv('GAE_APPLICATION', None): ALLOWED_HOSTS = [ 'really-site-312814.an.r.appspot.com', ] else: DEBUG = True ALLOWED_HOSTS = ['*'] import yaml with open(os.path.join(BASE_DIR, 'secrets', 'secret_dev.yaml')) as file: objs = yaml.safe_load(file) for obj in objs: os.environ[obj] = objs[obj] SECRET_KEY = os.environ['DJANGO_SECRET_KEY'] # Application definition INSTALLED_APPS = [ 'django.contrib.admin', 'django.contrib.auth', 'django.contrib.contenttypes', 'django.contrib.sessions', 'livereload', 'django.contrib.messages', 'django.contrib.staticfiles', 'django.contrib.sitemaps', 'mysite',
def get_install_config(self): return yaml.safe_load(self.api_client.get_cluster_install_config(self.id))
if rank: article['popularity'] = int(rank[0]['Ranking']) else: article['popularity'] = 0 logging.debug("Add to list") parsed_articles.append(article) # Sort by title parsed_articles = sorted(parsed_articles, key=lambda x: x['Title']) all_tags = [] all_langs = [] with open(path.join(doc_directory, "toc.yml"), 'r') as stream: main_toc = yaml.safe_load(stream) all_topics = defaultdict(list) for article in parsed_articles: # toc_link=list(find_all(main_toc,article['file_url'],'item')) # if len(toc_link) == 0: # main_toc= update_toc(article, main_toc) # Sort the tags article['tags'] = sorted(article['tags']) # Define the primnary topic for the article article['topic'] = categories[article['category'][0]] # Build topic array
}, { 'name': 'TestNetwork_5', 'vlanId': 5 }, { 'name': 'TestNetwork_9', 'vlanId': 9 }, { 'name': 'TestNetwork_10', 'vlanId': 10 }, ] DICT_PARAMS_WITH_CHANGES = yaml.safe_load(YAML_PARAMS_WITH_CHANGES)["data"] class EthernetNetworkModuleSpec(unittest.TestCase, OneViewBaseTestCase): """ OneViewBaseTestCase provides the mocks used in this test case """ def setUp(self): self.configure_mocks(self, EthernetNetworkModule) self.resource = self.mock_ov_client.ethernet_networks def test_should_create_new_ethernet_network(self): self.resource.get_by.return_value = [] self.resource.create.return_value = DEFAULT_ENET_TEMPLATE self.mock_ansible_module.params = PARAMS_FOR_PRESENT
def test_should_include(self, builder_context: PipelineBuilderContext) -> None: """Test calculating configuration for the given prescription unit.""" prescription_str = f"""\ name: GroupStep type: step.Group should_include: adviser_pipeline: true match: group: - package_version: name: numpy version: '!=1.21.4' index_url: not: 'https://pypi.org/simple' - package_version: name: tensorflow version: '~=2.7.0' - package_version: name: pandas index_url: 'https://thoth-station.ninja' develop: true - package_version: name: werkzeug version: '==1.0.0' index_url: 'https://pypi.org/simple' develop: false run: stack_info: - type: INFO message: Running the unit. link: 'https://thoth-station.ninja' """ prescription = yaml.safe_load(prescription_str) PRESCRIPTION_GROUP_STEP_SCHEMA(prescription) GroupStepPrescription.set_prescription(prescription) configurations = list( GroupStepPrescription.should_include(builder_context)) assert configurations == [ { "match": { "package_version": { "index_url": { "not": "https://pypi.org/simple" }, "name": "numpy", "version": "!=1.21.4", }, "state": { "resolved_dependencies": [ { "name": "tensorflow", "version": "~=2.7.0" }, { "develop": True, "index_url": "https://thoth-station.ninja", "name": "pandas" }, { "develop": False, "index_url": "https://pypi.org/simple", "name": "werkzeug", "version": "==1.0.0", }, ] }, }, "multi_package_resolution": True, "package_name": "numpy", "prescription": { "run": False }, "run": { "stack_info": [{ "link": "https://thoth-station.ninja", "message": "Running the unit.", "type": "INFO" }] }, }, { "match": { "package_version": { "name": "tensorflow", "version": "~=2.7.0" }, "state": { "resolved_dependencies": [ { "index_url": { "not": "https://pypi.org/simple" }, "name": "numpy", "version": "!=1.21.4" }, { "develop": True, "index_url": "https://thoth-station.ninja", "name": "pandas" }, { "develop": False, "index_url": "https://pypi.org/simple", "name": "werkzeug", "version": "==1.0.0", }, ] }, }, "multi_package_resolution": True, "package_name": "tensorflow", "prescription": { "run": False }, "run": { "stack_info": [{ "link": "https://thoth-station.ninja", "message": "Running the unit.", "type": "INFO" }] }, }, { "match": { "package_version": { "develop": True, "index_url": "https://thoth-station.ninja", "name": "pandas" }, "state": { "resolved_dependencies": [ { "index_url": { "not": "https://pypi.org/simple" }, "name": "numpy", "version": "!=1.21.4" }, { "name": "tensorflow", "version": "~=2.7.0" }, { "develop": False, "index_url": "https://pypi.org/simple", "name": "werkzeug", "version": "==1.0.0", }, ] }, }, "multi_package_resolution": True, "package_name": "pandas", "prescription": { "run": False }, "run": { "stack_info": [{ "link": "https://thoth-station.ninja", "message": "Running the unit.", "type": "INFO" }] }, }, { "match": { "package_version": { "develop": False, "index_url": "https://pypi.org/simple", "name": "werkzeug", "version": "==1.0.0", }, "state": { "resolved_dependencies": [ { "index_url": { "not": "https://pypi.org/simple" }, "name": "numpy", "version": "!=1.21.4" }, { "name": "tensorflow", "version": "~=2.7.0" }, { "develop": True, "index_url": "https://thoth-station.ninja", "name": "pandas" }, ] }, }, "multi_package_resolution": True, "package_name": "werkzeug", "prescription": { "run": False }, "run": { "stack_info": [{ "link": "https://thoth-station.ninja", "message": "Running the unit.", "type": "INFO" }] }, }, ] for conf in configurations: GroupStepPrescription.CONFIGURATION_SCHEMA(conf)
def load_yaml(self, fp): if self._reader_format == "yaml": return pd.DataFrame(safe_load(fp))
#!/usr/bin/python import yaml import glob import os # Load old configurations try: sfconfig = yaml.safe_load(open("/etc/software-factory/sfconfig.yaml")) except IOError: sfconfig = {'nodepool': {}} images = yaml.safe_load(open("nodepool/images.yaml")) labels = yaml.safe_load(open("nodepool/labels.yaml")) # Prepare the new nodepool.yaml nodepool = { 'labels': labels['labels'], 'diskimages': [], 'providers': [], } # Import provider setting from sfconfig.yaml for provider in sfconfig["nodepool"].get("providers", []): new_provider = { "name": provider["name"], "cloud": provider["name"], "clean-floating-ips": True, "images": [], } if "network" in provider and provider.get("network") is not None: new_provider["networks"] = [{"name": provider["network"]}]
else: last_read_message_id = await process_messages( client, messages_list, config["media_types"], config["file_formats"], ) pagination_count = 0 messages_list = [] messages_list.append(message) if messages_list: last_read_message_id = await process_messages( client, messages_list, config["media_types"], config["file_formats"], ) await client.stop() config["last_read_message_id"] = last_read_message_id return config if __name__ == "__main__": f = open(os.path.join(THIS_DIR, "config.yaml")) config = yaml.safe_load(f) f.close() updated_config = asyncio.get_event_loop().run_until_complete( begin_import(config, pagination_limit=100)) update_config(updated_config)
import yaml from task_18_1 import send_show_command from task_18_2 import send_config_commands commands = [ "logging 10.255.255.1", "logging buffered 20010", "no logging console" ] command = "sh ip int br" def send_commands(device, *, config=None, show=None): if show and config: raise ValueError( "Можно передавать только один из аргументов show/config") elif show: return send_show_command(device, show) elif config: return send_config_commands(device, config) if __name__ == "__main__": commands = [ "logging 10.255.255.1", "logging buffered 20010", "no logging console" ] command = "sh ip int br" with open("devices.yaml") as f: devices = yaml.safe_load(f) r1 = devices[0] print(send_commands(r1, config=commands)) print(send_commands(r1, show=command))
def get_template_without_crds(path): template = yaml.safe_load(open(path)) template['objects'] = [ obj for obj in template['objects'] if obj['kind'] != 'CustomResourceDefinition' ] return template
def load(file): with open(file) as f: yaml.safe_load(f) new_dict = Dictionary() return new_dict
def run(numWorlds, numElems, paramsPath, outputDir, plotVerbosity, consoleVerbosity, params=None): if params is None: with open(paramsPath) as paramsFile: params = yaml.safe_load(paramsFile) # Setup params n = params["n"] w = params["w"] tmParams = params["tmParams"] tpParams = params["tpParams"] isOnline = params["isOnline"] onlineTrainingReps = params["onlineTrainingReps"] if isOnline else "N/A" completeSequenceLength = numElems**2 print( "Experiment parameters: " "(# worlds = {0}, # elements = {1}, n = {2}, w = {3}, " "online = {4}, onlineReps = {5})".format(numWorlds, numElems, n, w, isOnline, onlineTrainingReps)) print "Temporal memory parameters: {0}".format(tmParams) print "Temporal pooler parameters: {0}".format(tpParams) print # Setup experiment start = time.time() runner, exhaustiveAgents, randomAgents = setupExperiment( n, w, numElems, numWorlds, tmParams, tpParams) # Training phase # print "Training: (worlds: {0}, elements: {1})...".format(numWorlds, numElems) # print # if isOnline: # trainOnline(runner, exhaustiveAgents, completeSequenceLength, # onlineTrainingReps, consoleVerbosity) # else: # trainTwoPass(runner, exhaustiveAgents, completeSequenceLength, consoleVerbosity) # print "Done training." # print # Custom Cosyne training print "Training temporal memory and temporal pooler..." numSequences = 3 sequenceLength = 50 * (completeSequenceLength / numSequences) sequences = runner.generateSequences(sequenceLength, randomAgents, verbosity=consoleVerbosity, numSequences=numSequences) runner.feedLayers(sequences, tmLearn=True, tpLearn=True, verbosity=consoleVerbosity, showProgressInterval=SHOW_PROGRESS_INTERVAL) plotExperimentState(runner, plotVerbosity, numWorlds, numElems, isOnline, "Training") # Test TM and TP on randomly moving agents # runTestPhase(runner, randomAgents, numWorlds, numElems, # completeSequenceLength, consoleVerbosity) # plotExperimentState(runner, plotVerbosity, numWorlds, numElems, isOnline, "Testing") elapsed = int(time.time() - start) print "Total time: {0:2} seconds.".format(elapsed) # Write results to output file writeOutput(outputDir, runner, numElems, numWorlds, elapsed) if plotVerbosity >= 1: raw_input("Press any key to exit...")
pd.set_option('display.max_columns', 500) pd.set_option('display.width', 1000) import matplotlib matplotlib.use('Agg') import matplotlib.pyplot as plt graph_lag=graph_hp=graph_cpam='n' router_mode='3' topos_names=['rn001_001'] ######################################################################## # Globals ######################################################################## settings = yaml.safe_load(open('plugins/settings.yml')) ######################################################################## # Program ######################################################################## db = MySQLdb.connect(host = settings['mysql']['host'], port = settings['mysql']['port'], user = settings['mysql']['username'], passwd = settings['mysql']['password'], db = settings['mysql']['database'], ) posProg = 0 posTopo = 1 posMode = 2 posFormat = 3
def main(args=None): args = parser.parse_args(args) # read config file and default config with open('config/default.yaml') as f: default_config = utils.AttrDict(yaml.safe_load(f)) with open(args.config) as f: config = utils.AttrDict(yaml.safe_load(f)) if args.learning_rate is not None: args.reset_learning_rate = True # command-line parameters have higher precedence than config file for k, v in vars(args).items(): if v is not None: config[k] = v # set default values for parameters that are not defined for k, v in default_config.items(): config.setdefault(k, v) os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' # disable TensorFlow's debugging logs decoding_mode = any(arg is not None for arg in (args.decode, args.eval, args.align)) # enforce parameter constraints assert config.steps_per_eval % config.steps_per_checkpoint == 0, ( 'steps-per-eval should be a multiple of steps-per-checkpoint') assert decoding_mode or args.train, ( 'you need to specify at least one action (decode, eval, align, or train)') assert not (args.average and args.ensemble) if args.train and args.purge: utils.log('deleting previous model') shutil.rmtree(config.model_dir, ignore_errors=True) os.makedirs(config.model_dir, exist_ok=True) # copy config file to model directory config_path = os.path.join(config.model_dir, 'config.yaml') if args.train and not os.path.exists(config_path): with open(args.config) as config_file, open(config_path, 'w') as dest_file: content = config_file.read() content = re.sub(r'model_dir:.*?\n', 'model_dir: {}\n'.format(config.model_dir), content, flags=re.MULTILINE) dest_file.write(content) # also copy default config config_path = os.path.join(config.model_dir, 'default.yaml') if args.train and not os.path.exists(config_path): shutil.copy('config/default.yaml', config_path) # copy source code to model directory tar_path = os.path.join(config.model_dir, 'code.tar.gz') if args.train and not os.path.exists(tar_path): with tarfile.open(tar_path, "w:gz") as tar: for filename in os.listdir('translate'): if filename.endswith('.py'): tar.add(os.path.join('translate', filename), arcname=filename) logging_level = logging.DEBUG if args.verbose else logging.INFO # always log to stdout in decoding and eval modes (to avoid overwriting precious train logs) log_path = os.path.join(config.model_dir, config.log_file) logger = utils.create_logger(log_path if args.train else None) logger.setLevel(logging_level) utils.log('label: {}'.format(config.label)) utils.log('description:\n {}'.format('\n '.join(config.description.strip().split('\n')))) utils.log(' '.join(sys.argv)) # print command line try: # print git hash commit_hash = subprocess.check_output(['git', 'rev-parse', 'HEAD']).decode().strip() utils.log('commit hash {}'.format(commit_hash)) except: pass utils.log('tensorflow version: {}'.format(tf.__version__)) # log parameters utils.debug('program arguments') for k, v in sorted(config.items(), key=itemgetter(0)): utils.debug(' {:<20} {}'.format(k, pformat(v))) if isinstance(config.dev_prefix, str): config.dev_prefix = [config.dev_prefix] if config.tasks is not None: config.tasks = [utils.AttrDict(task) for task in config.tasks] tasks = config.tasks else: tasks = [config] for task in tasks: for parameter, value in config.items(): task.setdefault(parameter, value) task.encoders = [utils.AttrDict(encoder) for encoder in task.encoders] task.decoders = [utils.AttrDict(decoder) for decoder in task.decoders] for encoder_or_decoder in task.encoders + task.decoders: for parameter, value in task.items(): encoder_or_decoder.setdefault(parameter, value) device = None if config.no_gpu: device = '/cpu:0' device_id = None elif config.gpu_id is not None: device = '/gpu:{}'.format(config.gpu_id) device_id = config.gpu_id else: device_id = 0 # hide other GPUs so that TensorFlow won't use memory on them os.environ['CUDA_VISIBLE_DEVICES'] = '' if device_id is None else str(device_id) utils.log('creating model') utils.log('using device: {}'.format(device)) with tf.device(device): config.checkpoint_dir = os.path.join(config.model_dir, 'checkpoints') if config.weight_scale: if config.initializer == 'uniform': initializer = tf.random_uniform_initializer(minval=-config.weight_scale, maxval=config.weight_scale) else: initializer = tf.random_normal_initializer(stddev=config.weight_scale) else: initializer = None tf.get_variable_scope().set_initializer(initializer) # exempt from creating gradient ops config.decode_only = decoding_mode if config.tasks is not None: model = MultiTaskModel(**config) else: model = TranslationModel(**config) # count parameters utils.log('model parameters ({})'.format(len(tf.global_variables()))) parameter_count = 0 for var in tf.global_variables(): utils.log(' {} {}'.format(var.name, var.get_shape())) if not var.name.startswith('gradients'): # not counting parameters created by training algorithm (e.g. Adam) v = 1 for d in var.get_shape(): v *= d.value parameter_count += v utils.log('number of parameters: {:.2f}M'.format(parameter_count / 1e6)) tf_config = tf.ConfigProto(log_device_placement=False, allow_soft_placement=True) tf_config.gpu_options.allow_growth = config.allow_growth tf_config.gpu_options.per_process_gpu_memory_fraction = config.mem_fraction def average_checkpoints(main_sess, sessions): for var in tf.global_variables(): avg_value = sum(sess.run(var) for sess in sessions) / len(sessions) main_sess.run(var.assign(avg_value)) with tf.Session(config=tf_config) as sess: best_checkpoint = os.path.join(config.checkpoint_dir, 'best') if config.ensemble and len(config.checkpoints) > 1: model.initialize(config.checkpoints) elif config.average and len(config.checkpoints) > 1: model.initialize(reset=True) sessions = [tf.Session(config=tf_config) for _ in config.checkpoints] for sess_, checkpoint in zip(sessions, config.checkpoints): model.initialize(sess=sess_, checkpoints=[checkpoint]) average_checkpoints(sess, sessions) elif (not config.checkpoints and decoding_mode and (os.path.isfile(best_checkpoint + '.index') or os.path.isfile(best_checkpoint + '.index'))): # in decoding and evaluation mode, unless specified otherwise (by `checkpoints`), # try to load the best checkpoint model.initialize([best_checkpoint]) else: # loads last checkpoint, unless `reset` is true model.initialize(**config) if args.decode is not None: model.decode(**config) elif args.eval is not None: model.evaluate(on_dev=False, **config) elif args.align is not None: model.align(**config) elif args.train: try: model.train(**config) except KeyboardInterrupt: sys.exit()
if __name__ == "__main__": settings = Settings() if settings.input_debug: logging.basicConfig(level=logging.DEBUG) else: logging.basicConfig(level=logging.INFO) logging.debug(f"Using config: {settings.json()}") g = Github(settings.input_token.get_secret_value()) repo = g.get_repo(settings.github_repository) if not settings.github_event_path.is_file(): raise RuntimeError( f"No github event file available at: {settings.github_event_path}" ) contents = settings.github_event_path.read_text() github_event = PartialGitHubEvent.parse_raw(contents) translations_map: Dict[str, int] = yaml.safe_load(translations_path.read_text()) logging.debug(f"Using translations map: {translations_map}") sleep_time = random.random() * 10 # random number between 0 and 10 seconds pr = repo.get_pull(github_event.pull_request.number) logging.debug( f"Processing PR: {pr.number}, with anti-race condition sleep time: {sleep_time}" ) if pr.state == "open": logging.debug(f"PR is open: {pr.number}") label_strs = {label.name for label in pr.get_labels()} if lang_all_label in label_strs and awaiting_label in label_strs: logging.info( f"This PR seems to be a language translation and awaiting reviews: {pr.number}" ) if approved_label in label_strs: message = (
def load_config(self): """ Just load our config from the text passed in on init """ config = yaml.safe_load(self.config_text) or {} self.update_config(config)
def workflow_generator_simple( project, engine, n_traj, n_steps, round_n, longest = 5000, n_rounds = 1, modeller = None, sfkwargs = dict(), minlength = None, batchsize = 999999, batchwait = False, batchsleep = 5, progression = 'any', cpu_threads = 8, fixedlength = True, startontraj = 0, # TODO None and extra condition if set analysis_cfg = None, admd_profile = None, min_model_trajlength = 0, # TODO None and extra condition if set sampling_function_name = 'explore_macrostates', # these arent currently utilized randomly = False, mpi_ranks = 0, continuing = True, **kwargs,): '''Rename this function, block-style make simple workflow DAG but the function is complex. ''' logger.info("Starting workflow_generator_simple function") sampling_function = get_sampling_function( sampling_function_name, **sfkwargs, ) resource_requirements = dict(resource_name=project.configuration.name) # TODO calculate request qkwargs = dict(sleeptime=batchsleep, batchsize=batchsize, wait=batchwait) if progression == 'all': progress = lambda tasks: all([ta.is_done() for ta in tasks]) else: progress = lambda tasks: any([ta.is_done() for ta in tasks]) if n_rounds: assert isinstance(n_rounds, int) assert n_rounds > 0 logger.info("Going to do n_rounds: {}".format(n_rounds)) c = counter(n_rounds) tasks = list() # PREPARATION - Preprocess task setups logger.info("Using MD Engine: {0} {1}".format(engine, engine.name))#, project.generators[engine.name].__dict__) logger.info("Using fixed length? {}".format(fixedlength)) if minlength is None: minlength = n_steps logger.info("\nProject models\n - Number: {n_model}" .format(n_model=len(project.models))) logger.info("\nProject trajectories\n - Number: {n_traj}" .format(n_traj=len(project.trajectories))) logger.debug("\nProject trajectories\n - Lengths:\n{lengths}" .format(lengths=[t.length for t in project.trajectories])) # ROUND 1 - No pre-existing data if len(project.trajectories) == 0: notfirsttime = False logger.info("Defining first simulation tasks for new project") for traj in project.new_trajectory(engine['pdb_file'], n_steps, engine, n_traj): tasks.append(traj.run(**resource_requirements)) if admd_profile: # This should be path to an RC file tasks[-1].pre.insert(0, "source %s" % admd_profile) if not c.done: logger.info("Project first tasks queue") queue_tasks(project, tasks, **qkwargs) c.increment() logger.info("Project first tasks queued") logger.info("Queued First Tasks in new project") yield lambda: progress(tasks) logger.info("First Tasks are done") logger.info("Project first tasks done") else: notfirsttime = True mtask = None if modeller: if analysis_cfg: with open(analysis_cfg, 'r') as f: _margs = yaml.safe_load(f) #logger.info(pformat(_margs)) #update_margs = lambda rn: _margs[rn] update_margs = lambda rn: _margs[ max(list(filter( lambda mi: mi <= rn, _margs))) ] else: raise RuntimeError("Must specify an analysis cfg file to use modeller") # Start of CONTROL LOOP while not c.done: logger.info("Checking Extension Lengths") # TODO fix, this isn't a consistent name "trajectories" trajectories = set() # This loop will escape if all the trajectories # are / become full length priorext = -1 while priorext and not c.done: xtasks = list() #active_trajs = ~~~ after_n_trajs_trajs after_n_trajs_trajs = list(project.trajectories.sorted( lambda tj: tj.__time__))[startontraj:] logger.info( "Checking last {} trajectories for proper length".format( len(after_n_trajs_trajs)) ) xtasks = check_trajectory_minlength( project, minlength, after_n_trajs_trajs, n_steps, n_traj, resource_requirements=resource_requirements ) # NOTE we are tracking pre-existing extension tasks tnames = set() if len(trajectories) > 0: [tnames.add(_) for _ in set(zip(*trajectories)[0])] # NOTE so we only extend those who aren't already running queuethese = list() for xta in xtasks: tname = xta.trajectory.basename if tname not in tnames: tnames.add(tname) trajectories.add( (tname, xta) ) queuethese.append(xta) if queuethese: queue_tasks(project, queuethese, **qkwargs) yield lambda: progress(queuethese) # NOTE and remove any that have already completed removals = list() for tname, xta in trajectories: if xta.state in {"fail","halted","success","cancelled"}: removals.append( (tname, xta) ) for removal in removals: trajectories.remove(removal) if len(trajectories) == n_traj and priorext < n_traj: logger.info("Have full width of extensions") c.increment() # setting this to look at next round priorext = len(trajectories) logger.info("----------- On workload #{0}".format(c.n)) logger.info("Runtime main loop enter") tasks = list() if not modeller: logger.info("Extending project without modeller") trajectories = sampling_function(project, engine, n_steps, n_traj) logger.info("Runtime new trajectories defined") logger.info("Converting trajectories to tasks") [tasks.append(t.run(**resource_requirements)) for t in trajectories] logger.info("Runtime new tasks queueing") if tasks: queue_tasks(project, tasks, **qkwargs) logger.info("Runtime new tasks queued") c.increment() yield lambda: progress(tasks) logger.info("Runtime main no modeller done") else: if mtask is None: margs = update_margs(round_n) margs.update(resource_requirements) logger.info("Extending project with modeller") logger.info("Analysis args for this round will be: {}".format(pformat(margs))) trajectories = list(filter(lambda tj: tj.length >= min_model_trajlength, project.trajectories)) mtask = modeller.execute(trajectories, **margs) project.queue(mtask) yield lambda: progress(tasks) elif mtask.is_done(): # In current 1-workload per runtime model, shouldn't ever see this condition logger.info("Current modelling task is done") mtask = None else: # In current 1-workload per runtime model, shouldn't ever see this condition logger.info("Waiting on modelling task") yield lambda: progress(tasks)
from flask_cors import CORS # import strptime to avoid weird issues described @ http://bugs.python.org/msg221094 datetime.strptime('', '') # Create WSGI app instance - this __name__.split('.') handles # the case where the file is part of a package. app = Flask(__name__.split('.')[0]) # to allow cross domain resource sharing over all domains, # for more specific usage see @ https://github.com/corydolphin/flask-cors CORS(app) # load the configuration file try: app_config = yaml.safe_load(open('etc/config.yml')) except yaml.YAMLError as e: # exceptions can be specific to the possible errors app.logger.error('exception encountered while parsing the configuration file, check the logs') app.logger.exception(e) sys.exit(1) # when the file is loaded properly, load & set database configs for sqlalchemy database_host = app_config['database']['host'] database_port = app_config['database']['port'] database_user = app_config['database']['user_name'] database_password = app_config['database']['password'] database_name = app_config['database']['database_name'] # setup database uri # i used postgres as example database however any database can be used which is supported be sqlalchemy app.config['SQLALCHEMY_DATABASE_URI'] = 'postgres://%s:%s@%s:%s/%s' % \
def _load_config_file(cls, agent_name) -> Dict: """Load a config file.""" agent_config_file = Path(agent_name, DEFAULT_AEA_CONFIG_FILE) # type: ignore file_pointer = open(agent_config_file, mode="r", encoding="utf-8") agent_config_instance = yaml.safe_load(file_pointer) return agent_config_instance
def read_diffuse_component_yaml(yamlfile): """ Read the yaml file for the diffuse components """ diffuse_components = yaml.safe_load(open(yamlfile)) return diffuse_components
def _test_internal_route_reflector(self, backend='bird', bgpconfig_as_num=64514, peer_as_num=64514): """ Run a multi-host test using an internal route reflector. """ with DockerHost('host1', additional_docker_options=CLUSTER_STORE_DOCKER_OPTIONS, start_calico=False) as host1, \ DockerHost('host2', additional_docker_options=CLUSTER_STORE_DOCKER_OPTIONS, start_calico=False) as host2, \ DockerHost('host3', additional_docker_options=CLUSTER_STORE_DOCKER_OPTIONS, start_calico=False) as host3: # Start all hosts using specific backends. host1.start_calico_node("--backend=%s" % backend) host2.start_calico_node("--backend=%s" % backend) host3.start_calico_node("--backend=%s" % backend) # Set the default AS number - as this is used by the RR mesh, and # turn off the node-to-node mesh (do this from any host). update_bgp_config(host1, nodeMesh=False, asNum=bgpconfig_as_num) # Create a workload on each host in the same network. network1 = host1.create_network("subnet1") workload_host1 = host1.create_workload("workload1", network=network1) workload_host2 = host2.create_workload("workload2", network=network1) workload_host3 = host3.create_workload("workload3", network=network1) # Allow network to converge (which it won't) self.assert_false(workload_host1.check_can_ping(workload_host2.ip, retries=5)) # Make host2 act as a route reflector. node2 = host2.calicoctl("get Node %s -o yaml" % host2.get_hostname()) node2cfg = yaml.safe_load(node2) logger.info("host2 Node: %s", node2cfg) node2cfg['spec']['bgp']['routeReflectorClusterID'] = '224.0.0.2' node2cfg['metadata']['labels'] = { 'routeReflectorClusterID': node2cfg['spec']['bgp']['routeReflectorClusterID'], } host2.add_resource(node2cfg) # Configure peerings - note, NOT a full mesh - from the # other nodes to the route reflector. host2.add_resource({ 'apiVersion': 'projectcalico.org/v3', 'kind': 'BGPPeer', 'metadata': { 'name': 'rr-peerings', }, 'spec': { 'nodeSelector': '!has(routeReflectorClusterID)', 'peerSelector': 'has(routeReflectorClusterID)', }, }) # Allow network to converge (which it now will). self.assert_true(workload_host1.check_can_ping(workload_host2.ip, retries=20)) # And check connectivity in both directions. self.assert_ip_connectivity(workload_list=[workload_host1, workload_host2, workload_host3], ip_pass_list=[workload_host1.ip, workload_host2.ip, workload_host3.ip], retries=5)
import logging from nuimo_menue.config import NuimoMenueConfiguration import yaml class NuimoOpenHabConfiguration(NuimoMenueConfiguration): def __init__(self, config): super(NuimoOpenHabConfiguration, self).__init__(config["key_mapping"]) self.config = config def __getitem__(self, item): return self.config[item] project_root_dir = os.path.dirname(sys.argv[0]) NUIMO_OPENHAB_CONFIG_PATH = os.getenv('NUIMO_OPENHAB_CONFIG_PATH', os.path.join(project_root_dir, 'config.yml')) with open(NUIMO_OPENHAB_CONFIG_PATH, 'r') as ymlfile: rawConfig = yaml.safe_load(ymlfile) loggers = [] if rawConfig["log_file"]: loggers.append(logging.FileHandler(filename=rawConfig["log_file"])) if rawConfig["log_stdout"]: loggers.append(logging.StreamHandler(stream=sys.stdout)) formatter = logging.Formatter('%(asctime)s.%(msecs)03d %(levelname)s %(module)s - %(funcName)s: %(message)s', "%Y-%m-%d %H:%M:%S") for logger in loggers: logger.setFormatter(formatter) if loggers: logging.basicConfig(handlers=loggers, level=rawConfig["log_level"]) logging.info("Loaded config file from %s", NUIMO_OPENHAB_CONFIG_PATH) sys.modules[__name__] = NuimoOpenHabConfiguration(rawConfig)
def __init__(self, *args, **kwargs): """Set up all the resources.""" super().__init__(*args, **kwargs) # Load the resources from the yaml files in /resources/ self.resources = { path.stem: yaml.safe_load(path.read_text()) for path in RESOURCES_PATH.rglob("*.yaml") } # Sort the resources alphabetically self.resources = dict( sorted(self.resources.items(), key=self._sort_key_disregard_the)) # Parse out all current tags resource_tags = { "topics": set(), "payment_tiers": set(), "difficulty": set(), "type": set(), } for resource_name, resource in self.resources.items(): css_classes = [] for tag_type in resource_tags.keys(): # Store the tags into `resource_tags` tags = resource.get("tags", {}).get(tag_type, []) for tag in tags: tag = tag.title() tag = tag.replace("And", "and") resource_tags[tag_type].add(tag) # Make a CSS class friendly representation too, while we're already iterating. for tag in tags: css_tag = to_kebabcase(f"{tag_type}-{tag}") css_classes.append(css_tag) # Now add the css classes back to the resource, so we can use them in the template. self.resources[resource_name]["css_classes"] = " ".join( css_classes) # Set up all the filter checkbox metadata self.filters = { "Difficulty": { "filters": sorted(resource_tags.get("difficulty")), "icon": "fas fa-brain", "hidden": False, }, "Type": { "filters": sorted(resource_tags.get("type")), "icon": "fas fa-photo-video", "hidden": False, }, "Payment tiers": { "filters": sorted(resource_tags.get("payment_tiers")), "icon": "fas fa-dollar-sign", "hidden": True, }, "Topics": { "filters": sorted(resource_tags.get("topics")), "icon": "fas fa-lightbulb", "hidden": True, } } # The bottom topic should always be "Other". self.filters["Topics"]["filters"].remove("Other") self.filters["Topics"]["filters"].append("Other") # A complete list of valid filter names self.valid_filters = { "topics": [ to_kebabcase(topic) for topic in self.filters["Topics"]["filters"] ], "payment_tiers": [ to_kebabcase(tier) for tier in self.filters["Payment tiers"]["filters"] ], "type": [to_kebabcase(type_) for type_ in self.filters["Type"]["filters"]], "difficulty": [ to_kebabcase(tier) for tier in self.filters["Difficulty"]["filters"] ], }
from pprint import pprint import argo_workflows from argo_workflows.api import workflow_service_api from argo_workflows.model.io_argoproj_workflow_v1alpha1_workflow_create_request import \ IoArgoprojWorkflowV1alpha1WorkflowCreateRequest import requests import yaml configuration = argo_workflows.Configuration(host="https://127.0.0.1:2746") configuration.verify_ssl = False resp = requests.get('https://raw.githubusercontent.com/argoproj/argo-workflows/master/examples/hello-world.yaml') manifest = yaml.safe_load(resp.text) api_client = argo_workflows.ApiClient(configuration) api_instance = workflow_service_api.WorkflowServiceApi(api_client) api_response = api_instance.create_workflow( namespace='argo', body=IoArgoprojWorkflowV1alpha1WorkflowCreateRequest(workflow=manifest, _check_type=False), _check_return_type=False) pprint(api_response)