def apply_default_config(config): if config.get("cloudwatchlogs"): cloudwatchlogs_default_config = { 'region': 'eu-central-1', 'log_level': 'INFO', 'groupname': 'monocyte_logs' } yamlreader.data_merge(cloudwatchlogs_default_config, config['cloudwatchlogs']) log_level_map = { 'DEBUG': logging.DEBUG, 'INFO': logging.INFO, 'WARN': logging.WARN, 'ERROR': logging.ERROR } cloudwatchlogs_default_config['log_level'] = log_level_map[ cloudwatchlogs_default_config['log_level'].upper()] config['cloudwatchlogs'] = cloudwatchlogs_default_config default_config = { "handler_names": [ "cloudformation.Stack", "ec2.Instance", "ec2.Volume", "rds2.Instance", "rds2.Snapshot", "dynamodb.Table", "s3.Bucket", "acm.Certificate" ], "ignored_resources": { "cloudformation": ["cloudtrail-logging"] }, "ignored_regions": ["cn-north-1", "us-gov-west-1"], "allowed_regions_prefixes": ["eu"] } for key in default_config.keys(): config[key] = config.get(key, default_config[key])
def load_config(global_config_dir=CFGDIR): global_config = {} if os.path.isdir(global_config_dir): global_config = yamlreader.yaml_load(global_config_dir, {}) user_config = {} user_config_dir = os.path.expanduser("~/.afp-cli") if os.path.isdir(user_config_dir): user_config = yamlreader.yaml_load(user_config_dir, {}) yamlreader.data_merge(global_config, user_config) return global_config
def load_config(global_config_dir=CFGDIR): global_config = {} if os.path.isdir(global_config_dir): global_config = yamlreader.yaml_load(global_config_dir, {}) user_config = {} user_config_dir = os.path.expanduser("~/.afp-cli") if os.path.isdir(user_config_dir): global_config = yamlreader.yaml_load(user_config_dir, {}) yamlreader.data_merge(global_config, user_config) return global_config
def main(arguments): cli_config = convert_arguments_to_config(arguments) config_path = get_config_path_from_args(arguments) whitelist_uri = get_whitelist_from_args(arguments) config = yamlreader.data_merge(read_config(config_path), cli_config) config = yamlreader.data_merge(config, load_whitelist(whitelist_uri)) apply_default_config(config) monocyte = Monocyte(**config) try: return monocyte.search_and_destroy_unwanted_resources() except Exception: monocyte.logger.exception("Error while running monocyte:") return 1
def parse_unknown_args(unknown): assert len( unknown ) % 2 == 0, "some custom config keys from arguments have no value assigned" import itertools custom_parameters = dict( itertools.zip_longest(*[iter(unknown)] * 2, fillvalue="")) formatted_custom_parameters = {} for key, value in custom_parameters.items(): new_dict = safe_load(value) while key: new_dict, key = get_dict(new_dict, key) formatted_custom_parameters = yamlreader.data_merge( formatted_custom_parameters, new_dict) return formatted_custom_parameters
def __init__(self, user, config, account_config, logger=None): default_config = { 'aws': { 'access_key': None, 'secret_key': None }, 'provider': { 'class': 'Provider' } } self.logger = logger or logging.getLogger(__name__) self.user = user self.application_config = data_merge(default_config, config) self.account_config = account_config self.provider = None self._setup_provider()
def merge_yamls(configs): data = {} for config in configs: path = config.get('path') with open(path, 'r') as f: text = f.read() text = interpolate_vars(text, config.get('module')) new_data = yaml.load(text, Loader=yaml.FullLoader) if new_data is not None: data = data_merge(data, new_data) path = os.path.join(RES_DIR, 'config.yaml') with open(path, 'w') as f: f.write( safe_dump(data, indent=2, default_flow_style=False, canonical=False)) CLEANUP_FILES_QUEUE.append(path)
def __main(): config_data = vars(__parse_args()) config_data = {key: config_data[key] for key in config_data if config_data[key] is not None} if 'config' in config_data: try: config_data = data_merge(yaml_load(config_data['config']), config_data) except YamlReaderError as exc: raise("Could not read configfile: %s" % exc) if len(config_data['input']) == 0: raise("No input direcotires configured.") if 'gendersfile' not in config_data: config_data['gendersfile'] = '/etc/genders' genders_generator = GenerateGenders( config_data.get('input'), config_data.get('gendersfile'), config_data.get('domain', {}), config_data.get('verbosity') ) genders_generator.generate_genders_file()
def _gen(): import io from os import path from yaml import safe_load from yamlreader import data_merge, yaml_load here = path.abspath(path.dirname(__file__)) files = [path.join(here, "swagger", p) for p in ["swagger.yml", "db.yml"] ] # , 'proxy.yml', 'storage.yml']] base = yaml_load(files) base["paths"] = OrderedDict( sorted(base["paths"].items(), key=lambda t: t[0])) with io.open(path.join(here, "swagger", "view.tmpl.yml"), "r", encoding="utf-8") as f: template = Template(str(f.read())) tags = base["tags"] def to_type(t): if t is None: return "string" if t is int: return "integer" if t is float: return "number" # integrate all views using the template for database, connector in manager.db.connectors.items(): db.resolve(database) # trigger filling up columns # add database tag tags.append( dict(name="db_" + database, description=connector.description or "")) for view, dbview in connector.views.items(): if not dbview.can_access() or dbview.query_type == "private": continue # if database != u'dummy' or view != u'b_items_verify': # continue for tag in dbview.tags: if tag not in tags: tags.append(tag) args = [] for arg in dbview.arguments: info = dbview.get_argument_info(arg) args.append( dict( name=arg, type=to_type(info.type), as_list=info.as_list, enum_values=None, description=info.description, example=info.example, )) for arg in (a for a in dbview.replacements if a not in secure_replacements): extra = dbview.valid_replacements.get(arg) arg_type = "string" enum_values = None if isinstance(extra, list): enum_values = extra if extra == int or extra == float: arg_type = to_type(extra) args.append( dict( name=arg, type=arg_type, as_list=False, enum=enum_values, description="", )) filters = set() if "where" in dbview.replacements or "and_where" in dbview.replacements: # filter possible for k in dbview.filters.keys(): filters.add(k) if not filters: for k in list(dbview.columns.keys()): filters.add(k) if "agg_score" in dbview.replacements: # score query magic handling agg_score = connector.agg_score args.append( dict( name="agg", type="string", as_list=False, enum=agg_score.valid_replacements.get("agg"), )) props = [] for k, prop in dbview.columns.items(): p = prop.copy() p["name"] = k if "type" not in p or p["type"] == "categorical": p["type"] = "string" props.append(p) if dbview.idtype: # assume when id type given then we have ids props.append(dict(name="_id", type="integer")) if not any((p["name"] == "id" for p in props)): props.append(dict(name="id", type="string")) features = { "generic": dbview.query_type in ["generic", "helper", "table"], "desc": dbview.query_type in ["table"], "lookup": dbview.query_type in ["lookup"], "score": dbview.query_type in ["score"], } keys = { "database": database, "view": view, "type": dbview.query_type, "description": dbview.description or "", "summary": dbview.summary or "", "args": args, "empty": not args and not filters, "filters": filters, "features": features, "tags": dbview.tags or [], "props": props, "propsempty": not props, } view_yaml = template.render(**keys) # _log.info(view_yaml) part = safe_load(view_yaml) base = data_merge(base, part) # post process using extensions for p in manager.registry.list("tdp-swagger-postprocessor"): base = p.load().factory(base) return base
def test_merge_dict_to_list(self): self.assertEqual(data_merge([1, 2], {1:2}), [1, 2, {1: 2}])
def test_merge_complex_list_to_list(self): self.assertEqual(data_merge([1, 2], [{1:2}, 1]), [1, 2, {1: 2}, 1])
def test_merge_deep_dict_to_deep_dict(self): self.assertEqual(data_merge({1:2,3:{1:2}},{3:{2:5}}),{1: 2, 3: {1: 2, 2: 5}})
def test_merge_dict_to_dict(self): self.assertEqual(data_merge({1:2,3:[1,2,3]},{3:6}),{1:2,3:[1,2,3,6]})
def main(): parser = argparse.ArgumentParser( description='Policy gradient algorithms with demonstration data.') parser.add_argument('--configs', type=str, nargs='+', help='path to additional config parameters') parser.add_argument('--continue_task', type=str, default=None, help='task name to continue training') args, unknown = parser.parse_known_args() custom_parameters = parse_unknown_args(unknown) if args.continue_task is not None: config_path = os.path.join(args.continue_task, "config.yaml") if not os.path.exists(config_path): config_path = os.path.join('Runs', config_path) cfg = yamlreader.yaml_load(config_path) else: configs = None if args.configs is not None: configs = [] for config in args.configs: if '.yaml' not in config: config += '.yaml' configs.append('../configs/config/' + config) cfg = yamlreader.yaml_load(configs, default_path='../configs/default.yaml') cfg = yamlreader.data_merge(cfg, custom_parameters) # parse multiple runs multiple_runs = cfg['runs'] != 'single_run' runs_from = cfg['runs'][0] if multiple_runs else 1 runs_to = cfg['runs'][1] if multiple_runs else 2 original_seed = cfg['seed'] for run_no in range(runs_from, runs_to): run_seed = original_seed + run_no - 1 if cfg['algorithm'] == "IRL+DAPG": # start IRL print("\n----- Starting IRL for DAPG -----\n") cfg['algorithm'] = 'IRL' if cfg['based_IRL']['IRL_config'] is None: IRL_cfg = yamlreader.yaml_load('../configs/config/IRL.yaml') else: config_path = os.path.join('../configs/config', cfg['based_IRL']['IRL_config']) if '.yaml' not in config_path: config_path += '.yaml' IRL_cfg = yamlreader.yaml_load(config_path) IRL_cfg = yamlreader.data_merge(cfg, IRL_cfg) IRL_cfg = yamlreader.data_merge(IRL_cfg, custom_parameters) irl_task = parse_task(IRL_cfg)[1] train(IRL_cfg, run_no=run_no, multiple_runs=multiple_runs, seed=run_seed) # start DAPG based on IRL job print("\n----- Starting DAPG based on finished IRL job -----\n") cfg['algorithm'] = 'DAPG_based_IRL' cfg['based_IRL']['IRL_job'] = irl_task cfg['based_IRL']['IRL_run_no'] = run_no if multiple_runs else None cfg['use_DAPG'] = True train(cfg, run_no=run_no, multiple_runs=multiple_runs, seed=run_seed) # make run with initialisation if dump_paths_percentage is set start_from_initialisation = IRL_cfg['IRL'][ 'dump_paths_percentage'] is not None if start_from_initialisation: cfg['based_IRL']['get_paths_for_initialisation'] = True train(cfg, run_no=run_no, multiple_runs=multiple_runs, seed=run_seed) # reset variables cfg['algorithm'] = "IRL+DAPG" cfg['based_IRL']['IRL_job'] = None cfg['based_IRL']['IRL_run_no'] = None if start_from_initialisation: del cfg['based_IRL']['get_paths_for_initialisation'] else: train(cfg, run_no=run_no, multiple_runs=multiple_runs, seed=run_seed)
def test_merge_tuple_to_list(self): self.assertEqual(data_merge([1, 2], ({1:2}, 3)), [1, 2, ({1: 2}, 3)])
def test_merge_string_to_list(self): self.assertEqual(data_merge([1, 2], "hello"), [1, 2, "hello"])
preproc.debug = args.debug meta = {} body = "" for source_file in source_files: raw = "" # Read in md file as str for line in fileinput.input(source_file): raw += line # Split into head (yaml) and body (md) this_head, this_body = get_yaml_data(raw) # Load yaml str into metadata dict this_meta = yaml.safe_load(this_head) # Merge this metadata dict with others (function imported from pypi package yamlreader) meta = data_merge(meta, this_meta) # Merge this body str with others body = body + this_body preproc.init_vars_metadata(meta) preproc.init_vars_args(args) new_body = preproc.process(body) delim = "---" + os.linesep # Create new file. The metadata here is ugly but usable new_file = delim + yaml.safe_dump(meta) + delim + new_body if out_file: with open(out_file, "w") as f: f.write(new_file)