def wmts_dict_merge(target, *args): # Merge multiple dicts # Recursively merge dicts and set non-dict values obj = args[0]['layers'] conf = args[1]['layers'] if not isinstance(obj, dict): return obj for k, v in obj.items(): if k in conf and isinstance(conf[k], dict): foundSourceMisMatch = False if 'projections' in v and 'projections' in conf[k]: conf_projections = conf[k]['projections'] for projectionKey, projection in v['projections'].items(): source = projection['source'] if projectionKey in conf_projections: if 'source' in conf_projections[projectionKey]: if source != conf_projections[projectionKey][ 'source']: foundSourceMisMatch = True continue if foundSourceMisMatch: continue if k in target and isinstance(target[k], dict): dict_merge(target[k], v, conf[k]) else: target[k] = dict_merge(v, conf[k]) return target
def updateconf(self): log.info("updating config") self.collect_enabled_features() self.process_rules() self.kconf_enable = self.config.get_kconf_enable() self.kconf_module = self.config.get_kconf_module() self.kconf_setting = self.config.get_kconf_setting() self.allsym = self.config.get_kconf_all() # process enabled feature list for fname, fobj in self.enabled_features.iteritems(): self.kconf_enable |= fobj.get_kconf_enable() self.kconf_module |= fobj.get_kconf_module() dict_merge(self.kconf_setting, fobj.get_kconf_setting()) for x in self.kconf_enable.intersection(self.kconf_module): log.info("module forced builtin: %s: " % x) self.kconf_module.remove(x) self.kconf_unset = (self.allsym - self.kconf_enable - self.kconf_module - dict_keys(self.kconf_setting)) self.processed = True
def set_metadata(self, metadata): metadata = util.build_recursive(metadata) if 'Metadata' in metadata: metadata = metadata['Metadata'] self['Metadata'] = util.dict_merge(self.get('Metadata', {}), metadata) self.dirty_children()
def get_dataloader(config, device): """ Get train / valid dataloader. """ # read data ratings_df = pd.read_csv(config['ratings']) # sizes user_size = int(ratings_df.userId.max() + 1) item_size = int(ratings_df.itemId.max() + 1) # split (users, items, ratings) = (torch.from_numpy(ratings_df.userId.values).long().to(device), torch.from_numpy(ratings_df.itemId.values).long().to(device), torch.from_numpy(ratings_df.rating.values).float().to(device)) dataset = TensorDataset(users, items, ratings) valid_size = int(len(dataset) * config['args']['valid']) (trainset, validset) = random_split(dataset, [len(dataset) - valid_size, valid_size]) batch_size = config['args']['batch_size'] (trainloader, validloader) = (DataLoader(trainset, batch_size=batch_size), DataLoader(validset, batch_size=batch_size)) # update config config['args'] = dict_merge(config['args'], { 'user_size': user_size, 'item_size': item_size, }) return trainloader, validloader
def run(): # pylint: disable=too-many-locals """ Run! """ args = parse_args() # load config default_config = yaml.safe_load(open(args.default_config)) config = yaml.safe_load(open(args.config)) print('config loaded:', config) config = dict_merge(default_config, config) # init init_seed(config['args']['seed']) device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') # load data preprocess_src = 'preprocess.{}'.format(Path(config['preprocess']).stem) preprocess_module = importlib.import_module(preprocess_src).get_dataloader (trainloader, validloader) = preprocess_module(config, device) # init trainer print('config:', config) trainer_src = 'trainer.{}'.format(Path(config['trainer']).stem) trainer_class = importlib.import_module(trainer_src).Trainer trainer = trainer_class(config, device) # train! model_dir = Path(config['model_dir']) if not model_dir.is_dir(): model_dir.mkdir() trainer.train(trainloader, validloader, config['args']['epoch_size'], model_dir)
def process_requests(wv_product_dict): futures = [] with ThreadPoolExecutor() as executor: for wv_id, val in wv_product_dict.items(): c_id = val['conceptId'] futures.append(executor.submit(get_cmr_data, wv_id, c_id)) futures.append(executor.submit(get_cmr_umm_data, wv_id, c_id)) for f in futures: try: f.result() except Exception as e: print("%s: ERROR: %s" % (prog, e)) collection_data = dict_merge(cmr_data, cmr_umm_data) no_data_count = 0 for value in collection_data.values(): if len(value.keys()) < 1: # TODO eventually should probably delete empty dict entries. # For now, may be useful to see which products are not returning metadata no_data_count += 1 success_count = len(collection_data) - no_data_count print("%s: %s collections returned metadata." % (prog, success_count)) print("%s: %s collections did not reutrn any metadata." % (prog, no_data_count)) return collection_data
def get_cmr_data(concept_id_dict): concept_id = concept_id_dict["value"] if cmr_data.get(concept_id) is None: response = requests.get(cmr_collection_url + concept_id) entry = response.json().get('feed', {}).get('entry') if len(entry) == 1: cmr_data[concept_id] = process_entries(entry[0], cmr_keys_map) elif len(entry) > 1: print("%s: WARNING: multiple entries found for %s ", (prog, concept_id)) concept_id_dict = dict_merge(concept_id_dict, cmr_data[concept_id])
async def get_summoner_names_by_ids(self, summoner_ids): """Get summoner names by their ids Keyword arguments: summoner_ids -- list of summoner ids to query """ results = [] for subset in util.grouper(summoner_ids, 40): url = self.base_summoner_url + ','.join(str(summoner_id) for summoner_id in subset if summoner_id) + '/name' results.append(await self.get(url)) return util.dict_merge(results)
async def get_summoners_info_by_names(self, summoner_names): """Get info about summoners by summoner names Keyword arguments: summoner_names -- list of summoner names to query """ results = [] for subset in util.grouper(summoner_names, 40): url = self.base_summoner_url + 'by-name/' + ','.join(name for name in subset if name) results.append(await self.get(url)) return util.dict_merge(results)
def get_summoners_info_by_names(self, summoner_names): """Get info about summoners by summoner names Keyword arguments: summoner_names -- list of summoner names to query """ results = [] for subset in util.grouper(summoner_names, 40): url = self.base_url + "by-name/" + ",".join(name for name in subset if name) results.append(LeagueRequest.get(url)) return util.dict_merge(results)
def get_summoner_names_by_ids(self, summoner_ids): """Get summoner names by their ids Keyword arguments: summoner_ids -- list of summoner ids to query """ results = [] for subset in util.grouper(summoner_ids, 40): url = self.base_url + ",".join(str(summoner_id) for summoner_id in subset if summoner_id) + "/name" results.append(LeagueRequest.get(url)) return util.dict_merge(results)
def load(file, sections=[], **instargs): """Create a sMAP instance based on the representation stored in a file. The configuration file contains sections which refer to either reporting instances, or paths in the sMAP heirarchy. Any section whose name starts with ``/`` is treated as a resource name; sections starting with ``report`` are treated as reports. The file must contain at least one section named ``/``, which must contain a ``uuid`` key to set the root identifier for the source. :param string file: filename of the configuration file :param instargs: arguments passed to the :py:class:`~smap.core.SmapInstance` constructor. :return smap.core.SmapInstance: the created instancev :raise smap.loader.SmapLoadError: an error is encountered processing the file :raise smap.core.SmapError: some other error is encountered validating the loaded object """ found = None for l in ['', os.getcwd(), sys.prefix]: path = os.path.join(l, file) if os.path.isfile(path): found = path if not found: raise Exception("Config file %s not found." % file) print "Loading config file:", found conf = configobj.ConfigObj(found, indent_type=' ') # if there's a server section, override the default server # configuration with that if 'server' in conf: smapconf.SERVER = util.dict_merge(smapconf.SERVER, dict(((k.lower(), v) for (k, v) in conf['server'].iteritems()))) if 'logging' in conf: smapconf.LOGGING = util.dict_merge(smapconf.LOGGING, dict(((k.lower(), v) for (k, v) in conf['logging'].iteritems()))) # we need the root to have a uuid inst = core.SmapInstance(conf['/']['uuid'], **instargs) inst.loading = True reports = [] for s in conf: print "Loading section", s if s.startswith('report'): resource = conf[s].get('ReportResource', '/+') format = conf[s].get('Format', 'json') max_age = conf[s].get('MaxAge', None) max_age = int(max_age) if max_age != None else None dest = [conf[s]['ReportDeliveryLocation']] for i in xrange(0, 10): if 'ReportDeliveryLocation%i' % i in conf[s]: dest.append(conf[s]['ReportDeliveryLocation%i' % i]) reportinst = { 'ReportDeliveryLocation' : dest, 'ReportResource' : resource, 'Format': format, 'uuid' : inst.uuid(s), 'MaxAge' : max_age, } for o in ['MinPeriod', 'MaxPeriod']: if o in conf[s]: reportinst[o] = conf[s][o] for o in ['ClientCertificateFile', 'ClientPrivateKeyFile', 'CAFile']: if o in conf[s]: reportinst[i] = os.path.expanduser(conf[s][o]) reports.append(reportinst) continue elif not s.startswith('/'): # path sections must start with a '/' # other sections might be present and could be parsed by # other parts of the program print "Warning: skipping section", s, "since it does not begin with a '/'" continue elif len(sections) and not util.norm_path(s) in sections: # skip all but the listed sections if we were asked to continue s = util.norm_path(s) # build the UUID for the item props = util.build_recursive(dict(conf[s].items())) id = None if 'uuid' in conf[s]: key = None id = uuid.UUID(conf[s]['uuid']) elif 'key' in conf[s]: key = conf[s]['key'] else: # default to the path if key = s if key: id = inst.uuid(key) # raise SmapLoadError("Every config file section must have a uuid or a key!") # create the timeseries or collection if (s == '/' or conf[s].get("type", None) == 'Collection' or inst.get_collection(s) != None): if s == '/': c = inst.get_collection('/') elif inst.get_collection(s) != None: # sometimes you will have collections created twice, # for instance if a driver creates it and then we want # to tag it with metadata c = inst.get_collection(s) else: c = core.Collection(s, inst) inst.add_collection(s, c) elif conf[s].get("type", "Timeseries") == "Timeseries": if inst.get_timeseries(s) != None: c = inst.get_timeseries(s) else: try: props['Properties']['UnitofMeasure'] except KeyError: raise SmapLoadError("A Timeseries must have at least " "the Properites/UnitofMeasure key") # the Timeseries uses defaults if the conf file doesn't # contain the right sections. c = core.Timeseries(id, props['Properties']['UnitofMeasure'], data_type=props['Properties'].get('ReadingType', core.Timeseries.DEFAULTS['Properties/ReadingType']), timezone=props['Properties'].get('Timezone', core.Timeseries.DEFAULTS['Properties/Timezone']), buffersz=int(props.get('BufferSize', core.Timeseries.DEFAULTS['BufferSize']))) inst.add_timeseries(s, c) else: if not id: raise SmapLoadError("A driver must have a key or uuid to generate a namespace") # load a new driver manager layer newdrv = driver.SmapDriver.get_driver(inst, conf[s]['type'], s, id) # create a collection and add it at the attachment point c = inst.get_collection(s) if not c: c = core.Collection(s, inst) inst.add_collection(s, c) # Add config file specified checkers for the driver check = checkers.get(inst, newdrv, conf[s]) if check: inst.checkers.append(check) # get the driver to add its points newdrv.setup(conf[s]) # Metadata and Description are shared between both Collections # and Timeseries if props.has_key('Metadata'): # the driver may have added metadata; however config file # metadata overrides it c['Metadata'] = util.dict_merge(c.get('Metadata', {}), props['Metadata']) if props.has_key('Description'): c['Description'] = props['Description'] if key: setattr(c, 'key', key) # since the sections could come in any order, update the reporting # instance to make sure all the topics are set right. for reportinst in reports: if not inst.reports.update_report(reportinst): inst.reports.add_report(reportinst) inst.reports.update_subscriptions() inst.loading = False return inst
new_conf["layers"] = {} new_conf["sources"] = {} with open(output_file) as fp: output_data = json.load(fp) file_count = 0 for file in os.listdir(input_dir): try: if not file.endswith(".json"): continue file_count += 1 with open(os.path.join(input_dir, file)) as fp: data = json.load(fp) new_conf['layers'] = wmts_dict_merge(new_conf['layers'], data, output_data) new_conf["sources"] = dict_merge(new_conf["sources"], data['sources'], output_data['sources']) except Exception as e: sys.stderr.write("ERROR: %s: %s\n" % (os.path.join(input_dir, file), str(e))) sys.exit(1) new_conf = dict_merge(new_conf, output_data) json_options = {} json_options["indent"] = 2 json_options["separators"] = (',', ': ') with open(output_file, "w") as fp: json.dump(new_conf, fp, **json_options) print("%s: %s file(s) merged into %s" % (prog, file_count, os.path.basename(output_file)))
def get_recursive_dict(self, name): lst = dict_from(self.cf, name) for i in self.imports: dict_merge(lst, self.imports[i].get_recursive_dict(name)) return lst
experiments = {} custom_config = { "callbacks": { "on_episode_end": tune.function(on_episode_end) } } if args.dgx: custom_config["num_workers"] = os.cpu_count() - 4 custom_config["num_gpus"] = 16 custom_config["num_envs_per_worker"] = 1 if args.ppo: config = ppo.DEFAULT_CONFIG.copy() dict_merge(config, custom_config) dict_merge(config, alg_config.ppo["v1"]) experiments["ppo"] = { "run": "PPO", "env": DeepLogisticsA10M20x20D4, "stop": { "episode_reward_mean": 500 }, "config": config } if args.a3c: config = a3c.DEFAULT_CONFIG.copy() dict_merge(config, custom_config) if args.dgx:
input_dir = args[0] output_file = args[1] conf = {} file_count = 0 for root, dirs, files in os.walk(input_dir): dirs.sort() for file in sorted(files): try: if not file.endswith(".json"): continue file_count += 1 with open(os.path.join(root, file)) as fp: data = json.load(fp) dict_merge(conf, data) except Exception as e: sys.stderr.write("ERROR: %s: %s\n" % (os.path.join(root, file), str(e))) sys.exit(1) json_options = {} json_options["indent"] = 2 json_options["separators"] = (',', ': ') with open(output_file, "w") as fp: json.dump(conf, fp, **json_options) print("%s: %s file(s) merged into %s" % (prog, file_count, os.path.basename(output_file)))
def load(file, sections=[], **instargs): """Create a sMAP instance based on the representation stored in a file. The configuration file contains sections which refer to either reporting instances, or paths in the sMAP heirarchy. Any section whose name starts with ``/`` is treated as a resource name; sections starting with ``report`` are treated as reports. The file must contain at least one section named ``/``, which must contain a ``uuid`` key to set the root identifier for the source. :param string file: filename of the configuration file :param instargs: arguments passed to the :py:class:`~smap.core.SmapInstance` constructor. :return smap.core.SmapInstance: the created instancev :raise smap.loader.SmapLoadError: an error is encountered processing the file :raise smap.core.SmapError: some other error is encountered validating the loaded object """ found = None for l in ['', os.getcwd(), sys.prefix]: path = os.path.join(l, file) if os.path.isfile(path): found = path if not found: raise Exception("Config file %s not found." % file) print "Loading config file:", found conf = configobj.ConfigObj(found, indent_type=' ') # if there's a server section, override the default server # configuration with that if 'server' in conf: smapconf.SERVER = util.dict_merge( smapconf.SERVER, dict(((k.lower(), v) for (k, v) in conf['server'].iteritems()))) # we need the root to have a uuid inst = core.SmapInstance(conf['/']['uuid'], **instargs) inst.loading = True reports = [] for s in conf: print "Loading section", s if s.startswith('report'): resource = conf[s].get('ReportResource', '/+') format = conf[s].get('Format', 'json') max_age = conf[s].get('MaxAge', None) max_age = int(max_age) if max_age != None else None dest = [conf[s]['ReportDeliveryLocation']] for i in xrange(0, 10): if 'ReportDeliveryLocation%i' % i in conf[s]: dest.append(conf[s]['ReportDeliveryLocation%i' % i]) reportinst = { 'ReportDeliveryLocation': dest, 'ReportResource': resource, 'Format': format, 'uuid': inst.uuid(s), 'MaxAge': max_age, } for o in ['MinPeriod', 'MaxPeriod']: if o in conf[s]: reportinst[o] = conf[s][o] for o in [ 'ClientCertificateFile', 'ClientPrivateKeyFile', 'CAFile' ]: if o in conf[s]: reportinst[i] = os.path.expanduser(conf[s][o]) reports.append(reportinst) continue elif not s.startswith('/'): # path sections must start with a '/' # other sections might be present and could be parsed by # other parts of the program print "Warning: skipping section", s, "since it does not begin with a '/'" continue elif len(sections) and not util.norm_path(s) in sections: # skip all but the listed sections if we were asked to continue s = util.norm_path(s) # build the UUID for the item props = util.build_recursive(dict(conf[s].items())) id = None if 'uuid' in conf[s]: key = None id = uuid.UUID(conf[s]['uuid']) elif 'key' in conf[s]: key = conf[s]['key'] else: # default to the path if key = s if key: id = inst.uuid(key) # raise SmapLoadError("Every config file section must have a uuid or a key!") # create the timeseries or collection if (s == '/' or conf[s].get("type", None) == 'Collection' or inst.get_collection(s) != None): if s == '/': c = inst.get_collection('/') elif inst.get_collection(s) != None: # sometimes you will have collections created twice, # for instance if a driver creates it and then we want # to tag it with metadata c = inst.get_collection(s) else: c = core.Collection(s, inst) inst.add_collection(s, c) elif conf[s].get("type", "Timeseries") == "Timeseries": if inst.get_timeseries(s) != None: c = inst.get_timeseries(s) else: try: props['Properties']['UnitofMeasure'] except KeyError: raise SmapLoadError("A Timeseries must have at least " "the Properites/UnitofMeasure key") # the Timeseries uses defaults if the conf file doesn't # contain the right sections. c = core.Timeseries( id, props['Properties']['UnitofMeasure'], data_type=props['Properties'].get( 'ReadingType', core.Timeseries.DEFAULTS['Properties/ReadingType']), timezone=props['Properties'].get( 'Timezone', core.Timeseries.DEFAULTS['Properties/Timezone']), buffersz=int( props.get('BufferSize', core.Timeseries.DEFAULTS['BufferSize']))) inst.add_timeseries(s, c) else: if not id: raise SmapLoadError( "A driver must have a key or uuid to generate a namespace") # load a new driver manager layer newdrv = driver.SmapDriver.get_driver(inst, conf[s]['type'], s, id) # create a collection and add it at the attachment point c = inst.get_collection(s) if not c: c = core.Collection(s, inst) inst.add_collection(s, c) # Add config file specified checkers for the driver check = checkers.get(inst, newdrv, conf[s]) if check: inst.checkers.append(check) # get the driver to add its points newdrv.setup(conf[s]) # Metadata and Description are shared between both Collections # and Timeseries if props.has_key('Metadata'): # the driver may have added metadata; however config file # metadata overrides it c['Metadata'] = util.dict_merge(c.get('Metadata', {}), props['Metadata']) if props.has_key('Description'): c['Description'] = props['Description'] if key: setattr(c, 'key', key) # since the sections could come in any order, update the reporting # instance to make sure all the topics are set right. for reportinst in reports: if not inst.reports.update_report(reportinst): inst.reports.add_report(reportinst) inst.reports.update_subscriptions() inst.loading = False return inst