def load_wiki(wiki_config, config): # Merge in wiki defaults wiki_config = deep_merge.merge({}, config["wiki_defaults"], wiki_config, merge_lists=deep_merge.overwrite) for model_name, model_config in wiki_config.get('models', {}).items(): # Merge in model defaults model_config = deep_merge.merge( {}, config["model_defaults"], model_config) wiki_config['models'][model_name] = model_config return wiki_config
def advance(self, evaluator: Evaluator): self.iteration += 1 logger.info("Starting SPSA iteration %d." % self.iteration) if self.parameters is None: self.parameters = evaluator.problem.initial # Calculate objective if self.compute_objective: annotations = { "type": "objective" } objective_identifier = evaluator.submit(self.parameters, annotations = annotations) # Update step lengths gradient_length = self.gradient_factor / (self.iteration + self.gradient_offset)**self.gradient_exponent perturbation_length = self.perturbation_factor / self.iteration**self.perturbation_exponent # Sample direction from Rademacher distribution direction = self.random.randint(0, 2, len(self.parameters)) - 0.5 annotations = { "gradient_length": gradient_length, "perturbation_length": perturbation_length, "direction": direction, "type": "gradient" } # Schedule samples positive_parameters = np.copy(self.parameters) positive_parameters += direction * perturbation_length annotations = deep_merge.merge(annotations, { "type": "positive_gradient" }) positive_identifier = evaluator.submit(positive_parameters, annotations = annotations) negative_parameters = np.copy(self.parameters) negative_parameters -= direction * perturbation_length annotations = deep_merge.merge(annotations, { "type": "negative_gradient" }) negative_identifier = evaluator.submit(negative_parameters, annotations = annotations) # Wait for gradient run results evaluator.wait() if self.compute_objective: evaluator.clean(objective_identifier) positive_objective, positive_state = evaluator.get(positive_identifier) evaluator.clean(positive_identifier) negative_objective, negative_state = evaluator.get(negative_identifier) evaluator.clean(negative_identifier) g_k = (positive_objective - negative_objective) / (2.0 * perturbation_length) g_k *= direction**-1 # Update state self.parameters -= gradient_length * g_k
def submit(self, x, simulator_parameters = {}, annotations = {}, transient = False): if len(x) != self.number_of_parameters: raise RuntimeError("Invalid number of parameters: %d (expected %d)" % ( len(x), self.number_of_parameters )) identifier = self._create_identifier() response = self.problem.parameterize(x) if isinstance(response, tuple): parameters, cost = response else: parameters, cost = response, 1 parameters = deep_merge.merge(parameters, simulator_parameters) self.simulations[identifier] = { "identifier": identifier, "parameters": parameters, "x": x, "cost": cost, "annotations": annotations, "status": "pending", "transient": transient } self.pending.append(identifier) return identifier
def swagger_json(request): services = cornice.service.get_services() swagger = cornice_swagger.CorniceSwagger( services, pyramid_registry=request.registry ) swagger.base_path = '/api' swagger.summary_docstrings = True info = request.registry.settings['api_specs'] base = swagger.generate(**info, info=info) for path in base['paths'].keys(): for doc in (DESCRIPTIONS, RESPONSES): merge(base['paths'][path], doc[path.lstrip('/')]) base['definitions'] = request.registry.models # This is private endpoint del(base['paths']['/releases.json']['post']) return base
def main(): config_path = os.path.dirname(os.path.abspath(__file__)) + '/config.yaml' with open(config_path, 'r') as f: config = yamlconf.load(f) data = get_ores_data(config['ores_host']) data = deep_merge.merge(data, get_wikilabels_data(config['wikilabels_host'])) data = {'data': data, 'timestamp': time.time()} path = os.path.dirname(os.path.abspath(__file__)) + '/static/data.json' with open(path, 'w') as f: f.write(json.dumps(data))
def union_merge_observations(observations, id_column): """Merge all observations, returning the output as a list. """ id_map = collections.defaultdict(dict) for ob in observations: # Get the id value. ob_id = ob[id_column] # Merge the contents, with later entries taking precedence when keys # match. id_map[ob_id] = deep_merge.merge(id_map[ob_id], ob) return id_map.values()
def intersect_merge_observations(observation_sets, id_column): """Intersect all observations, returning the output as an iterable. """ observation_maps = [ {ob[id_column]: ob for ob in observation_set} for observation_set in observation_sets] for id_ in observation_maps[0]: # Key exists in all sets if sum(id_ in om for om in observation_maps) == len(observation_maps): new_ob = {} for observation_map in observation_maps: new_ob = deep_merge.merge(new_ob, observation_map[id_]) yield new_ob
def intersect_merge_observations(observation_sets, id_column): """Intersect all observations, returning the output as an iterable. """ observation_maps = [{ob[id_column]: ob for ob in observation_set} for observation_set in observation_sets] for id_ in observation_maps[0]: # Key exists in all sets if sum(id_ in om for om in observation_maps) == len(observation_maps): new_ob = {} for observation_map in observation_maps: new_ob = deep_merge.merge(new_ob, observation_map[id_]) yield new_ob
def uniq_roles(mapping, bind, value): """ Makes roles unique """ roles_map = dict() for v in value: id_ = v.get('id', '') if not id_: id_ = v.get('name') if not id_: continue if id_ in roles_map: roles = roles_map[id_].pop('roles', []) roles.extend(v.get('roles', [])) roles_map[id_] = deep_merge.merge(roles_map[id_], v) roles_map[id_]['roles'] = list(set(roles)) else: roles_map[id_] = v return list(roles_map.values())
import deep_merge print(deep_merge.merge({'a': {'b': 10}}, {'a': {'c': 5}}))
def _load_modules(self, root_dir: str, module_loader_registry: ModuleLoaderRegistry, dir_filter: Callable[[str], bool], module_load_context: Optional[str], keys_referenced_as_modules: Set[str], ignore_unresolved_params: bool = False) -> bool: """ Load modules which have not already been loaded and can be loaded (don't have unresolved parameters). :param ignore_unresolved_params: If true, not-yet-loaded modules will be loaded even if they are passed parameters that are not fully resolved. :return: True if there were modules that were not loaded due to unresolved parameters. """ all_module_definitions = {} all_module_evaluations_context = {} skipped_a_module = False for file in list(self.out_definitions.keys()): # Don't process a file in a directory other than the directory we're processing. For example, # if we're down dealing with <top_dir>/<module>/something.tf, we don't want to rescan files # up in <top_dir>. if os.path.dirname(file) != root_dir: continue # Don't process a file reference which has already been processed if file.endswith("]"): continue file_data = self.out_definitions.get(file) if file_data is None: continue module_calls = file_data.get("module") if not module_calls or not isinstance(module_calls, list): continue for module_index, module_call in enumerate(module_calls): if not isinstance(module_call, dict): continue # There should only be one module reference per outer dict, but... safety first for module_call_name, module_call_data in module_call.items(): if not isinstance(module_call_data, dict): continue module_address = (file, module_index, module_call_name) if module_address in self._loaded_modules: continue # Variables being passed to module, "source" and "version" are reserved specified_vars = { k: v[0] for k, v in module_call_data.items() if k != "source" and k != "version" } if not ignore_unresolved_params: has_unresolved_params = False for k, v in specified_vars.items(): if not is_acceptable_module_param( v) or not is_acceptable_module_param(k): has_unresolved_params = True break if has_unresolved_params: skipped_a_module = True continue self._loaded_modules.add(module_address) source = module_call_data.get("source") if not source or not isinstance(source, list): continue source = source[0] if not isinstance(source, str): logging.debug( f"Skipping loading of {module_call_name} as source is not a string, it is: {source}" ) continue # Special handling for local sources to make sure we aren't double-parsing if source.startswith("./") or source.startswith("../"): source = os.path.normpath( os.path.join( os.path.dirname( _remove_module_dependency_in_path(file)), source)) version = module_call_data.get("version", "latest") if version and isinstance(version, list): version = version[0] try: with module_loader_registry.load( root_dir, source, version) as content: if not content.loaded(): continue self._internal_dir_load( directory=content.path(), module_loader_registry=module_loader_registry, dir_filter=dir_filter, specified_vars=specified_vars, module_load_context=module_load_context, keys_referenced_as_modules= keys_referenced_as_modules) module_definitions = { path: self.out_definitions[path] for path in list(self.out_definitions.keys()) if os.path.dirname(path) == content.path() } if not module_definitions: continue # NOTE: Modules are put into the main TF definitions structure "as normal" with the # notable exception of the file name. For loaded modules referrer information is # appended to the file name to create this format: # <file_name>[<referred_file>#<referrer_index>] # For example: # /the/path/module/my_module.tf[/the/path/main.tf#0] # The referrer and index allow a module allow a module to be loaded multiple # times with differing data. # # In addition, the referring block will have a "__resolved__" key added with a # list pointing to the location of the module data that was resolved. For example: # "__resolved__": ["/the/path/module/my_module.tf[/the/path/main.tf#0]"] resolved_loc_list = module_call_data.get( RESOLVED_MODULE_ENTRY_NAME) if resolved_loc_list is None: resolved_loc_list = [] module_call_data[ RESOLVED_MODULE_ENTRY_NAME] = resolved_loc_list # NOTE: Modules can load other modules, so only append referrer information where it # has not already been added. keys = list(module_definitions.keys()) for key in keys: if key.endswith("]") or file.endswith("]"): continue keys_referenced_as_modules.add(key) new_key = f"{key}[{file}#{module_index}]" module_definitions[ new_key] = module_definitions[key] del module_definitions[key] del self.out_definitions[key] if new_key not in resolved_loc_list: resolved_loc_list.append(new_key) resolved_loc_list.sort( ) # For testing, need predictable ordering deep_merge.merge(all_module_definitions, module_definitions) except Exception as e: logging.warning( "Unable to load module (source=\"%s\" version=\"%s\"): %s", source, version, e) pass if all_module_definitions: deep_merge.merge(self.out_definitions, all_module_definitions) deep_merge.merge(self.out_evaluations_context, all_module_evaluations_context) return skipped_a_module
import deep_merge from django.conf import settings from django.utils.module_loading import import_string DEFAULT_UPLOAD_SETTINGS = { 'FORM': 'data_ingest.forms.UploadForm', 'INGESTOR': 'data_ingest.ingestors.Ingestor', 'TEMPLATE': 'data_ingest/upload.html', 'LIST_TEMPLATE': 'data_ingest/upload_list.html', 'DETAIL_TEMPLATE': 'data_ingest/upload_detail.html', 'MODEL': 'data_ingest.models.DefaultUpload', 'DESTINATION': 'data_ingest/', 'DESTINATION_FORMAT': 'json', 'VALIDATION_SCHEMA': None, } UPLOAD_SETTINGS = deep_merge.merge(DEFAULT_UPLOAD_SETTINGS, getattr(settings, 'DATA_INGEST', {})) upload_form_class = import_string(UPLOAD_SETTINGS['FORM']) upload_model_class = import_string(UPLOAD_SETTINGS['MODEL']) ingestor_class = import_string(UPLOAD_SETTINGS['INGESTOR'])
with open("../node_modules/cloverleaf/data/sites.json", 'r') as json_file: sites = json.load(json_file) with open("../data/logos.json", 'r') as json_file: logos = json.load(json_file) with open("../node_modules/cloverleaf/unit_tests/configs.json", 'r') as json_file: configs = json.load(json_file) with open("../node_modules/cloverleaf/unit_tests/results.json", 'r') as json_file: results = json.load(json_file) sites = deep_merge.merge(sites, logos) def status_code(driver, url): js = ''' let callback = arguments[0]; let xhr = new XMLHttpRequest(); xhr.open('GET', ''' + "'" + url.replace("'", "\\\'") + "'" + ''', true); xhr.onload = function () { if (this.readyState === 4) { callback(this.status); } }; xhr.onerror = function () { callback('error');
def _load_modules(self, root_dir: str, module_loader_registry: ModuleLoaderRegistry, dir_filter: Callable[[str], bool], module_load_context: Optional[str]): all_module_definitions = {} all_module_evaluations_context = {} for file in list(self.out_definitions.keys()): file_data = self.out_definitions.get(file) module_calls = file_data.get("module") if not module_calls or not isinstance(module_calls, list): continue for module_index, module_call in enumerate(module_calls): if not isinstance(module_call, dict): continue # There should only be one module reference per outer dict, but... safety first for module_call_name, module_call_data in module_call.items(): if not isinstance(module_call_data, dict): continue source = module_call_data.get("source") if not source or not isinstance(source, list): continue source = source[0] # Special handling for local sources to make sure we aren't double-parsing if source.startswith("./") or source.startswith("../"): source = os.path.normpath( os.path.join( os.path.dirname( _remove_module_dependency_in_path(file)), source)) version = module_call_data.get("version", "latest") if version and isinstance(version, list): version = version[0] try: with module_loader_registry.load( root_dir, source, version) as content: if not content.loaded(): continue # Variables being passed to module, "source" and "version" are reserved specified_vars = { k: v[0] for k, v in module_call_data.items() if k != "source" and k != "version" } if not dir_filter(os.path.abspath(content.path())): continue self._internal_dir_load( directory=content.path(), module_loader_registry=module_loader_registry, dir_filter=dir_filter, specified_vars=specified_vars, module_load_context=module_load_context) module_definitions = { path: self.out_definitions[path] for path in list(self.out_definitions.keys()) if os.path.dirname(path) == content.path() } if not module_definitions: continue # NOTE: Modules are put into the main TF definitions structure "as normal" with the # notable exception of the file name. For loaded modules referrer information is # appended to the file name to create this format: # <file_name>[<referred_file>#<referrer_index>] # For example: # /the/path/module/my_module.tf[/the/path/main.tf#0] # The referrer and index allow a module allow a module to be loaded multiple # times with differing data. # # In addition, the referring block will have a "__resolved__" key added with a # list pointing to the location of the module data that was resolved. For example: # "__resolved__": ["/the/path/module/my_module.tf[/the/path/main.tf#0]"] resolved_loc_list = module_call_data.get( RESOLVED_MODULE_ENTRY_NAME) if resolved_loc_list is None: resolved_loc_list = [] module_call_data[ RESOLVED_MODULE_ENTRY_NAME] = resolved_loc_list # NOTE: Modules can load other modules, so only append referrer information where it # has not already been added. keys = list(module_definitions.keys()) for key in keys: if key.endswith("]"): continue new_key = f"{key}[{file}#{module_index}]" module_definitions[new_key] = \ module_definitions[key] del module_definitions[key] del self.out_definitions[key] resolved_loc_list.append(new_key) deep_merge.merge(all_module_definitions, module_definitions) except Exception as e: logging.warning( "Unable to load module (source=\"%s\" version=\"%s\"): %s", source, version, e) pass if all_module_definitions: deep_merge.merge(self.out_definitions, all_module_definitions) deep_merge.merge(self.out_evaluations_context, all_module_evaluations_context)
def load_wiki(wiki, config): wiki = deep_merge.merge({}, config["wiki_defaults"], wiki, merge_lists=deep_merge.overwrite) result = collections.OrderedDict() if 'models' not in wiki: wiki['models'] = {} if isinstance(wiki["models"], list): wiki["models"] = {name: {} for name in wiki["models"]} for model_name in ['reverted', 'damaging', 'goodfaith']: if model_name not in wiki['models']: continue model = wiki["models"][model_name] # Do not apply default configs for RandomForest models # Because it doesn't make sense for them if not model.get('rf'): model = deep_merge.merge({}, config["model_defaults"], model) for case in model['tuning_params']: value = model['tuning_params'][case] if isinstance(value, str): model['tuning_params'][case] = '"%s"' % value # Normalize for old label. if model_name == 'reverted': model['label'] = 'reverted_for_damage' else: model['label'] = model_name target_prediction = 'false' if model_name == 'goodfaith' else 'true' model['label_weight'] = \ "{}=$({}_weight)".format(target_prediction, model_name) model['algorithm'] = 'rf' if 'rf' in model else 'gradient_boosting' model[ 'class_name'] = 'RandomForest' if 'rf' in model else 'GradientBoosting' result[model_name] = model wiki["models"] = result # Sort sample types result = collections.OrderedDict() for sample_type in ['quarry_url', 'labeling_campaign']: for sample in wiki.get('samples', {}): if sample_type not in wiki['samples'][sample]: continue result[sample] = wiki['samples'][sample] wiki['samples'] = result # Normalize options if 'default_sample_bigger_sample' not in wiki: wiki['default_sample_bigger_sample'] = wiki['default_sample'] if 'sample_to_build_review' not in wiki: wiki['sample_to_build_review'] = wiki['default_sample'] return wiki
def run(self, identifier, parameters): """ Runs a MATSim simulation. """ if identifier in self.simulations: raise RuntimeError( "A simulation with identifier %s already exists." % identifier) simulation_path = "%s/%s" % (self.working_directory, identifier) if os.path.exists(simulation_path): shutil.rmtree(simulation_path) os.mkdir(simulation_path) simulation_parameters = {} simulation_parameters = deep_merge.merge(simulation_parameters, self.parameters) simulation_parameters = deep_merge.merge(simulation_parameters, parameters) parameters = simulation_parameters # Rewrite configuration if "iterations" in parameters: if "controler.lastIteration" in parameters["config"]: logger.warn( "Overwriting 'controler.lastIteration' for simulation %s" % identifier) if "controler.writeEventsInterval" in parameters["config"]: logger.warn( "Overwriting 'controler.writeEventsInterval' for simulation %s" % identifier) if "controler.writePlansInterval" in parameters["config"]: logger.warn( "Overwriting 'controler.writePlansInterval' for simulation %s" % identifier) parameters["config"]["controler.lastIteration"] = parameters[ "iterations"] parameters["config"]["controler.writeEventsInterval"] = parameters[ "iterations"] parameters["config"]["controler.writePlansInterval"] = parameters[ "iterations"] if "random_seed" in parameters: if "global.random_seed" in parameters["config"]: logger.warn( "Overwriting 'global.random_seed' for simulation %s" % identifier) parameters["config"]["global.random_seed"] = parameters[ "random_seed"] if "restart" in parameters: if "plans.inputPlansFile" in parameters["config"]: logger.warn( "Overwriting 'plans.inputPlansFile' for simulation %s" % identifier) restart_path = "%s/%s" % (self.working_directory, parameters["restart"]) parameters["config"][ "plans.inputPlansFile"] = "%s/output/output_plans.xml.gz" % restart_path if "controler.outputDirectory" in parameters["config"]: logger.warn( "Overwriting 'controler.outputDirectory' for simulation %s" % identifier) parameters["config"][ "controler.outputDirectory"] = "%s/output" % simulation_path # Construct command line arguments if not "class_path" in parameters: raise RuntimeError( "Parameter 'class_path' must be set for the MATSim simulator.") if not "main_class" in parameters: raise RuntimeError( "Parameter 'main_class' must be set for the MATSim simulator.") arguments = [ parameters["java"], "-Xmx%s" % parameters["memory"], "-cp", parameters["class_path"], parameters["main_class"] ] + parameters["prefix_arguments"] + parameters["arguments"] for key, value in parameters["config"].items(): arguments += ["--config:%s" % key, str(value)] arguments += parameters["postfix_arguments"] stdout = open("%s/simulation_output.log" % simulation_path, "w+") stderr = open("%s/simulation_error.log" % simulation_path, "w+") logger.info("Starting simulation %s:" % identifier) logger.info(" ".join(arguments)) self.simulations[identifier] = { "process": sp.Popen(arguments, stdout=stdout, stderr=stderr), "arguments": arguments, "status": "running", "progress": -1, "iterations": parameters["iterations"] if "iterations" in parameters else None }
def advance(self, evaluator: Evaluator): self.iteration += 1 logger.info("Starting FDSA iteration %d." % self.iteration) # Calculate objective if self.compute_objective: annotations = {"type": "objective"} objective_identifier = evaluator.submit(self.parameters, annotations=annotations) # Update lengths gradient_length = self.gradient_factor / ( self.iteration + self.gradient_offset)**self.gradient_exponent perturbation_length = self.perturbation_factor / self.iteration**self.perturbation_exponent annotations = { "gradient_length": gradient_length, "perturbation_length": perturbation_length, "type": "gradient" } # I) Calculate gradients gradient = np.zeros((len(self.parameters), )) gradient_information = [] # Schedule all necessary runs for d in range(len(self.parameters)): annotations = deep_merge.merge(annotations, {"dimension": d}) positive_parameters = np.copy(self.parameters) positive_parameters[d] += perturbation_length annotations = deep_merge.merge(annotations, {"type": "positive_gradient"}) positive_identifier = evaluator.submit(positive_parameters, annotations=annotations) negative_parameters = np.copy(self.parameters) negative_parameters[d] -= perturbation_length annotations = deep_merge.merge(annotations, {"sign": "negative_gradient"}) negative_identifier = evaluator.submit(negative_parameters, annotations=annotations) gradient_information.append( (positive_parameters, positive_identifier, negative_parameters, negative_identifier)) # Wait for gradient run results evaluator.wait() if self.compute_objective: evaluator.clean(objective_identifier) for d, item in enumerate(gradient_information): positive_parameters, positive_identifier, negative_parameters, negative_identifier = item positive_objective, positive_state = evaluator.get( positive_identifier) evaluator.clean(positive_identifier) negative_objective, negative_state = evaluator.get( negative_identifier) evaluator.clean(negative_identifier) gradient[d] = (positive_objective - negative_objective) / (2.0 * perturbation_length) # II) Update state self.parameters -= gradient_length * gradient