def main(): # get data from args gcm_id = request.args.get("gcm_id") new_data = request.args.get("user_data") file_path = "user_data/" + gcm_id if os.path.isfile(file_path): user_history = pickle.load(open(file_path, "rb")) existing_data = user_history['data'] threshold = user_history['threshold'] data_dict = update_dict(existing_data, new_data) else: data_dict = update_dict({}, new_data) threshold = DEFAULT_THRESHOLD data_dict = check_prediction(data_dict, threshold) user_json = { 'data': data_dict, 'threshold': threshold, } pickle.dump(user_json, open(file_path, "wb")) return jsonify(**data_dict)
def setUI(data): ''' takes { key: value, 'sub.key': value } ''' update_dict(settings.ui, data) return settings.ui
def setUI(data): """ takes { key: value, 'sub.key': value } """ update_dict(settings.ui, data) return settings.ui
def update(self, data): self.optimizer.zero_grad() self.model.train() self.model.set_data(data) self.model.update() loss_dict = self.model.loss_dict update_dict(self.loss_dict, loss_dict) self.optimizer.step() self.iteration += 1 return loss_dict
def fetch_server_data_for_account(server, account, ucaas=False): xsi = create_xsi_tool_for_account(server, account) utils.update_dict(account, xsi.get_directory_data()) kwargs = {"ucaas": ucaas} account["dm_config"] = xsi.get_dm_config(**kwargs) if not ucaas: for x in ["username", "password"]: account["xmpp_" + x] = utils.node_value(account["dm_config"], "protocols/xmpp/" "credentials/%s" % x) xmpp = create_xmpp_tool_for_account(account) utils.update_dict(account, xmpp.get_xmpp_data())
def setPreferences(data): ''' takes { key: value, 'sub.key': value } ''' update_dict(settings.preferences, data) if 'username' in data: u = state.user() u.update_name() u.save() return settings.preferences
def setPreferences(data): """ takes { key: value, 'sub.key': value } """ update_dict(settings.preferences, data) if "username" in data: u = state.user() u.update_name() u.save() return settings.preferences
def save(self, name=None, extra_info=None): epoch = self.epoch if name is None: name = f'model_{epoch:04d}' savepath = pjoin(self.ckpt_dir, "%s.pt" % name) state = { 'epoch': epoch, 'iteration': self.iteration, 'model': self.model.state_dict(), 'optimizer': self.optimizer.state_dict() } if isinstance(extra_info, dict): update_dict(state, extra_info) torch.save(state, savepath) self.log_string("Saving model at epoch {}, path {}".format( epoch, savepath))
def get_products_from_url(url, location): """This method is meant to extract products from a get response :param url - site to search url :param location - SearchPlace location :return Dictionary with objects""" response = requests.get( url, headers={ 'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:57.0) Gecko/20100101 Firefox/57.0' }) response = response.text response = get_products(response, location) product_dict = {} update_dict(product_dict, response, location.name) return product_dict
def create_nml(self): fv3_nml = os.path.join(self.workdir, 'input.nml') # Read in the namelist that has all the base settings. with open(self.config.paths.base_nml.format(n=self.config), 'r') as nml_file: base_nml = f90nml.read(nml_file) # Update the base namelist with settings for the current configuration # Send self.nml, a Namespace object, as dict to update_dict. # Update_dict modifies dict in place. utils.update_dict(base_nml, self.nml) with open(fv3_nml, 'w') as fn: base_nml.write(fn)
def train_all_models_with_cv_balance(config, k=10): train_df, test_df = data_acquisition(config) cv_df = cv_data_acquisition(config) cv = KFold(n_splits=k, random_state=1, shuffle=True) # cv_evaluator = Evaluator(test_path, cv_df) evaluator = Evaluator(config.args.test_data_path, test_df) # we won't save these they are for baseline majority_classifier(test_df, evaluator, 0) throw_a_die_classifier(test_df, evaluator) count = 0 model_dict = {} ngrams = [(1, 1), (1, 2), (1, 3)] for ngram in ngrams: cv_X_train_counts, cv_X_train_tf, y_cv_res = data_balance(cv_df) X_train_tf, X_test_tfidf, y_tr_res, y_ts_res = train_test_split( cv_X_train_tf, y_cv_res) # X_train_tf, X_test_tfidf = ngram_text_tf_idf(train_df.text, test_df.text, ngram) modles = get_all_Models(size=cv_X_train_counts.shape) # modles = {} evaluator.set_label(y_ts_res) for model in modles: model_name = model + str(ngram) clf, mic, mac, cm, acc, pred = train_eval_model( modles[model][utils.CLF], model_name, X_train_tf, y_tr_res, X_test_tfidf, evaluator, feature_selection=False) model_dict[model_name] = {utils.CLF: clf} model_dict = utils.update_dict(model_dict, model_name, clf, mic, mac, cm, acc, pred) scores = cross_val_score(model_dict[model_name][utils.CLF], cv_X_train_tf, y_cv_res, cv=cv, n_jobs=-1) mean_cv = scores.mean() model_dict[model_name][utils.CROSS_VALIDATION] = mean_cv count += 1 # print(f'mean Cross validation : {mean_cv}') print("\n") print(f"# of models ={count} with balancing") if config.args.print_info: for model in model_dict: print(f" model = {model}", end=' ') print(f"cv={model_dict[model][utils.CROSS_VALIDATION]}") print("\n") return model_dict, evaluator
def remove_call_forwards(self, **kwargs): """Remove all call forwards defined on the server. """ for forward_type in "always,busy,noanswer,notreachable".split(","): args = utils.update_dict(kwargs, { "forward_type": forward_type, "number": "", "active": False }) self.set_call_forwarding(**args)
def _write_config(self): """Write model configuration file.""" self.model_config['model_dir'] = self.model_dir self.wrapper_config['dict_path'] = self.dict_path model_defaults = get_default_args(Seq2SeqModel.__init__) self.model_config = update_dict(model_defaults, self.model_config) config = {} config['model'] = self.model_config config['wrapper'] = self.wrapper_config create_folder(self.model_dir) with open(self.config_path, 'w', encoding='utf8') as f: f.write(json.dumps(config, indent=2))
def _xsi_http(self, fn, **kwargs): api_endpoint = kwargs.get("api_endpoint", None) location = kwargs.get("location", None) parse = kwargs.get("parse", True) data = kwargs.get("data", None) auth = kwargs.get("auth", None) or (self.username, self.password) url = location if location else ( "%s/com.broadsoft.xsi-actions/v2.0" "/user/%s/%s" % (self.xsp_url, self.username, api_endpoint)) r = fn(url, verify=True, data=data, auth=auth) if len(r.history) > 0 and r.history[0].is_redirect: # requests bug (loses authorization header when redirected) return self._xsi_http( fn, **utils.update_dict( kwargs, {"location": r.history[0].headers["Location"]})) assert str(r.status_code)[0] == '2', \ "HTTP request %s failed (%s): %s" % (url, str(r.status_code), r.content or "") return utils.xml_tree(r.content) if parse else True
def load_plan(cls, config_filepath): """Loads a plan from file.""" with open(config_filepath) as fout: new_params = yaml.load(fout) update_dict(cls.params, new_params)
def update(self, item_id: int, data: dict) -> models.TasksList: item = self.get(item_id) self.delete(item_id) update_dict(item.__dict__, data) self._repo.add(item) return item
def main(cla): # Load the user-defined settings, and script settings # ---------------------------------------------------- user_config = cla.user_config print(f"user config: {user_config}") script_config = cla.script_config print(f"script config: {script_config}") if not script_config: ushdir = os.path.join(user_config['paths']['homerrfs'], 'configs') script_config = checks.load_config_file( os.path.join(ushdir, 'fv3_script.yml')) # Update script config with user-supplied config file # ---------------------------------------------------- utils.update_dict(script_config, user_config, quiet=cla.quiet) # Create Namespace of config for easier syntax # --------------------------------------------- config = argparse.Namespace() utils.namespace(config, script_config) # Now config and script_config contain identical information in Namespace # and dict formats, respectively. # Load each of the standard YAML config files # -------------------------------------------- # # Reminder: # checks.load_config_section(arg) takes a two-element list as it's input. # arg = [file_name, section_name(s)] # grid = cla.grid_config if not grid: grid = checks.load_config_section([ config.paths.grid.format(n=config), [config.grid_name, config.grid_gen_method], ]) machine = cla.machine_config if not machine: machine_path = config.paths.machine.format(n=config) machine = checks.load_config_section([ machine_path, config.machine, ]) namelist = cla.nml_config if not namelist: namelist = checks.load_config_section([ config.paths.namelist.format(n=config), config.phys_pkg, ]) # Update each of the provided configure files with user-supplied settings # ------------------------------------------------------------------------ for cfg in ['grid', 'machine', 'namelist']: utils.update_dict(locals()[cfg][0], script_config.get(cfg), quiet=cla.quiet) # Set up a kwargs dict for Forecast object # ----------------------------------------- fcst_kwargs = { 'grid': grid[0], 'nml': namelist[0], 'overwrite': cla.overwrite, } # Create the Forecast object # --------------------------- fcst = Forecast( config=script_config, machine=machine[0], starttime=cla.start_date, **fcst_kwargs, ) # Run the forecast job # --------------------- fcst.run(dry_run=cla.dry_run)
def __init__(self, hyperparameters): hp = hyperparameters self.hyperparameters = hp print( "LstmMlpSupersensesModel: Building model with the following hyperparameters:" ) pprint(hp.__dict__) self.features = build_features(hp) names = lambda features: [f.name for f in features] self.model = LstmMlpMulticlassModel( input_vocabularies={ feat.name: feat.vocab for feat in chain(self.features.list_enum_features(), self.features.list_string_features()) }, input_embeddings={ feat.name: feat.embedding for feat in self.features.list_features_with_embedding( include_auto=False) }, output_vocabulary=vocabs.PSS if self.hyperparameters.allow_empty_prediction else vocabs.PSS_WITHOUT_NONE, hyperparameters=LstmMlpMulticlassModel.HyperParameters( **update_dict(hp.__dict__, { 'lstm_input_fields': names(self.features.list_lstm_features()), 'mlp_input_fields': names(self.features.list_mlp_features(include_refs=False)), 'token_neighbour_types': names(self.features.list_ref_features()), 'input_embeddings_to_allow_partial': names(self.features.list_default_zero_vec_features()), 'input_embeddings_to_update': { name: True for name in names( self.features.list_updatable_features()) }, 'input_embeddings_default_dim': None, 'input_embedding_dims': { f.name: f.dim for f in self.features.list_features_with_embedding() }, 'n_labels_to_predict': len(self.hyperparameters.labels_to_predict) }, del_keys=[ 'use_token', 'lemmas_from', 'update_lemmas_embd', 'use_ud_xpos', 'use_govobj', 'use_ud_dep', 'use_ner', 'use_lexcat', 'token_embd_dim', 'ner_embd_dim', 'token_internal_embd_dim', 'ud_xpos_embd_dim', 'ud_deps_embd_dim', 'spacy_ner_embd_dim', 'govobj_config_embd_dim', 'lexcat_embd_dim', 'update_token_embd', 'use_prep_onehot', 'use_token_internal', 'labels_to_predict', 'mask_by', 'mask_mwes', 'allow_empty_prediction' ])))
def test_update_dict(a, b, expected): update_dict(a, b) assert a == expected
def set_params(self, **params): for key, value in params.items(): update_dict(self.params, key.split('__'), value) return self
def train_all_models_with_cv(config, k=10): # train test data train_df, test_df = data_acquisition(config) X_train_counts, X_test_counts, X_train_tf, X_test_tfidf, count_vect = data_pre_processing( train_df.text, test_df.text) cv_df = cv_data_acquisition(config) cv_X_train_counts, cv_X_train_tf, cv_count_vect = cv_data_pre_processing( cv_df.text) cv = KFold(n_splits=k, random_state=1, shuffle=True) evaluator = Evaluator(config.args.test_data_path, test_df) # trying out different classifiers # we won't save these they are for baseline majority_classifier(test_df, evaluator, 0) throw_a_die_classifier(test_df, evaluator) model_dict = {} count = 0 ngrams = [(1, 1), (1, 2), (1, 3)] for ngram in ngrams: modles = get_all_Models(size=test_df.shape) X_train_tf, X_test_tfidf = ngram_text_tf_idf(train_df.text, test_df.text, ngram) for model in modles: model_name = model + str(ngram) clf, mic, mac, cm, acc, pred = train_eval_model( modles[model][utils.CLF], model_name, X_train_tf, train_df.label, X_test_tfidf, evaluator, feature_selection=False) model_dict[model_name] = {utils.CLF: clf} model_dict = utils.update_dict(model_dict, model_name, clf, mic, mac, cm, acc, pred) scores = cross_val_score(model_dict[model_name][utils.CLF], cv_X_train_tf, cv_df.label, scoring='accuracy', cv=cv, n_jobs=-1) mean_cv = np.mean(scores) model_dict[model_name][utils.CROSS_VALIDATION] = mean_cv count += 1 # print(f'mean Cross validation : {mean_cv}') print("\n") print(f"# of models ={count} without balancing") if config.args.print_info: for model in model_dict: print(f" model = {model}", end=' ') print(f"cv={model_dict[model][utils.CROSS_VALIDATION]}") print("\n") return model_dict, count_vect, evaluator
def train_all_models(config): train_df, test_df = data_acquisition(config) X_train_counts, X_test_counts, X_train_tf, X_test_tfidf, count_vect = data_pre_processing( train_df.text, test_df.text) evaluator = Evaluator(config.args.test_data_path, test_df) # trying out different classifiers # we won't save these they are for baseline majority_classifier(test_df, evaluator, 0) throw_a_die_classifier(test_df, evaluator) model_dict = get_all_Models(size=test_df.shape) count = 0 for model in model_dict: clf, mic, mac, cm, acc, pred = train_eval_model( model_dict[model][utils.CLF], model, X_train_tf, train_df.label, X_test_tfidf, evaluator, feature_selection=False) model_dict = utils.update_dict(model_dict, model, clf, mic, mac, cm, acc, pred) count += 1 ngrams = [(1, 2), (1, 3)] for ngram in ngrams: modles = get_all_Models(size=test_df.shape) X_train_tf, X_test_tfidf = ngram_text_tf_idf(train_df.text, test_df.text, ngram) for model in modles: model_name = model + str(ngram) clf, mic, mac, cm, acc, pred = train_eval_model( modles[model][utils.CLF], model_name, X_train_tf, train_df.label, X_test_tfidf, evaluator, feature_selection=False) model_dict[model_name] = {utils.CLF: clf} model_dict = utils.update_dict(model_dict, model_name, clf, mic, mac, cm, acc, pred) count += 1 from datetime import datetime start_time = datetime.now() if config.args.with_bert: bert = Bert_init() model_name = 'bert' clf, mic, mac, cm, acc, pred = train_eval_model( clf=bert, name=model_name, X_train_tf=train_df.text, label=train_df.label, X_test_tfidf=test_df.text, evaluator=evaluator) end_time = datetime.now() print('Duration: {}'.format(end_time - start_time)) model_dict[model_name] = {utils.CLF: clf} model_dict = utils.update_dict(model_dict, model_name, clf, mic, mac, cm, acc, pred) count += 1 print(f"# of models ={count}") return model_dict, count_vect, evaluator