def upgrade_pip(config_dir): config_instance = Config(config_dir) with prefix('workon %s' % config_instance.project_name): run('pip install -U pip')
from utils.ArticlesHandler import ArticlesHandler from utils import solve, embedding_matrix_2_kNN, get_rate, accuracy, precision, recall, f1_score from utils.Trainer_graph import TrainerGraph from utils import Config, accuracy_sentence_based import time import numpy as np # from utils.postprocessing.SelectLabelsPostprocessor import SelectLabelsPostprocessor from utils.Trainer_graph import TrainerGraph from sklearn import svm from sklearn.ensemble import RandomForestClassifier from sklearn.metrics import accuracy_score config = Config('config/') debut = time.time() handler = ArticlesHandler(config) # Save in a pickle file. To open, use the pickle dataloader. #handler.articles.save("../Dataset/train_fake.pkl") # Only recompute labels: # handler.articles.compute_labels() C = handler.get_tensor() # select_labels = SelectLabelsPostprocessor(config, handler.articles) # handler.add_postprocessing(select_labels, "label-selection") # handler.postprocess() labels = np.array(handler.articles.labels) all_labels = np.array(handler.articles.labels_untouched) if config.learning.method_learning == "FaBP": assert max(labels) == 2, "FaBP can only be used for binary classification."
def test_loading_same_directory(self): c = Config(Path.cwd() / 'utils' / 'config.yml') self.assertEqual(c.mysql.host, 'localhost') self.assertEqual(c.other.preprocessing_queue[0], 'preprocessing.scale_and_center')
te_count += len(th) if td0 is None: td0 = diff else: td0 = np.concatenate((td0, diff)) fp.close() print("Err {0} {1} {2:.6f} {3:.6f} {4}".format( datetime.datetime.now() - T0, a, t_loss / t_count, te_loss / te_count, np.median(np.array(td0)))) saver.save(sess, cfg.netFile) if __name__ == '__main__': config_file = "config_2.json" cfg = Config(config_file) cfg.num_output = list(map(int, cfg.num_output.split(','))) iterations = 100000 if len(sys.argv) > 1: cfg.mode = int(sys.argv[1]) if len(sys.argv) > 2: iterations = int(sys.argv[2]) run(cfg, iterations)
def run_possibilities(dataset_path, logs_path, possibilities): x_train_labeled, x_train_unlabeled, y_train_labeled, x_val, y_val = get_data( dataset_path=dataset_path, normalization=NORMALIZATION, unlabeled_percentage=UNLABELED_PERCENTAGE, seed=SEED) _, evaluation_mapping, _ = timit.get_phone_mapping() n_classes = get_number_of_classes() for consistency_loss, schedule, sigma, consistency_scale, stabilization_scale, xi in possibilities: hparams = { 'consistency_loss': consistency_loss, 'schedule': schedule, 'sigma': sigma, 'consistency_scale': consistency_scale, 'stabilization_scale': stabilization_scale, 'xi': xi } for k, v in hparams.items(): print(f'{k}={v}, ', end='') print() config = Config(version='mono_directional', n_hidden_layers=N_HIDDEN_LAYERS, n_units=N_UNITS, n_epochs=N_EPOCHS, batch_size=BATCH_SIZE, unlabeled_percentage=UNLABELED_PERCENTAGE, optimizer=OPTIMIZER, consistency_loss=consistency_loss, consistency_scale=consistency_scale, stabilization_scale=stabilization_scale, xi=xi, sigma=sigma, schedule=schedule, schedule_length=SCHEDULE_LENGTH, normalization=NORMALIZATION, seed=SEED) logs_path_ = logs_path / str(config) if logs_path_.is_dir( ): # skip what already done (e.g. in case of crashes) print('already done, skipping...') continue logs_path_.mkdir(parents=True) logs_path_ = str(logs_path_) model = DualStudent(n_classes=n_classes, n_hidden_layers=config.n_hidden_layers, n_units=config.n_units, consistency_loss=config.consistency_loss, consistency_scale=config.consistency_scale, stabilization_scale=config.stabilization_scale, xi=config.xi, padding_value=PADDING_VALUE, sigma=config.sigma, schedule=config.schedule, schedule_length=config.schedule_length, version=config.version) model.compile(optimizer=get_optimizer(config.optimizer)) model.train(x_labeled=x_train_labeled, x_unlabeled=x_train_unlabeled, y_labeled=y_train_labeled, n_epochs=config.n_epochs, batch_size=config.batch_size, seed=config.seed) results = model.test(x=x_val, y=y_val, batch_size=config.batch_size, evaluation_mapping=evaluation_mapping) with tf.summary.create_file_writer(logs_path_).as_default(): hp.hparams(hparams) for k, v in results.items(): tf.summary.scalar(k, v, step=N_EPOCHS)
while i < len(filenames): sys.stdout.write('\r>> converting images %d/%d' % (i + 1, len(filenames))) sys.stdout.flush() filename = filenames[i] input_byte, target_byte, input_shape, target_shape = read_image( dataset_dir, filename) example = convert_to_example(input_byte, target_byte, input_shape, target_shape) serialized = example.SerializeToString() tfrecord_writer.write(serialized) i += 1 print('\n%s image converted' % (mode)) if __name__ == '__main__': parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument('--config', type=str, default='config/pix2pix.yml', help='config file name') args = parser.parse_args() Config(args.config) create_tfrecord()
from threading import Timer from lib.HealthMonitor.HealthMonitor.HealthMapper import HealthMapper from utils import Interaction, TaskThread, Cache, Config CF = Config() class HealthDispatcher(object): def __init__(self, **kwargs): super().__init__() self._request_inter = Interaction("receive") self._sender_inter = Interaction("sender") self._handlers = { "alive": TaskThread(target=kwargs.get("alive", self._alive), name="alive"), "status": TaskThread(target=kwargs.get("status", self._status), name="status"), "init": TaskThread(target=kwargs.get("init", self._init), name="init"), } self._cache = Cache("cluster_manager") self._cache['node_load'] = {} self._mapper = HealthMapper() # self._cache['fd'] = {} def dispatch(self, data, address):
def main(inputs, paths): print """ """ task = { 'KIND_RUN' : 'TEST_CP2K', 'TEMPLATE_FILE' : 'FSSH_CORE.template', 'FORCEFIELD_FILE' : 'FSSH_FF.template', 'TEST' : 'YES' } inputs.update(task) list_propagation = ['FSSH','BORN_OPPENHEIMER', 'TEST_HOP','FROZEN_HAMILTONIAN','CLASSICAL_PATH','GALILEAN'] #list_propagation = ['FSSH'] list_decoherences = ['NO_DECO_CORR', 'INSTANT_COLLAPSE', 'DAMPING'] list_trivial_corr = ['TRIVIAL_HOP_CORRECT', 'UNMODIFIED_SURF_HOP'] list_analytics = ['T', 'F'] #list_analytics = ['T'] # list_collapse = ['T', 'F'] #list_analytics = ['T'] list_com = ['T', 'F'] #list_collapse = ['T'] list_first_diabat = [1, 2] #list_first_diabat = [1] list_rescaling = ['SIMPLE', 'SIMPLE_QSYS','NACV'] #list_rescaling = ['NACV'] list_nacv = ['TEST', 'CONTRIBUTION','TOTAL','FAST'] #list_nacv = ['TOTAL'] list_reversal = ['NEVER','ALWAYS','TRUHLAR','SUBOTNIK'] #list_reversal = ['ALWAYS'] mega_list = [ { 'PROPAGATION' : prop, # 'COLLAPSE' : collapse, 'DECOHERENCES': deco, 'TRIVIAL_CORR' : trivial, 'ANALYTICS' : analytics, 'FIRST_DIABAT': diabat, 'METHOD_RESCALING' : rescaling, 'METHOD_ADIAB_NACV' : nacv, 'METHOD_REVERSAL' : reversal, 'CENTER_OF_MASS' : com } for prop in list_propagation for trivial in list_trivial_corr for deco in list_decoherences for analytics in list_analytics for diabat in list_first_diabat for rescaling in list_rescaling for nacv in list_nacv for reversal in list_reversal for com in list_com ] systems = ['dimer', 'trimer', 'dimer_solvent'] # SET_UP THE DIRECTORY, CHECK ANY SUBDIR IS PRESENT bucket = Bucket(inputs) bucket.name() paths.update({'bucket': bucket.path}) task = Dir(inputs.get('INPUT_INFO')) paths.update( {'task' : task.path} ) templates = Dir('templates', paths) templates.checkdir() templates.clean() bin = Dir('bin', paths) bin.checkdir() ndir= 0 for system in systems: system_input = InputFile( paths.get('task') + system + '/input') os.system(' cp -r %s/initial/ %s' % (paths.get('task') + system, paths.get('bucket'))) initial = Dir( 'initial', paths) initial.checkdir() paths.update({'initial' : initial.path}) inputs.update(system_input.dict) inputs.update({'STEPS': 2}) dict_prev = {} for dict in mega_list: if dict.get('PROPAGATION') == 'TEST_HOP': inputs.update({'STEPS':1}) if system_input.dict.get('FILE_INIT') != 'initial_dimer': dict.update({'ANALYTICS' : 'F' }) system = system_input.dict.get('SYSTEM') if system == 'CRYSTAL': from utils import CP2KOSFSSH as Config elif system == 'SOLVENT': from utils import CP2KOSwSolventFSSH as Config else: sys.exit() config = Config( inputs, paths, INIT = 1, **dict) print "GO FOR RUN %d" % ndir ndir = config.run(ndir) if os.path.exists('run-%d' % (ndir -1) ): # pass os.system('rm -rf run-%d' % (ndir - 1)) else: print " ERROR IN CP2K FOR THOSE PARAMETERS:" print dict print " TO BE COMPARED WITH:" print dict_prev sys.exit() dict_prev = dict
def __init__(self): self.__config = Config() pathlib.Path(self.__config.get_model_path()).mkdir(parents=True, exist_ok=True) pathlib.Path(self.__config.get_prediction_path()).mkdir(parents=True, exist_ok=True)
parser.add_argument( '--app_prof', choices=['dureader_debug', 'cmrc2018_debug', 'dureader', 'cmrc2018'], default='cmrc2018', help='choose config profile to use') parser.add_argument('--params_prof', choices=['qanet', 'default'], default='qanet', help='choose params profile to use') args = parser.parse_args() dic = { '../data/configs.yaml': args.app_prof, '../data/params.yaml': args.params_prof } config = Config(dic) server = Server(args, config) app = Flask(__name__) @app.route('/') def index(): return 'Index Page' @app.route('/answer', methods=['GET', 'POST']) def answer(): args = request.args question = args.get('question', None)
def main(args): dataset_config = Config(args.dataset_config) model_config = Config(args.model_config) exp_dir = Path("experiments") / model_config.type exp_dir = exp_dir.joinpath( f"epochs_{args.epochs}_batch_size_{args.batch_size}_learning_rate_{args.learning_rate}" ) if not exp_dir.exists(): exp_dir.mkdir(parents=True) if args.fix_seed: torch.manual_seed(777) torch.backends.cudnn.deterministic = True torch.backends.cudnn.benchmark = False tokenizer = get_tokenizer(dataset_config, model_config) tr_dl, val_dl = get_data_loaders(dataset_config, tokenizer, args.batch_size) model = VDCNN(num_classes=model_config.num_classes, embedding_dim=model_config.embedding_dim, k_max=model_config.k_max, vocab=tokenizer.vocab) loss_fn = nn.CrossEntropyLoss() opt = optim.Adam(params=model.parameters(), lr=args.learning_rate) scheduler = ReduceLROnPlateau(opt, patience=5) device = torch.device( 'cuda') if torch.cuda.is_available() else torch.device('cpu') model.to(device) writer = SummaryWriter(f'{exp_dir}/runs') checkpoint_manager = CheckpointManager(exp_dir) summary_manager = SummaryManager(exp_dir) best_val_loss = 1e+10 for epoch in tqdm(range(args.epochs), desc='epochs'): tr_loss = 0 tr_acc = 0 model.train() for step, mb in tqdm(enumerate(tr_dl), desc='steps', total=len(tr_dl)): x_mb, y_mb = map(lambda elm: elm.to(device), mb) opt.zero_grad() y_hat_mb = model(x_mb) mb_loss = loss_fn(y_hat_mb, y_mb) mb_loss.backward() opt.step() with torch.no_grad(): mb_acc = acc(y_hat_mb, y_mb) tr_loss += mb_loss.item() tr_acc += mb_acc.item() if (epoch * len(tr_dl) + step) % args.summary_step == 0: val_loss = evaluate(model, val_dl, {'loss': loss_fn}, device)['loss'] writer.add_scalars('loss', { 'train': tr_loss / (step + 1), 'val': val_loss }, epoch * len(tr_dl) + step) model.train() else: tr_loss /= (step + 1) tr_acc /= (step + 1) tr_summary = {'loss': tr_loss, 'acc': tr_acc} val_summary = evaluate(model, val_dl, { 'loss': loss_fn, 'acc': acc }, device) scheduler.step(val_summary['loss']) tqdm.write( f"epoch: {epoch+1}\n" f"tr_loss: {tr_summary['loss']:.3f}, val_loss: {val_summary['loss']:.3f}\n" f"tr_acc: {tr_summary['acc']:.2%}, val_acc: {val_summary['acc']:.2%}" ) val_loss = val_summary['loss'] is_best = val_loss < best_val_loss if is_best: state = { 'epoch': epoch + 1, 'model_state_dict': model.state_dict(), 'opt_state_dict': opt.state_dict() } summary = {'train': tr_summary, 'validation': val_summary} summary_manager.update(summary) summary_manager.save('summary.json') checkpoint_manager.save_checkpoint(state, 'best.tar') best_val_loss = val_loss
train_mse = np.square(self.calculate_rmse(X)) train_objective = train_mse * X.shape[0] + self.calc_regularization() epoch_convergence = {"train_objective": train_objective, "train_mse": train_mse} self.record(epoch_convergence) self.current_epoch += 1 self.save_params() """ raise NotImplementedError def run_epoch(self, data: np.array): raise NotImplementedError def predict_on_pair(self, user: int, item: int): raise NotImplementedError def save_params(self): raise NotImplementedError if __name__ == '__main__': baseline_config = Config(lr=0.001, gamma=0.001, epochs=10) train, validation = get_data() baseline_model = Baseline(baseline_config) baseline_model.fit(train) print(baseline_model.calculate_rmse(validation))
}, 'fetch_os': False }, 'toucan': { 'enable': True, 'api': 'http://toucan:3000', 'min_retry_time': 5, 'max_retry_time': 60 * 5, 'max_retry_count': 20 }, 'pid_file': 'aucote.pid', 'default_config': 'aucote_cfg_default.yaml', } # global cfg cfg = Config(_DEFAULT) async def start_toucan(default_config): """ Initialize Toucan Args: default_config: Returns: None """ Toucan.min_retry_time = cfg['toucan.min_retry_time'] Toucan.max_retry_time = cfg['toucan.max_retry_time']
import pandas as pd from pathlib import Path from utils import Config from sklearn.model_selection import train_test_split # dataset data_dir = Path("data") train = pd.read_csv(data_dir / "kor_pair_train.csv").filter( items=["question1", "question2", "is_duplicate"]) test = pd.read_csv(data_dir / "kor_pair_test.csv").filter( items=["question1", "question2", "is_duplicate"]) dataset = pd.concat([train, test], ignore_index=True, sort=False) train, test = train_test_split(dataset, test_size=.1, random_state=777) train, validation = train_test_split(train, test_size=.1, random_state=777) train.to_csv(data_dir / "train.txt", sep="\t", index=False) validation.to_csv(data_dir / "validation.txt", sep="\t", index=False) test.to_csv(data_dir / "test.txt", sep="\t", index=False) config = Config({ "train": str(data_dir / "train.txt"), "validation": str(data_dir / "validation.txt"), "test": str(data_dir / "test.txt"), }) config.save(data_dir / "config.json")
from model.metric import evaluate, acc from utils import Config, CheckpointManager, SummaryManager parser = argparse.ArgumentParser() parser.add_argument('--data_dir', default='data', help="Directory containing config.json of data") parser.add_argument('--model_dir', default='experiments/base_model', help="Directory containing config.json of model") if __name__ == '__main__': args = parser.parse_args() data_dir = Path(args.data_dir) model_dir = Path(args.model_dir) data_config = Config(json_path=data_dir / 'config.json') model_config = Config(json_path=model_dir / 'config.json') # tokenizer with open(data_config.vocab, mode='rb') as io: vocab = pickle.load(io) pad_sequence = PadSequence(length=model_config.length, pad_val=vocab.to_indices(vocab.padding_token)) tokenizer = Tokenizer(vocab=vocab, split_fn=MeCab().morphs, pad_fn=pad_sequence) # model model = SenCNN(num_classes=model_config.num_classes, vocab=tokenizer.vocab) # training
token_to_idx=ptr_bert_vocab.token_to_idx) # save vocab with open(ptr_vocab_path.with_suffix('.pkl'), mode="wb") as io: pickle.dump(vocab, io) else: print('Already you have pytorch_model_skt_vocab.json!') if not ptr_tokenizer_path.exists(): urlretrieve('https://kobert.blob.core.windows.net/models/kobert/tokenizer/tokenizer_78b3253a26.model', filename=ptr_tokenizer_path) else: print('Already you have pytorch_model_skt_tokenizer.model') ptr_config = Config({'config': str(ptr_config_path), 'bert': str(ptr_bert_path), 'tokenizer': str(ptr_tokenizer_path), 'vocab': str(ptr_vocab_path.with_suffix('.pkl'))}) ptr_config.save(ptr_dir / "config_skt.json") if args.type == 'etri': # loading BertTokenizer ptr_config_path = ptr_dir / 'bert_config_etri.json' ptr_tokenizer_path = ptr_dir / "vocab.korean.rawtext.list" ptr_bert_path = ptr_dir / "pytorch_model_etri.bin" ptr_tokenizer = ETRITokenizer.from_pretrained( ptr_tokenizer_path, do_lower_case=False ) # generate vocab idx_to_token = list(ptr_tokenizer.vocab.keys()) token_to_idx = {token: idx for idx, token in enumerate(idx_to_token)}
For the full list of settings and their values, see https://docs.djangoproject.com/en/3.1/ref/settings/ """ import os from pathlib import Path from corsheaders.defaults import default_methods, default_headers from common.admin_site import admin_site from utils import Config BASE_DIR = Path(__file__).resolve(strict=True).parent.parent.parent # Quick-start development settings - unsuitable for production # See https://docs.djangoproject.com/en/3.1/howto/deployment/checklist/ CONFIG_FILE = os.path.join(BASE_DIR, '..', 'config.yml') config = Config(CONFIG_FILE) admin_site.site_title = config.get('SITE_TITLE', 'Django Template Project') admin_site.site_header = config.get('SITE_HEADER', 'Django Template Project') SECRET_KEY = config.get('SECRET_KEY', raise_error=True) DEBUG = config.get('DEBUG', False, cast=bool) ALLOWED_HOSTS = config.get('ALLOWED_HOSTS', cast=list) INSTALLED_APPS = [ 'administration', 'website', 'common', 'rest_framework_swagger', 'rest_framework', 'rest_framework.authtoken', 'drf_yasg',
from geeteventbus.subscriber import subscriber from utils import Logger, XEvent, Config from pluginbase import PluginBase from functools import partial import os here = os.path.abspath(os.path.dirname(__file__)) get_path = partial(os.path.join, here) plugin_base = PluginBase(package='plugins', searchpath=[ os.path.join(os.getcwd(), 'plugins', 'user'), os.path.join(os.getcwd(), 'plugins', 'system') ]) conf = Config() lg = Logger(__file__) class Core(subscriber): def __init__(self, eb): super().__init__() self.eb = eb self.plugins = plugin_base.make_plugin_source(searchpath=[ os.path.join(os.getcwd(), 'plugins', 'user'), os.path.join(os.getcwd(), 'plugins', 'system') ], identifier=__name__) for plugin_name in self.plugins.list_plugins():
cv2.line(image, pt['prev'], pt['curr'], (0, 0, 0), 1) cv2.imwrite('tmp.jpg', image) image = QtGui.QPixmap('tmp.jpg') self.scene.history.append(3) self.scene.addPixmap(image) def save_img(self): if type(self.output_img): fileName, _ = QtWidgets.QFileDialog.getSaveFileName( self, "Save File", QtCore.QDir.currentPath()) cv2.imwrite(fileName + '.jpg', self.output_img) def undo(self): self.scene.undo() def clear(self): self.scene.reset_items() self.scene.reset() if type(self.image): self.scene.addPixmap(self.image) if __name__ == '__main__': config = Config('demo.yaml') os.environ["CUDA_VISIBLE_DEVICES"] = str(config.GPU_NUM) model = Model(config) app = QtWidgets.QApplication(sys.argv) ex = Ex(model, config) sys.exit(app.exec_())
def main(log_dir, dataset_dir, out_dir): models = os.listdir(log_dir) config_path = os.path.join(log_dir, models[0]) config = Config(config_path, "READ") model_path_list = [ os.path.join(log_dir, each, "xgb.model") for each in models ] dataset_fmt = "/data/slowmoyang/QGJets/npz/root_{}_{}/dijet_test_set.npz" datasets = [ dataset_fmt.format(min_pt, min_pt + 100) for min_pt in range(100, 901, 100) ] auc_matrix = [[] for min_pt in range(100, 1000, 100)] for model_idx, model_path in enumerate(model_path_list): clf = load_classifier(model_path) model_min_pt, _ = parse_model_path(model_path) for dataset_path in datasets: _, dataset_min_pt, _ = parse_dataset_path(dataset_path) x, y_true, _ = load_dataset(dataset_path, features=config.feature_names) y_score = clf.predict_proba(x)[:, 1] auc = metrics.roc_auc_score(y_true=y_true, y_score=y_score) auc_matrix[model_idx].append(auc) auc_mat = np.array(auc_matrix, np.float32) print(auc_mat) path_fmt = os.path.join(out_dir, "auc_heatmap.{ext}") np.save(path_fmt.format(ext="npy"), auc_mat) sns.set(font_scale=1.2) fig, ax = plt.subplots() fig.set_figheight(8) fig.set_figwidth(12) labels = [ "{}\n~ {}\n".format(min_pt, min_pt + 100) for min_pt in range(100, 1000, 100) ] sns.palplot(sns.cubehelix_palette()) ax = sns.heatmap(auc_mat.transpose(), annot=True, ax=ax, cmap="coolwarm", vmax=0.9, vmin=0.7, xticklabels=labels, yticklabels=labels, fmt=".3f") ax.set_title("AUC of BDT", fontsize=24) ax.set_ylabel("Test set", fontsize=18) ax.set_xlabel("Training set", fontsize=18) ax.invert_yaxis() fig.savefig(path_fmt.format(ext="png")) fig.savefig(path_fmt.format(ext="pdf"), format="pdf")
DATA_SIZE = args.datasize # demo, small, large BATCH_SIZE = 1 # dummy test = MINDDataset(path.join(DATA_SIZE,'test/news.tsv'), path.join(DATA_SIZE,'test/behaviors.tsv'), 'all_embeddings.vec', 'large', batch_size=BATCH_SIZE, model=args.pretrained_model, subset='test') test.init_news() print('finish loading data', flush = True) # build the model # news_encoder_parameters = {'n_classes': len(train._class2id), 'n_subclasses': len(train._subclass2id), 'class_embedding_dim': 50, 'subclass_embedding_dim': 30, 'news_repr_dim': 400, 'distil_dropout': 0.1, 'class_dropout': 0, 'entity_embedding_dim': 100} news_encoder_parameters = {'n_classes': 19, 'n_subclasses': 294, 'class_embedding_dim': 50, 'subclass_embedding_dim': 30, 'news_repr_dim': 400, 'distil_dropout': 0.1, 'class_dropout': 0, 'entity_embedding_dim': 100} self_attention_hyperparameters['hidden_size'] = news_encoder_parameters['news_repr_dim'] print(self_attention_hyperparameters) self_attention_config = Config(self_attention_hyperparameters) model = NewsRec(self_attention_config, news_encoder_parameters, args.pretrained_model, args.scorer).to(device) if torch.cuda.device_count() > 1: print("Let's use", torch.cuda.device_count(), "GPUs!") model = nn.DataParallel(model) print('finish building the model', flush = True) try: print(args.checkpoint_name) load_checkpoint(model, map_location = device, path = args.checkpoint_name) print('checkpoint loaded', flush = True) except: print('WARNING: failed to load any checkpoints.', flush = True) evaluate(test, model, 1, subset='test')
pass logger_file_handler = logging.FileHandler(logging_file) logger.addHandler(logger_file_handler) logger.info('Arguments: {}'.format(args)) mean, std = get_mean_std(args.dataset) if args.dataset in ['MNIST', 'FashionMNIST']: input_ch = 1 padded_im_size = 32 num_classes = 10 im_size = 28 epc_seed = 0 config = Config(input_ch=input_ch, padded_im_size=padded_im_size, num_classes=num_classes, im_size=im_size, epc_seed=epc_seed ) dataset_sizes = {'train': 6e4, 'test': 1e4} elif args.dataset in ['CIFAR10', 'CIFAR100']: input_ch = 3 padded_im_size = 32 if args.dataset == 'CIFAR10': num_classes = 10 elif args.dataset == 'CIFAR100': num_classes = 100 else: raise Exception('Should not have reached here') im_size = 32 epc_seed = 0 config = Config(input_ch=input_ch,
def __init__(self, client): self.client = client self.config = Config("config.toml")
intersection_group = set(self.items_dict[first_item].keys()).intersection(set(self.items_dict[second_item].keys())) if len(intersection_group) > 1: if (sim > 0): writer.writerow([np.int16(first_item),np.int16(second_item),np.float32(round(sim, 6))]) def build_item_to_itm_corr(self, X): self.user_map = pd.Series(X[USER_COL_NAME_IN_DATAEST].unique()) df_movie_features = X.pivot(index=ITEM_COL_NAME_IN_DATASET, columns=USER_COL_NAME_IN_DATAEST, values=RATING_COL_NAME_IN_DATASET).fillna(0) self.item_map = pd.Series(df_movie_features.index) self.corr_matrix = np.corrcoef(df_movie_features) self.corr_matrix = pd.DataFrame(self.corr_matrix).set_index(self.item_map,self.item_map) self.corr_matrix.columns = [self.item_map][0].values def get_input_item_similarities(self, item): input_item_similarities = self.corr_csv[self.corr_csv['item_1'] == item].set_index('item_2', drop=False).reindex(columns=['item_2','sim']) input_item_similarities.rename(columns={'item_2': 'item'}, inplace=True) input_item_similarities2 = self.corr_csv[self.corr_csv['item_2'] == item].set_index('item_1', drop=False).reindex(columns=['item_1','sim']) input_item_similarities2.rename(columns={'item_1': 'item'}, inplace=True) input_item_similarities_full = pd.concat([input_item_similarities, input_item_similarities2]) input_item_similarities_full.drop_duplicates(inplace=True) return input_item_similarities_full if __name__ == '__main__': knn_config = Config(k=25) train, validation = get_data() knn = KnnItemSimilarity(knn_config) knn.fit(train) print(knn.calculate_rmse(validation))
import click import os import logging import coloredlogs import spiffymanagement from utils import Config log = logging.getLogger('spiffymanagement') coloredlogs.install( level='DEBUG', logger=log, fmt='%(asctime)s:%(msecs)03d %(name)s: %(levelname)s %(message)s') dirp = os.path.dirname(os.path.realpath(__file__)) cfg = Config(f'{dirp}/configs/serverCfgs.toml') @click.group() def spiffy(): cfg._load() @spiffy.command() @click.argument('server') def start(server): spiffymanagement.start(cfg, server) @spiffy.command() @click.argument('server') @click.option('-c', '--countdown')
def setUp(self): super(ToucanConsumerTest, self).setUp() self.cfg = Config() self.consumer = ToucanConsumer(self.cfg)
def test_loading_experiment(self): c = Config(Path.cwd() / 'experiments' / 'sample.yml') self.assertEqual(c.model.bias, True) self.assertEqual(c.training.location, 'aws.google.com') self.assertEqual(c.training.alert.every, '20 minutes')
def test_empty_config_len(self): config = Config() self.assertEqual(len(config), 0)
from torch import device from utils import Config vocab_size = 10000 embedding_dim = 200 config = Config({ "predictor_type": "model.retrival_baseline.Predictor_DualRNNEncoder", "num_epoch": 12, "vocab_size": vocab_size, "save_path": './save/retrival_baseline2.pt' }) config.train_set = Config({ "type": "utterance_with_cands", "kwargs": { "train": True, }, "loader": { "batch_size": 30, "shuffle": True, "num_workers": 8, } }) config.test_set = Config({ "type": "utterance_with_cands", "kwargs": { "train": False, }, "loader": { "batch_size": 1, "shuffle": False,
import networkx as nx import re import pickle import os import torch import yaml from utils import Voc, Config, dump_pickle FILEPATH = '/home/lanco/zhaoliang/KB/en_concept_net_extracted.csv' ROOTPATH = '/home/lanco/zhaoliang/KB/' edgeList = [] errorList = [] nodeList = set() relationList = [] config = Config(os.path.join(ROOTPATH, 'config.yml')) voc = Voc(config) try: with open(FILEPATH, 'r') as file: for index, line in enumerate(file): if index % 100000 == 0: print('processing %d' % index) lineSearch = re.search( "/a/\[/r/(.+)/,/c/en/(.+?)/.*,/c/en/(.+)/\]", line) if lineSearch != None and lineSearch.group( 1) != None and lineSearch.group(2) != None: voc.addWord(lineSearch.group(3)) voc.addWord(lineSearch.group(2)) if lineSearch.group(1) not in relationList: