def __init__(self, labels_num, data_cnf, model_cnf, tree_id=''): self.data_cnf, self.model_cnf = data_cnf.copy(), model_cnf.copy() model_name, data_name = model_cnf['name'], data_cnf['name'] self.model_path = os.path.join(model_cnf['path'], F'{model_name}-{data_name}{tree_id}') self.emb_init, self.level = get_word_emb(data_cnf['embedding']['emb_init']), model_cnf['level'] self.labels_num, self.models = labels_num, {} self.inter_group_size, self.top = model_cnf['k'], model_cnf['top'] self.groups_path = os.path.join(model_cnf['path'], F'{model_name}-{data_name}{tree_id}-cluster')
def main(data_cnf, model_cnf, mode, reg): yaml = YAML(typ='safe') data_cnf, model_cnf = yaml.load(Path(data_cnf)), yaml.load(Path(model_cnf)) model, model_name, data_name = None, model_cnf['name'], data_cnf['name'] model_path = os.path.join(model_cnf['path'], F'{model_name}-{data_name}') emb_init = get_word_emb(data_cnf['embedding']['emb_init']) logger.info(F'Model Name: {model_name}') if mode is None or mode == 'train': logger.info('Loading Training and Validation Set') train_x, train_labels = get_data(data_cnf['train']['texts'], data_cnf['train']['labels']) if 'size' in data_cnf['valid']: random_state = data_cnf['valid'].get('random_state', 1240) train_x, valid_x, train_labels, valid_labels = train_test_split( train_x, train_labels, test_size=data_cnf['valid']['size'], random_state=random_state) else: valid_x, valid_labels = get_data(data_cnf['valid']['texts'], data_cnf['valid']['labels']) mlb = get_mlb(data_cnf['labels_binarizer'], np.hstack((train_labels, valid_labels))) train_y, valid_y = mlb.transform(train_labels), mlb.transform( valid_labels) labels_num = len(mlb.classes_) logger.info(F'Number of Labels: {labels_num}') logger.info(F'Size of Training Set: {len(train_x)}') logger.info(F'Size of Validation Set: {len(valid_x)}') edges = set() if reg: classes = mlb.classes_.tolist() with open(data_cnf['hierarchy']) as fin: for line in fin: data = line.strip().split() p = data[0] if p not in classes: continue p_id = classes.index(p) for c in data[1:]: if c not in classes: continue c_id = classes.index(c) edges.add((p_id, c_id)) logger.info(F'Number of Edges: {len(edges)}') logger.info('Training') train_loader = DataLoader(MultiLabelDataset(train_x, train_y), model_cnf['train']['batch_size'], shuffle=True, num_workers=4) valid_loader = DataLoader(MultiLabelDataset(valid_x, valid_y, training=True), model_cnf['valid']['batch_size'], num_workers=4) model = Model(network=MATCH, labels_num=labels_num, model_path=model_path, emb_init=emb_init, mode='train', reg=reg, hierarchy=edges, **data_cnf['model'], **model_cnf['model']) opt_params = { 'lr': model_cnf['train']['learning_rate'], 'betas': (model_cnf['train']['beta1'], model_cnf['train']['beta2']), 'weight_decay': model_cnf['train']['weight_decay'] } model.train(train_loader, valid_loader, opt_params=opt_params, **model_cnf['train']) # CHANGE: inserted opt_params logger.info('Finish Training') if mode is None or mode == 'eval': logger.info('Loading Test Set') mlb = get_mlb(data_cnf['labels_binarizer']) labels_num = len(mlb.classes_) test_x, _ = get_data(data_cnf['test']['texts'], None) logger.info(F'Size of Test Set: {len(test_x)}') logger.info('Predicting') test_loader = DataLoader(MultiLabelDataset(test_x), model_cnf['predict']['batch_size'], num_workers=4) if model is None: model = Model(network=MATCH, labels_num=labels_num, model_path=model_path, emb_init=emb_init, mode='eval', **data_cnf['model'], **model_cnf['model']) scores, labels = model.predict(test_loader, k=model_cnf['predict'].get('k', 100)) logger.info('Finish Predicting') labels = mlb.classes_[labels] output_res(data_cnf['output']['res'], F'{model_name}-{data_name}', scores, labels)
def main(data_cnf, model_cnf, mode, tree_id): tree_id = F'-Tree-{tree_id}' if tree_id is not None else '' yaml = YAML(typ='safe') data_cnf, model_cnf = yaml.load(Path(data_cnf)), yaml.load(Path(model_cnf)) model, model_name, data_name = None, model_cnf['name'], data_cnf['name'] model_path = os.path.join(model_cnf['path'], F'{model_name}-{data_name}{tree_id}') emb_init = get_word_emb(data_cnf['embedding']['emb_init']) logger.info(F'Model Name: {model_name}') if mode is None or mode == 'train': logger.info('Loading Training and Validation Set') train_x, train_labels = get_data(data_cnf['train']['texts'], data_cnf['train']['labels']) if 'size' in data_cnf['valid']: random_state = data_cnf['valid'].get('random_state', 1240) train_x, valid_x, train_labels, valid_labels = train_test_split(train_x, train_labels, test_size=data_cnf['valid']['size'], random_state=random_state) else: valid_x, valid_labels = get_data(data_cnf['valid']['texts'], data_cnf['valid']['labels']) mlb = get_mlb(data_cnf['labels_binarizer'], np.hstack((train_labels, valid_labels))) train_y, valid_y = mlb.transform(train_labels), mlb.transform(valid_labels) labels_num = len(mlb.classes_) logger.info(F'Number of Labels: {labels_num}') logger.info(F'Size of Training Set: {len(train_x)}') logger.info(F'Size of Validation Set: {len(valid_x)}') logger.info('Training') if 'cluster' not in model_cnf: train_loader = DataLoader(MultiLabelDataset(train_x, train_y), model_cnf['train']['batch_size'], shuffle=True, num_workers=4) valid_loader = DataLoader(MultiLabelDataset(valid_x, valid_y, training=False), model_cnf['valid']['batch_size'], num_workers=4) model = Model(network=AttentionRNN, labels_num=labels_num, model_path=model_path, emb_init=emb_init, **data_cnf['model'], **model_cnf['model']) model.train(train_loader, valid_loader, **model_cnf['train']) else: model = FastAttentionXML(labels_num, data_cnf, model_cnf, tree_id) model.train(train_x, train_y, valid_x, valid_y, mlb) logger.info('Finish Training') if mode is None or mode == 'eval': logger.info('Loading Test Set') mlb = get_mlb(data_cnf['labels_binarizer']) labels_num = len(mlb.classes_) test_x, _ = get_data(data_cnf['test']['texts'], None) logger.info(F'Size of Test Set: {len(test_x)}') logger.info('Predicting') if 'cluster' not in model_cnf: test_loader = DataLoader(MultiLabelDataset(test_x), model_cnf['predict']['batch_size'], num_workers=4) if model is None: model = Model(network=AttentionRNN, labels_num=labels_num, model_path=model_path, emb_init=emb_init, **data_cnf['model'], **model_cnf['model']) scores, labels = model.predict(test_loader, k=model_cnf['predict'].get('k', 100)) else: if model is None: model = FastAttentionXML(labels_num, data_cnf, model_cnf, tree_id) scores, labels = model.predict(test_x) logger.info('Finish Predicting') labels = mlb.classes_[labels] output_res(data_cnf['output']['res'], F'{model_name}-{data_name}{tree_id}', scores, labels)
def build_tree_by_level( sparse_data_x, sparse_data_y, train_x: str, emb_init: str, mlb, indices: np.ndarray, eps: float, max_leaf: int, levels: list, label_emb: str, alg: str, groups_path: str, n_components: int = None, overlap_ratio: float = 0.0, head_split_ratio: float = 0.0, adj_th: int = None, random_state: int = None, ): os.makedirs(os.path.split(groups_path)[0], exist_ok=True) logger.info('Clustering') logger.info('Getting Labels Feature') if label_emb == 'tf-idf': sparse_x, sparse_labels = get_sparse_feature(sparse_data_x, sparse_data_y) with redirect_stderr(None): sparse_y = mlb.transform(sparse_labels) if indices is not None: sparse_x = sparse_x[indices] sparse_y = sparse_y[indices] labels_f = normalize(csr_matrix(sparse_y.T) @ csc_matrix(sparse_x)) elif label_emb == 'glove': emb_init = get_word_emb(emb_init) train_x, train_y = get_data(train_x, sparse_data_y) with redirect_stderr(None): train_y = mlb.transform(train_y) if indices is not None: train_x = train_x[indices] train_y = train_y[indices] labels_f = normalize(_get_labels_f(emb_init, train_x, train_y)) elif label_emb == 'spectral': _, sparse_labels = get_sparse_feature(sparse_data_x, sparse_data_y) sparse_y = mlb.transform(sparse_labels) logger.info('Build label adjacency matrix') adj = sparse_y.T @ sparse_y adj.setdiag(0) adj.eliminate_zeros() if adj_th is not None: logger.info(f"adj th: {adj_th}") ind1 = np.where(adj.data < adj_th) ind2 = np.where(adj.data >= adj_th) adj.data[ind1] = 0 adj.data[ind2] = 1 adj.eliminate_zeros() logger.info( f"Sparsity: {1 - (adj.count_nonzero() / adj.shape[0] ** 2)}") logger.info('Getting spectral embedding') labels_f = spectral_embedding(adj, n_components=n_components, norm_laplacian=adj_th is None, eigen_solver='amg', drop_first=False) labels_f = normalize(labels_f) else: raise ValueError(f"label_emb: {label_emb} is invalid") head_labels = None if head_split_ratio > 0: logger.info(f"head ratio: {head_split_ratio}") train_labels = np.load(sparse_data_y, allow_pickle=True) train_y = mlb.transform(train_labels) counts = np.sum(train_y, axis=0).A1 cnt_indices = np.argsort(counts)[::-1] head_labels = cnt_indices[:int(len(counts) * head_split_ratio)] logger.info(f"# of head labels: {len(head_labels)}") logger.info(f"# of tail labels: {len(counts) - len(head_labels)}") logger.info(F'Start Clustering {levels}') levels, q = [2**x for x in levels], None for i in range(len(levels) - 1, -1, -1): if os.path.exists(F'{groups_path}-Level-{i}.npy'): labels_list = np.load(F'{groups_path}-Level-{i}.npy', allow_pickle=True) q = [(labels_i, labels_f[labels_i]) for labels_i in labels_list] break if q is None: q = [(np.arange(labels_f.shape[0]), labels_f)] while q: labels_list = np.asarray([x[0] for x in q]) assert len(reduce(lambda a, b: a | set(b), labels_list, set())) == labels_f.shape[0] if len(labels_list) in levels: level = levels.index(len(labels_list)) groups = np.asarray(labels_list) a = set(groups[0]) b = set(groups[1]) n_nodes = [len(set(group)) for group in groups] logger.info(F'Finish Clustering Level-{level}') logger.info(f'# of node: {len(a)}, # of overlapped: {len(a & b)}') logger.info(f'max # of node: {max(n_nodes)}') logger.info(f'average # of node: {np.mean(n_nodes)}') if head_labels is not None: logger.info(f"Getting Cluster Centers") if sp.issparse(labels_f): centers = sp.vstack([ normalize(csr_matrix(labels_f[idx].mean(axis=0))) for idx in groups ]) else: centers = np.vstack([ normalize(labels_f[idx].mean(axis=0, keepdims=True)) for idx in groups ]) # Find tail groups # If all labels in a group are not in head labels, # this group is tail group tail_groups = [] for i, group in enumerate(groups): is_tail_group = True for label in group: if label in head_labels: is_tail_group = False break if is_tail_group: tail_groups.append(i) tail_groups = np.array(tail_groups) nearest_head_labels = np.argmax( centers[tail_groups] @ labels_f[head_labels].T, axis=1) if hasattr(nearest_head_labels, 'A1'): nearest_head_labels = nearest_head_labels.A1 for i, tail_group in enumerate(tail_groups): head_label = head_labels[nearest_head_labels[i]] group = groups[tail_group] groups[tail_group] = np.append(groups[tail_group], head_label) np.save(F'{groups_path}-Level-{level}.npy', groups) if level == len(levels) - 1: break else: logger.info(F'Finish Clustering {len(labels_list)}') next_q = [] for node_i, node_f in q: if len(node_i) > max_leaf: next_q += list( split_node(node_i, node_f, eps, alg, overlap_ratio, random_state)) q = next_q logger.info('Finish Clustering')
def main(data_cnf, model_cnf, mode, tree_id, output_suffix, dry_run): if not dry_run: mlflow.start_run() set_seed(tree_id) tree_id = F'-Tree-{tree_id}' if tree_id is not None else '' yaml = YAML(typ='safe') data_cnf_path = data_cnf model_cnf_path = model_cnf data_cnf, model_cnf = yaml.load(Path(data_cnf)), yaml.load(Path(model_cnf)) model, model_name, data_name = None, model_cnf['name'], data_cnf['name'] model_path = os.path.join( model_cnf['path'], F'{model_name}-{data_name}{tree_id}{output_suffix}') emb_init = get_word_emb(data_cnf['embedding']['emb_init']) logger.info(F'Model Name: {model_name}') is_split_head_tail = 'split_head_tail' in data_cnf is_random_forest = 'random_forest' in model_cnf is_spectral_clustering = 'spectral_clustering' in model_cnf is_transformer_model = model_name in TRANSFORMER_MODEL_NAMES if is_split_head_tail: split_ratio = data_cnf['split_head_tail'] head_model = None tail_model = None head_labels = None tail_labels = None elif is_random_forest: num_tree = model_cnf['random_forest']['num'] elif is_spectral_clustering: pass if mode is None or mode == 'train': if is_transformer_model: transformer_train( data_cnf, data_cnf_path, model_cnf, model_cnf_path, model_path, dry_run, ) elif is_split_head_tail: head_model, tail_model, head_labels, tail_labels = splitting_head_tail_train( data_cnf, data_cnf_path, model_cnf, model_cnf_path, emb_init, model_path, tree_id, output_suffix, dry_run, split_ratio, ) elif is_random_forest: random_forest_train( data_cnf, data_cnf_path, model_cnf, model_cnf_path, emb_init, model_path, tree_id, output_suffix, dry_run, num_tree, ) elif is_spectral_clustering: spectral_clustering_train( data_cnf, data_cnf_path, model_cnf, model_cnf_path, emb_init, model_path, tree_id, output_suffix, dry_run, ) else: default_train( data_cnf, data_cnf_path, model_cnf, model_cnf_path, emb_init, model_path, tree_id, output_suffix, dry_run, ) log_tag(dry_run, model_name, data_name, output_suffix) if mode is None or mode == 'eval': if is_transformer_model: transformer_eval( data_cnf, model_cnf, data_name, model_name, model_path, tree_id, output_suffix, dry_run, ) elif is_split_head_tail: splitting_head_tail_eval( data_cnf, model_cnf, data_name, model_name, model_path, emb_init, tree_id, output_suffix, dry_run, split_ratio, head_labels, tail_labels, head_model, tail_model, ) elif is_random_forest: random_forest_eval( data_cnf, model_cnf, data_name, model_name, model_path, emb_init, tree_id, output_suffix, dry_run, num_tree, ) elif is_spectral_clustering: spectral_clustering_eval( data_cnf, model_cnf, data_name, model_name, model_path, emb_init, tree_id, output_suffix, dry_run, ) else: default_eval( data_cnf, model_cnf, data_name, model_name, model_path, emb_init, tree_id, output_suffix, dry_run, )
def main(data_cnf, model_cnf, mode): model_name = os.path.split(model_cnf)[1].split(".")[0] # 設定log檔案位置 logfile("./logs/logfile_" + model_name + ".log") yaml = YAML(typ='safe') data_cnf, model_cnf = yaml.load(Path(data_cnf)), yaml.load(Path(model_cnf)) model, model_name, data_name = None, model_cnf['name'], data_cnf['name'] # model_path = model_cnf['path'] + "/" + model_cnf['name'] + '.h' model_path = r'E:\\PycharmProject\\CorNet\\' + model_name + '.h5' emb_init = get_word_emb(data_cnf['embedding']['emb_init']) logger.info(F'Model Name: {model_name}') # keras log file csv_logger = CSVLogger('./logs/' + model_name + '_log.csv', append=True, separator=',') if mode is None or mode == 'train': logger.info('Loading Training and Validation Set') train_x, train_labels = get_data(data_cnf['train']['texts'], data_cnf['train']['labels']) if 'size' in data_cnf['valid']: random_state = data_cnf['valid'].get('random_state', 1240) train_x, valid_x, train_labels, valid_labels = train_test_split(train_x, train_labels, test_size=data_cnf['valid']['size'], random_state=random_state) else: valid_x, valid_labels = get_data(data_cnf['valid']['texts'], data_cnf['valid']['labels']) mlb = get_mlb(data_cnf['labels_binarizer'], np.hstack((train_labels, valid_labels))) train_y, valid_y = mlb.transform(train_labels), mlb.transform(valid_labels) labels_num = len(mlb.classes_) logger.info(F'Number of Labels: {labels_num}') logger.info(F'Size of Training Set: {len(train_x)}') logger.info(F'Size of Validation Set: {len(valid_x)}') vocab_size = emb_init.shape[0] emb_size = emb_init.shape[1] # 可調參數 data_num = len(train_x) ks = 3 output_channel = model_cnf['model']['num_filters'] dynamic_pool_length = model_cnf['model']['dynamic_pool_length'] num_bottleneck_hidden = model_cnf['model']['bottleneck_dim'] drop_out = model_cnf['model']['dropout'] cornet_dim = model_cnf['model']['cornet_dim'] nb_cornet_block = model_cnf['model'].get('nb_cornet_block', 0) nb_epochs = model_cnf['train']['nb_epoch'] batch_size = model_cnf['train']['batch_size'] max_length = 500 input_tensor = Input(batch_shape=(batch_size, max_length), name='input') emb_data = Embedding(input_dim=vocab_size, output_dim=emb_size, input_length=max_length, weights=[emb_init], trainable=False, name='embedding1')(input_tensor) emb_data.trainable = False # emd_out_4d = keras.layers.core.RepeatVector(1)(emb_data) # unsqueeze_emb_data = tf.keras.layers.Reshape((1, 500, 300), input_shape=(500, 300))(emb_data) # emb_data = tf.expand_dims(emb_data, axis=1) # emb_data = Lambda(reshape_tensor, arguments={'shape': (1, max_length, 300)}, name='lambda1')( # emb_data) conv1_output = Convolution1D(output_channel, 2, padding='same', kernel_initializer=keras.initializers.glorot_uniform(seed=None), activation='relu', name='conv1')(emb_data) # conv1_output = Lambda(reshape_tensor, arguments={'shape': (batch_size, max_length, output_channel)}, # name='conv1_lambda')( # conv1_output) conv2_output = Convolution1D(output_channel, 4, padding='same', kernel_initializer=keras.initializers.glorot_uniform(seed=None), activation='relu', name='conv2')(emb_data) # conv2_output = Lambda(reshape_tensor, arguments={'shape': (batch_size, max_length, output_channel)}, # name='conv2_lambda')( # conv2_output) conv3_output = Convolution1D(output_channel, 8, padding='same', kernel_initializer=keras.initializers.glorot_uniform(seed=None), activation='relu', name='conv3')(emb_data) # conv3_output = Lambda(reshape_tensor, arguments={'shape': (batch_size, max_length, output_channel)}, # name='conv3_lambda')( # conv3_output) # pool1 = adapmaxpooling(conv1_output, dynamic_pool_length) pool1 = GlobalMaxPooling1D(name='globalmaxpooling1')(conv1_output) pool2 = GlobalMaxPooling1D(name='globalmaxpooling2')(conv2_output) pool3 = GlobalMaxPooling1D(name='globalmaxpooling3')(conv3_output) output = concatenate([pool1, pool2, pool3], axis=-1) # output = Dense(num_bottleneck_hidden, activation='relu',name='bottleneck')(output) output = Dropout(drop_out, name='dropout1')(output) output = Dense(labels_num, activation='softmax', name='dense_final', kernel_initializer=keras.initializers.glorot_uniform(seed=None))(output) if nb_cornet_block > 0: for i in range(nb_cornet_block): x_shortcut = output x = keras.layers.Activation('sigmoid', name='cornet_sigmoid_{0}'.format(i + 1))(output) x = Dense(cornet_dim, kernel_initializer='glorot_uniform', name='cornet_1st_dense_{0}'.format(i + 1))(x) # x = Dense(cornet_dim, kernel_initializer=keras.initializers.glorot_uniform(seed=None), # activation='sigmoid', name='cornet_1st_dense_{0}'.format(i + 1))(output) x = keras.layers.Activation('elu', name='cornet_elu_{0}'.format(i + 1))(x) x = Dense(labels_num, kernel_initializer='glorot_uniform', name='cornet_2nd_dense_{0}'.format(i + 1))(x) # x = Dense(labels_num, kernel_initializer=keras.initializers.glorot_uniform(seed=None), activation='elu', # name='cornet_2nd_dense_{0}'.format(i + 1))(x) output = Add()([x, x_shortcut]) model = Model(input_tensor, output) model.summary() model.compile(optimizer='adam', loss='binary_crossentropy', metrics=[tf.keras.metrics.Precision(top_k=5)]) # model.compile(optimizer='adam', loss='binary_crossentropy', metrics=[tf.keras.metrics.top_k_categorical_accuracy(k=5)]) model.fit_generator(steps_per_epoch=data_num / batch_size, generator=batch_generator(train_x, train_y, batch_size), validation_data=batch_generator(valid_x, valid_y, batch_size), validation_steps=valid_x.shape[0] / batch_size, nb_epoch=nb_epochs, callbacks=[csv_logger]) model.save(model_path) elif mode is None or mode == 'eval': logger.info('Loading Training and Validation Set') train_x, train_labels = get_data(data_cnf['train']['texts'], data_cnf['train']['labels']) if 'size' in data_cnf['valid']: # 如果有設定valid的size 則直接使用train的一部分作為valid random_state = data_cnf['valid'].get('random_state', 1240) train_x, valid_x, train_labels, valid_labels = train_test_split(train_x, train_labels, test_size=data_cnf['valid']['size'], random_state=random_state) else: valid_x, valid_labels = get_data(data_cnf['valid']['texts'], data_cnf['valid']['labels']) mlb = get_mlb(data_cnf['labels_binarizer'], np.hstack((train_labels, valid_labels))) train_y, valid_y = mlb.transform(train_labels), mlb.transform(valid_labels) labels_num = len(mlb.classes_) ################################################################################################## logger.info('Loading Test Set') logger.info('model path: ', model_path) mlb = get_mlb(data_cnf['labels_binarizer']) labels_num = len(mlb.classes_) test_x, test_label = get_data(data_cnf['test']['texts'], data_cnf['test']['labels']) logger.info(F'Size of Test Set: {len(test_x)}') test_y = mlb.transform(test_label).toarray() model = tf.keras.models.load_model(model_path) score = model.predict(test_x) print("p5: ", p5(test_y, score))
def main(data_cnf, model_cnf, mode): model_name = os.path.split(model_cnf)[1].split(".")[0] yaml = YAML(typ='safe') data_cnf, model_cnf = yaml.load(Path(data_cnf)), yaml.load(Path(model_cnf)) # 設定log檔案位置 logfile("./logs/logfile_{0}_cornet_{1}_cornet_dim_{2}.log".format( model_name, model_cnf['model']['n_cornet_blocks'], model_cnf['model']['cornet_dim'])) model, model_name, data_name = None, model_cnf['name'], data_cnf['name'] model_path = os.path.join( model_cnf['path'], F'{model_name}-{data_name}-{model_cnf["model"]["n_cornet_blocks"]}-{model_cnf["model"]["cornet_dim"]}' ) emb_init = get_word_emb(data_cnf['embedding']['emb_init']) logger.info(F'Model Name: {model_name}') # summary(model_dict[model_name]) if mode is None or mode == 'train': logger.info('Loading Training and Validation Set') train_x, train_labels = get_data(data_cnf['train']['texts'], data_cnf['train']['labels']) if 'size' in data_cnf['valid']: random_state = data_cnf['valid'].get('random_state', 1240) train_x, valid_x, train_labels, valid_labels = train_test_split( train_x, train_labels, test_size=data_cnf['valid']['size'], random_state=random_state) else: valid_x, valid_labels = get_data(data_cnf['valid']['texts'], data_cnf['valid']['labels']) mlb = get_mlb(data_cnf['labels_binarizer'], np.hstack((train_labels, valid_labels))) train_y, valid_y = mlb.transform(train_labels), mlb.transform( valid_labels) labels_num = len(mlb.classes_) logger.info(F'Number of Labels: {labels_num}') logger.info(F'Size of Training Set: {len(train_x)}') logger.info(F'Size of Validation Set: {len(valid_x)}') logger.info('Training') train_loader = DataLoader(MultiLabelDataset(train_x, train_y), model_cnf['train']['batch_size'], shuffle=True, num_workers=4) valid_loader = DataLoader(MultiLabelDataset(valid_x, valid_y, training=True), model_cnf['valid']['batch_size'], num_workers=4) if 'gpipe' not in model_cnf: model = Model(network=model_dict[model_name], labels_num=labels_num, model_path=model_path, emb_init=emb_init, **data_cnf['model'], **model_cnf['model']) else: model = GPipeModel(model_name, labels_num=labels_num, model_path=model_path, emb_init=emb_init, **data_cnf['model'], **model_cnf['model']) loss, p1, p5 = model.train(train_loader, valid_loader, **model_cnf['train']) np.save( model_cnf['np_loss'] + "{0}_cornet_{1}_cornet_dim_{2}.npy".format( model_name, model_cnf['model']['n_cornet_blocks'], model_cnf['model']['cornet_dim']), loss) np.save( model_cnf['np_p1'] + "{0}_cornet_{1}_cornet_dim_{2}.npy".format( model_name, model_cnf['model']['n_cornet_blocks'], model_cnf['model']['cornet_dim']), p1) np.save( model_cnf['np_p5'] + "{0}_cornet_{1}_cornet_dim_{2}.npy".format( model_name, model_cnf['model']['n_cornet_blocks'], model_cnf['model']['cornet_dim']), p5) logger.info('Finish Training') if mode is None or mode == 'eval': logger.info('Loading Test Set') logger.info('model path: ', model_path) mlb = get_mlb(data_cnf['labels_binarizer']) labels_num = len(mlb.classes_) test_x, _ = get_data(data_cnf['test']['texts'], None) logger.info(F'Size of Test Set: {len(test_x)}') logger.info('Predicting') test_loader = DataLoader(MultiLabelDataset(test_x), model_cnf['predict']['batch_size'], num_workers=4) if 'gpipe' not in model_cnf: if model is None: model = Model(network=model_dict[model_name], labels_num=labels_num, model_path=model_path, emb_init=emb_init, **data_cnf['model'], **model_cnf['model']) else: if model is None: model = GPipeModel(model_name, labels_num=labels_num, model_path=model_path, emb_init=emb_init, **data_cnf['model'], **model_cnf['model']) scores, labels = model.predict(test_loader, k=model_cnf['predict'].get('k', 3801)) logger.info('Finish Predicting') labels = mlb.classes_[labels] output_res(data_cnf['output']['res'], F'{model_name}-{data_name}', scores, labels)