def loadChkpt(network, optimizer, model, dev='cpu', root='.', params=hyperparams.Hyperparams(), best=0): chkptDirNm = "|".join("{}:{}".format(k, v) for k, v in params.paramDict.items()) chkptDir = os.path.join(root, chkptDirNm) lastNm = '{}Last.{}.chkpt.tar'.format(model, modelHash(network)) bestNm = '{}Best.{}.chkpt.tar'.format(model, modelHash(network)) loadPaths = [os.path.join(chkptDir, lastNm), os.path.join(root, lastNm)] if best: loadPaths += [ os.path.join(chkptDir, bestNm), os.path.join(root, bestNm) ] print('HYPERPARAMS', chkptDirNm) for path in loadPaths: if not os.path.exists(path): print('PATH DOES NOT EXIST:', path) continue state = ch.load(path, map_location=dev) network.load_state_dict(state['modelState']) if optimizer: optimizer.load_state_dict(state['optimizerState']) # if len(state['lossHist']) > 10: plt.plot(state['lossHist']) print("LOADED EPOCH {}, LOSS {}, BEST LOSS {} FROM {}".format( state['epoch'], state['lossHist'][-1] if state['lossHist'] else float('inf'), state['bestLoss'], path)) return state['epoch'], state['lossHist'], state['bestLoss'] return 0, [], float('inf')
def __init__(self, o, i, k, d, causal, params=hyperparams.Hyperparams(), stride=1): assert o == i super(HC, self).__init__() self.o = o self.conv = C(2 * o, i, k, d, causal, params, stride)
def __init__(self, params=hyperparams.Hyperparams()): super(AudioDec, self).__init__() s = 1 # causal d, F = params.d, params.F layers = [C(d, 2 * d, 1, 1, s, params)] ldfs = reversed(range(4)) if params.reversedDilation else range(4) ldfs = list(ldfs) for _ in range(1): #? layers += [HC(d, d, 3, 3**ldf, s, params) for ldf in ldfs] layers += [HC(d, d, 3, 1, s, params) for _ in range(2)] for _ in range(3): layers += [C(d, d, 1, 1, s, params), ch.nn.ReLU()] layers += [C(F, d, 1, 1, s, params), ch.nn.Sigmoid()] self.seq = ch.nn.Sequential(*layers)
def __init__(self, o, i, k, d, params=hyperparams.Hyperparams(), causal=0, s=2): super(D, self).__init__() self.tconv = ch.nn.ConvTranspose1d(out_channels=o, in_channels=i, kernel_size=k, dilation=d, stride=s) ch.nn.init.kaiming_normal_(self.tconv.weight.data)
def __init__(self, params=hyperparams.Hyperparams(), *args, **kwargs): super(SSRN, self).__init__(*args, **kwargs) self.params = params s = 0 # non causal c, F, Fp = params.c, params.F, params.Fp layers = [C(c, F, 1, 1, s)] for _ in range(1): #? layers += [HC(c, c, 3, 1, s), HC(c, c, 3, 3, s)] for _ in range(2): layers += [D(c, c, 2, 1), HC(c, c, 3, 1, s), HC(c, c, 3, 3, s)] layers += [C(2 * c, c, 1, 1, s)] layers += [HC(2 * c, 2 * c, 3, 1, s) for _ in range(2)] layers += [C(Fp, 2 * c, 1, 1, s)] for _ in range(2): layers += [C(Fp, Fp, 1, 1, s), ch.nn.ReLU()] layers += [C(Fp, Fp, 1, 1, s), ch.nn.Sigmoid()] self.seq = ch.nn.Sequential(*layers)
def __init__(self, params=hyperparams.Hyperparams()): super(AudioEnc, self).__init__() c = 1 # causal d, F = params.d, params.F layers = [ C(d, F, 1, 1, c, params), ch.nn.ReLU(), C(d, d, 1, 1, c, params), ch.nn.ReLU(), C(d, d, 1, 1, c, params) ] ldfs = reversed(range(4)) if params.reversedDilation else range(4) ldfs = list(ldfs) for _ in range(2): layers += [HC(d, d, 3, 3**ldf, c, params) for ldf in ldfs] layers += [HC(d, d, 3, 3, c, params) for _ in range(2)] self.seq = ch.nn.Sequential(*layers)
def __init__(self, root='../LJSpeech-1.1', ttmel=1, params=hyperparams.Hyperparams()): self.ttmel = ttmel self.params = params self.csvpath = os.path.join(root, 'metadata.csv') self.wavdir = os.path.join(root, 'wavs') with open(self.csvpath) as F: lines = F.read().split('\n') split = [l.split('|') for l in lines] split = [s for s in split if len(s) == 3] split = [(duid, _, dtxt.lower()) for duid, _, dtxt in split] self.valid = [(duid, _, dtxt) for duid, _, dtxt in split if sum(c in params.c2i for c in dtxt) == len(dtxt)] self.valid = np.array(self.valid)
def __init__(self, params=hyperparams.Hyperparams()): super(TextEnc, self).__init__() c = 0 # non causal d, e, alphabet = params.d, params.e, params.alphabet self.embed = ch.nn.Embedding(len(alphabet), e) ch.nn.init.kaiming_normal_(self.embed.weight.data) layers = [ C(2 * d, e, 1, 1, c, params), ch.nn.ReLU(), C(2 * d, 2 * d, 1, 1, c, params) ] ldfs = reversed(range(4)) if params.reversedDilation else range(4) ldfs = list(ldfs) for _ in range(2): layers += [HC(2 * d, 2 * d, 3, 3**ldf, c, params) for ldf in ldfs] layers += [HC(2 * d, 2 * d, 3, 1, c, params) for _ in range(2)] layers += [HC(2 * d, 2 * d, 1, 1, c, params) for _ in range(2)] self.seq = ch.nn.Sequential(*layers)
def saveChkpt(network, state, model, root='.', params=hyperparams.Hyperparams()): chkptDirNm = "|".join("{}:{}".format(k, v) for k, v in params.paramDict.items()) chkptDir = os.path.join(root, chkptDirNm) lastNm = '{}Last.{}.chkpt.tar'.format(model, modelHash(network)) bestNm = '{}Best.{}.chkpt.tar'.format(model, modelHash(network)) if not os.path.exists(chkptDir): os.makedirs(chkptDir) savePaths = [os.path.join(chkptDir, lastNm), os.path.join(root, lastNm)] # found copying files using shutil unreliable if state['lossHist'] and state['lossHist'][-1] <= state['bestLoss']: savePaths += [ os.path.join(chkptDir, bestNm), os.path.join(root, bestNm) ] [ch.save(state, path) for path in savePaths] print("HYPERPARAMS", chkptDirNm) print("SAVED EPOCH {}, LOSS {}, BEST LOSS {} TO {}".format( state['epoch'], state['lossHist'][-1] if state['lossHist'] else float('inf'), state['bestLoss'], savePaths))
import shutil import random import hyperparams as hy # solve default encoding problem from imp import reload defaultencoding = 'utf-8' if sys.getdefaultencoding() != defaultencoding: reload(sys) sys.setdefaultencoding(defaultencoding) # random seed torch.manual_seed(hy.seed_num) random.seed(hy.seed_num) # init hyperparams instance hyperparams = hy.Hyperparams() parser = argparse.ArgumentParser( description="Text Classification for sentence level.") # Data path parser.add_argument('-train_path', type=str, default=hyperparams.train_path, help='train data path') parser.add_argument('-dev_path', type=str, default=hyperparams.dev_path, help='dev data path') parser.add_argument('-test_path', type=str, default=hyperparams.test_path,
import pdb import tensorflow as tf import numpy as np import train_utils as tr_utils # import plotting as plot import hyperparams import cnn_model import dataloader import ldataloader import settings h = hyperparams.Hyperparams() hyperparams = h.getworkingHyperparams() TRAIN_FOLDS = [1,2,3,4,5,6] train_loader = Da('train', h.LABEL_MODE, TRAIN_FOLDS, h.TRAIN_SCENES, ldl_timesteps=settings.ldl_timesteps, ldl_blocks_per_batch=hyperparams["ldl_blocks_per_batch"], ldl_overlap=settings.ldl_overlap, epochs=epochs, features=h.NFEATURES, classes=h.NCLASSES) print("Filimport os
def test_ldl(ldl_buffer_rows): h = hyperparams.Hyperparams() hyp = h.getworkingHyperparams() import time from tqdm import tqdm TRAIN_FOLDS = [1, 2, 3, 4, 5, 6] train_loader = ldataloader.LineDataLoader('train', h.LABEL_MODE, TRAIN_FOLDS, h.TRAIN_SCENES, ldl_timesteps=settings.ldl_timesteps, ldl_blocks_per_batch=hyp["ldl_blocks_per_batch"], ldl_overlap=settings.ldl_overlap, ldl_buffer_rows=ldl_buffer_rows, epochs=1, features=h.NFEATURES, classes=h.NCLASSES, seed_by_epoch=False, seed=time.time()) _, d_timesteps, __ = test_compare.measure_directly(train_loader) batches = 0 while (True): batches = batches + 1 _x, _y = train_loader.next_batch() if _x is None: break ''' print("batches: " + str(batches)) print("Memory:" + str(train_loader.buffer_add_memory)) print("Timestepsy:" + str(train_loader.buffer_add_timesteps)) print("Buffer Iterations:" + str(train_loader.buffer_add_iterations)) #print(train_loader.buffer_x.shape) print(train_loader.buffer_x.shape) print("Buffer Timesteps:" + str(train_loader.buffer_add_timesteps)) print("Real Timesteps:" + str(d_timesteps)) print("bad:") print(train_loader.bad_sliced_lines) print("good") print(train_loader.good_sliced_lines) print("lin self.count_line_positions_taken_total" + str(train_loader.count_line_positions_taken_total)) print("max position sum" + str(train_loader.add_max_positions)) print("slices y" + str(train_loader.check_slices_y)) print("slices x" + str(train_loader.check_slices_xy)) print("count batches: " + str(train_loader.count_batches)) ''' slices_with_all_data = (d_timesteps / train_loader.buffer_x.shape[0]) /25 slices_in_buffer = train_loader.add_max_positions / 25 loss_due_to_buffer = 1 - (slices_in_buffer/slices_with_all_data) loss_due_to_rejeceted_data = (train_loader.bad_sliced_lines / slices_in_buffer) #print(slices_with_all_data) #print(loss_due_to_buffer) #print(loss_due_to_rejeceted_data) return slices_with_all_data, loss_due_to_buffer, loss_due_to_rejeceted_data
delimiter=';', fieldnames=fieldnames) if not file_exists: writer.writeheader() writer.writerow(dict) def print_log(self): for i in range(1, np.max(self.h.nr_conv_layers) + 1): fieldnames, dict = self._nth_conv_layer(i) self.create_csv_nth_layer(i, fieldnames, dict) for i in range(100): h = hp.Hyperparams() hyperparams = h.getworkingHyperparams() logger = Logger(hyperparams, h) y = np.ones((4, 13)) x = np.ones((4, 160, 49, 1)) with tf.Graph().as_default() as g: graphModel = cnn_model.GraphModel(hyperparams, logger) ''' with tf.Session(graph=g) as sess: init = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer()) sess.run(init) out = sess.run([graphModel.thresholded],feed_dict={graphModel.x: x, graphModel.y: y}) '''
def __init__(self, o, i, k, d, causal, params=hyperparams.Hyperparams(), stride=1): super(C, self).__init__() self.causal = causal self.params = params assert (k - 1) % 2 == 0 assert k > 0 if causal: self.pad = (k - 1) * d else: # print('filter',k,'dilation',d,'total pad',(k-1)*d,'half pad',(k-1)*d//2) self.pad = (k - 1) * d // 2 self.dilation = d self.k = k self.o = o self.i = i layers = [] if params.dropout: layers += [ch.nn.Dropout(p=params.dropout)] # have norm before convolution b/c many convolutions double # of channels # layer norm over sequence - pretty questionable b/c of padding # and different data distribution (also length) when generating # also I think layer norm requires size of layer?? to be known in advance # does it make sense to normalize over a small spatial window? # probably have speedup by doing layer norm first # if params.norm == 3: # layers += [ch.nn.LayerNorm(( # layer norm over channel if params.norm == 2: self.norm = ch.nn.LayerNorm((i, )) layers += [Transpose(2, 1), self.norm, Transpose(2, 1)] # batch norm over channel elif params.norm == 1: self.norm = ch.nn.BatchNorm1d(num_features=i) layers += [self.norm] if params.sep == 3: # and o == i: sqz = 4 self.reduce = ch.nn.Conv1d(out_channels=i // sqz, in_channels=i, kernel_size=1) self.conv = ch.nn.Conv1d(out_channels=o // sqz, in_channels=i // sqz, kernel_size=k, dilation=d, stride=stride, padding=self.pad) self.expand = ch.nn.Conv1d(out_channels=o, in_channels=o // sqz, kernel_size=1) ch.nn.init.kaiming_normal_(self.reduce.weight.data) ch.nn.init.kaiming_normal_(self.conv.weight.data) ch.nn.init.kaiming_normal_(self.expand.weight.data) layers += [self.reduce, self.conv, self.expand] elif params.sep in (1, 2) and k > 1: g = 4 if params.sep == 2 and o % g == 0 and i % g == 0: chanGroups = g else: chanGroups = 1 self.depthwise = ch.nn.Conv1d(out_channels=i, in_channels=i, kernel_size=k, dilation=d, stride=stride, padding=self.pad, groups=i) self.pointwise = ch.nn.Conv1d(out_channels=o, in_channels=i, kernel_size=1, groups=chanGroups) ch.nn.init.kaiming_normal_(self.depthwise.weight.data) ch.nn.init.kaiming_normal_(self.pointwise.weight.data) layers += [self.depthwise, self.pointwise] else: self.conv = ch.nn.Conv1d(out_channels=o, in_channels=i, kernel_size=k, dilation=d, stride=stride, padding=self.pad) ch.nn.init.kaiming_normal_(self.conv.weight.data) layers += [self.conv] if self.causal and self.pad: layers += [Slice(dim=-1, end=-self.pad)] self.block = ch.nn.Sequential(*layers)
def __init__(self, params=hyperparams.Hyperparams(), *args, **kwargs): super(Text2Mel, self).__init__(*args, **kwargs) self.params = params self.textEnc = TextEnc(params) self.audioEnc = AudioEnc(params) self.audioDec = AudioDec(params)
import json sys.path.append(os.path.join(os.path.dirname(__file__),"..")) sys.path.append(os.path.join(os.path.dirname(__file__),"../..")) sys.path.append(os.path.join(os.path.dirname(__file__),"../../..")) import data_helper as dh import hyperparams import Modeling from CommonLibs import ModelUtils from CommonLibs import Metrix from CommonLibs import OtherUtils prj_name = sys.argv[1] hp = hyperparams.Hyperparams(prj_name) def train(): # 初始化日志和模型路径 OtherUtils.initPaths(hp.train_params["model_path"], hp.train_params["log_path"]) # 初始化输入文件 train_data_helper = dh.TrainDataHelper(hp.model_params["max_seq_len"]) train_data_helper.initialize() vocab_size = train_data_helper.get_vocab_size() # 词汇量大小 train_datas = train_data_helper.read_input_file(hp.train_params["train_file"], type="train") train_data_size = len(train_datas) valid_datas = train_data_helper.read_input_file(hp.train_params["valid_file"], type="valid") valid_data_size = len(valid_datas)