def __call__(self, inputs, state): """Most basic RNN: input is [bs, dim] output is [bs, dim] """ logger.debug(self.name) with tf.variable_scope(self.name) as scope: dim = inputs.get_shape()[1] shift = _variable_on_cpu('shift', dim, initializer=tf.zeros_initializer()) scale = _variable_on_cpu('scale', dim, initializer=tf.ones_initializer()) logger.debug(shift.name) batch_mean, batch_var = tf.nn.moments(inputs, [0], name='moments') ema = tf.train.ExponentialMovingAverage(decay=0.5) if self.is_train: ema_apply_op = ema.apply([batch_mean, batch_var]) with tf.control_dependencies([ema_apply_op]): mean = tf.identity(batch_mean) var = tf.identity(batch_var) else: mean, var = lambda: (ema.average(batch_mean), ema.average(batch_var)) scope.reuse_variables() bn = tf.nn.batch_normalization(inputs, mean, var, shift, scale, self.eps) return bn, bn
def __init__(self, num_units, activation=tf.nn.relu6, skip_input=True, use_fp16=False, name=None): self._num_units = num_units self.use_fp16 = use_fp16 self.name = name self.skip_input = skip_input logger.debug(self.name)
def conv_layer(x, input_channels, num_kernels, kernel_size, stride_size, data_format, padding='VALID', with_bias=True, name=None): """conv_layer returns a 2d convolution layer. param: input_channels, num_kernels and kernel_size are required size of kernel and stride are list as: [height, width] data_format: "NHWC" or "NCHW" padding refer to tf.nn.conv2d padding """ assert input_channels is not None and num_kernels is not None assert isinstance(kernel_size, list) and len(kernel_size) == 2 assert isinstance(stride_size, list) and len(stride_size) == 2 assert isinstance(name, str) assert data_format in ["NHWC", "NCHW"] ic = input_channels # outpout channel oc = num_kernels kh, kw = kernel_size sh, sw = stride_size if data_format == "NHWC": stride = [1, sh, sw, 1] else: stride = [1, 1, sh, sw] logger.debug(name) with tf.name_scope(name): wgt = weight_variable('param_weight', [kh, kw, ic, oc]) DEBUG(variable_summaries(wgt)) DEBUG(tf.summary.histogram('pre_' + name, x)) conv = tf.nn.conv2d(x, wgt, stride, padding, data_format=data_format, name=name) DEBUG(tf.summary.histogram('post_' + name, conv)) if with_bias: bias = bias_variable('param_bias', [oc]) DEBUG(variable_summaries(bias)) conv = tf.nn.bias_add(conv, bias, data_format, 'add_bias') DEBUG(tf.summary.histogram('post_bias', conv)) return conv
def __init__(self, num_units, activation=tf.nn.relu6, is_train=True, eps=1e-5, use_fp16=False, name=None): self._num_units = num_units self.use_fp16 = use_fp16 self.eps = eps self.name = name self.is_train = is_train logger.debug(self.name)
def feed_dict(): '''feed inputs for ds2 ''' start_time = time.time() dat, lbl_ind, lbl_val, lbl_shp, utt = dataset.next_batch( ARGS.batch_size) duration = (time.time() - start_time) * 1000 logger.debug('reading data %.3f ms' % duration) return { input_data: dat, input_label_indice: lbl_ind, input_label_value: lbl_val, input_label_shape: lbl_shp, input_utt_lens: utt }
def main(_): set_debug_mode(ARGS.debug) logger.debug(ARGS) if tf.gfile.Exists(ARGS.log_dir): tf.gfile.DeleteRecursively(ARGS.log_dir) tf.gfile.MakeDirs(ARGS.log_dir) if not tf.gfile.Exists(ARGS.checkpoint_dir): tf.gfile.MakeDirs(ARGS.checkpoint_dir) os.environ["KMP_BLOCKTIME"] = "1" os.environ["KMP_SETTINGS"] = "1" os.environ["OMP_NUM_THREADS"] = "32" os.environ["MKL_NUM_THREADS"] = "32" os.environ["OMP_DYNAMIC"] = "false" os.environ["KMP_AFFINITY"] = "granularity=fine,verbose,compact,1,0" train_loop()
def seq_batch_norm(x, dim, seq_lens, eps=1e-5, name=None, is_train=True): '''sequence batch normalization input is [seq, bs, dim] ''' assert len(x.get_shape()) == 3, 'input should be 3-D' dtype = INFERENCE_DTYPE logger.debug(dtype) with tf.variable_scope(name) as scope: bncell = SeqBNCell(num_units=dim) #cell = tf.contrib.rnn.CompiledWrapper(bncell) #cell = tf.contrib.rnn.RNNCell(bncell) cell = tf.contrib.rnn.MultiRNNCell([bncell]) seqbn_output, _ = tf.nn.dynamic_rnn( cell, x, sequence_length=seq_lens, dtype=dtype, time_major=True, # input is [seq, bs, dim] swap_memory=True) return seqbn_output
def __call__(self, inputs, state): """Most basic RNN: input shape is [bs, dim] state shape is [bs, dim] output shaep is [bs, dim] """ logger.debug(self.name) assert self.skip_input, 'only support skip input mode yet' with tf.variable_scope(self.name): #print "rnn cell input size: ", inputs.get_shape().as_list() #print "rnn cell state size: ", state.get_shape().as_list() u_ = _variable_on_cpu('U', [self._num_units, self._num_units], initializer=tf.zeros_initializer(), use_fp16=self.use_fp16) resu = tf.matmul(state, u_) b_ = _variable_on_cpu('B', [self._num_units], tf.constant_initializer(0), use_fp16=self.use_fp16) output = relu_layer(resu + b_, capping=20) return output, output
def fc_layer(x, dim_out, dim_in=None, with_bias=True, name=None): '''fc_layer returns a full connected layer Wx + b param: if dim_in is None, will set as dim_out ''' logger.debug(name) if dim_in is None: dim_in = dim_out with tf.name_scope(name): DEBUG(tf.summary.histogram('pre_' + name, x)) x = tf.reshape(x, [-1, dim_in]) wgt = weight_variable('weight', [dim_in, dim_out]) DEBUG(variable_summaries(wgt)) fc_ = tf.matmul(x, wgt) DEBUG(tf.summary.histogram('post_' + name, fc_)) if with_bias: bias = bias_variable('bias', [dim_out]) DEBUG(variable_summaries(bias)) fc_ = tf.add(fc_, bias, 'add_bias') DEBUG(tf.summary.histogram('post_bias' + name, fc_)) return fc_
def train_loop(): '''ds2 train loop ''' def feed_dict(): '''feed inputs for ds2 ''' start_time = time.time() dat, lbl_ind, lbl_val, lbl_shp, utt = dataset.next_batch( ARGS.batch_size) duration = (time.time() - start_time) * 1000 logger.debug('reading data %.3f ms' % duration) return { input_data: dat, input_label_indice: lbl_ind, input_label_value: lbl_val, input_label_shape: lbl_shp, input_utt_lens: utt } with tf.name_scope('inputs'): # defulat input data shape is [bs, seq, 161] input_data = tf.placeholder(dtype=tf.float32, shape=[None, None, Dataset.freq_bins]) input_label_indice = tf.placeholder(dtype=tf.int64, shape=[None, None]) input_label_value = tf.placeholder(dtype=tf.int32, shape=[None]) input_label_shape = tf.placeholder(dtype=tf.int64, shape=[2]) input_utt_lens = tf.placeholder(dtype=tf.int32, shape=[None]) input_label = tf.SparseTensor(indices=input_label_indice, values=input_label_value, dense_shape=input_label_shape) #with tf.name_scope('ds2'): with tf.variable_scope('ds2'): loss_op = ds2.get_loss(input_data, input_label, input_utt_lens, ARGS.data_format) with tf.name_scope('train'): optimizer = tf.train.AdamOptimizer(ARGS.learning_rate) train_op = optimizer.minimize(loss_op) dataset = Dataset(use_dummy=ARGS.use_dummy) # Create a saver. TODO: how about only inference saver = tf.train.Saver(tf.global_variables()) sess = tf.Session(config=tf.ConfigProto( allow_soft_placement=False, #True, log_device_placement=False, inter_op_parallelism_threads=8, intra_op_parallelism_threads=32)) last_iter = init_variables(sess, saver, ARGS.checkpoint_dir) run_options = None run_metadata = None # Merge all the summaries and write them out to ARGS.log_dir if ARGS.debug: summary_op = tf.summary.merge_all() summary_writer = tf.summary.FileWriter(ARGS.log_dir, sess.graph) run_options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE) run_metadata = tf.RunMetadata() #assert not np.isnan(loss_value), 'Model diverged with loss = NaN' sum_time = 0 min_time = float('inf') for i in range(ARGS.max_iter): start_time = time.time() train_loss, _ = sess.run([loss_op, train_op], feed_dict=feed_dict(), options=run_options, run_metadata=run_metadata) duration = time.time() - start_time sum_time += duration if min_time > duration: min_time = duration # Save the model checkpoint periodically if i % ARGS.checkpoint_iter == 0 or (i + 1) == ARGS.max_iter: checkpoint_path = ARGS.checkpoint_dir + '/ds2_model' global_step = last_iter + i saver.save(sess, checkpoint_path, global_step=global_step) logger.info('save checkpoint to %s-%d' % (ARGS.checkpoint_dir, global_step)) if i == ARGS.profil_iter: # save timeline to a json and only save once filename = ARGS.log_dir + ('/timeline_iter%d' % i) + '.json' logger.debug('save timeline to: ' + filename) save_timeline(filename, run_metadata) # tfprof logger.debug('save TFprof to: ' + ARGS.log_dir) save_tfprof(ARGS.log_dir, sess.graph, run_metadata) if i % ARGS.loss_iter_interval == 0: format_str = 'iter %d, training loss = %.2f (%.3f sec/iter)' log_content = (i, train_loss, duration) if ARGS.debug: s = time.time() summary = sess.run(summary_op, feed_dict=feed_dict(), options=run_options, run_metadata=run_metadata) summary_writer.add_run_metadata(run_metadata, 'iter%03d' % i) summary_writer.add_summary(summary, i) d = time.time() - s format_str += ', summary %.3g sec/iter' log_content += (d, ) logger.info(format_str % log_content) if ARGS.debug: summary_writer.close() logger.info('avg time: %.3f ms, min_time: %.3f ms' % (sum_time * 1000 / ARGS.max_iter, min_time * 1000)) sess.close()