def run(content_fname,
style_fname,
output_path,
model,
iterations=100,
sr=16000,
hop_size=512,
frame_size=2048,
alpha=1e-3):
content, fs = librosa.load(content_fname, sr=sr)
style, fs = librosa.load(style_fname, sr=sr)
n_samples = (min(content.shape[0], style.shape[0]) // 512) * 512
content = utils.chop(content[:n_samples], hop_size, frame_size)
style = utils.chop(style[:n_samples], hop_size, frame_size)
if model == 'encoder':
content_features, style_features = compute_wavenet_encoder_features(
content=content, style=style)
result = compute_wavenet_encoder_stylization(
n_frames=content_features[0].shape[0],
n_samples=frame_size,
alpha=alpha,
content_features=content_features,
style_features=style_features,
iterations=iterations)
elif model == 'decoder':
content_features, style_features = compute_wavenet_decoder_features(
content=content, style=style)
result = compute_wavenet_decoder_stylization(
n_frames=content_features[0].shape[0],
n_samples=frame_size,
alpha=alpha,
content_features=content_features,
style_features=style_features,
iterations=iterations)
else:
raise ValueError('Unsupported model type: {}.'.format(model))
x = utils.unchop(result, hop_size, frame_size)
librosa.output.write_wav('prelimiter.wav', x, sr)
limited = utils.limiter(x)
output_fname = '{}/{}+{}.wav'.format(output_path,
content_fname.split('/')[-1],
style_fname.split('/')[-1])
librosa.output.write_wav(output_fname, limited, sr=sr)
def main(args):
prog = args[0]
if len(args) < 3:
usage(prog)
sys.exit()
conf_file = []
conf_file_a = args[1]
conf_file_b = args[2]
printf("Enter username: "******"Enter password: "******"Scanning files %s and %s\n", conf_file_a, conf_file_b)
ldap_a = utils.LdapConnecter(user, passwd, conf_file=conf_file_a)
ldap_b = utils.LdapConnecter(user, passwd, conf_file=conf_file_b)
printf("binding to ldap from conf %s\n", conf_file_a)
ldap_a.bind()
printf("binding to ldap from conf %s\n", conf_file_b)
ldap_b.bind()
printf("\n")
ldap_groups = [v for (k,v) in ldap_a.conf["roles"].items()]
for ldap_group in ldap_groups:
printf("ldap group %s\n", ldap_group)
printf("----------------------------------------------\n")
printf("\n")
members_a = ldap_a.get_group_members_ssos(ldap_group)
members_b = ldap_b.get_group_members_ssos(ldap_group)
a_ssos = set(members_a.keys())
b_ssos = set(members_b.keys())
n_common = len(a_ssos & b_ssos)
printf("common member count: %d\n", n_common )
printf(" missing from %s:\n", conf_file_a)
missing_from_a = sorted(list(b_ssos - a_ssos))
if len(missing_from_a) > 0:
max_col_len = max([len(sso) for sso in missing_from_a])
for sso in missing_from_a:
lsso = "%s" %((sso + ":").ljust(max_col_len +2),)
printf(" %s%s\n", lsso, members_b[sso])
else:
printf(" None\n")
printf("\n")
printf(" missing from %s:\n", conf_file_b)
missing_from_b = sorted(list(a_ssos - b_ssos))
if len(missing_from_b) > 0:
max_col_len = max([len(sso) for sso in missing_from_b])
for sso in missing_from_a:
lsso = "%s" % ((sso + ":").ljustt(max_col_len +2),)
printf(" %s%s\n", lsso, members_a[sso])
else:
printf(" None\n")
printf("\n")
printf("\n")
ldap_a.unbind()
ldap_b.unbind()
def main(args):
prog = args[0]
if len(args) < 3:
usage(prog)
sys.exit()
conf_file = []
conf_file_a = args[1]
conf_file_b = args[2]
printf("Enter username: "******"Enter password: "******"Scanning files %s and %s\n", conf_file_a, conf_file_b)
ldap_a = utils.LdapConnecter(user, passwd, conf_file=conf_file_a)
ldap_b = utils.LdapConnecter(user, passwd, conf_file=conf_file_b)
printf("binding to ldap from conf %s\n", conf_file_a)
ldap_a.bind()
printf("binding to ldap from conf %s\n", conf_file_b)
ldap_b.bind()
printf("\n")
ldap_groups = [v for (k, v) in ldap_a.conf["roles"].items()]
for ldap_group in ldap_groups:
printf("ldap group %s\n", ldap_group)
printf("----------------------------------------------\n")
printf("\n")
members_a = ldap_a.get_group_members_ssos(ldap_group)
members_b = ldap_b.get_group_members_ssos(ldap_group)
a_ssos = set(members_a.keys())
b_ssos = set(members_b.keys())
n_common = len(a_ssos & b_ssos)
printf("common member count: %d\n", n_common)
printf(" missing from %s:\n", conf_file_a)
missing_from_a = sorted(list(b_ssos - a_ssos))
if len(missing_from_a) > 0:
max_col_len = max([len(sso) for sso in missing_from_a])
for sso in missing_from_a:
lsso = "%s" % ((sso + ":").ljust(max_col_len + 2), )
printf(" %s%s\n", lsso, members_b[sso])
else:
printf(" None\n")
printf("\n")
printf(" missing from %s:\n", conf_file_b)
missing_from_b = sorted(list(a_ssos - b_ssos))
if len(missing_from_b) > 0:
max_col_len = max([len(sso) for sso in missing_from_b])
for sso in missing_from_a:
lsso = "%s" % ((sso + ":").ljustt(max_col_len + 2), )
printf(" %s%s\n", lsso, members_a[sso])
else:
printf(" None\n")
printf("\n")
printf("\n")
ldap_a.unbind()
ldap_b.unbind()
def reload(self):
'''Reloads and reconfigures itself'''
parser = self.__parser # shortcut
lvl = parser.get("MQTT", "mqtt_log")
log.setLevel(lvl)
self.__keepalive = parser.getint("MQTT", "mqtt_period")
self.__initial_T = self.__keepalive / 2
self.__period = self.__initial_T
self.setPeriod(self.__initial_T)
topics = utils.chop(parser.get("MQTT", "mqtt_topics"), ',')
self.__newtopics = [(topic, QOS) for topic in topics]
if self.__state == CONNECTED:
self.subscribe()
log.debug("Reload complete")
def HiStr(self):
"""
High part string representation in form "1 000 000".
Result:
High part string representation.
"""
# Sign.
hi = self.Hi()
sign = "-" if hi < 0 else ""
# Take absulute value.
if hi < 0:
hi = -hi
chopped = ut.chop(str(hi), -3)
merged = ut.merge(chopped, [Money.Delim] * (len(chopped) - 1))
return sign + reduce(lambda a, b: a + b, merged)
def usage(prog):
printf("usage is %s <conf_file>\n", prog)
printf("get the user group and roles based on the conf file\n")
printf("You will be prompted for your SSO user and passwd\n")
printf("\n")
if __name__ == "__main__":
prog = sys.argv[0]
if len(sys.argv) < 2:
usage(prog)
sys.exit()
conf_file = sys.argv[1]
printf("Enter username: "******"Enter password: "******"User %s is a member of %d groups\n", user, len(groups))
printf("--------------------------------------------------------\n")
for (group_cn, group_dn) in groups:
printf("%s\n", group_dn)
printf("\n")
printf("User %s has %d roles\n", user, len(roles))
printf("--------------------------------------------------------\n")
for role in roles:
printf("%s\n", role)
printf("\n")
def find_pole(self, LAMBD, common_weight):
"""
This function is used to find the poles of f by finding the zeros of sigma using the method described in [3].
:param LAMBD: Diagonal matrix of the initial poles.
:param common_weight: Determines whether there is a single weighting used for all ports.
:return:
roetter: The zeros of the sigma function (the new poles of f).
SERA: Transpose of roetter
"""
# Set up an offset so make room for potential D and E coefficients in Eq. (1) in [2].
if self.options['asymp'] == AsympOptions.NONE:
offs = 0
elif self.options['asymp'] == AsympOptions.D:
offs = 1
else:
offs = 2
# Finding out which starting poles are complex
cindex = self.find_which_poles_are_complex(LAMBD)
# Building system matrix
Dk = self.build_system_matrix(cindex, LAMBD)
# Use the Dk matrix to create something closer to Eq. (A.1) in [1] for each port entry.
if self.options['asymp'] in (AsympOptions.NONE, AsympOptions.D):
# Add a column of 1's after the Dk data
Dk = np.column_stack((Dk, np.ones(self.Ns))).copy()
elif self.options['asymp'] == AsympOptions.DE:
# Add a column of 1's after the Dk data and a column of the s data as in Eq. (A.3) in [1]
Dk = np.column_stack((Dk, np.ones(self.Ns), self.s)).copy()
# Scaling for last row of LS-problem (pole identification). Calculate Eq. (9) in [2]
scale = 0
for m in range(0, self.Nc):
if self.weight.shape[1] == 1:
scale = scale + (np.linalg.norm(
self.weight[:, 0] * np.transpose(self.f[m])))**2
else:
scale = scale + (np.linalg.norm(
self.weight[:, m] * np.transpose(self.f[m])))**2
scale = sqrt(scale) / self.Ns
# Use relaxed nontriviality constraint (removed a non-relaxed option entirely)
AA = np.zeros(
(self.Nc * (self.N + 1),
self.N + 1)) # Set up AA to be used in Eq. (6), (A.8) in [1]
bb = np.zeros((self.Nc * (self.N + 1),
1)) # Set up bb to be used in Eq. (6), (A.8) in [1]
Escale = np.zeros((self.N + 1, 1))
for n in range(self.Nc):
A = np.zeros((self.Ns, (self.N + offs) + self.N + 1),
dtype=np.complex)
# offs was set earlier based on the D, DE, or neither D nor E options
if common_weight == 1:
weig = self.weight[:, 0]
else:
weig = self.weight[:, n]
for m in range(self.N + offs): # Left Block in Eq. (10) in [3]
A[0:self.Ns, m] = weig * Dk[0:self.Ns, m]
inda = self.N + offs # Dk already contains '1' and 's' data if applicable for D/E opts.
for m in range(self.N + 1): # Right block in Eq. (10) in [3]
A[0:self.Ns,
inda + m] = -weig * Dk[0:self.Ns, m] * self.f[n, 0:self.Ns].T
A = np.vstack(
(np.real(A), np.imag(A))
) # Stack the real and imaginary components as in Eq. (7) in [3]
# Integral criterion for sigma (this takes care of Eq. (8) in [2]
offset = self.N + offs
if n + 1 == self.Nc:
A_temp = np.zeros((1, A.shape[1]))
for mm in range(self.N + 1):
A_temp[0, offset + mm] = np.real(scale * np.sum(Dk[:, mm]))
# A_temp is a vector accounting for Eq. (8) in [2]; it is stacked below the last row of A
# A is called [X -HvX] in Eq. (10) in [3]. Used to avoid the null solution by adding to the LS problem.
A = np.vstack((A, A_temp))
Q, R = np.linalg.qr(
A
) # Solve as in Eq. (10) in [3] to implement the fast implementation of VF
ind1 = self.N + offs
ind2 = self.N + offs + self.N + 1
R22 = R[ind1:ind2, ind1:ind2] # R22 is used in Eq. (11) in [3]
AA[(n) * (self.N + 1):(n + 1) * (self.N + 1), :] = R22.copy()
# Fill in the vector of R22's for Eq. (11) in 3
if n + 1 == self.Nc:
bb[((n) * (self.N + 1)):(n + 1) * (self.N + 1),
0] = Q[-1, self.N + offs:].conj().T * self.Ns * scale
# This covers the right side of Eq. (11) in [3], with scale from above taking care of Hv weighted
# with Eq. (8), (9) in [2].
x = self.calculate_x(
AA, bb,
Escale) # Calculate the C tilda residues in Eq. (11) in [3]
C = x[:-1].copy().astype(
np.complex
) # The C tilda residues become C sigma in Eq. (9) in [3]
D = x[-1].reshape(
1,
-1) # The last entry in the x vector becomes D in Eq. (9) in [3]
# We now change back to make C complex
for m in range(self.N):
if cindex[m] == PoleTypes.COMPLEX_FIRST:
r1 = C[m].copy()
r2 = C[m + 1].copy()
C[m] = r1 + 1j * r2
C[m + 1] = r1 - 1j * r2
if self.options['spy1']:
self.plot_sigma(LAMBD, C, D)
# We now calculate the zeros for sigma
m = -1
B = np.ones((LAMBD.shape[0], 1)) # B is set to a default vector of 1's
# LAMBD is calculated using the A hat for complex numbers in Eq. (B.2) in [1],
# which is A sigma in Eq. (9) in [3]
# B is changed to [[2],[0]] according to Eq. (B.2) in [1] for complex numbers
# c tilda prime is also adjustded for complex numbers according to Eq. (B.2) in [1]
for n in range(self.N):
m = m + 1
if m < self.N - 1:
if LAMBD.imag[m, m]:
LAMBD[m + 1, m] = -np.imag(LAMBD[m, m])
LAMBD[m, m + 1] = np.imag(LAMBD[m, m])
LAMBD[m, m] = np.real(LAMBD[m, m])
LAMBD[m + 1, m + 1] = LAMBD[m, m].copy()
B[m, 0] = 2
B[m + 1, 0] = 0
koko = C[m, 0]
C[m] = koko.real
C[m + 1] = koko.imag
m = m + 1
ZER = LAMBD - B * C.T / D[0, 0] # Eq. (9) in [3]
roetter = LA.eigvals(
ZER
) # The rest of Eq. (9) in [3] to find the zeros of sigma / new poles (called roetter)
roetter = chop(
roetter
) # Get rid of tiny imaginary numbers so the program doesn't think reals are complex
unstables = np.real(roetter) > 0
if self.options['stable']:
# Forcing unstable poles to be stable...
roetter[unstables] = roetter[unstables] - 2 * np.real(
roetter[unstables])
self.N = roetter.shape[
0] # Make sure N matches the new number of poles.
roetter = roetter[np.argsort(np.abs(roetter))] # Magnitude sort
roetter = roetter[np.argsort(
roetter.imag != 0, kind='stable')] # Move reals to the beginning
roetter = roetter - 2 * 1j * roetter.imag # Force the sorting to match that of MATLAB (negative imaginary
# then positive imaginary for complex conjugate pairs).
SERA = roetter.T # Both of these are the new poles
return SERA, roetter
def pass_check_Y(A, C, D):
"""
Input state-space model with diagonal A (poles) with complex conjugate formulation and return
a matrix with 2 rows, and each column denotes the start and end frequency of non-passive band
A violation extending to infinity is denoted by s_end=j*1e16
:param A: poles in a vector (num_poles)
:param C: residues in a 3D matrix (ports x ports x num_poles)
:param D: D values in a 2D matrix (ports x ports)
:return:
"""
# This will rearrange A, B, C
Nc = D.shape[0]
N = A.shape[0]
tell = 0
CC = np.zeros((Nc, Nc * N), dtype=complex)
B = np.ones((N, 1))
for col in range(Nc):
if col == 0:
AA = np.diagflat(A)
BB = B
else:
AA = LA.block_diag(AA, np.diagflat(A))
BB = LA.block_diag(BB, B)
for row in range(col, Nc):
CC[row, col * N:(col + 1) * N] = C[row, col, :]
CC[col, row * N:(row + 1) * N] = C[row, col, :]
A = AA.copy()
B = BB.copy()
C = CC.copy()
Acmplx = A.copy()
Bcmplx = B.copy()
Ccmplx = C.copy()
Dcmplx = D.copy()
A = chop(A) # Get rid of tiny imaginary numbers
TOL = 1e-5 # Use this instead of 0 to look for tiny sums
# Convert to real-only configuration
if np.sum(A - np.diagflat(np.diag(A))) < TOL:
cindex, A, B, C = separate_real_imag_in_state_space(A, B, C)
N = A.shape[0]
Nc = D.shape[0]
E = np.zeros((Nc, Nc))
if np.sum(LA.eig(D) == 0):
Ahat = LA.solve(A, np.eye(N))
Bhat = -Ahat @ B
Chat = C * Ahat
Dhat = D - C * Ahat * B
A = Ahat.copy()
B = Bhat.copy()
C = Chat.copy()
D = Dhat.copy()
S1 = A @ (B @ (np.linalg.matrix_power(D, -1)) @ C - A) # Eq. (23a) in [7]
wS1, wS2 = LA.eig(S1)
wS1 = np.sqrt(wS1) # See note below Eq. (23a) in [17]
if np.sum(LA.eig(Dcmplx) == 0) > 0:
ws1 = 1 / wS1
ind = np.nonzero(np.abs(wS1.imag) < 1e-6)
wS1 = (wS1[ind]).real
sing_w = np.sort(wS1)
if sing_w.shape[0] == 0:
intervals = np.array([])
return intervals
A = Acmplx.copy()
B = Bcmplx.copy()
C = Ccmplx.copy()
D = Dcmplx.copy()
# Establishing frequency list at midpoint of all bands defined by sing_w
midw = np.zeros((1 + sing_w.shape[0], 1), dtype=complex)
midw[0] = sing_w[0] / 2
midw[-1] = 2 * sing_w[-1]
for k in range(sing_w.shape[0] - 1):
midw[k + 1] = (sing_w[k] + sing_w[k + 1]) / 2
EE = np.zeros((Nc, midw.shape[0]), dtype=complex)
viol = np.zeros(midw.shape[0])
# Checking passivity at all midpoints
for k in range(midw.shape[0]):
sk = 1j * midw[k]
G = (fitcalcABCDE(sk[0], np.diag(A), B, C, D, E)).real
EE[:, k], EE_temp = LA.eig(G)
if np.any(EE[:, k] < 0, axis=0):
viol[k] = 1
else:
viol[k] = 0
# Establishing intervals for passivity violations:
intervals = np.empty((2, 0))
for k in range(midw.shape[0]):
if viol[k] == 1:
if k == 0:
intervals = (np.vstack((0, sing_w[0])))
elif k == midw.shape[0] - 1:
intervals = np.hstack((intervals, (np.vstack(
(sing_w[k - 1], 1e16)))))
else:
intervals = np.hstack((intervals, (np.vstack(
(sing_w[k - 1], sing_w[k])))))
if not np.any(intervals):
wintervals = intervals.copy()
return wintervals
# Collapsing overlapping bands:
killindex = 0
for k in range(1, intervals.shape[1]):
if intervals[1, k - 1] == intervals[0, k]: # An overlap exists
intervals[1, k - 1] = intervals[1, k]
intervals[:, k] = intervals[:, k - 1]
killindex = np.append(killindex, k - 1)
# Delete any intervals with killindex == 1
if np.any(killindex) != 0:
intervals = np.delete(intervals, killindex, axis=1)
wintervals = intervals.copy()
return wintervals
def train():
print('data_path: %s' % FLAGS.data_path)
raw_data = reader.ptb_raw_data(FLAGS.data_path)
train_data, valid_data, valid_nbest_data, vocab = raw_data
train_data = chop(train_data, vocab['<eos>'])
config = MediumConfig()
if FLAGS.init_scale: config.init_scale = FLAGS.init_scale
if FLAGS.learning_rate: config.learning_rate = FLAGS.learning_rate
if FLAGS.max_grad_norm: config.max_grad_norm = FLAGS.max_grad_norm
if FLAGS.num_layers: config.num_layers = FLAGS.num_layers
if FLAGS.num_steps: config.num_steps = FLAGS.num_steps
if FLAGS.hidden_size: config.hidden_size = FLAGS.hidden_size
if FLAGS.max_epoch: config.max_epoch = FLAGS.max_epoch
if FLAGS.max_max_epoch: config.max_max_epoch = FLAGS.max_max_epoch
if FLAGS.keep_prob: config.keep_prob = FLAGS.keep_prob
if FLAGS.lr_decay: config.lr_decay = FLAGS.lr_decay
if FLAGS.batch_size: config.batch_size = FLAGS.batch_size
if FLAGS.opt_method: config.opt_method = FLAGS.opt_method
if FLAGS.log_dir: config.log_dir = FLAGS.log_dir
config.h_max_log_smooth = FLAGS.h_max_log_smooth
config.vocab_size = len(vocab)
print('init_scale: %.2f' % config.init_scale)
print('learning_rate: %.2f' % config.learning_rate)
print('max_grad_norm: %.2f' % config.max_grad_norm)
print('num_layers: %d' % config.num_layers)
print('num_steps: %d' % config.num_steps)
print('hidden_size: %d' % config.hidden_size)
print('max_epoch: %d' % config.max_epoch)
print('max_max_epoch: %d' % config.max_max_epoch)
print('keep_prob: %.2f' % config.keep_prob)
print('lr_decay: %.2f' % config.lr_decay)
print('batch_size: %d' % config.batch_size)
print('vocab_size: %d' % config.vocab_size)
print('opt_method: %s' % config.opt_method)
print('log_dir: %s' % config.log_dir)
print('seed: %d' % FLAGS.seed)
sys.stdout.flush()
eval_config = MediumConfig()
eval_config.init_scale = config.init_scale
eval_config.learning_rate = config.learning_rate
eval_config.max_grad_norm = config.max_grad_norm
eval_config.num_layers = config.num_layers
eval_config.num_steps = config.num_steps
eval_config.hidden_size = config.hidden_size
eval_config.max_epoch = config.max_epoch
eval_config.max_max_epoch = config.max_max_epoch
eval_config.keep_prob = config.keep_prob
eval_config.lr_decay = config.lr_decay
eval_config.batch_size = 200
# eval_config.batch_size = config.batch_size
eval_config.vocab_size = len(vocab)
eval_config.h_max_log_smooth = config.h_max_log_smooth
prev = 0
with tf.Graph().as_default(), tf.Session() as session:
initializer = tf.random_uniform_initializer(-config.init_scale,
config.init_scale)
with tf.variable_scope("model", reuse=None, initializer=initializer):
m = PTBModel(is_training=True, config=config)
with tf.variable_scope("model", reuse=True, initializer=initializer):
mvalid = PTBModel(is_training=False, config=eval_config)
tf.initialize_all_variables().run()
if FLAGS.model_path:
saver = tf.train.Saver()
loss_list = []
train_perp_list = []
val_perp_list = []
val_f1_list = []
for i in range(config.max_max_epoch):
shuffle(train_data)
shuffled_data = list(itertools.chain(*train_data))
start_time = time.time()
lr_decay = config.lr_decay**max(i - config.max_epoch, 0.0)
if config.opt_method == "YF":
session.run(tf.assign(m.optimizer.lr_factor, lr_decay))
else:
m.assign_lr(session, config.learning_rate * lr_decay)
print("Epoch: %d Learning rate: %.3f" % (i + 1, session.run(m.lr)))
train_perplexity, loss = run_epoch(session,
m,
shuffled_data,
m.train_op,
verbose=True,
epoch_id=i)
loss_list += loss
print("Epoch: %d Train Perplexity: %.3f" %
(i + 1, train_perplexity))
valid_perplexity, _ = run_epoch(session, mvalid, valid_data,
tf.no_op())
print("Epoch: %d Valid Perplexity: %.3f" %
(i + 1, valid_perplexity))
valid_f1, num = run_epoch2(session, mvalid, valid_nbest_data,
tf.no_op(), vocab['<eos>'])
print("Epoch: %d Valid F1: %.2f (%d trees)" %
(i + 1, valid_f1, num))
print('It took %.2f seconds' % (time.time() - start_time))
#print("summary added step", i * len(loss) )
summ = tf.Summary(value=[
tf.Summary.Value(tag="eval_perp",
simple_value=valid_perplexity),
])
m.writer.add_summary(summ, i * len(loss))
summ = tf.Summary(value=[
tf.Summary.Value(tag="eval_F1", simple_value=valid_f1),
])
m.writer.add_summary(summ, i * len(loss))
train_perp_list.append([i * len(loss), train_perplexity])
val_perp_list.append([i * len(loss), valid_perplexity])
val_f1_list.append([i * len(loss), valid_f1])
if prev < valid_f1:
prev = valid_f1
if FLAGS.model_path:
print('Save a model to %s' % FLAGS.model_path)
saver.save(session, FLAGS.model_path)
pickle.dump(eval_config,
open(FLAGS.model_path + '.config', 'wb'))
sys.stdout.flush()
with open(config.log_dir + "/loss.txt", "w") as f:
np.savetxt(f, np.array(loss_list))
with open(config.log_dir + "/train_perp.txt", "w") as f:
np.savetxt(f, np.array(train_perp_list))
with open(config.log_dir + "/val_perp.txt", "w") as f:
np.savetxt(f, np.array(val_perp_list))
with open(config.log_dir + "/val_f1.txt", "w") as f:
np.savetxt(f, np.array(val_f1_list))
def train():
print('data_path: %s' % FLAGS.data_path)
raw_data = reader.ptb_raw_data3(FLAGS.data_path)
train_data, silver_path, valid_data, valid_nbest_data, vocab = raw_data
train_data = chop(train_data, vocab['<eos>'])
config = MediumConfig()
if FLAGS.init_scale: config.init_scale = FLAGS.init_scale
if FLAGS.learning_rate: config.learning_rate = FLAGS.learning_rate
if FLAGS.max_grad_norm: config.max_grad_norm = FLAGS.max_grad_norm
if FLAGS.num_layers: config.num_layers = FLAGS.num_layers
if FLAGS.num_steps: config.num_steps = FLAGS.num_steps
if FLAGS.hidden_size: config.hidden_size = FLAGS.hidden_size
if FLAGS.max_epoch: config.max_epoch = FLAGS.max_epoch
if FLAGS.max_max_epoch: config.max_max_epoch = FLAGS.max_max_epoch
if FLAGS.keep_prob: config.keep_prob = FLAGS.keep_prob
if FLAGS.lr_decay: config.lr_decay = FLAGS.lr_decay
if FLAGS.batch_size: config.batch_size = FLAGS.batch_size
config.vocab_size = len(vocab)
if FLAGS.silver: config.silver = FLAGS.silver
print('init_scale: %.2f' % config.init_scale)
print('learning_rate: %.2f' % config.learning_rate)
print('max_grad_norm: %.2f' % config.max_grad_norm)
print('num_layers: %d' % config.num_layers)
print('num_steps: %d' % config.num_steps)
print('hidden_size: %d' % config.hidden_size)
print('max_epoch: %d' % config.max_epoch)
print('max_max_epoch: %d' % config.max_max_epoch)
print('keep_prob: %.2f' % config.keep_prob)
print('lr_decay: %.2f' % config.lr_decay)
print('batch_size: %d' % config.batch_size)
print('vocab_size: %d' % config.vocab_size)
print('silver: %d' % config.silver)
sys.stdout.flush()
eval_config = MediumConfig()
eval_config.init_scale = config.init_scale
eval_config.learning_rate = config.learning_rate
eval_config.max_grad_norm = config.max_grad_norm
eval_config.num_layers = config.num_layers
eval_config.num_steps = config.num_steps
eval_config.hidden_size = config.hidden_size
eval_config.max_epoch = config.max_epoch
eval_config.max_max_epoch = config.max_max_epoch
eval_config.keep_prob = config.keep_prob
eval_config.lr_decay = config.lr_decay
eval_config.batch_size = 200
eval_config.vocab_size = len(vocab)
prev = 0 # record F1 scores
with tf.Graph().as_default(), tf.Session() as session:
initializer = tf.random_uniform_initializer(-config.init_scale,
config.init_scale)
with tf.variable_scope("model", reuse=None, initializer=initializer):
m = PTBModel(is_training=True, config=config)
with tf.variable_scope("model", reuse=True, initializer=initializer):
mvalid = PTBModel(is_training=False, config=eval_config)
tf.initialize_all_variables().run()
if FLAGS.model_path:
saver = tf.train.Saver()
silver_generator = reader.file_to_word_ids3(silver_path)
j = 0
for i in range(config.max_max_epoch):
shuffle(train_data)
shuffled_data = list(itertools.chain(*train_data))
start_time = time.time()
lr_decay = config.lr_decay ** max(i - config.max_epoch, 0.0)
m.assign_lr(session, config.learning_rate * lr_decay)
print("Epoch: %d Learning rate: %.3f" % (i + 1, session.run(m.lr)))
train_perplexity = run_epoch(session, m, shuffled_data, m.train_op,
verbose=True)
print("Epoch: %d Train Perplexity: %.3f" % (i + 1, train_perplexity))
valid_perplexity = run_epoch(session, mvalid, valid_data, tf.no_op())
print("Epoch: %d Valid Perplexity: %.3f" % (i + 1, valid_perplexity))
valid_f1, num = run_epoch2(session, mvalid, valid_nbest_data,
tf.no_op(), vocab['<eos>'])
print("Epoch: %d Valid F1: %.2f (%d trees)" % (i + 1, valid_f1, num))
if valid_f1 > prev:
prev = valid_f1
if FLAGS.model_path:
print('Save a model to %s' % FLAGS.model_path)
saver.save(session, FLAGS.model_path)
pickle.dump(eval_config, open(FLAGS.model_path + '.config', 'wb'))
print('It took %.2f seconds' % (time.time() - start_time))
sys.stdout.flush()
start_time = time.time()
for k in xrange(config.silver):
try:
silver_data = silver_generator.next()
except:
silver_generator = reader.file_to_word_ids3(silver_path)
silver_data = silver_generator.next()
j += 1
silver_data = chop(silver_data, vocab['<eos>'])
shuffle(silver_data)
silver_data = list(itertools.chain(*silver_data))
silver_perplexity = run_epoch(session, m, silver_data, m.train_op,
verbose=False)
print("Epoch: %d Silver(%d) Perplexity: %.3f" %
(i + 1, j, silver_perplexity))
valid_perplexity = run_epoch(session, mvalid, valid_data, tf.no_op())
print("Epoch: %d Silver(V) Perplexity: %.3f" % (i+1, valid_perplexity))
valid_f1, num = run_epoch2(session, mvalid, valid_nbest_data,
tf.no_op(), vocab['<eos>'])
print("Epoch: %d Silver(V) F1: %.2f (%d trees)" % (i+1, valid_f1, num))
if valid_f1 > prev:
prev = valid_f1
if FLAGS.model_path:
print('Save a model to %s' % FLAGS.model_path)
saver.save(session, FLAGS.model_path)
pickle.dump(eval_config, open(FLAGS.model_path + '.config', 'wb'))
print('It took %.2f seconds' % (time.time() - start_time))
sys.stdout.flush()
