def __init__(self, batch_size, version):
super(CycleGAN_CNN, self).__init__(batch_size, version)
self.encoder_X = ConverterA_CNN(name="encoder_cnn")
self.decoder_X = ConverterB_CNN(name="decoder_cnn")
self.discriminator = SeqDiscriminator_CNN(class_num=62, mode=1, fixed_length=False, name='discriminator')
self.data_X = datamanager(time_major=False, seed=19940610)
self.data_Y = datamanager(time_major=False, seed=19940610)
self.data_Y.shuffle_train(seed=19931028)
self.build_network()
self.build_sess()
self.build_dirs()
def sample(self, epoch):
data = datamanager(time_major=False, seed=0)
data.shuffle_train(0)
X = data(self.batch_size, var_list=["AccGyo", "lens", "labels", "XYZ"])
feed_dict = {
self.source_X: X["AccGyo"][:, :, :, None],
self.length_X: X["lens"]
}
logit = self.sess.run(self.G, feed_dict=feed_dict)
logit = np.array([filtering(seq, window=5) for seq in logit])
ori, target, lens = X["AccGyo"], X["XYZ"], X['lens']
for i in range(4):
idx = i * 3
plt.subplot(4, 3, 3 * i + 1)
plt.plot(ori[idx, :lens[idx], :])
plt.subplot(4, 3, 3 * i + 2)
# plt.plot(logit[idx, :lens[idx], :, 0])
plt.plot(logit[idx, :lens[idx], 0, 0], logit[idx, :lens[idx], 1,
0])
plt.xticks([])
plt.title(str(np.argmax(X["labels"][idx])))
plt.subplot(4, 3, 3 * i + 3)
# plt.plot(target[idx, :lens[idx], :])
plt.plot(target[idx, :lens[idx], 0], target[idx, :lens[idx], 1])
plt.xticks([])
plt.savefig(
os.path.join(self.fig_dir, "sample_epoch_{}.png".format(epoch)))
plt.clf()
def sample(self, epoch):
print "sample after epoch {}".format(epoch)
data = datamanager(time_major=False,seed=19940610)
data_X = data(self.batch_size, phase='train', var_list=["AccGyo", "lens", "labels", "XYZ"])
feed_dict={
self.source_X:data_X['AccGyo'][:,:,:,None],
self.len_X:data_X['lens']
}
data_X['enX_outputs'], data_X['deX_outputs'] = self.sess.run([self.encoder_outputs, self.decoder_outputs], feed_dict=feed_dict)
np.savez(os.path.join(self.fig_dir, "sample_epoch_{}.npz".format(epoch)), **data_X)
ori, lens, pred = data_X['XYZ'], data_X['lens'], data_X["enX_outputs"][:,:,:,0]
indexes, tmp = [], {}
for i, label in enumerate(np.argmax(data_X['labels'], axis=1)):
if not tmp.has_key(label):
indexes.append(i)
tmp[label] = 0
for i in range(4):
for j in range(3):
idx = i*3 + j
pic_idx = indexes[idx]
plt.subplot(4,6, idx*2+1)
plt.plot(ori[pic_idx, :lens[pic_idx], 0], ori[pic_idx, :lens[pic_idx], 1], color='g')
plt.xticks([])
plt.subplot(4,6, idx*2+2)
plt.plot(pred[pic_idx, :lens[pic_idx], 0], pred[pic_idx, :lens[pic_idx], 1], color='r')
plt.xticks([])
plt.savefig(os.path.join(self.fig_dir, "sample_epoch_{}.png".format(epoch)))
plt.clf()
def __init__(self):
"""Generate reduc plot html"""
# Get all tags with raw data that are not cut
dm = datamanager.datamanager()
dm.gettags()
# Sort tags with latest one first
self.tags = dm.tags[::-1]
self.htmldir = 'browser'
def sample(self, epoch):
print "sample after epoch {}".format(epoch)
data = datamanager(seed=19931028)
data_X = data(self.batch_size, phase='train', var_list=["AccGyo", "lens", "labels", "XYZ"])
feed_dict={
self.source_X:data_X['AccGyo'],
self.len_X:data_X['lens']
}
data_X['enX_outputs'], data_X['deX_outputs'] = self.sess.run([self.en_outputs_1, self.de_outputs_1], feed_dict=feed_dict)
np.savez(os.path.join(self.fig_dir, "sample_epoch_{}.npz".format(epoch)), **data_X)
def __init__(self, batch_size, version):
super(DTN, self).__init__(batch_size, version)
self.classifier_X = SeqDiscriminator_CNN(class_num=62,
mode=0,
fixed_length=False,
name="cls_x_cnn")
self.classifier_Y = SeqDiscriminator_CNN(class_num=62,
mode=0,
fixed_length=False,
name="cls_y_cnn")
self.autoencoder_X = AutoEncoder_CNN(input_depth=6,
output_depth=6,
fixed_length=False,
name="ae_X")
self.autoencoder_Y = AutoEncoder_CNN(input_depth=3,
output_depth=3,
fixed_length=False,
name="ae_Y")
self.encoder = ConverterA_CNN(name="encoder_XtoY")
self.discriminator = SeqDiscriminator_CNN(class_num=None,
mode=1,
fixed_length=False,
name="discriminator")
self.data_X = datamanager(time_major=False)
self.data_Y = datamanager(time_major=False, seed=1)
self.build_placeholder()
self.build_classifier()
# self.build_dtn()
# self.build_ae()
self.build_gan()
self.build_optimizer()
self.build_summary()
self.build_sess()
self.build_dirs()
def __init__(self, batch_size, version='saegan', gpu='0'):
super(SAEGAN, self).__init__(batch_size, version, gpu)
self.data_img = datamanager('CT_img',
train_ratio=0.8,
expand_dim=3,
seed=0)
self.data_seq = datamanager('CT_seq',
train_ratio=0.8,
expand_dim=3,
seed=1)
self.sample_data_Img = self.data_img(self.batch_size,
phase='test',
var_list=['data'])
self.sample_data_Seq = self.data_seq(self.batch_size,
phase='test',
var_list=['data'])
self.len_latent = 64
# self.autoencoder_img = AutoEncoder_Image(self.len_latent, name='AEI')
# self.autoencoder_seq = AutoEncoder_Seq(self.len_latent, name='AES')
self.autoencoder_img = Variational_AutoEncoder_Image(self.len_latent,
name='VAEI')
self.autoencoder_seq = Variational_AutoEncoder_Seq(self.len_latent,
name='VAES')
self.latent_discriminator = Latent_Discriminator(name='LD')
self.class_num = 20
self.latent_classifier = Latent_Classifier(class_num=self.class_num,
name='LC')
self.build_placeholder()
self.build_network()
self.build_optimizer()
self.build_dirs(appendix="../")
self.build_sess()
def test(self):
data = datamanager(time_major=False)
loss = 0
for _ in range(107):
X = self.data_X(self.batch_size, var_list=["lens", "AccGyo", "XYZ"])
feed_dict = {
self.source_X: X["AccGyo"][:,:,:,None],
self.len_X: X["lens"],
self.target_X: X["AccGyo"][:,:,:,None],
self.mid_X: X["XYZ"][:,:,:,None]
}
loss += self.sess.run(self.transform_loss, feed_dict=feed_dict)
loss /= float(107 * self.batch_size)
print loss
def __init__(self, batch_size, gan_type='gan', version="cyclegan"):
self.gan_type = gan_type
super(CycleGAN, self).__init__(batch_size, version)
self.data_Seq = datamanager('CT', train_ratio=0.8, expand_dim=3, seed=0)
self.data_Img = datamanager('CT_img', train_ratio=0.8, expand_dim=3, seed=1)
self.sample_data_Seq = self.data_Seq(self.batch_size, phase='test', var_list=['data'])
self.sample_data_Img = self.data_Img(self.batch_size, phase='test', var_list=['data'])
self.critic_iter = 3
self.generator_SeqtoImg = CNN_Generator_SeqtoImg(output_dim=1, name="cnn_generator_SeqtoImg")
self.generator_ImgtoSeq = CNN_Generator_ImgtoSeq(output_dim=1, name="cnn_generator_ImgtoSeq")
self.discriminator_Seq = CNN_Discriminator_Seq(name="cnn_discriminator_Seq")
self.discriminator_Img = CNN_Discriminator_Img(name="cnn_discriminator_Img")
self.build_placeholder()
self.build_network()
self.build_optimizer()
self.build_summary()
self.build_sess()
self.build_dirs()
def get_latent(self):
data = datamanager(time_major=self.time_major)
for i in range(107):
X = data(64, phase='train', var_list=["lens", "AccGyo", "XYZ"])
if not self.time_major:
X["XYZ"] = X["XYZ"]
X["AccGyo"] = X["AccGyo"]
feed_dict = {
self.source: X[self.input_pointer],
self.length: X["lens"],
self.target: X[self.output_pointer],
}
latent = self.sess.run(self.autoencoder.latent,
feed_dict=feed_dict)
print latent.shape
def __init__(self, class_num, data_dim, batch_size, version):
super(Classifier, self).__init__(batch_size, version)
self.data = datamanager(time_major=False)
self.class_num = class_num
self.data_dim = data_dim
# CNN classifier
self.classifier = SeqDiscriminator_CNN(class_num=class_num,
mode=0,
fixed_length=False,
name="CNN_classifier")
self.build_network()
self.build_sess()
self.build_dirs()
def gen_latent(self):
data = datamanager(time_major=False)
train_x, train_y = [], []
for _ in range(108):
X = data(self.batch_size,
phase='train',
var_list=[self.input_pointer, "lens", "labels"])
feed_dict = {
self.source: X[self.input_pointer][:, :, :, None],
self.length: X["lens"]
}
train_x.append(self.sess.run(self.embedding, feed_dict=feed_dict))
train_y.append(np.argmax(X["labels"], axis=1))
train_x = np.concatenate(train_x, axis=0)
train_y = np.concatenate(train_y)[:data.train_num]
test_x, test_y = [], []
for _ in range(27):
X = data(self.batch_size,
phase='test',
var_list=[self.input_pointer, "lens", "labels"])
feed_dict = {
self.source: X[self.input_pointer][:, :, :, None],
self.length: X["lens"]
}
test_x.append(self.sess.run(self.embedding, feed_dict=feed_dict))
test_y.append(np.argmax(X["labels"], axis=1))
test_x = np.concatenate(test_x, axis=0)[:data.test_num, :]
test_y = np.concatenate(test_y)[:data.test_num]
print train_x.shape, train_y.shape, test_x.shape, test_y.shape
to_save = {
"train_embedding": train_x,
"train_label": train_y,
"test_embedding": test_x,
"test_label": test_y,
"classifier_name": self.version,
"data_seed": 19940610
}
print to_save["classifier_name"]
np.savez(
"/home/scw4750/songbinxu/autoencoder/data/" + self.version +
"_emb.npz", **to_save)
def __init__(self, batch_size, version):
super(DAE_GAN, self).__init__(batch_size, version)
self.data = datamanager(time_major=False)
self.autoencoder = AutoEncoder_CNN(6,
6,
100,
fixed_length=False,
name='autoencoder')
# self.encoder = SeqDiscriminator_CNN(class_num=None, mode=1, fixed_length=False, name='encoder')
# self.decoder = dynamic_decoder([64,64], 6, 'lstm', name='decoder')
self.time_major = False
self.build_placeholder()
self.build_network()
self.build_optimizer()
self.build_sess()
def Seq2Seq_CNN(mode):
from autoencoder import ConverterA_CNN, AE
from encoders import encoder_bi
from decoders import dynamic_decoder
from datamanager import datamanager
for fold_id in range(5):
autoencoder = ConverterA_CNN(name="encoderdecoder_CNN")
data = datamanager(time_major=False,
expand_dim=True,
train_ratio=None,
fold_k=5,
seed=233)
ae = AE(autoencoder, data, 64, "Seq2Seq_CNN/fold_{}".format(fold_id))
if mode == 'train':
ae.train(epoches=100)
elif mode == 'test':
# ae.saver.restore(ae.sess, os.path.join(ae.model_dir, "model.ckpt-10699"))
ae.load_model()
ae.test()
tf.reset_default_graph()
def __init__(self, batch_size, noise_dim=100, version="CGAN"):
super(CGAN, self).__init__(batch_size, version)
self.noise_dim = noise_dim
self.data = datamanager('CT', train_ratio=0.8, expand_dim=3, seed=0)
self.data_test = self.data(self.batch_size,
'test',
var_list=['data', 'labels'])
self.class_num = self.data.class_num
self.Generator = Generator(output_dim=1, name='G')
self.Discriminator = Discriminator(name='D')
self.build_placeholder()
self.build_gan()
self.build_optimizer()
self.build_summary()
self.build_sess()
self.build_dirs()
def __init__(self, batch_size, version):
super(infoGAN, self).__init__(batch_size, version or 'infoGAN')
self.noise_dim = 100
self.SUPERVISED = True
self.data = datamanager("data/CharacterTrajectories/CharacterTrajectories.npz",
train_ratio=1.0, fold_k=None, norm=None, expand_dim=3, seed=0)
self.class_num = self.data.class_num
# code
self.len_code = 2
self.Generator = Generator(output_dim=1, name='G')
self.Discriminator = Discriminator(name='D')
self.Classifier = Classifier(class_num=self.class_num + self.len_code, name='C')
self.build_placeholder()
self.build_gan()
self.build_optimizer()
self.build_summary()
self.build_sess()
self.build_dirs()
def Seq2Seq_RNN(mode):
from autoencoder import AutoEncoder_RNN, AE
from encoders import encoder_bi
from decoders import dynamic_decoder
from datamanager import datamanager
tmp = []
for fold_id in range(5):
autoencoder = AutoEncoder_RNN(encoder_bi, {
"hidden_units": [64, 64],
"cell_type": "gru"
},
dynamic_decoder, {
"hidden_units": [64, 64],
"cell_type": "gru"
},
input_depth=6,
output_depth=3,
embedding_dim=50,
name="encoderdecoder_gru")
data = datamanager(time_major=True,
expand_dim=False,
train_ratio=None,
fold_k=5,
seed=233)
ae = AE(autoencoder, data, 64, "Seq2Seq_GRU/fold_{}".format(fold_id))
if mode == 'train':
ae.train(epoches=100)
elif mode == 'test':
ae.saver.restore(ae.sess,
os.path.join(ae.model_dir, "model.ckpt-10699"))
a, b = ae.test()
tmp.append([a, b])
tf.reset_default_graph()
print tmp
def __init__(self):
self.mg = datamanager()
self.companies = pd.read_pickle('../Database/naver_companies.pickle')
def tsne(self):
data_img = datamanager('CT_img', train_ratio=0.8, expand_dim=3, seed=0)
data_seq = datamanager('CT_seq', train_ratio=0.8, expand_dim=3, seed=1)
img_embs, seq_embs, img_labels, seq_labels = [], [], [], []
for i in range(data_img.train_num // self.batch_size + 1):
img = data_img(self.batch_size,
phase='train',
var_list=['data', 'labels'])
seq = data_seq(self.batch_size,
phase='train',
var_list=['data', 'labels'])
feed_dict = {
self.source_img: img['data'],
self.source_seq: seq['data']
}
emb_img, emb_seq = self.sess.run(
[self.emb_img_test, self.emb_seq_test], feed_dict=feed_dict)
img_embs.append(emb_img)
seq_embs.append(emb_seq)
img_labels.append(np.argmax(img['labels'], 1))
seq_labels.append(np.argmax(seq['labels'], 1))
embs = np.concatenate(img_embs + seq_embs, axis=0)
img_labels = np.concatenate(img_labels)
seq_labels = np.concatenate(seq_labels)
domain_labels = np.array([0] * (len(img_embs) * self.batch_size) +
[1] * (len(seq_embs) * self.batch_size))
print embs.shape, domain_labels.shape
from sklearn.manifold import TSNE
model = TSNE(n_components=2, random_state=0)
embs = model.fit_transform(embs)
# np.save(os.path.join(self.fig_dir, "tsne", 'emb.npy'), embs)
# embs = np.load(os.path.join(self.fig_dir, "tsne", 'emb.npy'))
plt.scatter(embs[:, 0], embs[:, 1], c=domain_labels)
plt.colorbar()
keys = [
'a', 'b', 'c', 'd', 'e', 'g', 'h', 'l', 'm', 'n', 'o', 'p', 'q',
'r', 's', 'u', 'v', 'w', 'y', 'z'
]
img_labels_dict = {}
for i in range(200):
img_labels_dict[img_labels[i]] = i
seq_labels_dict = {}
for i in range(200):
seq_labels_dict[seq_labels[i]] = i + len(img_labels)
for i, v in img_labels_dict.iteritems():
plt.text(embs[v][0] - 5,
embs[v][1] + 5,
s=keys[i],
fontsize=10,
color='b')
for i, v in seq_labels_dict.iteritems():
plt.text(embs[v][0] + 5,
embs[v][1] - 5,
s=keys[i],
fontsize=10,
color='r')
plt.savefig(os.path.join(self.fig_dir, "tsne", "tsne.png"))
plt.clf()
import numpy as np
import bmxdata
from datamanager import datamanager
from reduce_plot import genplots
import os
import cPickle as cP
import astropy.units as u
from astropy.coordinates import EarthLocation, AltAz,SkyCoord
from astropy.time import Time
import time
telescope_loc = EarthLocation(lat=40.87792*u.deg, lon=-72.85852*u.deg, height=0*u.m)
# Initialize empty data manager so we can use its functions
dm = datamanager()
dm.gettags()
class reduce(object):
def __init__(self, tag):
""" Init with a tag string E.g.
r = reduce('170928_1000')
"""
# Store tag string and get filename
self.tag = tag
self.rawfname = dm.getrawfname(tag)
self.redfname = dm.getreducedfname(tag)
print('loading data...')
t = time.time()