# Load data
print("Loading data...")
trainset = Dataset('../../data/'+FLAGS.dataset+'/train.ss')
devset = Dataset('../../data/'+FLAGS.dataset+'/dev.ss')
testset = Dataset('../../data/'+FLAGS.dataset+'/test.ss')
alldata = np.concatenate([trainset.t_docs, devset.t_docs, testset.t_docs], axis=0)
embeddingpath = '../../data/'+FLAGS.dataset+'/embedding.txt'
embeddingfile, wordsdict = data_helpers.load_embedding(embeddingpath, alldata, FLAGS.embedding_dim)
del alldata
print("Loading data finished...")
usrdict, prddict = trainset.get_usr_prd_dict()
trainbatches = trainset.batch_iter(usrdict, prddict, wordsdict, FLAGS.n_class, FLAGS.batch_size,
FLAGS.num_epochs, FLAGS.max_sen_len, FLAGS.max_doc_len)
devset.genBatch(usrdict, prddict, wordsdict, FLAGS.batch_size,
FLAGS.max_sen_len, FLAGS.max_doc_len, FLAGS.n_class)
testset.genBatch(usrdict, prddict, wordsdict, FLAGS.batch_size,
FLAGS.max_sen_len, FLAGS.max_doc_len, FLAGS.n_class)
with tf.Graph().as_default():
session_config = tf.ConfigProto(
allow_soft_placement=FLAGS.allow_soft_placement,
log_device_placement=FLAGS.log_device_placement
)
session_config.gpu_options.allow_growth = True
sess = tf.Session(config=session_config)
with sess.as_default():
huapataha = huapahata(
max_sen_len = FLAGS.max_sen_len,
max_doc_len = FLAGS.max_doc_len,
fs = open('data/wordlist.txt')
alldata = fs.readlines()
alldata = [item.strip() for item in alldata]
fs.close()
estime = time.time()
print "================= load wordsdict ===============", estime - etime
embeddingpath = 'data/embeding1'
embeddingfile, wordsdict = data_helpers.load_embedding(embeddingpath, alldata,
FLAGS.embedding_dim)
print type(embeddingfile)
del alldata
stime = time.time()
print "================= load word2vec ===============", stime - estime
testset.genBatch(wordsdict, FLAGS.batch_size, FLAGS.max_sen_len,
FLAGS.max_doc_len, FLAGS.n_class, True)
sstime = time.time()
print "================= testset genBatch ===============", sstime - stime
print "satrt*********************************************************"
print("Loading data finished...")
graph = tf.Graph()
with graph.as_default():
session_config = tf.ConfigProto(
allow_soft_placement=FLAGS.allow_soft_placement,
log_device_placement=FLAGS.log_device_placement)
session_config.gpu_options.allow_growth = True
sess = tf.Session(config=session_config)
with sess.as_default():