def puzzle1(actions): """Run puzzle 1.""" shuffler = Shuffler(CARD_COUNT) shuffler.shuffle(actions) print("PUZZLE 1") print("========") print("Position of card 2019: {0}".format(shuffler.stack.index(2019)))
def smart_shuffle(tracks): """ Run the smart shuffler :param tracks: the tracks to shuffle :return: results tuple (sort, script and div) """ spotify = session['spotify'] features = spotify.get_audio_features(tracks) shuffler = Shuffler(tracks, features) sort = shuffler.get_sort() script, div = shuffler.get_charts(request.MOBILE) results = Results(sort=tuple(sort), script=script, div=div) return results
def test_cut(self): shuffler = Shuffler(10) shuffler.cut(3) self.assertEqual(shuffler.stack, [3, 4, 5, 6, 7, 8, 9, 0, 1, 2]) shuffler.reset_stack() shuffler.cut(-4) self.assertEqual(shuffler.stack, [6, 7, 8, 9, 0, 1, 2, 3, 4, 5])
def test_shuffle_example4(self): shuffler = Shuffler(10) shuffler.shuffle(( "deal into new stack", "cut -2", "deal with increment 7", "cut 8", "cut -4", "deal with increment 7", "cut 3", "deal with increment 9", "deal with increment 3", "cut -1" )) self.assertEqual(shuffler.stack, [9, 2, 5, 8, 1, 4, 7, 0, 3, 6])
def test_reset_stack(self): shuffler = Shuffler() self.assertEqual(shuffler.stack, []) shuffler.reset_stack(7) self.assertEqual(shuffler.stack, [0, 1, 2, 3, 4, 5, 6]) shuffler.stack[0] = -1 shuffler.reset_stack() self.assertEqual(shuffler.stack, [0, 1, 2, 3, 4, 5, 6])
def __init__(self, batch_size=50, topic_num=30, hidden_size=100, corpus_path="corpus/avaliable", debug=False, fea_dim=20, train_epoch_num=10, processor="gpu"): self.processor = processor self.train_epoch_num = train_epoch_num self.fea_dim = fea_dim self.batch_size = batch_size self.topic_num = topic_num self.hidden_size = hidden_size self.corpus_path = modify_path(corpus_path) self.save_step = 60 self.model_path = "./model/LSTM/topic_num(%d)-hidden_size(%d)-batch_size(%d)-fead_dim(%d)/" % ( self.topic_num, self.hidden_size, self.batch_size, self.fea_dim) self.model_data_path = self.model_path + "data/" self.model_save_path = self.model_path + "model/" self.summary_save_path = self.model_path + "summary/" if debug == True: if os.path.exists(self.model_path): shutil.rmtree(self.model_path) # creat directory for _dir in [ self.model_path, self.model_data_path, self.model_save_path, self.summary_save_path ]: if not os.path.exists(_dir): os.mkdir(_dir) self.logger = get_logger("LSTM") self.shuffler = Shuffler() self.load_lda() self.build_graph() self.saver = tf.train.Saver()
class LDA_LSTM: def __init__(self, batch_size=50, topic_num=30, hidden_size=100, corpus_path="corpus/avaliable", debug=False, fea_dim=20, train_epoch_num=10, processor="gpu"): self.processor = processor self.train_epoch_num = train_epoch_num self.fea_dim = fea_dim self.batch_size = batch_size self.topic_num = topic_num self.hidden_size = hidden_size self.corpus_path = modify_path(corpus_path) self.save_step = 60 self.model_path = "./model/LSTM/topic_num(%d)-hidden_size(%d)-batch_size(%d)-fead_dim(%d)/" % ( self.topic_num, self.hidden_size, self.batch_size, self.fea_dim) self.model_data_path = self.model_path + "data/" self.model_save_path = self.model_path + "model/" self.summary_save_path = self.model_path + "summary/" if debug == True: if os.path.exists(self.model_path): shutil.rmtree(self.model_path) # creat directory for _dir in [ self.model_path, self.model_data_path, self.model_save_path, self.summary_save_path ]: if not os.path.exists(_dir): os.mkdir(_dir) self.logger = get_logger("LSTM") self.shuffler = Shuffler() self.load_lda() self.build_graph() self.saver = tf.train.Saver() def shuffle(self, input_file_list, output_file): self.logger.info("start shuffle") self.shuffler.shuffle_mul_file( [self.corpus_path + _file for _file in input_file_list], self.model_data_path + output_file) self.logger.info("shuffle Done!") def shuffle_train_data(self): self.shuffle(["mix.txt", "neg.txt", "pos.txt"], "train.txt") def shuffle_test_data(self): self.shuffle(["mix_test.txt", "neg_test.txt", "pos_test.txt"], "test.txt") def build_graph(self): self.logger.info("start building graph") english_input = tf.placeholder(tf.int32, [self.batch_size, self.topic_num], name="english_input") chinese_input = tf.placeholder(tf.int32, [self.batch_size, self.topic_num], name="chinese_input") Y = tf.placeholder(tf.float32, [self.batch_size], name="scores") # embedding layer with tf.variable_scope("embdding"): en_embeddings = [] zh_embeddings = [] for i in range(self.topic_num): english_ids = tf.slice(english_input, [0, i], [self.batch_size, 1]) chinese_ids = tf.slice(chinese_input, [0, i], [self.batch_size, 1]) embedding_en = tf.Variable( tf.random_normal([self.fea_dim, self.hidden_size]), name="en_topic_%d_embedding" % (i + 1), dtype=tf.float32) embedding_zh = tf.Variable( tf.random_normal([self.fea_dim, self.hidden_size]), name="zh_topic_%d_embedding" % (i + 1), dtype=tf.float32) en_embeddings.append( tf.nn.embedding_lookup(embedding_en, english_ids)) zh_embeddings.append( tf.nn.embedding_lookup(embedding_zh, chinese_ids)) english_embedding = tf.concat(en_embeddings, 1) chinese_embedding = tf.concat(zh_embeddings, 1) english_embedding = tf.reshape( english_embedding, [self.batch_size, self.topic_num, self.hidden_size]) chinese_embedding = tf.reshape( chinese_embedding, [self.batch_size, self.topic_num, self.hidden_size]) # lstm layer two_lstm_outputs = [] for i in range(2): with tf.variable_scope("lstm-%s" % chr(ord('a') + i)): if i == 0: X = english_embedding else: X = chinese_embedding cell = BasicLSTMCell(num_units=self.hidden_size) initial_state = cell.zero_state(self.batch_size, tf.float32) outputs, _states = tf.nn.dynamic_rnn( cell, X, initial_state=initial_state, dtype=tf.float32) outputs = tf.slice(outputs, [0, self.topic_num - 1, 0], [self.batch_size, 1, self.hidden_size]) two_lstm_outputs.append( tf.reshape(outputs, [-1, self.hidden_size])) # concat and reshape output # concat_outputs = tf.concat(two_lstm_outputs, 1) # concat_outputs = tf.reshape(concat_outputs, [-1, 2*self.hidden_size]) # full connected layer # w = tf.Variable(tf.random_normal([2*self.hidden_size, 1]), name="weight", dtype=tf.float32) # b = tf.Variable(tf.constant(1.0), name="bias", dtype=tf.float32) # y = tf.matmul(concat_outputs, w) + b # y = tf.exp(-tf.nn.relu(y)) # get lstm_a output and lstm_b output lstm_a_output = two_lstm_outputs[0] lstm_b_output = two_lstm_outputs[1] # cosine similariy numerator = tf.reduce_sum(lstm_a_output * lstm_b_output, 1) denominator = tf.sqrt(tf.reduce_sum( tf.square(lstm_a_output), 1)) * tf.sqrt( tf.reduce_sum(tf.square(lstm_b_output), 1)) y = 1 - (tf.acos((numerator / denominator)) / tf.constant(3.141592653)) # Euclidean distance # y = tf.exp(-tf.sqrt(tf.reduce_sum(tf.square(lstm_a_output - lstm_b_output), 1))) # reshape y y = tf.reshape(y, [self.batch_size]) self.global_step = tf.Variable(0, trainable=False) self.learning_rate = tf.train.exponential_decay(0.1, self.global_step, 10, 2, staircase=False) self.loss_op = tf.reduce_mean(tf.square(y - Y)) self.train_op = tf.train.AdamOptimizer(learning_rate=0.1).minimize( self.loss_op, global_step=self.global_step) tf.summary.scalar("loss", self.loss_op) tf.summary.histogram("prediction", y) tf.summary.histogram("labels", Y) self.prediction = y self.init = tf.global_variables_initializer() # 导出图 # print("exporting meta graph......") # tf.train.export_meta_graph(filename=self.model_path+"model.ckpt.meta") self.logger.info("Done! building graph") def load_stopwords(self): self.logger.info("start load stopwords") with open("stopwords.txt", encoding="utf-8") as fd: txt = fd.read() self.stopwords = set(txt.split("\n")) self.logger.info("Done! load stopwords") def gen_feed_dict(self, english_batch, chinese_batch, scores): return { "english_input:0": english_batch, "chinese_input:0": chinese_batch, "scores:0": scores } def train(self): if self.processor == "gpu": print("using gpu...") elif self.processor == "cpu": print("using cpu...") os.environ['CUDA_VISIBLE_DEVICES'] = '' with tf.Session() as sess: self.logger.info("start initialization...") sess.run(self.init) self.logger.info("Done! initialization...") merged = tf.summary.merge_all() self.logger.info("Done! merge all summary") file_writer = tf.summary.FileWriter(self.summary_save_path, sess.graph) self.logger.info("Done! create summary <FileWriter>") for train_epoch in range(1, self.train_epoch_num + 1): self.shuffle_train_data() fea_batches = self.gen_fea_batch(self.model_data_path + "train.txt") for english_batch, chinese_batch, labels in fea_batches: feed_dict = self.gen_feed_dict(english_batch, chinese_batch, labels) loss, _, summary, prediction, global_step = sess.run( [ self.loss_op, self.train_op, merged, self.prediction, self.global_step ], feed_dict=feed_dict) step = global_step % self.save_step print( "cur_epoch/total_epoch: (%3d)/(%3d), current_step/save_step: (%4d)/(%4d), global_step: %4d" % (train_epoch, self.train_epoch_num, self.save_step if step == 0 else step, self.save_step, global_step)) if global_step % self.save_step == 0: print("saving model....") self.saver.save(sess, self.model_save_path + "model.ckpt") # write summary file_writer.add_summary(summary, global_step) def txt_2_fea_soft_hot(self, english_batch, chinese_batch): english_batch = [english.split() for english in english_batch] chinese_batch = [chinese.split() for chinese in chinese_batch] english_bow_batch = [ self.english_lda.id2word.doc2bow(english_words) for english_words in english_batch ] chinese_bow_batch = [ self.chinese_lda.id2word.doc2bow(chinese_words) for chinese_words in chinese_batch ] english_topics_batch = [ self.english_lda[english_bow] for english_bow in english_bow_batch ] chinese_topics_batch = [ self.chinese_lda[chinese_bow] for chinese_bow in chinese_bow_batch ] english_batch = [] chinese_batch = [] for i in range(len(english_topics_batch)): english_input = np.zeros(shape=(self.topic_num, self.topic_num), dtype=np.float32) chinese_input = np.zeros(shape=(self.topic_num, self.topic_num), dtype=np.float32) english_topics = english_topics_batch[i] chinese_topics = chinese_topics_batch[i] for item in english_topics: index = item[0] score = item[1] english_input[index][index] = score for item in chinese_topics: index = item[0] score = item[1] chinese_input[index][index] = score english_batch.append(english_input) chinese_batch.append(chinese_input) return english_batch, chinese_batch def txt_2_fea_one_hot(self, english_batch, chinese_batch): english_batch = [[ word for word in english.split() if len(word.strip()) > 0 ] for english in english_batch] chinese_batch = [[ word for word in chinese.split() if len(word.strip()) > 0 ] for chinese in chinese_batch] english_bow_batch = [ self.english_lda.id2word.doc2bow(english_words) for english_words in english_batch ] chinese_bow_batch = [ self.chinese_lda.id2word.doc2bow(chinese_words) for chinese_words in chinese_batch ] english_topics_batch = [ self.english_lda[english_bow] for english_bow in english_bow_batch ] chinese_topics_batch = [ self.chinese_lda[chinese_bow] for chinese_bow in chinese_bow_batch ] english_batch = [] chinese_batch = [] seg_len = 1.0 / (self.fea_dim - 1) for i in range(len(english_topics_batch)): english_input = np.zeros(shape=(self.topic_num), dtype=np.int32) chinese_input = np.zeros(shape=(self.topic_num), dtype=np.int32) english_topics = english_topics_batch[i] chinese_topics = chinese_topics_batch[i] for item in english_topics: index = item[0] score = item[1] score_index = int(score / seg_len) english_input[index] = score_index for item in chinese_topics: index = item[0] score = item[1] score_index = int(score / seg_len) chinese_input[index] = score_index english_batch.append(english_input) chinese_batch.append(chinese_input) return english_batch, chinese_batch def gen_raw_batch(self, file_name): with open(file_name, encoding="utf-8", errors="ignore") as fd: english_batch = [] chinese_batch = [] while True: score_list = [] for _ in range(self.batch_size): english = fd.readline().strip() chinese = fd.readline().strip() if len(english) == 0 or len(chinese) == 0: return score = float(fd.readline().strip()) fd.readline() english_batch.append(english) chinese_batch.append(chinese) score_list.append(score) if len(english_batch) != self.batch_size or len( chinese_batch) != self.batch_size: return if len(score_list) != self.batch_size: return yield english_batch, chinese_batch, np.array(score_list, dtype=np.float32) english_batch = [] chinese_batch = [] def gen_fea_batch(self, file_name): for a, b, c in self.gen_raw_batch(file_name): a_fea, b_fea = self.txt_2_fea_one_hot(a, b) yield a_fea, b_fea, c def load_lda(self): self.logger.info("start loading lda model") lda = LDA(topic_num=self.topic_num) lda.load_model() self.english_lda = lda.english_model self.chinese_lda = lda.chinese_model self.logger.info("Done! loading lda model") def predict_raw(self, english_raw, chinese_raw): english_batch, chinese_batch = self.txt_2_fea_one_hot( english_raw, chinese_raw) with tf.Session() as sess: prediction = sess.run([self.prediction], feed_dict={ "english_input:0": english_batch, "chinese_input:0": chinese_batch }) return prediction def predict_batch(self, english_batch, chinese_batch): with tf.Session() as sess: prediction = sess.run([self.prediction], feed_dict={ "english_input:0": english_batch, "chinese_input:0": chinese_batch }) return prediction def test(self): max_buffer_size = 20 * 1024 en_writer = BufferWriter("result/en.txt", max_buffer_size=max_buffer_size) zh_writer = BufferWriter("result/zh.txt", max_buffer_size=max_buffer_size) labels_writer = BufferWriter("result/labels.txt", max_buffer_size=max_buffer_size) predict_writer = BufferWriter("result/predict.txt", max_buffer_size=max_buffer_size) with tf.Session() as sess: saver = tf.train.Saver() saver.restore(sess, self.model_path + "model/model.ckpt") for en_batch, zh_batch, labels in self.gen_raw_batch("train.txt"): en_fea_batch, zh_fea_batch = self.txt_2_fea_one_hot( en_batch, zh_batch) prediction = sess.run( [self.prediction], feed_dict={ "english_input:0": en_fea_batch, "chinese_input:0": zh_fea_batch }) en_writer.update("\n".join(en_batch)) zh_writer.update("\n".join(zh_batch)) labels_writer.update("\n".join(map(str, labels))) predict_writer.update("\n".join(map(str, prediction))) # close buffer writter for writer in [en_writer, zh_writer, labels_writer, predict_writer]: writer.close()
def test_shuffle_example3(self): shuffler = Shuffler(10) shuffler.shuffle(("deal with increment 7", "deal with increment 9", "cut -2")) self.assertEqual(shuffler.stack, [6, 3, 0, 7, 4, 1, 8, 5, 2, 9])
def test_shuffle_example2(self): shuffler = Shuffler(10) shuffler.shuffle(("cut 6", "deal with increment 7", "deal into new stack")) self.assertEqual(shuffler.stack, [3, 0, 7, 4, 1, 8, 5, 2, 9, 6])
def test_shuffle_example1(self): shuffler = Shuffler(10) shuffler.shuffle(("deal with increment 7", "deal into new stack", "deal into new stack")) self.assertEqual(shuffler.stack, [0, 3, 6, 9, 2, 5, 8, 1, 4, 7])
def test_increment(self): shuffler = Shuffler(10) shuffler.deal_with_increment(3) self.assertEqual(shuffler.stack, [0, 7, 4, 1, 8, 5, 2, 9, 6, 3])
def test_new_stack(self): shuffler = Shuffler(10) shuffler.deal_into_new_stack() self.assertEqual(shuffler.stack, [9, 8, 7, 6, 5, 4, 3, 2, 1, 0])
total_duration = 0 for item in playlist: item["logfilename"] = "%s/session.log" % item["sessiondir"] print "preparing %s" % item["sessiondir"] item["tr_log"] = TrLogReader(item["logfilename"]).get_log(reduced_passivity=True) item["orchestra"] = Orchestra(server, item["sessiondir"], item["tr_log"], item["args"]) total_duration += item["orchestra"].estimated_duration print "-" * 33 print "%-19s%s\n" % ("TOTAL DURATION", datetime.timedelta(seconds=total_duration)) if args.start: count = args.start print "WARNING: shuffle disabled" else: shuffler = Shuffler(range(len(playlist))) while True: # print "\n\n\nnum threads: %s\n\n" % threading.active_count() # print "\nthreads:%s\n" % "\n".join(map(str, threading.enumerate())) log_open_files() if args.start: item = playlist[count % len(playlist)] count += 1 else: item = playlist[shuffler.next()] print "playing %s" % item["sessiondir"] orchestra = item["orchestra"] if len(orchestra.chunks) == 0: raise Exception("No chunks to play. Unsupported file format?")