def inference(): configs = read_config(FLAGS.config_path) class_ids = read_config(FLAGS.class_config) img_data = read_data(FLAGS.data_dir, int(configs['width']), int(configs['height'])) cyc_GAN = CycleGAN(configs) cyc_GAN.build_model() # You must define saver after you build a tensorflow graph saver = tf.train.Saver() with tf.Session() as sess: saver.restore(sess, FLAGS.model_path) meta_data = meta_initializer( int(class_ids[FLAGS.t]), [int(configs['width']), int(configs['height'])]) idx = 0 for img in img_data: img_vec = np.expand_dims(img, axis=0) if int(class_ids[FLAGS.f]) < int(class_ids[FLAGS.t]): mode = "AtoB" else: mode = "BtoA" output_img = sess.run(cyc_GAN.predict(mode), feed_dict={ cyc_GAN.sample_vector: img_vec, cyc_GAN.sample_meta_data: meta_data }) output_img = sess.run(tf.image.encode_jpeg(tf.squeeze(output_img))) with open(join(FLAGS.output_dir, str(idx) + '.jpg'), 'wb') as f: f.write(output_img) idx += 1
def main(args): # Read data from local disk print("[info] Reading data from disk ...") X, y = read_data(path=args['data_path']) # Normalize data # X = min_max_normalization(X=X) # Build model print("[info] Building model ...") model = LinearRegression(args=args) # Fitting model with data print("[info] learning ...") model.learn(X=X, y=y, output_log='logs/linear_regression.csv') # Plot the training progress plot_progress(path='log', save=True) # Evaluate on test set model.evaluate()
from utils.read_data import read_data from utils.plot import plot_var from utils.var import var, select_lr if __name__ == "__main__": df = read_data("data/cee.xls", f="1963Q3") lag = select_lr(df, maxlags=15) print("LR ratio: ", lag) standard_res = var(df, lags=4) plot_var(standard_res, impulse="FF", response=df.columns, folder="cee/ex", autocorr=True) ic = "bic" standard_res = var(df, trend="c", lags=2) plot_var(standard_res, impulse="FF", folder=f"cee/{ic}") new_order = ["Y", "P", "PCOM", "FF", "TR", "NBR", "M1"] reindexed = df.reindex(columns=new_order) standard_res = var(reindexed, lags=4) plot_var(standard_res, impulse="FF", response=df.columns, folder="cee/swapped", autocorr=True)
def train(config): """Train a en->fr translation model using WMT data.""" # Prepare WMT data. print("Preparing WMT data in %s" % config.data_dir) en_embd_name = "" en_train, fr_train, en_dev, fr_dev, _, _, embd_mat_en, embd_mat_fr = data_utils.prepare_wmt_data( config.data_dir, config.en_vocab_size, config.fr_vocab_size) #config.embedding_en_path, config.embedding_fr_path, 'enc_embedding', 'dec_embedding') with tf.Session() as sess: if not os.path.exists(FLAGS.model_dir): os.makedirs(FLAGS.model_dir) # Create model. print("Creating %d layers of %d units." % (config.num_layers, config.size)) model = create_model(sess, config, False) #if not config.probabilistic: # self.kl_rate_update(0.0) train_writer = tf.summary.FileWriter(os.path.join( FLAGS.model_dir, "train"), graph=sess.graph) dev_writer = tf.summary.FileWriter(os.path.join( FLAGS.model_dir, "test"), graph=sess.graph) # Read data into buckets and compute their sizes. print("Reading development and training data (limit: %d)." % config.max_train_data_size) dev_set = read_data.read_data(en_dev, fr_dev, config) train_set = read_data.read_data(en_train, fr_train, config, config.max_train_data_size) train_bucket_sizes = [ len(train_set[b]) for b in xrange(len(config.buckets)) ] train_total_size = float(sum(train_bucket_sizes)) # A bucket scale is a list of increasing numbers from 0 to 1 that we'll use # to select a bucket. Length of [scale[i], scale[i+1]] is proportional to # the size if i-th training bucket, as used later. train_buckets_scale = [ sum(train_bucket_sizes[:i + 1]) / train_total_size for i in xrange(len(train_bucket_sizes)) ] # This is the training loop. step_time, loss = 0.0, 0.0 KL_loss = 0.0 current_step = model.global_step.eval() step_loss_summaries = [] step_KL_loss_summaries = [] overall_start_time = time.time() while True: # Choose a bucket according to data distribution. We pick a random number # in [0, 1] and use the corresponding interval in train_buckets_scale. random_number_01 = np.random.random_sample() bucket_id = min([ i for i in xrange(len(train_buckets_scale)) if train_buckets_scale[i] > random_number_01 ]) # Get a batch and make a step. start_time = time.time() encoder_inputs, decoder_inputs, target_weights = model.get_batch( train_set, bucket_id) _, step_loss, step_KL_loss, _ = model.step(sess, encoder_inputs, decoder_inputs, target_weights, bucket_id, False, config.probabilistic) if config.anneal and model.global_step.eval( ) > config.kl_rate_rise_time and model.kl_rate < 1: new_kl_rate = model.kl_rate.eval() + config.kl_rate_rise_factor sess.run(model.kl_rate_update, feed_dict={'new_kl_rate': new_kl_rate}) step_time += (time.time() - start_time) / config.steps_per_checkpoint step_loss_summaries.append( tf.Summary(value=[ tf.Summary.Value(tag="step loss", simple_value=float(step_loss)) ])) step_KL_loss_summaries.append( tf.Summary(value=[ tf.Summary.Value(tag="KL step loss", simple_value=float(step_KL_loss)) ])) loss += step_loss / config.steps_per_checkpoint KL_loss += step_KL_loss / config.steps_per_checkpoint current_step = model.global_step.eval() # Once in a while, we save checkpoint, print statistics, and run evals. if current_step % config.steps_per_checkpoint == 0: # Print statistics for the previous epoch. perplexity = math.exp( float(loss)) if loss < 300 else float("inf") print( "global step %d learning rate %.4f step-time %.2f perplexity " "%.2f" % (model.global_step.eval(), model.learning_rate.eval(), step_time, perplexity)) print( "global step %d learning rate %.4f step-time %.2f KL divergence " "%.2f" % (model.global_step.eval(), model.learning_rate.eval(), step_time, KL_loss)) wall_time = time.time() - overall_start_time print("time passed: {0}".format(wall_time)) # Add perplexity, KL divergence to summary and stats. perp_summary = tf.Summary(value=[ tf.Summary.Value(tag="train perplexity", simple_value=perplexity) ]) train_writer.add_summary(perp_summary, current_step) KL_loss_summary = tf.Summary(value=[ tf.Summary.Value(tag="KL divergence", simple_value=KL_loss) ]) train_writer.add_summary(KL_loss_summary, current_step) for i, summary in enumerate(step_loss_summaries): train_writer.add_summary(summary, current_step - 200 + i) step_loss_summaries = [] for i, summary in enumerate(step_KL_loss_summaries): train_writer.add_summary(summary, current_step - 200 + i) step_KL_loss_summaries = [] # Save checkpoint and zero timer and loss. checkpoint_path = os.path.join(FLAGS.model_dir, FLAGS.model_name + ".ckpt") model.saver.save(sess, checkpoint_path, global_step=model.global_step) step_time, loss, KL_loss = 0.0, 0.0, 0.0 # Run evals on development set and print their perplexity. eval_losses = [] eval_KL_losses = [] eval_bucket_num = 0 for bucket_id in xrange(len(config.buckets)): if len(dev_set[bucket_id]) == 0: print(" eval: empty bucket %d" % (bucket_id)) continue eval_bucket_num += 1 encoder_inputs, decoder_inputs, target_weights = model.get_batch( dev_set, bucket_id) _, eval_loss, eval_KL_loss, _ = model.step( sess, encoder_inputs, decoder_inputs, target_weights, bucket_id, True, config.probabilistic) eval_losses.append(float(eval_loss)) eval_KL_losses.append(float(eval_KL_loss)) eval_ppx = math.exp( float(eval_loss)) if eval_loss < 300 else float("inf") print(" eval: bucket %d perplexity %.2f" % (bucket_id, eval_ppx)) eval_perp_summary = tf.Summary(value=[ tf.Summary.Value(tag="eval perplexity for bucket {0}". format(bucket_id), simple_value=eval_ppx) ]) dev_writer.add_summary(eval_perp_summary, current_step) mean_eval_loss = sum(eval_losses) / float(eval_bucket_num) mean_eval_KL_loss = sum(eval_KL_losses) / float( eval_bucket_num) mean_eval_ppx = math.exp(float(mean_eval_loss)) print(" eval: mean perplexity {0}".format(mean_eval_ppx)) eval_loss_summary = tf.Summary(value=[ tf.Summary.Value(tag="mean eval loss", simple_value=float(mean_eval_ppx)) ]) dev_writer.add_summary(eval_loss_summary, current_step) eval_KL_loss_summary = tf.Summary(value=[ tf.Summary.Value(tag="mean eval loss", simple_value=float(mean_eval_KL_loss)) ]) dev_writer.add_summary(eval_KL_loss_summary, current_step)
len_argv = len(sys.argv) K = 10 L = 5 try: flag = sys.argv[1] K = int(sys.argv[2]) L = int(sys.argv[3]) except: pass print '*' * 30 # make generalization hierarchies gen_gh_trees() #read gentree tax att_trees = read_tree() #read record data = read_data() # APA need only GH for transaction if flag == 'k': get_result_K(att_trees, data) elif flag == 'l': get_result_L(att_trees, data) elif flag == 'data': get_result_dataset(att_trees, data) elif flag == 'apa': if len_argv > 2: get_result_one(att_trees, data, K, L) else: get_result_one(att_trees, data) elif flag == '': get_result_one(att_trees, data) else:
if __name__ == "__main__": FLAG = "" try: FLAG = sys.argv[1] except: pass # read record print "*" * 30 # make generalization hierarchies gen_gh_trees() # read gentree tax ATT_TREES = read_tree() # read record DATA = read_data() # Separation_Gen need only GH for transaction if FLAG == "k": get_result_k(ATT_TREES, DATA) elif FLAG == "l": get_result_l(ATT_TREES, DATA) elif FLAG == "data": get_result_dataset(ATT_TREES, DATA) elif FLAG == "": # cProfile.run('get_result_one(ATT_TREE, DATA, TYPE_ALG)') get_result_one(ATT_TREES, DATA) else: try: INPUT_K = int(FLAG) get_result_one(ATT_TREEs, DATA) except ValueError:
INPUT_K = 10 # read record RELAX = True if mode_config.model == "relax" else False if RELAX: print("Relax Mondrian") else: print("Strict Mondrian") # print("Adult data") # INTUITIVE_ORDER is an intuitive order for # categorical attributes. This order is produced # by the reading (from data set) order. DATA, INTUITIVE_ORDER = read_data() print(INTUITIVE_ORDER) FLAG = mode_config.flag if FLAG == 'k': get_result_k(DATA) elif FLAG == 'qi': get_result_qi(DATA) elif FLAG == 'data': get_result_dataset(DATA) elif FLAG == '': get_result_one(DATA, mode_config.k) else: try: get_result_one(DATA, 10) except ValueError:
""" run anatomize with given parameters """ # !/usr/bin/env python # coding=utf-8 # by Qiyuan Gong # [email protected] from anatomize import anatomize from utils.read_data import read_data import sys if __name__ == '__main__': L = 10 try: L = int(sys.argv[1]) except IndexError: pass # read record RAW_DATA = read_data() # remove duplicate items print "Begin Anatomizer" RESULT = anatomize(RAW_DATA, L) print "No. groups in result=%d" % len(RESULT) print "Finish Anatomizer!!"
def initialize_game(self, difficulty): self.canvas.clear() self.state = "game" self.difficulty = difficulty self.audio_controller = AudioController(self.difficulty) self.audio_controller.toggle() self.notes_down = [ ] # an array that keeps track of all notes that are currently being played on midi keyboard # The display for the gems, now bar, and bar lines self.canvas.add(Color(1, 1, 1)) rect = Rectangle(pos=(0, 0), size=(Window.width, Window.height), texture=Image('assets/bg_c.png').texture) self.canvas.add(rect) self.lane_manager = LaneManager() # Display the status of the game through the text labels self.canvas.add(Color(1, 1, 1)) self.hud = Rectangle(pos=(0, Window.height / 1.12), size=(Window.width / 4, Window.height / 9), texture=Image('assets/tophub.png').texture) self.canvas.add(self.hud) self.canvas.add(Color(1, 1, 1)) self.hud_score = Rectangle( pos=(Window.width / 1.4, Window.height / 1.12), size=(Window.width / 3.5, Window.height / 9), texture=Image('assets/topscore.png').texture) self.canvas.add(self.hud_score) self.score_label = score_label() self.add_widget(self.score_label) self.hp_label = hp_label() self.add_widget(self.hp_label) self.enemy_times = [] self.enemy_lanes = [] self.enemy_types = [] self.enemy_chords = [] self.enemy_manager = EnemyManager() self.canvas.add(self.enemy_manager) if difficulty == "easy": read_data("song_annotations/hallelujah_left_hand_test.txt", None, self.enemy_times, self.enemy_lanes, self.enemy_types, self.enemy_chords) self.song_length = 140 elif difficulty == "medium": read_data("song_annotations/falling_left_hand_test.txt", "song_annotations/falling_right_hand_test.txt", self.enemy_times, self.enemy_lanes, self.enemy_types, self.enemy_chords) self.song_length = 44 elif difficulty == "hard": read_data("song_annotations/hallelujah_left_hand_test.txt", "song_annotations/hallelujah_right_hand_test.txt", self.enemy_times, self.enemy_lanes, self.enemy_types, self.enemy_chords, inversions=True) self.song_length = 140 self.prev_time = time.time() self.elapsed_time = 0 self.note_index = 0 window_size = 4 # 4 seconds of notes are displayed x_scale = Window.width / window_size # pixels / sec # Create the player object which will store and control the state of the game self.player = Player(self.hp_label, self.score_label, self.enemy_times, self.enemy_lanes, self.enemy_types, self.enemy_chords, self.enemy_manager, self.audio_controller) self.canvas.add(self.player) self.player.toggle()
"""Returns a LaTeX pmatrix :a: numpy array :returns: LaTeX pmatrix as a string """ if len(a.shape) > 2: raise ValueError('pmatrix can at most display two dimensions') lines = str(a).replace('[', '').replace(']', '').splitlines() rv = [r'\begin{pmatrix}'] rv += [' ' + ' & '.join(l.split()) + r'\\' for l in lines] rv += [r'\end{pmatrix}'] return '\n'.join(rv) if __name__ == "__main__": df = read_data("data/bp.xls") scaled_df, unscaled_df = parse_data(df) pre_elas, post_elas = G_Y_multiplier(scaled_df, unscaled_df) print(f"Multiplier: {pre_elas} (pre) and {post_elas} (post)") dummy_df = scaled_df.copy() dummy_df["D"] = 0. dummy_df["D"]["1975Q2"] = 1. standard_res = var(dummy_df, lags=4) plot_var(standard_res, impulse="G", response=dummy_df.columns, folder="bp/base",
if __name__ == '__main__': # set K=10 as default FLAG = '' DATA_SELECT = '' # gen_even_BMS_tree(5) try: DATA_SELECT = sys.argv[1] FLAG = sys.argv[2] except IndexError: pass INPUT_K = 10 if DATA_SELECT == 'i': print "INFORMS data" DATA = read_data(1) ATT_TREE = read_tree(2) else: print "BMS-WebView data" DATA = read_data(0) ATT_TREE = read_tree(0) # read generalization hierarchy # read record # remove duplicate items for i in range(len(DATA)): DATA[i] = list(set(DATA[i])) print "Begin Partition" if FLAG == 'k': get_result_k(ATT_TREE, DATA) elif FLAG == 'data': get_result_dataset(ATT_TREE, DATA)
def parse_data(df: pd.DataFrame) -> pd.DataFrame: new_df = df.copy() pop = new_df["POP"] for transf_var in transform: trans_data = np.log(new_df[transf_var] / pop) new_df[name_map[transf_var]] = trans_data return new_df.drop(transform, axis=1) if __name__ == "__main__": df = read_data("data/ramey.xls") parsed_df = parse_data(df)["1960Q1":] restr_var = ["G", "T", "Y"] restr_df = parsed_df[restr_var] res = var(restr_df, trend="ctt", lags=4) plot_var(res, impulse="G", folder="ramey/base") aug_var = restr_var + ["CONS", "INV"] aug_df = parsed_df[aug_var] res = var(aug_df, trend="ctt", lags=4) plot_var(res, impulse="G", folder="ramey/aug") war_var = ["WAR"] + aug_var war_df = parsed_df[war_var] res = var(war_df, trend="ctt", lags=4)
len_argv = len(sys.argv) K = 10 L = 5 try: flag = sys.argv[1] K = int(sys.argv[2]) L = int(sys.argv[3]) except: pass print '*'*30 # make generalization hierarchies gen_gh_trees() #read gentree tax att_trees = read_tree() #read record data = read_data() # PAA need only GH for transaction if flag == 'k': get_result_K(att_trees, data) elif flag == 'l': get_result_L(att_trees, data) elif flag == 'data': get_result_dataset(att_trees, data) elif flag == 'paa': if len_argv > 2: get_result_one(att_trees, data, K, L) else: get_result_one(att_trees, data) elif flag =='': get_result_one(att_trees, data) else:
FLAG = '' # gen_even_BMS_tree(5) try: TYPE_ALG = sys.argv[1] DATA_SELECT = sys.argv[2] FLAG = sys.argv[3] except IndexError: pass INPUT_K = 10 if TYPE_ALG == 'DA' or TYPE_ALG == 'da': print "Begin DA" else: print "Begin AA" if DATA_SELECT == 'i': print "INFORMS data" DATA = read_data(1) ATT_TREE = read_tree(2) else: print "BMS-WebView data" DATA = read_data(0) ATT_TREE = read_tree(0) print "*" * 30 # read generalization hierarchy # read record # remove duplicate items # DATA = DATA[:1000] # for i in range(len(DATA)): # if len(DATA[i]) <= 40: # DATA[i] = list(set(DATA[i])) # else: # DATA[i] = list(set(DATA[i][:40]))
import pandas as pd import matplotlib.pyplot as plt import seaborn as sns from utils.read_data import read_data from utils.plot import plot_series if __name__ == '__main__': df = read_data("data/rz.xls", time_name="quarter") plot_series(df, folder="rz") df.to_csv("data/parsed_rz.xls")